Reworked the tests, read more

I've brought the tests up to scratch, except for compounds im the
parser, because I'm too damn tired to do that over SSH. It looks like
collections are right-recursive, whixh was unintended but still works
just fine.

I've also added the '--verbose' flag to the repl to control the
debugging output.

Several obscure bugs have been fixed, and comments have been tweaked.

Mustfail tests are still needed, but that's a low priority. See #142.

Fixed #151
This commit is contained in:
2024-11-12 22:04:07 +11:00
parent b74aa63c1c
commit be7e4ddd18
22 changed files with 364 additions and 102 deletions

View File

@@ -60,5 +60,3 @@ jobs:
- name: run the tests
if: (matrix.commands.gdb == true && matrix.platforms.gdb_enabled == false) != true
run: ${{ matrix.commands.exec }}

View File

@@ -26,8 +26,8 @@ This repository holds the reference implementation for Toy version 2.x, written
//print is a built-in keyword, that can handle complex expressions
print 6 * 7;
//strings can be concatenated with the .. operator
print "Hello" .. "world!";
//strings can be concatenated with the .. operator, and substringed with the [] operator
print "Hello" .. "world!"[3, 3]; //[index, length] - this prints "low"
//variables are declared easily
var foobar = 42;
@@ -40,7 +40,10 @@ var foobar = 42;
//the types default to 'any' but can be specified if needed (same with constants)
var immutable: string const = "Foobar";
//more examples to be added as the features are implemented
//the assert keyword can check an expression, and takes an optional second parameter
assert immutable == "Fizzbuzz", "This message is sent to the terminal by default";
//NOTE: This section will be expanded as more features are implemented
```
# Building

View File

@@ -27,8 +27,8 @@ repl: source
.PHONY: tests
tests: clean test-cases test-integrations
.PHONY: test-all
test-all: clean test-cases test-integrations
#.PHONY: test-all
#test-all: clean test-cases test-integrations
.PHONY: test-cases
test-cases:
@@ -39,8 +39,8 @@ test-integrations:
$(MAKE) -C $(TOY_INTEGRATIONSDIR) -k
#same as above, but with GDB
.PHONY: test-gdb
test-gdb: clean test-cases-gdb test-integrations-gdb
.PHONY: tests-gdb
tests-gdb: clean test-cases-gdb test-integrations-gdb
.PHONY: test-cases-gdb
test-cases-gdb:

View File

@@ -40,4 +40,3 @@ TOY_API Toy_Array* Toy_resizeArray(Toy_Array* array, unsigned int capacity);
#ifndef TOY_ARRAY_PUSHBACK
#define TOY_ARRAY_PUSHBACK(array, value) (TOY_ARRAY_EXPAND(array),(array)->data[(array)->count++] = (value))
#endif

View File

@@ -11,4 +11,3 @@ typedef struct Toy_Bytecode {
TOY_API Toy_Bytecode Toy_compileBytecode(Toy_Ast* ast);
TOY_API void Toy_freeBytecode(Toy_Bytecode bc);

View File

@@ -23,7 +23,7 @@ const Toy_KeywordTypeTuple keywordTuples[] = {
{TOY_TOKEN_TYPE_STRING, "string"},
{TOY_TOKEN_TYPE_ARRAY, "array"},
{TOY_TOKEN_TYPE_TABLE, "table"},
{TOY_TOKEN_TYPE_FUNCTION, "function"}, //TODO: type??
{TOY_TOKEN_TYPE_FUNCTION, "function"},
{TOY_TOKEN_TYPE_OPAQUE, "opaque"},
{TOY_TOKEN_TYPE_ANY, "any"},

View File

@@ -41,11 +41,10 @@ typedef enum Toy_OpcodeType {
TOY_OPCODE_PRINT,
TOY_OPCODE_CONCAT,
TOY_OPCODE_INDEX,
//TODO: clear the program stack?
//TODO: clear the program stack - much needed
//meta instructions
TOY_OPCODE_PASS,
TOY_OPCODE_ERROR,
TOY_OPCODE_EOF = 255,
} Toy_OpcodeType;

View File

@@ -563,7 +563,7 @@ static Toy_AstFlag compound(Toy_Bucket** bucketHandle, Toy_Parser* parser, Toy_A
advance(parser);
if (parser->previous.type == TOY_TOKEN_OPERATOR_COMMA) {
parsePrecedence(bucketHandle, parser, rootHandle, PREC_GROUP);
parsePrecedence(bucketHandle, parser, rootHandle, PREC_GROUP); //NOT +1, as compounds are right-recursive
return TOY_AST_FLAG_COMPOUND_COLLECTION;
}
else if (parser->previous.type == TOY_TOKEN_OPERATOR_BRACKET_LEFT) {

View File

@@ -272,8 +272,6 @@ void Toy_stringifyValue(Toy_Value value, Toy_callbackType callback) {
case TOY_VALUE_STRING: {
Toy_String* str = TOY_VALUE_AS_STRING(value);
//TODO: decide on how long strings, etc. live for in memory
if (str->type == TOY_STRING_NODE) {
char* buffer = Toy_getStringRawBuffer(str);
callback(buffer);

View File

@@ -189,7 +189,7 @@ static void processAccess(Toy_VM* vm) {
//find and push the value
Toy_Value value = Toy_accessScope(vm->scope, TOY_VALUE_AS_STRING(name));
Toy_pushStack(&vm->stack, value);
Toy_pushStack(&vm->stack, Toy_copyValue(value));
//cleanup
Toy_freeValue(name);

View File

@@ -49,4 +49,4 @@ TOY_API void Toy_freeVM(Toy_VM* vm);
TOY_API void Toy_resetVM(Toy_VM* vm); //prepares for another run without deleting stack, scope and memory
//TODO: inject extra data
//TODO: inject extra data (hook system for external libraries)

36
tests/README.md Normal file
View File

@@ -0,0 +1,36 @@
# Test Instructions
To run these tests, execute the following commands from the repo's root:
`make tests`
`make test-cases`
`make test-integrations`
Alternatively, to run these tests under GDB, execute the following commands from the repo's root:
`make tests-gdb`
`make test-cases-gdb`
`make test-integrations-gdb`
Remember that `make clean` will remove the build artifacts after testing, and `make tests` and `make-tests-gdb` automatically invoke `make clean` before they begin.
## Benchmarks
For testing and comparing different potential solutions. This may be left in an incomplete state, so it might not work out of the box.
## Cases
For testing individual pieces of the source code in isolation. These are essentially the unit tests.
## Integrations
This compiles the source and repl files into a library and executable, then runs each `*.toy` file through the repl to ensure the Toy code works in practice. These are essentially integration tests.
## Mustfails
These have situations which will raise errors of some kind, to ensure that common user errors are handled gracefully. This is not yet implemented.
## Standalone
These are one-file programs that are not intended to test the source directly. Instead, these can cover a number of situations, such as the exact behavior of GitHub's workflow runners, or to generate repetitive code predictably, etc.

View File

@@ -185,7 +185,71 @@ int test_type_emission(Toy_Bucket** bucketHandle) {
}
}
//emit print keyword
//emit compound
{
//build the AST
Toy_Ast* ast = NULL;
Toy_Ast* right = NULL;
Toy_private_emitAstValue(bucketHandle, &ast, TOY_VALUE_FROM_INTEGER(42));
Toy_private_emitAstValue(bucketHandle, &right, TOY_VALUE_FROM_INTEGER(69));
Toy_private_emitAstCompound(bucketHandle, &ast, TOY_AST_FLAG_COMPOUND_COLLECTION, right);
//check if it worked
if (
ast == NULL ||
ast->type != TOY_AST_COMPOUND ||
ast->compound.left == NULL ||
ast->compound.left->type != TOY_AST_VALUE ||
TOY_VALUE_IS_INTEGER(ast->compound.left->value.value) != true ||
TOY_VALUE_AS_INTEGER(ast->compound.left->value.value) != 42 ||
ast->compound.right == NULL ||
ast->compound.right->type != TOY_AST_VALUE ||
TOY_VALUE_IS_INTEGER(ast->compound.right->value.value) != true ||
TOY_VALUE_AS_INTEGER(ast->compound.right->value.value) != 69 ||
false)
{
fprintf(stderr, TOY_CC_ERROR "ERROR: failed to emit a compound as 'Toy_Ast', state unknown\n" TOY_CC_RESET);
return -1;
}
}
//emit keyword assert
{
//build the AST
Toy_Ast* ast = NULL;
Toy_Ast* child = NULL;
Toy_Ast* msg = NULL;
Toy_private_emitAstValue(bucketHandle, &child, TOY_VALUE_FROM_INTEGER(42));
Toy_private_emitAstValue(bucketHandle, &msg, TOY_VALUE_FROM_INTEGER(69));
Toy_private_emitAstAssert(bucketHandle, &ast, child, msg);
//check if it worked
if (
ast == NULL ||
ast->type != TOY_AST_ASSERT ||
ast->assert.child == NULL ||
ast->assert.child->type != TOY_AST_VALUE ||
TOY_VALUE_IS_INTEGER(ast->assert.child->value.value) != true ||
TOY_VALUE_AS_INTEGER(ast->assert.child->value.value) != 42 ||
ast->assert.message == NULL ||
ast->assert.message->type != TOY_AST_VALUE ||
TOY_VALUE_IS_INTEGER(ast->assert.message->value.value) != true ||
TOY_VALUE_AS_INTEGER(ast->assert.message->value.value) != 69 ||
false)
{
fprintf(stderr, TOY_CC_ERROR "ERROR: failed to emit a keyword 'assert' as 'Toy_Ast', state unknown\n" TOY_CC_RESET);
return -1;
}
}
//emit keyword print
{
//build the AST
Toy_Ast* ast = NULL;
@@ -207,54 +271,11 @@ int test_type_emission(Toy_Bucket** bucketHandle) {
ast->print.child->binary.right->type != TOY_AST_VALUE ||
TOY_VALUE_AS_INTEGER(ast->print.child->binary.right->value.value) != 69)
{
fprintf(stderr, TOY_CC_ERROR "ERROR: failed to emit a print as 'Toy_Ast', state unknown\n" TOY_CC_RESET);
fprintf(stderr, TOY_CC_ERROR "ERROR: failed to emit a keyword 'print' as 'Toy_Ast', state unknown\n" TOY_CC_RESET);
return -1;
}
}
//emit and append blocks of code
{
//initialize the root block
Toy_Ast* block = NULL;
Toy_private_initAstBlock(bucketHandle, &block);
//loop over the ast emissions, appending each one as you go
for (int i = 0; i < 5; i++) {
//build the AST
Toy_Ast* ast = NULL;
Toy_Ast* right = NULL;
Toy_private_emitAstValue(bucketHandle, &ast, TOY_VALUE_FROM_INTEGER(42));
Toy_private_emitAstValue(bucketHandle, &right, TOY_VALUE_FROM_INTEGER(69));
Toy_private_emitAstBinary(bucketHandle, &ast, TOY_AST_FLAG_ADD, right);
Toy_private_emitAstGroup(bucketHandle, &ast);
Toy_private_appendAstBlock(bucketHandle, block, ast);
}
//check if it worked
Toy_Ast* iter = block;
while(iter != NULL) {
if (
iter->type != TOY_AST_BLOCK ||
iter->block.child == NULL ||
iter->block.child->type != TOY_AST_GROUP ||
iter->block.child->group.child == NULL ||
iter->block.child->group.child->type != TOY_AST_BINARY ||
iter->block.child->group.child->binary.flag != TOY_AST_FLAG_ADD ||
iter->block.child->group.child->binary.left->type != TOY_AST_VALUE ||
TOY_VALUE_AS_INTEGER(iter->block.child->group.child->binary.left->value.value) != 42 ||
iter->block.child->group.child->binary.right->type != TOY_AST_VALUE ||
TOY_VALUE_AS_INTEGER(iter->block.child->group.child->binary.right->value.value) != 69)
{
fprintf(stderr, TOY_CC_ERROR "ERROR: failed to emit a block as 'Toy_Ast', state unknown\n" TOY_CC_RESET);
return -1;
}
iter = iter->block.next;
}
}
//emit var declare
{
//build the AST
@@ -331,6 +352,48 @@ int test_type_emission(Toy_Bucket** bucketHandle) {
}
}
//emit and append blocks of code (at the bottom of this test function, so everything else is checked first)
{
//initialize the root block
Toy_Ast* block = NULL;
Toy_private_initAstBlock(bucketHandle, &block);
//loop over the ast emissions, appending each one as you go
for (int i = 0; i < 5; i++) {
//build the AST
Toy_Ast* ast = NULL;
Toy_Ast* right = NULL;
Toy_private_emitAstValue(bucketHandle, &ast, TOY_VALUE_FROM_INTEGER(42));
Toy_private_emitAstValue(bucketHandle, &right, TOY_VALUE_FROM_INTEGER(69));
Toy_private_emitAstBinary(bucketHandle, &ast, TOY_AST_FLAG_ADD, right);
Toy_private_emitAstGroup(bucketHandle, &ast);
Toy_private_appendAstBlock(bucketHandle, block, ast);
}
//check if it worked
Toy_Ast* iter = block;
while(iter != NULL) {
if (
iter->type != TOY_AST_BLOCK ||
iter->block.child == NULL ||
iter->block.child->type != TOY_AST_GROUP ||
iter->block.child->group.child == NULL ||
iter->block.child->group.child->type != TOY_AST_BINARY ||
iter->block.child->group.child->binary.flag != TOY_AST_FLAG_ADD ||
iter->block.child->group.child->binary.left->type != TOY_AST_VALUE ||
TOY_VALUE_AS_INTEGER(iter->block.child->group.child->binary.left->value.value) != 42 ||
iter->block.child->group.child->binary.right->type != TOY_AST_VALUE ||
TOY_VALUE_AS_INTEGER(iter->block.child->group.child->binary.right->value.value) != 69)
{
fprintf(stderr, TOY_CC_ERROR "ERROR: failed to emit a block as 'Toy_Ast', state unknown\n" TOY_CC_RESET);
return -1;
}
iter = iter->block.next;
}
}
return 0;
}
@@ -339,24 +402,23 @@ int main() {
//run each test set, returning the total errors given
int total = 0, res = 0;
{
#if TOY_BITNESS == 64
res = test_sizeof_ast_64bit();
total += res;
if (res == 0) {
printf(TOY_CC_NOTICE "All good\n" TOY_CC_RESET);
}
#elif TOY_BITNESS == 32
res = test_sizeof_ast_32bit();
total += res;
#else
res = -1;
fprintf(stderr, TOY_CC_WARN "WARNING: Skipping test_sizeof_ast_**bit(); Can't determine the 'bitness' of this platform (seems to be %d)\n" TOY_CC_RESET, TOY_BITNESS);
#endif
if (res == 0) {
printf(TOY_CC_NOTICE "All good\n" TOY_CC_RESET);
}
#else
fprintf(stderr, TOY_CC_WARN "WARNING: Skipping test_sizeof_ast_*bit(); Can't determine the 'bitness' of this platform (seems to be %d)\n" TOY_CC_RESET, TOY_BITNESS);
#endif
else if (res > 0) {
total += res;
}
}
{
Toy_Bucket* bucketHandle = Toy_allocateBucket(TOY_BUCKET_IDEAL);

View File

@@ -455,6 +455,11 @@ int test_binary(Toy_Bucket** bucketHandle) {
return 0;
}
int test_compound(Toy_Bucket** bucketHandle) {
//TODO: fix test_compound()
return 0;
}
int test_precedence(Toy_Bucket** bucketHandle) {
//test term-factor precedence
{
@@ -624,8 +629,6 @@ int main() {
total += res;
}
//TODO: assign & compare?
{
Toy_Bucket* bucket = Toy_allocateBucket(TOY_BUCKET_IDEAL);
res = test_values(&bucket);
@@ -656,6 +659,16 @@ int main() {
total += res;
}
{
Toy_Bucket* bucket = Toy_allocateBucket(TOY_BUCKET_IDEAL);
res = test_compound(&bucket);
Toy_freeBucket(&bucket);
if (res == 0) {
printf(TOY_CC_NOTICE "All good\n" TOY_CC_RESET);
}
total += res;
}
{
Toy_Bucket* bucket = Toy_allocateBucket(TOY_BUCKET_IDEAL);
res = test_precedence(&bucket);

View File

@@ -175,7 +175,136 @@ static void callbackUtil(const char* msg) {
}
}
int test_keywords(Toy_Bucket** bucketHandle) {
int test_keyword_assert(Toy_Bucket** bucketHandle) {
//test assert true
{
//setup
Toy_setAssertFailureCallback(callbackUtil);
const char* source = "assert true;";
Toy_Lexer lexer;
Toy_bindLexer(&lexer, source);
Toy_Parser parser;
Toy_bindParser(&parser, &lexer);
Toy_Ast* ast = Toy_scanParser(bucketHandle, &parser);
Toy_Bytecode bc = Toy_compileBytecode(ast);
Toy_VM vm;
Toy_initVM(&vm);
Toy_bindVM(&vm, bc.ptr);
//run
Toy_runVM(&vm);
//check
if (callbackUtilReceived != NULL)
{
fprintf(stderr, TOY_CC_ERROR "ERROR: Unexpected assert message '%s' found, source: %s\n" TOY_CC_RESET, callbackUtilReceived != NULL ? callbackUtilReceived : "NULL", source);
//cleanup and return
Toy_resetAssertFailureCallback();
free(callbackUtilReceived);
Toy_freeVM(&vm);
return -1;
}
//teadown
Toy_resetAssertFailureCallback();
free(callbackUtilReceived);
callbackUtilReceived = NULL;
Toy_freeVM(&vm);
}
//test assert false
{
//setup
Toy_setAssertFailureCallback(callbackUtil);
const char* source = "assert false;";
Toy_Lexer lexer;
Toy_bindLexer(&lexer, source);
Toy_Parser parser;
Toy_bindParser(&parser, &lexer);
Toy_Ast* ast = Toy_scanParser(bucketHandle, &parser);
Toy_Bytecode bc = Toy_compileBytecode(ast);
Toy_VM vm;
Toy_initVM(&vm);
Toy_bindVM(&vm, bc.ptr);
//run
Toy_runVM(&vm);
//check
if (callbackUtilReceived == NULL ||
strcmp(callbackUtilReceived, "assertion failed") != 0)
{
fprintf(stderr, TOY_CC_ERROR "ERROR: Unexpected assert failure message '%s' found, source: %s\n" TOY_CC_RESET, callbackUtilReceived != NULL ? callbackUtilReceived : "NULL", source);
//cleanup and return
Toy_resetAssertFailureCallback();
free(callbackUtilReceived);
Toy_freeVM(&vm);
return -1;
}
//teadown
Toy_resetAssertFailureCallback();
free(callbackUtilReceived);
callbackUtilReceived = NULL;
Toy_freeVM(&vm);
}
//test assert false with message
{
//setup
Toy_setAssertFailureCallback(callbackUtil);
const char* source = "assert false, \"You passed a false to assert\";";
Toy_Lexer lexer;
Toy_bindLexer(&lexer, source);
Toy_Parser parser;
Toy_bindParser(&parser, &lexer);
Toy_Ast* ast = Toy_scanParser(bucketHandle, &parser);
Toy_Bytecode bc = Toy_compileBytecode(ast);
Toy_VM vm;
Toy_initVM(&vm);
Toy_bindVM(&vm, bc.ptr);
//run
Toy_runVM(&vm);
//check
if (callbackUtilReceived == NULL ||
strcmp(callbackUtilReceived, "You passed a false to assert") != 0)
{
fprintf(stderr, TOY_CC_ERROR "ERROR: Unexpected assert failure message '%s' found, source: %s\n" TOY_CC_RESET, callbackUtilReceived != NULL ? callbackUtilReceived : "NULL", source);
//cleanup and return
Toy_resetAssertFailureCallback();
free(callbackUtilReceived);
Toy_freeVM(&vm);
return -1;
}
//teadown
Toy_resetAssertFailureCallback();
free(callbackUtilReceived);
callbackUtilReceived = NULL;
Toy_freeVM(&vm);
}
return 0;
}
int test_keyword_print(Toy_Bucket** bucketHandle) {
//test print
{
//setup
@@ -198,7 +327,7 @@ int test_keywords(Toy_Bucket** bucketHandle) {
//run
Toy_runVM(&vm);
//check the final state of the stack
//check
if (callbackUtilReceived == NULL ||
strcmp(callbackUtilReceived, "42") != 0)
{
@@ -240,7 +369,7 @@ int test_keywords(Toy_Bucket** bucketHandle) {
//run
Toy_runVM(&vm);
//check the final state of the stack
//check
if (callbackUtilReceived == NULL ||
strcmp(callbackUtilReceived, "Hello world!") != 0)
{
@@ -282,7 +411,7 @@ int test_keywords(Toy_Bucket** bucketHandle) {
//run
Toy_runVM(&vm);
//check the final state of the stack
//check
if (callbackUtilReceived == NULL ||
strcmp(callbackUtilReceived, "Helloworld!") != 0)
{
@@ -329,7 +458,7 @@ int test_scope(Toy_Bucket** bucketHandle) {
//run
Toy_runVM(&vm);
//check the final state of the stack
//check
Toy_String* key = Toy_createNameStringLength(bucketHandle, "foobar", 6, TOY_VALUE_ANY, false);
if (vm.stack == NULL ||
@@ -376,7 +505,7 @@ int test_scope(Toy_Bucket** bucketHandle) {
//run
Toy_runVM(&vm);
//check the final state of the stack
//check
Toy_String* key = Toy_createNameStringLength(bucketHandle, "foobar", 6, TOY_VALUE_UNKNOWN, false);
if (vm.stack == NULL ||
@@ -512,7 +641,17 @@ int main() {
{
Toy_Bucket* bucket = Toy_allocateBucket(TOY_BUCKET_IDEAL);
res = test_keywords(&bucket);
res = test_keyword_print(&bucket);
Toy_freeBucket(&bucket);
if (res == 0) {
printf(TOY_CC_NOTICE "All good\n" TOY_CC_RESET);
}
total += res;
}
{
Toy_Bucket* bucket = Toy_allocateBucket(TOY_BUCKET_IDEAL);
res = test_keyword_assert(&bucket);
Toy_freeBucket(&bucket);
if (res == 0) {
printf(TOY_CC_NOTICE "All good\n" TOY_CC_RESET);

View File

@@ -33,7 +33,7 @@ all: source repl run
run: $(TEST_SCRIPTFILES:.toy=.toy-run)
%.toy-run: %.toy
$(TEST_OUTDIR)/$(TEST_REPLNAME) -f ../$<
$(TEST_OUTDIR)/$(TEST_REPLNAME) -f ../$< --verbose
#same as above, but with gdb
gdb: source repl run-gdb
@@ -41,7 +41,7 @@ gdb: source repl run-gdb
run-gdb: $(TEST_SCRIPTFILES:.toy=.toy-gdb)
%.toy-gdb: %.toy
gdb $(TEST_OUTDIR)/$(TEST_REPLNAME) -ix gdb_init -ex=run --batch --return-child-result --args "$(TEST_OUTDIR)/$(TEST_REPLNAME)" "-f" "../$<"
gdb $(TEST_OUTDIR)/$(TEST_REPLNAME) -ix gdb_init -ex=run --batch --return-child-result --args "$(TEST_OUTDIR)/$(TEST_REPLNAME)" "-f" "../$<" "--verbose"
#compile the source and repl first
source: $(TEST_OBJDIR) $(TEST_OUTDIR)

View File

@@ -1,3 +0,0 @@
//basic expressions with no side effects (other than debug stack dumps)
(1 + 2) * (3 + 4);

View File

@@ -0,0 +1,14 @@
//NOTE: these tests are all passing - failing tests can be found under the 'mustfails' directory
//basic assert statement
assert true;
//assert on a string (tests for it's truthiness)
assert "Hello world";
//assert on a condition
assert 1 < 2;
//assert with an optional message
assert true, "Assertion message";

View File

@@ -13,7 +13,9 @@ print "Hello" .. "world!";
//print with escaped characters
print "\tHello\nworld";
//print from a leaf string
//print from a substring string
print "Hello world"[0,5];
//print from a substring, after a concat
print ("hello" .. "world")[2,6];

View File

@@ -15,5 +15,3 @@ print question; //42
print question; //42
}
print question; //42
//TODO: scope test case

View File

@@ -50,7 +50,7 @@ print !true; //false
print !false; //true
//precedence
print true && false || true; //TODO: a warning is needed for this
print true && false || true; //TODO: a grouping warning is needed for this
//types
var a: int;
@@ -67,4 +67,9 @@ var c: int const = 42;
print c;
//indexing
var s = "Hello" .. "world!";
print s[3, 3];
//TODO: type casting