From 5fd933a15e58fa68bbe71672969c46c2f693a89e Mon Sep 17 00:00:00 2001 From: Kayne Ruse Date: Tue, 10 Sep 2024 20:19:11 +1000 Subject: [PATCH] Implemented AST, ensured bucket memory worked --- .notes/Reminders.txt | 3 + scripts/.gitkeep | 0 source/toy_ast.c | 76 ++++++++++++ source/toy_ast.h | 99 +++++++++++++++ source/toy_console_colors.h | 2 + source/toy_lexer.c | 2 +- source/toy_memory.h | 19 ++- source/toy_opcodes.h | 2 + source/toy_parser.c | 0 source/toy_parser.h | 0 source/toy_token_types.h | 2 +- source/toy_value.h | 2 +- tests/cases/test_ast.c | 241 ++++++++++++++++++++++++++++++++++++ 13 files changed, 435 insertions(+), 13 deletions(-) create mode 100644 .notes/Reminders.txt create mode 100644 scripts/.gitkeep create mode 100644 source/toy_parser.c create mode 100644 source/toy_parser.h create mode 100644 tests/cases/test_ast.c diff --git a/.notes/Reminders.txt b/.notes/Reminders.txt new file mode 100644 index 0000000..6de52d6 --- /dev/null +++ b/.notes/Reminders.txt @@ -0,0 +1,3 @@ +Add these to the docs somewhere: + +double pointers are referred to as handles \ No newline at end of file diff --git a/scripts/.gitkeep b/scripts/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/source/toy_ast.c b/source/toy_ast.c index da59574..7ee9045 100644 --- a/source/toy_ast.c +++ b/source/toy_ast.c @@ -1,2 +1,78 @@ #include "toy_ast.h" +void Toy_private_initAstBlock(Toy_Bucket** bucket, Toy_Ast** handle) { + (*handle) = (Toy_Ast*)Toy_partBucket(bucket, sizeof(Toy_Ast)); + + (*handle)->block.type = TOY_AST_BLOCK; + (*handle)->block.child = NULL; + (*handle)->block.next = NULL; + (*handle)->block.tail = NULL; +} + +void Toy_private_appendAstBlock(Toy_Bucket** bucket, Toy_Ast** handle, Toy_Ast* child) { + //type check + + //first, check if we're an empty head + if ((*handle)->block.child == NULL) { + (*handle)->block.child = child; + return; //NOTE: first call on an empty head skips any memory allocations + } + + //run (or jump) until we hit the current tail + Toy_Ast* iter = (*handle)->block.tail ? (*handle)->block.tail : (*handle); + + while(iter->block.next != NULL) { + iter = iter->block.next; + } + + //append a new link to the chain + Toy_private_initAstBlock(bucket, &(iter->block.next)); + + //store the child in the new link, prep the tail pointer + iter->block.next->block.child = child; + (*handle)->block.tail = iter->block.next; +} + +void Toy_private_emitAstValue(Toy_Bucket** bucket, Toy_Ast** handle, Toy_Value value) { + (*handle) = (Toy_Ast*)Toy_partBucket(bucket, sizeof(Toy_Ast)); + + (*handle)->value.type = TOY_AST_VALUE; + (*handle)->value.value = value; +} + +//TODO: flag range checks +void Toy_private_emitAstUnary(Toy_Bucket** bucket, Toy_Ast** handle, Toy_AstFlag flag, Toy_Ast* child) { + (*handle) = (Toy_Ast*)Toy_partBucket(bucket, sizeof(Toy_Ast)); + + (*handle)->unary.type = TOY_AST_UNARY; + (*handle)->unary.flag = flag; + (*handle)->unary.child = child; +} + +void Toy_private_emitAstBinary(Toy_Bucket** bucket, Toy_Ast** handle, Toy_AstFlag flag, Toy_Ast* left, Toy_Ast* right) { + (*handle) = (Toy_Ast*)Toy_partBucket(bucket, sizeof(Toy_Ast)); + + (*handle)->binary.type = TOY_AST_BINARY; + (*handle)->binary.flag = flag; + (*handle)->binary.left = left; + (*handle)->binary.right = right; +} + +void Toy_private_emitAstGroup(Toy_Bucket** bucket, Toy_Ast** handle, Toy_Ast* child) { + (*handle) = (Toy_Ast*)Toy_partBucket(bucket, sizeof(Toy_Ast)); + + (*handle)->group.type = TOY_AST_GROUP; + (*handle)->group.child = child; +} + +void Toy_private_emitAstPass(Toy_Bucket** bucket, Toy_Ast** handle) { + (*handle) = (Toy_Ast*)Toy_partBucket(bucket, sizeof(Toy_Ast)); + + (*handle)->pass.type = TOY_AST_PASS; +} + +void Toy_private_emitAstError(Toy_Bucket** bucket, Toy_Ast** handle) { + (*handle) = (Toy_Ast*)Toy_partBucket(bucket, sizeof(Toy_Ast)); + + (*handle)->error.type = TOY_AST_ERROR; +} diff --git a/source/toy_ast.h b/source/toy_ast.h index 24ad45b..cdf8065 100644 --- a/source/toy_ast.h +++ b/source/toy_ast.h @@ -1,9 +1,108 @@ #pragma once #include "toy_common.h" +#include "toy_memory.h" +#include "toy_value.h" + +//each major type typedef enum Toy_AstType { + TOY_AST_BLOCK, + + TOY_AST_VALUE, + TOY_AST_UNARY, + TOY_AST_BINARY, + TOY_AST_GROUP, + TOY_AST_PASS, TOY_AST_ERROR, } Toy_AstType; +//flags are handled differently by different types +typedef enum Toy_AstFlag { + //binary flags + TOY_AST_FLAG_ADD, + TOY_AST_FLAG_SUBTRACT, + TOY_AST_FLAG_MULTIPLY, + TOY_AST_FLAG_DIVIDE, + TOY_AST_FLAG_MODULO, + TOY_AST_FLAG_COMPARE_EQUAL, + TOY_AST_FLAG_COMPARE_NOT, + TOY_AST_FLAG_COMPARE_LESS, + TOY_AST_FLAG_COMPARE_LESS_EQUAL, + TOY_AST_FLAG_COMPARE_GREATER, + TOY_AST_FLAG_COMPARE_GREATER_EQUAL, + TOY_AST_FLAG_AND, + TOY_AST_FLAG_OR, + + //unary flags + TOY_AST_FLAG_NEGATE, + TOY_AST_FLAG_INCREMENT, + TOY_AST_FLAG_DECREMENT, + + // TOY_AST_FLAG_TERNARY, +} Toy_AstFlag; + +//the root AST type +typedef union Toy_Ast Toy_Ast; + +void Toy_private_initAstBlock(Toy_Bucket** bucket, Toy_Ast** handle); +void Toy_private_appendAstBlock(Toy_Bucket** bucket, Toy_Ast** handle, Toy_Ast* child); + +void Toy_private_emitAstValue(Toy_Bucket** bucket, Toy_Ast** handle, Toy_Value value); +void Toy_private_emitAstUnary(Toy_Bucket** bucket, Toy_Ast** handle, Toy_AstFlag flag, Toy_Ast* child); +void Toy_private_emitAstBinary(Toy_Bucket** bucket, Toy_Ast** handle,Toy_AstFlag flag, Toy_Ast* left, Toy_Ast* right); +void Toy_private_emitAstGroup(Toy_Bucket** bucket, Toy_Ast** handle, Toy_Ast* child); + +void Toy_private_emitAstPass(Toy_Bucket** bucket, Toy_Ast** handle); +void Toy_private_emitAstError(Toy_Bucket** bucket, Toy_Ast** handle); + +typedef struct Toy_AstBlock { + Toy_AstType type; + Toy_Ast* child; //begin encoding the line + Toy_Ast* next; //'next' is either an AstBlock or null + Toy_Ast* tail; //'tail' - either points to the tail of the current list, or null; only used by the head of a list as an optimisation +} Toy_AstBlock; + +typedef struct Toy_AstValue { + Toy_AstType type; + Toy_Value value; +} Toy_AstValue; + +typedef struct Toy_AstUnary { + Toy_AstType type; + Toy_AstFlag flag; + Toy_Ast* child; +} Toy_AstUnary; + +typedef struct Toy_AstBinary { + Toy_AstType type; + Toy_Ast* left; + Toy_AstFlag flag; + Toy_Ast* right; +} Toy_AstBinary; + +typedef struct Toy_AstGroup { + Toy_AstType type; + Toy_Ast* child; +} Toy_AstGroup; + +typedef struct Toy_AstPass { + Toy_AstType type; +} Toy_AstPass; + +typedef struct Toy_AstError { + Toy_AstType type; + //TODO: more data regarding the error +} Toy_AstError; + +union Toy_Ast { + Toy_AstType type; //4 + Toy_AstBlock block; //12 + Toy_AstValue value; //12 + Toy_AstUnary unary; //12 + Toy_AstBinary binary; //16 + Toy_AstGroup group; //8 + Toy_AstPass pass; //4 + Toy_AstError error; //4 +}; //16 diff --git a/source/toy_console_colors.h b/source/toy_console_colors.h index ceb319b..0ece69e 100644 --- a/source/toy_console_colors.h +++ b/source/toy_console_colors.h @@ -37,6 +37,7 @@ NOTE: you need both font AND background for these to work #define TOY_CC_BACK_WHITE "47m" //useful +#define TOY_CC_DEBUG TOY_CC_FONT_BLUE TOY_CC_BACK_BLACK #define TOY_CC_NOTICE TOY_CC_FONT_GREEN TOY_CC_BACK_BLACK #define TOY_CC_WARN TOY_CC_FONT_YELLOW TOY_CC_BACK_BLACK #define TOY_CC_ERROR TOY_CC_FONT_RED TOY_CC_BACK_BLACK @@ -67,6 +68,7 @@ NOTE: you need both font AND background for these to work #define TOY_CC_BACK_WHITE //useful +#define TOY_CC_DEBUG TOY_CC_FONT_BLUE TOY_CC_BACK_BLACK #define TOY_CC_NOTICE TOY_CC_FONT_GREEN TOY_CC_BACK_BLACK #define TOY_CC_WARN TOY_CC_FONT_YELLOW TOY_CC_BACK_BLACK #define TOY_CC_ERROR TOY_CC_FONT_RED TOY_CC_BACK_BLACK diff --git a/source/toy_lexer.c b/source/toy_lexer.c index 5102095..2af2006 100644 --- a/source/toy_lexer.c +++ b/source/toy_lexer.c @@ -261,7 +261,7 @@ Toy_Token Toy_private_scanLexer(Toy_Lexer* lexer) { case '/': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_DIVIDE_ASSIGN : TOY_TOKEN_OPERATOR_DIVIDE); case '%': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_MODULO_ASSIGN : TOY_TOKEN_OPERATOR_MODULO); - case '!': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_COMPARE_NOT : TOY_TOKEN_OPERATOR_INVERT); + case '!': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_COMPARE_NOT : TOY_TOKEN_OPERATOR_NEGATE); case '=': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_COMPARE_EQUAL : TOY_TOKEN_OPERATOR_ASSIGN); case '<': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_COMPARE_LESS_EQUAL : TOY_TOKEN_OPERATOR_COMPARE_LESS); diff --git a/source/toy_memory.h b/source/toy_memory.h index 26d8381..55d4c81 100644 --- a/source/toy_memory.h +++ b/source/toy_memory.h @@ -24,6 +24,15 @@ TOY_API void* Toy_reallocate(void* pointer, size_t oldSize, size_t newSize); //immobile "bucket" memory structure for custom allocators +#define TOY_BUCKET_INIT(type, bucket, capacity) \ + Toy_initBucket(&(bucket), sizeof(type)*(capacity)) + +#define TOY_BUCKET_PART(type, bucket) \ + (type*)Toy_partBucket(&(bucket), sizeof(type)) + +#define TOY_BUCKET_FREE(bucket) \ + Toy_freeBucket(&(bucket)) + typedef struct Toy_Bucket { struct Toy_Bucket* next; void* contents; @@ -34,13 +43,3 @@ typedef struct Toy_Bucket { TOY_API void Toy_initBucket(Toy_Bucket** bucketHandle, size_t capacity); TOY_API void* Toy_partBucket(Toy_Bucket** bucketHandle, size_t space); TOY_API void Toy_freeBucket(Toy_Bucket** bucketHandle); - -#define TOY_BUCKET_INIT(type, bucket, capacity) \ - Toy_initBucket(&(bucket), sizeof(type)*(capacity)) - -#define TOY_BUCKET_PART(type, bucket) \ - Toy_partBucket(&(bucket), sizeof(type)) - -#define TOY_BUCKET_FREE(bucket) \ - Toy_freeBucket(&(bucket)) - diff --git a/source/toy_opcodes.h b/source/toy_opcodes.h index 22d7ecb..07df503 100644 --- a/source/toy_opcodes.h +++ b/source/toy_opcodes.h @@ -1,6 +1,8 @@ #pragma once typedef enum Toy_OpcodeType { + // + TOY_OPCODE_PASS, TOY_OPCODE_ERROR, TOY_OPCODE_EOF, diff --git a/source/toy_parser.c b/source/toy_parser.c new file mode 100644 index 0000000..e69de29 diff --git a/source/toy_parser.h b/source/toy_parser.h new file mode 100644 index 0000000..e69de29 diff --git a/source/toy_token_types.h b/source/toy_token_types.h index e3095a7..0a80c44 100644 --- a/source/toy_token_types.h +++ b/source/toy_token_types.h @@ -84,7 +84,7 @@ typedef enum Toy_TokenType { //other operators TOY_TOKEN_OPERATOR_AND, TOY_TOKEN_OPERATOR_OR, - TOY_TOKEN_OPERATOR_INVERT, + TOY_TOKEN_OPERATOR_NEGATE, TOY_TOKEN_OPERATOR_QUESTION, TOY_TOKEN_OPERATOR_COLON, diff --git a/source/toy_value.h b/source/toy_value.h index afacf39..f7afab1 100644 --- a/source/toy_value.h +++ b/source/toy_value.h @@ -14,7 +14,7 @@ typedef enum Toy_ValueType { TOY_VALUE_OPAQUE, } Toy_ValueType; -//4 bytes in size +//8 bytes in size typedef struct Toy_Value { union { bool boolean; //1 diff --git a/tests/cases/test_ast.c b/tests/cases/test_ast.c new file mode 100644 index 0000000..5da6b78 --- /dev/null +++ b/tests/cases/test_ast.c @@ -0,0 +1,241 @@ +#include "toy_ast.h" +#include "toy_console_colors.h" + +#include + +int test_sizeof_ast() { +#define TEST_SIZEOF(type, size) \ + if (sizeof(type) != size) { \ + fprintf(stderr, TOY_CC_ERROR "ERROR: sizeof(" #type ") is %d, expected %d\n" TOY_CC_RESET, (int)sizeof(type), size); \ + ++err; \ + } + + //count errors + int err = 0; + + //run for each type + TEST_SIZEOF(Toy_AstType, 4); + TEST_SIZEOF(Toy_AstBlock, 16); + TEST_SIZEOF(Toy_AstValue, 12); + TEST_SIZEOF(Toy_AstUnary, 12); + TEST_SIZEOF(Toy_AstBinary, 16); + TEST_SIZEOF(Toy_AstGroup, 8); + TEST_SIZEOF(Toy_AstPass, 4); + TEST_SIZEOF(Toy_AstError, 4); + TEST_SIZEOF(Toy_Ast, 16); + +#undef TEST_SIZEOF + + return -err; +} + +int test_type_emission() { + //emit value + { + //bucket setup + Toy_Bucket* bucket = NULL; + TOY_BUCKET_INIT(Toy_Ast, bucket, 8); + + //emit to an AST + Toy_Ast* ast = NULL; + Toy_private_emitAstValue(&bucket, &ast, TOY_VALUE_TO_INTEGER(42)); + + //check if it worked + if ( + ast == NULL || + ast->type != TOY_AST_VALUE || + TOY_VALUE_AS_INTEGER(ast->value.value) != 42) + { + fprintf(stderr, TOY_CC_ERROR "ERROR: failed to emit a value as 'Toy_Ast', state unknown\n" TOY_CC_RESET); + return -1; + } + + //bucket free + TOY_BUCKET_FREE(bucket); + } + + //emit unary + { + //bucket setup + Toy_Bucket* bucket = NULL; + TOY_BUCKET_INIT(Toy_Ast, bucket, 8); + + //build the AST + Toy_Ast* ast = NULL; + Toy_Ast* child = NULL; + Toy_private_emitAstValue(&bucket, &child, TOY_VALUE_TO_INTEGER(42)); + Toy_private_emitAstUnary(&bucket, &ast, TOY_AST_FLAG_NEGATE, child); + + //check if it worked + if ( + ast == NULL || + ast->type != TOY_AST_UNARY || + ast->unary.flag != TOY_AST_FLAG_NEGATE || + ast->unary.child->type != TOY_AST_VALUE || + TOY_VALUE_AS_INTEGER(ast->unary.child->value.value) != 42) + { + fprintf(stderr, TOY_CC_ERROR "ERROR: failed to emit a unary as 'Toy_Ast', state unknown\n" TOY_CC_RESET); + return -1; + } + + //bucket free + TOY_BUCKET_FREE(bucket); + } + + //emit binary + { + //bucket setup + Toy_Bucket* bucket = NULL; + TOY_BUCKET_INIT(Toy_Ast, bucket, 8); + + //build the AST + Toy_Ast* ast = NULL; + Toy_Ast* left = NULL; + Toy_Ast* right = NULL; + Toy_private_emitAstValue(&bucket, &left, TOY_VALUE_TO_INTEGER(42)); + Toy_private_emitAstValue(&bucket, &right, TOY_VALUE_TO_INTEGER(69)); + Toy_private_emitAstBinary(&bucket, &ast, TOY_AST_FLAG_ADD, left, right); + + //check if it worked + if ( + ast == NULL || + ast->type != TOY_AST_BINARY || + ast->binary.flag != TOY_AST_FLAG_ADD || + ast->binary.left->type != TOY_AST_VALUE || + TOY_VALUE_AS_INTEGER(ast->binary.left->value.value) != 42 || + ast->binary.right->type != TOY_AST_VALUE || + TOY_VALUE_AS_INTEGER(ast->binary.right->value.value) != 69) + { + fprintf(stderr, TOY_CC_ERROR "ERROR: failed to emit a binary as 'Toy_Ast', state unknown\n" TOY_CC_RESET); + return -1; + } + + //bucket free + TOY_BUCKET_FREE(bucket); + } + + //emit group + { + //bucket setup + Toy_Bucket* bucket = NULL; + TOY_BUCKET_INIT(Toy_Ast, bucket, 8); + + //build the AST + Toy_Ast* ast = NULL; + Toy_Ast* addition = NULL; + Toy_Ast* left = NULL; + Toy_Ast* right = NULL; + Toy_private_emitAstValue(&bucket, &left, TOY_VALUE_TO_INTEGER(42)); + Toy_private_emitAstValue(&bucket, &right, TOY_VALUE_TO_INTEGER(69)); + Toy_private_emitAstBinary(&bucket, &addition, TOY_AST_FLAG_ADD, left, right); + Toy_private_emitAstGroup(&bucket, &ast, addition); + + //check if it worked + if ( + ast == NULL || + ast->type != TOY_AST_GROUP || + ast->group.child == NULL || + ast->group.child->type != TOY_AST_BINARY || + ast->group.child->binary.flag != TOY_AST_FLAG_ADD || + ast->group.child->binary.left->type != TOY_AST_VALUE || + TOY_VALUE_AS_INTEGER(ast->group.child->binary.left->value.value) != 42 || + ast->group.child->binary.right->type != TOY_AST_VALUE || + TOY_VALUE_AS_INTEGER(ast->group.child->binary.right->value.value) != 69) + { + fprintf(stderr, TOY_CC_ERROR "ERROR: failed to emit a group as 'Toy_Ast', state unknown\n" TOY_CC_RESET); + return -1; + } + + //bucket free + TOY_BUCKET_FREE(bucket); + } + + //emit and append blocks of code + { + //bucket setup + Toy_Bucket* bucket = NULL; + TOY_BUCKET_INIT(Toy_Ast, bucket, 8); + + //initialize the root block + Toy_Ast* block = NULL; + Toy_private_initAstBlock(&bucket, &block); + + //loop over the ast emissions, appending each one as you go + for (int i = 0; i < 5; i++) { + //build the AST + Toy_Ast* ast = NULL; + Toy_Ast* addition = NULL; + Toy_Ast* left = NULL; + Toy_Ast* right = NULL; + Toy_private_emitAstValue(&bucket, &left, TOY_VALUE_TO_INTEGER(42)); + Toy_private_emitAstValue(&bucket, &right, TOY_VALUE_TO_INTEGER(69)); + Toy_private_emitAstBinary(&bucket, &addition, TOY_AST_FLAG_ADD, left, right); + Toy_private_emitAstGroup(&bucket, &ast, addition); + + Toy_private_appendAstBlock(&bucket, &block, ast); + } + + //check if it worked + Toy_Ast* iter = block; + + while(iter != NULL) { + if ( + iter->type != TOY_AST_BLOCK || + iter->block.child == NULL || + iter->block.child->type != TOY_AST_GROUP || + iter->block.child->group.child == NULL || + iter->block.child->group.child->type != TOY_AST_BINARY || + iter->block.child->group.child->binary.flag != TOY_AST_FLAG_ADD || + iter->block.child->group.child->binary.left->type != TOY_AST_VALUE || + TOY_VALUE_AS_INTEGER(iter->block.child->group.child->binary.left->value.value) != 42 || + iter->block.child->group.child->binary.right->type != TOY_AST_VALUE || + TOY_VALUE_AS_INTEGER(iter->block.child->group.child->binary.right->value.value) != 69) + { + fprintf(stderr, TOY_CC_ERROR "ERROR: failed to emit a block as 'Toy_Ast', state unknown\n" TOY_CC_RESET); + return -1; + } + + iter = iter->block.next; + } + + //additional check: count the bucket's total allocations + Toy_Bucket* biter = bucket; + int total = 0; + + while(biter != NULL) { + total += biter->count; + biter = biter->next; + } + + if (total != 25 * (int)sizeof(Toy_Ast)) { + fprintf(stderr, TOY_CC_ERROR "ERROR: unexpected number of allocations found in a bucket, expected %d, found %d\n" TOY_CC_RESET, 25 * (int)sizeof(Toy_Ast), total); + return -1; + } + + //bucket free + TOY_BUCKET_FREE(bucket); + } + + return 0; +} + +int main() { + //run each test set, returning the total errors given + int total = 0, res = 0; + + res = test_sizeof_ast(); + total += res; + + if (res == 0) { + printf(TOY_CC_NOTICE "All good\n" TOY_CC_RESET); + } + + res = test_type_emission(); + total += res; + + if (res == 0) { + printf(TOY_CC_NOTICE "All good\n" TOY_CC_RESET); + } + + return total; +} \ No newline at end of file