Added lexer, implemented tests build system

2026-04-15 14:54:07 +10:00 · 2024-08-11 20:15:09 +10:00
parent 82f63013d8
commit a912b6a29c
10 changed files with 632 additions and 14 deletions
--- a/source/toy.h
+++ b/source/toy.h
@@ -2,9 +2,11 @@

 //general utilities
 #include "toy_common.h"
+#include "toy_console_colors.h"

 //building blocks
 //

 //pipeline
-//
+#include "toy_lexer.h"
+
--- a/source/toy_console_colors.h
+++ b/source/toy_console_colors.h
@@ -0,0 +1,75 @@
+#pragma once
+
+/* toy_console_colors.h - console utility
+
+This file provides a number of macros that can set the color of text in a console
+window. These are used for convenience only. They are supposed to be dropped into
+a printf()'s first argument, like so:
+
+    printf(TOY_CC_NOTICE "Hello world" TOY_CC_RESET);
+
+NOTE: you need both font AND background for these to work
+
+*/
+
+//platform/compiler-specific instructions
+#if defined(__linux__) || defined(__MINGW32__) || defined(__GNUC__)
+
+//fonts color
+#define TOY_CC_FONT_BLACK      "\033[30;"
+#define TOY_CC_FONT_RED        "\033[31;"
+#define TOY_CC_FONT_GREEN      "\033[32;"
+#define TOY_CC_FONT_YELLOW     "\033[33;"
+#define TOY_CC_FONT_BLUE       "\033[34;"
+#define TOY_CC_FONT_PURPLE     "\033[35;"
+#define TOY_CC_FONT_DGREEN     "\033[6;"
+#define TOY_CC_FONT_WHITE      "\033[7;"
+#define TOY_CC_FONT_CYAN       "\x1b[36m"
+
+//background color
+#define TOY_CC_BACK_BLACK      "40m"
+#define TOY_CC_BACK_RED        "41m"
+#define TOY_CC_BACK_GREEN      "42m"
+#define TOY_CC_BACK_YELLOW     "43m"
+#define TOY_CC_BACK_BLUE       "44m"
+#define TOY_CC_BACK_PURPLE     "45m"
+#define TOY_CC_BACK_DGREEN     "46m"
+#define TOY_CC_BACK_WHITE      "47m"
+
+//useful
+#define TOY_CC_NOTICE TOY_CC_FONT_GREEN TOY_CC_BACK_BLACK
+#define TOY_CC_WARN TOY_CC_FONT_YELLOW TOY_CC_BACK_BLACK
+#define TOY_CC_ERROR TOY_CC_FONT_RED TOY_CC_BACK_BLACK
+#define TOY_CC_RESET "\033[0m"
+
+//for unsupported platforms, these become no-ops
+#else
+
+//fonts color
+#define TOY_CC_FONT_BLACK
+#define TOY_CC_FONT_RED
+#define TOY_CC_FONT_GREEN
+#define TOY_CC_FONT_YELLOW
+#define TOY_CC_FONT_BLUE
+#define TOY_CC_FONT_PURPLE
+#define TOY_CC_FONT_DGREEN
+#define TOY_CC_FONT_WHITE
+#define TOY_CC_FONT_CYAN
+
+//background color
+#define TOY_CC_BACK_BLACK
+#define TOY_CC_BACK_RED
+#define TOY_CC_BACK_GREEN
+#define TOY_CC_BACK_YELLOW
+#define TOY_CC_BACK_BLUE
+#define TOY_CC_BACK_PURPLE
+#define TOY_CC_BACK_DGREEN
+#define TOY_CC_BACK_WHITE
+
+//useful
+#define TOY_CC_NOTICE TOY_CC_FONT_GREEN TOY_CC_BACK_BLACK
+#define TOY_CC_WARN TOY_CC_FONT_YELLOW TOY_CC_BACK_BLACK
+#define TOY_CC_ERROR TOY_CC_FONT_RED TOY_CC_BACK_BLACK
+#define TOY_CC_RESET
+
+#endif
--- a/source/toy_lexer.c
+++ b/source/toy_lexer.c
@@ -0,0 +1,350 @@
+#include "toy_lexer.h"
+#include "toy_keywords.h"
+#include "toy_console_colors.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+
+//static generic utility functions
+static void cleanLexer(Toy_Lexer* lexer) {
+	lexer->start = 0;
+	lexer->current = 0;
+	lexer->line = 1;
+	lexer->source = NULL;
+}
+
+static bool isAtEnd(Toy_Lexer* lexer) {
+	return lexer->source[lexer->current] == '\0';
+}
+
+static char peek(Toy_Lexer* lexer) {
+	return lexer->source[lexer->current];
+}
+
+static char peekNext(Toy_Lexer* lexer) {
+	if (isAtEnd(lexer)) return '\0';
+	return lexer->source[lexer->current + 1];
+}
+
+static char advance(Toy_Lexer* lexer) {
+	if (isAtEnd(lexer)) {
+		return '\0';
+	}
+
+	//new line
+	if (lexer->source[lexer->current] == '\n') {
+		lexer->line++;
+	}
+
+	lexer->current++;
+	return lexer->source[lexer->current - 1];
+}
+
+static void eatWhitespace(Toy_Lexer* lexer) {
+	const char c = peek(lexer);
+
+	switch(c) {
+		case ' ':
+		case '\r':
+		case '\n':
+		case '\t':
+			advance(lexer);
+			break;
+
+		//comments
+		case '/':
+			//eat the line
+			if (peekNext(lexer) == '/') {
+				while (!isAtEnd(lexer) && advance(lexer) != '\n');
+				break;
+			}
+
+			//eat the block
+			if (peekNext(lexer) == '*') {
+				advance(lexer);
+				advance(lexer);
+				while(!isAtEnd(lexer) && !(peek(lexer) == '*' && peekNext(lexer) == '/')) advance(lexer);
+				advance(lexer);
+				advance(lexer);
+				break;
+			}
+		return;
+
+		default:
+			return;
+	}
+
+	//tail recursion
+	eatWhitespace(lexer);
+}
+
+static bool isDigit(Toy_Lexer* lexer) {
+	return peek(lexer) >= '0' && peek(lexer) <= '9';
+}
+
+static bool isAlpha(Toy_Lexer* lexer) {
+	return
+		(peek(lexer) >= 'A' && peek(lexer) <= 'Z') ||
+		(peek(lexer) >= 'a' && peek(lexer) <= 'z') ||
+		peek(lexer) == '_'
+	;
+}
+
+static bool match(Toy_Lexer* lexer, char c) {
+	if (peek(lexer) == c) {
+		advance(lexer);
+		return true;
+	}
+
+	return false;
+}
+
+//token generators
+static Toy_Token makeErrorToken(Toy_Lexer* lexer, char* msg) {
+	Toy_Token token;
+
+	token.type = TOY_TOKEN_ERROR;
+	token.length = strlen(msg);
+	token.line = lexer->line;
+	token.lexeme = msg;
+
+	return token;
+}
+
+static Toy_Token makeToken(Toy_Lexer* lexer, Toy_TokenType type) {
+	Toy_Token token;
+
+	token.type = type;
+	token.length = lexer->current - lexer->start;
+	token.line = lexer->line;
+	token.lexeme = &lexer->source[lexer->current - token.length];
+
+	return token;
+}
+
+static Toy_Token makeIntegerOrFloat(Toy_Lexer* lexer) {
+	Toy_TokenType type = TOY_TOKEN_LITERAL_INTEGER; //assume we're reading an integer
+
+	//the character '_' can be inserted into numbers as a separator
+	while(isDigit(lexer) || peek(lexer) == '_') advance(lexer);
+
+	if (peek(lexer) == '.' && (peekNext(lexer) >= '0' && peekNext(lexer) <= '9')) { //peekNext(lexer) == digit
+		type = TOY_TOKEN_LITERAL_FLOAT; //change the assumption to reading a float
+		advance(lexer); //eat the '.'
+
+		//'_' again
+		while(isDigit(lexer) || peek(lexer) == '_') advance(lexer);
+	}
+
+	//make the token
+	Toy_Token token;
+
+	token.type = type;
+	token.length = lexer->current - lexer->start;
+	token.line = lexer->line;
+	token.lexeme = &lexer->source[lexer->start];
+
+	return token;
+}
+
+static bool isEscapableCharacter(char c) {
+	switch (c) {
+		case 'n':
+		case 't':
+		case '\\':
+		case '"':
+			return true;
+
+		default:
+			return false;
+	}
+}
+
+static Toy_Token makeString(Toy_Lexer* lexer, char terminator) {
+	while (!isAtEnd(lexer)) {
+		//stop if you've hit the terminator
+		if (peek(lexer) == terminator) {
+			advance(lexer); //eat the terminator
+			break;
+		}
+
+		//skip escaped control characters
+		if (peek(lexer) == '\\' && isEscapableCharacter(peekNext(lexer))) {
+			advance(lexer);
+			advance(lexer);
+			continue;
+		}
+
+		//otherwise
+		advance(lexer);
+	}
+
+	if (isAtEnd(lexer)) {
+		return makeErrorToken(lexer, "Unterminated string");
+	}
+
+	//make the token
+	Toy_Token token;
+
+	token.type = TOY_TOKEN_LITERAL_STRING;
+	token.length = lexer->current - lexer->start - 2; //-1 to omit the quotes
+	token.line = lexer->line;
+	token.lexeme = &lexer->source[lexer->start + 1]; //+1 to omit the first quote
+
+	return token;
+}
+
+static Toy_Token makeKeywordOrIdentifier(Toy_Lexer* lexer) {
+	advance(lexer); //first letter can only be alpha
+
+	while(isDigit(lexer) || isAlpha(lexer)) {
+		advance(lexer);
+	}
+
+	//scan for a keyword
+	for (int i = 0; Toy_private_keywords[i].keyword; i++) {
+		//WONTFIX: could squeeze miniscule performance gain from this, but ROI isn't worth it
+		if (strlen(Toy_private_keywords[i].keyword) == (size_t)(lexer->current - lexer->start) && !strncmp(Toy_private_keywords[i].keyword, &lexer->source[lexer->start], lexer->current - lexer->start)) {
+			//make token (keyword)
+			Toy_Token token;
+
+			token.type = Toy_private_keywords[i].type;
+			token.length = lexer->current - lexer->start;
+			token.line = lexer->line;
+			token.lexeme = &lexer->source[lexer->start];
+
+			return token;
+		}
+	}
+
+	//make token (identifier)
+	Toy_Token token;
+
+	token.type = TOY_TOKEN_IDENTIFIER;
+	token.length = lexer->current - lexer->start;
+	token.line = lexer->line;
+	token.lexeme = &lexer->source[lexer->start];
+
+	return token;
+}
+
+//exposed functions
+void Toy_bindLexer(Toy_Lexer* lexer, const char* source) {
+	cleanLexer(lexer);
+	lexer->source = source;
+}
+
+Toy_Token Toy_private_scanLexer(Toy_Lexer* lexer) {
+	eatWhitespace(lexer);
+
+	lexer->start = lexer->current;
+
+	if (isAtEnd(lexer)) return makeToken(lexer, TOY_TOKEN_EOF);
+
+	if (isDigit(lexer)) return makeIntegerOrFloat(lexer);
+	if (isAlpha(lexer)) return makeKeywordOrIdentifier(lexer);
+
+	char c = advance(lexer);
+
+	switch(c) {
+		case '(': return makeToken(lexer, TOY_TOKEN_OPERATOR_PAREN_LEFT);
+		case ')': return makeToken(lexer, TOY_TOKEN_OPERATOR_PAREN_RIGHT);
+		case '[': return makeToken(lexer, TOY_TOKEN_OPERATOR_BRACKET_LEFT);
+		case ']': return makeToken(lexer, TOY_TOKEN_OPERATOR_BRACKET_RIGHT);
+		case '{': return makeToken(lexer, TOY_TOKEN_OPERATOR_BRACE_LEFT);
+		case '}': return makeToken(lexer, TOY_TOKEN_OPERATOR_BRACE_RIGHT);
+
+		case '+': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_ADD_ASSIGN : match(lexer, '+') ? TOY_TOKEN_OPERATOR_INCREMENT : TOY_TOKEN_OPERATOR_ADD);
+		case '-': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_SUBTRACT_ASSIGN : match(lexer, '-') ? TOY_TOKEN_OPERATOR_DECREMENT : TOY_TOKEN_OPERATOR_SUBTRACT);
+		case '*': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_MULTIPLY_ASSIGN : TOY_TOKEN_OPERATOR_MULTIPLY);
+		case '/': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_DIVIDE_ASSIGN : TOY_TOKEN_OPERATOR_DIVIDE);
+		case '%': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_MODULO_ASSIGN : 	TOY_TOKEN_OPERATOR_MODULO);
+
+		case '!': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_COMPARE_NOT : TOY_TOKEN_OPERATOR_INVERT);
+		case '=': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_COMPARE_EQUAL : TOY_TOKEN_OPERATOR_ASSIGN);
+
+		case '<': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_COMPARE_LESS_EQUAL : TOY_TOKEN_OPERATOR_COMPARE_LESS);
+		case '>': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_COMPARE_GREATER_EQUAL : TOY_TOKEN_OPERATOR_COMPARE_GREATER);
+
+		case '&': //TOY_TOKEN_OPERATOR_AMPERSAND is unused
+			if (match(lexer, '&')) {
+				return makeToken(lexer, TOY_TOKEN_OPERATOR_AND);
+			} else {
+				return makeErrorToken(lexer, "Unexpected '&'");
+			}
+
+		case '|': //TOY_TOKEN_OPERATOR_PIPE is unused
+			if (match(lexer, '|')) {
+				return makeToken(lexer, TOY_TOKEN_OPERATOR_OR);
+			} else {
+				return makeErrorToken(lexer, "Unexpected '|'");
+			}
+
+		case '?': return makeToken(lexer, TOY_TOKEN_OPERATOR_QUESTION);
+		case ':': return makeToken(lexer, TOY_TOKEN_OPERATOR_COLON);
+		case ';': return makeToken(lexer, TOY_TOKEN_OPERATOR_SEMICOLON);
+		case ',': return makeToken(lexer, TOY_TOKEN_OPERATOR_COMMA);
+
+		case '.':
+			if (match(lexer, '.')) {
+				if (match(lexer, '.')) {
+					return makeToken(lexer, TOY_TOKEN_OPERATOR_REST); //three dots
+				}
+				else {
+					return makeToken(lexer, TOY_TOKEN_OPERATOR_CONCAT); //two dots
+				}
+			}
+			else {
+				return makeToken(lexer, TOY_TOKEN_OPERATOR_DOT); //one dot
+			}
+
+		case '"':
+			return makeString(lexer, c);
+
+		default: {
+			return makeErrorToken(lexer, "Unknown token value found in lexer");
+		}
+	}
+}
+
+static void trim(char** s, int* l) { //util
+	while( isspace(( (*((unsigned char**)(s)))[(*l) - 1] )) ) (*l)--;
+	while(**s && isspace( **(unsigned char**)(s)) ) { (*s)++; (*l)--; }
+}
+
+//for debugging
+void Toy_private_printToken(Toy_Token* token) {
+	//print errors
+	if (token->type == TOY_TOKEN_ERROR) {
+		printf(TOY_CC_ERROR "Error\t%d\t%.*s\n" TOY_CC_RESET, token->line, token->length, token->lexeme);
+		return;
+	}
+
+	//read pass token, even though it isn't generated
+	if (token->type == TOY_TOKEN_PASS) {
+		printf(TOY_CC_NOTICE "Error\t%d\t%.*s\n" TOY_CC_RESET, token->line, token->length, token->lexeme);
+		return;
+	}
+
+	//print the line number
+	printf("\t%d\t%d\t", token->type, token->line);
+
+	//print based on type
+	if (token->type == TOY_TOKEN_IDENTIFIER || token->type == TOY_TOKEN_LITERAL_INTEGER || token->type == TOY_TOKEN_LITERAL_FLOAT || token->type == TOY_TOKEN_LITERAL_STRING) {
+		printf("%.*s\t", token->length, token->lexeme);
+	} else {
+		const char* keyword = Toy_private_findKeywordByType(token->type);
+
+		if (keyword != NULL) {
+			printf("%s", keyword);
+		} else {
+			char* str = (char*)token->lexeme; //strip const-ness for trimming
+			int length = token->length;
+			trim(&str, &length);
+			printf("%.*s", length, str);
+		}
+	}
+
+	printf("\n");
+}
--- a/source/toy_lexer.h
+++ b/source/toy_lexer.h
@@ -21,5 +21,5 @@ typedef struct {

 TOY_API void Toy_bindLexer(Toy_Lexer* lexer, const char* source);
 TOY_API Toy_Token Toy_private_scanLexer(Toy_Lexer* lexer);
-TOY_API void Toy_private_printToken(Toy_Token* token);
+TOY_API void Toy_private_printToken(Toy_Token* token); //debugging

--- a/source/toy_token_types.h
+++ b/source/toy_token_types.h
@@ -65,27 +65,38 @@ typedef enum Toy_TokenType {
 	TOY_TOKEN_OPERATOR_DECREMENT,
 	TOY_TOKEN_OPERATOR_ASSIGN,

-	//logical operators
-	TOY_TOKEN_OPERATOR_EQUAL,
-	TOY_TOKEN_OPERATOR_NOT_EQUAL,_EQUAL
+	//comparator operators
+	TOY_TOKEN_OPERATOR_COMPARE_EQUAL,
+	TOY_TOKEN_OPERATOR_COMPARE_NOT,
+	TOY_TOKEN_OPERATOR_COMPARE_LESS,
+	TOY_TOKEN_OPERATOR_COMPARE_LESS_EQUAL,
+	TOY_TOKEN_OPERATOR_COMPARE_GREATER,
+	TOY_TOKEN_OPERATOR_COMPARE_GREATER_EQUAL,
+
+	//structural operators
 	TOY_TOKEN_OPERATOR_PAREN_LEFT,
 	TOY_TOKEN_OPERATOR_PAREN_RIGHT,
 	TOY_TOKEN_OPERATOR_BRACKET_LEFT,
 	TOY_TOKEN_OPERATOR_BRACKET_RIGHT,
 	TOY_TOKEN_OPERATOR_BRACE_LEFT,
 	TOY_TOKEN_OPERATOR_BRACE_RIGHT,
-	TOY_TOKEN_OPERATOR_SEMICOLON,
-	TOY_TOKEN_OPERATOR_COMMA,

-	//other operators (context sensitive)
+	//other operators
+	TOY_TOKEN_OPERATOR_AND,
+	TOY_TOKEN_OPERATOR_OR,
+	TOY_TOKEN_OPERATOR_INVERT,
 	TOY_TOKEN_OPERATOR_QUESTION,
 	TOY_TOKEN_OPERATOR_COLON,

+	TOY_TOKEN_OPERATOR_SEMICOLON,
+	TOY_TOKEN_OPERATOR_COMMA,
+
 	TOY_TOKEN_OPERATOR_DOT, // .
 	TOY_TOKEN_OPERATOR_CONCAT, // ..
 	TOY_TOKEN_OPERATOR_REST, // ...

 	//unused operators
+	TOY_TOKEN_OPERATOR_AMPERSAND, // &
 	TOY_TOKEN_OPERATOR_PIPE, // |

 	//meta tokens