mirror of
https://github.com/krgamestudios/Toy.git
synced 2026-04-15 14:54:07 +10:00
Added lexer, implemented tests build system
This commit is contained in:
4
.github/workflows/comprehensive-tests.yml
vendored
4
.github/workflows/comprehensive-tests.yml
vendored
@@ -26,5 +26,5 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: make test (placeholder)
|
||||
run: echo ""
|
||||
- name: make test
|
||||
run: make tests
|
||||
|
||||
63
makefile
63
makefile
@@ -1,4 +1,63 @@
|
||||
export CFLAGS+=-std=c17 -pedantic -Werror
|
||||
#directories
|
||||
export TOY_SOURCEDIR=source
|
||||
export TOY_OUTDIR=out
|
||||
export TOY_OBJDIR=obj
|
||||
|
||||
all:
|
||||
#compiler settings
|
||||
CFLAGS+=-std=c17 -pedantic -Werror
|
||||
LIBS=-lm
|
||||
|
||||
#file names
|
||||
TOY_SOURCEFILES=$(wildcard $(TOY_SOURCEDIR)/*.c)
|
||||
|
||||
#targets
|
||||
all: clean tests
|
||||
@echo no targets ready
|
||||
|
||||
.PHONY: tests
|
||||
tests:
|
||||
$(MAKE) -C tests
|
||||
|
||||
#util targets
|
||||
$(TOY_OUTDIR):
|
||||
mkdir $(TOY_OUTDIR)
|
||||
|
||||
$(TOY_OBJDIR):
|
||||
mkdir $(TOY_OBJDIR)
|
||||
|
||||
$(TOY_OBJDIR)/%.o: $(TOY_SOURCEDIR)/%.c
|
||||
$(CC) -c -o $@ $< $(addprefix -I,$(TOY_SOURCEDIR)) $(CFLAGS)
|
||||
|
||||
#util commands
|
||||
.PHONY: clean
|
||||
clean:
|
||||
ifeq ($(shell uname),Linux)
|
||||
find . -type f -name '*.o' -delete
|
||||
find . -type f -name '*.a' -delete
|
||||
find . -type f -name '*.exe' -delete
|
||||
find . -type f -name '*.dll' -delete
|
||||
find . -type f -name '*.lib' -delete
|
||||
find . -type f -name '*.so' -delete
|
||||
find . -type f -name '*.dylib' -delete
|
||||
find . -type d -name 'out' -delete
|
||||
find . -type d -name 'obj' -delete
|
||||
else ifeq ($(OS),Windows_NT)
|
||||
del *.o *.a *.exe *.dll *.lib *.so *.dylib
|
||||
del /s out
|
||||
del /s obj
|
||||
else ifeq ($(shell uname),Darwin)
|
||||
find . -type f -name '*.o' -delete
|
||||
find . -type f -name '*.a' -delete
|
||||
find . -type f -name '*.exe' -delete
|
||||
find . -type f -name '*.dll' -delete
|
||||
find . -type f -name '*.lib' -delete
|
||||
find . -type f -name '*.so' -delete
|
||||
find . -type f -name '*.dylib' -delete
|
||||
find . -type d -name 'out' -delete
|
||||
find . -type d -name 'obj' -delete
|
||||
else
|
||||
@echo "Deletion failed - what platform is this?"
|
||||
endif
|
||||
|
||||
.PHONY: rebuild
|
||||
rebuild: clean all
|
||||
|
||||
@@ -2,9 +2,11 @@
|
||||
|
||||
//general utilities
|
||||
#include "toy_common.h"
|
||||
#include "toy_console_colors.h"
|
||||
|
||||
//building blocks
|
||||
//
|
||||
|
||||
//pipeline
|
||||
//
|
||||
#include "toy_lexer.h"
|
||||
|
||||
|
||||
75
source/toy_console_colors.h
Normal file
75
source/toy_console_colors.h
Normal file
@@ -0,0 +1,75 @@
|
||||
#pragma once
|
||||
|
||||
/* toy_console_colors.h - console utility
|
||||
|
||||
This file provides a number of macros that can set the color of text in a console
|
||||
window. These are used for convenience only. They are supposed to be dropped into
|
||||
a printf()'s first argument, like so:
|
||||
|
||||
printf(TOY_CC_NOTICE "Hello world" TOY_CC_RESET);
|
||||
|
||||
NOTE: you need both font AND background for these to work
|
||||
|
||||
*/
|
||||
|
||||
//platform/compiler-specific instructions
|
||||
#if defined(__linux__) || defined(__MINGW32__) || defined(__GNUC__)
|
||||
|
||||
//fonts color
|
||||
#define TOY_CC_FONT_BLACK "\033[30;"
|
||||
#define TOY_CC_FONT_RED "\033[31;"
|
||||
#define TOY_CC_FONT_GREEN "\033[32;"
|
||||
#define TOY_CC_FONT_YELLOW "\033[33;"
|
||||
#define TOY_CC_FONT_BLUE "\033[34;"
|
||||
#define TOY_CC_FONT_PURPLE "\033[35;"
|
||||
#define TOY_CC_FONT_DGREEN "\033[6;"
|
||||
#define TOY_CC_FONT_WHITE "\033[7;"
|
||||
#define TOY_CC_FONT_CYAN "\x1b[36m"
|
||||
|
||||
//background color
|
||||
#define TOY_CC_BACK_BLACK "40m"
|
||||
#define TOY_CC_BACK_RED "41m"
|
||||
#define TOY_CC_BACK_GREEN "42m"
|
||||
#define TOY_CC_BACK_YELLOW "43m"
|
||||
#define TOY_CC_BACK_BLUE "44m"
|
||||
#define TOY_CC_BACK_PURPLE "45m"
|
||||
#define TOY_CC_BACK_DGREEN "46m"
|
||||
#define TOY_CC_BACK_WHITE "47m"
|
||||
|
||||
//useful
|
||||
#define TOY_CC_NOTICE TOY_CC_FONT_GREEN TOY_CC_BACK_BLACK
|
||||
#define TOY_CC_WARN TOY_CC_FONT_YELLOW TOY_CC_BACK_BLACK
|
||||
#define TOY_CC_ERROR TOY_CC_FONT_RED TOY_CC_BACK_BLACK
|
||||
#define TOY_CC_RESET "\033[0m"
|
||||
|
||||
//for unsupported platforms, these become no-ops
|
||||
#else
|
||||
|
||||
//fonts color
|
||||
#define TOY_CC_FONT_BLACK
|
||||
#define TOY_CC_FONT_RED
|
||||
#define TOY_CC_FONT_GREEN
|
||||
#define TOY_CC_FONT_YELLOW
|
||||
#define TOY_CC_FONT_BLUE
|
||||
#define TOY_CC_FONT_PURPLE
|
||||
#define TOY_CC_FONT_DGREEN
|
||||
#define TOY_CC_FONT_WHITE
|
||||
#define TOY_CC_FONT_CYAN
|
||||
|
||||
//background color
|
||||
#define TOY_CC_BACK_BLACK
|
||||
#define TOY_CC_BACK_RED
|
||||
#define TOY_CC_BACK_GREEN
|
||||
#define TOY_CC_BACK_YELLOW
|
||||
#define TOY_CC_BACK_BLUE
|
||||
#define TOY_CC_BACK_PURPLE
|
||||
#define TOY_CC_BACK_DGREEN
|
||||
#define TOY_CC_BACK_WHITE
|
||||
|
||||
//useful
|
||||
#define TOY_CC_NOTICE TOY_CC_FONT_GREEN TOY_CC_BACK_BLACK
|
||||
#define TOY_CC_WARN TOY_CC_FONT_YELLOW TOY_CC_BACK_BLACK
|
||||
#define TOY_CC_ERROR TOY_CC_FONT_RED TOY_CC_BACK_BLACK
|
||||
#define TOY_CC_RESET
|
||||
|
||||
#endif
|
||||
350
source/toy_lexer.c
Normal file
350
source/toy_lexer.c
Normal file
@@ -0,0 +1,350 @@
|
||||
#include "toy_lexer.h"
|
||||
#include "toy_keywords.h"
|
||||
#include "toy_console_colors.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <ctype.h>
|
||||
|
||||
//static generic utility functions
|
||||
static void cleanLexer(Toy_Lexer* lexer) {
|
||||
lexer->start = 0;
|
||||
lexer->current = 0;
|
||||
lexer->line = 1;
|
||||
lexer->source = NULL;
|
||||
}
|
||||
|
||||
static bool isAtEnd(Toy_Lexer* lexer) {
|
||||
return lexer->source[lexer->current] == '\0';
|
||||
}
|
||||
|
||||
static char peek(Toy_Lexer* lexer) {
|
||||
return lexer->source[lexer->current];
|
||||
}
|
||||
|
||||
static char peekNext(Toy_Lexer* lexer) {
|
||||
if (isAtEnd(lexer)) return '\0';
|
||||
return lexer->source[lexer->current + 1];
|
||||
}
|
||||
|
||||
static char advance(Toy_Lexer* lexer) {
|
||||
if (isAtEnd(lexer)) {
|
||||
return '\0';
|
||||
}
|
||||
|
||||
//new line
|
||||
if (lexer->source[lexer->current] == '\n') {
|
||||
lexer->line++;
|
||||
}
|
||||
|
||||
lexer->current++;
|
||||
return lexer->source[lexer->current - 1];
|
||||
}
|
||||
|
||||
static void eatWhitespace(Toy_Lexer* lexer) {
|
||||
const char c = peek(lexer);
|
||||
|
||||
switch(c) {
|
||||
case ' ':
|
||||
case '\r':
|
||||
case '\n':
|
||||
case '\t':
|
||||
advance(lexer);
|
||||
break;
|
||||
|
||||
//comments
|
||||
case '/':
|
||||
//eat the line
|
||||
if (peekNext(lexer) == '/') {
|
||||
while (!isAtEnd(lexer) && advance(lexer) != '\n');
|
||||
break;
|
||||
}
|
||||
|
||||
//eat the block
|
||||
if (peekNext(lexer) == '*') {
|
||||
advance(lexer);
|
||||
advance(lexer);
|
||||
while(!isAtEnd(lexer) && !(peek(lexer) == '*' && peekNext(lexer) == '/')) advance(lexer);
|
||||
advance(lexer);
|
||||
advance(lexer);
|
||||
break;
|
||||
}
|
||||
return;
|
||||
|
||||
default:
|
||||
return;
|
||||
}
|
||||
|
||||
//tail recursion
|
||||
eatWhitespace(lexer);
|
||||
}
|
||||
|
||||
static bool isDigit(Toy_Lexer* lexer) {
|
||||
return peek(lexer) >= '0' && peek(lexer) <= '9';
|
||||
}
|
||||
|
||||
static bool isAlpha(Toy_Lexer* lexer) {
|
||||
return
|
||||
(peek(lexer) >= 'A' && peek(lexer) <= 'Z') ||
|
||||
(peek(lexer) >= 'a' && peek(lexer) <= 'z') ||
|
||||
peek(lexer) == '_'
|
||||
;
|
||||
}
|
||||
|
||||
static bool match(Toy_Lexer* lexer, char c) {
|
||||
if (peek(lexer) == c) {
|
||||
advance(lexer);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
//token generators
|
||||
static Toy_Token makeErrorToken(Toy_Lexer* lexer, char* msg) {
|
||||
Toy_Token token;
|
||||
|
||||
token.type = TOY_TOKEN_ERROR;
|
||||
token.length = strlen(msg);
|
||||
token.line = lexer->line;
|
||||
token.lexeme = msg;
|
||||
|
||||
return token;
|
||||
}
|
||||
|
||||
static Toy_Token makeToken(Toy_Lexer* lexer, Toy_TokenType type) {
|
||||
Toy_Token token;
|
||||
|
||||
token.type = type;
|
||||
token.length = lexer->current - lexer->start;
|
||||
token.line = lexer->line;
|
||||
token.lexeme = &lexer->source[lexer->current - token.length];
|
||||
|
||||
return token;
|
||||
}
|
||||
|
||||
static Toy_Token makeIntegerOrFloat(Toy_Lexer* lexer) {
|
||||
Toy_TokenType type = TOY_TOKEN_LITERAL_INTEGER; //assume we're reading an integer
|
||||
|
||||
//the character '_' can be inserted into numbers as a separator
|
||||
while(isDigit(lexer) || peek(lexer) == '_') advance(lexer);
|
||||
|
||||
if (peek(lexer) == '.' && (peekNext(lexer) >= '0' && peekNext(lexer) <= '9')) { //peekNext(lexer) == digit
|
||||
type = TOY_TOKEN_LITERAL_FLOAT; //change the assumption to reading a float
|
||||
advance(lexer); //eat the '.'
|
||||
|
||||
//'_' again
|
||||
while(isDigit(lexer) || peek(lexer) == '_') advance(lexer);
|
||||
}
|
||||
|
||||
//make the token
|
||||
Toy_Token token;
|
||||
|
||||
token.type = type;
|
||||
token.length = lexer->current - lexer->start;
|
||||
token.line = lexer->line;
|
||||
token.lexeme = &lexer->source[lexer->start];
|
||||
|
||||
return token;
|
||||
}
|
||||
|
||||
static bool isEscapableCharacter(char c) {
|
||||
switch (c) {
|
||||
case 'n':
|
||||
case 't':
|
||||
case '\\':
|
||||
case '"':
|
||||
return true;
|
||||
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static Toy_Token makeString(Toy_Lexer* lexer, char terminator) {
|
||||
while (!isAtEnd(lexer)) {
|
||||
//stop if you've hit the terminator
|
||||
if (peek(lexer) == terminator) {
|
||||
advance(lexer); //eat the terminator
|
||||
break;
|
||||
}
|
||||
|
||||
//skip escaped control characters
|
||||
if (peek(lexer) == '\\' && isEscapableCharacter(peekNext(lexer))) {
|
||||
advance(lexer);
|
||||
advance(lexer);
|
||||
continue;
|
||||
}
|
||||
|
||||
//otherwise
|
||||
advance(lexer);
|
||||
}
|
||||
|
||||
if (isAtEnd(lexer)) {
|
||||
return makeErrorToken(lexer, "Unterminated string");
|
||||
}
|
||||
|
||||
//make the token
|
||||
Toy_Token token;
|
||||
|
||||
token.type = TOY_TOKEN_LITERAL_STRING;
|
||||
token.length = lexer->current - lexer->start - 2; //-1 to omit the quotes
|
||||
token.line = lexer->line;
|
||||
token.lexeme = &lexer->source[lexer->start + 1]; //+1 to omit the first quote
|
||||
|
||||
return token;
|
||||
}
|
||||
|
||||
static Toy_Token makeKeywordOrIdentifier(Toy_Lexer* lexer) {
|
||||
advance(lexer); //first letter can only be alpha
|
||||
|
||||
while(isDigit(lexer) || isAlpha(lexer)) {
|
||||
advance(lexer);
|
||||
}
|
||||
|
||||
//scan for a keyword
|
||||
for (int i = 0; Toy_private_keywords[i].keyword; i++) {
|
||||
//WONTFIX: could squeeze miniscule performance gain from this, but ROI isn't worth it
|
||||
if (strlen(Toy_private_keywords[i].keyword) == (size_t)(lexer->current - lexer->start) && !strncmp(Toy_private_keywords[i].keyword, &lexer->source[lexer->start], lexer->current - lexer->start)) {
|
||||
//make token (keyword)
|
||||
Toy_Token token;
|
||||
|
||||
token.type = Toy_private_keywords[i].type;
|
||||
token.length = lexer->current - lexer->start;
|
||||
token.line = lexer->line;
|
||||
token.lexeme = &lexer->source[lexer->start];
|
||||
|
||||
return token;
|
||||
}
|
||||
}
|
||||
|
||||
//make token (identifier)
|
||||
Toy_Token token;
|
||||
|
||||
token.type = TOY_TOKEN_IDENTIFIER;
|
||||
token.length = lexer->current - lexer->start;
|
||||
token.line = lexer->line;
|
||||
token.lexeme = &lexer->source[lexer->start];
|
||||
|
||||
return token;
|
||||
}
|
||||
|
||||
//exposed functions
|
||||
void Toy_bindLexer(Toy_Lexer* lexer, const char* source) {
|
||||
cleanLexer(lexer);
|
||||
lexer->source = source;
|
||||
}
|
||||
|
||||
Toy_Token Toy_private_scanLexer(Toy_Lexer* lexer) {
|
||||
eatWhitespace(lexer);
|
||||
|
||||
lexer->start = lexer->current;
|
||||
|
||||
if (isAtEnd(lexer)) return makeToken(lexer, TOY_TOKEN_EOF);
|
||||
|
||||
if (isDigit(lexer)) return makeIntegerOrFloat(lexer);
|
||||
if (isAlpha(lexer)) return makeKeywordOrIdentifier(lexer);
|
||||
|
||||
char c = advance(lexer);
|
||||
|
||||
switch(c) {
|
||||
case '(': return makeToken(lexer, TOY_TOKEN_OPERATOR_PAREN_LEFT);
|
||||
case ')': return makeToken(lexer, TOY_TOKEN_OPERATOR_PAREN_RIGHT);
|
||||
case '[': return makeToken(lexer, TOY_TOKEN_OPERATOR_BRACKET_LEFT);
|
||||
case ']': return makeToken(lexer, TOY_TOKEN_OPERATOR_BRACKET_RIGHT);
|
||||
case '{': return makeToken(lexer, TOY_TOKEN_OPERATOR_BRACE_LEFT);
|
||||
case '}': return makeToken(lexer, TOY_TOKEN_OPERATOR_BRACE_RIGHT);
|
||||
|
||||
case '+': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_ADD_ASSIGN : match(lexer, '+') ? TOY_TOKEN_OPERATOR_INCREMENT : TOY_TOKEN_OPERATOR_ADD);
|
||||
case '-': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_SUBTRACT_ASSIGN : match(lexer, '-') ? TOY_TOKEN_OPERATOR_DECREMENT : TOY_TOKEN_OPERATOR_SUBTRACT);
|
||||
case '*': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_MULTIPLY_ASSIGN : TOY_TOKEN_OPERATOR_MULTIPLY);
|
||||
case '/': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_DIVIDE_ASSIGN : TOY_TOKEN_OPERATOR_DIVIDE);
|
||||
case '%': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_MODULO_ASSIGN : TOY_TOKEN_OPERATOR_MODULO);
|
||||
|
||||
case '!': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_COMPARE_NOT : TOY_TOKEN_OPERATOR_INVERT);
|
||||
case '=': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_COMPARE_EQUAL : TOY_TOKEN_OPERATOR_ASSIGN);
|
||||
|
||||
case '<': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_COMPARE_LESS_EQUAL : TOY_TOKEN_OPERATOR_COMPARE_LESS);
|
||||
case '>': return makeToken(lexer, match(lexer, '=') ? TOY_TOKEN_OPERATOR_COMPARE_GREATER_EQUAL : TOY_TOKEN_OPERATOR_COMPARE_GREATER);
|
||||
|
||||
case '&': //TOY_TOKEN_OPERATOR_AMPERSAND is unused
|
||||
if (match(lexer, '&')) {
|
||||
return makeToken(lexer, TOY_TOKEN_OPERATOR_AND);
|
||||
} else {
|
||||
return makeErrorToken(lexer, "Unexpected '&'");
|
||||
}
|
||||
|
||||
case '|': //TOY_TOKEN_OPERATOR_PIPE is unused
|
||||
if (match(lexer, '|')) {
|
||||
return makeToken(lexer, TOY_TOKEN_OPERATOR_OR);
|
||||
} else {
|
||||
return makeErrorToken(lexer, "Unexpected '|'");
|
||||
}
|
||||
|
||||
case '?': return makeToken(lexer, TOY_TOKEN_OPERATOR_QUESTION);
|
||||
case ':': return makeToken(lexer, TOY_TOKEN_OPERATOR_COLON);
|
||||
case ';': return makeToken(lexer, TOY_TOKEN_OPERATOR_SEMICOLON);
|
||||
case ',': return makeToken(lexer, TOY_TOKEN_OPERATOR_COMMA);
|
||||
|
||||
case '.':
|
||||
if (match(lexer, '.')) {
|
||||
if (match(lexer, '.')) {
|
||||
return makeToken(lexer, TOY_TOKEN_OPERATOR_REST); //three dots
|
||||
}
|
||||
else {
|
||||
return makeToken(lexer, TOY_TOKEN_OPERATOR_CONCAT); //two dots
|
||||
}
|
||||
}
|
||||
else {
|
||||
return makeToken(lexer, TOY_TOKEN_OPERATOR_DOT); //one dot
|
||||
}
|
||||
|
||||
case '"':
|
||||
return makeString(lexer, c);
|
||||
|
||||
default: {
|
||||
return makeErrorToken(lexer, "Unknown token value found in lexer");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void trim(char** s, int* l) { //util
|
||||
while( isspace(( (*((unsigned char**)(s)))[(*l) - 1] )) ) (*l)--;
|
||||
while(**s && isspace( **(unsigned char**)(s)) ) { (*s)++; (*l)--; }
|
||||
}
|
||||
|
||||
//for debugging
|
||||
void Toy_private_printToken(Toy_Token* token) {
|
||||
//print errors
|
||||
if (token->type == TOY_TOKEN_ERROR) {
|
||||
printf(TOY_CC_ERROR "Error\t%d\t%.*s\n" TOY_CC_RESET, token->line, token->length, token->lexeme);
|
||||
return;
|
||||
}
|
||||
|
||||
//read pass token, even though it isn't generated
|
||||
if (token->type == TOY_TOKEN_PASS) {
|
||||
printf(TOY_CC_NOTICE "Error\t%d\t%.*s\n" TOY_CC_RESET, token->line, token->length, token->lexeme);
|
||||
return;
|
||||
}
|
||||
|
||||
//print the line number
|
||||
printf("\t%d\t%d\t", token->type, token->line);
|
||||
|
||||
//print based on type
|
||||
if (token->type == TOY_TOKEN_IDENTIFIER || token->type == TOY_TOKEN_LITERAL_INTEGER || token->type == TOY_TOKEN_LITERAL_FLOAT || token->type == TOY_TOKEN_LITERAL_STRING) {
|
||||
printf("%.*s\t", token->length, token->lexeme);
|
||||
} else {
|
||||
const char* keyword = Toy_private_findKeywordByType(token->type);
|
||||
|
||||
if (keyword != NULL) {
|
||||
printf("%s", keyword);
|
||||
} else {
|
||||
char* str = (char*)token->lexeme; //strip const-ness for trimming
|
||||
int length = token->length;
|
||||
trim(&str, &length);
|
||||
printf("%.*s", length, str);
|
||||
}
|
||||
}
|
||||
|
||||
printf("\n");
|
||||
}
|
||||
@@ -21,5 +21,5 @@ typedef struct {
|
||||
|
||||
TOY_API void Toy_bindLexer(Toy_Lexer* lexer, const char* source);
|
||||
TOY_API Toy_Token Toy_private_scanLexer(Toy_Lexer* lexer);
|
||||
TOY_API void Toy_private_printToken(Toy_Token* token);
|
||||
TOY_API void Toy_private_printToken(Toy_Token* token); //debugging
|
||||
|
||||
|
||||
@@ -65,27 +65,38 @@ typedef enum Toy_TokenType {
|
||||
TOY_TOKEN_OPERATOR_DECREMENT,
|
||||
TOY_TOKEN_OPERATOR_ASSIGN,
|
||||
|
||||
//logical operators
|
||||
TOY_TOKEN_OPERATOR_EQUAL,
|
||||
TOY_TOKEN_OPERATOR_NOT_EQUAL,_EQUAL
|
||||
//comparator operators
|
||||
TOY_TOKEN_OPERATOR_COMPARE_EQUAL,
|
||||
TOY_TOKEN_OPERATOR_COMPARE_NOT,
|
||||
TOY_TOKEN_OPERATOR_COMPARE_LESS,
|
||||
TOY_TOKEN_OPERATOR_COMPARE_LESS_EQUAL,
|
||||
TOY_TOKEN_OPERATOR_COMPARE_GREATER,
|
||||
TOY_TOKEN_OPERATOR_COMPARE_GREATER_EQUAL,
|
||||
|
||||
//structural operators
|
||||
TOY_TOKEN_OPERATOR_PAREN_LEFT,
|
||||
TOY_TOKEN_OPERATOR_PAREN_RIGHT,
|
||||
TOY_TOKEN_OPERATOR_BRACKET_LEFT,
|
||||
TOY_TOKEN_OPERATOR_BRACKET_RIGHT,
|
||||
TOY_TOKEN_OPERATOR_BRACE_LEFT,
|
||||
TOY_TOKEN_OPERATOR_BRACE_RIGHT,
|
||||
TOY_TOKEN_OPERATOR_SEMICOLON,
|
||||
TOY_TOKEN_OPERATOR_COMMA,
|
||||
|
||||
//other operators (context sensitive)
|
||||
//other operators
|
||||
TOY_TOKEN_OPERATOR_AND,
|
||||
TOY_TOKEN_OPERATOR_OR,
|
||||
TOY_TOKEN_OPERATOR_INVERT,
|
||||
TOY_TOKEN_OPERATOR_QUESTION,
|
||||
TOY_TOKEN_OPERATOR_COLON,
|
||||
|
||||
TOY_TOKEN_OPERATOR_SEMICOLON,
|
||||
TOY_TOKEN_OPERATOR_COMMA,
|
||||
|
||||
TOY_TOKEN_OPERATOR_DOT, // .
|
||||
TOY_TOKEN_OPERATOR_CONCAT, // ..
|
||||
TOY_TOKEN_OPERATOR_REST, // ...
|
||||
|
||||
//unused operators
|
||||
TOY_TOKEN_OPERATOR_AMPERSAND, // &
|
||||
TOY_TOKEN_OPERATOR_PIPE, // |
|
||||
|
||||
//meta tokens
|
||||
|
||||
47
tests/cases/test_lexer.c
Normal file
47
tests/cases/test_lexer.c
Normal file
@@ -0,0 +1,47 @@
|
||||
#include "toy_lexer.h"
|
||||
#include "toy_console_colors.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
int main() {
|
||||
{
|
||||
//source code sample to operate on
|
||||
char* source = "print null;";
|
||||
|
||||
//test the lexer
|
||||
Toy_Lexer lexer;
|
||||
Toy_bindLexer(&lexer, source);
|
||||
|
||||
//get each token
|
||||
Toy_Token print = Toy_private_scanLexer(&lexer);
|
||||
Toy_Token null = Toy_private_scanLexer(&lexer);
|
||||
Toy_Token semi = Toy_private_scanLexer(&lexer);
|
||||
Toy_Token eof = Toy_private_scanLexer(&lexer);
|
||||
|
||||
//test each token is correct
|
||||
if (strncmp(print.lexeme, "print", print.length)) {
|
||||
fprintf(stderr, TOY_CC_ERROR "ERROR: print lexeme is wrong: %s" TOY_CC_RESET, print.lexeme);
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
if (strncmp(null.lexeme, "null", null.length)) {
|
||||
fprintf(stderr, TOY_CC_ERROR "ERROR: null lexeme is wrong: %s" TOY_CC_RESET, null.lexeme);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (strncmp(semi.lexeme, ";", semi.length)) {
|
||||
fprintf(stderr, TOY_CC_ERROR "ERROR: semicolon lexeme is wrong: %s" TOY_CC_RESET, semi.lexeme);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (eof.type != TOY_TOKEN_EOF) {
|
||||
fprintf(stderr, TOY_CC_ERROR "ERROR: Failed to find EOF token" TOY_CC_RESET);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
printf(TOY_CC_NOTICE "All good\n" TOY_CC_RESET);
|
||||
return 0;
|
||||
}
|
||||
74
tests/makefile
Normal file
74
tests/makefile
Normal file
@@ -0,0 +1,74 @@
|
||||
#compiler settings
|
||||
CFLAGS=-g -Wall -Werror -Wno-unused-parameter -Wno-unused-function -Wno-unused-variable
|
||||
LIBS=-lm
|
||||
|
||||
#directories
|
||||
TEST_SOURCEDIR=../$(TOY_SOURCEDIR)
|
||||
TEST_CASESDIR=cases
|
||||
|
||||
TEST_OUTDIR=out
|
||||
TEST_OBJDIR=obj
|
||||
|
||||
#file names
|
||||
TEST_SOURCEFILES=$(wildcard $(TEST_SOURCEDIR)/*.c)
|
||||
TEST_CASESFILES=$(wildcard $(TEST_CASESDIR)/*.c)
|
||||
|
||||
#build the object files, compile the test cases, and run
|
||||
all: clean build-source build-cases $(addprefix $(TEST_OUTDIR)/,$(notdir $(TEST_CASESFILES:%.c=%.exe)))
|
||||
|
||||
#targets for each set of source files
|
||||
.PHONY: build-source
|
||||
build-source: $(TEST_OUTDIR) $(TEST_OBJDIR) $(addprefix $(TEST_OBJDIR)/,$(notdir $(TEST_SOURCEFILES:.c=.o)))
|
||||
|
||||
.PHONY: build-cases
|
||||
build-cases: $(TEST_OUTDIR) $(TEST_OBJDIR) $(addprefix $(TEST_OBJDIR)/,$(notdir $(TEST_CASESFILES:.c=.o)))
|
||||
|
||||
#util targets
|
||||
$(TEST_OUTDIR):
|
||||
mkdir $(TEST_OUTDIR)
|
||||
|
||||
$(TEST_OBJDIR):
|
||||
mkdir $(TEST_OBJDIR)
|
||||
|
||||
#compilation step
|
||||
$(TEST_OBJDIR)/%.o: $(TEST_SOURCEDIR)/%.c
|
||||
$(CC) -c -o $@ $< $(addprefix -I,$(TEST_SOURCEDIR)) $(CFLAGS) -fdata-sections -ffunction-sections
|
||||
|
||||
$(TEST_OBJDIR)/%.o: $(TEST_CASESDIR)/%.c
|
||||
$(CC) -c -o $@ $< $(addprefix -I,$(TEST_SOURCEDIR) $(TEST_CASESDIR)) $(CFLAGS) -fdata-sections -ffunction-sections
|
||||
|
||||
#final linking step (with extra flags to strip dead code)
|
||||
$(TEST_OUTDIR)/%.exe: $(TEST_OBJDIR)/%.o
|
||||
@$(CC) -o $@ $(shell find $(TEST_OBJDIR) -name '*.o') $(CFLAGS) $(LIBS) -Wl,--gc-sections
|
||||
$@
|
||||
|
||||
#util commands
|
||||
.PHONY: clean
|
||||
clean:
|
||||
ifeq ($(shell uname),Linux)
|
||||
find . -type f -name '*.o' -delete
|
||||
find . -type f -name '*.a' -delete
|
||||
find . -type f -name '*.exe' -delete
|
||||
find . -type f -name '*.dll' -delete
|
||||
find . -type f -name '*.lib' -delete
|
||||
find . -type f -name '*.so' -delete
|
||||
find . -type f -name '*.dylib' -delete
|
||||
find . -type d -name 'out' -delete
|
||||
find . -type d -name 'obj' -delete
|
||||
else ifeq ($(OS),Windows_NT)
|
||||
del *.o *.a *.exe *.dll *.lib *.so *.dylib
|
||||
del /s out
|
||||
del /s obj
|
||||
else ifeq ($(shell uname),Darwin)
|
||||
find . -type f -name '*.o' -delete
|
||||
find . -type f -name '*.a' -delete
|
||||
find . -type f -name '*.exe' -delete
|
||||
find . -type f -name '*.dll' -delete
|
||||
find . -type f -name '*.lib' -delete
|
||||
find . -type f -name '*.so' -delete
|
||||
find . -type f -name '*.dylib' -delete
|
||||
find . -type d -name 'out' -delete
|
||||
find . -type d -name 'obj' -delete
|
||||
else
|
||||
@echo "Deletion failed - what platform is this?"
|
||||
endif
|
||||
Reference in New Issue
Block a user