krgamestudios/Toy
1
0
Fork 0
mirror of https://github.com/krgamestudios/Toy.git synced 2026-04-15 14:54:07 +10:00
Code Issues Packages Projects Releases Wiki Activity

Moved keywords into the lexer

Browse Source
This commit is contained in:
Kayne Ruse
2024-10-14 12:41:53 +11:00
parent 694e262ee2
commit 425ef7e3e0
3 changed files with 83 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
source/toy_keywords.c
View File

@@ -1,77 +0,0 @@
#include "toy_keywords.h"
#include <string.h>
const Toy_KeywordTypeTuple Toy_private_keywords[] = {
//null
{TOY_TOKEN_NULL, "null"},
//types
{TOY_TOKEN_TYPE_TYPE, "type"},
{TOY_TOKEN_TYPE_BOOLEAN, "bool"},
{TOY_TOKEN_TYPE_INTEGER, "int"},
{TOY_TOKEN_TYPE_FLOAT, "float"},
{TOY_TOKEN_TYPE_STRING, "string"},
// TOY_TOKEN_TYPE_ARRAY,
// TOY_TOKEN_TYPE_DICTIONARY,
// TOY_TOKEN_TYPE_FUNCTION,
{TOY_TOKEN_TYPE_OPAQUE, "opaque"},
{TOY_TOKEN_TYPE_ANY, "any"},
//keywords and reserved words
{TOY_TOKEN_KEYWORD_AS, "as"},
{TOY_TOKEN_KEYWORD_ASSERT, "assert"},
{TOY_TOKEN_KEYWORD_BREAK, "break"},
{TOY_TOKEN_KEYWORD_CLASS, "class"},
{TOY_TOKEN_KEYWORD_CONST, "const"},
{TOY_TOKEN_KEYWORD_CONTINUE, "continue"},
{TOY_TOKEN_KEYWORD_DO, "do"},
{TOY_TOKEN_KEYWORD_ELSE, "else"},
{TOY_TOKEN_KEYWORD_EXPORT, "export"},
{TOY_TOKEN_KEYWORD_FOR, "for"},
{TOY_TOKEN_KEYWORD_FOREACH, "foreach"},
{TOY_TOKEN_KEYWORD_FUNCTION, "fn"},
{TOY_TOKEN_KEYWORD_IF, "if"},
{TOY_TOKEN_KEYWORD_IMPORT, "import"},
{TOY_TOKEN_KEYWORD_IN, "in"},
{TOY_TOKEN_KEYWORD_OF, "of"},
{TOY_TOKEN_KEYWORD_PRINT, "print"},
{TOY_TOKEN_KEYWORD_RETURN, "return"},
{TOY_TOKEN_KEYWORD_TYPEAS, "typeas"},
{TOY_TOKEN_KEYWORD_TYPEOF, "typeof"},
{TOY_TOKEN_KEYWORD_VAR, "var"},
{TOY_TOKEN_KEYWORD_WHILE, "while"},
{TOY_TOKEN_KEYWORD_YIELD, "yield"},
//literal values
{TOY_TOKEN_LITERAL_TRUE, "true"},
{TOY_TOKEN_LITERAL_FALSE, "false"},
{TOY_TOKEN_EOF, NULL},
};
const char* Toy_private_findKeywordByType(const Toy_TokenType type) {
if (type == TOY_TOKEN_EOF) {
return "EOF";
}
for(int i = 0; Toy_private_keywords[i].keyword; i++) {
if (Toy_private_keywords[i].type == type) {
return Toy_private_keywords[i].keyword;
}
}
return NULL;
}
Toy_TokenType Toy_private_findTypeByKeyword(const char* keyword) {
const int length = strlen(keyword);
for (int i = 0; Toy_private_keywords[i].keyword; i++) {
if (!strncmp(keyword, Toy_private_keywords[i].keyword, length)) {
return Toy_private_keywords[i].type;
}
}
return TOY_TOKEN_EOF;
}

16
source/toy_keywords.h
View File

@@ -1,16 +0,0 @@
#pragma once
#include "toy_token_types.h"
#include "toy_common.h"
typedef struct {
const Toy_TokenType type;
const char* keyword;
} Toy_KeywordTypeTuple;
extern const Toy_KeywordTypeTuple Toy_private_keywords[];
//access
const char* Toy_private_findKeywordByType(const Toy_TokenType type);
Toy_TokenType Toy_private_findTypeByKeyword(const char* keyword);

87
source/toy_lexer.c
View File

@@ -1,11 +1,90 @@
#include "toy_lexer.h"
#include "toy_keywords.h"
#include "toy_console_colors.h"
#include <stdio.h>
#include <string.h>
#include <ctype.h>
//keyword data
typedef struct {
const Toy_TokenType type;
const char* keyword;
} Toy_KeywordTypeTuple;
const Toy_KeywordTypeTuple keywordTuples[] = {
//null
{TOY_TOKEN_NULL, "null"},
//types
{TOY_TOKEN_TYPE_TYPE, "type"},
{TOY_TOKEN_TYPE_BOOLEAN, "bool"},
{TOY_TOKEN_TYPE_INTEGER, "int"},
{TOY_TOKEN_TYPE_FLOAT, "float"},
{TOY_TOKEN_TYPE_STRING, "string"},
// TOY_TOKEN_TYPE_ARRAY,
// TOY_TOKEN_TYPE_DICTIONARY,
// TOY_TOKEN_TYPE_FUNCTION,
{TOY_TOKEN_TYPE_OPAQUE, "opaque"},
{TOY_TOKEN_TYPE_ANY, "any"},
//keywords and reserved words
{TOY_TOKEN_KEYWORD_AS, "as"},
{TOY_TOKEN_KEYWORD_ASSERT, "assert"},
{TOY_TOKEN_KEYWORD_BREAK, "break"},
{TOY_TOKEN_KEYWORD_CLASS, "class"},
{TOY_TOKEN_KEYWORD_CONST, "const"},
{TOY_TOKEN_KEYWORD_CONTINUE, "continue"},
{TOY_TOKEN_KEYWORD_DO, "do"},
{TOY_TOKEN_KEYWORD_ELSE, "else"},
{TOY_TOKEN_KEYWORD_EXPORT, "export"},
{TOY_TOKEN_KEYWORD_FOR, "for"},
{TOY_TOKEN_KEYWORD_FOREACH, "foreach"},
{TOY_TOKEN_KEYWORD_FUNCTION, "fn"},
{TOY_TOKEN_KEYWORD_IF, "if"},
{TOY_TOKEN_KEYWORD_IMPORT, "import"},
{TOY_TOKEN_KEYWORD_IN, "in"},
{TOY_TOKEN_KEYWORD_OF, "of"},
{TOY_TOKEN_KEYWORD_PRINT, "print"},
{TOY_TOKEN_KEYWORD_RETURN, "return"},
{TOY_TOKEN_KEYWORD_TYPEAS, "typeas"},
{TOY_TOKEN_KEYWORD_TYPEOF, "typeof"},
{TOY_TOKEN_KEYWORD_VAR, "var"},
{TOY_TOKEN_KEYWORD_WHILE, "while"},
{TOY_TOKEN_KEYWORD_YIELD, "yield"},
//literal values
{TOY_TOKEN_LITERAL_TRUE, "true"},
{TOY_TOKEN_LITERAL_FALSE, "false"},
{TOY_TOKEN_EOF, NULL},
};
const char* Toy_private_findKeywordByType(const Toy_TokenType type) {
if (type == TOY_TOKEN_EOF) {
return "EOF";
}
for(int i = 0; keywordTuples[i].keyword; i++) {
if (keywordTuples[i].type == type) {
return keywordTuples[i].keyword;
}
}
return NULL;
}
Toy_TokenType Toy_private_findTypeByKeyword(const char* keyword) {
const int length = strlen(keyword);
for (int i = 0; keywordTuples[i].keyword; i++) {
if (!strncmp(keyword, keywordTuples[i].keyword, length)) {
return keywordTuples[i].type;
}
}
return TOY_TOKEN_EOF;
}
//static generic utility functions
static void cleanLexer(Toy_Lexer* lexer) {
lexer->start = 0;
@@ -203,13 +282,13 @@ static Toy_Token makeKeywordOrName(Toy_Lexer* lexer) {
}
//scan for a keyword
for (int i = 0; Toy_private_keywords[i].keyword; i++) {
for (int i = 0; keywordTuples[i].keyword; i++) {
//WONTFIX: could squeeze miniscule performance gain from this, but ROI isn't worth it
if (strlen(Toy_private_keywords[i].keyword) == (lexer->current - lexer->start) && !strncmp(Toy_private_keywords[i].keyword, &lexer->source[lexer->start], lexer->current - lexer->start)) {
if (strlen(keywordTuples[i].keyword) == (lexer->current - lexer->start) && !strncmp(keywordTuples[i].keyword, &lexer->source[lexer->start], lexer->current - lexer->start)) {
//make token (keyword)
Toy_Token token;
token.type = Toy_private_keywords[i].type;
token.type = keywordTuples[i].type;
token.length = lexer->current - lexer->start;
token.line = lexer->line;
token.lexeme = &lexer->source[lexer->start];