Minimal build of the refstrings in the literal structure

.github/workflows/c-cpp.yml

@@ -2,7 +2,7 @@ name: Comprehensive Tests
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "dev" ]
   pull_request:
     branches: [ "main" ]
 

source/literal.c

@@ -21,7 +21,7 @@ static unsigned int hashString(const char* string, int length) {
 	return hash;
 }
 
-static unsigned int hash(unsigned int x) {
+static unsigned int hashUInt(unsigned int x) {
     x = ((x >> 16) ^ x) * 0x45d9f3b;
     x = ((x >> 16) ^ x) * 0x45d9f3b;
     x = (x >> 16) ^ x;
@@ -30,11 +30,18 @@ static unsigned int hash(unsigned int x) {
 
 //exposed functions
 void freeLiteral(Literal literal) {
+	//refstrings
 	if (IS_STRING(literal)) {
-		FREE_ARRAY(char, AS_STRING(literal), literal.as.string.length + 1);
+		deleteRefString(AS_STRING(literal));
 		return;
 	}
 
+	if (IS_IDENTIFIER(literal)) {
+		deleteRefString(AS_IDENTIFIER(literal));
+		return;
+	}
+
+	//compounds
 	if (IS_ARRAY(literal) || literal.type == LITERAL_DICTIONARY_INTERMEDIATE || literal.type == LITERAL_TYPE_INTERMEDIATE) {
 		freeLiteralArray(AS_ARRAY(literal));
 		FREE(LiteralArray, AS_ARRAY(literal));
@@ -47,17 +54,13 @@ void freeLiteral(Literal literal) {
 		return;
 	}
 
+	//complex literals
 	if (IS_FUNCTION(literal)) {
 		popScope(AS_FUNCTION(literal).scope);
 		AS_FUNCTION(literal).scope = NULL;
 		FREE_ARRAY(unsigned char, AS_FUNCTION(literal).bytecode, AS_FUNCTION(literal).length);
 	}
 
-	if (IS_IDENTIFIER(literal)) {
-		FREE_ARRAY(char, AS_IDENTIFIER(literal), literal.as.identifier.length + 1);
-		return;
-	}
-
 	if (IS_TYPE(literal)) {
 		for (int i = 0; i < AS_TYPE(literal).count; i++) {
 			freeLiteral(((Literal*)(AS_TYPE(literal).subtypes))[i]);
@@ -80,12 +83,12 @@ bool _isTruthy(Literal x) {
 	return true;
 }
 
-Literal _toStringLiteral(char* str, int length) {
-	return ((Literal){LITERAL_STRING, { .string.ptr = (char*)str, .string.length = length }});
+Literal _toStringLiteral(RefString* ptr) {
+	return ((Literal){LITERAL_STRING, { .string.ptr = ptr }});
 }
 
-Literal _toIdentifierLiteral(char* str, int length) {
-	return ((Literal){LITERAL_IDENTIFIER,{ .identifier.ptr = (char*)str, .identifier.length = length, .identifier.hash = hashString(str, length) }});
+Literal _toIdentifierLiteral(RefString* ptr) {
+	return ((Literal){LITERAL_IDENTIFIER,{ .identifier.ptr = ptr, .identifier.hash = hashString(toCString(ptr), lengthRefString(ptr)) }});
 }
 
 Literal* _typePushSubtype(Literal* lit, Literal subtype) {
@@ -112,7 +115,7 @@ Literal copyLiteral(Literal original) {
 			return original;
 
 		case LITERAL_STRING: {
-			return TO_STRING_LITERAL(copyString(AS_STRING(original), original.as.string.length), original.as.string.length);
+			return TO_STRING_LITERAL(copyRefString(AS_STRING(original)));
 		}
 
 		case LITERAL_ARRAY: {
@@ -152,7 +155,7 @@ Literal copyLiteral(Literal original) {
 		}
 
 		case LITERAL_IDENTIFIER: {
-			 return TO_IDENTIFIER_LITERAL(copyString(AS_IDENTIFIER(original), original.as.identifier.length), original.as.identifier.length);
+			 return TO_IDENTIFIER_LITERAL(copyRefString(AS_IDENTIFIER(original)));
 		}
 
 		case LITERAL_TYPE: {
@@ -212,14 +215,6 @@ Literal copyLiteral(Literal original) {
 	}
 }
 
-char* copyString(char* original, int length) {
-	//make a local copy of the char array
-	char* buffer = ALLOCATE(char, length + 1);
-	strncpy(buffer, original, length);
-	buffer[length] = '\0';
-	return buffer;
-}
-
 bool literalsAreEqual(Literal lhs, Literal rhs) {
 	//utility for other things
 	if (lhs.type != rhs.type) {
@@ -250,10 +245,7 @@ bool literalsAreEqual(Literal lhs, Literal rhs) {
 			return AS_FLOAT(lhs) == AS_FLOAT(rhs);
 
 		case LITERAL_STRING:
-			if (lhs.as.string.length != rhs.as.string.length) {
-				return false;
-			}
-			return !strncmp(AS_STRING(lhs), AS_STRING(rhs), lhs.as.string.length);
+			return equalsRefString(AS_STRING(lhs), AS_STRING(rhs));
 
 		case LITERAL_ARRAY:
 		case LITERAL_DICTIONARY_INTERMEDIATE: //BUGFIX
@@ -299,11 +291,11 @@ bool literalsAreEqual(Literal lhs, Literal rhs) {
 
 		case LITERAL_IDENTIFIER:
 			//check shortcuts
-			if (HASH_I(lhs) != HASH_I(rhs) && lhs.as.identifier.length != rhs.as.identifier.length) {
+			if (HASH_I(lhs) != HASH_I(rhs)) {
 				return false;
 			}
 
-			return !strncmp(AS_IDENTIFIER(lhs), AS_IDENTIFIER(rhs), lhs.as.identifier.length);
+			return equalsRefString(AS_IDENTIFIER(lhs), AS_IDENTIFIER(rhs));
 
 		case LITERAL_TYPE:
 			//check types
@@ -359,20 +351,20 @@ int hashLiteral(Literal lit) {
 			return AS_BOOLEAN(lit) ? 1 : 0;
 
 		case LITERAL_INTEGER:
-			return hash((unsigned int)AS_INTEGER(lit));
+			return hashUInt((unsigned int)AS_INTEGER(lit));
 
 		case LITERAL_FLOAT:
-			return hash(*(unsigned int*)(&AS_FLOAT(lit)));
+			return hashUInt(*(unsigned int*)(&AS_FLOAT(lit)));
 
 		case LITERAL_STRING:
-			return hashString(AS_STRING(lit), strlen(AS_STRING(lit)));
+			return hashString(toCString(AS_STRING(lit)), lengthRefString(AS_STRING(lit)));
 
 		case LITERAL_ARRAY: {
 			unsigned int res = 0;
 			for (int i = 0; i < AS_ARRAY(lit)->count; i++) {
 				res += hashLiteral(AS_ARRAY(lit)->literals[i]);
 			}
-			return hash(res);
+			return hashUInt(res);
 		}
 
 		case LITERAL_DICTIONARY: {
@@ -383,7 +375,7 @@ int hashLiteral(Literal lit) {
 					res += hashLiteral(AS_DICTIONARY(lit)->entries[i].value);
 				}
 			}
-			return hash(res);
+			return hashUInt(res);
 		}
 
 		case LITERAL_FUNCTION:
@@ -464,10 +456,10 @@ void printLiteralCustom(Literal literal, void (printFn)(const char*)) {
 		case LITERAL_STRING: {
 			char buffer[MAX_STRING_LENGTH];
 			if (!quotes) {
-				snprintf(buffer, MAX_STRING_LENGTH, "%.*s", (int)strlen(AS_STRING(literal)), AS_STRING(literal));
+				snprintf(buffer, MAX_STRING_LENGTH, "%.*s", lengthRefString(AS_STRING(literal)), toCString(AS_STRING(literal)));
 			}
 			else {
-				snprintf(buffer, MAX_STRING_LENGTH, "%c%.*s%c", quotes, (int)strlen(AS_STRING(literal)), AS_STRING(literal), quotes);
+				snprintf(buffer, MAX_STRING_LENGTH, "%c%.*s%c", quotes, lengthRefString(AS_STRING(literal)), toCString(AS_STRING(literal)), quotes);
 			}
 			printFn(buffer);
 		}
@@ -573,7 +565,7 @@ void printLiteralCustom(Literal literal, void (printFn)(const char*)) {
 
 		case LITERAL_IDENTIFIER: {
 			char buffer[256];
-			snprintf(buffer, 256, "%.*s", (int)strlen( AS_IDENTIFIER(literal) ), AS_IDENTIFIER(literal));
+			snprintf(buffer, 256, "%.*s", lengthRefString(AS_IDENTIFIER(literal)), toCString(AS_IDENTIFIER(literal)));
 			printFn(buffer);
 		}
 		break;

source/literal.h

@@ -2,6 +2,8 @@
 
 #include "toy_common.h"
 
+#include "refstring.h"
+
 #include <string.h>
 
 typedef enum {
@@ -33,8 +35,8 @@ typedef struct {
 		int integer;
 		float number;
 		struct {
-			char* ptr;
-			int length;
+			RefString* ptr;
+			//string hash?
 		} string;
 
 		void* array;
@@ -47,8 +49,7 @@ typedef struct {
 		} function;
 
 		struct { //for variable names
-            char* ptr;
-            int length;
+            RefString* ptr;
 			int hash;
         } identifier;
 
@@ -62,7 +63,7 @@ typedef struct {
 
 		struct {
 			void* ptr;
-			int tag;
+			int tag; //TODO: remove tags?
 		} opaque;
 	} as;
 } Literal;
@@ -95,11 +96,11 @@ typedef struct {
 #define TO_BOOLEAN_LITERAL(value)			((Literal){LITERAL_BOOLEAN,		{ .boolean = value }})
 #define TO_INTEGER_LITERAL(value)			((Literal){LITERAL_INTEGER,		{ .integer = value }})
 #define TO_FLOAT_LITERAL(value)				((Literal){LITERAL_FLOAT,		{ .number = value }})
-#define TO_STRING_LITERAL(value, l)			_toStringLiteral(value, l)
+#define TO_STRING_LITERAL(value)			_toStringLiteral(value)
 #define TO_ARRAY_LITERAL(value)				((Literal){LITERAL_ARRAY,		{ .array = value }})
 #define TO_DICTIONARY_LITERAL(value)		((Literal){LITERAL_DICTIONARY,	{ .dictionary = value }})
 #define TO_FUNCTION_LITERAL(value, l)		((Literal){LITERAL_FUNCTION,	{ .function.bytecode = value, .function.scope = NULL, .function.length = l }})
-#define TO_IDENTIFIER_LITERAL(value, l)		_toIdentifierLiteral(value, l)
+#define TO_IDENTIFIER_LITERAL(value)		_toIdentifierLiteral(value)
 #define TO_TYPE_LITERAL(value, c)			((Literal){ LITERAL_TYPE,		{ .type.typeOf = value, .type.constant = c, .type.subtypes = NULL, .type.capacity = 0, .type.count = 0 }})
 #define TO_OPAQUE_LITERAL(value, t)			((Literal){ LITERAL_OPAQUE,		{ .opaque.ptr = value, .opaque.tag = t }})
 
@@ -114,13 +115,12 @@ TOY_API void freeLiteral(Literal literal);
 
 //BUGFIX: macros are not functions
 TOY_API bool _isTruthy(Literal x);
-TOY_API Literal _toStringLiteral(char* str, int length);
-TOY_API Literal _toIdentifierLiteral(char* str, int length);
+TOY_API Literal _toStringLiteral(RefString* ptr);
+TOY_API Literal _toIdentifierLiteral(RefString* ptr);
 TOY_API Literal* _typePushSubtype(Literal* lit, Literal subtype);
 
 //utils
 TOY_API Literal copyLiteral(Literal original);
-TOY_API char* copyString(char* original, int length);
 TOY_API bool literalsAreEqual(Literal lhs, Literal rhs);
 TOY_API int hashLiteral(Literal lit);
 

source/memory.c

@@ -1,4 +1,5 @@
 #include "memory.h"
+#include "refstring.h"
 
 #include "console_colors.h"
 
@@ -6,7 +7,7 @@
 #include <stdlib.h>
 
 //default allocator
-void* defaultAllocator(void* pointer, size_t oldSize, size_t newSize) {
+void* defaultMemoryAllocator(void* pointer, size_t oldSize, size_t newSize) {
 	if (newSize == 0 && oldSize == 0) {
 		//causes issues, so just skip out with a NO-OP
 		return NULL;
@@ -28,10 +29,20 @@ void* defaultAllocator(void* pointer, size_t oldSize, size_t newSize) {
 	return mem;
 }
 
-//exposed API
-static AllocatorFn allocator = defaultAllocator;
+//static variables
+static MemoryAllocatorFn allocator;
 
-void setAllocator(AllocatorFn fn) {
+//preload
+static void __attribute__((constructor)) preloadMemoryAllocator() {
+	setMemoryAllocator(defaultMemoryAllocator);
+}
+
+//exposed API
+void* reallocate(void* pointer, size_t oldSize, size_t newSize) {
+	return allocator(pointer, oldSize, newSize);
+}
+
+void setMemoryAllocator(MemoryAllocatorFn fn) {
 	if (fn == NULL) {
 		fprintf(stderr, ERROR "[internal] Memory allocator error (can't be null)\n" RESET);
 		exit(-1);
@@ -43,8 +54,5 @@ void setAllocator(AllocatorFn fn) {
 	}
 
 	allocator = fn;
+	setRefStringAllocatorFn(fn);
 }
-
-void* reallocate(void* pointer, size_t oldSize, size_t newSize) {
-	return allocator(pointer, oldSize, newSize);
-}

source/memory.h

@@ -11,7 +11,8 @@
 #define FREE_ARRAY(type, pointer, oldCount) reallocate((type*)pointer, sizeof(type) * (oldCount), 0)
 
 //implementation details
-typedef void* (*AllocatorFn)(void* pointer, size_t oldSize, size_t newSize);
-TOY_API void setAllocator(AllocatorFn);
-
 void* reallocate(void* pointer, size_t oldSize, size_t newSize);
+
+//assign the memory allocator
+typedef void* (*MemoryAllocatorFn)(void* pointer, size_t oldSize, size_t newSize);
+TOY_API void setMemoryAllocator(MemoryAllocatorFn);

source/refstring.c

@@ -0,0 +1,86 @@
+#include "refstring.h"
+
+#include <string.h>
+#include <assert.h>
+
+//test variable sizes based on platform (safety)
+#define STATIC_ASSERT(test_for_true) static_assert((test_for_true), "(" #test_for_true ") failed")
+
+STATIC_ASSERT(sizeof(RefString) == 12);
+STATIC_ASSERT(sizeof(int) == 4);
+STATIC_ASSERT(sizeof(char) == 1);
+
+//memory allocation
+static RefStringAllocatorFn allocate;
+
+void setRefStringAllocatorFn(RefStringAllocatorFn allocator) {
+	allocate = allocator;
+}
+
+//API
+RefString* createRefString(char* cstring) {
+	int length = strlen(cstring);
+
+	//allocate the memory area (including metadata space)
+	RefString* refString = (RefString*)allocate(NULL, 0, sizeof(int) * 2 + sizeof(char) * length + 1);
+
+	//set the data
+	refString->refcount = 1;
+	refString->length = length;
+	strncpy(refString->data, cstring, refString->length);
+
+	refString->data[refString->length] = '\0'; //string terminator
+
+	return refString;
+}
+
+void deleteRefString(RefString* refString) {
+	if (refString->refcount > 0) {
+		//decrement, then check
+		refString->refcount--;
+		if (refString->refcount <= 0) {
+			allocate(refString, sizeof(int) * 2 + sizeof(char) * refString->length + 1, 0);
+		}
+	}
+}
+
+int countRefString(RefString* refString) {
+	return refString->refcount;
+}
+
+int lengthRefString(RefString* refString) {
+	return refString->length;
+}
+
+RefString* copyRefString(RefString* refString) {
+	refString->refcount++;
+	return refString;
+}
+
+RefString* deepCopyRefString(RefString* refString) {
+	//deep copy, which can be modified immediately
+	RefString* newRefString = (RefString*)allocate(NULL, 0, sizeof(int) * 2 + refString->length + 1);
+
+	memcpy(newRefString, refString, refString->length + 1);
+
+	return newRefString;
+}
+
+char* toCString(RefString* refString) {
+	return refString->data;
+}
+
+bool equalsRefString(RefString* lhs, RefString* rhs) {
+	//same pointer
+	if (lhs == rhs) {
+		return true;
+	}
+
+	//different length
+	if (lhs->length != rhs->length) {
+		return false;
+	}
+
+	//same string
+	return strncmp(lhs->data, rhs->data, lhs->length) == 0;
+}

source/refstring.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <stdbool.h>
+#include <stddef.h>
+
+//memory allocation hook
+typedef void* (*RefStringAllocatorFn)(void* pointer, size_t oldSize, size_t newSize);
+void setRefStringAllocatorFn(RefStringAllocatorFn);
+
+//the RefString structure
+typedef struct RefString {
+	int refcount;
+	int length;
+	char data[1];
+} RefString;
+
+//API
+RefString* createRefString(char* cstring);
+void deleteRefString(RefString* refString);
+int countRefString(RefString* refString);
+int lengthRefString(RefString* refString);
+RefString* copyRefString(RefString* refString);
+RefString* deepCopyRefString(RefString* refString);
+char* toCString(RefString* refString);
+bool equalsRefString(RefString* lhs, RefString* rhs);

test/makefile

@@ -5,8 +5,11 @@ CFLAGS +=$(addprefix -I,$(IDIR)) -g -Wall -W -Wno-unused-parameter -Wno-unused-f
 LIBS +=
 
 ODIR = obj
-TARGETS = $(wildcard ../source/*.c) $(wildcard ../repl/lib_*.c)
-TESTS = $(wildcard *.c)
+#TARGETS = $(wildcard ../source/*.c) $(wildcard ../repl/lib_*.c)
+
+TARGETS = ../source/memory.c ../source/refstring.c ../source/literal.c ../source/literal_array.c ../source/literal_dictionary.c ../source/scope.c
+
+TESTS = $(wildcard test_*.c)
 OBJ = $(addprefix $(ODIR)/,$(TARGETS:../source/%.c=%.o)) $(addprefix $(ODIR)/,$(TESTS:.c=.o))
 
 .PRECIOUS:  $(TESTS:%.c=../$(TOY_OUTDIR)/%.exe)

test/test_literal.c

@@ -29,22 +29,18 @@ int main() {
 
 	{
 		//test string literals
-		char* buffer = ALLOCATE(char, 128);
+		char* buffer = "Hello world";
 
-		snprintf(buffer, 128, "Hello world");
-
-		Literal literal = TO_STRING_LITERAL(buffer, 128);
+		Literal literal = TO_STRING_LITERAL(createRefString(buffer));
 
 		freeLiteral(literal);
 	}
 
 	{
 		//test identifier literals
-		char* buffer = ALLOCATE(char, 128);
+		char buffer[] = "Hello world";
 
-		snprintf(buffer, 128, "foobar");
-
-		Literal literal = TO_IDENTIFIER_LITERAL(buffer, 128);
+		Literal literal = TO_IDENTIFIER_LITERAL(createRefString(buffer));
 
 		freeLiteral(literal);
 	}

test/test_literal_array.c

@@ -46,10 +46,10 @@ int main() {
 		char* str_raw = "hello world";
 		char* idn_raw = "foobar";
 
-		Literal string = TO_STRING_LITERAL(copyString(str_raw, strlen(str_raw)), strlen(str_raw));
-		Literal identifier = TO_IDENTIFIER_LITERAL(copyString(idn_raw, strlen(idn_raw)), strlen(idn_raw));
+		Literal string = TO_STRING_LITERAL(createRefString(str_raw));
+		Literal identifier = TO_IDENTIFIER_LITERAL(createRefString(idn_raw));
 
-		//[string, string]
+		//[string : string]
 		Literal type = TO_TYPE_LITERAL(LITERAL_DICTIONARY, false);
 		TYPE_PUSH_SUBTYPE(&type, TO_TYPE_LITERAL(LITERAL_STRING, false));
 		TYPE_PUSH_SUBTYPE(&type, TO_TYPE_LITERAL(LITERAL_STRING, false));

test/test_literal_dictionary.c

@@ -18,8 +18,8 @@ int main() {
 		char* idn_raw = "foobar";
 		char* str_raw = "hello world";
 
-		Literal identifier = TO_IDENTIFIER_LITERAL(copyString(idn_raw, strlen(idn_raw)), strlen(idn_raw));
-		Literal string = TO_STRING_LITERAL(copyString(str_raw, strlen(str_raw)), strlen(str_raw));
+		Literal identifier = TO_IDENTIFIER_LITERAL(createRefString(idn_raw));
+		Literal string = TO_STRING_LITERAL(createRefString(str_raw));
 
 		LiteralDictionary dictionary;
 		initLiteralDictionary(&dictionary);

test/test_memory.c

@@ -63,7 +63,7 @@ int main() {
 	testMemoryAllocation();
 
 	//test the custom allocator
-	setAllocator(allocator);
+	setMemoryAllocator(allocator);
 	testMemoryAllocation();
 
 	if (callCount != 8) {

test/test_scope.c

@@ -16,7 +16,7 @@ int main() {
 		//prerequisites
 		char* idn_raw = "foobar";
 
-		Literal identifier = TO_IDENTIFIER_LITERAL(copyString(idn_raw, strlen(idn_raw)), strlen(idn_raw));
+		Literal identifier = TO_IDENTIFIER_LITERAL(createRefString(idn_raw));
 		Literal value = TO_INTEGER_LITERAL(42);
 		Literal type = TO_TYPE_LITERAL(value.type, false);
 
@@ -46,7 +46,7 @@ int main() {
 		//prerequisites
 		char* idn_raw = "foobar";
 
-		Literal identifier = TO_IDENTIFIER_LITERAL(copyString(idn_raw, strlen(idn_raw)), strlen(idn_raw));
+		Literal identifier = TO_IDENTIFIER_LITERAL(createRefString(idn_raw));
 		Literal type = TO_TYPE_LITERAL(LITERAL_INTEGER, false);
 
 		//test declarations & assignments