Implemented garbage collection

As a whole, this is still tentative.

scripts/benchpress.toy

@@ -1,12 +1,12 @@
 //calculate the nth fibonacci number, and print it
 
-var counter: int = 0;
+var counter: Int = 0;
 
-var first: int = 1;
-var second: int = 0;
+var first: Int = 1;
+var second: Int = 0;
 
 while (counter < 100_000) {
-	var third: int = first + second;
+	var third: Int = first + second;
 	first = second;
 	second = third;
 

scripts/fizzbuzz.toy

@@ -1,9 +1,9 @@
 //standard example, using 'while' instead of 'for', because it's not ready yet
 
-var counter: int = 0;
+var counter: Int = 0;
 
 while (++counter <= 100) {
-	var result: string = "";
+	var result: String = "";
 
 	if (counter % 3 == 0) {
 		result = result .. "fizz";

scripts/hello_world.toy

@@ -9,4 +9,5 @@ var b = 69;
 var c;
 var d;
 
+//BUG: still causes a segfault
 c, d = swap(a, b);

scripts/leapyear.toy

@@ -1,5 +1,5 @@
 //find the leap years
-fn isLeapYear(n: int) {
+fn isLeapYear(n: Int) {
 	if (n % 400 == 0) return true;
 	if (n % 100 == 0) return false;
 	return n % 4 == 0;

scripts/tally.toy

@@ -1,5 +1,5 @@
 fn makeCounter() {
-	var counter: int = 0;
+	var counter: Int = 0;
 
 	fn increment() {
 		return ++counter;

scripts/test_compounds.toy

@@ -1,35 +0,0 @@
-
-//array outside a table
-var a = [
-	[1, 2, 3],
-	["alpha": 1, "beta": 2, "gamma": 3],
-	[7, 8, 9],
-];
-
-print a;
-
-//table outside an array
-
-var t = [
-	"alpha": [1,2,3],
-	"beta": [4,5,6],
-	"gamma": [7,8,9],
-];
-
-print t;
-
-//we need to go deeper
-
-var deeper = [
-	[1, 2, 3],
-	[
-		"alpha": [1,2,3],
-		"beta": [4,5,6],
-		"gamma": [7,[
-			"delta":10,"epsilon":11,"foxtrot":12
-		],9],
-	],
-	[7, 8, 9],
-];
-
-print deeper;

scripts/test_tables.toy

@@ -1,14 +0,0 @@
-//snipped out of the tests, this seems fine?
-
-//nested
-var example = [
-	"outer": ["inner": true],
-	"alpha": 1,
-	"beta": 2,
-	"gamma": 3
-];
-
-print example;
-assert example == ["alpha": 1, "beta": 2, "gamma": 3, "outer": ["inner": true]], "nested tables failed";
-
-return example;

source/toy_bucket.c

@@ -26,21 +26,29 @@ Toy_Bucket* Toy_allocateBucket(unsigned int capacity) {
 
 unsigned char* Toy_partitionBucket(Toy_Bucket** bucketHandle, unsigned int amount) {
 	//the endpoint must be aligned to the word size, otherwise you'll get a bus error from moving pointers
-	amount = (amount + 3) & ~3;
+	amount = (amount + 3) & ~3; //NOTE: this also leaves the lowest two bits as zero
 
 	assert((*bucketHandle) != NULL && "Expected a 'Toy_Bucket', received NULL");
-	assert((*bucketHandle)->capacity >= amount && "ERROR: Failed to partition a 'Toy_Bucket', requested amount is too high");
+	assert((*bucketHandle)->capacity >= (amount + 4) && "ERROR: Failed to partition a 'Toy_Bucket', requested amount is too high");
 
 	//if you're out of space in this bucket, allocate another one
-	if ((*bucketHandle)->capacity < (*bucketHandle)->count + amount) {
+	if ((*bucketHandle)->capacity < (*bucketHandle)->count + amount + 4) { //+4 for the metadata header
 		Toy_Bucket* tmp = Toy_allocateBucket((*bucketHandle)->capacity);
 		tmp->next = (*bucketHandle); //it's buckets all the way down
 		(*bucketHandle) = tmp;
 	}
 
-	//track the new count, and return the specified memory space
-	(*bucketHandle)->count += amount;
-	return ((*bucketHandle)->data + (*bucketHandle)->count - amount);
+	//use a 4-byte metadata header to hold the size of this partition, for GC
+	*((unsigned int*)((*bucketHandle)->data + (*bucketHandle)->count)) = amount;
+
+	//track the new metadata, and return the requested memory space
+	(*bucketHandle)->count += amount + 4;
+	return ((*bucketHandle)->data + (*bucketHandle)->count - amount); //metadata is before the pointer's address
+}
+
+void Toy_releaseBucketPartition(unsigned char* ptr) {
+	*((int*)(ptr-4)) |= 1; //flips the low-bit within the header
+	//no checks here, for technical reasons
 }
 
 void Toy_freeBucket(Toy_Bucket** bucketHandle) {
@@ -58,3 +66,55 @@ void Toy_freeBucket(Toy_Bucket** bucketHandle) {
 	//for safety
 	(*bucketHandle) = NULL;
 }
+
+TOY_API void Toy_collectBucketGarbage(Toy_Bucket** bucketHandle) {
+	//clear whatever this handle is pointing to
+	if ((*bucketHandle) == NULL) {
+		return;
+	}
+
+	Toy_Bucket* link = *bucketHandle;
+	while (link) {
+		//find non-free partitions
+		unsigned char* ptr = link->data;
+
+		bool gc = true;
+
+		while (ptr - link->data < link->count) { //for each partition
+			if ( (*((int*)ptr) & 1) == 0) { //is this partition still in use?
+				gc = false;
+				break;
+			}
+			ptr += (*((int*)(ptr)) ^ 1) + 4; //XOR to remove the 'free' flag from the size
+		}
+
+		//free this link, if its been entirely released
+		if (gc) {
+			//if link is the head
+			if (link == (*bucketHandle)) {
+				//if there's nowhere to go, don't delete the whole bucket
+				if ((*bucketHandle)->next == NULL) {
+					return;
+				}
+				else {
+					(*bucketHandle) = (*bucketHandle)->next;
+					free(link);
+					link = (*bucketHandle);
+				}
+			}
+			else {
+				//find the prev and free this link before continuing
+				Toy_Bucket* it = (*bucketHandle);
+				while (it->next != link) {
+					it = it->next;
+				}
+				it->next = link->next;
+				free(link);
+				link = it->next;
+			}
+		}
+		else {
+			link = link->next;
+		}
+	}
+}

source/toy_bucket.h

@@ -18,8 +18,11 @@ typedef struct Toy_Bucket {  //32 | 64 BITNESS
 
 TOY_API Toy_Bucket* Toy_allocateBucket(unsigned int capacity);
 TOY_API unsigned char* Toy_partitionBucket(Toy_Bucket** bucketHandle, unsigned int amount);
+TOY_API void Toy_releaseBucketPartition(unsigned char* ptr);
 TOY_API void Toy_freeBucket(Toy_Bucket** bucketHandle);
 
+TOY_API void Toy_collectBucketGarbage(Toy_Bucket** bucketHandle);
+
 //standard capacity sizes
 #ifndef TOY_BUCKET_1KB
 #define TOY_BUCKET_1KB (1 << 10)

source/toy_function.c

@@ -44,4 +44,6 @@ TOY_API void Toy_freeFunction(Toy_Function* fn) {
 	else if (fn->type == TOY_FUNCTION_NATIVE) {
 		fn->native.callback = NULL;
 	}
+
+	Toy_releaseBucketPartition((void*)fn);
 }

source/toy_scope.c

@@ -227,17 +227,29 @@ void Toy_private_incrementScopeRefCount(Toy_Scope* scope) {
 }
 
 void Toy_private_decrementScopeRefCount(Toy_Scope* scope) {
-	for (Toy_Scope* iter = scope; iter; iter = iter->next) {
+	Toy_Scope* iter = scope;
+
+	while (iter) {
 		iter->refCount--;
-		if (iter->refCount == 0 && iter->data != NULL) {
+		if (iter->refCount == 0) {
 			//free the scope entries when this scope is no longer needed
-			for (unsigned int i = 0; i < iter->capacity; i++) {
-				if (iter->data[i].psl > 0) {
-					Toy_freeString(&(iter->data[i].key));
-					Toy_freeValue(iter->data[i].value);
+			if (iter->data != NULL) {
+				for (unsigned int i = 0; i < iter->capacity; i++) {
+					if (iter->data[i].psl > 0) {
+						Toy_freeString(&(iter->data[i].key));
+						Toy_freeValue(iter->data[i].value);
+					}
 				}
+				free(iter->data);
 			}
-			free(iter->data);
+
+			//free the scope itself, fixing the iterator for the next loop
+			Toy_Scope* empty = iter;
+			iter = iter->next;
+			Toy_releaseBucketPartition((void*)empty);
+		}
+		else {
+			iter = iter->next;
 		}
 	}
 }

source/toy_string.c

@@ -23,6 +23,9 @@ static void decrementRefCount(Toy_String* str) {
 		decrementRefCount(str->node.left);
 		decrementRefCount(str->node.right);
 	}
+	if (str->info.refCount == 0) {
+		Toy_releaseBucketPartition((void*)str);
+	}
 }
 
 //exposed functions
@@ -43,10 +46,10 @@ Toy_String* Toy_toStringLength(Toy_Bucket** bucketHandle, const char* cstring, u
 }
 
 Toy_String* Toy_createStringLength(Toy_Bucket** bucketHandle, const char* cstring, unsigned int length) {
-	Toy_String* ret = (Toy_String*)Toy_partitionBucket(bucketHandle, sizeof(Toy_String));
+	Toy_String* ret = (Toy_String*)Toy_partitionBucket(bucketHandle, sizeof(Toy_String) + length + 1);
 
 	if (length > 0) {
-		ret->leaf.data = (char*)Toy_partitionBucket(bucketHandle, length + 1);
+		ret->leaf.data = (char*)(ret + 1); //increments by 1 'string', to the length +1
 		strncpy((char*)(ret->leaf.data), cstring, length);
 		((char*)(ret->leaf.data))[length] = '\0'; //don't forget the null
 		ret->info.length = length;

source/toy_vm.c

@@ -1097,7 +1097,10 @@ void Toy_resetVM(Toy_VM* vm, bool preserveScope, bool preserveStack) {
 		Toy_resetStack(&vm->stack); //NOTE: has a realloc()
 	}
 
-	//NOTE: buckets are not altered during resets
+	//not sure how often to call teh GC
+	if (vm->memoryBucket) {
+		Toy_collectBucketGarbage(&vm->memoryBucket);
+	}
 }
 
 void Toy_initVM(Toy_VM* vm) {

tests/units/test_bucket.c

@@ -31,7 +31,7 @@ int test_buckets(void) {
 		Toy_partitionBucket(&bucket, sizeof(int));
 
 		//check
-		if (bucket == NULL || bucket->count != 4 * sizeof(int)) {
+		if (bucket == NULL || bucket->count != 4 * (sizeof(int) +4)) { //+4 take the metadata into account
 			fprintf(stderr, TOY_CC_ERROR "ERROR: failed to partition 'Toy_Bucket' correctly: count is %d, expected %d\n" TOY_CC_RESET, (int)(bucket->count), (int)(4*sizeof(int)));
 			return -1;
 		}
@@ -43,7 +43,7 @@ int test_buckets(void) {
 	//test partitioning a bucket, several times, with an internal expansion
 	{
 		//init
-		Toy_Bucket* bucket = Toy_allocateBucket(sizeof(int) * 4);
+		Toy_Bucket* bucket = Toy_allocateBucket((sizeof(int)+4) * 4); //+4 take the metadata into account
 
 		//grab some memory
 		Toy_partitionBucket(&bucket, sizeof(int));
@@ -55,11 +55,11 @@ int test_buckets(void) {
 
 		//checks - please note that the top-most bucket is what is being filled - older buckets are further along
 		if (
-			bucket->capacity != 4 * sizeof(int) ||
-			bucket->count != 2 * sizeof(int) ||
+			bucket->capacity != 4 * (sizeof(int)+4) ||
+			bucket->count != 2 * (sizeof(int)+4) ||
 			bucket->next == NULL ||
-			bucket->next->capacity != 4 * sizeof(int) ||
-			bucket->next->count != 4 * sizeof(int))
+			bucket->next->capacity != 4 * (sizeof(int)+4) ||
+			bucket->next->count != 4 * (sizeof(int)+4))
 		{
 			fprintf(stderr, TOY_CC_ERROR "ERROR: failed to expand 'Toy_Bucket' correctly\n" TOY_CC_RESET);
 			return -1;
@@ -72,6 +72,149 @@ int test_buckets(void) {
 	return 0;
 }
 
+int test_garbage_collection(void) {
+	//release one element in one bucket link
+	{
+		//init
+		Toy_Bucket* bucket = Toy_allocateBucket(sizeof(int) * 32);
+
+		//dummy data, producing 4 entries
+		unsigned char* ptr1 = Toy_partitionBucket(&bucket, sizeof(int));
+		unsigned char* ptr2 = Toy_partitionBucket(&bucket, sizeof(int));
+		unsigned char* ptr3 = Toy_partitionBucket(&bucket, sizeof(int));
+		unsigned char* ptr4 = Toy_partitionBucket(&bucket, sizeof(int));
+
+		//release exactly one chunk of data
+		(void)ptr1;
+		(void)ptr2;
+		Toy_releaseBucketPartition(ptr3);
+		(void)ptr4;
+
+
+		//check the state of the bucket's data
+		if (
+			bucket->capacity != 32 * sizeof(int) ||
+			bucket->count != 4 * (sizeof(int)+4) ||
+			bucket->next != NULL ||
+			((unsigned int*)(bucket->data))[0] != 4 ||
+			((unsigned int*)(bucket->data))[1] != 0 ||
+			((unsigned int*)(bucket->data))[2] != 4 ||
+			((unsigned int*)(bucket->data))[3] != 0 ||
+			((unsigned int*)(bucket->data))[4] != 5 || //nth bit is altered here
+			((unsigned int*)(bucket->data))[5] != 0 ||
+			((unsigned int*)(bucket->data))[6] != 4 ||
+			((unsigned int*)(bucket->data))[7] != 0
+		)
+		{
+			fprintf(stderr, TOY_CC_ERROR "ERROR: failed simple memory partition release in 'Toy_Bucket'\n" TOY_CC_RESET);
+			Toy_freeBucket(&bucket);
+			return -1;
+		}
+
+		//cleanup
+		Toy_freeBucket(&bucket);
+	}
+
+	//release one element in many bucket links
+	{
+		//init
+		Toy_Bucket* bucket = Toy_allocateBucket(sizeof(int) * 32);
+
+		//partition the bucket 100 times, for dummy data
+		for (int i = 0; i < 50; i++) {
+			Toy_partitionBucket(&bucket, sizeof(int));
+		}
+
+		unsigned char* ptr = Toy_partitionBucket(&bucket, sizeof(int)); //grab the 51st element
+
+		for (int i = 0; i < 49; i++) {
+			Toy_partitionBucket(&bucket, sizeof(int));
+		}
+
+		//16 integers to a link, check for 7 links
+		if (
+			bucket->next == NULL ||
+			bucket->next->next == NULL ||
+			bucket->next->next->next == NULL ||
+			bucket->next->next->next->next == NULL ||
+			bucket->next->next->next->next->next == NULL ||
+			bucket->next->next->next->next->next->next == NULL ||
+			bucket->next->next->next->next->next->next->next != NULL) //there is no 8th link
+		{
+			fprintf(stderr, TOY_CC_ERROR "ERROR: failed to set up 'Toy_Bucket' to 'release one element in many bucket links'\n" TOY_CC_RESET);
+			Toy_freeBucket(&bucket);
+			return -1;
+		}
+
+		Toy_releaseBucketPartition(ptr);
+
+		//check the 3rd element in the 4th link
+		if (
+			((int*)(bucket->next->next->next->data + 2 * (sizeof(int)+4) ))[0] != 5 ||
+			((int*)(bucket->next->next->next->data + 2 * (sizeof(int)+4) ))[1] != 0
+		)
+		{
+			fprintf(stderr, TOY_CC_ERROR "ERROR: failed to release one element in many bucket links\n" TOY_CC_RESET);
+			Toy_freeBucket(&bucket);
+			return -1;
+		}
+
+		//cleanup
+		Toy_freeBucket(&bucket);
+	}
+
+	//garbage collection on a chain
+	{
+		//init
+		Toy_Bucket* bucket = Toy_allocateBucket(sizeof(int) * 32);
+
+		//partition the bucket 100 times, for dummy data
+		for (int i = 0; i < 100; i++) {
+			Toy_partitionBucket(&bucket, sizeof(int));
+		}
+
+		//16 integers to a link, check for 7 links
+		if (
+			bucket->next == NULL ||
+			bucket->next->next == NULL ||
+			bucket->next->next->next == NULL ||
+			bucket->next->next->next->next == NULL ||
+			bucket->next->next->next->next->next == NULL ||
+			bucket->next->next->next->next->next->next == NULL ||
+			bucket->next->next->next->next->next->next->next != NULL) //there is no 8th link
+		{
+			fprintf(stderr, TOY_CC_ERROR "ERROR: failed to set up 'Toy_Bucket' to test garbage collection on a link\n" TOY_CC_RESET);
+			Toy_freeBucket(&bucket);
+			return -1;
+		}
+
+		//grab link pointers
+		Toy_Bucket* third = bucket->next->next;
+		Toy_Bucket* fourth = bucket->next->next->next;
+		Toy_Bucket* fifth = bucket->next->next->next->next;
+
+		//free all elements in this link
+		for (int i = 0; i < 16; i++) {
+			Toy_releaseBucketPartition((fourth->data + i*8 + 4));
+		}
+
+		//run the GC
+		Toy_collectBucketGarbage(&bucket);
+
+		//check
+		if (third->next != fifth) {
+			fprintf(stderr, TOY_CC_ERROR "ERROR: failed to remove a chain link from 'Toy_Bucket' correctly\n" TOY_CC_RESET);
+			Toy_freeBucket(&bucket);
+			return -1;
+		}
+
+		//cleanup
+		Toy_freeBucket(&bucket);
+	}
+
+	return 0;
+}
+
 int main(void) {
 	//run each test set, returning the total errors given
 	int total = 0, res = 0;
@@ -85,5 +228,14 @@ int main(void) {
 		}
 	}
 
+	{
+		res = test_garbage_collection();
+		total += res;
+
+		if (res == 0) {
+			printf(TOY_CC_NOTICE "All good\n" TOY_CC_RESET);
+		}
+	}
+
 	return total;
 }

tests/units/test_string.c

@@ -56,7 +56,7 @@ int test_string_allocation(void) {
 
 		//inspect the bucket
 		if (bucket->capacity != 1024 ||
-			bucket->count != sizeof(Toy_String) + 12 ||
+			bucket->count != sizeof(Toy_String) + 12 + 4 || //+4 for bucket metadata
 			bucket->next != NULL)
 		{
 			fprintf(stderr, TOY_CC_ERROR "ERROR: Unexpected bucket state after 'Toy_createStringLength'\n" TOY_CC_RESET);