Revert "New repo layout and unified test"

.gitignore

@@ -0,0 +1,4 @@
+/bin/
+/pkg/
+/out/
+/in/

asl_test.go

@@ -1,327 +0,0 @@
-package main
-
-import (
-	"io/ioutil"
-	"testing"
-)
-
-type Want struct {
-    tokens []string
-    parser string
-}
-
-type Got struct {
-    tokens []Token
-    parser string
-}
-
-func TestArray(t *testing.T) {
-	got := getCompiled(t, "test/array.asl")
-    want := &Want{
-        []string{"var","x","=","[","1",",","2",",","3","]",";","var","y","=","x","[","1","]",";"},
-        "x = [1,2,3];\r\ny = (x select (1));\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-func TestAssignResult(t *testing.T) {
-	got := getCompiled(t, "test/assign_result.asl")
-    want := &Want{
-        []string{"var", "x", "=", "foo", "(", "1", ",", "2", ",", "3", ")", ";", "y", "=", "bar", "(", "1", ",", "2", ",", "3", ")", ";"},
-	    "x = ([1, 2, 3] call foo);\r\ny = ([1, 2, 3] call bar);\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-func TestAssignment(t *testing.T) {
-	got := getCompiled(t, "test/assignment.asl")
-    want := &Want{
-        []string{"x", "=", "1", ";"},
-	    "x = 1;\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-func TestBuildinFunctionCall(t *testing.T) {
-	got := getCompiled(t, "test/buildin_func.asl")
-    want := &Want{
-        []string{"var","_x","=","setHit","(","getVar","(","player",",","foo",")","(","bar",")",")","(","\"head\"",",","\"tail\"",")",";"},
-	    "_x = (([player, foo] getVar bar) setHit [\"head\", \"tail\"]);\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-func TestCode(t *testing.T) {
-	got := getCompiled(t, "test/code.asl")
-    want := &Want{
-        []string{"var", "x", "=", "code", "(", "\"var x = 5;\"", ")", ";"},
-	    "x = {x = 5;};\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-func TestComment(t *testing.T) {
-	got := getCompiled(t, "test/comment.asl")
-    want := &Want{
-        []string{"var","x","=","1",";"},
-	    "x = 1;\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-
-func TestExitWith(t *testing.T) {
-	got := getCompiled(t, "test/exitwith.asl")
-    want := &Want{
-        []string{"exitwith","{","}"},
-	    "if (true) exitWith {\r\n};\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-func TestExpression1(t *testing.T) {
-	got := getCompiled(t, "test/expression1.asl")
-    want := &Want{
-        []string{"x","=","(","(","1","+","2","+","3",")","*","4","/","2",")","+","foo","(","1",",","2",",","3",")",";"},
-	    "x = ((1+2+3)*4/2)+([1, 2, 3] call foo);\r\n",
-	}
-	equal(t, got, want)
-}
-
-func TestExpression2(t *testing.T) {
-	got := getCompiled(t, "test/expression2.asl")
-    want := &Want{
-        []string{"var","x","=","true","|","|","(","3",">","=","4","&","&","5","<","8",")",";"},
-	    "x = true||(3>=4&&5<8);\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-func TestExpression3(t *testing.T) {
-	got := getCompiled(t, "test/expression3.asl")
-    want := &Want{
-        []string{"var","x","=","-","(","1","+","(","2","+","3",")",")","/","(","6","*","(","someVariable","+","99","-","100",")",")","-","(","20",")","+","!","anotherVariable","+","foo","(",")",";"},
-	    "x = -(1+(2+3))/(6*(someVariable+99-100))-(20)+!anotherVariable+([] call foo);\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-func TestFor(t *testing.T) {
-	got := getCompiled(t, "test/for.asl")
-    want := &Want{
-        []string{"for","var","i","=","0",";","i","<","100",";","i","=","i","+","1","{","}"},
-	    "for [{i=0}, {i<100}, {i=i+1}] do {\r\n};\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-func TestForeach(t *testing.T) {
-	got := getCompiled(t, "test/foreach.asl")
-    want := &Want{
-        []string{"foreach", "unit", "=", ">", "allUnits", "{", "}"},
-	    "{\r\nunit = _x;\r\n} forEach (allUnits);\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-func TestFunction(t *testing.T) {
-	got := getCompiled(t, "test/func.asl")
-    want := &Want{
-        []string{"func", "TestFunction", "(", "param0", ",", "param1", ")", "{", "return", "true", ";", "}"},
-	    "TestFunction = {\r\nparams [\"param0\",\"param1\"];\r\nreturn true;\r\n};\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-func TestFunctionCall(t *testing.T) {
-	got := getCompiled(t, "test/func_call.asl")
-    want := &Want{
-        []string{"func","myFunc","(","a",",","b",")","{","return","a",">","b",";","}","myFunc","(","1","+","3","/","4",",","2","-","(","66","*","22",")","/","3","-","(","(","123",")",")",")",";"},
-	    "myFunc = {\r\nparams [\"a\",\"b\"];\r\nreturn a>b;\r\n};\r\n[1+3/4, 2-(66*22)/3-((123))] call myFunc;\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-func TestFunctionParams(t *testing.T) {
-	got := getCompiled(t, "test/func_params.asl")
-    want := &Want{
-        []string{"func","myFunc","(","a","=","1",",","b","=","2",")","{","return","a","+","b",";","}"},
-	    "myFunc = {\r\nparams [[\"a\",1],[\"b\",2]];\r\nreturn a+b;\r\n};\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-func TestIdentifier(t *testing.T) {
-	got := getCompiled(t, "test/identifier.asl")
-    want := &Want{
-        []string{"var","format","=","\"should not be for mat!\"",";"},
-	    "format = \"should not be for mat!\";\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-func TestIf(t *testing.T) {
-	got := getCompiled(t, "test/if.asl")
-    want := &Want{
-        []string{"if","a","<","b","{","}"},
-	    "if (a<b) then {\r\n};\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-func TestInlineCode(t *testing.T) {
-	got := getCompiled(t, "test/inline_code.asl")
-    want := &Want{
-        []string{"var","inline_code","=","code","(","\"var a = 1;var b = 2;if a < b {foo();}\"",")",";"},
-	    "inline_code = {a = 1;b = 2;if (a<b) then {[] call foo;};};\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-func TestNegation(t *testing.T) {
-	got := getCompiled(t, "test/negation.asl")
-    want := &Want{
-        []string{"var","x","=","!","foo","(",")",";"},
-	    "x = !([] call foo);\r\n",
-	}
-
-	equal(t, got, want)
-}
-
-func TestOperator(t *testing.T) {
-	got := getCompiled(t, "test/operator.asl")
-    want := &Want{
-        []string{"if","x","=","=","y","&","&","x","!","=","y","&","&","x","<","=","y","&","&","x",">","=","y","&","&","x","<","y","&","&","x",">","y","{","}"},
-	    "if (x==y&&x!=y&&x<=y&&x>=y&&x<y&&x>y) then {\r\n};\r\n",
-    }
-
-
-	equal(t, got, want)
-}
-
-func TestPreprocessor(t *testing.T) {
-	got := getCompiled(t, "test/preprocessor.asl")
-    want := &Want{
-        []string{"#define HELLO_WORLD \"Hello World!\"", "hint", "(", ")", "(", "HELLO_WORLD", ")", ";"},
-	    "\r\n#define HELLO_WORLD \"Hello World!\"\r\nhint HELLO_WORLD;\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-func TestSwitch(t *testing.T) {
-    got := getCompiled(t, "test/switch.asl")
-    want := &Want{
-        []string{"switch","x","{","case","1",":","x","=","1",";","case","2",":","x","=","2",";","default",":","x","=","3",";","}"},
-	    "switch (x) do {\r\ncase 1:\r\n{\r\nx = 1;\r\n};\r\ncase 2:\r\n{\r\nx = 2;\r\n};\r\ndefault:\r\n{\r\nx = 3;\r\n};\r\n};\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-func TestTryCatch(t *testing.T) {
-	got := getCompiled(t, "test/try_catch.asl")
-    want := &Want{
-        []string{"try","{","}","catch","{","}"},
-	    "try {\r\n} catch {\r\n};\r\n",
-    }
-	equal(t, got, want)
-}
-
-func TestWaitUntil(t *testing.T) {
-	got := getCompiled(t, "test/waituntil.asl")
-    want := &Want{
-        []string{"waituntil","(","x","=","x","+","1",";","x","<","100",")",";"},
-	    "waitUntil {x=x+1;x<100};\r\n",
-    }
-
-	equal(t, got, want)
-}
-
-
-func TestWhile(t *testing.T) {
-	got := getCompiled(t, "test/while.asl")
-    want := &Want{
-        []string{"while", "true", "{", "}"},
-	    "while {true} do {\r\n};",
-    }
-
-	equal(t, got, want)
-}
-
-func getCompiled(t *testing.T, file string) *Got {
-	code, err := ioutil.ReadFile(file)
-
-	if err != nil {
-		t.Error("Could not read test file: " + file)
-		t.FailNow()
-	}
-
-	tokens := Tokenize(code)
-	compiler := Compiler{}
-	parsed := compiler.Parse(tokens, true)
-
-	got := &Got{tokens, parsed}
-
-	return got
-}
-
-func compareLength(t *testing.T, got *Got, want *Want) {
-	if len(got.tokens) != len(want.tokens) {
-		t.Error("Length of tokens got and expected tokens not equal, was:")
-		gotlist, wantlist := "", ""
-
-		for i := range got.tokens {
-			gotlist += (got.tokens)[i].Token + " "
-		}
-
-		for i := range want.tokens {
-			wantlist += (want.tokens)[i] + " "
-		}
-
-		t.Log(gotlist)
-		t.Log("expected:")
-		t.Log(wantlist)
-		t.FailNow()
-	}
-
-
-}
-
-func compareTokens(t *testing.T, got *Got, want *Want) {
-	for i := range got.tokens {
-		if (got.tokens)[i].Token != (want.tokens)[i] {
-			t.Error("Tokens do not match: " + (got.tokens)[i].Token + " != " + (want.tokens)[i])
-		}
-	}
-}
-
-func equal(t *testing.T, got *Got, want *Want) {
-	compareLength(t, got, want)
-	compareTokens(t, got, want)
-
-	if got.parser != want.parser {
-		t.Error("Parsed does not equal, got:")
-		t.Log(got.parser)
-		t.Log("expected:")
-		t.Log(want.parser)
-		t.FailNow()
-	}
-}

src/main/asl.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"parser"
+	"tokenizer"
 	"fmt"
 	"io/ioutil"
 	"path/filepath"
@@ -13,7 +15,6 @@ const (
     extension = ".asl"
     sqfextension = ".sqf"
     PathSeparator = string(os.PathSeparator)
-	new_line = "\r\n"
 )
 
 type ASLFile struct {
@@ -100,8 +101,8 @@ func compile(path string) {
             continue
         }
         
-    	token := Tokenize(code)
+    	token := tokenizer.Tokenize(code)
-    	compiler := Compiler{}
+    	compiler := parser.Compiler{}
     	sqf := compiler.Parse(token, pretty)
     	
     	os.MkdirAll(filepath.FromSlash(path+PathSeparator+aslFiles[i].out), 0777)

src/parser/parser.go

@@ -1,8 +1,14 @@
-package main
+package parser
+
+import (
+	"tokenizer"
+)
+
+const new_line = "\r\n"
 
 // Parses tokens, validates code to a specific degree
 // and writes SQF code into desired location.
-func (c *Compiler) Parse(token []Token, prettyPrinting bool) string {
+func (c *Compiler) Parse(token []tokenizer.Token, prettyPrinting bool) string {
 	if !c.initParser(token, prettyPrinting) {
 	    return ""
 	}
@@ -307,7 +313,7 @@ func (c *Compiler) parseInlineCode() string {
     
     if len(code) > 2 {
         compiler := Compiler{}
-        output = "{"+compiler.Parse(Tokenize([]byte(code[1:len(code)-1])), false)+"}"
+        output = "{"+compiler.Parse(tokenizer.Tokenize([]byte(code[1:len(code)-1])), false)+"}"
     }
     
     c.expect(")")

src/parser/parserHelper.go

@@ -1,11 +1,12 @@
-package main
+package parser
 
 import (
     "strconv"
+    "tokenizer"
 )
 
 type Compiler struct {
-    tokens []Token
+    tokens []tokenizer.Token
     tokenIndex int
     out string
     offset int
@@ -13,7 +14,7 @@ type Compiler struct {
 }
 
 // Initilizes the parser.
-func (c *Compiler) initParser(token []Token, prettyPrinting bool) bool {
+func (c *Compiler) initParser(token []tokenizer.Token, prettyPrinting bool) bool {
 	if len(token) == 0 {
 		return false
 	}
@@ -59,7 +60,7 @@ func (c *Compiler) next() {
 }
 
 // Returns current token or throws, if no more tokens are available.
-func (c *Compiler) get() Token {
+func (c *Compiler) get() tokenizer.Token {
 	if c.tokenIndex >= len(c.tokens) {
 		panic("No more tokens")
 	}
@@ -73,7 +74,7 @@ func (c *Compiler) end() bool {
 }
 
 // Checks if two strings match.
-func (c *Compiler) tokenEqual(a string, b Token) bool {
+func (c *Compiler) tokenEqual(a string, b tokenizer.Token) bool {
 	return a == b.Token
 }
 

src/parser/parser_test.go

@@ -0,0 +1,186 @@
+package parser_test
+
+import (
+    "tokenizer"
+	"parser"
+	"io/ioutil"
+	"testing"
+)
+
+func TestParserDeclaration(t *testing.T) {
+	got := getCompiled(t, "test/tokenizer_var.asl")
+	want := "x = 1;\r\narray = [1,2,3];\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserAssignment(t *testing.T) {
+	got := getCompiled(t, "test/parser_assignment.asl")
+	want := "x = 1;\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserIf(t *testing.T) {
+	got := getCompiled(t, "test/tokenizer_if.asl")
+	want := "if (a<b) then {\r\n};\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserWhile(t *testing.T) {
+	got := getCompiled(t, "test/tokenizer_while.asl")
+	want := "while {true} do {\r\n};"
+
+	equal(t, got, want)
+}
+
+func TestParserFor(t *testing.T) {
+	got := getCompiled(t, "test/tokenizer_for.asl")
+	want := "for [{i=0}, {i<100}, {i=i+1}] do {\r\n};\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserForeach(t *testing.T) {
+	got := getCompiled(t, "test/tokenizer_foreach.asl")
+	want := "{\r\nunit = _x;\r\n} forEach (allUnits);\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserSwitch(t *testing.T) {
+    got := getCompiled(t, "test/tokenizer_switch.asl")
+	want := "switch (x) do {\r\ncase 1:\r\n{\r\nx = 1;\r\n};\r\ncase 2:\r\n{\r\nx = 2;\r\n};\r\ndefault:\r\n{\r\nx = 3;\r\n};\r\n};\r\n"
+	
+	equal(t, got, want)
+}
+
+func TestParserFunction(t *testing.T) {
+	got := getCompiled(t, "test/tokenizer_func.asl")
+	want := "TestFunction = {\r\nparams [\"param0\",\"param1\"];\r\nreturn true;\r\n};\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserAssignResult(t *testing.T) {
+	got := getCompiled(t, "test/parser_assign_result.asl")
+	want := "x = ([1, 2, 3] call foo);\r\ny = ([1, 2, 3] call bar);\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserExpression(t *testing.T) {
+	got := getCompiled(t, "test/parser_expression.asl")
+	want := "x = -(1+(2+3))/(6*(someVariable+99-100))-(20)+!anotherVariable+([] call foo);\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserExpression2(t *testing.T) {
+	got := getCompiled(t, "test/parser_expression2.asl")
+	want := "x = true||(3>=4&&5<8);\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserFunctionCall(t *testing.T) {
+	got := getCompiled(t, "test/parser_func_call.asl")
+	want := "myFunc = {\r\nparams [\"a\",\"b\"];\r\nreturn a>b;\r\n};\r\n[1+3/4, 2-(66*22)/3-((123))] call myFunc;\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserBuildinFunctionCall(t *testing.T) {
+	got := getCompiled(t, "test/parser_buildin_func.asl")
+	want := "_x = (([player, foo] getVar bar) setHit [\"head\", \"tail\"]);\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserOperator(t *testing.T) {
+	got := getCompiled(t, "test/parser_operator.asl")
+	want := "if (x==y&&x!=y&&x<=y&&x>=y&&x<y&&x>y) then {\r\n};\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserTryCatch(t *testing.T) {
+	got := getCompiled(t, "test/parser_try_catch.asl")
+	want := "try {\r\n} catch {\r\n};\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserNegationFunctionCall(t *testing.T) {
+	got := getCompiled(t, "test/parser_negation.asl")
+	want := "x = !([] call foo);\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserExitWith(t *testing.T) {
+	got := getCompiled(t, "test/parser_exitwith.asl")
+	want := "if (true) exitWith {\r\n};\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserWaitUntil(t *testing.T) {
+	got := getCompiled(t, "test/parser_waituntil.asl")
+	want := "waitUntil {x=x+1;x<100};\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserArray(t *testing.T) {
+	got := getCompiled(t, "test/parser_array.asl")
+	want := "x = [1,2,3];\r\ny = (x select (1));\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserFunctionParams(t *testing.T) {
+	got := getCompiled(t, "test/parser_func_params.asl")
+	want := "myFunc = {\r\nparams [[\"a\",1],[\"b\",2]];\r\nreturn a+b;\r\n};\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserInlineCode(t *testing.T) {
+	got := getCompiled(t, "test/parser_code.asl")
+	want := "inline_code = {a = 1;b = 2;if (a<b) then {[] call foo;};};\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserPreprocessor(t *testing.T) {
+	got := getCompiled(t, "test/tokenizer_preprocessor.asl")
+	want := "\r\n#define HELLO_WORLD \"Hello World!\"\r\nhint HELLO_WORLD;\r\n"
+
+	equal(t, got, want)
+}
+
+func getCompiled(t *testing.T, file string) string {
+	code, err := ioutil.ReadFile(file)
+
+	if err != nil {
+		t.Error("Could not read test file: " + file)
+		t.FailNow()
+	}
+
+	tokens := tokenizer.Tokenize(code)
+	compiler := parser.Compiler{}
+
+	return compiler.Parse(tokens, true)
+}
+
+func equal(t *testing.T, got, want string) {
+	if got != want {
+		t.Error("Results do not equal, got:")
+		t.Log(got)
+		t.Log("expected:")
+		t.Log(want)
+		t.FailNow()
+	}
+}

src/tokenizer/tokenizer.go

@@ -1,4 +1,4 @@
-package main
+package tokenizer
 
 import (
 	"strings"
@@ -55,7 +55,7 @@ var keywords = []string{
 var whitespace = []byte{' ', '\n', '\t', '\r'}
 var identifier = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"
 var preprocessor = byte('#')
-var new_line_byte = []byte{'\r', '\n'}
+var new_line = []byte{'\r', '\n'}
 
 // Tokenizes the given byte array into syntax tokens,
 // which can be parsed later.
@@ -68,7 +68,7 @@ func Tokenize(code []byte) []Token {
 		c := code[i]
 		column++
 		
-		if byteArrayContains(new_line_byte, c) {
+		if byteArrayContains(new_line, c) {
 		    line++
 		    column = 0
 		}
@@ -161,7 +161,7 @@ func preprocessorLine(code []byte, i *int, lineNr, column int) Token {
     for *i < len(code) {
         c = code[*i]
         
-        if byteArrayContains(new_line_byte, c) {
+        if byteArrayContains(new_line, c) {
             break
         }
         
@@ -172,7 +172,7 @@ func preprocessorLine(code []byte, i *int, lineNr, column int) Token {
     // read all new line characters (\r and \n)
     c = code[*i]
     
-    for byteArrayContains(new_line_byte, c) {
+    for byteArrayContains(new_line, c) {
         (*i)++
         c = code[*i]
     }

src/tokenizer/tokenizer_test.go

@@ -0,0 +1,134 @@
+package tokenizer_test
+
+import (
+    "tokenizer"
+	"io/ioutil"
+	"testing"
+)
+
+func TestTokenizerVar(t *testing.T) {
+	got := getTokens(t, "test/tokenizer_var.asl")
+	want := []string{"var", "x", "=", "1", ";", "var", "array", "=", "[", "1", ",", "2", ",", "3", "]", ";"}
+
+	compareLength(t, &got, &want)
+	compareTokens(t, &got, &want)
+}
+
+func TestTokenizerIf(t *testing.T) {
+	got := getTokens(t, "test/tokenizer_if.asl")
+	want := []string{"if", "a", "<", "b", "{", "}"}
+
+	compareLength(t, &got, &want)
+	compareTokens(t, &got, &want)
+}
+
+func TestTokenizerWhile(t *testing.T) {
+	got := getTokens(t, "test/tokenizer_while.asl")
+	want := []string{"while", "true", "{", "}"}
+
+	compareLength(t, &got, &want)
+	compareTokens(t, &got, &want)
+}
+
+func TestTokenizerFor(t *testing.T) {
+	got := getTokens(t, "test/tokenizer_for.asl")
+	want := []string{"for", "var", "i", "=", "0", ";", "i", "<", "100", ";", "i", "=", "i", "+", "1", "{", "}"}
+
+	compareLength(t, &got, &want)
+	compareTokens(t, &got, &want)
+}
+
+func TestTokenizerForach(t *testing.T) {
+	got := getTokens(t, "test/tokenizer_foreach.asl")
+	want := []string{"foreach", "unit", "=", ">", "allUnits", "{", "}"}
+
+	compareLength(t, &got, &want)
+	compareTokens(t, &got, &want)
+}
+
+func TestTokenizerSwitch(t *testing.T) {
+	got := getTokens(t, "test/tokenizer_switch.asl")
+	want := []string{"switch", "x", "{", "case", "1", ":", "x", "=", "1", ";", "case", "2", ":", "x", "=", "2", ";", "default", ":", "x", "=", "3", ";", "}"}
+
+	compareLength(t, &got, &want)
+	compareTokens(t, &got, &want)
+}
+
+func TestTokenizerFunction(t *testing.T) {
+	got := getTokens(t, "test/tokenizer_func.asl")
+	want := []string{"func", "TestFunction", "(", "param0", ",", "param1", ")", "{", "return", "true", ";", "}"}
+
+	compareLength(t, &got, &want)
+	compareTokens(t, &got, &want)
+}
+
+func TestTokenizerExpression(t *testing.T) {
+	got := getTokens(t, "test/tokenizer_expr.asl")
+	want := []string{"x", "=", "(", "(", "1", "+", "2", "+", "3", ")", "*", "4", "/", "2", ")", "+", "foo", "(", "1", ",", "2", ",", "3", ")", ";"}
+
+	compareLength(t, &got, &want)
+	compareTokens(t, &got, &want)
+}
+
+func TestTokenizerIdentifier(t *testing.T) {
+	got := getTokens(t, "test/tokenizer_identifier.asl")
+	want := []string{"var", "format", "=", "\"should not be for mat!\"", ";"}
+
+	compareLength(t, &got, &want)
+	compareTokens(t, &got, &want)
+}
+
+func TestTokenizerInlineCode(t *testing.T) {
+	got := getTokens(t, "test/tokenizer_code.asl")
+	want := []string{"var", "x", "=", "code", "(", "\"var x = 5;\"", ")", ";"}
+
+	compareLength(t, &got, &want)
+	compareTokens(t, &got, &want)
+}
+
+func TestTokenizerPreprocessor(t *testing.T) {
+	got := getTokens(t, "test/tokenizer_preprocessor.asl")
+	want := []string{"#define HELLO_WORLD \"Hello World!\"", "hint", "(", ")", "(", "HELLO_WORLD", ")", ";"}
+
+	compareLength(t, &got, &want)
+	compareTokens(t, &got, &want)
+}
+
+func compareLength(t *testing.T, got *[]tokenizer.Token, want *[]string) {
+	if len(*got) != len(*want) {
+		t.Error("Length of tokens got and expected tokens not equal, was:")
+		gotlist, wantlist := "", ""
+
+		for i := range *got {
+			gotlist += (*got)[i].Token + " "
+		}
+
+		for i := range *want {
+			wantlist += (*want)[i] + " "
+		}
+
+		t.Log(gotlist)
+		t.Log("expected:")
+		t.Log(wantlist)
+		t.FailNow()
+	}
+}
+
+func compareTokens(t *testing.T, got *[]tokenizer.Token, want *[]string) {
+	for i := range *got {
+		if (*got)[i].Token != (*want)[i] {
+			t.Error("Tokens do not match: " + (*got)[i].Token + " != " + (*want)[i])
+		}
+	}
+}
+
+func getTokens(t *testing.T, file string) []tokenizer.Token {
+	code, err := ioutil.ReadFile(file)
+
+	if err != nil {
+		t.Error("Could not read test file: " + file)
+		t.FailNow()
+	}
+
+	return tokenizer.Tokenize(code)
+}

test/tokenizer_var.asl

@@ -7,3 +7,4 @@ comment
 */
 
 var x = 1;
+var array = [1, 2, 3];