Merge pull request #19 from nikolauska/new-repo-example

New repo layout and unified test

.gitignore

@@ -1,4 +0,0 @@
-/bin/
-/pkg/
-/out/
-/in/

asl.go

@@ -1,8 +1,6 @@
 package main
 
 import (
-	"parser"
-	"tokenizer"
 	"fmt"
 	"io/ioutil"
 	"path/filepath"
@@ -15,6 +13,7 @@ const (
     extension = ".asl"
     sqfextension = ".sqf"
     PathSeparator = string(os.PathSeparator)
+	new_line = "\r\n"
 )
 
 type ASLFile struct {
@@ -101,8 +100,8 @@ func compile(path string) {
             continue
         }
 
-    	token := tokenizer.Tokenize(code)
-    	compiler := parser.Compiler{}
+    	token := Tokenize(code)
+    	compiler := Compiler{}
     	sqf := compiler.Parse(token, pretty)
 
     	os.MkdirAll(filepath.FromSlash(path+PathSeparator+aslFiles[i].out), 0777)

asl_test.go

@@ -0,0 +1,327 @@
+package main
+
+import (
+	"io/ioutil"
+	"testing"
+)
+
+type Want struct {
+    tokens []string
+    parser string
+}
+
+type Got struct {
+    tokens []Token
+    parser string
+}
+
+func TestArray(t *testing.T) {
+	got := getCompiled(t, "test/array.asl")
+    want := &Want{
+        []string{"var","x","=","[","1",",","2",",","3","]",";","var","y","=","x","[","1","]",";"},
+        "x = [1,2,3];\r\ny = (x select (1));\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+func TestAssignResult(t *testing.T) {
+	got := getCompiled(t, "test/assign_result.asl")
+    want := &Want{
+        []string{"var", "x", "=", "foo", "(", "1", ",", "2", ",", "3", ")", ";", "y", "=", "bar", "(", "1", ",", "2", ",", "3", ")", ";"},
+	    "x = ([1, 2, 3] call foo);\r\ny = ([1, 2, 3] call bar);\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+func TestAssignment(t *testing.T) {
+	got := getCompiled(t, "test/assignment.asl")
+    want := &Want{
+        []string{"x", "=", "1", ";"},
+	    "x = 1;\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+func TestBuildinFunctionCall(t *testing.T) {
+	got := getCompiled(t, "test/buildin_func.asl")
+    want := &Want{
+        []string{"var","_x","=","setHit","(","getVar","(","player",",","foo",")","(","bar",")",")","(","\"head\"",",","\"tail\"",")",";"},
+	    "_x = (([player, foo] getVar bar) setHit [\"head\", \"tail\"]);\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+func TestCode(t *testing.T) {
+	got := getCompiled(t, "test/code.asl")
+    want := &Want{
+        []string{"var", "x", "=", "code", "(", "\"var x = 5;\"", ")", ";"},
+	    "x = {x = 5;};\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+func TestComment(t *testing.T) {
+	got := getCompiled(t, "test/comment.asl")
+    want := &Want{
+        []string{"var","x","=","1",";"},
+	    "x = 1;\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+
+func TestExitWith(t *testing.T) {
+	got := getCompiled(t, "test/exitwith.asl")
+    want := &Want{
+        []string{"exitwith","{","}"},
+	    "if (true) exitWith {\r\n};\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+func TestExpression1(t *testing.T) {
+	got := getCompiled(t, "test/expression1.asl")
+    want := &Want{
+        []string{"x","=","(","(","1","+","2","+","3",")","*","4","/","2",")","+","foo","(","1",",","2",",","3",")",";"},
+	    "x = ((1+2+3)*4/2)+([1, 2, 3] call foo);\r\n",
+	}
+	equal(t, got, want)
+}
+
+func TestExpression2(t *testing.T) {
+	got := getCompiled(t, "test/expression2.asl")
+    want := &Want{
+        []string{"var","x","=","true","|","|","(","3",">","=","4","&","&","5","<","8",")",";"},
+	    "x = true||(3>=4&&5<8);\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+func TestExpression3(t *testing.T) {
+	got := getCompiled(t, "test/expression3.asl")
+    want := &Want{
+        []string{"var","x","=","-","(","1","+","(","2","+","3",")",")","/","(","6","*","(","someVariable","+","99","-","100",")",")","-","(","20",")","+","!","anotherVariable","+","foo","(",")",";"},
+	    "x = -(1+(2+3))/(6*(someVariable+99-100))-(20)+!anotherVariable+([] call foo);\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+func TestFor(t *testing.T) {
+	got := getCompiled(t, "test/for.asl")
+    want := &Want{
+        []string{"for","var","i","=","0",";","i","<","100",";","i","=","i","+","1","{","}"},
+	    "for [{i=0}, {i<100}, {i=i+1}] do {\r\n};\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+func TestForeach(t *testing.T) {
+	got := getCompiled(t, "test/foreach.asl")
+    want := &Want{
+        []string{"foreach", "unit", "=", ">", "allUnits", "{", "}"},
+	    "{\r\nunit = _x;\r\n} forEach (allUnits);\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+func TestFunction(t *testing.T) {
+	got := getCompiled(t, "test/func.asl")
+    want := &Want{
+        []string{"func", "TestFunction", "(", "param0", ",", "param1", ")", "{", "return", "true", ";", "}"},
+	    "TestFunction = {\r\nparams [\"param0\",\"param1\"];\r\nreturn true;\r\n};\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+func TestFunctionCall(t *testing.T) {
+	got := getCompiled(t, "test/func_call.asl")
+    want := &Want{
+        []string{"func","myFunc","(","a",",","b",")","{","return","a",">","b",";","}","myFunc","(","1","+","3","/","4",",","2","-","(","66","*","22",")","/","3","-","(","(","123",")",")",")",";"},
+	    "myFunc = {\r\nparams [\"a\",\"b\"];\r\nreturn a>b;\r\n};\r\n[1+3/4, 2-(66*22)/3-((123))] call myFunc;\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+func TestFunctionParams(t *testing.T) {
+	got := getCompiled(t, "test/func_params.asl")
+    want := &Want{
+        []string{"func","myFunc","(","a","=","1",",","b","=","2",")","{","return","a","+","b",";","}"},
+	    "myFunc = {\r\nparams [[\"a\",1],[\"b\",2]];\r\nreturn a+b;\r\n};\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+func TestIdentifier(t *testing.T) {
+	got := getCompiled(t, "test/identifier.asl")
+    want := &Want{
+        []string{"var","format","=","\"should not be for mat!\"",";"},
+	    "format = \"should not be for mat!\";\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+func TestIf(t *testing.T) {
+	got := getCompiled(t, "test/if.asl")
+    want := &Want{
+        []string{"if","a","<","b","{","}"},
+	    "if (a<b) then {\r\n};\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+func TestInlineCode(t *testing.T) {
+	got := getCompiled(t, "test/inline_code.asl")
+    want := &Want{
+        []string{"var","inline_code","=","code","(","\"var a = 1;var b = 2;if a < b {foo();}\"",")",";"},
+	    "inline_code = {a = 1;b = 2;if (a<b) then {[] call foo;};};\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+func TestNegation(t *testing.T) {
+	got := getCompiled(t, "test/negation.asl")
+    want := &Want{
+        []string{"var","x","=","!","foo","(",")",";"},
+	    "x = !([] call foo);\r\n",
+	}
+
+	equal(t, got, want)
+}
+
+func TestOperator(t *testing.T) {
+	got := getCompiled(t, "test/operator.asl")
+    want := &Want{
+        []string{"if","x","=","=","y","&","&","x","!","=","y","&","&","x","<","=","y","&","&","x",">","=","y","&","&","x","<","y","&","&","x",">","y","{","}"},
+	    "if (x==y&&x!=y&&x<=y&&x>=y&&x<y&&x>y) then {\r\n};\r\n",
+    }
+
+
+	equal(t, got, want)
+}
+
+func TestPreprocessor(t *testing.T) {
+	got := getCompiled(t, "test/preprocessor.asl")
+    want := &Want{
+        []string{"#define HELLO_WORLD \"Hello World!\"", "hint", "(", ")", "(", "HELLO_WORLD", ")", ";"},
+	    "\r\n#define HELLO_WORLD \"Hello World!\"\r\nhint HELLO_WORLD;\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+func TestSwitch(t *testing.T) {
+    got := getCompiled(t, "test/switch.asl")
+    want := &Want{
+        []string{"switch","x","{","case","1",":","x","=","1",";","case","2",":","x","=","2",";","default",":","x","=","3",";","}"},
+	    "switch (x) do {\r\ncase 1:\r\n{\r\nx = 1;\r\n};\r\ncase 2:\r\n{\r\nx = 2;\r\n};\r\ndefault:\r\n{\r\nx = 3;\r\n};\r\n};\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+func TestTryCatch(t *testing.T) {
+	got := getCompiled(t, "test/try_catch.asl")
+    want := &Want{
+        []string{"try","{","}","catch","{","}"},
+	    "try {\r\n} catch {\r\n};\r\n",
+    }
+	equal(t, got, want)
+}
+
+func TestWaitUntil(t *testing.T) {
+	got := getCompiled(t, "test/waituntil.asl")
+    want := &Want{
+        []string{"waituntil","(","x","=","x","+","1",";","x","<","100",")",";"},
+	    "waitUntil {x=x+1;x<100};\r\n",
+    }
+
+	equal(t, got, want)
+}
+
+
+func TestWhile(t *testing.T) {
+	got := getCompiled(t, "test/while.asl")
+    want := &Want{
+        []string{"while", "true", "{", "}"},
+	    "while {true} do {\r\n};",
+    }
+
+	equal(t, got, want)
+}
+
+func getCompiled(t *testing.T, file string) *Got {
+	code, err := ioutil.ReadFile(file)
+
+	if err != nil {
+		t.Error("Could not read test file: " + file)
+		t.FailNow()
+	}
+
+	tokens := Tokenize(code)
+	compiler := Compiler{}
+	parsed := compiler.Parse(tokens, true)
+
+	got := &Got{tokens, parsed}
+
+	return got
+}
+
+func compareLength(t *testing.T, got *Got, want *Want) {
+	if len(got.tokens) != len(want.tokens) {
+		t.Error("Length of tokens got and expected tokens not equal, was:")
+		gotlist, wantlist := "", ""
+
+		for i := range got.tokens {
+			gotlist += (got.tokens)[i].Token + " "
+		}
+
+		for i := range want.tokens {
+			wantlist += (want.tokens)[i] + " "
+		}
+
+		t.Log(gotlist)
+		t.Log("expected:")
+		t.Log(wantlist)
+		t.FailNow()
+	}
+
+
+}
+
+func compareTokens(t *testing.T, got *Got, want *Want) {
+	for i := range got.tokens {
+		if (got.tokens)[i].Token != (want.tokens)[i] {
+			t.Error("Tokens do not match: " + (got.tokens)[i].Token + " != " + (want.tokens)[i])
+		}
+	}
+}
+
+func equal(t *testing.T, got *Got, want *Want) {
+	compareLength(t, got, want)
+	compareTokens(t, got, want)
+
+	if got.parser != want.parser {
+		t.Error("Parsed does not equal, got:")
+		t.Log(got.parser)
+		t.Log("expected:")
+		t.Log(want.parser)
+		t.FailNow()
+	}
+}

parser.go

@@ -1,14 +1,8 @@
-package parser
-
-import (
-	"tokenizer"
-)
-
-const new_line = "\r\n"
+package main
 
 // Parses tokens, validates code to a specific degree
 // and writes SQF code into desired location.
-func (c *Compiler) Parse(token []tokenizer.Token, prettyPrinting bool) string {
+func (c *Compiler) Parse(token []Token, prettyPrinting bool) string {
 	if !c.initParser(token, prettyPrinting) {
 	    return ""
 	}
@@ -313,7 +307,7 @@ func (c *Compiler) parseInlineCode() string {
 
     if len(code) > 2 {
         compiler := Compiler{}
-        output = "{"+compiler.Parse(tokenizer.Tokenize([]byte(code[1:len(code)-1])), false)+"}"
+        output = "{"+compiler.Parse(Tokenize([]byte(code[1:len(code)-1])), false)+"}"
     }
 
     c.expect(")")

parserHelper.go

@@ -1,12 +1,11 @@
-package parser
+package main
 
 import (
     "strconv"
-    "tokenizer"
 )
 
 type Compiler struct {
-    tokens []tokenizer.Token
+    tokens []Token
     tokenIndex int
     out string
     offset int
@@ -14,7 +13,7 @@ type Compiler struct {
 }
 
 // Initilizes the parser.
-func (c *Compiler) initParser(token []tokenizer.Token, prettyPrinting bool) bool {
+func (c *Compiler) initParser(token []Token, prettyPrinting bool) bool {
 	if len(token) == 0 {
 		return false
 	}
@@ -60,7 +59,7 @@ func (c *Compiler) next() {
 }
 
 // Returns current token or throws, if no more tokens are available.
-func (c *Compiler) get() tokenizer.Token {
+func (c *Compiler) get() Token {
 	if c.tokenIndex >= len(c.tokens) {
 		panic("No more tokens")
 	}
@@ -74,7 +73,7 @@ func (c *Compiler) end() bool {
 }
 
 // Checks if two strings match.
-func (c *Compiler) tokenEqual(a string, b tokenizer.Token) bool {
+func (c *Compiler) tokenEqual(a string, b Token) bool {
 	return a == b.Token
 }
 

src/parser/parser_test.go

@@ -1,186 +0,0 @@
-package parser_test
-
-import (
-    "tokenizer"
-	"parser"
-	"io/ioutil"
-	"testing"
-)
-
-func TestParserDeclaration(t *testing.T) {
-	got := getCompiled(t, "test/tokenizer_var.asl")
-	want := "x = 1;\r\narray = [1,2,3];\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserAssignment(t *testing.T) {
-	got := getCompiled(t, "test/parser_assignment.asl")
-	want := "x = 1;\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserIf(t *testing.T) {
-	got := getCompiled(t, "test/tokenizer_if.asl")
-	want := "if (a<b) then {\r\n};\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserWhile(t *testing.T) {
-	got := getCompiled(t, "test/tokenizer_while.asl")
-	want := "while {true} do {\r\n};"
-
-	equal(t, got, want)
-}
-
-func TestParserFor(t *testing.T) {
-	got := getCompiled(t, "test/tokenizer_for.asl")
-	want := "for [{i=0}, {i<100}, {i=i+1}] do {\r\n};\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserForeach(t *testing.T) {
-	got := getCompiled(t, "test/tokenizer_foreach.asl")
-	want := "{\r\nunit = _x;\r\n} forEach (allUnits);\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserSwitch(t *testing.T) {
-    got := getCompiled(t, "test/tokenizer_switch.asl")
-	want := "switch (x) do {\r\ncase 1:\r\n{\r\nx = 1;\r\n};\r\ncase 2:\r\n{\r\nx = 2;\r\n};\r\ndefault:\r\n{\r\nx = 3;\r\n};\r\n};\r\n"
-	
-	equal(t, got, want)
-}
-
-func TestParserFunction(t *testing.T) {
-	got := getCompiled(t, "test/tokenizer_func.asl")
-	want := "TestFunction = {\r\nparams [\"param0\",\"param1\"];\r\nreturn true;\r\n};\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserAssignResult(t *testing.T) {
-	got := getCompiled(t, "test/parser_assign_result.asl")
-	want := "x = ([1, 2, 3] call foo);\r\ny = ([1, 2, 3] call bar);\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserExpression(t *testing.T) {
-	got := getCompiled(t, "test/parser_expression.asl")
-	want := "x = -(1+(2+3))/(6*(someVariable+99-100))-(20)+!anotherVariable+([] call foo);\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserExpression2(t *testing.T) {
-	got := getCompiled(t, "test/parser_expression2.asl")
-	want := "x = true||(3>=4&&5<8);\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserFunctionCall(t *testing.T) {
-	got := getCompiled(t, "test/parser_func_call.asl")
-	want := "myFunc = {\r\nparams [\"a\",\"b\"];\r\nreturn a>b;\r\n};\r\n[1+3/4, 2-(66*22)/3-((123))] call myFunc;\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserBuildinFunctionCall(t *testing.T) {
-	got := getCompiled(t, "test/parser_buildin_func.asl")
-	want := "_x = (([player, foo] getVar bar) setHit [\"head\", \"tail\"]);\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserOperator(t *testing.T) {
-	got := getCompiled(t, "test/parser_operator.asl")
-	want := "if (x==y&&x!=y&&x<=y&&x>=y&&x<y&&x>y) then {\r\n};\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserTryCatch(t *testing.T) {
-	got := getCompiled(t, "test/parser_try_catch.asl")
-	want := "try {\r\n} catch {\r\n};\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserNegationFunctionCall(t *testing.T) {
-	got := getCompiled(t, "test/parser_negation.asl")
-	want := "x = !([] call foo);\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserExitWith(t *testing.T) {
-	got := getCompiled(t, "test/parser_exitwith.asl")
-	want := "if (true) exitWith {\r\n};\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserWaitUntil(t *testing.T) {
-	got := getCompiled(t, "test/parser_waituntil.asl")
-	want := "waitUntil {x=x+1;x<100};\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserArray(t *testing.T) {
-	got := getCompiled(t, "test/parser_array.asl")
-	want := "x = [1,2,3];\r\ny = (x select (1));\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserFunctionParams(t *testing.T) {
-	got := getCompiled(t, "test/parser_func_params.asl")
-	want := "myFunc = {\r\nparams [[\"a\",1],[\"b\",2]];\r\nreturn a+b;\r\n};\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserInlineCode(t *testing.T) {
-	got := getCompiled(t, "test/parser_code.asl")
-	want := "inline_code = {a = 1;b = 2;if (a<b) then {[] call foo;};};\r\n"
-
-	equal(t, got, want)
-}
-
-func TestParserPreprocessor(t *testing.T) {
-	got := getCompiled(t, "test/tokenizer_preprocessor.asl")
-	want := "\r\n#define HELLO_WORLD \"Hello World!\"\r\nhint HELLO_WORLD;\r\n"
-
-	equal(t, got, want)
-}
-
-func getCompiled(t *testing.T, file string) string {
-	code, err := ioutil.ReadFile(file)
-
-	if err != nil {
-		t.Error("Could not read test file: " + file)
-		t.FailNow()
-	}
-
-	tokens := tokenizer.Tokenize(code)
-	compiler := parser.Compiler{}
-
-	return compiler.Parse(tokens, true)
-}
-
-func equal(t *testing.T, got, want string) {
-	if got != want {
-		t.Error("Results do not equal, got:")
-		t.Log(got)
-		t.Log("expected:")
-		t.Log(want)
-		t.FailNow()
-	}
-}

src/tokenizer/tokenizer_test.go

@@ -1,134 +0,0 @@
-package tokenizer_test
-
-import (
-    "tokenizer"
-	"io/ioutil"
-	"testing"
-)
-
-func TestTokenizerVar(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_var.asl")
-	want := []string{"var", "x", "=", "1", ";", "var", "array", "=", "[", "1", ",", "2", ",", "3", "]", ";"}
-
-	compareLength(t, &got, &want)
-	compareTokens(t, &got, &want)
-}
-
-func TestTokenizerIf(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_if.asl")
-	want := []string{"if", "a", "<", "b", "{", "}"}
-
-	compareLength(t, &got, &want)
-	compareTokens(t, &got, &want)
-}
-
-func TestTokenizerWhile(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_while.asl")
-	want := []string{"while", "true", "{", "}"}
-
-	compareLength(t, &got, &want)
-	compareTokens(t, &got, &want)
-}
-
-func TestTokenizerFor(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_for.asl")
-	want := []string{"for", "var", "i", "=", "0", ";", "i", "<", "100", ";", "i", "=", "i", "+", "1", "{", "}"}
-
-	compareLength(t, &got, &want)
-	compareTokens(t, &got, &want)
-}
-
-func TestTokenizerForach(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_foreach.asl")
-	want := []string{"foreach", "unit", "=", ">", "allUnits", "{", "}"}
-
-	compareLength(t, &got, &want)
-	compareTokens(t, &got, &want)
-}
-
-func TestTokenizerSwitch(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_switch.asl")
-	want := []string{"switch", "x", "{", "case", "1", ":", "x", "=", "1", ";", "case", "2", ":", "x", "=", "2", ";", "default", ":", "x", "=", "3", ";", "}"}
-
-	compareLength(t, &got, &want)
-	compareTokens(t, &got, &want)
-}
-
-func TestTokenizerFunction(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_func.asl")
-	want := []string{"func", "TestFunction", "(", "param0", ",", "param1", ")", "{", "return", "true", ";", "}"}
-
-	compareLength(t, &got, &want)
-	compareTokens(t, &got, &want)
-}
-
-func TestTokenizerExpression(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_expr.asl")
-	want := []string{"x", "=", "(", "(", "1", "+", "2", "+", "3", ")", "*", "4", "/", "2", ")", "+", "foo", "(", "1", ",", "2", ",", "3", ")", ";"}
-
-	compareLength(t, &got, &want)
-	compareTokens(t, &got, &want)
-}
-
-func TestTokenizerIdentifier(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_identifier.asl")
-	want := []string{"var", "format", "=", "\"should not be for mat!\"", ";"}
-
-	compareLength(t, &got, &want)
-	compareTokens(t, &got, &want)
-}
-
-func TestTokenizerInlineCode(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_code.asl")
-	want := []string{"var", "x", "=", "code", "(", "\"var x = 5;\"", ")", ";"}
-
-	compareLength(t, &got, &want)
-	compareTokens(t, &got, &want)
-}
-
-func TestTokenizerPreprocessor(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_preprocessor.asl")
-	want := []string{"#define HELLO_WORLD \"Hello World!\"", "hint", "(", ")", "(", "HELLO_WORLD", ")", ";"}
-
-	compareLength(t, &got, &want)
-	compareTokens(t, &got, &want)
-}
-
-func compareLength(t *testing.T, got *[]tokenizer.Token, want *[]string) {
-	if len(*got) != len(*want) {
-		t.Error("Length of tokens got and expected tokens not equal, was:")
-		gotlist, wantlist := "", ""
-
-		for i := range *got {
-			gotlist += (*got)[i].Token + " "
-		}
-
-		for i := range *want {
-			wantlist += (*want)[i] + " "
-		}
-
-		t.Log(gotlist)
-		t.Log("expected:")
-		t.Log(wantlist)
-		t.FailNow()
-	}
-}
-
-func compareTokens(t *testing.T, got *[]tokenizer.Token, want *[]string) {
-	for i := range *got {
-		if (*got)[i].Token != (*want)[i] {
-			t.Error("Tokens do not match: " + (*got)[i].Token + " != " + (*want)[i])
-		}
-	}
-}
-
-func getTokens(t *testing.T, file string) []tokenizer.Token {
-	code, err := ioutil.ReadFile(file)
-
-	if err != nil {
-		t.Error("Could not read test file: " + file)
-		t.FailNow()
-	}
-
-	return tokenizer.Tokenize(code)
-}

test/comment.asl

@@ -7,4 +7,3 @@ comment
 */
 
 var x = 1;
-var array = [1, 2, 3];

tokenizer.go

@@ -1,4 +1,4 @@
-package tokenizer
+package main
 
 import (
 	"strings"
@@ -55,7 +55,7 @@ var keywords = []string{
 var whitespace = []byte{' ', '\n', '\t', '\r'}
 var identifier = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"
 var preprocessor = byte('#')
-var new_line = []byte{'\r', '\n'}
+var new_line_byte = []byte{'\r', '\n'}
 
 // Tokenizes the given byte array into syntax tokens,
 // which can be parsed later.
@@ -68,7 +68,7 @@ func Tokenize(code []byte) []Token {
 		c := code[i]
 		column++
 
-		if byteArrayContains(new_line, c) {
+		if byteArrayContains(new_line_byte, c) {
 		    line++
 		    column = 0
 		}
@@ -161,7 +161,7 @@ func preprocessorLine(code []byte, i *int, lineNr, column int) Token {
     for *i < len(code) {
         c = code[*i]
 
-        if byteArrayContains(new_line, c) {
+        if byteArrayContains(new_line_byte, c) {
             break
         }
 
@@ -172,7 +172,7 @@ func preprocessorLine(code []byte, i *int, lineNr, column int) Token {
     // read all new line characters (\r and \n)
     c = code[*i]
 
-    for byteArrayContains(new_line, c) {
+    for byteArrayContains(new_line_byte, c) {
         (*i)++
         c = code[*i]
     }