From 765a3fde5bd07361b6eb63cd9baa72c83f2ffd3f Mon Sep 17 00:00:00 2001
From: Marvin Blum <marvin@marvinblum.de>
Date: Sat, 10 Oct 2015 14:38:33 +0200
Subject: [PATCH] Started testing with tokenizer.

---
 in/simple.asl             |  8 +---
 src/asl/parser.go         |  2 +-
 src/asl/parserHelper.go   | 11 +++++
 src/asl/tokenizer.go      |  8 ----
 src/asl/tokenizer_test.go | 95 +++++++++++++++++++++++++++++++++++++++
 test/tokenizer_each.asl   |  3 ++
 test/tokenizer_for.asl    |  3 ++
 test/tokenizer_func.asl   |  3 ++
 test/tokenizer_if.asl     |  3 ++
 test/tokenizer_var.asl    |  9 ++++
 test/tokenizer_while.asl  |  3 ++
 11 files changed, 132 insertions(+), 16 deletions(-)
 create mode 100644 src/asl/tokenizer_test.go
 create mode 100644 test/tokenizer_each.asl
 create mode 100644 test/tokenizer_for.asl
 create mode 100644 test/tokenizer_func.asl
 create mode 100644 test/tokenizer_if.asl
 create mode 100644 test/tokenizer_var.asl
 create mode 100644 test/tokenizer_while.asl

diff --git a/in/simple.asl b/in/simple.asl
index 7d3a355..b2dcf18 100644
--- a/in/simple.asl
+++ b/in/simple.asl
@@ -1,7 +1 @@
-func foo(){
-    return 1;
-}
-
-var _x = foo();
-
-foo();
+var x = foo();
diff --git a/src/asl/parser.go b/src/asl/parser.go
index 296d2e9..d1e7499 100644
--- a/src/asl/parser.go
+++ b/src/asl/parser.go
@@ -312,7 +312,7 @@ func parseExpression(out bool) string {
 		} else if accept(")") {
 			openingBrackets--
 		}
-
+		
 		next()
 	}
 
diff --git a/src/asl/parserHelper.go b/src/asl/parserHelper.go
index 05eb849..0cbb64d 100644
--- a/src/asl/parserHelper.go
+++ b/src/asl/parserHelper.go
@@ -4,6 +4,7 @@ var tokens []Token
 var tokenIndex int
 var out string
 var offset int
+//var pretty bool
 
 // Initilizes the parser.
 func initParser(token []Token) {
@@ -33,6 +34,16 @@ func expect(token string) {
 	next()
 }
 
+// Returns true, if the next token matches expected one.
+// Does not throw parse errors and checks if token is available.
+func seek(token string) bool {
+    if tokenIndex+1 >= len(tokens) {
+        return false
+    }
+    
+    return tokenEqual(token, tokens[tokenIndex+1])
+}
+
 // Increases token counter, so that the next token is compared.
 func next() {
 	tokenIndex++
diff --git a/src/asl/tokenizer.go b/src/asl/tokenizer.go
index 5dce409..4ca0f16 100644
--- a/src/asl/tokenizer.go
+++ b/src/asl/tokenizer.go
@@ -1,7 +1,6 @@
 package asl
 
 import (
-	"fmt"
 	"strings"
 )
 
@@ -49,8 +48,6 @@ func Tokenize(code []byte) []Token {
 	tokens := make([]Token, 0)
 	token, mask, isstring := "", false, false
 
-	fmt.Println("CODE:\n"+string(code)) // TODO: remove
-
 	for i := range code {
 		c := code[i]
 		
@@ -89,11 +86,6 @@ func Tokenize(code []byte) []Token {
 		
 		mask = false
 	}
-	
-	fmt.Println("TOKENS:") // TODO: remove
-	for t := range tokens {
-	    fmt.Println(tokens[t].token)
-	}
 
 	return tokens
 }
diff --git a/src/asl/tokenizer_test.go b/src/asl/tokenizer_test.go
new file mode 100644
index 0000000..ae95a00
--- /dev/null
+++ b/src/asl/tokenizer_test.go
@@ -0,0 +1,95 @@
+package asl
+
+import (
+    "testing"
+    "io/ioutil"
+)
+
+func TestVar(t *testing.T) {
+    got := getTokens(t, "test/tokenizer_var.asl")
+	want := []string{"var", "x", "=", "1", ";"}
+	
+	compareLength(t, &got, &want)
+	compareTokens(t, &got, &want)
+}
+
+func TestIf(t *testing.T) {
+    got := getTokens(t, "test/tokenizer_if.asl")
+	want := []string{"if", "a", "<", "b", "{", "}"}
+	
+	compareLength(t, &got, &want)
+	compareTokens(t, &got, &want)
+}
+
+func TestWhile(t *testing.T) {
+    got := getTokens(t, "test/tokenizer_while.asl")
+	want := []string{"while", "true", "{", "}"}
+	
+	compareLength(t, &got, &want)
+	compareTokens(t, &got, &want)
+}
+
+//for var i = 0; i < 100; i = i+1 {
+//}
+func TestFor(t *testing.T) {
+    got := getTokens(t, "test/tokenizer_for.asl")
+	want := []string{"for", "var", "i", "=", "0", ";", "i", "<", "100", ";", "i", "=", "i+1", "{", "}"}
+	
+	compareLength(t, &got, &want)
+	compareTokens(t, &got, &want)
+}
+
+func TestEach(t *testing.T) {
+    got := getTokens(t, "test/tokenizer_each.asl")
+	want := []string{"each", "allUnits", "{", "}"}
+	
+	compareLength(t, &got, &want)
+	compareTokens(t, &got, &want)
+}
+
+func TestFunction(t *testing.T) {
+    got := getTokens(t, "test/tokenizer_func.asl")
+	want := []string{"func", "TestFunction", "(", "param0", ",", "param1", ")", "{", "return", "true", ";", "}"}
+	
+	compareLength(t, &got, &want)
+	compareTokens(t, &got, &want)
+}
+
+func compareLength(t *testing.T, got *[]Token, want *[]string) {
+    if len(*got) != len(*want) {
+	    t.Error("Length of tokens got and expected tokens not equal, was:")
+	    gotlist, wantlist := "", ""
+	    
+	    for i := range *got {
+	        gotlist += (*got)[i].token+" "
+	    }
+	    
+	    for i := range *want {
+	        wantlist += (*want)[i]+" "
+	    }
+	    
+	    t.Log(gotlist)
+	    t.Log("expected:")
+	    t.Log(wantlist)
+	    t.FailNow()
+	}
+}
+
+func compareTokens(t *testing.T, got *[]Token, want *[]string) {
+    for i := range *got {
+	    if (*got)[i].token != (*want)[i] {
+	        t.Error("Tokens do not match: "+(*got)[i].token+" != "+(*want)[i])
+	    }
+	}
+}
+
+func getTokens(t *testing.T, file string) []Token {
+    code, err := ioutil.ReadFile(file)
+    
+    if err != nil {
+        t.Error("Could not read test file: "+file)
+        t.FailNow()
+    }
+    
+	return Tokenize(code)
+}
diff --git a/test/tokenizer_each.asl b/test/tokenizer_each.asl
new file mode 100644
index 0000000..344d883
--- /dev/null
+++ b/test/tokenizer_each.asl
@@ -0,0 +1,3 @@
+each allUnits {
+    // ...
+}
diff --git a/test/tokenizer_for.asl b/test/tokenizer_for.asl
new file mode 100644
index 0000000..176d111
--- /dev/null
+++ b/test/tokenizer_for.asl
@@ -0,0 +1,3 @@
+for var i = 0; i < 100; i = i+1 {
+    // ...
+}
diff --git a/test/tokenizer_func.asl b/test/tokenizer_func.asl
new file mode 100644
index 0000000..fab8a0f
--- /dev/null
+++ b/test/tokenizer_func.asl
@@ -0,0 +1,3 @@
+func TestFunction(param0, param1) {
+    return true;
+}
diff --git a/test/tokenizer_if.asl b/test/tokenizer_if.asl
new file mode 100644
index 0000000..9e18f01
--- /dev/null
+++ b/test/tokenizer_if.asl
@@ -0,0 +1,3 @@
+if a < b {
+    // ...
+}
diff --git a/test/tokenizer_var.asl b/test/tokenizer_var.asl
new file mode 100644
index 0000000..a701b14
--- /dev/null
+++ b/test/tokenizer_var.asl
@@ -0,0 +1,9 @@
+// single line comment
+
+/*
+multi
+line
+comment
+*/
+
+var x = 1;
diff --git a/test/tokenizer_while.asl b/test/tokenizer_while.asl
new file mode 100644
index 0000000..3b65d3f
--- /dev/null
+++ b/test/tokenizer_while.asl
@@ -0,0 +1,3 @@
+while true {
+    // ...
+}