Revert "New repo layout and unified test"

src/parser/parser.go

@@ -0,0 +1,528 @@
+package parser
+
+import (
+	"tokenizer"
+)
+
+const new_line = "\r\n"
+
+// Parses tokens, validates code to a specific degree
+// and writes SQF code into desired location.
+func (c *Compiler) Parse(token []tokenizer.Token, prettyPrinting bool) string {
+	if !c.initParser(token, prettyPrinting) {
+	    return ""
+	}
+
+	for c.tokenIndex < len(token) {
+		c.parseBlock()
+	}
+
+	return c.out
+}
+
+func (c *Compiler) parseBlock() {
+    if c.get().Preprocessor {
+        c.parsePreprocessor()
+    } else if c.accept("var") {
+		c.parseVar()
+	} else if c.accept("if") {
+		c.parseIf()
+	} else if c.accept("while") {
+		c.parseWhile()
+	} else if c.accept("switch") {
+		c.parseSwitch()
+	} else if c.accept("for") {
+		c.parseFor()
+	} else if c.accept("foreach") {
+		c.parseForeach()
+	} else if c.accept("func") {
+		c.parseFunction()
+	} else if c.accept("return") {
+		c.parseReturn()
+	} else if c.accept("try") {
+		c.parseTryCatch()
+	} else if c.accept("exitwith") {
+	    c.parseExitWith()
+	} else if c.accept("waituntil") {
+	    c.parseWaitUntil()
+	} else if c.accept("case") || c.accept("default") {
+		return
+	} else {
+		c.parseStatement()
+	}
+
+	if !c.end() && !c.accept("}") {
+		c.parseBlock()
+	}
+}
+
+func (c *Compiler) parsePreprocessor() {
+    // we definitely want a new line before and after
+    c.appendOut(new_line+c.get().Token+new_line, false)
+    c.next()
+}
+
+func (c *Compiler) parseVar() {
+	c.expect("var")
+	c.appendOut(c.get().Token, false)
+	c.next()
+
+	if c.accept("=") {
+		c.next()
+		c.appendOut(" = ", false)
+
+		if c.accept("[") {
+			c.parseArray()
+		} else {
+			c.parseExpression(true)
+		}
+	}
+
+	c.expect(";")
+	c.appendOut(";", true)
+}
+
+func (c *Compiler) parseArray() {
+	c.expect("[")
+	c.appendOut("[", false)
+
+	if !c.accept("]") {
+		c.parseExpression(true)
+
+		for c.accept(",") {
+			c.next()
+			c.appendOut(",", false)
+			c.parseExpression(true)
+		}
+	}
+
+	c.expect("]")
+	c.appendOut("]", false)
+}
+
+func (c *Compiler) parseIf() {
+	c.expect("if")
+	c.appendOut("if (", false)
+	c.parseExpression(true)
+	c.appendOut(") then {", true)
+	c.expect("{")
+	c.parseBlock()
+	c.expect("}")
+
+	if c.accept("else") {
+		c.next()
+		c.expect("{")
+		c.appendOut("} else {", true)
+		c.parseBlock()
+		c.expect("}")
+	}
+
+	c.appendOut("};", true)
+}
+
+func (c *Compiler) parseWhile() {
+	c.expect("while")
+	c.appendOut("while {", false)
+	c.parseExpression(true)
+	c.appendOut("} do {", true)
+	c.expect("{")
+	c.parseBlock()
+	c.expect("}")
+	c.appendOut("};", false)
+}
+
+func (c *Compiler) parseSwitch() {
+	c.expect("switch")
+	c.appendOut("switch (", false)
+	c.parseExpression(true)
+	c.appendOut(") do {", true)
+	c.expect("{")
+	c.parseSwitchBlock()
+	c.expect("}")
+	c.appendOut("};", true)
+}
+
+func (c *Compiler) parseSwitchBlock() {
+	if c.accept("}") {
+		return
+	}
+
+	if c.accept("case") {
+		c.next()
+		c.appendOut("case ", false)
+		c.parseExpression(true)
+		c.expect(":")
+		c.appendOut(":", true)
+
+		if !c.accept("case") && !c.accept("}") && !c.accept("default") {
+			c.appendOut("{", true)
+			c.parseBlock()
+			c.appendOut("};", true)
+		}
+	} else if c.accept("default") {
+		c.next()
+		c.expect(":")
+		c.appendOut("default:", true)
+
+		if !c.accept("}") {
+			c.appendOut("{", true)
+			c.parseBlock()
+			c.appendOut("};", true)
+		}
+	}
+
+	c.parseSwitchBlock()
+}
+
+func (c *Compiler) parseFor() {
+	c.expect("for")
+	c.appendOut("for [{", false)
+
+	// var in first assignment is optional
+	if c.accept("var") {
+		c.next()
+	}
+
+	c.parseExpression(true)
+	c.expect(";")
+	c.appendOut("}, {", false)
+	c.parseExpression(true)
+	c.expect(";")
+	c.appendOut("}, {", false)
+	c.parseExpression(true)
+	c.appendOut("}] do {", true)
+	c.expect("{")
+	c.parseBlock()
+	c.expect("}")
+	c.appendOut("};", true)
+}
+
+func (c *Compiler) parseForeach() {
+	c.expect("foreach")
+	element := c.get().Token
+	c.next()
+	c.expect("=")
+	c.expect(">")
+	expr := c.parseExpression(false)
+	c.expect("{")
+	c.appendOut("{", true)
+	c.appendOut(element+" = _x;", true)
+	c.parseBlock()
+	c.expect("}")
+	c.appendOut("} forEach ("+expr+");", true)
+}
+
+func (c *Compiler) parseFunction() {
+	c.expect("func")
+	c.appendOut(c.get().Token+" = {", true)
+	c.next()
+	c.expect("(")
+	c.parseFunctionParameter()
+	c.expect(")")
+	c.expect("{")
+	c.parseBlock()
+	c.expect("}")
+	c.appendOut("};", true)
+}
+
+func (c *Compiler) parseFunctionParameter() {
+	// empty parameter list
+	if c.accept("{") {
+		return
+	}
+	
+	c.appendOut("params [", false)
+
+	for !c.accept(")") {
+		name := c.get().Token
+		c.next()
+		
+		if c.accept("=") {
+		    c.next()
+		    value := c.get().Token
+		    c.next()
+		    c.appendOut("[\""+name+"\","+value+"]", false)
+		} else {
+		    c.appendOut("\""+name+"\"", false)
+		}
+
+		if !c.accept(")") {
+			c.expect(",")
+			c.appendOut(",", false)
+		}
+	}
+	
+	c.appendOut("];", true)
+}
+
+func (c *Compiler) parseReturn() {
+	c.expect("return")
+	c.appendOut("return ", false)
+	c.parseExpression(true)
+	c.expect(";")
+	c.appendOut(";", true)
+}
+
+func (c *Compiler) parseTryCatch() {
+	c.expect("try")
+	c.expect("{")
+	c.appendOut("try {", true)
+	c.parseBlock()
+	c.expect("}")
+	c.expect("catch")
+	c.expect("{")
+	c.appendOut("} catch {", true)
+	c.parseBlock()
+	c.expect("}")
+	c.appendOut("};", true)
+}
+
+func (c *Compiler) parseExitWith() {
+    c.expect("exitwith")
+    c.expect("{")
+    c.appendOut("if (true) exitWith {", true)
+    c.parseBlock()
+    c.expect("}")
+    c.appendOut("};", true)
+}
+
+func (c *Compiler) parseWaitUntil() {
+    c.expect("waituntil")
+    c.expect("(")
+    c.appendOut("waitUntil {", false)
+    c.parseExpression(true)
+    
+    if c.accept(";") {
+        c.next()
+        c.appendOut(";", false)
+        c.parseExpression(true)
+    }
+    
+    c.expect(")")
+    c.expect(";")
+    c.appendOut("};", true)
+}
+
+func (c *Compiler) parseInlineCode() string {
+    c.expect("code")
+    c.expect("(")
+    
+    code := c.get().Token
+    c.next()
+    output := "{}"
+    
+    if len(code) > 2 {
+        compiler := Compiler{}
+        output = "{"+compiler.Parse(tokenizer.Tokenize([]byte(code[1:len(code)-1])), false)+"}"
+    }
+    
+    c.expect(")")
+    
+    return output
+}
+
+// Everything that does not start with a keyword.
+func (c *Compiler) parseStatement() {
+	// empty block
+	if c.accept("}") || c.accept("case") || c.accept("default") {
+		return
+	}
+
+	// variable or function name
+	name := c.get().Token
+	c.next()
+
+	if c.accept("=") {
+		c.appendOut(name, false)
+		c.parseAssignment()
+	} else {
+		c.parseFunctionCall(true, name)
+		c.expect(";")
+		c.appendOut(";", true)
+	}
+
+	if !c.end() {
+		c.parseBlock()
+	}
+}
+
+func (c *Compiler) parseAssignment() {
+	c.expect("=")
+	c.appendOut(" = ", false)
+	c.parseExpression(true)
+	c.expect(";")
+	c.appendOut(";", true)
+}
+
+func (c *Compiler) parseFunctionCall(out bool, name string) string {
+	output := ""
+
+	c.expect("(")
+	leftParams, leftParamCount := c.parseParameter(false)
+	c.expect(")")
+
+	if c.accept("(") {
+		// buildin function
+		c.next()
+		rightParams, rightParamCount := c.parseParameter(false)
+		c.expect(")")
+
+		if leftParamCount > 1 {
+			leftParams = "[" + leftParams + "]"
+		}
+
+		if rightParamCount > 1 {
+			rightParams = "[" + rightParams + "]"
+		}
+
+		if leftParamCount > 0 {
+			output = leftParams + " " + name + " " + rightParams
+		} else {
+			output = name + " " + rightParams
+		}
+	} else {
+		output = "[" + leftParams + "] call " + name
+	}
+
+	if out {
+		c.appendOut(output, false)
+	}
+
+	return output
+}
+
+func (c *Compiler) parseParameter(out bool) (string, int) {
+	output := ""
+	count := 0
+
+	for !c.accept(")") {
+		output += c.parseExpression(out)
+		count++
+
+		if !c.accept(")") {
+			c.expect(",")
+			output += ", "
+		}
+	}
+
+	if out {
+		c.appendOut(output, false)
+	}
+
+	return output, count
+}
+
+func (c *Compiler) parseExpression(out bool) string {
+	output := c.parseArith()
+
+	for c.accept("<") || c.accept(">") || c.accept("&") || c.accept("|") || c.accept("=") || c.accept("!") {
+		if c.accept("<") {
+			output += "<"
+			c.next()
+		} else if c.accept(">") {
+			output += ">"
+			c.next()
+		} else if c.accept("&") {
+			c.next()
+			c.expect("&")
+			output += "&&"
+		} else if c.accept("|") {
+			c.next()
+			c.expect("|")
+			output += "||"
+		} else if c.accept("=") {
+			output += "="
+			c.next()
+		} else {
+			c.next()
+			c.expect("=")
+			output += "!="
+		}
+
+		if c.accept("=") {
+			output += "="
+			c.next()
+		}
+
+		output += c.parseExpression(false)
+	}
+
+	if out {
+		c.appendOut(output, false)
+	}
+
+	return output
+}
+
+func (c *Compiler) parseIdentifier() string {
+	output := ""
+
+	if c.accept("code") {
+	    output += c.parseInlineCode()
+	} else if c.seek("(") && !c.accept("!") && !c.accept("-") {
+		name := c.get().Token
+		c.next()
+		output = "(" + c.parseFunctionCall(false, name) + ")"
+	} else if c.seek("[") {
+	    output += "("+c.get().Token
+	    c.next()
+	    c.expect("[")
+	    output += " select ("+c.parseExpression(false)+"))"
+	    c.expect("]")
+	} else if c.accept("!") || c.accept("-") {
+		output = c.get().Token
+		c.next()
+		output += c.parseTerm()
+	} else {
+		output = c.get().Token
+		c.next()
+	}
+
+	return output
+}
+
+func (c *Compiler) parseTerm() string {
+	if c.accept("(") {
+		c.expect("(")
+		output := "(" + c.parseExpression(false) + ")"
+		c.expect(")")
+
+		return output
+	}
+
+	return c.parseIdentifier()
+}
+
+func (c *Compiler) parseFactor() string {
+	output := c.parseTerm()
+
+	for c.accept("*") || c.accept("/") { // TODO: modulo?
+		if c.accept("*") {
+			output += "*"
+		} else {
+			output += "/"
+		}
+
+		c.next()
+		output += c.parseExpression(false)
+	}
+
+	return output
+}
+
+func (c *Compiler) parseArith() string {
+	output := c.parseFactor()
+
+	for c.accept("+") || c.accept("-") {
+		if c.accept("+") {
+			output += "+"
+		} else {
+			output += "-"
+		}
+
+		c.next()
+		output += c.parseExpression(false)
+	}
+
+	return output
+}

src/parser/parserHelper.go

@@ -0,0 +1,88 @@
+package parser
+
+import (
+    "strconv"
+    "tokenizer"
+)
+
+type Compiler struct {
+    tokens []tokenizer.Token
+    tokenIndex int
+    out string
+    offset int
+    pretty bool
+}
+
+// Initilizes the parser.
+func (c *Compiler) initParser(token []tokenizer.Token, prettyPrinting bool) bool {
+	if len(token) == 0 {
+		return false
+	}
+
+	c.tokens = token
+	c.tokenIndex = 0
+	c.out = ""
+	c.offset = 0
+	c.pretty = prettyPrinting
+	
+	return true
+}
+
+// Returns true, if current token matches expected one.
+// Does not throw parse errors and checks if token is available.
+func (c *Compiler) accept(token string) bool {
+	return c.tokenIndex < len(c.tokens) && c.tokenEqual(token, c.get())
+}
+
+// Hard version of "accept".
+// Throws if current token does not match expected one.
+func (c *Compiler) expect(token string) {
+	if !c.tokenEqual(token, c.get()) {
+		panic("Parse error, expected '" + token + "' but was '" + c.get().Token + "' in line "+strconv.Itoa(c.get().Line)+" at "+strconv.Itoa(c.get().Column))
+	}
+
+	c.next()
+}
+
+// Returns true, if the next token matches expected one.
+// Does not throw parse errors and checks if token is available.
+func (c *Compiler) seek(token string) bool {
+	if c.tokenIndex+1 >= len(c.tokens) {
+		return false
+	}
+
+	return c.tokenEqual(token, c.tokens[c.tokenIndex+1])
+}
+
+// Increases token counter, so that the next token is compared.
+func (c *Compiler) next() {
+	c.tokenIndex++
+}
+
+// Returns current token or throws, if no more tokens are available.
+func (c *Compiler) get() tokenizer.Token {
+	if c.tokenIndex >= len(c.tokens) {
+		panic("No more tokens")
+	}
+
+	return c.tokens[c.tokenIndex]
+}
+
+// Returns true if the end of input code was reached.
+func (c *Compiler) end() bool {
+	return c.tokenIndex == len(c.tokens)
+}
+
+// Checks if two strings match.
+func (c *Compiler) tokenEqual(a string, b tokenizer.Token) bool {
+	return a == b.Token
+}
+
+// Appends the output string to current SQF code output.
+func (c *Compiler) appendOut(str string, newLine bool) {
+	c.out += str
+
+	if newLine && c.pretty {
+		c.out += "\r\n"
+	}
+}

src/parser/parser_test.go

@@ -0,0 +1,186 @@
+package parser_test
+
+import (
+    "tokenizer"
+	"parser"
+	"io/ioutil"
+	"testing"
+)
+
+func TestParserDeclaration(t *testing.T) {
+	got := getCompiled(t, "test/tokenizer_var.asl")
+	want := "x = 1;\r\narray = [1,2,3];\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserAssignment(t *testing.T) {
+	got := getCompiled(t, "test/parser_assignment.asl")
+	want := "x = 1;\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserIf(t *testing.T) {
+	got := getCompiled(t, "test/tokenizer_if.asl")
+	want := "if (a<b) then {\r\n};\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserWhile(t *testing.T) {
+	got := getCompiled(t, "test/tokenizer_while.asl")
+	want := "while {true} do {\r\n};"
+
+	equal(t, got, want)
+}
+
+func TestParserFor(t *testing.T) {
+	got := getCompiled(t, "test/tokenizer_for.asl")
+	want := "for [{i=0}, {i<100}, {i=i+1}] do {\r\n};\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserForeach(t *testing.T) {
+	got := getCompiled(t, "test/tokenizer_foreach.asl")
+	want := "{\r\nunit = _x;\r\n} forEach (allUnits);\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserSwitch(t *testing.T) {
+    got := getCompiled(t, "test/tokenizer_switch.asl")
+	want := "switch (x) do {\r\ncase 1:\r\n{\r\nx = 1;\r\n};\r\ncase 2:\r\n{\r\nx = 2;\r\n};\r\ndefault:\r\n{\r\nx = 3;\r\n};\r\n};\r\n"
+	
+	equal(t, got, want)
+}
+
+func TestParserFunction(t *testing.T) {
+	got := getCompiled(t, "test/tokenizer_func.asl")
+	want := "TestFunction = {\r\nparams [\"param0\",\"param1\"];\r\nreturn true;\r\n};\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserAssignResult(t *testing.T) {
+	got := getCompiled(t, "test/parser_assign_result.asl")
+	want := "x = ([1, 2, 3] call foo);\r\ny = ([1, 2, 3] call bar);\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserExpression(t *testing.T) {
+	got := getCompiled(t, "test/parser_expression.asl")
+	want := "x = -(1+(2+3))/(6*(someVariable+99-100))-(20)+!anotherVariable+([] call foo);\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserExpression2(t *testing.T) {
+	got := getCompiled(t, "test/parser_expression2.asl")
+	want := "x = true||(3>=4&&5<8);\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserFunctionCall(t *testing.T) {
+	got := getCompiled(t, "test/parser_func_call.asl")
+	want := "myFunc = {\r\nparams [\"a\",\"b\"];\r\nreturn a>b;\r\n};\r\n[1+3/4, 2-(66*22)/3-((123))] call myFunc;\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserBuildinFunctionCall(t *testing.T) {
+	got := getCompiled(t, "test/parser_buildin_func.asl")
+	want := "_x = (([player, foo] getVar bar) setHit [\"head\", \"tail\"]);\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserOperator(t *testing.T) {
+	got := getCompiled(t, "test/parser_operator.asl")
+	want := "if (x==y&&x!=y&&x<=y&&x>=y&&x<y&&x>y) then {\r\n};\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserTryCatch(t *testing.T) {
+	got := getCompiled(t, "test/parser_try_catch.asl")
+	want := "try {\r\n} catch {\r\n};\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserNegationFunctionCall(t *testing.T) {
+	got := getCompiled(t, "test/parser_negation.asl")
+	want := "x = !([] call foo);\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserExitWith(t *testing.T) {
+	got := getCompiled(t, "test/parser_exitwith.asl")
+	want := "if (true) exitWith {\r\n};\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserWaitUntil(t *testing.T) {
+	got := getCompiled(t, "test/parser_waituntil.asl")
+	want := "waitUntil {x=x+1;x<100};\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserArray(t *testing.T) {
+	got := getCompiled(t, "test/parser_array.asl")
+	want := "x = [1,2,3];\r\ny = (x select (1));\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserFunctionParams(t *testing.T) {
+	got := getCompiled(t, "test/parser_func_params.asl")
+	want := "myFunc = {\r\nparams [[\"a\",1],[\"b\",2]];\r\nreturn a+b;\r\n};\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserInlineCode(t *testing.T) {
+	got := getCompiled(t, "test/parser_code.asl")
+	want := "inline_code = {a = 1;b = 2;if (a<b) then {[] call foo;};};\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserPreprocessor(t *testing.T) {
+	got := getCompiled(t, "test/tokenizer_preprocessor.asl")
+	want := "\r\n#define HELLO_WORLD \"Hello World!\"\r\nhint HELLO_WORLD;\r\n"
+
+	equal(t, got, want)
+}
+
+func getCompiled(t *testing.T, file string) string {
+	code, err := ioutil.ReadFile(file)
+
+	if err != nil {
+		t.Error("Could not read test file: " + file)
+		t.FailNow()
+	}
+
+	tokens := tokenizer.Tokenize(code)
+	compiler := parser.Compiler{}
+
+	return compiler.Parse(tokens, true)
+}
+
+func equal(t *testing.T, got, want string) {
+	if got != want {
+		t.Error("Results do not equal, got:")
+		t.Log(got)
+		t.Log("expected:")
+		t.Log(want)
+		t.FailNow()
+	}
+}