Merge remote-tracking branch 'refs/remotes/DeKugelschieber/master'

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+**1.2.0**
+
+* better error output
+* concurrent compiling
+* errors are handled per file and won't stop the whole compilation
+* function name check for build in functions
+* simpler syntax for "null" and unary buildin functions
+
+**1.1.1**
+
+* arrays can now be declared within expressions
+* code keyword bug fix
+
 **1.1.0**
 
 * changed syntax of foreach

README.md

@@ -36,6 +36,19 @@ asl [-v|-r|-pretty|--help] <input directory> <output directory>
 asl ./missions/myMission/myScripts ./missions/myMission/compiledScripts
 ```
 
+Since 1.2.0 ASL requires a [supportInfo](https://community.bistudio.com/wiki/supportInfo) file, which must be generated, named "types" and placed right next to the binary. So if you have the asl.exe there must also be a file called types right next to it. The content looks like:
+
+```
+...
+t:DIARY_RECORD
+t:LOCATION
+b:ARRAY waypointattachobject SCALAR,OBJECT
+b:OBJECT,GROUP enableattack BOOL
+...
+```
+
+A current types file will be delivered with the current release, but not updated when Arma is.
+
 ## Syntax
 
 ### Comments
@@ -68,6 +81,9 @@ var one = array[0];
 
 // accessing using a statement:
 var zwo = array[33/3-2];
+
+// it is possble to use arrays in expressions:
+var emptyArray = one-[0];
 ```
 
 ### Control structures
@@ -138,6 +154,8 @@ var _x = add();
 // result in _x is 0
 ```
 
+When trying to define a function with a name that exists in the build in function set, you'll get an compile error.
+
 ### Call build in commands
 
 To call SQF build in commands (like hint, getDir, addItem, ...) we have to use a different syntax.
@@ -158,6 +176,13 @@ foo(x, y, z)(1, 2, 3);
 [x, y, z] foo [1, 2, 3];
 ```
 
+If the build in function does not accept parameters or only on one side (unary function), it can be called with a single pair of brackets:
+
+```
+hint("your text");
+shownWatch();
+```
+
 ### Special functions
 
 There are some special functions in SQF, which also require special syntax in ASL. The examples presented here show how they are written in ASL and what the output will look like. Remember that ASL is case sensitive!
@@ -231,7 +256,7 @@ Keywords should not be used as identifiers. Here is a full list of all keywords
 | var |
 | if |
 | while |
-| witch |
+| switch |
 | for |
 | foreach |
 | func |
@@ -248,22 +273,18 @@ Keywords should not be used as identifiers. Here is a full list of all keywords
 
 ## What's missing?
 
-The following features are not implemented yet, but will be in 1.1.0 or a future version:
+The following features are not implemented yet, but will be in 1.3.0 or a future version:
 
 * scopes
 * else if
-* arrays within expressions (like someArray-[someEntity])
+* selector in expression
 
 scopes won't be supported, since they are a stupid concept and can be replaced by functions.
-There is a simple workaround for arrays within expressions:
+
+Selectors in expressions do not work (yet):
 
 ```
-// want: ... forEach allCurators-[myCurator];
-
-var myCuratorArray = [myCurator]; // or a more complex expression within array
-foreach allCurators-myCuratorArray {
-	// ...
-}
+var x = ([1, 2, 3]-[1, 2])[0]; // should result in 3, but does not work
 ```
 
 ## Contribute
@@ -278,6 +299,11 @@ For further information you can read the SQF tutorial and documentation of scrip
 * [Scripting commands](https://community.bistudio.com/wiki/Category:Scripting_Commands_Arma_3)
 * [Scripting preprocessor](https://community.bistudio.com/wiki/PreProcessor_Commands)
 
+Interesting pages to visit:
+
+* [Bohemia forum topic](https://forums.bistudio.com/topic/185649-asl-arma-scripting-language-compiler/)
+* [Armaholic page](http://www.armaholic.com/page.php?id=29720)
+
 ## License
 
 MIT

ToDo.md

@@ -6,14 +6,15 @@
 * ~~pretty/minified printing~~
 * ~~usage~~
 * ~~recursive compiling~~
-* concurrent compiling
+* ~~concurrent compiling~~
 * ~~inline buildin function call -> foo(a)(bar(x)(y));~~
 * ~~negative values e.g. -1, operator !~~
 * ~~tokenizer splits commands like "format" -> for, mat~~
 * ~~try ... catch~~
-* better error reporting
-    - recover panic
-    - line, column
+* ~~better error reporting~~
+    - ~~recover panic~~
+    - ~~line, column~~
+* type check for functions
 
 ## Special cases
 

src/main/asl.go

@@ -1,19 +1,21 @@
 package main
 
 import (
-	"parser"
-	"tokenizer"
 	"fmt"
 	"io/ioutil"
-	"path/filepath"
 	"os"
+	"parser"
+	"path/filepath"
 	"strings"
+	"tokenizer"
+	"types"
 )
 
 const (
-    version = "1.1.0"
+	version       = "1.2.0"
 	extension     = ".asl"
 	sqfextension  = ".sqf"
+	typeinfo      = "types"
 	PathSeparator = string(os.PathSeparator)
 )
 
@@ -41,10 +43,14 @@ func usage() {
 	fmt.Println("<output directory> output directory, directory structure will be created corresponding to input directory")
 }
 
+// Parses compiler flags.
 func flags(flag string) bool {
 	flag = strings.ToLower(flag)
 
-    if flag[0] == '-' {
+	if flag[0] != '-' {
+		return false
+	}
+
 	if flag == "-v" {
 		fmt.Println("asl version " + version)
 		exit = true
@@ -60,9 +66,16 @@ func flags(flag string) bool {
 	return true
 }
 
-    return false
+// Loads types from types file.
+// If none is provided, an error will be printed.
+func loadTypes() {
+	if err := types.LoadTypes(typeinfo); err != nil {
+		fmt.Println("No 'types' file provided. Please add type information to this file from 'supportInfo' script command output.")
+		exit = true
+	}
 }
 
+// Creates a list of all ASL files to compile.
 func readAslFiles(path string) {
 	dir, err := ioutil.ReadDir(path)
 
@@ -90,28 +103,57 @@ func readAslFiles(path string) {
 	}
 }
 
-func compile(path string) {
-    for i := 0; i < len(aslFiles); i++ {
-        out := filepath.FromSlash(path+PathSeparator+aslFiles[i].out+PathSeparator+aslFiles[i].newname+sqfextension)
-        fmt.Println(aslFiles[i].in+" -> "+out)
-        code, err := ioutil.ReadFile(aslFiles[i].in)
-        
-        if err != nil {
-            fmt.Println("Error reading file: "+aslFiles[i].in)
-            continue
+// Recovers and prints thrown error.
+func recoverCompileError(file string, waiter chan bool) {
+	if r := recover(); r != nil {
+		fmt.Println("Compile error in file "+file+":", r)
 	}
 
-    	token := tokenizer.Tokenize(code)
+	waiter <- true // the show must go on
+}
+
+// Compiles a single ASL file.
+func compileFile(path string, file ASLFile, waiter chan bool) {
+	defer recoverCompileError(file.in, waiter)
+
+	// read file
+	out := filepath.FromSlash(path + PathSeparator + file.out + PathSeparator + file.newname + sqfextension)
+	fmt.Println(file.in + " -> " + out)
+	code, err := ioutil.ReadFile(file.in)
+
+	if err != nil {
+		fmt.Println("Error reading file: " + file.in)
+		return
+	}
+
+	// compile
+	token := tokenizer.Tokenize(code, false)
 	compiler := parser.Compiler{}
 	sqf := compiler.Parse(token, pretty)
 
-    	os.MkdirAll(filepath.FromSlash(path+PathSeparator+aslFiles[i].out), 0777)
+	os.MkdirAll(filepath.FromSlash(path+PathSeparator+file.out), 0777)
 	err = ioutil.WriteFile(out, []byte(sqf), 0666)
 
 	if err != nil {
-    	    fmt.Println("Error writing file: "+aslFiles[i].out)
+		fmt.Println("Error writing file: " + file.out)
 		fmt.Println(err)
 	}
+
+	waiter <- true // done
+}
+
+// Compiles ASL files concurrently.
+func compile(path string) {
+	waiter := make(chan bool, len(aslFiles))
+
+	// fire compile
+	for i := 0; i < len(aslFiles); i++ {
+		go compileFile(path, aslFiles[i], waiter)
+	}
+
+	// wait until all files are compiled
+	for i := 0; i < len(aslFiles); i++ {
+		<-waiter
 	}
 }
 
@@ -125,7 +167,15 @@ func main() {
 	}
 
 	var i int
-	for i = 1; i < len(args) && flags(args[i]); i++ {}
+	for i = 1; i < len(args) && flags(args[i]); i++ {
+	}
+
+	if exit {
+		return
+	}
+
+	// load type information
+	loadTypes()
 
 	if exit {
 		return

src/parser/parser.go

@@ -1,7 +1,9 @@
 package parser
 
 import (
+	"errors"
 	"tokenizer"
+	"types"
 )
 
 const new_line = "\r\n"
@@ -70,34 +72,35 @@ func (c *Compiler) parseVar() {
 	if c.accept("=") {
 		c.next()
 		c.appendOut(" = ", false)
-
-		if c.accept("[") {
-			c.parseArray()
-		} else {
 		c.parseExpression(true)
 	}
-	}
 
 	c.expect(";")
 	c.appendOut(";", true)
 }
 
-func (c *Compiler) parseArray() {
+func (c *Compiler) parseArray(out bool) string {
+	output := ""
 	c.expect("[")
-	c.appendOut("[", false)
+	output += "["
 
 	if !c.accept("]") {
-		c.parseExpression(true)
+		output += c.parseExpression(false)
 
 		for c.accept(",") {
 			c.next()
-			c.appendOut(",", false)
-			c.parseExpression(true)
+			output += "," + c.parseExpression(false)
 		}
 	}
 
 	c.expect("]")
-	c.appendOut("]", false)
+	output += "]"
+
+	if out {
+		c.appendOut(output, false)
+	}
+
+	return output
 }
 
 func (c *Compiler) parseIf() {
@@ -214,6 +217,12 @@ func (c *Compiler) parseForeach() {
 
 func (c *Compiler) parseFunction() {
 	c.expect("func")
+
+	// check for build in function
+	if buildin := types.GetFunction(c.get().Token); buildin != nil {
+		panic(errors.New(c.get().Token + " is a build in function, choose a different name"))
+	}
+
 	c.appendOut(c.get().Token+" = {", true)
 	c.next()
 	c.expect("(")
@@ -313,7 +322,7 @@ func (c *Compiler) parseInlineCode() string {
 
 	if len(code) > 2 {
 		compiler := Compiler{}
-        output = "{"+compiler.Parse(tokenizer.Tokenize([]byte(code[1:len(code)-1])), false)+"}"
+		output = "{" + compiler.Parse(tokenizer.Tokenize([]byte(code[1:len(code)-1]), true), false) + "}"
 	}
 
 	c.expect(")")
@@ -358,30 +367,22 @@ func (c *Compiler) parseFunctionCall(out bool, name string) string {
 	output := ""
 
 	c.expect("(")
-	leftParams, leftParamCount := c.parseParameter(false)
+	paramsStr, paramCount := c.parseParameter(false)
 	c.expect(")")
 
-	if c.accept("(") {
 	// buildin function
-		c.next()
-		rightParams, rightParamCount := c.parseParameter(false)
-		c.expect(")")
+	buildin := types.GetFunction(name)
 
-		if leftParamCount > 1 {
-			leftParams = "[" + leftParams + "]"
-		}
-
-		if rightParamCount > 1 {
-			rightParams = "[" + rightParams + "]"
-		}
-
-		if leftParamCount > 0 {
-			output = leftParams + " " + name + " " + rightParams
+	if buildin != nil {
+		if buildin.Type == types.NULL {
+			output = name
+		} else if buildin.Type == types.UNARY {
+			output = c.parseUnaryFunction(name, paramsStr, paramCount)
 		} else {
-			output = name + " " + rightParams
+			output = c.parseBinaryFunction(name, paramsStr, buildin, paramCount)
 		}
 	} else {
-		output = "[" + leftParams + "] call " + name
+		output = "[" + paramsStr + "] call " + name
 	}
 
 	if out {
@@ -391,12 +392,49 @@ func (c *Compiler) parseFunctionCall(out bool, name string) string {
 	return output
 }
 
+func (c *Compiler) parseUnaryFunction(name, paramsStr string, paramCount int) string {
+	output := ""
+
+	if paramCount == 1 {
+		output = name + " " + paramsStr
+	} else {
+		output = "[" + paramsStr + "] call " + name
+	}
+
+	return output
+}
+
+func (c *Compiler) parseBinaryFunction(name string, leftParamsStr string, buildin *types.FunctionType, paramCount int) string {
+	output := ""
+
+	c.next()
+	rightParamsStr, rightParamCount := c.parseParameter(false)
+	c.expect(")")
+
+	if paramCount > 1 {
+		leftParamsStr = "[" + leftParamsStr + "]"
+	}
+
+	if rightParamCount > 1 {
+		rightParamsStr = "[" + rightParamsStr + "]"
+	}
+
+	if paramCount > 0 {
+		output = leftParamsStr + " " + name + " " + rightParamsStr
+	} else {
+		output = name + " " + rightParamsStr
+	}
+
+	return output
+}
+
 func (c *Compiler) parseParameter(out bool) (string, int) {
 	output := ""
 	count := 0
 
 	for !c.accept(")") {
-		output += c.parseExpression(out)
+		expr := c.parseExpression(out)
+		output += expr
 		count++
 
 		if !c.accept(")") {
@@ -463,6 +501,8 @@ func (c *Compiler) parseIdentifier() string {
 		name := c.get().Token
 		c.next()
 		output = "(" + c.parseFunctionCall(false, name) + ")"
+	} else if c.accept("[") {
+		output += c.parseArray(false)
 	} else if c.seek("[") {
 		output += "(" + c.get().Token
 		c.next()

src/parser/parser_helper.go

@@ -1,6 +1,7 @@
 package parser
 
 import (
+	"errors"
 	"strconv"
 	"tokenizer"
 )
@@ -38,7 +39,7 @@ func (c *Compiler) accept(token string) bool {
 // Throws if current token does not match expected one.
 func (c *Compiler) expect(token string) {
 	if !c.tokenEqual(token, c.get()) {
-		panic("Parse error, expected '" + token + "' but was '" + c.get().Token + "' in line "+strconv.Itoa(c.get().Line)+" at "+strconv.Itoa(c.get().Column))
+		panic(errors.New("Parse error, expected '" + token + "' but was '" + c.get().Token + "' in line " + strconv.Itoa(c.get().Line) + " at " + strconv.Itoa(c.get().Column)))
 	}
 
 	c.next()
@@ -62,7 +63,7 @@ func (c *Compiler) next() {
 // Returns current token or throws, if no more tokens are available.
 func (c *Compiler) get() tokenizer.Token {
 	if c.tokenIndex >= len(c.tokens) {
-		panic("No more tokens")
+		panic(errors.New("No more tokens"))
 	}
 
 	return c.tokens[c.tokenIndex]

src/parser/parser_test.go

@@ -1,166 +1,200 @@
 package parser_test
 
 import (
-    "tokenizer"
-	"parser"
 	"io/ioutil"
+	"parser"
 	"testing"
+	"tokenizer"
+	"types"
+)
+
+const (
+	types_file = "../../test/types"
 )
 
 func TestParserDeclaration(t *testing.T) {
-	got := getCompiled(t, "test/tokenizer_var.asl")
+	got := getCompiled(t, "../../test/tokenizer_var.asl")
 	want := "x = 1;\r\narray = [1,2,3];\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserAssignment(t *testing.T) {
-	got := getCompiled(t, "test/parser_assignment.asl")
+	got := getCompiled(t, "../../test/parser_assignment.asl")
 	want := "x = 1;\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserIf(t *testing.T) {
-	got := getCompiled(t, "test/tokenizer_if.asl")
+	got := getCompiled(t, "../../test/tokenizer_if.asl")
 	want := "if (a<b) then {\r\n};\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserWhile(t *testing.T) {
-	got := getCompiled(t, "test/tokenizer_while.asl")
+	got := getCompiled(t, "../../test/tokenizer_while.asl")
 	want := "while {true} do {\r\n};"
 
 	equal(t, got, want)
 }
 
 func TestParserFor(t *testing.T) {
-	got := getCompiled(t, "test/tokenizer_for.asl")
+	got := getCompiled(t, "../../test/tokenizer_for.asl")
 	want := "for [{i=0}, {i<100}, {i=i+1}] do {\r\n};\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserForeach(t *testing.T) {
-	got := getCompiled(t, "test/tokenizer_foreach.asl")
+	got := getCompiled(t, "../../test/tokenizer_foreach.asl")
 	want := "{\r\nunit = _x;\r\n} forEach (allUnits);\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserSwitch(t *testing.T) {
-    got := getCompiled(t, "test/tokenizer_switch.asl")
+	got := getCompiled(t, "../../test/tokenizer_switch.asl")
 	want := "switch (x) do {\r\ncase 1:\r\n{\r\nx = 1;\r\n};\r\ncase 2:\r\n{\r\nx = 2;\r\n};\r\ndefault:\r\n{\r\nx = 3;\r\n};\r\n};\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserFunction(t *testing.T) {
-	got := getCompiled(t, "test/tokenizer_func.asl")
+	got := getCompiled(t, "../../test/tokenizer_func.asl")
 	want := "TestFunction = {\r\nparams [\"param0\",\"param1\"];\r\nreturn true;\r\n};\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserAssignResult(t *testing.T) {
-	got := getCompiled(t, "test/parser_assign_result.asl")
+	got := getCompiled(t, "../../test/parser_assign_result.asl")
 	want := "x = ([1, 2, 3] call foo);\r\ny = ([1, 2, 3] call bar);\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserExpression(t *testing.T) {
-	got := getCompiled(t, "test/parser_expression.asl")
+	got := getCompiled(t, "../../test/parser_expression.asl")
 	want := "x = -(1+(2+3))/(6*(someVariable+99-100))-(20)+!anotherVariable+([] call foo);\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserExpression2(t *testing.T) {
-	got := getCompiled(t, "test/parser_expression2.asl")
+	got := getCompiled(t, "../../test/parser_expression2.asl")
 	want := "x = true||(3>=4&&5<8);\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserFunctionCall(t *testing.T) {
-	got := getCompiled(t, "test/parser_func_call.asl")
+	got := getCompiled(t, "../../test/parser_func_call.asl")
 	want := "myFunc = {\r\nparams [\"a\",\"b\"];\r\nreturn a>b;\r\n};\r\n[1+3/4, 2-(66*22)/3-((123))] call myFunc;\r\n"
 
 	equal(t, got, want)
 }
 
-func TestParserBuildinFunctionCall(t *testing.T) {
-	got := getCompiled(t, "test/parser_buildin_func.asl")
-	want := "_x = (([player, foo] getVar bar) setHit [\"head\", \"tail\"]);\r\n"
+func TestParserNullBuildinFunctionCall(t *testing.T) {
+	types.LoadTypes(types_file)
+
+	got := getCompiled(t, "../../test/parser_null_buildin_func.asl")
+	want := "_volume = (radioVolume);\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserUnaryBuildinFunctionCall(t *testing.T) {
+	types.LoadTypes(types_file)
+
+	got := getCompiled(t, "../../test/parser_unary_buildin_func.asl")
+	want := "_isReady = (unitReady soldier);\r\n"
+
+	equal(t, got, want)
+}
+
+func TestParserBinaryBuildinFunctionCall(t *testing.T) {
+	types.LoadTypes(types_file)
+
+	got := getCompiled(t, "../../test/parser_binary_buildin_func.asl")
+	want := "someCar setHit [\"motor\", 1];\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserOperator(t *testing.T) {
-	got := getCompiled(t, "test/parser_operator.asl")
+	got := getCompiled(t, "../../test/parser_operator.asl")
 	want := "if (x==y&&x!=y&&x<=y&&x>=y&&x<y&&x>y) then {\r\n};\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserTryCatch(t *testing.T) {
-	got := getCompiled(t, "test/parser_try_catch.asl")
+	got := getCompiled(t, "../../test/parser_try_catch.asl")
 	want := "try {\r\n} catch {\r\n};\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserNegationFunctionCall(t *testing.T) {
-	got := getCompiled(t, "test/parser_negation.asl")
+	got := getCompiled(t, "../../test/parser_negation.asl")
 	want := "x = !([] call foo);\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserExitWith(t *testing.T) {
-	got := getCompiled(t, "test/parser_exitwith.asl")
+	got := getCompiled(t, "../../test/parser_exitwith.asl")
 	want := "if (true) exitWith {\r\n};\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserWaitUntil(t *testing.T) {
-	got := getCompiled(t, "test/parser_waituntil.asl")
+	got := getCompiled(t, "../../test/parser_waituntil.asl")
 	want := "waitUntil {x=x+1;x<100};\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserArray(t *testing.T) {
-	got := getCompiled(t, "test/parser_array.asl")
+	got := getCompiled(t, "../../test/parser_array.asl")
 	want := "x = [1,2,3];\r\ny = (x select (1));\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserFunctionParams(t *testing.T) {
-	got := getCompiled(t, "test/parser_func_params.asl")
+	got := getCompiled(t, "../../test/parser_func_params.asl")
 	want := "myFunc = {\r\nparams [[\"a\",1],[\"b\",2]];\r\nreturn a+b;\r\n};\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserInlineCode(t *testing.T) {
-	got := getCompiled(t, "test/parser_code.asl")
+	got := getCompiled(t, "../../test/parser_code.asl")
 	want := "inline_code = {a = 1;b = 2;if (a<b) then {[] call foo;};};\r\n"
 
 	equal(t, got, want)
 }
 
 func TestParserPreprocessor(t *testing.T) {
-	got := getCompiled(t, "test/tokenizer_preprocessor.asl")
+	types.LoadTypes(types_file)
+
+	got := getCompiled(t, "../../test/tokenizer_preprocessor.asl")
 	want := "\r\n#define HELLO_WORLD \"Hello World!\"\r\nhint HELLO_WORLD;\r\n"
 
 	equal(t, got, want)
 }
 
+func TestParserExpressionArray(t *testing.T) {
+	got := getCompiled(t, "../../test/parser_expression_array.asl")
+	want := "x = [1,2,3]-[2,3];\r\n"
+
+	equal(t, got, want)
+}
+
 func getCompiled(t *testing.T, file string) string {
 	code, err := ioutil.ReadFile(file)
 
@@ -169,7 +203,7 @@ func getCompiled(t *testing.T, file string) string {
 		t.FailNow()
 	}
 
-	tokens := tokenizer.Tokenize(code)
+	tokens := tokenizer.Tokenize(code, false)
 	compiler := parser.Compiler{}
 
 	return compiler.Parse(tokens, true)

src/tokenizer/tokenizer.go

@@ -11,7 +11,8 @@ type Token struct {
 	Column       int
 }
 
-var delimiter = []byte{
+var (
+	delimiter = []byte{
 		'=',
 		';',
 		'{',
@@ -32,7 +33,7 @@ var delimiter = []byte{
 		'*',
 		'/'} // TODO: modulo?
 
-var keywords = []string{
+	keywords = []string{
 		"var",
 		"if",
 		"while",
@@ -51,14 +52,19 @@ var keywords = []string{
 		"waituntil",
 		"code"}
 
-var whitespace = []byte{' ', '\n', '\t', '\r'}
-var identifier = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"
-var preprocessor = byte('#')
-var new_line = []byte{'\r', '\n'}
+	whitespace   = []byte{' ', '\n', '\t', '\r'}
+	identifier   = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"
+	preprocessor = byte('#')
+	new_line     = []byte{'\r', '\n'}
+)
 
 // Tokenizes the given byte array into syntax tokens,
 // which can be parsed later.
-func Tokenize(code []byte) []Token {
+func Tokenize(code []byte, doStripSlashes bool) []Token {
+	if doStripSlashes {
+		code = stripSlashes(code)
+	}
+
 	code = removeComments(code)
 	tokens := make([]Token, 0)
 	token, mask, isstring, line, column := "", false, false, 0, 0
@@ -114,6 +120,28 @@ func Tokenize(code []byte) []Token {
 	return tokens
 }
 
+// Removes slashes from input code.
+// This is used for the "code" keyword for correct strings in resulting code.
+func stripSlashes(code []byte) []byte {
+	newcode := make([]byte, len(code))
+	j, mask := 0, false
+
+	for i := 0; i < len(code); i++ {
+		c := code[i]
+
+		if c == '\\' && !mask {
+			mask = true
+			continue
+		}
+
+		newcode[j] = code[i]
+		mask = false
+		j++
+	}
+
+	return newcode
+}
+
 // Removes all comments from input byte array.
 // Comments are single line comments, starting with // (two slashes),
 // multi line comments with /* ... */ (slash star, star slash).

src/tokenizer/tokenizer_test.go

@@ -1,13 +1,13 @@
 package tokenizer_test
 
 import (
-    "tokenizer"
 	"io/ioutil"
 	"testing"
+	"tokenizer"
 )
 
 func TestTokenizerVar(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_var.asl")
+	got := getTokens(t, "../../test/tokenizer_var.asl")
 	want := []string{"var", "x", "=", "1", ";", "var", "array", "=", "[", "1", ",", "2", ",", "3", "]", ";"}
 
 	compareLength(t, &got, &want)
@@ -15,7 +15,7 @@ func TestTokenizerVar(t *testing.T) {
 }
 
 func TestTokenizerIf(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_if.asl")
+	got := getTokens(t, "../../test/tokenizer_if.asl")
 	want := []string{"if", "a", "<", "b", "{", "}"}
 
 	compareLength(t, &got, &want)
@@ -23,7 +23,7 @@ func TestTokenizerIf(t *testing.T) {
 }
 
 func TestTokenizerWhile(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_while.asl")
+	got := getTokens(t, "../../test/tokenizer_while.asl")
 	want := []string{"while", "true", "{", "}"}
 
 	compareLength(t, &got, &want)
@@ -31,7 +31,7 @@ func TestTokenizerWhile(t *testing.T) {
 }
 
 func TestTokenizerFor(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_for.asl")
+	got := getTokens(t, "../../test/tokenizer_for.asl")
 	want := []string{"for", "var", "i", "=", "0", ";", "i", "<", "100", ";", "i", "=", "i", "+", "1", "{", "}"}
 
 	compareLength(t, &got, &want)
@@ -39,7 +39,7 @@ func TestTokenizerFor(t *testing.T) {
 }
 
 func TestTokenizerForach(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_foreach.asl")
+	got := getTokens(t, "../../test/tokenizer_foreach.asl")
 	want := []string{"foreach", "unit", "=", ">", "allUnits", "{", "}"}
 
 	compareLength(t, &got, &want)
@@ -47,7 +47,7 @@ func TestTokenizerForach(t *testing.T) {
 }
 
 func TestTokenizerSwitch(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_switch.asl")
+	got := getTokens(t, "../../test/tokenizer_switch.asl")
 	want := []string{"switch", "x", "{", "case", "1", ":", "x", "=", "1", ";", "case", "2", ":", "x", "=", "2", ";", "default", ":", "x", "=", "3", ";", "}"}
 
 	compareLength(t, &got, &want)
@@ -55,7 +55,7 @@ func TestTokenizerSwitch(t *testing.T) {
 }
 
 func TestTokenizerFunction(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_func.asl")
+	got := getTokens(t, "../../test/tokenizer_func.asl")
 	want := []string{"func", "TestFunction", "(", "param0", ",", "param1", ")", "{", "return", "true", ";", "}"}
 
 	compareLength(t, &got, &want)
@@ -63,7 +63,7 @@ func TestTokenizerFunction(t *testing.T) {
 }
 
 func TestTokenizerExpression(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_expr.asl")
+	got := getTokens(t, "../../test/tokenizer_expr.asl")
 	want := []string{"x", "=", "(", "(", "1", "+", "2", "+", "3", ")", "*", "4", "/", "2", ")", "+", "foo", "(", "1", ",", "2", ",", "3", ")", ";"}
 
 	compareLength(t, &got, &want)
@@ -71,7 +71,7 @@ func TestTokenizerExpression(t *testing.T) {
 }
 
 func TestTokenizerIdentifier(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_identifier.asl")
+	got := getTokens(t, "../../test/tokenizer_identifier.asl")
 	want := []string{"var", "format", "=", "\"should not be for mat!\"", ";"}
 
 	compareLength(t, &got, &want)
@@ -79,7 +79,7 @@ func TestTokenizerIdentifier(t *testing.T) {
 }
 
 func TestTokenizerInlineCode(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_code.asl")
+	got := getTokens(t, "../../test/tokenizer_code.asl")
 	want := []string{"var", "x", "=", "code", "(", "\"var x = 5;\"", ")", ";"}
 
 	compareLength(t, &got, &want)
@@ -87,8 +87,17 @@ func TestTokenizerInlineCode(t *testing.T) {
 }
 
 func TestTokenizerPreprocessor(t *testing.T) {
-	got := getTokens(t, "test/tokenizer_preprocessor.asl")
-	want := []string{"#define HELLO_WORLD \"Hello World!\"", "hint", "(", ")", "(", "HELLO_WORLD", ")", ";"}
+	got := getTokens(t, "../../test/tokenizer_preprocessor.asl")
+	want := []string{"#define HELLO_WORLD \"Hello World!\"", "hint", "(", "HELLO_WORLD", ")", ";"}
+
+	compareLength(t, &got, &want)
+	compareTokens(t, &got, &want)
+}
+
+func TestTokenizerMask(t *testing.T) {
+	got := getTokens(t, "../../test/tokenizer_mask.asl")
+	want := []string{"var", "x", "=", "\"Hello \\\"World\\\"\"", ";",
+		"var", "y", "=", "code", "(", "\"var z = \\\"Hello \\\\\"World\\\\\"\\\";\"", ")", ";"}
 
 	compareLength(t, &got, &want)
 	compareTokens(t, &got, &want)
@@ -130,5 +139,5 @@ func getTokens(t *testing.T, file string) []tokenizer.Token {
 		t.FailNow()
 	}
 
-	return tokenizer.Tokenize(code)
+	return tokenizer.Tokenize(code, false)
 }

src/types/loader.go

@@ -0,0 +1,146 @@
+package types
+
+import (
+	"io/ioutil"
+	"strings"
+)
+
+const (
+	// type for object types
+	TYPE  = 1
+	NAN   = "NaN"
+	ARRAY = "ARRAY"
+
+	// types for functions
+	NULL   = 2
+	UNARY  = 3
+	BINARY = 4
+
+	win_new_line  = "\r\n"
+	unix_new_line = "\n"
+)
+
+type FunctionType struct {
+	Name      string
+	Type      int // one of the constants NULL, UNARY, BINARY
+	ArgsLeft  int
+	ArgsRight int // number of args on left side for binary functions
+}
+
+var functions []FunctionType
+
+// Returns function type information by name.
+// If not found, the parameter will be nil.
+func GetFunction(name string) *FunctionType {
+	name = strings.ToLower(name)
+
+	for _, function := range functions {
+		if function.Name == name {
+			return &function
+		}
+	}
+
+	return nil
+}
+
+// Loads type information from file.
+// The format is specified by 'supportInfo' command: https://community.bistudio.com/wiki/supportInfo
+func LoadTypes(path string) error {
+	content, err := ioutil.ReadFile(path)
+
+	if err != nil {
+		return err
+	}
+
+	data := strings.Replace(win_new_line, unix_new_line, string(content), -1) // make this work on windows and unix
+	functions = make([]FunctionType, 0)
+	parseTypes(data)
+
+	return nil
+}
+
+func parseTypes(content string) {
+	lines := strings.Split(content, unix_new_line)
+
+	for _, line := range lines {
+		if len(line) < 3 {
+			continue
+		}
+
+		if line[0] == 'n' {
+			parseNullFunction(line)
+		} else if line[0] == 'u' {
+			parseUnaryFunction(line)
+		} else if line[0] == 'b' {
+			parseBinaryFunction(line)
+		}
+	}
+}
+
+func parseNullFunction(line string) {
+	parts := getParts(line)
+	functions = append(functions, FunctionType{parts[0], NULL, 0, 0})
+}
+
+func parseUnaryFunction(line string) {
+	parts := getParts(line)
+
+	if len(parts) < 2 {
+		return
+	}
+
+	args := getArgs(parts[1])
+
+	var argsCount int
+
+	if args[0] != ARRAY {
+		argsCount = len(args) - getNaNArgs(args)
+	}
+
+	functions = append(functions, FunctionType{parts[0], UNARY, argsCount, 0})
+}
+
+func parseBinaryFunction(line string) {
+	parts := getParts(line)
+
+	if len(parts) < 3 {
+		return
+	}
+
+	argsLeft := getArgs(parts[0])
+	argsRight := getArgs(parts[2])
+
+	var argsLeftCount int
+	var argsRightCount int
+
+	if argsLeft[0] != ARRAY {
+		argsLeftCount = len(argsLeft) - getNaNArgs(argsLeft)
+	}
+
+	if argsRight[0] != ARRAY {
+		argsRightCount = len(argsRight) - getNaNArgs(argsRight)
+	}
+
+	functions = append(functions, FunctionType{parts[1], BINARY, argsLeftCount, argsRightCount})
+}
+
+func getParts(line string) []string {
+	line = line[2:]
+	return strings.Split(line, " ")
+}
+
+func getArgs(part string) []string {
+	return strings.Split(part, ",")
+}
+
+func getNaNArgs(args []string) int {
+	nan := 0
+
+	for _, arg := range args {
+		if arg == NAN {
+			nan++
+		}
+	}
+
+	return nan
+}

src/types/loader_test.go

@@ -0,0 +1,18 @@
+package types_test
+
+import (
+	"testing"
+	"types"
+)
+
+func TestTypesGetFunction(t *testing.T) {
+	if err := types.LoadTypes("../../test/types"); err != nil {
+		t.Error(err)
+	}
+
+	function := types.GetFunction("hint")
+
+	if function == nil {
+		t.Error("Function 'hint' not found in type list")
+	}
+}

test/parser_binary_buildin_func.asl

@@ -0,0 +1 @@
+setHit(someCar)("motor", 1);

test/parser_buildin_func.asl

@@ -1 +0,0 @@
-var _x = setHit(getVar(player, foo)(bar))("head", "tail");

test/parser_expression_array.asl

@@ -0,0 +1 @@
+var x = [1, 2, 3]-[2, 3];

test/parser_null_buildin_func.asl

@@ -0,0 +1 @@
+var _volume = radioVolume();

test/parser_unary_buildin_func.asl

@@ -0,0 +1 @@
+var _isReady = unitReady(soldier);

test/tokenizer_mask.asl

@@ -0,0 +1,2 @@
+var x = "Hello \"World\"";
+var y = code("var z = \"Hello \\"World\\"\";");

test/tokenizer_preprocessor.asl

@@ -1,2 +1,2 @@
 #define HELLO_WORLD "Hello World!"
-hint()(HELLO_WORLD);
+hint(HELLO_WORLD);

tools/README.md

@@ -3,12 +3,13 @@ A visual tool set to ease the work of asl developers.
 Maintained by yours truly: [654wak654](https://github.com/654wak654/)
 
 **Notepad++ Syntax Higligthing**: https://github.com/DeKugelschieber/asl/blob/master/tools/asl.xml
+
 Feel free to contribute with another text editor's syntax higligthing plugin.
 
 ##ASL GUI <img src="https://img.shields.io/badge/version-1.1.0.0-orange.svg" alt="version">
 An optional Java interface to make the compile procces of ASL faster and more user-friendly. It's released under the MIT licence just like the core project. It also helps with error reporting of asl.
 
-**Version 1.1.00**
+**Version 1.1.0.0**
 - New Arma 3-themed look, and error reporting straight from asl.
 - Program now depends on asl.exe, they need to be in the same directory.