marvin/asl
1
0
Fork 0
mirror of https://github.com/Kugelschieber/asl.git synced 2026-01-18 12:00:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Now splits by windows and unix new lines, added parsing structure.

Browse Source
This commit is contained in:
Marvin Blum
2015-12-17 23:23:49 +01:00
parent 05e6937201
commit 3f6b854a09
2 changed files with 122 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
src/tokenizer/tokenizer.go
View File

@@ -11,50 +11,52 @@ type Token struct {
Column int
}
var delimiter = []byte{
'=',
';',
'{',
'}',
'(',
')',
'[',
']',
'<',
'>',
'!',
',',
':',
'&',
'|',
'+',
'-',
'*',
'/'} // TODO: modulo?
var (
delimiter = []byte{
'=',
';',
'{',
'}',
'(',
')',
'[',
']',
'<',
'>',
'!',
',',
':',
'&',
'|',
'+',
'-',
'*',
'/'} // TODO: modulo?
var keywords = []string{
"var",
"if",
"while",
"switch",
"for",
"foreach",
"func",
"true",
"false",
"case",
"default",
"return",
"try",
"catch",
"exitwith",
"waituntil",
"code"}
keywords = []string{
"var",
"if",
"while",
"switch",
"for",
"foreach",
"func",
"true",
"false",
"case",
"default",
"return",
"try",
"catch",
"exitwith",
"waituntil",
"code"}
var whitespace = []byte{' ', '\n', '\t', '\r'}
var identifier = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"
var preprocessor = byte('#')
var new_line = []byte{'\r', '\n'}
whitespace = []byte{' ', '\n', '\t', '\r'}
identifier = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"
preprocessor = byte('#')
new_line = []byte{'\r', '\n'}
)
// Tokenizes the given byte array into syntax tokens,
// which can be parsed later.