golexers

const (
	STATE_POSSIBLEREGEX = STATE_CUSTOM // like normal but affects parsing '/'
	STATE_REGEX         = STATE_CUSTOM + 1
)

const (
	STRING_PROCESS = iota
	STRING_IGNORE
)

const (
	STATE_NORMAL = iota
	STATE_CHARLITERAL
	STATE_STRINGLITERAL
	STATE_RAWSTRINGLITERAL
	STATE_LONGSTRINGLITERAL
	STATE_EOLCOMMENT
	STATE_MLCOMMENT
	STATE_CUSTOM // keep last - first for a language specific state
)

const (
	IS_WORD    = 1 << 8
	IS_STRING  = 1 << 9
	IS_COMMENT = 1 << 10
)

const (
	INVALID      TokenType = -2
	END                    = -1
	KEYWORD                = 1 | IS_WORD
	KEYWORD_TYPE           = 2 | IS_WORD // a keyword (or known identifier) which is a type e.g. types in go
	IDENTIFIER             = 3 | IS_WORD
	BUILTIN                = 4 | IS_WORD // a built in function as used in go or perl
	PUNCTUATION            = 5
	LITERAL                = 6 // literal but not string or char
	CHARLITERAL            = 7
	STRING                 = 8 | IS_STRING // inside a string but not a word
	STRINGWORD             = 9 | IS_WORD | IS_STRING
	COMMENT                = 10 | IS_COMMENT // inside a commant but not a aord
	COMMENTWORD            = 11 | IS_WORD | IS_COMMENT
	NEWLINE                = 100
)

const (
	STATE_STARTTAG   = STATE_CUSTOM
	STATE_ATTRSTRING = STATE_CUSTOM + 1
)

const STATE_PERL_AFTER_END = STATE_CUSTOM

const (
	STATE_VERBATIMSTRING = STATE_CUSTOM
)