tokenizer

package
v0.0.0-...-03b1f2e Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Nov 21, 2021 License: MIT Imports: 6 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var NullToken = Token{
	Type: NULLTOKEN,
}

Functions

This section is empty.

Types

type Token

type Token struct {
	Type  TokenType
	Value string
	Line  int
}

func (Token) String

func (t Token) String() string

type TokenError

type TokenError struct {
	// contains filtered or unexported fields
}

TokenError is error

func (TokenError) Error

func (e TokenError) Error() string

type TokenType

type TokenType int
const (
	NULLTOKEN    TokenType = iota // No Token
	LEFTPAREN                     // "("
	RIGHTPAREN                    // ")"
	LEFTBRACE                     // "{"
	RIGHTBRACE                    // "}"
	COMMA                         // ","
	DOT                           // "."
	MINUS                         // "-"
	PLUS                          // "+"
	SEMICOLON                     // ";"
	DIVIDE                        // "/"
	MULTIPLY                      // "*"
	BANG                          // "!"
	BANGEQUAL                     // "!="
	EQUAL                         // "="
	DOUBLEEQUAL                   // "=="
	GREATER                       // ">"
	GREATEREQUAL                  // ">="
	LESS                          // "<"
	LESSEQUAL                     // "<="
	IDENTIFIER                    // E.g. "i"
	STRING                        // E.g. "Hello"
	NUMBER                        // E.g. 42, 3.14
	AND                           // &&
	OR                            // "||"
	CLASS                         // "class"
	ELSE                          // "else"
	FALSE                         // "false"
	FUN                           // "fun"
	FOR                           // "for"
	IF                            // "if"
	NIL                           // "nil"
	PRINT                         // "print"
	RETURN                        // "return"
	SUPER                         // "super"
	THIS                          // "this"
	TRUE                          // "true"
	VAR                           // "var"
	WHILE                         // "while"
	EOF                           // "EOF"
	COMMENT                       // "// This is a comment"
)

func (TokenType) String

func (t TokenType) String() string

type Tokenizer

type Tokenizer interface {
	GetToken() (Token, error)
}

Tokenizer is interface for token generation

func NewTokenizer

func NewTokenizer(source io.Reader) Tokenizer

NewTokenizer creates new instance of tokenizer

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL