57 lines
1.1 KiB
Odin
57 lines
1.1 KiB
Odin
package toml
|
|
|
|
import "core:text/scanner"
|
|
import "core:unicode/utf8"
|
|
|
|
Tokenizer :: struct {
|
|
scanner: scanner.Scanner,
|
|
}
|
|
|
|
EOF :: scanner.EOF
|
|
Ident :: scanner.Ident
|
|
Int :: scanner.Int
|
|
Float :: scanner.Float
|
|
String :: scanner.String
|
|
Comment :: scanner.Comment
|
|
New_Line :: -9
|
|
|
|
tokenizer_init :: proc(tokenizer: ^Tokenizer, src: string) {
|
|
tokenizer^ = {}
|
|
scanner.init(&tokenizer.scanner, src)
|
|
tokenizer.scanner.flags = {.Scan_Ints, .Scan_Floats, .Scan_Strings, .Scan_Idents}
|
|
tokenizer.scanner.whitespace = {' ', '\t'}
|
|
}
|
|
|
|
is_ident :: #force_inline proc(r: rune) -> bool {
|
|
switch r {
|
|
case 'a' ..= 'z', 'A' ..= 'Z', '0' ..= '9', '-', '_':
|
|
return true
|
|
case:
|
|
return false
|
|
}
|
|
}
|
|
|
|
next_token :: proc(tokenizer: ^Tokenizer) -> rune {
|
|
ch := scanner.scan(&tokenizer.scanner)
|
|
tok := ch
|
|
|
|
switch ch {
|
|
case EOF, Int, Float, String:
|
|
case '\r':
|
|
if scanner.peek(&tokenizer.scanner) != '\n' {
|
|
scanner.error(&tokenizer.scanner, "invalid line ending, expected \n or \r\n")
|
|
} else {
|
|
scanner.next(&tokenizer.scanner)
|
|
}
|
|
tok = New_Line
|
|
case '\n':
|
|
tok = New_Line
|
|
case:
|
|
if is_ident(tok) {
|
|
ch = scanner.next(&tokenizer.scanner)
|
|
}
|
|
}
|
|
|
|
return tok
|
|
}
|