package toml

import "core:text/scanner"
import "core:unicode/utf8"

Tokenizer :: struct {
	scanner: scanner.Scanner,
}

EOF :: scanner.EOF
Ident :: scanner.Ident
Int :: scanner.Int
Float :: scanner.Float
String :: scanner.String
Comment :: scanner.Comment
New_Line :: -9

tokenizer_init :: proc(tokenizer: ^Tokenizer, src: string) {
	tokenizer^ = {}
	scanner.init(&tokenizer.scanner, src)
	tokenizer.scanner.flags = {.Scan_Ints, .Scan_Floats, .Scan_Strings, .Scan_Idents}
	tokenizer.scanner.whitespace = {' ', '\t'}
}

is_ident :: #force_inline proc(r: rune) -> bool {
	switch r {
	case 'a' ..= 'z', 'A' ..= 'Z', '0' ..= '9', '-', '_':
		return true
	case:
		return false
	}
}

next_token :: proc(tokenizer: ^Tokenizer) -> rune {
	ch := scanner.scan(&tokenizer.scanner)
	tok := ch

	switch ch {
	case EOF, Int, Float, String:
	case '\r':
		if scanner.peek(&tokenizer.scanner) != '\n' {
			scanner.error(&tokenizer.scanner, "invalid line ending, expected \n or \r\n")
		} else {
			scanner.next(&tokenizer.scanner)
		}
		tok = New_Line
	case '\n':
		tok = New_Line
	case:
		if is_ident(tok) {
			ch = scanner.next(&tokenizer.scanner)
		}
	}

	return tok
}