diff --git a/src/compiler/compiler.go b/src/compiler/compiler.go index 660e461..ac8dbbc 100644 --- a/src/compiler/compiler.go +++ b/src/compiler/compiler.go @@ -1,9 +1,9 @@ package main +// TODO previous token start and end + import ( - "bufio" "fmt" - "io" "log" "os" "strings" @@ -23,6 +23,7 @@ const ( IdentifierEnd SectionStart SectionEnd + EOF ) type Token struct { @@ -57,6 +58,8 @@ func toString (ttype *TokenType) string { return "SectionStart" case SectionEnd: return "SectionEnd" + case EOF: + return "EndOfFile" default: return "NOT_DEFINED" } @@ -72,11 +75,12 @@ func (e LexingErr) Error() string { return fmt.Sprintf("%d:%d - %s", e.row, e.column, e.message) } -var row int32 = 1 -var column int32 = 1 -var previous_row int32 = -1 -var previous_col int32 = -1 +var row int32 = 1 +var column int32 = 1 +var previous_row int32 = -1 +var previous_col int32 = -1 var can_have_text bool = false +var textStarted bool = false func makePostTextToken(ttype TokenType, tokenLen int32) { if (len(strings.Trim(string(buffer), " \n")) - 1 > 0) { @@ -109,31 +113,29 @@ func makePostTextToken(ttype TokenType, tokenLen int32) { buffer = []rune{} } -func tokenize(contents string) error { +func tokenize(runes []rune) error { tokens = []Token{} buffer = []rune{} - previous_row = -1 - previous_col = -1 - reader := bufio.NewReader( - strings.NewReader(contents), - ) - for { - c, sz, err := reader.ReadRune() + for i := 0; i < len(runes); i++ { // TODO previous token start and end - if err != nil { - if err == io.EOF { - break - } else { - log.Fatal(err) - } - } - _ = sz + c := runes[i] + if (c == '\n') { row += 1 column = 1 } buffer = append(buffer, c) + if !textStarted { + if c == '\n' { + previous_row += 1 + previous_col = 1 + } else if (c == ' ') { + previous_col += 1 + } else { + textStarted = true + } + } trimmedBuffer := strings.Trim(string(buffer), " \n") if (len(trimmedBuffer) > 2) { @@ -144,14 +146,20 @@ func tokenize(contents string) error { return LexingErr{"Text cannot be here", previous_row, previous_col} } makePostTextToken(SectionStart, 2) + previous_row = row + previous_col = column can_have_text = true + textStarted = false continue case "<|": if (len(trimmedBuffer) - 2 > 0 && !can_have_text) { return LexingErr{"Text cannot be here", previous_row, previous_col} } makePostTextToken(SectionEnd, 2) + previous_row = row + previous_col = column can_have_text = false + textStarted = false continue } } @@ -199,33 +207,44 @@ func tokenize(contents string) error { ) previous_row = row previous_col = column + fmt.Printf("--> %d:%d\n", row, column) + textStarted = false buffer = []rune{} case '+': if (len(trimmedBuffer) - 1 > 0 && !can_have_text) { return LexingErr{"Text cannot be here", previous_row, previous_col} } makePostTextToken(ElementPlusStart, 1) + previous_row = row + previous_col = column can_have_text = true + textStarted = false case '-': - // fmt.Printf("%s %d\n", trimmedBuffer, len(trimmedBuffer)) if (len(trimmedBuffer) - 1 > 0 && !can_have_text) { return LexingErr{"Text cannot be here", previous_row, previous_col} } makePostTextToken(ElementDashStart, 1) + previous_row = row + previous_col = column can_have_text = true + textStarted = false case '>': if (len(trimmedBuffer) - 1 > 0 && !can_have_text) { return LexingErr{"Text cannot be here", previous_row, previous_col} } makePostTextToken(QuestionEnd, 1) + previous_row = row + previous_col = column can_have_text = false } column += 1 } + makePostTextToken(EOF, 0) return nil } +var fileRunes []rune + func main() { - // openAndExtractFileContents log.Println("Compilation started") file, err := os.ReadFile("./input.mdem") @@ -234,12 +253,13 @@ func main() { } fileContents := string(file) - err = tokenize(fileContents) + err = tokenize([]rune(fileContents)) if (err != nil) { fmt.Printf("%s\n", err.Error()) return } + // prettyPrintTokens for i := 0; i < len(tokens); i++ { token := tokens[i] content := token.content