format

src/compiler/compiler.go

@@ -1,9 +1,9 @@
 package main
 
+// TODO previous token start and end
+
 import (
-	"bufio"
 	"fmt"
-	"io"
 	"log"
 	"os"
 	"strings"
@@ -23,6 +23,7 @@ const (
 	IdentifierEnd             
 	SectionStart
 	SectionEnd
+	EOF
 )
 
 type Token struct {
@@ -57,6 +58,8 @@ func toString (ttype *TokenType) string {
 		return "SectionStart"             
 	case SectionEnd:
 		return "SectionEnd"             
+	case EOF:
+		return "EndOfFile"             
 	default:
 		return "NOT_DEFINED"
 	}
@@ -77,6 +80,7 @@ var	column int32       =  1
 var	previous_row int32 =    -1
 var	previous_col int32 =    -1
 var can_have_text bool = false
+var textStarted bool =   false
 
 func makePostTextToken(ttype TokenType, tokenLen int32) {
 	if (len(strings.Trim(string(buffer), " \n")) - 1 > 0) {
@@ -109,31 +113,29 @@ func makePostTextToken(ttype TokenType, tokenLen int32) {
 	buffer = []rune{}
 }
 
-func tokenize(contents string) error {
+func tokenize(runes []rune) error {
 	tokens = []Token{} 
 	buffer = []rune{} 
 
-	previous_row = -1
-	previous_col = -1
-	reader := bufio.NewReader(
-		strings.NewReader(contents),
-	)
-	for {
-		c, sz, err := reader.ReadRune()
+	for i := 0; i < len(runes); i++ {
 		// TODO previous token start and end
-		if err != nil {
-			if err == io.EOF {
-				break
-			} else {
-				log.Fatal(err)
-			}
-		}
-		_ = sz
+		c := runes[i]
+
 		if (c == '\n') {
 			row += 1
 			column = 1
 		}
 		buffer = append(buffer, c)
+		if !textStarted {
+			if c == '\n' {
+				previous_row += 1
+				previous_col = 1
+			} else if (c == ' ') {
+				previous_col += 1
+			} else {
+				textStarted = true 
+			}
+		}
 
 		trimmedBuffer := strings.Trim(string(buffer), " \n")
 		if (len(trimmedBuffer) > 2) {
@@ -144,14 +146,20 @@ func tokenize(contents string) error {
 					return LexingErr{"Text cannot be here", previous_row, previous_col}
 				}
 				makePostTextToken(SectionStart, 2)
+				previous_row = row
+				previous_col = column
 				can_have_text = true
+				textStarted = false
 				continue
 			case "<|":
 				if (len(trimmedBuffer) - 2 > 0 && !can_have_text) {
 					return LexingErr{"Text cannot be here", previous_row, previous_col}
 				}
 				makePostTextToken(SectionEnd, 2)
+				previous_row = row
+				previous_col = column
 				can_have_text = false 
+				textStarted = false
 				continue
 			}
 		}
@@ -199,33 +207,44 @@ func tokenize(contents string) error {
 			)
 			previous_row = row
 			previous_col = column
+			fmt.Printf("--> %d:%d\n", row, column)
+			textStarted = false
 			buffer = []rune{}
 		case '+':
 			if (len(trimmedBuffer) - 1 > 0 && !can_have_text) {
 				return LexingErr{"Text cannot be here", previous_row, previous_col}
 			}
 			makePostTextToken(ElementPlusStart, 1)
+			previous_row = row
+			previous_col = column
 			can_have_text = true
+			textStarted = false
 		case '-':
-			// fmt.Printf("%s %d\n", trimmedBuffer, len(trimmedBuffer))
 			if (len(trimmedBuffer) - 1 > 0 && !can_have_text) {
 				return LexingErr{"Text cannot be here", previous_row, previous_col}			}
 			makePostTextToken(ElementDashStart, 1)
+			previous_row = row
+			previous_col = column
 			can_have_text = true
+			textStarted = false
 		case '>':
 			if (len(trimmedBuffer) - 1 > 0 && !can_have_text) {
 				return LexingErr{"Text cannot be here", previous_row, previous_col}			
 			}
 			makePostTextToken(QuestionEnd, 1)
+			previous_row = row
+			previous_col = column
 			can_have_text = false 
 		}
 		column += 1
 	}
+	makePostTextToken(EOF, 0)
 	return nil
 }
 
+var fileRunes []rune
+
 func main() {
-	// openAndExtractFileContents
 	log.Println("Compilation started")
 
 	file, err := os.ReadFile("./input.mdem")
@@ -234,12 +253,13 @@ func main() {
 	}
 	fileContents := string(file)
 
-	err = tokenize(fileContents)
+	err = tokenize([]rune(fileContents))
 	if (err != nil) {
 		fmt.Printf("%s\n", err.Error())	
 		return
 	}
 
+	// prettyPrintTokens
 	for i := 0; i < len(tokens); i++ {
 		token := tokens[i]
 		content := token.content