mdemory/src/compiler/lexer.go

package main 

import (
	"strings"
)

var tokens []Token
var buffer []rune
var	row int32          =     1
var	column int32       =     1
var	previousRow int32 =     -1
var	previousColumn int32 =  -1
var textStarted bool =   false

type TokenType int

const (
	TextFragment TokenType = iota
	QuestionEnd
	ElementDashStart
	ElementPlusStart
	Identifier              
	IdentifierStart
	IdentifierEnd
	SectionIdentifierStart
	SectionStart
	SectionEnd
	SOF
	EOF
)

type Token struct {
	tokenType   TokenType;
	content        string;
	row		        int32;
	column          int32;
} 

func makePostTextToken(ttype TokenType, tokenLen int32, textType TokenType) {
	if (len(strings.Trim(string(buffer), " \n\t")) - 1 > 0) {
		textFragment := []rune{}
		for i := 0; i < len(buffer) - int(tokenLen); i++ {
			element := buffer[i]
			textFragment = append(textFragment, element)
		}
		tokens = append(
			tokens,
			Token{
				tokenType: textType,
				content: string(textFragment),
				row: int32(previousRow),
				column: int32(previousColumn),
			},
		)
	}
	tokens = append(
		tokens,
		Token{
			tokenType: ttype,
			content: string(buffer[len(buffer)-int(tokenLen):]),
			row: int32(row),
			column: int32(column),
		},
	)
	previousRow = row
	previousColumn = column
	buffer = []rune{}
}

func tokenize(fileRunes []rune) ( []Token, error ) {
	tokens = []Token{} 
	buffer = []rune{} 

	for i := 0; i < len(fileRunes); i++ {
		c := fileRunes[i]

		if (c == '\n') {
			row += 1
			column = 1
		}
		buffer = append(buffer, c)
		if !textStarted {
			if c == '\n' {
				previousRow += 1
				previousColumn = 1
			} else if (c == ' ') {
				previousColumn += 1
			} else {
				textStarted = true 
			}
		}

		switch c {
		case '[':
			makePostTextToken(IdentifierStart, 1, TextFragment)
			previousRow = row
			previousColumn = column
			textStarted = false
		case ']':
			if (len(buffer) - 1 > 1) {
				textFragment := []rune{}
				trimmedStr := strings.Trim(string(buffer), " \n\t")
				for i := 0; i <  len(trimmedStr) - 1; i++ {
					element := trimmedStr[i]
					textFragment = append(textFragment, rune(element))
				}
				tokens = append(
					tokens,
					Token{
						tokenType: Identifier,
						content: string(textFragment),
						row: int32(previousRow),
						column: int32(previousColumn),
					},
				)
			}
			tokens = append(
				tokens,
				Token{
					tokenType: IdentifierEnd,
					content: "]",
					row: int32(row),
					column: int32(column),
				},
			)
			previousRow = row
			previousColumn = column
			textStarted = false
			buffer = []rune{}
		case '#':
			makePostTextToken(SectionIdentifierStart, 1, TextFragment)
			previousRow = row
			previousColumn = column
			textStarted = false
		case '{':
			makePostTextToken(SectionStart, 1, Identifier)
			previousRow = row
			previousColumn = column
			textStarted = false
		case '}':
			makePostTextToken(SectionEnd, 1, TextFragment)
			previousRow = row
			previousColumn = column
			textStarted = false
		case '-':
			makePostTextToken(ElementDashStart, 1, TextFragment)
			previousRow = row
			previousColumn = column
			textStarted = false
		case '>':
			makePostTextToken(QuestionEnd, 1, TextFragment)
			previousRow = row
			previousColumn = column
		case '+':
			makePostTextToken(ElementPlusStart, 1, TextFragment)
			previousRow = row
			previousColumn = column
			textStarted = false
		}
		column += 1
	}
	makePostTextToken(EOF, 0, TextFragment)
	return tokens, nil
}

func ToString (ttype *TokenType) string {
	switch *ttype {
	case TextFragment:
		return "TextFragment"
	case QuestionEnd:
		return "QuestionEnd"               
	case ElementDashStart:
		return "ElementDashStart"          
	case ElementPlusStart:
		return "ElementPlusStart"          
	case Identifier:
		return "Identifier"                
	case IdentifierStart:
		return "IdentifierStart"            
	case IdentifierEnd:
		return "IdentifierEnd"             
	case SectionIdentifierStart:
		return "SectionIdentifierStart"             
	case SectionStart:
		return "SectionStart"             
	case SectionEnd:
		return "SectionEnd"             
	case EOF:
		return "EndOfFile"             
	default:
		return "NOT_DEFINED"
	}
}