mdemory/src/compiler/lexer/lexer.go

package lexer 

import (
	"fmt"
	"strings"
)

var buffer []rune
var	row             int32 =     1
var	column          int32 =     1
var	previousRow     int32 =    -1
var	previousColumn  int32 =    -1
var textStarted     bool  = false

type TokenType int

const (
	TextFragment TokenType = iota
	QuestionEnd
	ElementDashStart
	ElementPlusStart
	Identifier              
	IdentifierStart
	IdentifierEnd
	SectionIdentifierStart
	SectionStart
	SectionEnd
	SOF
	EOF
)

type Token struct {
	TokenType   TokenType;
	Content        string;
	Row		        int32;
	Column          int32;
} 

func (token Token)ToString() string {
	content := token.Content
	if (token.TokenType == TextFragment) {
		content = strings.Replace(
			strings.Trim(content, " "),
			"\n",
			"\\n",
			-1,
		)
	}
	return fmt.Sprintf(
		"%s: \"%s\" %d:%d\n",
		ToString(&token.TokenType),
		content,
		token.Row,
		token.Column,
	)
}

var tokens []Token

func makePostTextToken(ttype TokenType, tokenLen int32, textType TokenType) {
	if (len(strings.Trim(string(buffer), " \n\t")) - 1 > 0) {
		textFragment := []rune{}
		for i := 0; i < len(buffer) - int(tokenLen); i++ {
			element := buffer[i]
			textFragment = append(textFragment, element)
		}
		tokens = append(
			tokens,
			Token{
				TokenType: textType,
				Content: string(textFragment),
				Row: int32(previousRow),
				Column: int32(previousColumn),
			},
		)
	}
	tokens = append(
		tokens,
		Token{
			TokenType: ttype,
			Content: string(buffer[len(buffer)-int(tokenLen):]),
			Row: int32(row),
			Column: int32(column),
		},
	)
	previousRow = row
	previousColumn = column
	buffer = []rune{}
}

func TokenizeMdem(fileRunes []rune) ( []Token, error ) {
	tokens = []Token{} 
	buffer = []rune{} 

	for i := 0; i < len(fileRunes); i++ {
		c := fileRunes[i]

		if (c == '\n') {
			row += 1
			column = 1
		}
		buffer = append(buffer, c)
		if !textStarted {
			if c == '\n' {
				previousRow += 1
				previousColumn = 1
			} else if (c == ' ') {
				previousColumn += 1
			} else {
				textStarted = true 
			}
		}

		switch c {
		case '[':
			makePostTextToken(IdentifierStart, 1, TextFragment)
			previousRow = row
			previousColumn = column
			textStarted = false
		case ']':
			if (len(buffer) - 1 > 1) {
				textFragment := []rune{}
				trimmedStr := strings.Trim(string(buffer), " \n\t")
				for i := 0; i <  len(trimmedStr) - 1; i++ {
					element := trimmedStr[i]
					textFragment = append(textFragment, rune(element))
				}
				tokens = append(
					tokens,
					Token{
						TokenType: Identifier,
						Content: string(textFragment),
						Row: int32(previousRow),
						Column: int32(previousColumn),
					},
				)
			}
			tokens = append(
				tokens,
				Token{
					TokenType: IdentifierEnd,
					Content: "]",
					Row: int32(row),
					Column: int32(column),
				},
			)
			previousRow = row
			previousColumn = column
			textStarted = false
			buffer = []rune{}
		case '#':
			makePostTextToken(SectionIdentifierStart, 1, TextFragment)
			previousRow = row
			previousColumn = column
			textStarted = false
		case '{':
			makePostTextToken(SectionStart, 1, Identifier)
			previousRow = row
			previousColumn = column
			textStarted = false
		case '}':
			makePostTextToken(SectionEnd, 1, TextFragment)
			previousRow = row
			previousColumn = column
			textStarted = false
		case '-':
			makePostTextToken(ElementDashStart, 1, TextFragment)
			previousRow = row
			previousColumn = column
			textStarted = false
		case '>':
			makePostTextToken(QuestionEnd, 1, TextFragment)
			previousRow = row
			previousColumn = column
		case '+':
			makePostTextToken(ElementPlusStart, 1, TextFragment)
			previousRow = row
			previousColumn = column
			textStarted = false
		}
		column += 1
	}
	makePostTextToken(EOF, 0, TextFragment)
	return tokens, nil
}

func ToString (ttype *TokenType) string {
	switch *ttype {
	case TextFragment: return "TextFragment"
	case QuestionEnd: return "QuestionEnd"               
	case ElementDashStart: return "ElementDashStart"          
	case ElementPlusStart: return "ElementPlusStart"          
	case Identifier: return "Identifier"                
	case IdentifierStart: return "IdentifierStart"            
	case IdentifierEnd: return "IdentifierEnd"             
	case SectionIdentifierStart: return "SectionIdentifierStart"             
	case SectionStart: return "SectionStart"             
	case SectionEnd: return "SectionEnd"             
	case EOF: return "EndOfFile"             
	default: return "NOT_DEFINED"
	}
}