package main import ( "strings" ) var tokens []Token var buffer []rune var row int32 = 1 var column int32 = 1 var previousRow int32 = -1 var previousColumn int32 = -1 var textStarted bool = false type TokenType int const ( TextFragment TokenType = iota QuestionEnd ElementDashStart ElementPlusStart Identifier IdentifierStart IdentifierEnd SectionIdentifierStart SectionStart SectionEnd SOF EOF ) type Token struct { tokenType TokenType; content string; row int32; column int32; } func makePostTextToken(ttype TokenType, tokenLen int32, textType TokenType) { if (len(strings.Trim(string(buffer), " \n\t")) - 1 > 0) { textFragment := []rune{} for i := 0; i < len(buffer) - int(tokenLen); i++ { element := buffer[i] textFragment = append(textFragment, element) } tokens = append( tokens, Token{ tokenType: textType, content: string(textFragment), row: int32(previousRow), column: int32(previousColumn), }, ) } tokens = append( tokens, Token{ tokenType: ttype, content: string(buffer[len(buffer)-int(tokenLen):]), row: int32(row), column: int32(column), }, ) previousRow = row previousColumn = column buffer = []rune{} } func tokenize(fileRunes []rune) ( []Token, error ) { tokens = []Token{} buffer = []rune{} for i := 0; i < len(fileRunes); i++ { c := fileRunes[i] if (c == '\n') { row += 1 column = 1 } buffer = append(buffer, c) if !textStarted { if c == '\n' { previousRow += 1 previousColumn = 1 } else if (c == ' ') { previousColumn += 1 } else { textStarted = true } } switch c { case '[': makePostTextToken(IdentifierStart, 1, TextFragment) previousRow = row previousColumn = column textStarted = false case ']': if (len(buffer) - 1 > 1) { textFragment := []rune{} trimmedStr := strings.Trim(string(buffer), " \n\t") for i := 0; i < len(trimmedStr) - 1; i++ { element := trimmedStr[i] textFragment = append(textFragment, rune(element)) } tokens = append( tokens, Token{ tokenType: Identifier, content: string(textFragment), row: int32(previousRow), column: int32(previousColumn), }, ) } tokens = append( tokens, Token{ tokenType: IdentifierEnd, content: "]", row: int32(row), column: int32(column), }, ) previousRow = row previousColumn = column textStarted = false buffer = []rune{} case '#': makePostTextToken(SectionIdentifierStart, 1, TextFragment) previousRow = row previousColumn = column textStarted = false case '{': makePostTextToken(SectionStart, 1, Identifier) previousRow = row previousColumn = column textStarted = false case '}': makePostTextToken(SectionEnd, 1, TextFragment) previousRow = row previousColumn = column textStarted = false case '-': makePostTextToken(ElementDashStart, 1, TextFragment) previousRow = row previousColumn = column textStarted = false case '>': makePostTextToken(QuestionEnd, 1, TextFragment) previousRow = row previousColumn = column case '+': makePostTextToken(ElementPlusStart, 1, TextFragment) previousRow = row previousColumn = column textStarted = false } column += 1 } makePostTextToken(EOF, 0, TextFragment) return tokens, nil } func ToString (ttype *TokenType) string { switch *ttype { case TextFragment: return "TextFragment" case QuestionEnd: return "QuestionEnd" case ElementDashStart: return "ElementDashStart" case ElementPlusStart: return "ElementPlusStart" case Identifier: return "Identifier" case IdentifierStart: return "IdentifierStart" case IdentifierEnd: return "IdentifierEnd" case SectionIdentifierStart: return "SectionIdentifierStart" case SectionStart: return "SectionStart" case SectionEnd: return "SectionEnd" case EOF: return "EndOfFile" default: return "NOT_DEFINED" } }