kristoferssolo/mdemory
1
0
Fork 0
mirror of https://github.com/jorenchik/mdemory.git synced 2026-03-22 00:26:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

lexer refactoring

Browse Source
This commit is contained in:
jorenchik
2024-09-06 19:56:52 +03:00
parent 1fdbff42f2
commit 1a2c961807
1 changed files with 43 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
src/compiler/lexer/lexer.go
View File

@@ -13,7 +13,6 @@ var previousColumn int32 = -1
var textStarted bool = false var textStarted bool = false
type TokenType int type TokenType int
const ( const (
TextFragment TokenType = iota TextFragment TokenType = iota
QuestionEnd QuestionEnd
@@ -36,7 +35,7 @@ type Token struct {
Column int32; Column int32;
} }
func (token Token)ToString() string { func (token Token) ToString() string {
content := token.Content content := token.Content
if (token.TokenType == TextFragment) { if (token.TokenType == TextFragment) {
content = strings.Replace( content = strings.Replace(
@@ -57,7 +56,11 @@ func (token Token)ToString() string {
var tokens []Token var tokens []Token
func makePostTextToken(ttype TokenType, tokenLen int32, textType TokenType) { func makeTokenWithTokenBuffer(
ttype TokenType,
tokenLen int32,
textType TokenType,
) {
if (len(strings.Trim(string(buffer), " \n\t")) - 1 > 0) { if (len(strings.Trim(string(buffer), " \n\t")) - 1 > 0) {
textFragment := []rune{} textFragment := []rune{}
for i := 0; i < len(buffer) - int(tokenLen); i++ { for i := 0; i < len(buffer) - int(tokenLen); i++ {
@@ -95,11 +98,14 @@ func TokenizeMdem(fileRunes []rune) ( []Token, error ) {
for i := 0; i < len(fileRunes); i++ { for i := 0; i < len(fileRunes); i++ {
c := fileRunes[i] c := fileRunes[i]
// AdvancePointer
if (c == '\n') { if (c == '\n') {
row += 1 row += 1
column = 1 column = 1
} }
buffer = append(buffer, c) buffer = append(buffer, c)
// SkipWhitetext
if !textStarted { if !textStarted {
if c == '\n' { if c == '\n' {
previousRow += 1 previousRow += 1
@@ -111,76 +117,53 @@ func TokenizeMdem(fileRunes []rune) ( []Token, error ) {
} }
} }
// EmitTokens
switch c { switch c {
case '[': case '[':
makePostTextToken(IdentifierStart, 1, TextFragment) makeTokenWithTokenBuffer(IdentifierStart, 1, TextFragment)
previousRow = row previousRow = row
previousColumn = column previousColumn = column
textStarted = false textStarted = false
case ']': case ']':
if (len(buffer) - 1 > 1) { makeTokenWithTokenBuffer(IdentifierEnd, 1, Identifier)
textFragment := []rune{}
trimmedStr := strings.Trim(string(buffer), " \n\t")
for i := 0; i < len(trimmedStr) - 1; i++ {
element := trimmedStr[i]
textFragment = append(textFragment, rune(element))
}
tokens = append(
tokens,
Token{
TokenType: Identifier,
Content: string(textFragment),
Row: int32(previousRow),
Column: int32(previousColumn),
},
)
}
tokens = append(
tokens,
Token{
TokenType: IdentifierEnd,
Content: "]",
Row: int32(row),
Column: int32(column),
},
)
previousRow = row previousRow = row
previousColumn = column previousColumn = column
textStarted = false textStarted = false
buffer = []rune{}
case '#': case '#':
makePostTextToken(SectionIdentifierStart, 1, TextFragment) makeTokenWithTokenBuffer(SectionIdentifierStart, 1, TextFragment)
previousRow = row previousRow = row
previousColumn = column previousColumn = column
textStarted = false textStarted = false
case '{': case '{':
makePostTextToken(SectionStart, 1, Identifier) makeTokenWithTokenBuffer(SectionStart, 1, Identifier)
previousRow = row previousRow = row
previousColumn = column previousColumn = column
textStarted = false textStarted = false
case '}': case '}':
makePostTextToken(SectionEnd, 1, TextFragment) makeTokenWithTokenBuffer(SectionEnd, 1, TextFragment)
previousRow = row previousRow = row
previousColumn = column previousColumn = column
textStarted = false textStarted = false
case '-': case '-':
makePostTextToken(ElementDashStart, 1, TextFragment) makeTokenWithTokenBuffer(ElementDashStart, 1, TextFragment)
previousRow = row previousRow = row
previousColumn = column previousColumn = column
textStarted = false textStarted = false
case '>': case '>':
makePostTextToken(QuestionEnd, 1, TextFragment) makeTokenWithTokenBuffer(QuestionEnd, 1, TextFragment)
previousRow = row previousRow = row
previousColumn = column previousColumn = column
case '+': case '+':
makePostTextToken(ElementPlusStart, 1, TextFragment) makeTokenWithTokenBuffer(ElementPlusStart, 1, TextFragment)
previousRow = row previousRow = row
previousColumn = column previousColumn = column
textStarted = false textStarted = false
} }
column += 1 column += 1
} }
makePostTextToken(EOF, 0, TextFragment)
// EmitEOF
makeTokenWithTokenBuffer(EOF, 0, TextFragment)
return tokens, nil return tokens, nil
} }
@@ -197,6 +180,6 @@ func ToString (ttype *TokenType) string {
case SectionStart: return "SectionStart" case SectionStart: return "SectionStart"
case SectionEnd: return "SectionEnd" case SectionEnd: return "SectionEnd"
case EOF: return "EndOfFile" case EOF: return "EndOfFile"
default: return "NOT_DEFINED" default: return "NOT_RECOGNIZED"
} }
} }