diff --git a/src/compiler/compiler.go b/src/compiler/compiler.go index 5ca3dba..9e46bb0 100644 --- a/src/compiler/compiler.go +++ b/src/compiler/compiler.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "os" + "github.com/jorenchik/mdemory/src/compiler/lexer" ) func main() { @@ -16,7 +17,7 @@ func main() { } fileContents := string(file) - tokens, err := tokenizeMdem([]rune(fileContents)) + tokens, err := lexer.TokenizeMdem([]rune(fileContents)) if (err != nil) { fmt.Printf("%s\n", err.Error()) return @@ -29,7 +30,7 @@ func main() { } automata = parserAutomata() - err = validateGrammar() + err = validateGrammar(tokens) if (err != nil) { log.Fatal(err.Error()) } diff --git a/src/compiler/lexer.go b/src/compiler/lexer/lexer.go similarity index 82% rename from src/compiler/lexer.go rename to src/compiler/lexer/lexer.go index 8932cb0..a4f5070 100644 --- a/src/compiler/lexer.go +++ b/src/compiler/lexer/lexer.go @@ -1,4 +1,4 @@ -package main +package lexer import ( "fmt" @@ -30,15 +30,15 @@ const ( ) type Token struct { - tokenType TokenType; - content string; - row int32; - column int32; + TokenType TokenType; + Content string; + Row int32; + Column int32; } func (token Token)ToString() string { - content := token.content - if (token.tokenType == TextFragment) { + content := token.Content + if (token.TokenType == TextFragment) { content = strings.Replace( strings.Trim(content, " "), "\n", @@ -48,10 +48,10 @@ func (token Token)ToString() string { } return fmt.Sprintf( "%s: \"%s\" %d:%d\n", - ToString(&token.tokenType), + ToString(&token.TokenType), content, - token.row, - token.column, + token.Row, + token.Column, ) } @@ -67,20 +67,20 @@ func makePostTextToken(ttype TokenType, tokenLen int32, textType TokenType) { tokens = append( tokens, Token{ - tokenType: textType, - content: string(textFragment), - row: int32(previousRow), - column: int32(previousColumn), + TokenType: textType, + Content: string(textFragment), + Row: int32(previousRow), + Column: int32(previousColumn), }, ) } tokens = append( tokens, Token{ - tokenType: ttype, - content: string(buffer[len(buffer)-int(tokenLen):]), - row: int32(row), - column: int32(column), + TokenType: ttype, + Content: string(buffer[len(buffer)-int(tokenLen):]), + Row: int32(row), + Column: int32(column), }, ) previousRow = row @@ -88,7 +88,7 @@ func makePostTextToken(ttype TokenType, tokenLen int32, textType TokenType) { buffer = []rune{} } -func tokenizeMdem(fileRunes []rune) ( []Token, error ) { +func TokenizeMdem(fileRunes []rune) ( []Token, error ) { tokens = []Token{} buffer = []rune{} @@ -128,20 +128,20 @@ func tokenizeMdem(fileRunes []rune) ( []Token, error ) { tokens = append( tokens, Token{ - tokenType: Identifier, - content: string(textFragment), - row: int32(previousRow), - column: int32(previousColumn), + TokenType: Identifier, + Content: string(textFragment), + Row: int32(previousRow), + Column: int32(previousColumn), }, ) } tokens = append( tokens, Token{ - tokenType: IdentifierEnd, - content: "]", - row: int32(row), - column: int32(column), + TokenType: IdentifierEnd, + Content: "]", + Row: int32(row), + Column: int32(column), }, ) previousRow = row diff --git a/src/compiler/parser.go b/src/compiler/parser.go index 3c986bc..0ed4144 100644 --- a/src/compiler/parser.go +++ b/src/compiler/parser.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" "log" + "github.com/jorenchik/mdemory/src/compiler/lexer" ) type Question interface { @@ -60,7 +61,7 @@ type QuestionElement struct { content string; } -var automata map[TokenType][]TokenType +var automata map[lexer.TokenType][]lexer.TokenType type CompilerErr struct { message string; @@ -72,7 +73,7 @@ func (e CompilerErr) Error() string { return fmt.Sprintf("%d:%d - %s", e.row, e.column, e.message) } -func contains(s []TokenType, e TokenType) bool { +func contains(s []lexer.TokenType, e lexer.TokenType) bool { for _, a := range s { if a == e { return true @@ -81,65 +82,71 @@ func contains(s []TokenType, e TokenType) bool { return false } -func parserAutomata() map[TokenType][]TokenType { - automata := make(map[TokenType][]TokenType) - automata[TextFragment] = []TokenType{ - QuestionEnd, ElementDashStart, ElementPlusStart, SectionIdentifierStart, SectionStart, EOF, SectionEnd, +func parserAutomata() map[lexer.TokenType][]lexer.TokenType { + automata := make(map[lexer.TokenType][]lexer.TokenType) + automata[lexer.TextFragment] = []lexer.TokenType{ + lexer.QuestionEnd, + lexer.ElementDashStart, + lexer.ElementPlusStart, + lexer.SectionIdentifierStart, + lexer.SectionStart, + lexer.EOF, + lexer.SectionEnd, } - automata[QuestionEnd] = []TokenType{ - ElementDashStart, ElementPlusStart, + automata[lexer.QuestionEnd] = []lexer.TokenType{ + lexer.ElementDashStart, lexer.ElementPlusStart, } - automata[ElementDashStart] = []TokenType{ - IdentifierStart, TextFragment, + automata[lexer.ElementDashStart] = []lexer.TokenType{ + lexer.IdentifierStart, lexer.TextFragment, } - automata[ElementPlusStart] = []TokenType{ - TextFragment, + automata[lexer.ElementPlusStart] = []lexer.TokenType{ + lexer.TextFragment, } - automata[Identifier] = []TokenType{ - IdentifierEnd, SectionStart, + automata[lexer.Identifier] = []lexer.TokenType{ + lexer.IdentifierEnd, lexer.SectionStart, } - automata[IdentifierStart] = []TokenType{ - Identifier, + automata[lexer.IdentifierStart] = []lexer.TokenType{ + lexer.Identifier, } - automata[IdentifierEnd] = []TokenType{ - TextFragment, + automata[lexer.IdentifierEnd] = []lexer.TokenType{ + lexer.TextFragment, } - automata[SectionIdentifierStart] = []TokenType{ - Identifier, + automata[lexer.SectionIdentifierStart] = []lexer.TokenType{ + lexer.Identifier, } - automata[SectionStart] = []TokenType{ - ElementDashStart, SectionIdentifierStart, EOF, + automata[lexer.SectionStart] = []lexer.TokenType{ + lexer.ElementDashStart, lexer.SectionIdentifierStart, lexer.EOF, } - automata[SectionEnd] = []TokenType{ - SectionIdentifierStart, ElementDashStart, EOF, + automata[lexer.SectionEnd] = []lexer.TokenType{ + lexer.SectionIdentifierStart, lexer.ElementDashStart, lexer.EOF, } - automata[SOF] = []TokenType{ - ElementDashStart, SectionIdentifierStart, EOF, + automata[lexer.SOF] = []lexer.TokenType{ + lexer.ElementDashStart, lexer.SectionIdentifierStart, lexer.EOF, } - automata[EOF] = []TokenType{} + automata[lexer.EOF] = []lexer.TokenType{} return automata } -func validateGrammar() error { +func validateGrammar(tokens []lexer.Token) error { for i := 0; i < len(tokens) - 1; i++ { token := tokens[i] nextToken := tokens[i + 1] - if (!contains(automata[token.tokenType], nextToken.tokenType)) { + if (!contains(automata[token.TokenType], nextToken.TokenType)) { return CompilerErr{ message: fmt.Sprintf( "Token %s cannot precede %s\n", - ToString(&token.tokenType), - ToString(&nextToken.tokenType), + lexer.ToString(&token.TokenType), + lexer.ToString(&nextToken.TokenType), ), - row: token.row, - column: token.column, + row: token.Row, + column: token.Column, } } } return nil } -func ParseQuestions(tokens []Token) ([]Question, error) { +func ParseQuestions(tokens []lexer.Token) ([]Question, error) { questions := []Question{} section := "" i := 0 @@ -147,25 +154,25 @@ func ParseQuestions(tokens []Token) ([]Question, error) { if (i >= len(tokens)) { break } - if (tokens[i].tokenType == ElementDashStart) { - id := tokens[i + 2].content - question := tokens[i + 4].content + if (tokens[i].TokenType == lexer.ElementDashStart) { + id := tokens[i + 2].Content + question := tokens[i + 4].Content quesitonElements := []QuestionElement{} i += 6 for { if (i + 1 >= len(tokens) || - !(tokens[i].tokenType == ElementDashStart || - tokens[i].tokenType == ElementPlusStart) || - tokens[i+1].tokenType == IdentifierStart) { + !(tokens[i].TokenType == lexer.ElementDashStart || + tokens[i].TokenType == lexer.ElementPlusStart) || + tokens[i+1].TokenType == lexer.IdentifierStart) { break } questionElement := QuestionElement{} - if (tokens[i].tokenType == ElementDashStart) { + if (tokens[i].TokenType == lexer.ElementDashStart) { questionElement.isDash = true } else { questionElement.isDash = false } - questionElement.content = tokens[i+1].content + questionElement.content = tokens[i+1].Content quesitonElements = append(quesitonElements, questionElement) i += 2 } @@ -197,23 +204,23 @@ func ParseQuestions(tokens []Token) ([]Question, error) { } questions = append(questions, question) } - } else if (tokens[i].tokenType == SectionIdentifierStart) { - section = tokens[i + 1].content + } else if (tokens[i].TokenType == lexer.SectionIdentifierStart) { + section = tokens[i + 1].Content i += 3; - } else if (tokens[i].tokenType == SectionEnd) { + } else if (tokens[i].TokenType == lexer.SectionEnd) { section = "" i += 1 - } else if (tokens[i].tokenType == EOF) { + } else if (tokens[i].TokenType == lexer.EOF) { break } else { log.Fatalf( "Not handled: %s", - ToString(&tokens[i].tokenType), + lexer.ToString(&tokens[i].TokenType), ) return nil, CompilerErr{ message: "", - row: tokens[i].row, - column: tokens[i].column, + row: tokens[i].Row, + column: tokens[i].Column, } } } diff --git a/src/mdemory-app/go.mod b/src/mdemory-app/go.mod index 070cbe7..09b32f0 100644 --- a/src/mdemory-app/go.mod +++ b/src/mdemory-app/go.mod @@ -1,8 +1,6 @@ module mdemory-app -go 1.21 - -toolchain go1.22.5 +go 1.22.5 require github.com/wailsapp/wails/v2 v2.9.1 @@ -36,4 +34,6 @@ require ( golang.org/x/text v0.15.0 // indirect ) +replace github.com/jorenchik/mdemory/src/compiler => ../compiler + // replace github.com/wailsapp/wails/v2 v2.9.1 => /home/jorenchik/go/pkg/mod