kristoferssolo/mdemory
1
0
Fork 0
mirror of https://github.com/jorenchik/mdemory.git synced 2026-03-22 00:26:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Lexer as a package

Browse Source
This commit is contained in:
jorenchik
2024-08-04 21:23:24 +03:00
parent eeb42b6a32
commit 1164a10846
4 changed files with 89 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/compiler/compiler.go
View File

@@ -4,6 +4,7 @@ import (
"fmt"
"log"
"os"
"github.com/jorenchik/mdemory/src/compiler/lexer"
)
func main() {
@@ -16,7 +17,7 @@ func main() {
}
fileContents := string(file)
tokens, err := tokenizeMdem([]rune(fileContents))
tokens, err := lexer.TokenizeMdem([]rune(fileContents))
if (err != nil) {
fmt.Printf("%s\n", err.Error())
return
@@ -29,7 +30,7 @@ func main() {
}
automata = parserAutomata()
err = validateGrammar()
err = validateGrammar(tokens)
if (err != nil) {
log.Fatal(err.Error())
}

54
src/compiler/lexer.go → src/compiler/lexer/lexer.go
View File

@@ -1,4 +1,4 @@
package main
package lexer
import (
"fmt"
@@ -30,15 +30,15 @@ const (
)
type Token struct {
tokenType TokenType;
content string;
row int32;
column int32;
TokenType TokenType;
Content string;
Row int32;
Column int32;
}
func (token Token)ToString() string {
content := token.content
if (token.tokenType == TextFragment) {
content := token.Content
if (token.TokenType == TextFragment) {
content = strings.Replace(
strings.Trim(content, " "),
"\n",
@@ -48,10 +48,10 @@ func (token Token)ToString() string {
}
return fmt.Sprintf(
"%s: \"%s\" %d:%d\n",
ToString(&token.tokenType),
ToString(&token.TokenType),
content,
token.row,
token.column,
token.Row,
token.Column,
)
}
@@ -67,20 +67,20 @@ func makePostTextToken(ttype TokenType, tokenLen int32, textType TokenType) {
tokens = append(
tokens,
Token{
tokenType: textType,
content: string(textFragment),
row: int32(previousRow),
column: int32(previousColumn),
TokenType: textType,
Content: string(textFragment),
Row: int32(previousRow),
Column: int32(previousColumn),
},
)
}
tokens = append(
tokens,
Token{
tokenType: ttype,
content: string(buffer[len(buffer)-int(tokenLen):]),
row: int32(row),
column: int32(column),
TokenType: ttype,
Content: string(buffer[len(buffer)-int(tokenLen):]),
Row: int32(row),
Column: int32(column),
},
)
previousRow = row
@@ -88,7 +88,7 @@ func makePostTextToken(ttype TokenType, tokenLen int32, textType TokenType) {
buffer = []rune{}
}
func tokenizeMdem(fileRunes []rune) ( []Token, error ) {
func TokenizeMdem(fileRunes []rune) ( []Token, error ) {
tokens = []Token{}
buffer = []rune{}
@@ -128,20 +128,20 @@ func tokenizeMdem(fileRunes []rune) ( []Token, error ) {
tokens = append(
tokens,
Token{
tokenType: Identifier,
content: string(textFragment),
row: int32(previousRow),
column: int32(previousColumn),
TokenType: Identifier,
Content: string(textFragment),
Row: int32(previousRow),
Column: int32(previousColumn),
},
)
}
tokens = append(
tokens,
Token{
tokenType: IdentifierEnd,
content: "]",
row: int32(row),
column: int32(column),
TokenType: IdentifierEnd,
Content: "]",
Row: int32(row),
Column: int32(column),
},
)
previousRow = row

105
src/compiler/parser.go
View File

@@ -4,6 +4,7 @@ import (
"fmt"
"strings"
"log"
"github.com/jorenchik/mdemory/src/compiler/lexer"
)
type Question interface {
@@ -60,7 +61,7 @@ type QuestionElement struct {
content string;
}
var automata map[TokenType][]TokenType
var automata map[lexer.TokenType][]lexer.TokenType
type CompilerErr struct {
message string;
@@ -72,7 +73,7 @@ func (e CompilerErr) Error() string {
return fmt.Sprintf("%d:%d - %s", e.row, e.column, e.message)
}
func contains(s []TokenType, e TokenType) bool {
func contains(s []lexer.TokenType, e lexer.TokenType) bool {
for _, a := range s {
if a == e {
return true
@@ -81,65 +82,71 @@ func contains(s []TokenType, e TokenType) bool {
return false
}
func parserAutomata() map[TokenType][]TokenType {
automata := make(map[TokenType][]TokenType)
automata[TextFragment] = []TokenType{
QuestionEnd, ElementDashStart, ElementPlusStart, SectionIdentifierStart, SectionStart, EOF, SectionEnd,
func parserAutomata() map[lexer.TokenType][]lexer.TokenType {
automata := make(map[lexer.TokenType][]lexer.TokenType)
automata[lexer.TextFragment] = []lexer.TokenType{
lexer.QuestionEnd,
lexer.ElementDashStart,
lexer.ElementPlusStart,
lexer.SectionIdentifierStart,
lexer.SectionStart,
lexer.EOF,
lexer.SectionEnd,
}
automata[QuestionEnd] = []TokenType{
ElementDashStart, ElementPlusStart,
automata[lexer.QuestionEnd] = []lexer.TokenType{
lexer.ElementDashStart, lexer.ElementPlusStart,
}
automata[ElementDashStart] = []TokenType{
IdentifierStart, TextFragment,
automata[lexer.ElementDashStart] = []lexer.TokenType{
lexer.IdentifierStart, lexer.TextFragment,
}
automata[ElementPlusStart] = []TokenType{
TextFragment,
automata[lexer.ElementPlusStart] = []lexer.TokenType{
lexer.TextFragment,
}
automata[Identifier] = []TokenType{
IdentifierEnd, SectionStart,
automata[lexer.Identifier] = []lexer.TokenType{
lexer.IdentifierEnd, lexer.SectionStart,
}
automata[IdentifierStart] = []TokenType{
Identifier,
automata[lexer.IdentifierStart] = []lexer.TokenType{
lexer.Identifier,
}
automata[IdentifierEnd] = []TokenType{
TextFragment,
automata[lexer.IdentifierEnd] = []lexer.TokenType{
lexer.TextFragment,
}
automata[SectionIdentifierStart] = []TokenType{
Identifier,
automata[lexer.SectionIdentifierStart] = []lexer.TokenType{
lexer.Identifier,
}
automata[SectionStart] = []TokenType{
ElementDashStart, SectionIdentifierStart, EOF,
automata[lexer.SectionStart] = []lexer.TokenType{
lexer.ElementDashStart, lexer.SectionIdentifierStart, lexer.EOF,
}
automata[SectionEnd] = []TokenType{
SectionIdentifierStart, ElementDashStart, EOF,
automata[lexer.SectionEnd] = []lexer.TokenType{
lexer.SectionIdentifierStart, lexer.ElementDashStart, lexer.EOF,
}
automata[SOF] = []TokenType{
ElementDashStart, SectionIdentifierStart, EOF,
automata[lexer.SOF] = []lexer.TokenType{
lexer.ElementDashStart, lexer.SectionIdentifierStart, lexer.EOF,
}
automata[EOF] = []TokenType{}
automata[lexer.EOF] = []lexer.TokenType{}
return automata
}
func validateGrammar() error {
func validateGrammar(tokens []lexer.Token) error {
for i := 0; i < len(tokens) - 1; i++ {
token := tokens[i]
nextToken := tokens[i + 1]
if (!contains(automata[token.tokenType], nextToken.tokenType)) {
if (!contains(automata[token.TokenType], nextToken.TokenType)) {
return CompilerErr{
message: fmt.Sprintf(
"Token %s cannot precede %s\n",
ToString(&token.tokenType),
ToString(&nextToken.tokenType),
lexer.ToString(&token.TokenType),
lexer.ToString(&nextToken.TokenType),
),
row: token.row,
column: token.column,
row: token.Row,
column: token.Column,
}
}
}
return nil
}
func ParseQuestions(tokens []Token) ([]Question, error) {
func ParseQuestions(tokens []lexer.Token) ([]Question, error) {
questions := []Question{}
section := ""
i := 0
@@ -147,25 +154,25 @@ func ParseQuestions(tokens []Token) ([]Question, error) {
if (i >= len(tokens)) {
break
}
if (tokens[i].tokenType == ElementDashStart) {
id := tokens[i + 2].content
question := tokens[i + 4].content
if (tokens[i].TokenType == lexer.ElementDashStart) {
id := tokens[i + 2].Content
question := tokens[i + 4].Content
quesitonElements := []QuestionElement{}
i += 6
for {
if (i + 1 >= len(tokens) ||
!(tokens[i].tokenType == ElementDashStart ||
tokens[i].tokenType == ElementPlusStart) ||
tokens[i+1].tokenType == IdentifierStart) {
!(tokens[i].TokenType == lexer.ElementDashStart ||
tokens[i].TokenType == lexer.ElementPlusStart) ||
tokens[i+1].TokenType == lexer.IdentifierStart) {
break
}
questionElement := QuestionElement{}
if (tokens[i].tokenType == ElementDashStart) {
if (tokens[i].TokenType == lexer.ElementDashStart) {
questionElement.isDash = true
} else {
questionElement.isDash = false
}
questionElement.content = tokens[i+1].content
questionElement.content = tokens[i+1].Content
quesitonElements = append(quesitonElements, questionElement)
i += 2
}
@@ -197,23 +204,23 @@ func ParseQuestions(tokens []Token) ([]Question, error) {
}
questions = append(questions, question)
}
} else if (tokens[i].tokenType == SectionIdentifierStart) {
section = tokens[i + 1].content
} else if (tokens[i].TokenType == lexer.SectionIdentifierStart) {
section = tokens[i + 1].Content
i += 3;
} else if (tokens[i].tokenType == SectionEnd) {
} else if (tokens[i].TokenType == lexer.SectionEnd) {
section = ""
i += 1
} else if (tokens[i].tokenType == EOF) {
} else if (tokens[i].TokenType == lexer.EOF) {
break
} else {
log.Fatalf(
"Not handled: %s",
ToString(&tokens[i].tokenType),
lexer.ToString(&tokens[i].TokenType),
)
return nil, CompilerErr{
message: "",
row: tokens[i].row,
column: tokens[i].column,
row: tokens[i].Row,
column: tokens[i].Column,
}
}
}

6
src/mdemory-app/go.mod
View File

@@ -1,8 +1,6 @@
module mdemory-app
go 1.21
toolchain go1.22.5
go 1.22.5
require github.com/wailsapp/wails/v2 v2.9.1
@@ -36,4 +34,6 @@ require (
golang.org/x/text v0.15.0 // indirect
)
replace github.com/jorenchik/mdemory/src/compiler => ../compiler
// replace github.com/wailsapp/wails/v2 v2.9.1 => /home/jorenchik/go/pkg/mod