#pragma once

#include <string>
#include <vector>

#include "result.h"

enum class TokenType {
    TextFragment,
    QuestionEnd,
	MatchGroupEnd,
    ElementDashStart,
    ElementPlusStart,
    ElementOrderModifier,
    Identifier,
    IdentifierStart,
    IdentifierEnd,
    SectionIdentifierStart,
    SectionStart,
    SectionEnd,
    SOF,
    EndOfFile
};

struct Token {
    TokenType tokenType;
    std::string content;
    int32_t row;
    int32_t column;

    std::string ToString() const;
    static std::string ToString(const TokenType* ttype);
};

Result<std::vector<Token>> TokenizeMdem(const std::string& fileRunes);
