From 586ad72e70bfe3e8e94c58846d342e6833b79219 Mon Sep 17 00:00:00 2001
From: jorenchik <jorens.stekels@gmail.com>
Date: Sun, 29 Sep 2024 11:32:11 +0300
Subject: [PATCH] text content cleaning happens in parsing & adjusted the
 approach

---
 src/cpp/include/stringUtils.h      |  2 ++
 src/cpp/qtapp/main.cpp             | 24 +-----------------------
 src/cpp/transpiler/CMakeLists.txt  |  1 +
 src/cpp/transpiler/parser.cpp      | 13 ++++++++-----
 src/cpp/transpiler/stringUtils.cpp | 25 +++++++++++++++++++++++++
 5 files changed, 37 insertions(+), 28 deletions(-)
 create mode 100644 src/cpp/include/stringUtils.h
 create mode 100644 src/cpp/transpiler/stringUtils.cpp

diff --git a/src/cpp/include/stringUtils.h b/src/cpp/include/stringUtils.h
new file mode 100644
index 0000000..d182ec9
--- /dev/null
+++ b/src/cpp/include/stringUtils.h
@@ -0,0 +1,2 @@
+#pragma once
+std::string cleanContent(std::string answer);
diff --git a/src/cpp/qtapp/main.cpp b/src/cpp/qtapp/main.cpp
index 1416c0a..173b063 100644
--- a/src/cpp/qtapp/main.cpp
+++ b/src/cpp/qtapp/main.cpp
@@ -10,7 +10,6 @@
 #include <qtoolbutton.h>
 #include <qwindow.h>
 #include <qwindowdefs.h>
-#include <regex>
 #include <string>
 
 #include <QApplication>
@@ -37,6 +36,7 @@
 #include "api.h"
 #include "parser.h"
 #include "trainWindow.h"
+#include "stringUtils.h"
 
 struct Page {
     int start;
@@ -158,21 +158,6 @@ QToolButton            *load;
 QToolButton            *practice;
 
 
-const std::regex doubleSpaceExp(
-	"  ",
-	std::regex_constants::ECMAScript | std::regex_constants::icase
-);
-
-const std::regex tabExp(
-	"\t",
-	std::regex_constants::ECMAScript | std::regex_constants::icase
-);
-
-const std::regex newLineExp(
-	"\n",
-	std::regex_constants::ECMAScript | std::regex_constants::icase
-);
-
 void CreateMdems(std::vector<Question*>& questions) {
 	hMdemScroll->removeItem(mdemSpacer);
 
@@ -191,12 +176,6 @@ void CreateMdems(std::vector<Question*>& questions) {
 		}
 	}
 
-	auto transformAnswer = [](std::string answer) -> std::string {
-		answer = std::regex_replace(answer, doubleSpaceExp, " ");
-		answer = std::regex_replace(answer, tabExp, "");
-		answer = std::regex_replace(answer, newLineExp, "");
-		return answer;
-	};
 
 	for (size_t i = 0; i < questions.size(); ++i) {
 		if (MultiElementQuestion* mw = dynamic_cast<MultiElementQuestion*>(questions[i])) {
@@ -206,7 +185,6 @@ void CreateMdems(std::vector<Question*>& questions) {
 			auto choices = mw->Choices;
 			for (size_t k = 0; k < choices.size(); ++k) {
 				auto answer = choices[k].Answer;
-				answer = transformAnswer(answer);
 				answer = std::format("- {}", answer); 
 				if (k < mdems[i]->backLabels.size()) {
 					mdems[i]->backLabels[k]->setText(QString::fromStdString(answer));
diff --git a/src/cpp/transpiler/CMakeLists.txt b/src/cpp/transpiler/CMakeLists.txt
index ca3fafb..cbe6f28 100644
--- a/src/cpp/transpiler/CMakeLists.txt
+++ b/src/cpp/transpiler/CMakeLists.txt
@@ -10,6 +10,7 @@ add_library(
     parser.cpp
 	time.cpp
 	api.cpp
+	stringUtils.cpp
 )
 
 add_executable(transpiler ${SOURCES})
diff --git a/src/cpp/transpiler/parser.cpp b/src/cpp/transpiler/parser.cpp
index 959beaf..69d82de 100644
--- a/src/cpp/transpiler/parser.cpp
+++ b/src/cpp/transpiler/parser.cpp
@@ -11,6 +11,7 @@
 #include "lexer.h"
 #include "result.h"
 #include "parser.h"
+#include "stringUtils.h"
 
 struct QuestionElement {
 	bool isDash;
@@ -317,19 +318,21 @@ Result<std::vector<Question*>> ParseQuestions(const std::vector<Token>& tokens)
 					auto *question = new GroupQuestion();
 					question->ID           = id;
 					question->QuestionText = questionText;
-					question->Section = section;
+					question->Section      = section;
 					int32_t k = -1;
 					for (size_t i = 0; i < questionElements.size(); ++i) {
 						auto questionElement = questionElements[i];
 						if (questionElement.isGroup) {
 							++k;
 							auto group = Group();
-							group.name = questionElement.content;
+							group.name = cleanContent(questionElement.content);
 							question->Groups.push_back(group);
 						} else {
 							if (k >= 0) {
 								question->Groups[k].elements.push_back(
-									questionElement.content
+									cleanContent(
+										questionElement.content
+									)
 								);
 							}
 						}
@@ -341,11 +344,11 @@ Result<std::vector<Question*>> ParseQuestions(const std::vector<Token>& tokens)
 				} else {
 					auto *question = new MultiElementQuestion();
 					question->ID = id;
-					question->QuestionText = questionText;
+					question->QuestionText = cleanContent(questionText);
 					question->Section = section;
 					for (const auto& elem : questionElements) {
 						Choice choice;
-						choice.Answer = elem.content;
+						choice.Answer = cleanContent(elem.content);
 						choice.IsCorrect = !elem.isDash;
 						question->Choices.push_back(choice);
 					}
diff --git a/src/cpp/transpiler/stringUtils.cpp b/src/cpp/transpiler/stringUtils.cpp
new file mode 100644
index 0000000..fe3493d
--- /dev/null
+++ b/src/cpp/transpiler/stringUtils.cpp
@@ -0,0 +1,25 @@
+#include <regex>
+
+#include "stringUtils.h"
+
+const std::regex doubleOrMoreSpaceExp(
+	"\\s\\s+",
+	std::regex_constants::ECMAScript | std::regex_constants::icase
+);
+
+const std::regex tabExp(
+	"\\t+",
+	std::regex_constants::ECMAScript | std::regex_constants::icase
+);
+
+const std::regex newLineExp(
+	"\\n+",
+	std::regex_constants::ECMAScript | std::regex_constants::icase
+);
+
+std::string cleanContent(std::string answer) {
+	answer = std::regex_replace(answer, newLineExp, "");
+	answer = std::regex_replace(answer, tabExp, " ");
+	answer = std::regex_replace(answer, doubleOrMoreSpaceExp, " ");
+	return answer;
+}