Finally solved this myself after making many styles of parsers suffering from the same bug... Working (cleaner but still needs more cleaning) code Here
if(input > 47 & input < 123) should be if(input > 44 & input < 123) to include the '.' thanks for no help.
Modifying This example code almost works for me, and is a starting point to learn more.
I've been trying to parse some files from the game Europa Universalis IV. The files contain text like this:
In case someone wants to help debug Here is a browser version with 2 processed OuterEntities as example.
Once everything works the data would be put into a database after every object is finished. For testing I just write my progress to a text file.
I have tried this a couple times in different ways that almost always end up looking the same. By now the process is split into parts, and most of them work.
I start by collecting all the "OuterEntities" (First level brackets) in each of the GameFiles (just one for fast testing), this step is easy. I put those outer entities into a list in a GameFile object (ToString overrides are just for testing) :
GameFile
internal record GameFile
{
public List<string> UnprocessedOuterEntites = new();
public string Name;
public string Path;
public List<OuterEntity> OuterEntities = new();
public override string ToString()
{
StringBuilder sb = new();
sb.AppendLine(Name);
sb.AppendLine(Path);
foreach(OuterEntity entity in OuterEntities)
{
sb.AppendLine(entity.ToString());
}
foreach(string entity in UnprocessedOuterEntites)
{
sb.AppendLine(entity);
}
return sb.ToString();
}
OuterEntity
internal class OuterEntity
{
public List<string> UnprocessedInnerEntities = new();
public string? Name = "";
public List<InnerEntity> InnerEntities = new();
public void Clear()
{
this.UnprocessedInnerEntities.Clear();
this.InnerEntities.Clear();
this.Name = null;
}
public override string ToString()
{
StringBuilder sb = new();
sb.Append(Name);
foreach (InnerEntity entity in InnerEntities)
{
sb.Append(entity.ToString());
}
foreach(string entity in UnprocessedInnerEntities)
{
sb.Append(entity);
}
return sb.ToString();
}
}
Those OuterEntities contain InnerEntities, which contain values or more InnerEntities (Nested brackets) :
Inner Entity
internal class InnerEntity
{
public List<string> UnprocessedInnerEntities = new();
public string? Name;
public string? Value;
public List<InnerEntity> InnerEntities = new();
public void Clear()
{
this.Name = null;
this.Value = null;
this.InnerEntities = new();
this.UnprocessedInnerEntities = new();
}
public override string ToString()
{
StringBuilder sb = new StringBuilder();
sb.AppendLine(Name);
if (InnerEntities.Count > 0)
{
foreach (InnerEntity entity in InnerEntities)
{
sb.Append(entity.ToString());
}
}
else
{
sb.AppendLine(Value);
}
foreach (string String in UnprocessedInnerEntities)
{
sb.AppendLine(String);
}
return sb.ToString();
}
The program is (for testing) run like this:
internal class Program{
private static Program program = new();
public List<GameFile> gameFiles = new();
public static void Main()
{
program.Run();
}
private void Run()
{
string[] paths = Directory.GetFiles("Source", "*.txt", SearchOption.AllDirectories);
foreach (string path in paths)
{
Process.GetOuterEntities(path,this);
}
Process.ProcessOuterEntities(this);
Process.ProcessInnerEntities(this);
Test.PrintOuterEntities(this);
}}
The OuterEntities are collected and processed:
public static void GetOuterEntities(string path, Program program)
{
string[] lines = System.IO.File.ReadAllLines(path);
GameFile file = new GameFile() { Path = path };
StringBuilder sb = new StringBuilder();
int currentPos = 0;
int lastPos = 0;
int brackets = 0;
bool inBrackets = false;
string st = RemoveComments(lines);
OuterEntity entity = new OuterEntity();
foreach (char c in st)
{
if (c == '{')
{
inBrackets = true;
brackets++;
}
else if (c == '}')
{
brackets--;
}
if (brackets == 0 && inBrackets)//End Outer
{
sb.Clear();
inBrackets = false;
for (int i = lastPos; i <= currentPos; i++)
{
if (st[i] != '\n')
{
sb.Append(st[i]);
}
}
file.UnprocessedOuterEntites.Add(sb.ToString());
program.gameFiles.Add(file);
lastPos = currentPos + 1;
}
currentPos++;
}
public static void ProcessOuterEntities(Program program)
{
foreach (GameFile file in program.gameFiles)
{
foreach (string UnprocessedOuterEntity in file.UnprocessedOuterEntites)
{
file.OuterEntities.Add(ProcessOuterEntity(UnprocessedOuterEntity));
}
file.UnprocessedOuterEntites.Clear();
}
}
private static OuterEntity ProcessOuterEntity(string UnprocessedOuterEntity)
{
OuterEntity processedOuterEntity = new();
int currentPos = 0;
int lastPos = 0;
int brackets = 0;
bool inBrackets = false;
StringBuilder sb = new StringBuilder();
foreach (char c in UnprocessedOuterEntity)
{
if (c == '{')
{
if (processedOuterEntity.Name == "")
{
for (int i = lastPos; i <= currentPos; i++)
{
if (UnprocessedOuterEntity[i] != '\n')
{
sb.Append(UnprocessedOuterEntity[i]);
}
}
processedOuterEntity.Name = sb.ToString();
sb.Clear();
lastPos = currentPos + 1;
}
inBrackets = true;
brackets++;
}
else if (c == '}')
{
brackets--;
}
if (brackets == 0 && inBrackets)
{
inBrackets = false;
for (int i = lastPos; i <= currentPos; i++)
{
sb.Append(UnprocessedOuterEntity[i]);
}
processedOuterEntity.UnprocessedInnerEntities.Add(sb.ToString());
sb.Clear();
lastPos = currentPos + 1;
}
currentPos++;
}
return processedOuterEntity;
}
After this I fail to process the inner entities.
This is the currently not working part:
private static InnerEntity ProcessInnerEntity(string UnprocessedInnerEntity, bool WaitingForValue = false)
{
InnerEntity processedInnerEntity = new();
int currentPos = 0;
int lastPos = 0;
int brackets = 0;
bool inBrackets = false;
bool waitingForValue = WaitingForValue;
StringBuilder sb = new StringBuilder();
foreach (char c in UnprocessedInnerEntity)
{
if (c == '{')//Brackets opening
{
inBrackets = true;
brackets++;
}
else if (c == '}')//Brackets closing
{
brackets--;
}
else if (c == '=')//Name behind, Value or NestedEntity in front
{
for (int i = lastPos; i <= currentPos; i++)
{
sb.Append(UnprocessedInnerEntity[i]);
}
processedInnerEntity.Name = sb.ToString();
sb.Clear();
lastPos = currentPos + 1;
waitingForValue = true;
if (inBrackets)//In a nested bracket, find InnerEntity
{
processedInnerEntity.InnerEntities.Add(ProcessInnerEntity(UnprocessedInnerEntity.Substring(currentPos + 1), waitingForValue));//Search everything in front
}
}
else if (c<47 | c > 123)//Not text
{
if (waitingForValue)//After =
{
for (int i = lastPos; i <= currentPos; i++)
{
if (c > 47 & c < 123)//Text
{
waitingForValue = false;
sb.Append(UnprocessedInnerEntity[i]);
}
}
if (!waitingForValue)//Has value
{
processedInnerEntity.Value = sb.ToString();
sb.Clear();
}
}
}
if (brackets == 0 && inBrackets)//Brackets closed
{
inBrackets = false;
currentPos++;
break;
}
currentPos++;
}
return processedInnerEntity;
}
The rest of the code can be found here
The debug file is written like this
internal class Test
{
public static void PrintOuterEntities(Program program)
{
List<string> strings = new();
foreach (GameFile gameFile in program.gameFiles)
{
strings.Add(gameFile.ToString());
}
foreach (string String in strings)
{
File.WriteAllText("test.txt", String);
}
}
}
Current output looks like this:
source\common\ideas\00_country_ideas.txt
HLR_ideas = { 1 diplomatic_reputation =
2 } bonus =
2 } bonus =
GER_ideas = { 0.3 infantry_power =
0.15 } bonus =
0.15 } bonus =
0.15 } bonus =
0.15 } bonus =
I want a list of OuterEntities containing their InnerEntities like this:
OuterEntity //Object
Name HLR_Ideas
InnerEntities //List
Name start
InnerEntities
Name possible_policy
Value 1
Name diplomatic_reputation
Value 2
Name bonus
InnerEntities
Name administrative_efficiency
Value 0.05
Name trigger
InnerEntities
Name tag
Value HLR
Name free
Value yes
Name hlr_imperial_throne
InnerEntities
Name legitimacy
Value 1.5
Name hlr_kaiserliche_armee
InnerEntities
Name land_morale
Value 0.15
Name hlr_imperial_diplomacy
InnerEntities
Name improve_relation_modifier
Value 0.33
Name hlr_centralised_empire
InnerEntities
Name global_tax_modifier
Value 0.2
Name hlr_roman_heritage
InnerEntities
Name core_creation
Value -0.2
Name hlr_adopting_the_goosestep
InnerEntities
Name discipline
Value 0.05
Name hlr_onwards_and_upwards
InnerEntities
Name governing_capacity_modifier
Value 0.1
Is there something obvious I'm doing wrong in the last part? Are there existing wheels I don't know about I'm trying to reinvent? Do I just learn Perl or the fancy non regular regular expressions I see out there?