Implement proper parser for NEI

ArcToolkitCLI/Commands/Converters/NeiConverter.cs

@@ -10,12 +10,17 @@ using CommandLine;
 namespace ArcToolkitCLI.Commands.Converters
 {
     [Verb("nei", HelpText = "Convert NEI files into CSV and back")]
-    class NeiConverter : IConverterCommand, IInputOptions, IOutputOptions
+    internal class NeiConverter : IConverterCommand, IInputOptions, IOutputOptions
     {
-        private static readonly byte[] NEI_KEY = { 0xAA, 0xC9, 0xD2, 0x35, 0x22, 0x87, 0x20, 0xF2, 0x40, 0xC5, 0x61, 0x7C, 0x01, 0xDF, 0x66, 0x54 };
-        private static readonly byte[] NEI_MAGIC = { 0x77, 0x73, 0x76, 0xFF };
+        private static readonly byte[] NEI_KEY =
+            {0xAA, 0xC9, 0xD2, 0x35, 0x22, 0x87, 0x20, 0xF2, 0x40, 0xC5, 0x61, 0x7C, 0x01, 0xDF, 0x66, 0x54};
+
+        private static readonly byte[] NEI_MAGIC = {0x77, 0x73, 0x76, 0xFF};
         private static readonly Encoding ShiftJisEncoding = Encoding.GetEncoding(932);
 
+        [Option('s', "separator", Default = ';', HelpText = "Value separator of the CSV file")]
+        public char ValueSeparator { get; set; }
+
 
         public int Run()
         {
@@ -29,7 +34,7 @@ namespace ArcToolkitCLI.Commands.Converters
 
             foreach (var file in files)
             {
-                int result = 0;
+                var result = 0;
                 switch (file.Extension.ToLowerInvariant())
                 {
                     case ".nei":
@@ -49,41 +54,124 @@ namespace ArcToolkitCLI.Commands.Converters
             return 0;
         }
 
-        int ToNei(string filePath)
+        [Value(0, HelpText = "Input NEI or CSV files")]
+        public IEnumerable<string> Input { get; set; }
+
+        public string Output { get; set; }
+
+        private List<List<string>> ParseCSV(TextReader tr)
         {
-            string nameNoExt = Path.GetFileNameWithoutExtension(filePath);
+            var buffer = new StringBuilder();
+            var whitespaceBuffer = new StringBuilder();
+            var isQuoted = false;
+            var quoteLevel = 0;
+            var result = new List<List<string>>();
+            var line = new List<string>();
+
+            int nextChar;
+            while ((nextChar = tr.Read()) != -1)
+            {
+                var c = (char) nextChar;
 
-            List<string[]> values = new List<string[]>();
+                if (c == '\n' && !isQuoted)
+                {
+                    if (buffer.Length != 0)
+                    {
+                        line.Add(buffer.ToString());
+                        buffer.Clear();
+                    }
 
-            int cols = 0;
-            int rows = 0;
+                    if (line.Count != 0)
+                        result.Add(line);
 
-            using (var tr = File.OpenText(filePath))
-            {
-                string line;
+                    line = new List<string>();
+                    whitespaceBuffer.Clear();
+                    continue;
+                }
+
+                var isWhitespace = char.IsWhiteSpace(c);
+                var shouldSeparate = c == ValueSeparator && (!isQuoted || quoteLevel % 2 == 0);
+
+                if (isWhitespace)
+                {
+                    whitespaceBuffer.Append(c);
+                    continue;
+                }
+
+                if (whitespaceBuffer.Length != 0)
+                {
+                    if (buffer.Length > 0 && !shouldSeparate)
+                        buffer.Append(whitespaceBuffer);
+                    whitespaceBuffer.Clear();
+                }
+
+                if (shouldSeparate)
+                {
+                    line.Add(buffer.ToString());
+                    buffer.Clear();
+                    quoteLevel = 0;
+                    isQuoted = false;
+                }
+                else if (c == '"')
+                {
+                    if (buffer.Length == 0 && quoteLevel == 0)
+                    {
+                        isQuoted = true;
+                        quoteLevel++;
+                        whitespaceBuffer.Clear();
+                        continue;
+                    }
 
-                while ((line = tr.ReadLine()) != null)
+                    if (isQuoted)
+                        quoteLevel++;
+
+                    if (!isQuoted || quoteLevel % 2 == 1)
+                        buffer.Append(c);
+                }
+                else
                 {
-                    var colValues = line.Split(new []{ValueSeparator}, StringSplitOptions.None);
-                    if(colValues.Length == 0)
+                    if (isQuoted && quoteLevel != 0 && quoteLevel % 2 == 0)
+                    {
+                        line.Clear();
+                        tr.ReadLine();
                         continue;
-                    cols = Math.Max(cols, colValues.Length);
-                    values.Add(colValues);
-                    rows++;
+                    }
+
+                    buffer.Append(c);
                 }
             }
 
-            byte[][] encodedValues = new byte[rows * cols][];
+            if (buffer.Length != 0)
+                line.Add(buffer.ToString());
+
+            if (line.Count != 0)
+                result.Add(line);
+
+            return result;
+        }
+
+        private int ToNei(string filePath)
+        {
+            var nameNoExt = Path.GetFileNameWithoutExtension(filePath);
+
+            List<List<string>> values;
+            using (var tr = File.OpenText(filePath))
+            {
+                values = ParseCSV(tr);
+            }
+
+            var cols = values.Max(l => l.Count);
+            var rows = values.Count;
+
+            var encodedValues = new byte[rows * cols][];
 
             for (var rowIndex = 0; rowIndex < values.Count; rowIndex++)
             {
                 var row = values[rowIndex];
-                for (int colIndex = 0; colIndex < cols; colIndex++)
-                {
-                    encodedValues[colIndex + rowIndex * cols] = colIndex < row.Length
+                for (var colIndex = 0; colIndex < cols; colIndex++)
+                    encodedValues[colIndex + rowIndex * cols] = colIndex < row.Count
                         ? ShiftJisEncoding.GetBytes(row[colIndex])
                         : new byte[0];
-                }
             }
 
             using (var ms = new MemoryStream())
@@ -94,7 +182,7 @@ namespace ArcToolkitCLI.Commands.Converters
                     bw.Write(cols);
                     bw.Write(rows);
 
-                    int totalLength = 0;
+                    var totalLength = 0;
                     foreach (var encodedValue in encodedValues)
                     {
                         var len = encodedValue.Length;
@@ -109,7 +197,7 @@ namespace ArcToolkitCLI.Commands.Converters
                     for (var i = 0; i < encodedValues.Length; i++)
                     {
                         var encodedValue = encodedValues[i];
-                        if(encodedValue.Length == 0)
+                        if (encodedValue.Length == 0)
                             continue;
 
                         bw.Write(encodedValue);
@@ -125,11 +213,18 @@ namespace ArcToolkitCLI.Commands.Converters
             return 0;
         }
 
-        int ToCSV(string filePath)
+        private string Escape(string value)
+        {
+            if (!value.Contains(ValueSeparator) && !value.Contains('\n'))
+                return value;
+            return $"\"{value.Replace("\"", "\"\"")}\"";
+        }
+
+        private int ToCSV(string filePath)
         {
-            string nameNoExt = Path.GetFileNameWithoutExtension(filePath);
+            var nameNoExt = Path.GetFileNameWithoutExtension(filePath);
 
-            byte[] neiData = Encryption.DecryptBytes(File.ReadAllBytes(filePath), NEI_KEY);
+            var neiData = Encryption.DecryptBytes(File.ReadAllBytes(filePath), NEI_KEY);
 
             using (var ms = new MemoryStream(neiData))
             {
@@ -146,7 +241,7 @@ namespace ArcToolkitCLI.Commands.Converters
 
                     var strLengths = new int[cols * rows];
 
-                    for (int cell = 0; cell < cols * rows; cell++)
+                    for (var cell = 0; cell < cols * rows; cell++)
                     {
                         br.ReadInt32(); // Total length of all strings because why not
                         strLengths[cell] = br.ReadInt32();
@@ -154,7 +249,7 @@ namespace ArcToolkitCLI.Commands.Converters
 
                     var values = new string[cols * rows];
 
-                    for (int cell = 0; cell < cols * rows; cell++)
+                    for (var cell = 0; cell < cols * rows; cell++)
                     {
                         var len = strLengths[cell];
                         values[cell] = ShiftJisEncoding.GetString(br.ReadBytes(len), 0, Math.Max(len - 1, 0));
@@ -162,14 +257,15 @@ namespace ArcToolkitCLI.Commands.Converters
 
                     using (var tw = File.CreateText(Path.Combine(Output, $"{nameNoExt}.csv")))
                     {
-                        for (int row = 0; row < rows; row++)
+                        for (var row = 0; row < rows; row++)
                         {
-                            for (int col = 0; col < cols; col++)
+                            for (var col = 0; col < cols; col++)
                             {
-                                tw.Write(values[row * cols + col]);
-                                if(col != cols - 1)
+                                tw.Write(Escape(values[row * cols + col]));
+                                if (col != cols - 1)
                                     tw.Write(ValueSeparator);
                             }
+
                             tw.WriteLine();
                         }
                     }
@@ -178,13 +274,5 @@ namespace ArcToolkitCLI.Commands.Converters
 
             return 0;
         }
-
-        [Value(0, HelpText = "Input NEI or CSV files")]
-        public IEnumerable<string> Input { get; set; }
-
-        public string Output { get; set; }
-
-        [Option('s', "separator", Default = ";", HelpText = "Value separator of the CSV file")]
-        public string ValueSeparator { get; set; }
     }
-}
+}