Implement NEI to CSV parsing

ArcToolkitCLI/Commands/Converters/NeiConverter.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
+using System.Text;
 using ArcToolkitCLI.Commands.Options;
 using ArcToolkitCLI.Util;
 using CommandLine;
@@ -10,6 +12,11 @@ namespace ArcToolkitCLI.Commands.Converters
     [Verb("nei", HelpText = "Convert NEI files into CSV and back")]
     class NeiConverter : IConverterCommand, IInputOptions, IOutputOptions
     {
+        private static readonly byte[] NEI_KEY = { 0xAA, 0xC9, 0xD2, 0x35, 0x22, 0x87, 0x20, 0xF2, 0x40, 0xC5, 0x61, 0x7C, 0x01, 0xDF, 0x66, 0x54 };
+        private static readonly byte[] NEI_MAGIC = { 0x77, 0x73, 0x76, 0xFF };
+        private static readonly Encoding ShiftJisEncoding = Encoding.GetEncoding(932);
+
+
         public int Run()
         {
             var files = Glob.EnumerateFiles(Input);
@@ -22,13 +29,16 @@ namespace ArcToolkitCLI.Commands.Converters
 
             foreach (var file in files)
             {
+                int result = 0;
                 switch (file.Extension.ToLowerInvariant())
                 {
-                    case "nei":
-                        ToCSV(file.FullName);
+                    case ".nei":
+                        if ((result = ToCSV(file.FullName)) != 0)
+                            return result;
                         break;
-                    case "csv":
-                        ToNei(file.FullName);
+                    case ".csv":
+                        if ((result = ToNei(file.FullName)) != 0)
+                            return result;
                         break;
                     default:
                         Console.WriteLine($"File {file.FullName} is neither .nei nor .csv file. Skipping...");
@@ -39,17 +49,69 @@ namespace ArcToolkitCLI.Commands.Converters
             return 0;
         }
 
-        void ToNei(string filePath)
+        int ToNei(string filePath)
         {
-
+            return 0;
         }
 
-        void ToCSV(string filePath)
+        int ToCSV(string filePath)
         {
+            string nameNoExt = Path.GetFileNameWithoutExtension(filePath);
+
+            byte[] neiData = Encryption.DecryptBytes(File.ReadAllBytes(filePath), NEI_KEY);
+
+            using (var ms = new MemoryStream(neiData))
+            {
+                using (var br = new BinaryReader(ms))
+                {
+                    if (!br.ReadBytes(4).SequenceEqual(NEI_MAGIC))
+                    {
+                        Console.WriteLine($"File {filePath} is not a valid NEI file");
+                        return 1;
+                    }
+
+                    var cols = br.ReadUInt32();
+                    var rows = br.ReadUInt32();
+
+                    var strLengths = new int[cols * rows * 4];
 
+                    for (int cell = 0; cell < cols * rows; cell++)
+                    {
+                        br.ReadInt32(); // Total length of all strings
+                        strLengths[cell] = br.ReadInt32();
+                    }
+
+                    var values = new string[cols * rows];
+
+                    for (int cell = 0; cell < cols * rows; cell++)
+                    {
+                        var len = strLengths[cell];
+                        values[cell] = ShiftJisEncoding.GetString(br.ReadBytes(len), 0, Math.Max(len - 1, 0));
+                    }
+
+                    using (var sw = new StreamWriter(File.Create(Path.Combine(Output, $"{nameNoExt}.csv"))))
+                    {
+                        for (int row = 0; row < rows; row++)
+                        {
+                            for (int col = 0; col < cols; col++)
+                            {
+                                sw.Write(values[row * cols + col]);
+                                sw.Write(ValueSeparator);
+                            }
+                            sw.WriteLine();
+                        }
+                    }
+                }
+
+            }
+
+            return 0;
         }
 
         public IEnumerable<string> Input { get; set; }
         public string Output { get; set; }
+
+        [Option('s', "separator", Default = ";", HelpText = "Value separator of the CSV file")]
+        public string ValueSeparator { get; set; }
     }
 }

ArcToolkitCLI/Util/Encryption.cs

@@ -46,7 +46,7 @@ namespace ArcToolkitCLI.Util
 
         public static byte[] GenerateIV(byte[] ivSeed)
         {
-            uint[] seed = { 0x075BCD15, 0x159A55E5, 0x1F123BB5, BitConverter.ToUInt32(ivSeed, 1) ^ 0xBFBFBFBF };
+            uint[] seed = { 0x075BCD15, 0x159A55E5, 0x1F123BB5, BitConverter.ToUInt32(ivSeed, 0) ^ 0xBFBFBFBF };
 
             for (int i = 0; i < 4; i++)
             {
@@ -66,8 +66,9 @@ namespace ArcToolkitCLI.Util
 
         public static byte[] DecryptBytes(byte[] encryptedBytes, byte[] key)
         {
-            var ivSeed = new byte[5];
-            Array.Copy(encryptedBytes, encryptedBytes.Length - 5, ivSeed, 0, 5);
+            var extraDataSize = encryptedBytes[encryptedBytes.Length - 5] ^ encryptedBytes[encryptedBytes.Length - 4];
+            var ivSeed = new byte[4];
+            Array.Copy(encryptedBytes, encryptedBytes.Length - 4, ivSeed, 0, 4);
 
             byte[] iv = GenerateIV(ivSeed);
 
@@ -76,10 +77,10 @@ namespace ArcToolkitCLI.Util
                 rijndael.Padding = PaddingMode.None;
 
                 using (ICryptoTransform decryptor = rijndael.CreateDecryptor(key, iv))
-                using (var mem = new MemoryStream(encryptedBytes, 0, encryptedBytes.Length - 5))
+                using (var mem = new MemoryStream(encryptedBytes, 0, encryptedBytes.Length - extraDataSize - 5))
                 using (var stream = new CryptoStream(mem, decryptor, CryptoStreamMode.Read))
                 {
-                    var output = new byte[encryptedBytes.Length - 5];
+                    var output = new byte[encryptedBytes.Length - extraDataSize - 5];
 
                     stream.Read(output, 0, output.Length);