ArcToolkitCLI/ArcToolkitCLI/Commands/Converters/NeiConverter.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using ArcToolkitCLI.Commands.Options;
using ArcToolkitCLI.Util;
using CommandLine;

namespace ArcToolkitCLI.Commands.Converters
{
    [Verb("nei", HelpText = "Convert NEI files into CSV and back")]
    internal class NeiConverter : IConverterCommand, IInputOptions, IOutputOptions
    {
        private static readonly byte[] NEI_KEY =
        {
            0xAA,
            0xC9,
            0xD2,
            0x35,
            0x22,
            0x87,
            0x20,
            0xF2,
            0x40,
            0xC5,
            0x61,
            0x7C,
            0x01,
            0xDF,
            0x66,
            0x54
        };

        private static readonly byte[] NEI_MAGIC =
        {
            0x77,
            0x73,
            0x76,
            0xFF
        };

        private static readonly Encoding ShiftJisEncoding = Encoding.GetEncoding(932);

        [Option('s', "separator", Default = ';', HelpText = "Value separator of the CSV file")]
        public char ValueSeparator { get; set; }


        public int Run()
        {
            var files = Glob.EnumerateFiles(Input);

            if (!files.Any())
            {
                Console.WriteLine("No files specified. Run `convert help nei` for help.");
                return 0;
            }

            foreach (var file in files)
            {
                var result = 0;
                switch (file.Extension.ToLowerInvariant())
                {
                    case ".nei":
                        if ((result = ToCSV(file.FullName)) != 0)
                            return result;
                        break;
                    case ".csv":
                        if ((result = ToNei(file.FullName)) != 0)
                            return result;
                        break;
                    default:
                        Console.WriteLine($"File {file.FullName} is neither .nei nor .csv file. Skipping...");
                        break;
                }
            }

            return 0;
        }

        [Value(0, HelpText = "Input NEI or CSV files")]
        public IEnumerable<string> Input { get; set; }

        public string Output { get; set; }

        private List<List<string>> ParseCSV(TextReader tr)
        {
            var buffer = new StringBuilder();
            var whitespaceBuffer = new StringBuilder();
            var isQuoted = false;
            var quoteLevel = 0;
            var result = new List<List<string>>();
            var line = new List<string>();

            int nextChar;
            while ((nextChar = tr.Read()) != -1)
            {
                var c = (char)nextChar;

                if (c == '\n' && !isQuoted)
                {
                    if (buffer.Length != 0)
                    {
                        line.Add(buffer.ToString());
                        buffer.Clear();
                    }

                    if (line.Count != 0)
                        result.Add(line);

                    line = new List<string>();
                    whitespaceBuffer.Clear();
                    continue;
                }

                var isWhitespace = char.IsWhiteSpace(c);
                var shouldSeparate = c == ValueSeparator && (!isQuoted || quoteLevel % 2 == 0);

                if (isWhitespace)
                {
                    whitespaceBuffer.Append(c);
                    continue;
                }

                if (whitespaceBuffer.Length != 0)
                {
                    if (buffer.Length > 0 && !shouldSeparate)
                        buffer.Append(whitespaceBuffer);
                    whitespaceBuffer.Clear();
                }

                if (shouldSeparate)
                {
                    line.Add(buffer.ToString());
                    buffer.Clear();
                    quoteLevel = 0;
                    isQuoted = false;
                }
                else if (c == '"')
                {
                    if (buffer.Length == 0 && quoteLevel == 0)
                    {
                        isQuoted = true;
                        quoteLevel++;
                        whitespaceBuffer.Clear();
                        continue;
                    }

                    if (isQuoted)
                        quoteLevel++;

                    if (!isQuoted || quoteLevel % 2 == 1)
                        buffer.Append(c);
                }
                else
                {
                    if (isQuoted && quoteLevel != 0 && quoteLevel % 2 == 0)
                    {
                        line.Clear();
                        tr.ReadLine();
                        continue;
                    }

                    buffer.Append(c);
                }
            }

            if (buffer.Length != 0)
                line.Add(buffer.ToString());

            if (line.Count != 0)
                result.Add(line);

            return result;
        }

        private int ToNei(string filePath)
        {
            var nameNoExt = Path.GetFileNameWithoutExtension(filePath);

            List<List<string>> values;
            using (var tr = File.OpenText(filePath))
                values = ParseCSV(tr);

            var cols = values.Max(l => l.Count);
            var rows = values.Count;

            var encodedValues = new byte[rows * cols][];

            for (var rowIndex = 0; rowIndex < values.Count; rowIndex++)
            {
                var row = values[rowIndex];
                for (var colIndex = 0; colIndex < cols; colIndex++)
                    encodedValues[colIndex + rowIndex * cols] = colIndex < row.Count
                        ? ShiftJisEncoding.GetBytes(row[colIndex])
                        : new byte[0];
            }

            using var ms = new MemoryStream();
            using var bw = new BinaryWriter(ms);
            bw.Write(NEI_MAGIC);
            bw.Write(cols);
            bw.Write(rows);

            var totalLength = 0;
            foreach (var encodedValue in encodedValues)
            {
                var len = encodedValue.Length;
                if (len != 0)
                    len++;

                bw.Write(encodedValue.Length == 0 ? 0 : totalLength);
                bw.Write(len);
                totalLength += len;
            }

            for (var i = 0; i < encodedValues.Length; i++)
            {
                var encodedValue = encodedValues[i];
                if (encodedValue.Length == 0)
                    continue;

                bw.Write(encodedValue);
                if (i != encodedValue.Length - 1)
                    bw.Write((byte)0x00);
            }

            var data = ms.ToArray();
            File.WriteAllBytes(Path.Combine(Output, $"{nameNoExt}.nei"), Encryption.EncryptBytes(data, NEI_KEY));
            return 0;
        }

        private string Escape(string value)
        {
            if (!value.Contains(ValueSeparator) && !value.Contains('\n'))
                return value;
            return $"\"{value.Replace("\"", "\"\"")}\"";
        }

        private int ToCSV(string filePath)
        {
            var nameNoExt = Path.GetFileNameWithoutExtension(filePath);

            var neiData = Encryption.DecryptBytes(File.ReadAllBytes(filePath), NEI_KEY);

            using var ms = new MemoryStream(neiData);
            using var br = new BinaryReader(ms);
            if (!br.ReadBytes(4).SequenceEqual(NEI_MAGIC))
            {
                Console.WriteLine($"File {filePath} is not a valid NEI file");
                return 1;
            }

            var cols = br.ReadUInt32();
            var rows = br.ReadUInt32();

            var strLengths = new int[cols * rows];

            for (var cell = 0; cell < cols * rows; cell++)
            {
                br.ReadInt32(); // Total length of all strings because why not
                strLengths[cell] = br.ReadInt32();
            }

            var values = new string[cols * rows];

            for (var cell = 0; cell < cols * rows; cell++)
            {
                var len = strLengths[cell];
                values[cell] = ShiftJisEncoding.GetString(br.ReadBytes(len), 0, Math.Max(len - 1, 0));
            }

            using var tw = File.CreateText(Path.Combine(Output, $"{nameNoExt}.csv"));
            for (var row = 0; row < rows; row++)
            {
                for (var col = 0; col < cols; col++)
                {
                    tw.Write(Escape(values[row * cols + col]));
                    if (col != cols - 1)
                        tw.Write(ValueSeparator);
                }

                tw.WriteLine();
            }

            return 0;
        }
    }
}