<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

namespace JConverter
{
    internal class MplusConverter
    {
        private readonly Config _config;
        private SpssDataTransformer _spssDataTransformer;

        public MplusConverter(string inFile, Config config)
        {
            _config = config;
            InFile = inFile;
            var inFileProcessed = InFile.Replace(" ", "_");
            OutDatFile = inFileProcessed + ".dat";
            OutInpFile = inFileProcessed + ".inp";
        }

        public string OutInpFile { get; }

        public string OutDatFile { get; }

        public string InFile { get; }

        public void ProcessFile()
        {
            ConfirmInputFileExists();
            ConfirmOutFilesDontExist();
            var data = ParseAndTransformData();
            CreateTransformedDatFile(data);
            CreateInpFile();
        }

        private void ConfirmInputFileExists()
        {
            if (!File.Exists(InFile)) throw new Exception("The file does not exist: " + InFile);
        }

        private void ConfirmOutFilesDontExist()
        {
            if (File.Exists(OutDatFile))
                throw new Exception("The .dat file already exists, please delete it first: " + OutDatFile);
            if (File.Exists(OutInpFile))
                throw new Exception("The .inp file already exists, please delete it first: " + OutInpFile);
        }

        private IEnumerable<string> ParseAndTransformData()
        {
            var data = ReadInputFile();
            _spssDataTransformer = new SpssDataTransformer(_config, data);
            _spssDataTransformer.TransformData();
            return data;
        }

        private string[] ReadInputFile() => File.ReadAllLines(InFile);

        private void CreateTransformedDatFile(IEnumerable<string> data)
            => File.WriteAllText(OutDatFile, GenerateTransformedDatData(data));

        private string GenerateTransformedDatData(IEnumerable<string> data)
            => string.Join(_config.NewLine, data.Where(x => x != null));

        private void CreateInpFile() => File.WriteAllText(OutInpFile, GenerateInpData());

        private string GenerateInpData()
            =>
                new InpDataGenerator(_config, _spssDataTransformer.VariableNames, new FileInfo(OutDatFile).Name)
                    .GenerateInpData();

        internal class SpssDataTransformer
        {
            private static readonly Regex NonNumerical = new Regex(@"[^\d,.-]+", RegexOptions.Compiled);
            private readonly Config _config;
            private readonly string[] _data;
            private int _amountOfColumns;

            public SpssDataTransformer(Config config, string[] data)
            {
                _config = config;
                _data = data;
            }

            public List<string> VariableNames { get; private set; } = new List<string>();

            public void TransformData()
            {
                foreach (var lInfo in _data.Select((x, i) => Tuple.Create(i, x)))
                    _data[lInfo.Item1] = TransformLine(lInfo);
            }

            private string TransformLine(Tuple<int, string> line)
            {
                var columns = line.Item2.Split(_config.ColumnSplitter);
                VerifyAmountOfColumns(line, columns);
                return columns.Any(x => NonNumerical.IsMatch(x))
                    ? ProcessVariableNamesLine(line, columns)
                    : ProcessValueLine(columns);
            }

            private void VerifyAmountOfColumns(Tuple<int, string> line, string[] columns)
            {
                if (line.Item1 == 0)
                    _amountOfColumns = columns.Length;
                else if (columns.Length != _amountOfColumns)
                    throw new Exception(
                        $"{HumanReadableLineNumber(line.Item1)} has {columns.Length} columns but should be {_amountOfColumns}");
            }

            private string ProcessVariableNamesLine(Tuple<int, string> line, string[] columns)
            {
                if (line.Item1 != 0)
                    throw new NotSupportedException(
                        $"There are non numerical characters on another line than the first. {HumanReadableLineNumber(line.Item1)}");
                VariableNames = columns.ToList();
                return null;
            }

            private string ProcessValueLine(string[] columns)
            {
                foreach (var entry in columns.Select((x, i) => Tuple.Create(i, x)))
                    columns[entry.Item1] = ProcessLine(entry);
                return string.Join(_config.ColumnJoiner, columns);
            }

            private string ProcessLine(Tuple<int, string> column)
            {
                var r = column.Item2;
                r = ReplaceReplacements(r);
                r = ReplaceEmpty(r);
                return r;
            }

            private string ReplaceEmpty(string column)
            {
                if (_config.HasEmptyReplacement() && string.IsNullOrWhiteSpace(column))
                    return _config.EmptyReplacement;
                return column;
            }

            private string ReplaceReplacements(string column)
            {
                if (_config.Replacements == null) return column;
                return _config.Replacements.Aggregate(column,<commit_msg>Skip non numerical test<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

namespace JConverter
{
    internal class MplusConverter
    {
        private readonly Config _config;
        private SpssDataTransformer _spssDataTransformer;

        public MplusConverter(string inFile, Config config)
        {
            _config = config;
            InFile = inFile;
            var inFileProcessed = InFile.Replace(" ", "_");
            OutDatFile = inFileProcessed + ".dat";
            OutInpFile = inFileProcessed + ".inp";
        }

        public string OutInpFile { get; }

        public string OutDatFile { get; }

        public string InFile { get; }

        public void ProcessFile()
        {
            ConfirmInputFileExists();
            ConfirmOutFilesDontExist();
            var data = ParseAndTransformData();
            CreateTransformedDatFile(data);
            CreateInpFile();
        }

        private void ConfirmInputFileExists()
        {
            if (!File.Exists(InFile)) throw new Exception("The file does not exist: " + InFile);
        }

        private void ConfirmOutFilesDontExist()
        {
            if (File.Exists(OutDatFile))
                throw new Exception("The .dat file already exists, please delete it first: " + OutDatFile);
            if (File.Exists(OutInpFile))
                throw new Exception("The .inp file already exists, please delete it first: " + OutInpFile);
        }

        private IEnumerable<string> ParseAndTransformData()
        {
            var data = ReadInputFile();
            _spssDataTransformer = new SpssDataTransformer(_config, data);
            _spssDataTransformer.TransformData();
            return data;
        }

        private string[] ReadInputFile() => File.ReadAllLines(InFile);

        private void CreateTransformedDatFile(IEnumerable<string> data)
            => File.WriteAllText(OutDatFile, GenerateTransformedDatData(data));

        private string GenerateTransformedDatData(IEnumerable<string> data)
            => string.Join(_config.NewLine, data.Where(x => x != null));

        private void CreateInpFile() => File.WriteAllText(OutInpFile, GenerateInpData());

        private string GenerateInpData()
            =>
                new InpDataGenerator(_config, _spssDataTransformer.VariableNames, new FileInfo(OutDatFile).Name)
                    .GenerateInpData();

        internal class SpssDataTransformer
        {
            private static readonly Regex NonNumerical = new Regex(@"[^\d,.-]+", RegexOptions.Compiled);
            private readonly Config _config;
            private readonly string[] _data;
            private int _amountOfColumns;

            public SpssDataTransformer(Config config, string[] data)
            {
                _config = config;
                _data = data;
            }

            public List<string> VariableNames { get; private set; } = new List<string>();

            public void TransformData()
            {
                foreach (var lInfo in _data.Select((x, i) => Tuple.Create(i, x)))
                    _data[lInfo.Item1] = TransformLine(lInfo);
            }

            private string TransformLine(Tuple<int, string> line)
            {
                var columns = line.Item2.Split(_config.ColumnSplitter);
                VerifyAmountOfColumns(line, columns);
                return columns.Any(x => NonNumerical.IsMatch(x))
                    ? ProcessVariableNamesLine(line, columns)
                    : ProcessValueLine(columns);
            }

            private void VerifyAmountOfColumns(Tuple<int, string> line, string[] columns)
            {
                if (line.Item1 == 0)
                    _amountOfColumns = columns.Length;
                else if (columns.Length != _amountOfColumns)
                    throw new Exception(
                        $"{HumanReadableLineNumber(line.Item1)} has {columns.Length} columns but should be {_amountOfColumns}");
            }

            private string ProcessVariableNamesLine(Tuple<int, string> line, string[] columns)
            {
                if (line.Item1 != 0)
                {
                    return null;
                    throw new NotSupportedException(
                        $"There are non numerical characters on another line than the first. {HumanReadableLineNumber(line.Item1)}");
                }
                VariableNames = columns.ToList();
                return null;
            }

            private string ProcessValueLine(string[] columns)
            {
                foreach (var entry in columns.Select((x, i) => Tuple.Create(i, x)))
                    columns[entry.Item1] = ProcessLine(entry);
                return string.Join(_config.ColumnJoiner, columns);
            }

            private string ProcessLine(Tuple<int, string> column)
            {
                var r = column.Item2;
                r = ReplaceReplacements(r);
                r = ReplaceEmpty(r);
                return r;
            }

            private string ReplaceEmpty(string column)
            {
                if (_config.HasEmptyReplacement() && string.IsNullOrWhiteSpace(column))
                    return _config.EmptyReplacement;
                return column;
            }

            private string ReplaceReplacements(string column)
            {
                if (_config.Replacements == null) return column;
                return _config.Replacements.Aggregate(column,
{'fraction_non_alphanumeric': 0.08626427993163624, 'fraction_numerical': 0.0017990465053521634, 'mean_word_length': 1.8146835443037974, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2114821', 'n_tokens_mistral': 2834, 'n_tokens_neox': 2684, 'n_words': 664}