<commit_before>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Text;
using System.Text.RegularExpressions;
using NLog;

namespace kgrep {

    public class ReplaceTokens : IFileAction {
        protected static Logger logger = LogManager.GetCurrentClassLogger();
        public IHandleOutput sw = new WriteStdout();
        protected int _countOfMatchesInFile = 0;
        protected int _lineNumber = 0;
        private Command _command;
        private readonly Pickup _pickup;

        public ReplaceTokens() {
            _pickup = new Pickup();
        }

        public virtual string ApplyCommandsToInputFiles(ParseCommandFile rf, List<string> inputFilenames) {
            try {
                string line;
                string alteredLine;
                Stopwatch timer;

                foreach (string filename in inputFilenames) {
                    timer = Stopwatch.StartNew();
                    logger.Debug("Replace Matches - Processing input file:{0}", filename);
                    IHandleInput sr = (new ReadFileFactory()).GetSource((filename));
                    _lineNumber = 0;
                    _countOfMatchesInFile = 0;
                    while ((line = sr.ReadLine()) != null) {
                        _lineNumber++;
                        // TODO: Print Info after every 5000 lines, i.e. I'm still processing
                        alteredLine = ApplyCommandsToLine(line, rf.CommandList);
                        if (!String.IsNullOrEmpty(alteredLine)) sw.Write(alteredLine);
                    }
                    sr.Close();
                    timer.Stop();
                    logger.Info("File {0} found {1} matches on {2} input lines [{3:d} ms]"
                                , filename, _countOfMatchesInFile, _lineNumber, timer.ElapsedMilliseconds);
                }
            } catch (Exception e) {
                Console.WriteLine("{0}", e.Message);
            }
            return sw.Close();
        }

        public string ApplyCommandsToLine(string argline, List<Command> commandList) {
            string line = argline;
            foreach (Command command in commandList) {
                if (command.IsReplaceFirstMatchCommand) {
                   line = (ApplyCommandsFirstMatch(line, command));
                   if (command.SubjectString.IsMatch(argline)) break;
                }
                else 
                   line = ApplyCommandsAllMatches(line, command);
            }
            return line;
        }

        public string ApplyCommandsAllMatches(string line, Command command) {
            if (isCandidateForReplacement(line, command)) {
                if (command.Style == Command.CommandType.Pickup)
                    _pickup.CollectAllPickupsInLine(line, command);
                else
                    line = ApplySingleCommand(line, command);
            }
            return line;
        }

        public string ApplyCommandsFirstMatch(string line, Command command) {
            if (isCandidateForReplacement(line, command)) {
                if (command.SubjectString.IsMatch(line)) {
                    if (command.Style == Command.CommandType.Normal || command.Style == Command.CommandType.Anchored) {
                        _pickup.CollectAllPickupsInLine(line, command);
                        line = command.SubjectString.Replace(line, _pickup.ReplacePickupsWithStoredValue(command.ReplacementString)); //,1);
                    }
                    _pickup.CollectAllPickupsInLine(line, command);
                }
            }
            return line;
        }

        private string ApplySingleCommand(string line, Command command) {
            _command = command;
            _pickup.CollectAllPickupsInLine(line, command);
            line = ReplaceIt(command.SubjectString, line, _pickup.ReplacePickupsWithStoredValue(command.ReplacementString));
            return line;
        }

        private string ReplaceIt(Regex re, string source, string target) {
            int count = re.Matches(source).Count;
            if (count>0) {
                logger.Debug("   At line {0} found {1} occurances of '{2}' in '{3}'", _lineNumber, count, re.ToString(), source);
                _countOfMatchesInFile += count;
                return re.Replace(source, target);
            }
            return source;
        }

        private bool isCandidateForReplacement(string line, Command command) {
            // Has a matching AnchorString?
            if (command.AnchorString.Length > 0) {
                if (Regex.IsMatch(line, command.AnchorString))
                    return true;
                logger.Trace("   is not a Command candidate");
                return false;
            }
            return true;
        }
    }
}<commit_msg>Refactoring - Doing Object Calisthenics<commit_after>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Text;
using System.Text.RegularExpressions;
using NLog;

namespace kgrep {

    public class ReplaceTokens : IFileAction {
        private static Logger logger = LogManager.GetCurrentClassLogger();
        public IHandleOutput sw = new WriteStdout();
        private int _countOfMatchesInFile = 0;
        private int _lineNumber = 0;
        private Command _command;
        private readonly Pickup _pickup;

        public ReplaceTokens() {
            _pickup = new Pickup();
        }

        public virtual string ApplyCommandsToInputFiles(ParseCommandFile rf, List<string> inputFilenames) {
            try {
                string line;
                string alteredLine;
                Stopwatch timer;

                foreach (string filename in inputFilenames) {
                    timer = Stopwatch.StartNew();
                    logger.Debug("Replace Matches - Processing input file:{0}", filename);
                    IHandleInput sr = (new ReadFileFactory()).GetSource((filename));
                    _lineNumber = 0;
                    _countOfMatchesInFile = 0;
                    while ((line = sr.ReadLine()) != null) {
                        _lineNumber++;
                        // TODO: Print Info after every 5000 lines, i.e. I'm still processing
                        alteredLine = ApplyCommandsToLine(line, rf.CommandList);
                        if (!String.IsNullOrEmpty(alteredLine)) sw.Write(alteredLine);
                    }
                    sr.Close();
                    timer.Stop();
                    logger.Info("File {0} found {1} matches on {2} input lines [{3:d} ms]"
                                , filename, _countOfMatchesInFile, _lineNumber, timer.ElapsedMilliseconds);
                }
            } catch (Exception e) {
                Console.WriteLine("{0}", e.Message);
            }
            return sw.Close();
        }

        public string ApplyCommandsToLine(string argline, List<Command> commandList) {
            string line = argline;
            foreach (Command command in commandList) {
                if ( ! isCandidateForReplacement(line, command)) break;
                if (command.IsReplaceFirstMatchCommand) {
                   line = ApplyCommandsFirstMatch(line, command);
                   if (command.SubjectString.IsMatch(argline)) break;
                }
                else 
                   line = ApplyCommandsAllMatches(line, command);
            }
            return line;
        }

        public string ApplyCommandsAllMatches(string line, Command command) {
            if (command.Style == Command.CommandType.Pickup)
                _pickup.CollectAllPickupsInLine(line, command);
            else
                line = ApplySingleCommand(line, command);
            return line;
        }

        public string ApplyCommandsFirstMatch(string line, Command command) {
            if ( ! command.SubjectString.IsMatch(line))
                return line;

            if (command.Style == Command.CommandType.Normal || command.Style == Command.CommandType.Anchored) {
                line = ApplySingleCommand(line, command);
            }
            _pickup.CollectAllPickupsInLine(line, command);
            return line;
        }

        private string ApplySingleCommand(string line, Command command) {
            _command = command;
            _pickup.CollectAllPickupsInLine(line, command);
            line = ReplaceIt(command.SubjectString, line, _pickup.ReplacePickupsWithStoredValue(command.ReplacementString));
            return line;
        }

        private string ReplaceIt(Regex re, string source, string target) {
            int count = re.Matches(source).Count;
            if (count>0) {
                logger.Debug("   At line {0} found {1} occurances of '{2}' in '{3}'", _lineNumber, count, re.ToString(), source);
                _countOfMatchesInFile += count;
                return re.Replace(source, target);
            }
            return source;
        }

        private bool isCandidateForReplacement(string line, Command command) {
            if (command.AnchorString.Length == 0)   // no anchor present
                return true;

            if (Regex.IsMatch(line, command.AnchorString))
                return true;
            logger.Trace("   is not a Command candidate");
            return false;
        }
    }
}
{'fraction_non_alphanumeric': 0.08576622821088653, 'fraction_numerical': 0.0043845578013046735, 'mean_word_length': 1.7068017366136035, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1030538', 'n_tokens_mistral': 2351, 'n_tokens_neox': 2214, 'n_words': 563}