<commit_before>using System;
using System.IO;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml.Serialization;
using System.Diagnostics;

namespace kgrep
{
    // Usage: kgrep matchpattern [topattern] filename
    // Finds ALL occurances of matchpattern in filename.
    // All output is written to stdout (console).
    class kgrep
    {
        static void Main(string[] args) {
            string filename;
            string matchpattern;
            string topattern = null;
            const string READ_STDIN = "stdin";  // can be any literal as long as it's not a valid file name
            List<Replacement> repList = new List<Replacement>();

            // cat filename|kgrep matchpattern
            if (args.Length == 1) {  
                matchpattern = args[0];
                ScanAndPrintTokens(READ_STDIN, matchpattern);  // read from stdin
            }
            
            // kgrep matchpattern filename OR cat filename|kgrep matchpattern topattern
            else if (args.Length == 2) {
                filename = args[1];
                switch (args[0])
	            {
                    case "-f":  // kgrep -f replacementFile
                        repList = (new ReadReplacementFile(filename)).GetReplacements();
                        SearchAndReplaceTokens(READ_STDIN, repList);
                        break;
                    default:
                        matchpattern = args[0];
                        if (File.Exists(filename))
                            ScanAndPrintTokens(filename, matchpattern);  // kgrep matchpattern filename 
                        else 
                            SingleFindAndReplace(matchpattern, topattern);
                        break;
                }
            }

            // kgrep matchpattern filename1 .... filenameN
            else if (args.Length > 2) {
                matchpattern = args[0];
                topattern = args[1];
                if (File.Exists(topattern)) { // no pattern given, it's a file instead
                    for (int i = 1; i < args.Length; i = i + 1) {
                        if (File.Exists(args[i])) {
                            ScanAndPrintTokens(args[i], matchpattern);  // kgrep matchpattern filename1 ... filenameN
                        }
                    }
                }
                else {  // kgrep matchpattern topattern
                    SingleFindAndReplace(matchpattern, topattern);
                }
            } 
            else {
                Console.WriteLine("kgrep (Kevin's grep) v0.03");
                Console.WriteLine("Usage: kgrep matchpattern [topattern] filename1 ... filenameN");
                Console.WriteLine("       kgrep -f patternfile filename1 ... filenameN");
                Console.WriteLine("       cat filename|kgrep matchpattern [topattern]");
            }
            //Console.ReadLine();
        }

        // kgrep matchpattern topattern
        static void SingleFindAndReplace(string fromPattern, string toPattern) {
            List<Replacement> repList = new List<Replacement>();
            Replacement rep = new Replacement(false, "", fromPattern, toPattern);
            repList.Add(rep);
            SearchAndReplaceTokens("stdin", repList);
        }

        // kgrep being used as a scanner/grep.
        static void ScanAndPrintTokens(string filename, string matchpattern) {
            string line;
            Regex r = null;
            HandleInput sr = new HandleInput(filename);

            try {
                KgrepEngine engine = new KgrepEngine();
                r = new Regex(matchpattern);
                while ((line = sr.ReadLine()) != null) {
                    string alteredLine = engine.ScanForTokens(line, matchpattern, "\n");
                    if (!String.IsNullOrEmpty(alteredLine)) Console.WriteLine(alteredLine);
                }
            }
            catch (Exception e) {
                Console.WriteLine("{0}", e.Message);
            }
            finally {
                sr.Close();
            }
        }

        // Kgrep being used as sed.
        static void SearchAndReplaceTokens(string filename, List<Replacement> repList) {
            HandleInput sr = new HandleInput(filename);
            if (repList.Count == 0)
                return;

             try {
                 KgrepEngine engine = new KgrepEngine();
                 string line;
                 while ((line = sr.ReadLine()) != null) {<commit_msg>For consistency and readability, let matchpattern be first argument.<commit_after>using System;
using System.IO;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml.Serialization;
using System.Diagnostics;

namespace kgrep
{
    // Finds ALL occurances of matchpattern in filename.
    // All output is written to stdout (console).
    class kgrep
    {
        static void Main(string[] args) {
            string filename;
            string matchpattern;
            string topattern = null;
            const string READ_STDIN = "stdin";  // can be any literal as long as it's not a valid file name

            // cat filename|kgrep matchpattern
            if (args.Length == 1) {  
                matchpattern = args[0];
                ScanAndPrintTokens(READ_STDIN, matchpattern);  
            }
            
            else if (args.Length == 2) {
                filename = args[1];
                switch (args[0])
	            {
                    case "-f":  // cat filename|kgrep -f replacementFile
                        List<Replacement> repList = new List<Replacement>();
                        repList = (new ReadReplacementFile(filename)).GetReplacements();
                        SearchAndReplaceTokens(repList, READ_STDIN);
                        break;
                    default:
                        matchpattern = args[0];
                        if (File.Exists(filename))
                            ScanAndPrintTokens(matchpattern, filename);  // kgrep matchpattern filename 
                        else 
                            SingleFindAndReplaceStdin(matchpattern, topattern); // cat filename|kgrep matchpattern topattern
                        break;
                }
            }

            // kgrep matchpattern filename1 .... filenameN
            else if (args.Length > 2) {
                matchpattern = args[0];
                topattern = args[1];
                if (File.Exists(topattern))
                    ScanAndPrintFromMultipleFiles(matchpattern, args);
                else  
                    SingleFindAndReplaceStdin(matchpattern, topattern);
            } 
            else {
                Usage();
            }
        }

        private static void Usage() {
            Console.WriteLine("kgrep (Kevin's grep) v0.04");
            Console.WriteLine("Usage: kgrep matchpattern [topattern] filename1 ... filenameN");
            Console.WriteLine("       kgrep -f patternfile filename1 ... filenameN");
            Console.WriteLine("       cat filename|kgrep matchpattern [topattern]");
        }

        // kgrep matchpattern filename1 ... filenameN
        private static void ScanAndPrintFromMultipleFiles(string matchpattern, string[] filelist) {
            for (int i = 1; i < filelist.Length; i = i + 1) {
                if (File.Exists(filelist[i])) {
                    ScanAndPrintTokens(filelist[i], matchpattern);  
                }
            }
        }

        // kgrep matchpattern topattern
        static void SingleFindAndReplaceStdin(string fromPattern, string toPattern) {
            List<Replacement> repList = new List<Replacement>();
            Replacement rep = new Replacement(false, "", fromPattern, toPattern);
            repList.Add(rep);
            SearchAndReplaceTokens(repList, "stdin");
        }

        // kgrep being used as a scanner/grep.
        static void ScanAndPrintTokens(string matchpattern, string filename) {
            HandleInput sr = new HandleInput(filename);

            try {
                KgrepEngine engine = new KgrepEngine();
                string line;
                while ((line = sr.ReadLine()) != null) {
                    string alteredLine = engine.ScanForTokens(line, matchpattern, "\n");
                    if (!String.IsNullOrEmpty(alteredLine)) Console.WriteLine(alteredLine);
                }
            }
            catch (Exception e) {
                Console.WriteLine("{0}", e.Message);
            }
            finally {
                sr.Close();
            }
        }

        // Kgrep being used as sed.
        static void SearchAndReplaceTokens(List<Replacement> repList, string filename) {
            HandleInput sr = new HandleInput(filename);
            if (repList.Count == 0)
                return;

             try {
                 KgrepEngine engine = new KgrepEngine();
                 string line;
                 while ((line = sr.ReadLine()) != null) {
{'fraction_non_alphanumeric': 0.08924232383476917, 'fraction_numerical': 0.004417936823503424, 'mean_word_length': 1.475396391470749, 'pattern_counts': {'":': 2, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1143342', 'n_tokens_mistral': 2269, 'n_tokens_neox': 2137, 'n_words': 574}