<commit_before>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Windows;
using System.Collections.ObjectModel;
using Cosmos.Compiler;
using Cosmos.Compiler.IL;


namespace TestApp {
    class Program {
        static void Main(string[] args)
        {
            try
            {
                DoScan(1);
                //DoScan(2);
            }
            catch(Exception E)
            {
                Console.WriteLine(E.ToString());
            }
            Console.ReadLine();
      }

        private static void DoScan(int aIdx)
        {
            var xSW = new Stopwatch();<commit_msg>removed readln, use ctrl-f5 instead<commit_after>﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Windows;
using System.Collections.ObjectModel;
using Cosmos.Compiler;
using Cosmos.Compiler.IL;


namespace TestApp {
    class Program {
        static void Main(string[] args)
        {
            try
            {
                DoScan(1);
                //DoScan(2);
            }
            catch(Exception E)
            {
                Console.WriteLine(E.ToString());
            }
      }

        private static void DoScan(int aIdx)
        {
            var xSW = new Stopwatch();
{'fraction_non_alphanumeric': 0.09305555555555556, 'fraction_numerical': 0.003472222222222222, 'mean_word_length': 1.7926356589147288, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7478886', 'n_tokens_mistral': 387, 'n_tokens_neox': 381, 'n_words': 94}