<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Globalization;

namespace CC.Common.Parser.Demo
{
  class Program
  {
    static void Main(string[] args)
    {
      // Parse DemoFile1 - Since it's fixed width we don't need to pad it
      StringSplitter split = new StringSplitter(StringSplitterEnum.PadNone);

      // Now we add our "field" definitions - zero based positioning
      split.AddRange("Date", new IntRange(0, 8));
      split.AddRange("Name", new IntRange(8, 47));
      split.AddRange("DeptNo", new IntRange(47, 53));
      split.AddRange("DoorCode", new IntRange(54, 57));
      split.AddRange("KeyCode", new IntRange(57, 60));


      // Read our file, line by line or all at once, doesn't matter
      string[] data = new string[] { };
      using (StreamReader sr = File.OpenText("DemoFile1.txt"))
      {
        string text = sr.ReadToEnd();
        // Split using UNIX EOL
        data = text.Split('\n');
        sr.Close();
      }

      // Go through each line
      foreach (string line in data)
      {
        // "stuff" the "parser"
        split.String = line;

        // Ask for the chunk back - case sensitive, but doesn't throw an error, you get a blank string
        string date = split["Date"];
        string name = split["Name"];
        string dept = split["DeptNo"];
        string code = split["DoorCode"];
        string key = split["KeyCode"];

        Console.WriteLine("Date: {0}", DateTime.ParseExact(date, "yyyyMMdd", CultureInfo.CurrentCulture).ToLongDateString());
        // splitter does not trim the strings
        Console.WriteLine("Name: {0}", name.Trim());
        Console.WriteLine("Dept: {0}", dept.Trim());
        Console.WriteLine("Door: {0}", code.Trim());
        Console.WriteLine("Code: {0}", key.Trim());
        Console.WriteLine();
      }
      
      // Wait for us
      Console.ReadLine();
    }
  }
}
<commit_msg>using constants<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Globalization;

namespace CC.Common.Parser.Demo
{
  class Program
  {
    private const string DATE = "Date";
    private const string NAME = "Name";
    private const string DEPT = "DeptNo";
    private const string DOOR = "DoorCode";
    private const string CODE = "KeyCode";

    static void Main(string[] args)
    {
      // Parse DemoFile1 - Since it's fixed width we don't need to pad it
      StringSplitter split = new StringSplitter(StringSplitterEnum.PadNone);

      // Now we add our "field" definitions - zero based positioning
      split.AddRange(DATE, new IntRange(0, 8));
      split.AddRange(NAME, new IntRange(8, 47));
      split.AddRange(DEPT, new IntRange(47, 53));
      split.AddRange(DOOR, new IntRange(54, 57));
      split.AddRange(CODE, new IntRange(57, 60));
      // you can also have overlaping fields
      split.AddRange("New", new IntRange(54, 60));

      // Read our file, line by line or all at once, doesn't matter
      string[] data = new string[] { };
      using (StreamReader sr = File.OpenText("DemoFile1.txt"))
      {
        string text = sr.ReadToEnd();
        // Split using UNIX EOL
        data = text.Split('\n');
        sr.Close();
      }

      // Go through each line
      foreach (string line in data)
      {
        // "stuff" the "parser"
        split.String = line;

        // Ask for the chunk back - case sensitive, but doesn't throw an error, you get a blank string
        string date = split[DATE];
        string name = split[NAME];
        string dept = split[DEPT];
        string code = split[DOOR];
        string key = split[CODE];

        Console.WriteLine("Date: {0}", DateTime.ParseExact(date, "yyyyMMdd", CultureInfo.CurrentCulture).ToLongDateString());
        // splitter does not trim the strings
        Console.WriteLine("Name: {0}", name.Trim());
        Console.WriteLine("Dept: {0}", dept.Trim());
        Console.WriteLine("Door: {0}", code.Trim());
        Console.WriteLine("Code: {0}", key.Trim());
        // And just for giggles
        Console.WriteLine("New : {0}", split["New"].Trim());
        Console.WriteLine();
      }
      
      // Wait for us
      Console.ReadLine();
    }
  }
}

{'fraction_non_alphanumeric': 0.12986720684222372, 'fraction_numerical': 0.011928876884987621, 'mean_word_length': 2.2772861356932155, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4129215', 'n_tokens_mistral': 1359, 'n_tokens_neox': 1256, 'n_words': 381}