<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Text.RegularExpressions;

namespace RCPA
{
  public class AnnotationFormat : IFileFormat<List<Annotation>>
  {
    private LineFormat<Annotation> _format { get; set; }

    public LineFormat<Annotation> Format
    {
      get
      {
        return _format;
      }
      set
      {
        _format = value;
      }
    }

    private string ignoreHeaderPattern;
    public AnnotationFormat(string ignoreHeaderPattern = null)
    {
      this.ignoreHeaderPattern = ignoreHeaderPattern;
      this.EndRegex = null;
    }

    public Regex EndRegex { get; set; }

    public virtual List<Annotation> ReadFromFile(string fileName)
    {
      var result = new List<Annotation>();

      using (StreamReader sr = new StreamReader(fileName))
      {
        string line;
        while ((line = sr.ReadLine()) != null)
        {
          if (string.IsNullOrEmpty(ignoreHeaderPattern))
          {
            break;
          }

          if (!Regex.Match(line, ignoreHeaderPattern).Success)
          {
            break;
          }
        }

        InitializeLineFormat(line);

        while ((line = sr.ReadLine()) != null)
        {
          if (line.Trim() == string.Empty)
          {
            break;
          }

          if(EndRegex != null && EndRegex.Match(line).Success)
          {
            break;
          }

          result.Add(Format.ParseString(line));
        }
      }

      return result;
    }


    public void InitializeLineFormat(string header)
    {
      _format = new LineFormat<Annotation>(AnnotationPropertyFactory.GetInstance(), header);
    }<commit_msg>add hasHeader option<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Text.RegularExpressions;

namespace RCPA
{
  public class AnnotationFormat : IFileFormat<List<Annotation>>
  {
    private LineFormat<Annotation> _format { get; set; }

    public LineFormat<Annotation> Format
    {
      get
      {
        return _format;
      }
      set
      {
        _format = value;
      }
    }

    private string ignoreHeaderPattern;
    private bool hasHeader;
    public AnnotationFormat(string ignoreHeaderPattern = null, bool hasHeader=true)
    {
      this.ignoreHeaderPattern = ignoreHeaderPattern;
      this.hasHeader = hasHeader;
      this.EndRegex = null;
    }

    public Regex EndRegex { get; set; }

    public virtual List<Annotation> ReadFromFile(string fileName)
    {
      var result = new List<Annotation>();

      using (StreamReader sr = new StreamReader(fileName))
      {
        string line;

        if (hasHeader)
        {
          while ((line = sr.ReadLine()) != null)
          {
            if (string.IsNullOrEmpty(ignoreHeaderPattern))
            {
              break;
            }

            if (!Regex.Match(line, ignoreHeaderPattern).Success)
            {
              break;
            }
          }

          InitializeLineFormat(line);
        }
        else
        {
          _format = null;
        }

        while ((line = sr.ReadLine()) != null)
        {
          if (line.Trim() == string.Empty)
          {
            break;
          }

          if(EndRegex != null && EndRegex.Match(line).Success)
          {
            break;
          }

          if(!hasHeader && _format == null)
          {
            var parts = line.Split('\t');
            var header = (from i in Enumerable.Range(1, parts.Length) select "X" + i.ToString()).Merge("\t");
            InitializeLineFormat(header);
          }

          result.Add(Format.ParseString(line));
        }
      }

      return result;
    }


    public void InitializeLineFormat(string header)
    {
      _format = new LineFormat<Annotation>(AnnotationPropertyFactory.GetInstance(), header);
    }
{'fraction_non_alphanumeric': 0.09994913530010173, 'fraction_numerical': 0.000254323499491353, 'mean_word_length': 1.83971119133574, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4763043', 'n_tokens_mistral': 1083, 'n_tokens_neox': 1052, 'n_words': 229}