<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
using System.Text.RegularExpressions;

namespace Kindle2OneNote
{
    public struct Clipping
    {
        public string Title { get; set; }
        public string Author { get; set; }
        public uint Page { get; set; }
        public uint LocationFrom { get; set; }
        public uint LocationTo { get; set; }
        public DateTime AddTime { get; set; }
        public string Content { get; set; }
    }

    public sealed class ClippingParser
    {
        private static volatile ClippingParser instance = null;
        private static object syncRoot = new Object();

        public const string pattern =
    @"(?<title>.*?) \((?<author>.*?)\)\r\n- .*?(?<page>\d+) \| Location (?<from>\d+)-(?<to>\d+) \| Added on (?<time>.*)\r\n\r\n(?<content>.*)\r\n={10}";

        private ClippingParser() { }

        public static ClippingParser Instance
        {
            get
            {
                if (instance == null)
                {
                    lock (syncRoot)
                    {
                        if (instance == null)
                            instance = new ClippingParser();
                    }
                }

                return instance;
            }
        }

        public List<Clipping> Parse(string fileContent)
        {
            Clipping clip = new Clipping();
            Regex regex = new Regex(pattern);
            Match match = regex.Match(fileContent);
            List<Clipping> clippings = new List<Clipping>();

            while (match.Success)
            {
                clip.Title = match.Groups["title"].Value;
                clip.Author = match.Groups["author"].Value;
                clip.Page = Convert.ToUInt32(match.Groups["page"].Value);
                clip.LocationFrom = Convert.ToUInt32(match.Groups["from"].Value);
                clip.LocationTo = Convert.ToUInt32(match.Groups["to"].Value);
                clip.AddTime = DateTime.Parse(match.Groups["time"].Value);
                clip.Content = match.Groups["content"].Value;
                clippings.Add(clip);

                match = match.NextMatch();
            }

            return clippings;
        }
    }
}<commit_msg>use new class to simplify upload process<commit_after>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
using System.Text.RegularExpressions;

namespace Kindle2OneNote
{
    public struct Clipping
    {

        public uint Page { get; set; }
        public uint LocationFrom { get; set; }
        public uint LocationTo { get; set; }
        public DateTime AddTime { get; set; }
        public string Content { get; set; }
    }

    public class BookWithClippings : IEquatable<BookWithClippings>
    {
        public string Title { get; set; }
        public string Author { get; set; }
        public List<Clipping> Clippings { get; set; }

        public BookWithClippings(string title, string author)
        {
            Title = title;
            Author = author;
        }

        public bool Equals(BookWithClippings other)
        {
            if (other == null)
                return false;

            return String.Equals(this.Title, other.Title) && String.Equals(this.Author, other.Author);
        }
    }

    public sealed class ClippingParser
    {
        private static volatile ClippingParser instance = null;
        private static object syncRoot = new Object();

        private static readonly int notFound = -1;
        public const string pattern =
    @"(?<title>.*?) \((?<author>.*?)\)\r\n- .*?(?<page>\d+) \| Location (?<from>\d+)-(?<to>\d+) \| Added on (?<time>.*)\r\n\r\n(?<content>.*)\r\n={10}";

        private ClippingParser() { }

        public static ClippingParser Instance
        {
            get
            {
                if (instance == null)
                {
                    lock (syncRoot)
                    {
                        if (instance == null)
                            instance = new ClippingParser();
                    }
                }

                return instance;
            }
        }

        public List<BookWithClippings> Parse(string fileContent)
        {
            int index = notFound;
            string title = "";
            string author = "";
            Clipping clip = new Clipping();
            BookWithClippings book = null;
            List<BookWithClippings> books = new List<BookWithClippings>();

            Regex regex = new Regex(pattern);
            Match match = regex.Match(fileContent);
            while (match.Success)
            {
                clip.Page = Convert.ToUInt32(match.Groups["page"].Value);
                clip.LocationFrom = Convert.ToUInt32(match.Groups["from"].Value);
                clip.LocationTo = Convert.ToUInt32(match.Groups["to"].Value);
                clip.AddTime = DateTime.Parse(match.Groups["time"].Value);
                clip.Content = match.Groups["content"].Value;

                title = match.Groups["title"].Value;
                author = match.Groups["author"].Value;
                book = new BookWithClippings(title, author);
                index = books.IndexOf(book);
                if (index == notFound)
                {
                    book.Clippings.Add(clip);
                    books.Add(book);
                }
                else
                {
                    books[index].Clippings.Add(clip);
                }
                match = match.NextMatch();
            }

            return books;
        }
    }
}
{'fraction_non_alphanumeric': 0.11441048034934498, 'fraction_numerical': 0.003318777292576419, 'mean_word_length': 1.5665620797848498, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4100125', 'n_tokens_mistral': 1551, 'n_tokens_neox': 1446, 'n_words': 350}