<commit_before>﻿using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Xml.Linq;

namespace BlogConverter
{
    class Program
    {
        static void Main(string[] args)
        {
            var root = XElement.Load(args[0]);
            XNamespace ns = "http://www.w3.org/2005/Atom";

            var entries = from entry in root.Elements(ns + "entry")
                          where entry.Element(ns + "category").Attribute("term").Value.Contains("kind#post")
                          select entry;
            foreach (var entry in entries)
            {
                string title = entry.Element(ns + "title").Value;
                title = Util.FixQuotes(title);

                var link = entry.Elements(ns + "link").Where(e => e.Attribute("rel").Value == "alternate").FirstOrDefault();
                if (link == null) continue;
                string fileName = Path.GetFileNameWithoutExtension(link.Attribute("href").Value);

                DateTime publishedDate = DateTime.Parse(entry.Element(ns + "published").Value);
                string publishedString = publishedDate.ToString("yyyy-MM-dd");

                var tags = from category in entry.Elements(ns + "category")
                           where category.Attribute("scheme").Value.Contains("ns#")
                           select category.Attribute("term").Value;

                string tagString = String.Join(" ", tags);
                Console.WriteLine(tagString);

                string content = entry.Element(ns + "content").Value;

                var converter = new HtmlToMarkDownConverter(content);
                string markDown = converter.Convert();

                string fullFileName = String.Format("{0}-{1}.markdown", publishedString, fileName);
                Console.WriteLine(fullFileName);

                string contentTemplate=
@"---
layout: post
title:  ""{0}""
categories: {1}
---

{2}
";

                File.WriteAllText(fullFileName, String.Format(contentTemplate, title, tagString, markDown));
            }
        }
    }
}<commit_msg>Enable comments<commit_after>﻿using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Xml.Linq;

namespace BlogConverter
{
    class Program
    {
        static void Main(string[] args)
        {
            var root = XElement.Load(args[0]);
            XNamespace ns = "http://www.w3.org/2005/Atom";

            var entries = from entry in root.Elements(ns + "entry")
                          where entry.Element(ns + "category").Attribute("term").Value.Contains("kind#post")
                          select entry;
            foreach (var entry in entries)
            {
                string title = entry.Element(ns + "title").Value;
                title = Util.FixQuotes(title);

                var link = entry.Elements(ns + "link").Where(e => e.Attribute("rel").Value == "alternate").FirstOrDefault();
                if (link == null) continue;
                string fileName = Path.GetFileNameWithoutExtension(link.Attribute("href").Value);

                DateTime publishedDate = DateTime.Parse(entry.Element(ns + "published").Value);
                string publishedString = publishedDate.ToString("yyyy-MM-dd");

                var tags = from category in entry.Elements(ns + "category")
                           where category.Attribute("scheme").Value.Contains("ns#")
                           select category.Attribute("term").Value;

                string tagString = String.Join(" ", tags);
                Console.WriteLine(tagString);

                string content = entry.Element(ns + "content").Value;

                var converter = new HtmlToMarkDownConverter(content);
                string markDown = converter.Convert();

                string fullFileName = String.Format("{0}-{1}.markdown", publishedString, fileName);
                Console.WriteLine(fullFileName);

                string contentTemplate=
@"---
layout: post
title:  ""{0}""
comments: true
categories: {1}
---

{2}
";

                File.WriteAllText(fullFileName, String.Format(contentTemplate, title, tagString, markDown));
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.12216364886545955, 'fraction_numerical': 0.0050424020169608065, 'mean_word_length': 2.0389972144846795, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1646727', 'n_tokens_mistral': 1134, 'n_tokens_neox': 1072, 'n_words': 243}