<commit_before>﻿using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text.RegularExpressions;
using System.Xml.XPath;
using WTM.Core.Application.Attributes;
using WTM.Core.Domain.WebsiteEntities.Base;

namespace WTM.Core.Application.Parsers
{
    public abstract class ParserBase<T> : IPageIdentifier
        where T : IWebsiteEntityBase, new()
    {
        private readonly IWebClient webClient;
        private readonly IHtmlParser htmlParser;

        public abstract string Identifier { get; }

        protected ParserBase(IWebClient webClient, IHtmlParser htmlParser)
        {
            this.webClient = webClient;
            this.htmlParser = htmlParser;
        }

        protected virtual T Parse(string parameter)
        {
            var uri = MakeUri(parameter);
            HtmlDocument document;
            using (var stream = webClient.GetStream(uri))
            {
                document = htmlParser.GetHtmlDocument(stream);
            }

            var instance = new T();

            BeforeParse(instance, document);
            Parse(instance, document);
            AfterParse(instance, document);

            return instance;
        }

        protected virtual Uri MakeUri(string parameter)
        {
            return new Uri(webClient.UriBase, Identifier + "/" + parameter);
        }

        protected virtual void BeforeParse(T instance, HtmlDocument htmlDocument)
        { }

        protected virtual void Parse(T instance, HtmlDocument htmlDocument)
        {
            var navigator = htmlDocument.CreateNavigator();
            if (navigator == null)
                throw new ArgumentException("Impossible to create navigator over HtmlDocument");

            ParseObject(instance, navigator);
        }

        protected virtual void AfterParse(T instance, HtmlDocument htmlDocument)
        { }

        //

        private readonly Type[] simpleTypes = {typeof(int), typeof(bool), typeof(string), typeof(DateTime)};

        private void ParseObject(object instance, XPathNavigator navigator)
        {
            var properties = GetPropertiesWithParserAttribute(instance);

            foreach (var property in properties)
            {
                ParseProperty(instance, property, navigator);
            }
        }

        private void ParseProperty(object instance, PropertyInfo property, XPathNavigator navigator)
        {
            var parseAttr = GetParserAttribute(property);
            var nodeIterator = GetXPathIterator(navigator, parseAttr.XPathExpression);

            if (IsSimpleType(property))
            {
                // Simple type
                ParseSimpleType(instance, property, parseAttr, nodeIterator);
            }
            else if (IsArrayType(property))<commit_msg>Remevoe useless methods before/after parse<commit_after>﻿using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text.RegularExpressions;
using System.Xml.XPath;
using WTM.Core.Application.Attributes;
using WTM.Core.Domain.WebsiteEntities.Base;

namespace WTM.Core.Application.Parsers
{
    public abstract class ParserBase<T> : IPageIdentifier
        where T : IWebsiteEntityBase, new()
    {
        private readonly IWebClient webClient;
        private readonly IHtmlParser htmlParser;

        public abstract string Identifier { get; }

        protected ParserBase(IWebClient webClient, IHtmlParser htmlParser)
        {
            this.webClient = webClient;
            this.htmlParser = htmlParser;
        }

        protected virtual T Parse(string parameter)
        {
            var uri = MakeUri(parameter);
            HtmlDocument document;
            using (var stream = webClient.GetStream(uri))
            {
                document = htmlParser.GetHtmlDocument(stream);
            }

            var instance = new T();

            Parse(instance, document);

            return instance;
        }

        protected virtual Uri MakeUri(string parameter)
        {
            return new Uri(webClient.UriBase, Identifier + "/" + parameter);
        }

        protected virtual void Parse(T instance, HtmlDocument htmlDocument)
        {
            var navigator = htmlDocument.CreateNavigator();
            if (navigator == null)
                throw new ArgumentException("Impossible to create navigator over HtmlDocument");

            ParseObject(instance, navigator);
        }

        //

        private readonly Type[] simpleTypes = {typeof(int), typeof(bool), typeof(string), typeof(DateTime)};

        private void ParseObject(object instance, XPathNavigator navigator)
        {
            var properties = GetPropertiesWithParserAttribute(instance);

            foreach (var property in properties)
            {
                ParseProperty(instance, property, navigator);
            }
        }

        private void ParseProperty(object instance, PropertyInfo property, XPathNavigator navigator)
        {
            var parseAttr = GetParserAttribute(property);
            var nodeIterator = GetXPathIterator(navigator, parseAttr.XPathExpression);

            if (IsSimpleType(property))
            {
                // Simple type
                ParseSimpleType(instance, property, parseAttr, nodeIterator);
            }
            else if (IsArrayType(property))
{'fraction_non_alphanumeric': 0.07472002937396732, 'fraction_numerical': 0.0, 'mean_word_length': 2.216056670602125, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2638403', 'n_tokens_mistral': 1319, 'n_tokens_neox': 1268, 'n_words': 339}