<commit_before>﻿using Abot2.Poco;
using Serilog;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;

namespace Abot2.Core
{
    /// <summary>
    /// Handles parsing html out of the raw html
    /// </summary>
    public interface IHtmlParser
    {
        /// <summary>
        /// Parses html to extract hyperlinks, converts each into an absolute url representation.
        /// </summary>
        IEnumerable<HyperLink> GetLinks(CrawledPage crawledPage);
    }

    public abstract class HyperLinkParser : IHtmlParser
    {
        protected CrawlConfiguration Config;
        protected Func<string, string> CleanUrlFunc;

        protected HyperLinkParser()
            :this(new CrawlConfiguration(), null)
        {

        }

        protected HyperLinkParser(CrawlConfiguration config, Func<string, string> cleanUrlFunc)
        {
            Config = config;
            CleanUrlFunc = cleanUrlFunc;
        }

        /// <summary>
        /// Parses html to extract hyperlinks, converts each into an absolute url
        /// </summary>
        public virtual IEnumerable<HyperLink> GetLinks(CrawledPage crawledPage)
        {
            CheckParams(crawledPage);

            var timer = Stopwatch.StartNew();

            var links = GetHyperLinks(crawledPage, GetRawHyperLinks(crawledPage));

            timer.Stop();
            Log.Debug("{0} parsed links from [{1}] in [{2}] milliseconds", ParserType, crawledPage.Uri, timer.ElapsedMilliseconds);

            return links;
        }

        #region Abstract

        protected abstract string ParserType { get; }

        protected abstract IEnumerable<HyperLink> GetRawHyperLinks(CrawledPage crawledPage);

        protected abstract string GetBaseHrefValue(CrawledPage crawledPage);

        protected abstract string GetMetaRobotsValue(CrawledPage crawledPage);

        #endregion

        protected virtual void CheckParams(CrawledPage crawledPage)
        {
            if (crawledPage == null)
                throw new ArgumentNullException("crawledPage");
        }

        protected virtual IEnumerable<HyperLink> GetHyperLinks(CrawledPage crawledPage, IEnumerable<HyperLink> rawLinks)
        {
            var finalList = new List<HyperLink>();
            if (rawLinks == null || !rawLinks.Any())
                return finalList;

            //Use the uri of the page that actually responded to the request instead of crawledPage.Uri (Issue 82).
            var uriToUse = crawledPage.HttpRequestMessage.RequestUri ?? crawledPage.Uri;

            //If html base tag exists use it instead of page uri for relative links
            var baseHref = GetBaseHrefValue(crawledPage);
            if (!string.IsNullOrEmpty(baseHref))
            {
                if (baseHref.StartsWith("//"))
                    baseHref = uriToUse.Scheme + ":" + baseHref;
                else if (baseHref.StartsWith("/"))
                    // '/' points to the root of the filesystem when running on Linux, and is as such
                    // considered an absolute URI
                    baseHref = uriToUse.GetLeftPart(UriPartial.Authority) + baseHref;

                if (Uri.TryCreate(uriToUse, baseHref, out Uri baseUri))
                    uriToUse = baseUri;
            }

            foreach (var rawLink in rawLinks)
            {
                try
                {
                    // Remove the url fragment part of the url if needed.
                    // This is the part after the # and is often not useful.<commit_msg>Restored use of the CrawledPage URI for handling base tags with deferred scheme.<commit_after>﻿using Abot2.Poco;
using Serilog;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;

namespace Abot2.Core
{
    /// <summary>
    /// Handles parsing html out of the raw html
    /// </summary>
    public interface IHtmlParser
    {
        /// <summary>
        /// Parses html to extract hyperlinks, converts each into an absolute url representation.
        /// </summary>
        IEnumerable<HyperLink> GetLinks(CrawledPage crawledPage);
    }

    public abstract class HyperLinkParser : IHtmlParser
    {
        protected CrawlConfiguration Config;
        protected Func<string, string> CleanUrlFunc;

        protected HyperLinkParser()
            :this(new CrawlConfiguration(), null)
        {

        }

        protected HyperLinkParser(CrawlConfiguration config, Func<string, string> cleanUrlFunc)
        {
            Config = config;
            CleanUrlFunc = cleanUrlFunc;
        }

        /// <summary>
        /// Parses html to extract hyperlinks, converts each into an absolute url
        /// </summary>
        public virtual IEnumerable<HyperLink> GetLinks(CrawledPage crawledPage)
        {
            CheckParams(crawledPage);

            var timer = Stopwatch.StartNew();

            var links = GetHyperLinks(crawledPage, GetRawHyperLinks(crawledPage));

            timer.Stop();
            Log.Debug("{0} parsed links from [{1}] in [{2}] milliseconds", ParserType, crawledPage.Uri, timer.ElapsedMilliseconds);

            return links;
        }

        #region Abstract

        protected abstract string ParserType { get; }

        protected abstract IEnumerable<HyperLink> GetRawHyperLinks(CrawledPage crawledPage);

        protected abstract string GetBaseHrefValue(CrawledPage crawledPage);

        protected abstract string GetMetaRobotsValue(CrawledPage crawledPage);

        #endregion

        protected virtual void CheckParams(CrawledPage crawledPage)
        {
            if (crawledPage == null)
                throw new ArgumentNullException("crawledPage");
        }

        protected virtual IEnumerable<HyperLink> GetHyperLinks(CrawledPage crawledPage, IEnumerable<HyperLink> rawLinks)
        {
            var finalList = new List<HyperLink>();
            if (rawLinks == null || !rawLinks.Any())
                return finalList;

            //Use the uri of the page that actually responded to the request instead of crawledPage.Uri (Issue 82).
            var uriToUse = crawledPage.HttpRequestMessage.RequestUri ?? crawledPage.Uri;

            //If html base tag exists use it instead of page uri for relative links
            var baseHref = GetBaseHrefValue(crawledPage);
            if (!string.IsNullOrEmpty(baseHref))
            {
                if (baseHref.StartsWith("//"))
                    baseHref = crawledPage.Uri.Scheme + ":" + baseHref;
                else if (baseHref.StartsWith("/"))
                    // '/' points to the root of the filesystem when running on Linux, and is as such
                    // considered an absolute URI
                    baseHref = uriToUse.GetLeftPart(UriPartial.Authority) + baseHref;

                if (Uri.TryCreate(uriToUse, baseHref, out Uri baseUri))
                    uriToUse = baseUri;
            }

            foreach (var rawLink in rawLinks)
            {
                try
                {
                    // Remove the url fragment part of the url if needed.
                    // This is the part after the # and is often not useful.
{'fraction_non_alphanumeric': 0.07981090100111235, 'fraction_numerical': 0.001946607341490545, 'mean_word_length': 2.0235393022278267, 'pattern_counts': {'":': 2, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4865572', 'n_tokens_mistral': 1894, 'n_tokens_neox': 1765, 'n_words': 545}