<commit_before>#region Copyright (c) 2016 <NAME>. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#endregion

namespace WebLinq
{
    using System;
    using System.Collections.Generic;
    using System.Collections.Specialized;
    using System.Linq;
    using System.Net.Mime;
    using System.Runtime.CompilerServices;
    using Fizzler.Systems.HtmlAgilityPack;
    using HtmlAgilityPack;
    using TryParsers;

    public interface IHtmlParser
    {
        ParsedHtml Parse(string html, Uri baseUrl);
    }

    public abstract class ParsedHtml
    {
        readonly Uri _baseUrl;
        readonly Lazy<Uri> _inlineBaseUrl;

        protected ParsedHtml() :
            this(null) {}

        protected ParsedHtml(Uri baseUrl)
        {
            _baseUrl = baseUrl;
            _inlineBaseUrl = new Lazy<Uri>(TryGetInlineBaseUrl);
        }

        public Uri BaseUrl => _baseUrl ?? _inlineBaseUrl.Value;

        Uri TryGetInlineBaseUrl()
        {
            var baseRef = QuerySelector("html > head > base[href]")?.GetAttributeValue("href");

            if (baseRef == null)
                return null;

            var baseUrl = TryParse.Uri(baseRef, UriKind.Absolute);

            return baseUrl.Scheme == Uri.UriSchemeHttp || baseUrl.Scheme == Uri.UriSchemeHttps
                 ? baseUrl : null;
        }

        public IEnumerable<HtmlObject> QuerySelectorAll(string selector) =>
            QuerySelectorAll(selector, null);

        public abstract IEnumerable<HtmlObject> QuerySelectorAll(string selector, HtmlObject context);

        public HtmlObject QuerySelector(string selector) =>
            QuerySelector(selector, null);

        public virtual HtmlObject QuerySelector(string selector, HtmlObject context) =>
            QuerySelectorAll(selector, context).FirstOrDefault();

        public abstract HtmlObject Root { get; }

        public override string ToString() => Root?.OuterHtml ?? string.Empty;
    }

    public abstract class HtmlObject
    {
        public abstract ParsedHtml Owner { get; }
        public abstract string Name { get; }
        public virtual bool HasAttributes => AttributeNames.Any();
        public abstract IEnumerable<string> AttributeNames { get; }
        public abstract bool HasAttribute(string name);
        public abstract string GetAttributeValue(string name);
        public abstract string OuterHtml { get; }
        public abstract string InnerHtml { get; }
        public abstract string InnerText { get; }
        public virtual bool HasChildElements => ChildElements.Any();
        public abstract IEnumerable<HtmlObject> ChildElements { get; }
        public override string ToString() => OuterHtml;
    }

    public enum HtmlControlType { Input, Select, TextArea }
    public enum HtmlDisabledFlag { Default, Disabled }
    public enum HtmlReadOnlyFlag { Default, ReadOnly }
    public enum HtmlFormMethod { Get, Post }

    public sealed class HapHtmlParser : IHtmlParser
    {
        readonly QueryContext _context;

        public HapHtmlParser(QueryContext context)
        {
            _context = context;
        }

        public ParsedHtml Parse(string html, Uri baseUrl)
        {
            var doc = new HtmlDocument();
            doc.LoadHtml2(html);
            return new HapParsedHtml(doc, baseUrl);
        }

        sealed class HapParsedHtml : ParsedHtml
        {
            readonly HtmlDocument _document;
            readonly ConditionalWeakTable<HtmlNode, HtmlObject> _map = new ConditionalWeakTable<HtmlNode, HtmlObject>();

            public HapParsedHtml(HtmlDocument document, Uri baseUrl) :
                base(baseUrl)
            {
                _document = document;
            }

            HtmlObject GetPublicObject(HtmlNode node)
            {
                HtmlObject obj;
                if (!_map.TryGetValue(node, out obj))
                    _map.Add(node, obj = new HapHtmlObject(node, this));
                return obj;
            }

            public override IEnumerable<HtmlObject> QuerySelectorAll(string selector, HtmlObject context) =>
                (_document.DocumentNode ?? ((HapHtmlObject)context).Node).QuerySelectorAll(selector).Select(GetPublicObject);

            public override HtmlObject Root => GetPublicObject(_document.DocumentNode);

            sealed class HapHtmlObject : HtmlObject
            {
                readonly HapParsedHtml _owner;
<commit_msg>Neglected cotext fix in QuerySelectorAll<commit_after>#region Copyright (c) 2016 <NAME>. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
#endregion

namespace WebLinq
{
    using System;
    using System.Collections.Generic;
    using System.Collections.Specialized;
    using System.Linq;
    using System.Net.Mime;
    using System.Runtime.CompilerServices;
    using Fizzler.Systems.HtmlAgilityPack;
    using HtmlAgilityPack;
    using TryParsers;

    public interface IHtmlParser
    {
        ParsedHtml Parse(string html, Uri baseUrl);
    }

    public abstract class ParsedHtml
    {
        readonly Uri _baseUrl;
        readonly Lazy<Uri> _inlineBaseUrl;

        protected ParsedHtml() :
            this(null) {}

        protected ParsedHtml(Uri baseUrl)
        {
            _baseUrl = baseUrl;
            _inlineBaseUrl = new Lazy<Uri>(TryGetInlineBaseUrl);
        }

        public Uri BaseUrl => _baseUrl ?? _inlineBaseUrl.Value;

        Uri TryGetInlineBaseUrl()
        {
            var baseRef = QuerySelector("html > head > base[href]")?.GetAttributeValue("href");

            if (baseRef == null)
                return null;

            var baseUrl = TryParse.Uri(baseRef, UriKind.Absolute);

            return baseUrl.Scheme == Uri.UriSchemeHttp || baseUrl.Scheme == Uri.UriSchemeHttps
                 ? baseUrl : null;
        }

        public IEnumerable<HtmlObject> QuerySelectorAll(string selector) =>
            QuerySelectorAll(selector, null);

        public abstract IEnumerable<HtmlObject> QuerySelectorAll(string selector, HtmlObject context);

        public HtmlObject QuerySelector(string selector) =>
            QuerySelector(selector, null);

        public virtual HtmlObject QuerySelector(string selector, HtmlObject context) =>
            QuerySelectorAll(selector, context).FirstOrDefault();

        public abstract HtmlObject Root { get; }

        public override string ToString() => Root?.OuterHtml ?? string.Empty;
    }

    public abstract class HtmlObject
    {
        public abstract ParsedHtml Owner { get; }
        public abstract string Name { get; }
        public virtual bool HasAttributes => AttributeNames.Any();
        public abstract IEnumerable<string> AttributeNames { get; }
        public abstract bool HasAttribute(string name);
        public abstract string GetAttributeValue(string name);
        public abstract string OuterHtml { get; }
        public abstract string InnerHtml { get; }
        public abstract string InnerText { get; }
        public virtual bool HasChildElements => ChildElements.Any();
        public abstract IEnumerable<HtmlObject> ChildElements { get; }
        public override string ToString() => OuterHtml;
    }

    public enum HtmlControlType { Input, Select, TextArea }
    public enum HtmlDisabledFlag { Default, Disabled }
    public enum HtmlReadOnlyFlag { Default, ReadOnly }
    public enum HtmlFormMethod { Get, Post }

    public sealed class HapHtmlParser : IHtmlParser
    {
        readonly QueryContext _context;

        public HapHtmlParser(QueryContext context)
        {
            _context = context;
        }

        public ParsedHtml Parse(string html, Uri baseUrl)
        {
            var doc = new HtmlDocument();
            doc.LoadHtml2(html);
            return new HapParsedHtml(doc, baseUrl);
        }

        sealed class HapParsedHtml : ParsedHtml
        {
            readonly HtmlDocument _document;
            readonly ConditionalWeakTable<HtmlNode, HtmlObject> _map = new ConditionalWeakTable<HtmlNode, HtmlObject>();

            public HapParsedHtml(HtmlDocument document, Uri baseUrl) :
                base(baseUrl)
            {
                _document = document;
            }

            HtmlObject GetPublicObject(HtmlNode node)
            {
                HtmlObject obj;
                if (!_map.TryGetValue(node, out obj))
                    _map.Add(node, obj = new HapHtmlObject(node, this));
                return obj;
            }

            public override IEnumerable<HtmlObject> QuerySelectorAll(string selector, HtmlObject context) =>
                (((HapHtmlObject)context)?.Node ?? _document.DocumentNode).QuerySelectorAll(selector).Select(GetPublicObject);

            public override HtmlObject Root => GetPublicObject(_document.DocumentNode);

            sealed class HapHtmlObject : HtmlObject
            {
                readonly HapParsedHtml _owner;

{'fraction_non_alphanumeric': 0.07640562248995984, 'fraction_numerical': 0.0018072289156626507, 'mean_word_length': 2.4289156626506023, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3344766', 'n_tokens_mistral': 2548, 'n_tokens_neox': 2452, 'n_words': 754}