<commit_before>﻿using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using Newtonsoft.Json;
using RohlikAPI.Helpers;
using RohlikAPI.Model;
using RohlikAPI.Model.JsonDeserialization;

namespace RohlikAPI
{
    internal class Products
    {
        private const string BaseUrl = "https://www.rohlik.cz/";
        private const string BaseSearchUrl = "https://www.rohlik.cz/hledat/";
        private readonly PersistentSessionHttpClient httpClient;
        private readonly PriceParser priceParser = new PriceParser();

        private readonly DateTime currentDate = DateTime.Now;

        internal Products(PersistentSessionHttpClient httpClient)
        {
            this.httpClient = httpClient;
        }

        internal IEnumerable<Product> Get(string category)
        {
            var allProductsString = GetAllProductsString(category, BaseUrl);
            return GetProductsFromHtmlString(allProductsString);
        }

        internal IEnumerable<Product> Search(string searchString)
        {
            var allProductsString = GetAllProductsString(searchString, BaseSearchUrl);
            return GetProductsFromHtmlString(allProductsString);
        }

        private IEnumerable<Product> GetProductsFromHtmlString(string htmlString)
        {
            var allProductsDocument = new HtmlDocument();
            allProductsDocument.LoadHtml(htmlString);

            var parsedProducts = ParseProducts(allProductsDocument);

            return parsedProducts;
        }

        private IEnumerable<Product> ParseProducts(HtmlDocument document)
        {
            var productNodes = document.DocumentNode.SelectNodes(@"//*[@class='product__grid_wrapper']/div[contains(@class,'base_product')]/article/div/div[contains(@class,'product__order')]");
            var parsedProducts = productNodes.AsParallel().Select(GetProductFromNode).Where(p => p != null);
            return parsedProducts;
        }

        private Product GetProductFromNode(HtmlNode productNode)
        {
            if (productNode == null) throw new ArgumentNullException(nameof(productNode));
            if (IsSoldOut(productNode))
            {
                return null;
            }
            var product = new Product();

            var aNode = productNode.SelectSingleNode(".//div/h3/a");

            var rawName = aNode.InnerText.Trim();
            product.Name = WebUtility.HtmlDecode(rawName);

            var imageNode = productNode.SelectSingleNode(".//header/a/img");
            var imageUrlLarge = imageNode.Attributes["data-replace"].Value;
            var imageUrlSmall = imageUrlLarge.Replace("260.jpg", "160.jpg");

            product.ImageUrl = imageUrlSmall;

            const string rohlikUrl = "https://rohlik.cz";
            product.ProductUrl = $"{rohlikUrl}{aNode.Attributes["href"].Value}";

            var priceNode = productNode.SelectSingleNode(".//div/strong[contains(@class,'font-15')]");
            product.Price = priceParser.ParsePrice(priceNode.InnerText);

            var pricePerUnitNode = productNode.SelectSingleNode(".//span[@class='grey font-11']/text()");
            var pricePerUnitString = pricePerUnitNode.InnerText.Trim().Trim('(',')');

            product.PricePerUnit = priceParser.ParsePrice(pricePerUnitString);
            product.Unit = pricePerUnitString.Split(new[] { "&nbsp;" }, StringSplitOptions.None).Last();

            var discountPriceNode = productNode.SelectSingleNode(".//span[@class='grey font-11']/del");

            if (discountPriceNode != null)
            {
                product.IsDiscounted = true;
                product.OriginalPrice = priceParser.ParsePrice(discountPriceNode.InnerText);
                var dateTimeNode = productNode.SelectSingleNode(".//div[@class='action-red action font-13']");
                if (dateTimeNode != null)
                {
                    product.DiscountedUntil = GetDateUntilDiscounted(dateTimeNode);
                }                
            }
            else
            {
                product.IsDiscounted = false;
            }

            return product;
        }

        private bool IsSoldOut(HtmlNode productNode)
        {
            var soldOutMessageNode = productNode.SelectSingleNode(".//div[@class='product__unavailable']");
            return soldOutMessageNode != null;
        }

        private DateTime? GetDateUntilDiscounted(HtmlNode dateTimeNode)
        {
            if (dateTimeNode == null)
            {
                throw new ArgumentNullException(nameof(dateTimeNode));
            }
            var dateTimeString = dateTimeNode.InnerText;
            dateTimeString = dateTimeString.Replace("Akce do ", "");
            dateTimeString = Regex.Replace(dateTimeString, @"\s", "");
            dateTimeString += currentDate.Year;
            
            try
            {
                var dateTime = DateTime.Parse(dateTimeString, new CultureInfo("cs-CZ"));<commit_msg>Fix not available products detection<commit_after>﻿using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using Newtonsoft.Json;
using RohlikAPI.Helpers;
using RohlikAPI.Model;
using RohlikAPI.Model.JsonDeserialization;

namespace RohlikAPI
{
    internal class Products
    {
        private const string BaseUrl = "https://www.rohlik.cz/";
        private const string BaseSearchUrl = "https://www.rohlik.cz/hledat/";
        private readonly PersistentSessionHttpClient httpClient;
        private readonly PriceParser priceParser = new PriceParser();

        private readonly DateTime currentDate = DateTime.Now;

        internal Products(PersistentSessionHttpClient httpClient)
        {
            this.httpClient = httpClient;
        }

        internal IEnumerable<Product> Get(string category)
        {
            var allProductsString = GetAllProductsString(category, BaseUrl);
            return GetProductsFromHtmlString(allProductsString);
        }

        internal IEnumerable<Product> Search(string searchString)
        {
            var allProductsString = GetAllProductsString(searchString, BaseSearchUrl);
            return GetProductsFromHtmlString(allProductsString);
        }

        private IEnumerable<Product> GetProductsFromHtmlString(string htmlString)
        {
            var allProductsDocument = new HtmlDocument();
            allProductsDocument.LoadHtml(htmlString);

            var parsedProducts = ParseProducts(allProductsDocument);

            return parsedProducts;
        }

        private IEnumerable<Product> ParseProducts(HtmlDocument document)
        {
            var productNodes = document.DocumentNode.SelectNodes(@"//*[@class='product__grid_wrapper']/div[contains(@class,'base_product')]/article/div/div[contains(@class,'product__order')]");
            var parsedProducts = productNodes.AsParallel().Select(GetProductFromNode).Where(p => p != null);
            return parsedProducts;
        }

        private Product GetProductFromNode(HtmlNode productNode)
        {
            if (productNode == null) throw new ArgumentNullException(nameof(productNode));
            if (IsSoldOut(productNode))
            {
                return null;
            }
            var product = new Product();

            var aNode = productNode.SelectSingleNode(".//div/h3/a");

            var rawName = aNode.InnerText.Trim();
            product.Name = WebUtility.HtmlDecode(rawName);

            var imageNode = productNode.SelectSingleNode(".//header/a/img");
            var imageUrlLarge = imageNode.Attributes["data-replace"].Value;
            var imageUrlSmall = imageUrlLarge.Replace("260.jpg", "160.jpg");

            product.ImageUrl = imageUrlSmall;

            const string rohlikUrl = "https://rohlik.cz";
            product.ProductUrl = $"{rohlikUrl}{aNode.Attributes["href"].Value}";

            var priceNode = productNode.SelectSingleNode(".//div/strong[contains(@class,'font-15')]");
            product.Price = priceParser.ParsePrice(priceNode.InnerText);

            var pricePerUnitNode = productNode.SelectSingleNode(".//span[@class='grey font-11']/text()");
            var pricePerUnitString = pricePerUnitNode.InnerText.Trim().Trim('(',')');

            product.PricePerUnit = priceParser.ParsePrice(pricePerUnitString);
            product.Unit = pricePerUnitString.Split(new[] { "&nbsp;" }, StringSplitOptions.None).Last();

            var discountPriceNode = productNode.SelectSingleNode(".//span[@class='grey font-11']/del");

            if (discountPriceNode != null)
            {
                product.IsDiscounted = true;
                product.OriginalPrice = priceParser.ParsePrice(discountPriceNode.InnerText);
                var dateTimeNode = productNode.SelectSingleNode(".//div[@class='action-red action font-13']");
                if (dateTimeNode != null)
                {
                    product.DiscountedUntil = GetDateUntilDiscounted(dateTimeNode);
                }                
            }
            else
            {
                product.IsDiscounted = false;
            }

            return product;
        }

        private bool IsSoldOut(HtmlNode productNode)
        {
            const string notAvailableClass = "--not-available";
            return productNode.InnerHtml.Contains(notAvailableClass);
        }

        private DateTime? GetDateUntilDiscounted(HtmlNode dateTimeNode)
        {
            if (dateTimeNode == null)
            {
                throw new ArgumentNullException(nameof(dateTimeNode));
            }
            var dateTimeString = dateTimeNode.InnerText;
            dateTimeString = dateTimeString.Replace("Akce do ", "");
            dateTimeString = Regex.Replace(dateTimeString, @"\s", "");
            dateTimeString += currentDate.Year;
            
            try
            {
                var dateTime = DateTime.Parse(dateTimeString, new CultureInfo("cs-CZ"));
{'fraction_non_alphanumeric': 0.10129437144538145, 'fraction_numerical': 0.0029417532849578347, 'mean_word_length': 2.378270950645909, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 6, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5071002', 'n_tokens_mistral': 2637, 'n_tokens_neox': 2514, 'n_words': 467}