<commit_before>﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using Newtonsoft.Json;
using RohlikAPI.Model;

namespace RohlikAPI
{
    public class Products
    {
        private const string BaseUrl = "https://www.rohlik.cz/";
        private readonly PersistentSessionHttpClient httpClient;

        internal Products(PersistentSessionHttpClient httpClient)
        {
            this.httpClient = httpClient;
        }

        internal IEnumerable<Product> Get(string category)
        {
            var allProductsString = GetAllProductsString(category);
            var allProductsDocument = new HtmlDocument();
            allProductsDocument.LoadHtml(allProductsString);

            var parsedProducts = ParseProducts(allProductsDocument);

            return parsedProducts;
        }

        private IEnumerable<Product> ParseProducts(HtmlDocument document)
        {
            var productNodes = document.DocumentNode.SelectNodes(@"//*[@class='productGridWrapper']/div[contains(@class,'baseProduct')]//div[@class='productContainerWrap']");
            var parsedProducts = productNodes.Select(GetProductFromNode).ToList();
            parsedProducts.RemoveAll(p => p == null);
            return parsedProducts;
        }

        private Product GetProductFromNode(HtmlNode productNode)
        {
            if (productNode == null) throw new ArgumentNullException(nameof(productNode));
            if (IsSoldOut(productNode))
            {
                return null;
            }
            var product = new Product();

            var aNode = productNode.SelectSingleNode("*/h3/a");
            
            product.Name = aNode.InnerText;

            const string rohlikUrl = "https://rohlik.cz";
            product.ProductUrl = $"{rohlikUrl}{aNode.Attributes["href"].Value}";

            var priceNode = productNode.SelectSingleNode("*/div/h6/strong");
            product.Price = ParsePrice(priceNode);

            var discountPriceNode = productNode.SelectSingleNode("*/div/h6/del");

            if (discountPriceNode != null)
            {
                product.IsDiscounted = true;
                product.OriginalPrice = ParsePrice(discountPriceNode);
                var dateTimeNode = productNode.SelectSingleNode(".//*[@class='center-content-bot']");
                product.DiscountedUntil = GetDateUntilDiscounted(dateTimeNode);
            }
            else
            {
                product.IsDiscounted = false;
            }

            return product;
        }

        private bool IsSoldOut(HtmlNode productNode)
        {
            var soldOutMessageNode = productNode.SelectSingleNode(".//*[@class='naMessage']");
            return soldOutMessageNode != null;
        }

        private double ParsePrice(HtmlNode priceNode)
        {
            if (priceNode == null) throw new ArgumentNullException(nameof(priceNode));

            var priceString = priceNode.InnerText;

            var cleanPriceString = Regex.Match(priceString, @"\d*?,\d*").Value;
            try
            {
                var price = double.Parse(cleanPriceString);
                return price;
            }
            catch (Exception ex)
            {
                throw new Exception($"Failed to parse product price from string '{priceString}'. Exception: {ex}");
            }
        }

        private DateTime GetDateUntilDiscounted(HtmlNode dateTimeNode)
        {
            var dateTimeString = dateTimeNode.InnerText;
            dateTimeString = dateTimeString.Replace("Do", "");
            dateTimeString = Regex.Replace(dateTimeString, @"\s", "");
            return DateTime.Parse(dateTimeString);
        }
<commit_msg>Improved dateTime parsing on different culture systems<commit_after>﻿using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Net.Http;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using Newtonsoft.Json;
using RohlikAPI.Model;

namespace RohlikAPI
{
    public class Products
    {
        private const string BaseUrl = "https://www.rohlik.cz/";
        private readonly PersistentSessionHttpClient httpClient;

        internal Products(PersistentSessionHttpClient httpClient)
        {
            this.httpClient = httpClient;
        }

        internal IEnumerable<Product> Get(string category)
        {
            var allProductsString = GetAllProductsString(category);
            var allProductsDocument = new HtmlDocument();
            allProductsDocument.LoadHtml(allProductsString);

            var parsedProducts = ParseProducts(allProductsDocument);

            return parsedProducts;
        }

        private IEnumerable<Product> ParseProducts(HtmlDocument document)
        {
            var productNodes = document.DocumentNode.SelectNodes(@"//*[@class='productGridWrapper']/div[contains(@class,'baseProduct')]//div[@class='productContainerWrap']");
            var parsedProducts = productNodes.Select(GetProductFromNode).ToList();
            parsedProducts.RemoveAll(p => p == null);
            return parsedProducts;
        }

        private Product GetProductFromNode(HtmlNode productNode)
        {
            if (productNode == null) throw new ArgumentNullException(nameof(productNode));
            if (IsSoldOut(productNode))
            {
                return null;
            }
            var product = new Product();

            var aNode = productNode.SelectSingleNode("*/h3/a");
            
            product.Name = aNode.InnerText;

            const string rohlikUrl = "https://rohlik.cz";
            product.ProductUrl = $"{rohlikUrl}{aNode.Attributes["href"].Value}";

            var priceNode = productNode.SelectSingleNode("*/div/h6/strong");
            product.Price = ParsePrice(priceNode);

            var discountPriceNode = productNode.SelectSingleNode("*/div/h6/del");

            if (discountPriceNode != null)
            {
                product.IsDiscounted = true;
                product.OriginalPrice = ParsePrice(discountPriceNode);
                var dateTimeNode = productNode.SelectSingleNode(".//*[@class='center-content-bot']");
                product.DiscountedUntil = GetDateUntilDiscounted(dateTimeNode);
            }
            else
            {
                product.IsDiscounted = false;
            }

            return product;
        }

        private bool IsSoldOut(HtmlNode productNode)
        {
            var soldOutMessageNode = productNode.SelectSingleNode(".//*[@class='naMessage']");
            return soldOutMessageNode != null;
        }

        private double ParsePrice(HtmlNode priceNode)
        {
            if (priceNode == null) throw new ArgumentNullException(nameof(priceNode));

            var priceString = priceNode.InnerText;

            var cleanPriceString = Regex.Match(priceString, @"\d*?,\d*").Value;
            try
            {
                var price = double.Parse(cleanPriceString);
                return price;
            }
            catch (Exception ex)
            {
                throw new Exception($"Failed to parse product price from string '{priceString}'. Exception: {ex}");
            }
        }

        private DateTime GetDateUntilDiscounted(HtmlNode dateTimeNode)
        {
            var dateTimeString = dateTimeNode.InnerText;
            dateTimeString = dateTimeString.Replace("Do", "");
            dateTimeString = Regex.Replace(dateTimeString, @"\s", "");
            try
            {
                var dateTime = DateTime.Parse(dateTimeString, CultureInfo.CurrentCulture);
                return dateTime;
            }
            catch (FormatException ex)
            {
                throw new FormatException($"Failed to parse datetime string: {dateTimeString}. Culture was {CultureInfo.CurrentCulture}. Ex: {ex}");
            }
        }

{'fraction_non_alphanumeric': 0.09661654135338346, 'fraction_numerical': 0.0007518796992481203, 'mean_word_length': 2.1054474708171207, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1031420', 'n_tokens_mistral': 2019, 'n_tokens_neox': 1939, 'n_words': 395}