<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using HtmlAgilityPack;

namespace GetQuote
{
  internal class Program
  {
    private static void Main()
    {
      // Create a request for the URL. 
      WebRequest request = WebRequest.Create(
        "http://www.badnuke.com/");
      // If required by the server, set the credentials.
      request.Credentials = CredentialCache.DefaultCredentials;
      // Get the response.
      WebResponse response = request.GetResponse();
      // Display the status.
      Console.WriteLine(((HttpWebResponse)response).StatusDescription);
      // Get the stream containing content returned by the server.
      Stream dataStream = response.GetResponseStream();
      // Open the stream using a StreamReader for easy access.
      StreamReader reader = new StreamReader(dataStream);
      // Read the content.
      string responseFromServer = reader.ReadToEnd();
      // Clean up the streams and the response.
      reader.Close();
      response.Close();
      // Display the content.
      //Console.WriteLine(responseFromServer);
      // parcours du DOM et recherche de <div class="text-center">
      Dictionary<string, string> dicoQuotes = new Dictionary<string, string>();
      var source = WebUtility.HtmlDecode(responseFromServer);
      HtmlDocument resultat = new HtmlDocument();
      resultat.LoadHtml(source);
      List<HtmlNode> quotes = resultat.DocumentNode.Descendants().Where
      (x => x.Name == "div" && x.Attributes["class"] != null &&
            x.Attributes["class"].Value.Contains("text-center")).ToList();
      if (quotes.Count != 0)
      {
        string tmpKey = quotes[0].InnerText.Trim().Trim('"');
        Console.WriteLine(tmpKey);
        string tmpValue = quotes[1].InnerText.Trim().Trim('"');
        Console.WriteLine(tmpValue);
        if (!dicoQuotes.ContainsKey(tmpKey))
        {
          dicoQuotes.Add(tmpKey, tmpValue);
        }
      }
      
      //foreach (HtmlNode htmlNode in quotes)
      //{
      //  string tmp = htmlNode.InnerText.Trim().Trim('"');
      //  Console.WriteLine(tmp);
      //}

      Console.WriteLine("Press a key to exit:");
      Console.ReadKey();
    }<commit_msg>Remove double quotes from web response<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using HtmlAgilityPack;

namespace GetQuote
{
  internal class Program
  {
    private static void Main()
    {
      // Create a request for the URL. 
      WebRequest request = WebRequest.Create(
        "http://www.badnuke.com/");
      // If required by the server, set the credentials.
      request.Credentials = CredentialCache.DefaultCredentials;
      // Get the response.
      WebResponse response = request.GetResponse();
      // Display the status.
      //Console.WriteLine(((HttpWebResponse)response).StatusDescription);
      // Get the stream containing content returned by the server.
      Stream dataStream = response.GetResponseStream();
      // Open the stream using a StreamReader for easy access.
      StreamReader reader = new StreamReader(dataStream);
      // Read the content.
      string responseFromServer = reader.ReadToEnd();
      // Clean up the streams and the response.
      reader.Close();
      response.Close();
      // Display the content.
      //Console.WriteLine(responseFromServer);
      // parcours du DOM et recherche de <div class="text-center">
      Dictionary<string, string> dicoQuotes = new Dictionary<string, string>();
      var source = WebUtility.HtmlDecode(responseFromServer);
      HtmlDocument resultat = new HtmlDocument();
      resultat.LoadHtml(source);
      List<HtmlNode> quotes = resultat.DocumentNode.Descendants().Where
      (x => x.Name == "div" && x.Attributes["class"] != null &&
            x.Attributes["class"].Value.Contains("text-center")).ToList();
      if (quotes.Count != 0)
      {
        string tmpKey = quotes[0].InnerText.Trim();
        tmpKey = tmpKey.Replace('"', ' ').Trim();
        if (tmpKey.StartsWith('"'.ToString()))
        {
          tmpKey = tmpKey.Substring(1, tmpKey.Length - 2);
        }
        Console.WriteLine(tmpKey);
        string tmpValue = quotes[1].InnerText.Trim();
        tmpValue = tmpValue.Replace('"', ' ').Trim();
        Console.WriteLine(tmpValue);
        if (!dicoQuotes.ContainsKey(tmpKey))
        {
          dicoQuotes.Add(tmpKey, tmpValue);
        }
      }
      
      //foreach (HtmlNode htmlNode in quotes)
      //{
      //  string tmp = htmlNode.InnerText.Trim().Trim('"');
      //  Console.WriteLine(tmp);
      //}

      Console.WriteLine("Press a key to exit:");
      Console.ReadKey();
    }
{'fraction_non_alphanumeric': 0.12581390464188197, 'fraction_numerical': 0.0016803192606595252, 'mean_word_length': 2.6048448145344434, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7348086', 'n_tokens_mistral': 1310, 'n_tokens_neox': 1292, 'n_words': 329}