<commit_before>module Ecco
  class LemondeAdapter < Adapter
    def initialize
      @root_uri = 'http://www.lemonde.fr'
      @article_block = get_html(root_uri).css('.titre_une')
      binding.pry
      @article_url = "http://www.lemonde.fr" + article_block.css('a').first.attr('href')
      @full_article_page = get_html(article_url)
    end

    def data
      SiteData.new(
        newspaper: "Le Monde",
        headline: article_block.css('a > h1 > text()').first,
        description: article_block.css('.description').inner_text,
        link: URI.parse(root_uri + article_block.css('a').first.attr('href')),
        image_url: URI.parse(article_block.css('img').first.attr('src')),
        article: full_article_page.css("div[itemprop=articleBody]").text
      )
    end
  end
end<commit_msg>chane to inner text<commit_after>module Ecco
  class LemondeAdapter < Adapter
    def initialize
      @root_uri = 'http://www.lemonde.fr'
      @article_block = get_html(root_uri).css('.titre_une')
      binding.pry
      @article_url = "http://www.lemonde.fr" + article_block.css('a').first.attr('href')
      @full_article_page = get_html(article_url)
    end

    def data
      SiteData.new(
        newspaper: "Le Monde",
        headline: article_block.css('a > h1 > text()').first.inner_text,
        description: article_block.css('.description').inner_text,
        link: URI.parse(root_uri + article_block.css('a').first.attr('href')),
        image_url: URI.parse(article_block.css('img').first.attr('src')),
        article: full_article_page.css("div[itemprop=articleBody]").text
      )
    end
  end
end
{'fraction_non_alphanumeric': 0.148673658235657, 'fraction_numerical': 0.0012338062924120913, 'mean_word_length': 3.212987012987013, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4412611', 'n_tokens_mistral': 545, 'n_tokens_neox': 542, 'n_words': 82}