<commit_before>require 'nokogiri'

class DataCarpentryScraper < Tess::Scrapers::Scraper
  # For more details of the API and available terms, see:
  # https://building.coursera.org/app-platform/catalog/

  def self.config
    {
        name: 'Data Carpentry Scraper',
        offline_url_mapping: {},
        root_url: 'http://www.datacarpentry.org',
    }
  end

  def scrape
    cp = add_content_provider(Tess::API::ContentProvider.new(
        { title: "Data Carpentry",
          url: "http://www.datacarpentry.org",
          image_url: "http://www.datacarpentry.org/assets/img/DC_logo_vision.png",
          description: "Data Carpentry's aim is to teach researchers basic concepts, skills, and tools for working with data so that they can get more done in less time, and with less pain.",
          content_provider_type: Tess::API::ContentProvider::PROVIDER_TYPE[:ORGANISATION]
        }))

    doc = Nokogiri::HTML(open_url(config[:root_url] + '/lessons'))
    exclude = [' Feeling Responsive']
    lessons = {}
    description = ''
    
    #first = doc.css('div.item-list').search('li')
    lesson_urls = doc.css('td > a').collect { |x| x.values }.flatten.select { |x| x.include?('github.io') }

    lesson_urls.each do |lesson_url|
      lesson = Nokogiri::HTML(open(lesson_url))
      title = lesson.css('h1').text
      unless exclude.include?(title)
        if title.empty?
          title = lesson.css('p').first.text.gsub('<p>#', '').gsub('</p>', '').gsub('#', '')
          description = lesson.css('p')[2]
        else
          description = lesson.css('p')[0].text
          if description == '======='
            description = lesson.css('p')[1].text
          end
        end
        descriptions = []
        index = 0
        while !description.include?('Content Contributors') and index < 5 do
          description = lesson.css('p')[index=index+1]
          descriptions << lesson.css('p')[index]
        end
        lessons[lesson_url] = {}
        lessons[lesson_url]['short_description'] = descriptions.join(' ')
        lessons[lesson_url]['long_description'] = lesson.css('p h1 a')
        lessons[lesson_url]['title'] = title
      end
    end

    lessons.each do |url, data|
      add_material(Tess::API::Material.new(
          { title: data['title'],
            url: url,
            short_description: data['short_description'],
            remote_updated_date: Time.now,
            remote_created_date: nil,
            content_provider_id: cp['id'],
            long_description: data['long_description']}))
    end
  end
end<commit_msg>Link DC content provider by object rather than ID<commit_after>require 'nokogiri'

class DataCarpentryScraper < Tess::Scrapers::Scraper
  # For more details of the API and available terms, see:
  # https://building.coursera.org/app-platform/catalog/

  def self.config
    {
        name: 'Data Carpentry Scraper',
        offline_url_mapping: {},
        root_url: 'http://www.datacarpentry.org',
    }
  end

  def scrape
    cp = add_content_provider(Tess::API::ContentProvider.new(
        { title: "Data Carpentry",
          url: "http://www.datacarpentry.org",
          image_url: "http://www.datacarpentry.org/assets/img/DC_logo_vision.png",
          description: "Data Carpentry's aim is to teach researchers basic concepts, skills, and tools for working with data so that they can get more done in less time, and with less pain.",
          content_provider_type: Tess::API::ContentProvider::PROVIDER_TYPE[:ORGANISATION]
        }))

    doc = Nokogiri::HTML(open_url(config[:root_url] + '/lessons'))
    exclude = [' Feeling Responsive']
    lessons = {}
    description = ''
    
    #first = doc.css('div.item-list').search('li')
    lesson_urls = doc.css('td > a').collect { |x| x.values }.flatten.select { |x| x.include?('github.io') }

    lesson_urls.each do |lesson_url|
      lesson = Nokogiri::HTML(open(lesson_url))
      title = lesson.css('h1').text
      unless exclude.include?(title)
        if title.empty?
          title = lesson.css('p').first.text.gsub('<p>#', '').gsub('</p>', '').gsub('#', '')
          description = lesson.css('p')[2]
        else
          description = lesson.css('p')[0].text
          if description == '======='
            description = lesson.css('p')[1].text
          end
        end
        descriptions = []
        index = 0
        while !description.include?('Content Contributors') and index < 5 do
          description = lesson.css('p')[index=index+1]
          descriptions << lesson.css('p')[index]
        end
        lessons[lesson_url] = {}
        lessons[lesson_url]['short_description'] = descriptions.join(' ')
        lessons[lesson_url]['long_description'] = lesson.css('p h1 a')
        lessons[lesson_url]['title'] = title
      end
    end

    lessons.each do |url, data|
      add_material(Tess::API::Material.new(
          { title: data['title'],
            url: url,
            short_description: data['short_description'],
            remote_updated_date: Time.now,
            remote_created_date: nil,
            content_provider: cp,
            long_description: data['long_description']}))
    end
  end
end
{'fraction_non_alphanumeric': 0.14426229508196722, 'fraction_numerical': 0.003085824493731919, 'mean_word_length': 2.6444132115249475, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3383934', 'n_tokens_mistral': 1599, 'n_tokens_neox': 1590, 'n_words': 331}