<commit_before>#!/usr/bin/env ruby

require 'open-uri'
require 'nokogiri'
require 'tess_api_client'
require 'geocoder'


$root_url = 'https://www.csc.fi/web/training/'
$owner_org = 'csc'
$events = {}
$debug = ScraperConfig.debug?

def parse_data()

    if $debug
      puts 'Opening local file.'
      begin
        f = File.open("html/csc_events.html")
        doc = Nokogiri::HTML(f)
        f.close
      rescue
        puts "Failed to open csc_events.html file."
      end
    else
      puts "Opening: #{$root_url}"
      doc = Nokogiri::HTML(open($root_url))
    end

    doc.css('div.csc-article').each do |article|
      categories = article.search('.koulutus-category').collect {|x| x.text.strip}.reject(&:empty?)
      header = article.search('h3 > a')[0]
      link = header['href']
      title = header.text
      description = article.search('.article-summary')[0].text

      $events[link] = {'title' => title,
                       'description' => description,
                       'categories' => categories,
                       'keywords' => categories}
    end
end


##################################################
# Main body of the script below, functions above #
##################################################

cp = ContentProvider.new(
    "CSC - IT Center for Science",
    "https://www.csc.fi",
    "https://www.csc.fi/documents/10180/161914/CSC_2012_LOGO_RGB_72dpi.jpg/c65ddc42-63fc-44da-8d0f-9f88c54779d7?t=1411391121769",
    "CSC - IT Center for Science Ltd. is a non-profit, state-owned company administered by the Finnish Ministry of Education and Culture. CSC maintains and develops the state-owned centralised IT infrastructure and uses it to provide nationwide IT services for research, libraries, archives, museums and culture as well as information, education and research management. 
    CSC has the task of promoting the operational framework of Finnish research, education, culture and administration. As a non-profit, government organisation, it is our duty to foster exemplary transparency, honesty and responsibility. Trust is the foundation of CSC's success. Customers, suppliers, owners and personnel alike must feel certain that we will fulfil our commitments and promises in an ethically sustainable manner.
    CSC has offices in Espoo's Keilaniemi and in the Renforsin Ranta business park in Kajaani."
    )
cp = Uploader.create_or_update_content_provider(cp)

# Actually run the code here...
parse_data
<commit_msg>Modified CSC scraper to use the new content provider and nodes API.<commit_after>#!/usr/bin/env ruby

require 'open-uri'
require 'nokogiri'
require 'tess_api_client'
require 'geocoder'


$root_url = 'https://www.csc.fi/web/training/'
$owner_org = 'csc'
$events = {}
$debug = ScraperConfig.debug?

def parse_data()

    if $debug
      puts 'Opening local file.'
      begin
        f = File.open("html/csc_events.html")
        doc = Nokogiri::HTML(f)
        f.close
      rescue
        puts "Failed to open csc_events.html file."
      end
    else
      puts "Opening: #{$root_url}"
      doc = Nokogiri::HTML(open($root_url))
    end

    doc.css('div.csc-article').each do |article|
      categories = article.search('.koulutus-category').collect {|x| x.text.strip}.reject(&:empty?)
      header = article.search('h3 > a')[0]
      link = header['href']
      title = header.text
      description = article.search('.article-summary')[0].text

      $events[link] = {'title' => title,
                       'description' => description,
                       'categories' => categories,
                       'keywords' => categories}
    end
end


##################################################
# Main body of the script below, functions above #
##################################################

cp = ContentProvider.new({
                             title: "CSC - IT Center for Science",
                             url: "https://www.csc.fi",
                             image_url: "https://www.csc.fi/documents/10180/161914/CSC_2012_LOGO_RGB_72dpi.jpg/c65ddc42-63fc-44da-8d0f-9f88c54779d7?t=1411391121769",
                             description: "CSC - IT Center for Science Ltd. is a non-profit, state-owned company administered by the Finnish Ministry of Education and Culture. CSC maintains and develops the state-owned centralised IT infrastructure and uses it to provide nationwide IT services for research, libraries, archives, museums and culture as well as information, education and research management.
    CSC has the task of promoting the operational framework of Finnish research, education, culture and administration. As a non-profit, government organisation, it is our duty to foster exemplary transparency, honesty and responsibility. Trust is the foundation of CSC's success. Customers, suppliers, owners and personnel alike must feel certain that we will fulfil our commitments and promises in an ethically sustainable manner.
    CSC has offices in Espoo's Keilaniemi and in the Renforsin Ranta business park in Kajaani.",
                             content_provider_type: ContentProvider::PROVIDER_TYPE[:ORGANISATION],
                             node: Node::NODE_NAMES[:FI],
                         })
cp = Uploader.create_or_update_content_provider(cp)

# Actually run the code here...
parse_data

{'fraction_non_alphanumeric': 0.13143071723619978, 'fraction_numerical': 0.01952684941794968, 'mean_word_length': 3.032551097653293, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 6, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5409225', 'n_tokens_mistral': 1570, 'n_tokens_neox': 1464, 'n_words': 471}