<commit_before>require 'tess_api_client'
require 'linkeddata'

# This scraper should use the XML API to get the URL of each course, then go to each individual
# course page to parse embedded RDFa data.

$materials = 'http://www.france-bioinformatique.fr/en/training_material'
$events = 'http://www.france-bioinformatique.fr/en/formations'
$root_url = 'http://www.france-bioinformatique.fr/en/'
$owner_org = 'ifb'
$lessons = {}
$debug = ScraperConfig.debug?

cp = ContentProvider.new({
                             title: "IFB French Institute of Bioinformatics",
                             url: "http://www.france-bioinformatique.fr/en",
                             image_url: "http://www.france-bioinformatique.fr/sites/default/files/ifb-logo_1.png",
                             description: "The French Institute of Bioinformatics (CNRS IFB) is a national service infrastructure in bioinformatics. IFB’s principal mission is to provide basic services and resources in bioinformatics for scientists and engineers working in the life sciences. IFB is the French node of the European research infrastructure, ELIXIR.",
                             content_provider_type: ContentProvider::PROVIDER_TYPE[:ORGANISATION],
                             node: Node::NODE_NAMES[:FR],
                             keywords: ['bioinformatics', 'infrastructure', 'Big Data', 'NGS']
                         })

cp = Uploader.create_or_update_content_provider(cp)

dump_file = File.open('parsed_ifb.json', 'w') if $debug

#Go through each Training Material, load RDFa, dump to JSON, interogate data, and upload to TeSS. 
rdfa = RDF::Graph.load($materials, format: :rdfa)
materials = RdfaExtractor.parse_rdfa(rdfa, 'CreativeWork')
#article = RdfaExtractor.parse_rdfa(rdfa, 'Article')


#write out to JSON for debug mode.
if $debug
    materials.each do |material|
        dump_file.write("#{material.to_json}")
    end
end

  
  # Create the new record
materials.each do |material|
    begin
        keywords = material['schema:keywords']
        keywords.delete('en') #Each has en meaning english in. Remove these
        upload_material = Material.new({
              title: material['schema:name'],
              url: material['@id'],
              short_description: material['schema:about'],
              doi: nil,
              remote_updated_date: Time.now,
              remote_created_date: material['dc:date'],
              content_provider_id: cp['id'],
              scientific_topic_names: keywords,
              keywords: keywords, #material['schema:learningResourceType'],
              licence: nil,
              difficulty_level: nil,
              contributors: [],
              authors: material['schema:author'],
              target_audience: material['schema:audience']
        })
        print "MATERIAL: #{material.inspect}" if $debug
        Uploader.create_or_update_material(upload_material)
    rescue => ex<commit_msg>Singular keywords weren't getting added. Cast as array<commit_after>require 'tess_api_client'
require 'linkeddata'

# This scraper should use the XML API to get the URL of each course, then go to each individual
# course page to parse embedded RDFa data.

$materials = 'http://www.france-bioinformatique.fr/en/training_material'
$events = 'http://www.france-bioinformatique.fr/en/formations'
$root_url = 'http://www.france-bioinformatique.fr/en/'
$owner_org = 'ifb'
$lessons = {}
$debug = ScraperConfig.debug?

cp = ContentProvider.new({
                             title: "IFB French Institute of Bioinformatics",
                             url: "http://www.france-bioinformatique.fr/en",
                             image_url: "http://www.france-bioinformatique.fr/sites/default/files/ifb-logo_1.png",
                             description: "The French Institute of Bioinformatics (CNRS IFB) is a national service infrastructure in bioinformatics. IFB’s principal mission is to provide basic services and resources in bioinformatics for scientists and engineers working in the life sciences. IFB is the French node of the European research infrastructure, ELIXIR.",
                             content_provider_type: ContentProvider::PROVIDER_TYPE[:ORGANISATION],
                             node: Node::NODE_NAMES[:FR],
                             keywords: ['bioinformatics', 'infrastructure', 'Big Data', 'NGS']
                         })

cp = Uploader.create_or_update_content_provider(cp)

dump_file = File.open('parsed_ifb.json', 'w') if $debug

#Go through each Training Material, load RDFa, dump to JSON, interogate data, and upload to TeSS. 
rdfa = RDF::Graph.load($materials, format: :rdfa)
materials = RdfaExtractor.parse_rdfa(rdfa, 'CreativeWork')
#article = RdfaExtractor.parse_rdfa(rdfa, 'Article')


#write out to JSON for debug mode.
if $debug
    materials.each do |material|
        dump_file.write("#{material.to_json}")
    end
end

  
  # Create the new record
materials.each do |material|
    begin
        keywords = material['schema:keywords']
        keywords.delete('en') #Each has en meaning english in. Remove these
        upload_material = Material.new({
              title: material['schema:name'],
              url: material['@id'],
              short_description: material['schema:about'],
              doi: nil,
              remote_updated_date: Time.now,
              remote_created_date: material['dc:date'],
              content_provider_id: cp['id'],
              scientific_topic_names: [keywords].flatten,
              keywords: [keywords].flatten, #material['schema:learningResourceType'],
              licence: nil,
              difficulty_level: nil,
              contributors: [],
              authors: material['schema:author'],
              target_audience: material['schema:audience']
        })
        print "MATERIAL: #{material.inspect}" if $debug
        Uploader.create_or_update_material(upload_material)
    rescue => ex
{'fraction_non_alphanumeric': 0.10328006728343145, 'fraction_numerical': 0.000336417157275021, 'mean_word_length': 2.6500920810313073, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 10, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3470529', 'n_tokens_mistral': 1672, 'n_tokens_neox': 1624, 'n_words': 436}