<commit_before>require 'open-uri'
tgac_mat = open 'https://tess.elixir-uk.org/api/3/action/organization_show?id=the-genome-analysis-centre&include_datasets=true', {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}
tgac_mats = JSON.parse(tgac_mat.read)

provider = tgac_mats['result']
materials = provider['packages']

cp = ContentProvider.new(
  provider['display_name'], #name
  provider['homepage'], #url
  provider['image_display_url'], #logo
  provider['description'], #description
)
cp.Uploader.create_or_update_content_provider(cp)
<commit_msg>completed TGAC transfer script using the two TeSS APIs<commit_after>require 'tess_api'
require 'open-uri'
tgac_mat = open 'https://tess.elixir-uk.org/api/3/action/organization_show?id=the-genome-analysis-centre&include_datasets=true', {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}
tgac_mats = JSON.parse(tgac_mat.read)

provider = tgac_mats['result']
materials = provider['packages']

cp = ContentProvider.new(
  provider['display_name'], #name
  provider['homepage'], #url
  provider['image_display_url'], #logo
  provider['description'], #description
)
cp = Uploader.create_or_update_content_provider(cp)

provider['packages'].each do |package|
  material = open "https://tess.elixir-uk.org/api/3/action/package_show?id=#{package['name']}", {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE} 
  material = JSON.parse(material.read) if material 
  material = material['result']
  new_material = Material.new
  new_material.title = material['title']  
  new_material.url = material['url']
  new_material.short_description = material['notes']
  new_material.keywords = material['tags']
  new_material.content_provider_id = cp['id']
  Uploader.create_or_update_material(new_material)
end


{'fraction_non_alphanumeric': 0.14269141531322505, 'fraction_numerical': 0.0017401392111368909, 'mean_word_length': 7.214285714285714, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3255590', 'n_tokens_mistral': 590, 'n_tokens_neox': 575, 'n_words': 87}