<commit_before>require 'csv'

EXPORT_VERSION = '1.0.0'
EXPORT_DATE = "2017-06-14"

namespace :open_data do
  desc 'Export all open data csvs'
  task export: [
    :export_projects,
    :export_versions,
    :export_dependencies,
    :export_repositories,
    :export_tags,
    :export_repository_dependencies
  ]

  desc 'Export projects open data csv'
  task export_projects: :environment do
    csv_file = File.open("data/projects-#{EXPORT_VERSION}-#{EXPORT_DATE}.csv",'w')
    csv_file = CSV.new(csv_file)
    csv_file << [
      'Platform',
      'Name',
      'Created Timestamp',
      'Updated Timestamp',
      'Description',
      'Keywords',
      'Homepage URL',
      'Licenses',
      'Repository URL',
      'Versions Count',
      'SourceRank',
      'Latest Release Publish Timestamp',
      'Latest Release Number',
      'Package Manager ID',
      'Dependent Projects Count',
      'Language',
      'Status',
      'Last synced Timestamp',
      'Dependent Repositories Count'
    ]

    Project.not_removed.includes(:repository).find_each do |project|
      csv_file << [
        project.platform,
        project.name,
        project.created_at,
        project.updated_at,
        project.description.try(:tr, "\r\n",' '),
        project.keywords_array.join(','),
        project.homepage.try(:strip),
        project.normalized_licenses.join(','),
        project.repository_url.try(:strip),
        project.versions_count,
        project.rank,
        project.latest_release_published_at,
        project.latest_release_number,
        project.pm_id,
        project.dependents_count,
        project.language,
        project.status,
        project.last_synced_at,
        project.dependent_repos_count,
      ]
    end
  end

  desc 'Export versions open data csv'
  task export_versions: :environment do
    csv_file = File.open("data/versions-#{EXPORT_VERSION}-#{EXPORT_DATE}.csv",'w')
    csv_file = CSV.new(csv_file)
    csv_file << [
      'Platform',
      'Project Name',
      'Number',
      'Published Timestamp',
      'Created Timestamp',
      'Updated Timestamp'
    ]

    Project.not_removed.includes(:versions).find_each do |project|
      project.versions.each do |version|<commit_msg>Remove newlines from project urls<commit_after>require 'csv'

EXPORT_VERSION = '1.0.0'
EXPORT_DATE = "2017-06-14"

namespace :open_data do
  desc 'Export all open data csvs'
  task export: [
    :export_projects,
    :export_versions,
    :export_dependencies,
    :export_repositories,
    :export_tags,
    :export_repository_dependencies
  ]

  desc 'Export projects open data csv'
  task export_projects: :environment do
    csv_file = File.open("data/projects-#{EXPORT_VERSION}-#{EXPORT_DATE}.csv",'w')
    csv_file = CSV.new(csv_file)
    csv_file << [
      'Platform',
      'Name',
      'Created Timestamp',
      'Updated Timestamp',
      'Description',
      'Keywords',
      'Homepage URL',
      'Licenses',
      'Repository URL',
      'Versions Count',
      'SourceRank',
      'Latest Release Publish Timestamp',
      'Latest Release Number',
      'Package Manager ID',
      'Dependent Projects Count',
      'Language',
      'Status',
      'Last synced Timestamp',
      'Dependent Repositories Count'
    ]

    Project.not_removed.includes(:repository).find_each do |project|
      csv_file << [
        project.platform,
        project.name,
        project.created_at,
        project.updated_at,
        project.description.try(:tr, "\r\n",' '),
        project.keywords_array.join(',').try(:tr, "\r\n",' '),
        project.homepage.try(:tr, "\r\n",' ').try(:strip),
        project.normalized_licenses.join(','),
        project.repository_url.try(:tr, "\r\n",' ').try(:strip),
        project.versions_count,
        project.rank,
        project.latest_release_published_at,
        project.latest_release_number,
        project.pm_id,
        project.dependents_count,
        project.language,
        project.status,
        project.last_synced_at,
        project.dependent_repos_count,
      ]
    end
  end

  desc 'Export versions open data csv'
  task export_versions: :environment do
    csv_file = File.open("data/versions-#{EXPORT_VERSION}-#{EXPORT_DATE}.csv",'w')
    csv_file = CSV.new(csv_file)
    csv_file << [
      'Platform',
      'Project Name',
      'Number',
      'Published Timestamp',
      'Created Timestamp',
      'Updated Timestamp'
    ]

    Project.not_removed.includes(:versions).find_each do |project|
      project.versions.each do |version|
{'fraction_non_alphanumeric': 0.12261041529334213, 'fraction_numerical': 0.004834102395078005, 'mean_word_length': 2.6768982229402263, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5415100', 'n_tokens_mistral': 1445, 'n_tokens_neox': 1387, 'n_words': 272}