<commit_before>class Repositories
  class Npm
    def self.project_names
      HTTParty.get("https://registry.npmjs.org/-/all/").parsed_response.keys[1..-1]
    end

    def self.project(name)
      HTTParty.get("http://registry.npmjs.org/#{name}").parsed_response
    end

    def self.keys
      ["_id", "_rev", "name", "dist-tags", "versions", "readme", "maintainers", "time", "readmeFilename", "license", "users", "_attachments"]
    end

    def self.mapping(project)
      {
        :name => project["name"],
        :published_at => project["time"],
      }
    end

    # TODO repo, authors, versions, licenses
  end
end<commit_msg>NPM importer<commit_after>class Repositories
  class Npm
    PLATFORM = 'npm'

    def self.project_names
      HTTParty.get("https://registry.npmjs.org/-/all/").parsed_response.keys[1..-1]
    end

    def self.project(name)
      HTTParty.get("http://registry.npmjs.org/#{name}").parsed_response
    end

    def self.keys
      ["_id", "_rev", "name", "description", "dist-tags", "versions", "readme", "maintainers", "time", "author", "repository", "users", "homepage", "keywords", "bugs", "readmeFilename", "_attachments"]
    end

    def self.mapping(project)
      {
        :name => project["name"],
        :description => project["description"],
        :homepage => project["homepage"],
        :keywords => project["keywords"].join(',')
      }
    end

    def self.save(project)
      mapped_project = mapping(project)
      project = Project.find_or_create_by({:name => mapped_project[:name], :platform => PLATFORM})
      project.update_attributes(mapped_project.slice(:description, :homepage, :keywords))
      project
    end

    def self.update(name)
      save(project(name))
    end

    # TODO repo, authors, versions, licenses
  end
end
{'fraction_non_alphanumeric': 0.16277777777777777, 'fraction_numerical': 0.0022222222222222222, 'mean_word_length': 3.1402298850574715, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '479094', 'n_tokens_mistral': 585, 'n_tokens_neox': 558, 'n_words': 111}