<commit_before>module RepoManifests
  def download_manifests(token = nil)
    return unless host_type == 'GitHub'
    body = parse_manifests(token)

    if body
      new_manifests = body["manifests"]
      sync_metadata(body)
    else
      new_manifests = nil
    end

    return if new_manifests.nil?

    new_manifests.each {|m| sync_manifest(m) }

    delete_old_manifests(new_manifests)

    repository_subscriptions.each(&:update_subscriptions)
  end

  def parse_manifests(token)
    r = Typhoeus::Request.new("https://librarian.libraries.io/v2/repos/#{full_name}",
      method: :get,
      params: { token: token },
      headers: { 'Accept' => 'application/json' }).run
    begin
      Oj.load(r.body)
    rescue Oj::ParseError
      nil
    end
  end

  def sync_metadata(body)
    if body && body['metadata']
      meta = body['metadata']

      self.has_readme       = meta['readme']['path']        if meta['readme']
      self.has_changelog    = meta['changelog']['path']     if meta['changelog']
      self.has_contributing = meta['contributing']['path']  if meta['contributing']
      self.has_license      = meta['license']['path']       if meta['license']
      self.has_coc          = meta['codeofconduct']['path'] if meta['codeofconduct']
      self.has_threat_model = meta['threatmodel']['path']   if meta['threatmodel']
      self.has_audit        = meta['audit']['path']         if meta['audit']

      save! if self.changed?
    end
  end

  def sync_manifest(m)
    args = {platform: m['platform'], kind: m['type'], filepath: m['filepath'], sha: m['sha']}

    unless manifests.find_by(args)
      manifest = manifests.create(args)
      dependencies = m['dependencies'].uniq{|dep| [dep['name'].try(:strip), dep['version'], dep['type']]}
      dependencies.each do |dep|
        platform = manifest.platform
        next unless dep.is_a?(Hash)
        project = Project.platform(platform).find_by_name(dep['name'])

        manifest.repository_dependencies.create({
          project_id: project.try(:id),
          project_name: dep['name'].try(:strip),
          platform: platform,
          requirements: dep['version'],
          kind: dep['type']
        })
      end
    end
  end

  def delete_old_manifests(new_manifests)
    existing_manifests = manifests.map{|m| [m.platform, m.filepath] }
    to_be_removed = existing_manifests - new_manifests.map{|m| [m["platform"], m["filepath"]] }
    to_be_removed.each do |m|
      manifests.where(platform: m[0], filepath: m[1]).each(&:destroy)
    end
    manifests.where.not(id: manifests.latest.map(&:id)).each(&:destroy)
  end
end
<commit_msg>Mostly working github dependency parsing with bibliothecary<commit_after>module RepoManifests
  def download_manifests(token = nil)
    return unless host_type == 'GitHub'

    file_list = get_file_list(token)
    return if file_list.empty?
    new_manifests = parse_manifests(file_list, token)
    sync_metadata(file_list.map{|file| file.path })

    return if new_manifests.empty?

    new_manifests.each {|m| sync_manifest(m) }

    delete_old_manifests(new_manifests)

    repository_subscriptions.each(&:update_subscriptions)
  end

  def get_file_list(token)
    tree = AuthToken.fallback_client(token).tree(full_name, default_branch, :recursive => true).tree
    tree.select{|item| item.type == 'blob' }
  end

  def parse_manifests(file_list, token)
    manifest_paths = Bibliothecary.identify_manifests(file_list.map{|file| file.path })

    manifest_paths.map do |manifest_path|
      contents = Base64.decode64 AuthToken.fallback_client(token).contents(full_name, path: manifest_path).content
      Bibliothecary.analyse_file(manifest_path, contents).first
    end.reject(&:empty?)
  end

  def sync_metadata(file_list)
    self.has_readme       = file_list.any?{|file| file.match(/^README/i) }
    self.has_changelog    = file_list.any?{|file| file.match(/^CHANGELOG/i) }
    self.has_contributing = file_list.any?{|file| file.match(/^CONTRIBUTING/i) }
    self.has_license      = file_list.any?{|file| file.match(/^LICENSE/i) }
    self.has_coc          = file_list.any?{|file| file.match(/^CODE[-_]OF[-_]CONDUCT/i) }
    self.has_threat_model = file_list.any?{|file| file.match(/^THREAT[-_]MODEL/i) }
    self.has_audit        = file_list.any?{|file| file.match(/^AUDIT/i) }
    save if self.changed?
  end

  def sync_manifest(m)
    args = {platform: m[:platform], kind: m[:type], filepath: m[:path]}

    unless manifests.find_by(args)
      manifest = manifests.create(args)
      dependencies = m[:dependencies].uniq{|dep| [dep[:name].try(:strip), dep[:requirement], dep[:type]]}
      dependencies.each do |dep|
        platform = manifest.platform
        next unless dep.is_a?(Hash)
        project = Project.platform(platform).find_by_name(dep[:name])

        manifest.repository_dependencies.create({
          project_id: project.try(:id),
          project_name: dep[:name].try(:strip),
          platform: platform,
          requirements: dep[:requirement],
          kind: dep[:type]
        })
      end
    end
  end

  def delete_old_manifests(new_manifests)
    existing_manifests = manifests.map{|m| [m.platform, m.filepath] }
    to_be_removed = existing_manifests - new_manifests.map{|m| [m["platform"], m["filepath"]] }
    to_be_removed.each do |m|
      manifests.where(platform: m[0], filepath: m[1]).each(&:destroy)
    end
    manifests.where.not(id: manifests.latest.map(&:id)).each(&:destroy)
  end
end

{'fraction_non_alphanumeric': 0.14032523296181254, 'fraction_numerical': 0.0016444363237712407, 'mean_word_length': 3.410958904109589, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5756311', 'n_tokens_mistral': 1874, 'n_tokens_neox': 1782, 'n_words': 311}