<commit_before>module RepositoryOwner
  class Gitlab < Base
    def avatar_url(size = 60)
      "https://gitlab.com/uploads/user/avatar/#{owner.uuid}/avatar.png"
    end

    def repository_url
      "https://gitlab.com/#{owner.login}"
    end

    def self.fetch_user(id_or_login)
      if id_or_login.to_i.zero?
        api_client.get("/users?username=#{id_or_login}").first
      else
        api_client.user(id_or_login)
      end
    rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS
      nil
    end

    def self.fetch_org(id_or_login)
      # TODO
    end

    def self.api_client(token = nil)
      ::Gitlab.client(endpoint: 'https://gitlab.com/api/v3', private_token: token || ENV['GITLAB_KEY'])
    end

    def api_client(token = nil)
      self.class.api_client(token)
    end

    def download_orgs
      # TODO
    end

    def download_repos
      # TODO
    end

    def download_members
      return unless owner.org?

      api_client.group_members(owner.login).each do |org|
        RepositoryCreateUserWorker.perform_async('GitLab', org.login)
      end<commit_msg>Scraping of orgs and repos for gitlab users<commit_after>module RepositoryOwner
  class Gitlab < Base
    def avatar_url(size = 60)
      "https://gitlab.com/uploads/user/avatar/#{owner.uuid}/avatar.png"
    end

    def repository_url
      "https://gitlab.com/#{owner.login}"
    end

    def self.fetch_user(id_or_login)
      if id_or_login.to_i.zero?
        api_client.get("/users?username=#{id_or_login}").first
      else
        api_client.user(id_or_login)
      end
    rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS
      nil
    end

    def self.fetch_org(id_or_login)
      # TODO
    end

    def self.api_client(token = nil)
      ::Gitlab.client(endpoint: 'https://gitlab.com/api/v3', private_token: token || ENV['GITLAB_KEY'])
    end

    def api_client(token = nil)
      self.class.api_client(token)
    end

    def download_orgs
      return if owner.org?

      # GitLab doesn't have an API to get a users public group memberships so we scrape it instead
      groups_html = Nokogiri::HTML(PackageManager::Base.get_json("https://gitlab.com/users/#{owner.login}/groups")['html'])
      links = groups_html.css('a.group-name').map{|l| l['href'][1..-1]}

      links.each do |org_login|
        RepositoryCreateOrgWorker.perform_async('GitLab', org_login)
      end
      true
    rescue *RepositoryHost::Gitlab::IGNORABLE_EXCEPTIONS
      nil
    end

    def download_repos
      if owner.org?
        repos = api_client.group_projects(owner.login).map(&:full_name)
      else
        # GitLab doesn't have an API to get a users public projects so we scrape it instead
        projects_html = Nokogiri::HTML(PackageManager::Base.get_json("https://gitlab.com/users/#{owner.login}/projects")['html'])
        links = projects_html.css('a.project').map{|l| l['href'][1..-1] }.uniq
      end

      links.each do |repo_name|
        CreateRepositoryWorker.perform_async('GitLab', repo_name)
      end
      true
    rescue *RepositoryHost::Github::IGNORABLE_EXCEPTIONS
      nil
    end

    def download_members
      return unless owner.org?

      api_client.group_members(owner.login).each do |org|
        RepositoryCreateUserWorker.perform_async('GitLab', org.login)
      end
{'fraction_non_alphanumeric': 0.11279563371740449, 'fraction_numerical': 0.0030321406913280777, 'mean_word_length': 2.844988344988345, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1671907', 'n_tokens_mistral': 1125, 'n_tokens_neox': 1066, 'n_words': 203}