<commit_before>require 'base64'

require 'fileutils'
require 'github_api'
require 'git'


require 'license_auto/config/config'
require 'license_auto/license/similarity'
require 'license_auto/license/readme'

class GithubCom < Website

  HOST = 'github.com'
  LANGUAGE = nil

  GIT_HASH_LENGTH = 40

  attr_reader :url

  ##
  # package: Hashie::Mash
  # user: string
  # repo: string
  # ref: string
  def initialize(package, user, repo, ref=nil, auto_pagination=false, last_commit=nil)
    super(package)
    @ref = ref
    @last_commit = last_commit
    @user = user
    @repo = repo
    @url = "https://github.com/#{user}/#{repo}"
    LicenseAuto.logger.debug(@url)

    @server =
        begin
          eval('WebMock')
          LicenseAuto.logger.debug("Running LicenseAuto in RSpec mode")
          Github.new(user: user, repo: repo)
        rescue NameError => e
          LicenseAuto.logger.debug("Running LicenseAuto in formal mode,username #{AUTO_CONF.github.username}")
          basic_auth = "#{AUTO_CONF.github.username}:#{AUTO_CONF.github.access_token}"
          server = Github.new(user: @user, repo: @repo, basic_auth: basic_auth, auto_pagination: auto_pagination)
          @repoinfo = server.repos.get
          if  @repoinfo.headers.status == 301
            response = HTTParty.get(@repoinfo.headers.location)
            if response["name"] && response["owner"]["login"]
              @user = response["owner"]["login"]
              @repo = response["name"]
              server = Github.new(user: @user, repo: @repo, basic_auth: basic_auth, auto_pagination: auto_pagination)
              @repoinfo = server.repos.get
            end
          end
          server
        end
  end

  def server
    @server
  end

  def ref=(ref=nil)
    @ref=ref
  end

  ##
  # @return LicenseInfoWrapper
  ## default_branch : nil, Get a specific version license
  ##                  true, Get a default branch license
  def get_license_info(default_branch = nil)
    pack_url = nil
    if default_branch
      possible_ref = nil
      pack_url = @url
    else
      possible_ref = @ref || match_versioned_ref
      pack_url = @url + "/tree/#{possible_ref}"
    end
    LicenseAuto.logger.debug("possible_ref: #{possible_ref}")
    # If possible_ref is nil, the Github API server will return the default branch contents
    contents = @server.repos.contents.get(path: '/', ref: possible_ref)

    license_files = []
    readme_files = []
    notice_files = []
    contents.each { |obj|
      if obj.type == 'file'
        filename_matcher = LicenseAuto::Matcher::FilepathName.new(obj.name)
        license_files.push(obj) if filename_matcher.match_license_file
        readme_files.push(obj) if filename_matcher.match_readme_file
        notice_files.push(obj) if filename_matcher.match_notice_file
      end
    }

    license_files = license_files.map { |obj|
      license_content = get_blobs(obj['sha'])
      license_name, sim_ratio = LicenseAuto::Similarity.new(license_content).most_license_sim
      LicenseAuto::LicenseWrapper.new(
          name: license_name,
          sim_ratio: sim_ratio,
          html_url: obj['html_url'],
          download_url: obj['download_url'],
          text: license_content
      )
    }

    readme_files = readme_files.map { |obj|
      readme_content = get_blobs(obj['sha'])
      license_content = LicenseAuto::Readme.new(obj['download_url'], readme_content).license_content
      LicenseAuto.logger.debug("readme_content:\n#{license_content}\n")
      if license_content.nil?
        next
      else
        license_name, sim_ratio = LicenseAuto::Similarity.new(license_content).most_license_sim
        LicenseAuto::LicenseWrapper.new(
            name: license_name,
            sim_ratio: sim_ratio,
            html_url: obj['html_url'],
            download_url: obj['download_url'],
            text: license_content
        )
      end
    }.compact

    notice_files = notice_files.map { |obj|
      notice_content = get_blobs(obj['sha'])
      LicenseAuto.logger.debug("notice_content:\n#{notice_content}\n")

      if notice_content.nil?
        next
      else
        LicenseAuto::NoticeWrapper.new(
            html_url: obj['html_url'],
            download_url: obj['download_url'],
            text: notice_content
        )
      end
    }.compact

    pack_wrapper = LicenseAuto::PackWrapper.new(
        homepage: nil,
        project_url: nil,
        source_url: pack_url || @url
    )
    if default_branch == nil and license_files.empty? and readme_files.empty?
      return get_license_info(true)
    end
    LicenseAuto::LicenseInfoWrapper.new(
        licenses: license_files,
        readmes: readme_files,
        notices: notice_files,
        pack: pack_wrapper
    )
  end

  def get_ref(ref)
    @server.git_data.references.get(ref: ref)
  end

  def match_versioned_ref()
    possible_ref = nil
    # If provided a Git SHA, use it directly
    if @package.version.size >= GIT_HASH_LENGTH
      possible_ref = @package.version
    else
      matcher = LicenseAuto::Matcher::FilepathName.new(@package.version)
      @server.repos.tags do |tag|
        matched = matcher.match_the_ref(tag.name)
        if matched
          possible_ref = tag.name
          break
        end
      end
    end
    possible_ref
  end

  def list_languages
    langs = @server.repos.languages
    LicenseAuto.logger.debug("All languaegs: #{langs}")
    langs
  end

  # @return
  # Array: [#<Hashie::Mash commit=#<Hashie::Mash sha="8065e5c64a22bd6d60e4df8d9be46b5805ec9355" url="https://api.github.com/repos/bower/bower/commits/8065e5c64a22bd6d60e4df8d9be46b5805ec9355"> name="v1.7.9" tarball_url="https://api.github.com/repos/bower/bower/tarball/v1.7.9" zipball_url="https://api.github.com/repos/bower/bower/zipball/v1.7.9">, #<Hashie::Mash commit=
  def list_tags
    @server.repos.tags.body
  end

  def list_commits
    commits = @server.repos.commits.list
  end

  def last_commit
    return @last_commit if @last_commit<commit_msg>get tree and last_commit add irregular case handle<commit_after>require 'base64'

require 'fileutils'
require 'github_api'
require 'git'


require 'license_auto/config/config'
require 'license_auto/license/similarity'
require 'license_auto/license/readme'

class GithubCom < Website

  HOST = 'github.com'
  LANGUAGE = nil

  GIT_HASH_LENGTH = 40

  attr_reader :url

  ##
  # package: Hashie::Mash
  # user: string
  # repo: string
  # ref: string
  def initialize(package, user, repo, ref=nil, auto_pagination=false, last_commit=nil)
    super(package)
    @ref = ref
    @last_commit = last_commit
    @user = user
    @repo = repo
    @url = "https://github.com/#{user}/#{repo}"
    LicenseAuto.logger.debug(@url)

    @server =
        begin
          eval('WebMock')
          LicenseAuto.logger.debug("Running LicenseAuto in RSpec mode")
          Github.new(user: user, repo: repo)
        rescue NameError => e
          LicenseAuto.logger.debug("Running LicenseAuto in formal mode,username #{AUTO_CONF.github.username}")
          basic_auth = "#{AUTO_CONF.github.username}:#{AUTO_CONF.github.access_token}"
          server = Github.new(user: @user, repo: @repo, basic_auth: basic_auth, auto_pagination: auto_pagination)
          @repoinfo = server.repos.get
          if  @repoinfo.headers.status == 301
            response = HTTParty.get(@repoinfo.headers.location)
            if response["name"] && response["owner"]["login"]
              @user = response["owner"]["login"]
              @repo = response["name"]
              server = Github.new(user: @user, repo: @repo, basic_auth: basic_auth, auto_pagination: auto_pagination)
              @repoinfo = server.repos.get
            end
          end
          server
        end
  end

  def server
    @server
  end

  def ref=(ref=nil)
    @ref=ref
  end

  ##
  # @return LicenseInfoWrapper
  ## default_branch : nil, Get a specific version license
  ##                  true, Get a default branch license
  def get_license_info(default_branch = nil)
    pack_url = nil
    if default_branch
      possible_ref = nil
      pack_url = @url
    else
      possible_ref = @ref || match_versioned_ref
      pack_url = @url + "/tree/#{possible_ref}"
    end
    LicenseAuto.logger.debug("possible_ref: #{possible_ref}")
    # If possible_ref is nil, the Github API server will return the default branch contents
    contents = @server.repos.contents.get(path: '/', ref: possible_ref)

    license_files = []
    readme_files = []
    notice_files = []
    contents.each { |obj|
      if obj.type == 'file'
        filename_matcher = LicenseAuto::Matcher::FilepathName.new(obj.name)
        license_files.push(obj) if filename_matcher.match_license_file
        readme_files.push(obj) if filename_matcher.match_readme_file
        notice_files.push(obj) if filename_matcher.match_notice_file
      end
    }

    license_files = license_files.map { |obj|
      license_content = get_blobs(obj['sha'])
      license_name, sim_ratio = LicenseAuto::Similarity.new(license_content).most_license_sim
      LicenseAuto::LicenseWrapper.new(
          name: license_name,
          sim_ratio: sim_ratio,
          html_url: obj['html_url'],
          download_url: obj['download_url'],
          text: license_content
      )
    }

    readme_files = readme_files.map { |obj|
      readme_content = get_blobs(obj['sha'])
      license_content = LicenseAuto::Readme.new(obj['download_url'], readme_content).license_content
      LicenseAuto.logger.debug("readme_content:\n#{license_content}\n")
      if license_content.nil?
        next
      else
        license_name, sim_ratio = LicenseAuto::Similarity.new(license_content).most_license_sim
        LicenseAuto::LicenseWrapper.new(
            name: license_name,
            sim_ratio: sim_ratio,
            html_url: obj['html_url'],
            download_url: obj['download_url'],
            text: license_content
        )
      end
    }.compact

    notice_files = notice_files.map { |obj|
      notice_content = get_blobs(obj['sha'])
      LicenseAuto.logger.debug("notice_content:\n#{notice_content}\n")

      if notice_content.nil?
        next
      else
        LicenseAuto::NoticeWrapper.new(
            html_url: obj['html_url'],
            download_url: obj['download_url'],
            text: notice_content
        )
      end
    }.compact

    pack_wrapper = LicenseAuto::PackWrapper.new(
        homepage: nil,
        project_url: nil,
        source_url: pack_url || @url
    )
    if default_branch == nil and license_files.empty? and readme_files.empty?
      return get_license_info(true)
    end
    LicenseAuto::LicenseInfoWrapper.new(
        licenses: license_files,
        readmes: readme_files,
        notices: notice_files,
        pack: pack_wrapper
    )
  end

  def get_ref(ref)
    @server.git_data.references.get(ref: ref)
  end

  def match_versioned_ref()
    possible_ref = nil
    # If provided a Git SHA, use it directly
    if @package.version.size >= GIT_HASH_LENGTH
      possible_ref = @package.version
    else
      matcher = LicenseAuto::Matcher::FilepathName.new(@package.version)
      @server.repos.tags do |tag|
        matched = matcher.match_the_ref(tag.name)
        if matched
          possible_ref = tag.name
          break
        end
      end
    end
    possible_ref
  end

  def list_languages
    langs = @server.repos.languages
    LicenseAuto.logger.debug("All languaegs: #{langs}")
    langs
  end

  # @return
  # Array: [#<Hashie::Mash commit=#<Hashie::Mash sha="8065e5c64a22bd6d60e4df8d9be46b5805ec9355" url="https://api.github.com/repos/bower/bower/commits/8065e5c64a22bd6d60e4df8d9be46b5805ec9355"> name="v1.7.9" tarball_url="https://api.github.com/repos/bower/bower/tarball/v1.7.9" zipball_url="https://api.github.com/repos/bower/bower/zipball/v1.7.9">, #<Hashie::Mash commit=
  def list_tags
    @server.repos.tags.body
  end

  def list_commits
    commits = @server.repos.commits.list
  end

  def last_commit
    return @last_commit if ( @last_commit && !@last_commit =~/{.*}/)
{'fraction_non_alphanumeric': 0.10789343041535661, 'fraction_numerical': 0.010921727618732417, 'mean_word_length': 2.7783682400750234, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3417613', 'n_tokens_mistral': 3928, 'n_tokens_neox': 3816, 'n_words': 741}