<commit_before>require 'semverly'

class GithubVersionCrawler < Versioneye::Crawl

  include HTTParty


  def self.logger
    if !defined?(@@log) || @@log.nil?
      @@log = Versioneye::DynLog.new("log/github_version_crawler.log", 10).log
    end
    @@log
  end


  # Crawle Release dates for Objective-C packages
  def self.crawl(language = Product::A_LANGUAGE_OBJECTIVEC, empty_release_dates = true, desc = true )
    products(language, empty_release_dates, desc).each do |product|
      add_version_to_product( product )
    end
  end


  # pulls and updates product versions from Github tags
  # params:
  #   api_client - initialized GithubVersionFetcher with own auth keys
  #   product    - Product model to update
  #   repo_owner - string, the name of github account
  #   repo_name  - string, the name of github repo
  def self.crawl_for_product(api_client, product, repo_owner, repo_name)
    if api_client.check_limit_or_pause == 0
      logger.error "crawl_for_product: hit ratelimit #{repo_owner}/#{repo_name} for #{product}"
      return
    end

    # fetch repo tags and attach commit dates
    tags = api_client.fetch_all_repo_tags(repo_owner, repo_name)
    tags = attach_commit_date(api_client, repo_owner, repo_name, tags)

    # save product versions
    tags.to_a.each do |tag|
      upsert_product_version(product, tag)
    end

    if product.save
      product.reload
      ProductService.update_version_data( product )

      logger.info "crawl_for_product: #{product} has now #{product.versions.size} version"
      true
    else
      logger.error "crawl_for_product: failed to save updated #{product}"
      logger.error "  reason: #{product.errors.full_messages.to_sentence}"
      false
    end
  rescue => e
    logger.error e.message
    logger.error e.backtrace.join("\n")
    false
  end


  # It fetches and attaches commit date for each Tag
  # Tags api doesnt return commit date
  def self.attach_commit_date(api_client, repo_owner, repo_name, tags)
    tags.to_a.map do |tag|
      tag_date = fetch_tag_commit_date(
        api_client, repo_owner, repo_name, tag[:commit][:sha]
      )

      if tag_date
        tag[:created_at] = tag_date
      else
        logger.error "crawl_for_product: failed to crawl all the commit dates for #{repo_owner}/#{repo_name}"
      end

    end

    tags
  end

  def self.fetch_tag_commit_date(api_client, repo_owner, repo_name, commit_sha)
    if api_client.check_limit_or_pause == 0
      logger.error "crawl_tag_commit_date: hit ratelimit for #{repo_owner}/#{repo_name} - #{commit_sha}"
      return
    end

    logger.info "crawl_tag_commit_date:  fetching commit date for #{repo_owner}/#{repo_name}:#{commit_sha}"
    commit_details = api_client.fetch_commit_details(repo_owner, repo_name, commit_sha)
    return if commit_details.nil?


    commit_details[:commit][:committer][:date]
  end

  # processes Tag and updates/inserts Product version
  # NB! it saves only valid SEMVER versions
  #
  # params:
  #   tag - Hashmap with response from Github tags endpoint
  #         expected structure: {name: String, created_at: String, commit: {:sha}}
  def self.upsert_product_version(product, tag)
    semver_label = process_version_label(tag[:name])
    if semver_label.to_s.empty?
      logger.warn "upsert_product_version: ignoring non-valid semver #{tag[:name]} for #{product.prod_key}"
      return
    end

    if !product.version_by_number( tag[:name] ).nil?
      logger.warn "Version #{tag[:name]} for #{product.prod_key} exist already."
      return
    end

    ver = product.versions.where(version: semver_label).first_or_initialize
    ver.update(
      tag: tag[:name],
      status: 'stable',
      released_at: tag[:created_at],
      released_string: tag[:created_at].to_s,
      commit_sha: tag[:commit][:sha].to_s
    )

    ver
  end

  def self.process_version_label(label)
    label = label.to_s.gsub(/\s+/, '').to_s.strip

    semver = SemVer.parse(label)
    return "" if semver.nil?
    semver.to_s
  end<commit_msg>Add a note<commit_after>require 'semverly'

class GithubVersionCrawler < Versioneye::Crawl

  include HTTParty


  def self.logger
    if !defined?(@@log) || @@log.nil?
      @@log = Versioneye::DynLog.new("log/github_version_crawler.log", 10).log
    end
    @@log
  end


  # Crawle Release dates for Objective-C packages
  def self.crawl(language = Product::A_LANGUAGE_OBJECTIVEC, empty_release_dates = true, desc = true )
    products(language, empty_release_dates, desc).each do |product|
      add_version_to_product( product )
    end
  end


  # pulls and updates product versions from Github tags
  # params:
  #   api_client - initialized GithubVersionFetcher with own auth keys
  #   product    - Product model to update
  #   repo_owner - string, the name of github account
  #   repo_name  - string, the name of github repo
  def self.crawl_for_product(api_client, product, repo_owner, repo_name)
    if api_client.check_limit_or_pause == 0
      logger.error "crawl_for_product: hit ratelimit #{repo_owner}/#{repo_name} for #{product}"
      return
    end

    # fetch repo tags and attach commit dates
    tags = api_client.fetch_all_repo_tags(repo_owner, repo_name)
    tags = attach_commit_date(api_client, repo_owner, repo_name, tags)

    # save product versions
    tags.to_a.each do |tag|
      version = upsert_product_version(product, tag)
      # TODO check license. RR or TG.
    end

    if product.save
      product.reload
      ProductService.update_version_data( product )

      logger.info "crawl_for_product: #{product} has now #{product.versions.size} version"
      true
    else
      logger.error "crawl_for_product: failed to save updated #{product}"
      logger.error "  reason: #{product.errors.full_messages.to_sentence}"
      false
    end
  rescue => e
    logger.error e.message
    logger.error e.backtrace.join("\n")
    false
  end


  # It fetches and attaches commit date for each Tag
  # Tags api doesnt return commit date
  def self.attach_commit_date(api_client, repo_owner, repo_name, tags)
    tags.to_a.map do |tag|
      tag_date = fetch_tag_commit_date(
        api_client, repo_owner, repo_name, tag[:commit][:sha]
      )

      if tag_date
        tag[:created_at] = tag_date
      else
        logger.error "crawl_for_product: failed to crawl all the commit dates for #{repo_owner}/#{repo_name}"
      end
    end

    tags
  end

  def self.fetch_tag_commit_date(api_client, repo_owner, repo_name, commit_sha)
    if api_client.check_limit_or_pause == 0
      logger.error "crawl_tag_commit_date: hit ratelimit for #{repo_owner}/#{repo_name} - #{commit_sha}"
      return
    end

    logger.info "crawl_tag_commit_date:  fetching commit date for #{repo_owner}/#{repo_name}:#{commit_sha}"
    commit_details = api_client.fetch_commit_details(repo_owner, repo_name, commit_sha)
    return if commit_details.nil?


    commit_details[:commit][:committer][:date]
  end

  # processes Tag and updates/inserts Product version
  # NB! it saves only valid SEMVER versions
  #
  # params:
  #   tag - Hashmap with response from Github tags endpoint
  #         expected structure: {name: String, created_at: String, commit: {:sha}}
  def self.upsert_product_version(product, tag)
    semver_label = process_version_label( tag[:name] )
    if semver_label.to_s.empty?
      logger.warn "upsert_product_version: ignoring non-valid semver #{tag[:name]} for #{product.prod_key}"
      return nil
    end

    if !product.version_by_number( tag[:name] ).nil?
      logger.warn "Version #{tag[:name]} for #{product.prod_key} exist already."
      return nil
    end

    ver = product.versions.where(version: semver_label).first_or_initialize
    ver.update(
      tag: tag[:name],
      status: 'stable',
      released_at: tag[:created_at],
      released_string: tag[:created_at].to_s,
      commit_sha: tag[:commit][:sha].to_s
    )

    ver
  end

  def self.process_version_label(label)
    label = label.to_s.gsub(/\s+/, '').to_s.strip

    semver = SemVer.parse(label)
    return "" if semver.nil?
    semver.to_s
  end
{'fraction_non_alphanumeric': 0.09701308318933596, 'fraction_numerical': 0.0009874105159219946, 'mean_word_length': 3.3634894991922457, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6338954', 'n_tokens_mistral': 2682, 'n_tokens_neox': 2588, 'n_words': 651}