<commit_before>require 'contracts'
require 'ostruct'
require 'date'
require 'json'

module HowIs
  ##
  # Represents a completed analysis of the repository being analyzed.
  class Analysis < OpenStruct
  end

  class Analyzer
    include Contracts::Core

    ##
    # Raised when attempting to export to an unsupported format.
    class UnsupportedImportFormat < StandardError
      def initialize(format)
        super("Unsupported import format: #{format}")
      end
    end

    ##
    # Generates and returns an analysis.
    Contract Fetcher::Results, C::KeywordArgs[analysis_class: C::Optional[Class]] => Analysis
    def call(data, analysis_class: Analysis)
      issues = data.issues
      pulls = data.pulls

      analysis_class.new(
        issues_url: "https://github.com/#{data.repository}/issues",
        pulls_url: "https://github.com/#{data.repository}/pulls",

        repository: data.repository,

        number_of_issues:  issues.length,
        number_of_pulls:   pulls.length,

        issues_with_label: with_label_links(num_with_label(issues), data.repository),
        issues_with_no_label: {'link' => nil, 'total' => num_with_no_label(issues)},

        average_issue_age: average_age_for(issues),
        average_pull_age:  average_age_for(pulls),

        oldest_issue: issue_or_pull_to_hash(oldest_for(issues)),
        oldest_pull: issue_or_pull_to_hash(oldest_for(pulls)),
      )
    end

    ##
    # Generates an analysis from a JSON report.
    def from_file(file)
      extension = file.split('.').last
      raise UnsupportedImportFormat, extension unless extension == 'json'

      hash = JSON.parse(open(file).read)
      hash = hash.map do |k, v|
        v = DateTime.parse(v) if k.end_with?('_date')

        [k, v]
      end.to_h

      %w[oldest_issue oldest_pull].each do |key|
        hash[key]['date'] = DateTime.parse(hash[key]['date'])
      end

      Analysis.new(hash)
    end

    # Given an Array of issues or pulls, return a Hash specifying how many
    # issues or pulls use each label.
    def num_with_label(issues_or_pulls)
      # Returned hash maps labels to frequency.
      # E.g., given 10 issues/pulls with label "label1" and 5 with label "label2",
      # {
      #   "label1" => 10,
      #   "label2" => 5
      # }

      hash = Hash.new(0)
      issues_or_pulls.each do |iop|
        next unless iop['labels']

        iop['labels'].each do |label|
          hash[label['name']] += 1
        end
      end
      hash
    end

    # Returns the number of issues with no label.
    def num_with_no_label(issues)
      issues.select { |x| x['labels'].empty? }.length
    end

    # Given an Array of dates, average the timestamps and return the date that
    # represents.
    def average_date_for(issues_or_pulls)
      timestamps = issues_or_pulls.map { |iop| Date.parse(iop['created_at']).strftime('%s').to_i }
      average_timestamp = timestamps.reduce(:+) / issues_or_pulls.length

      DateTime.strptime(average_timestamp.to_s, '%s')
    end

    # Given an Array of issues or pulls, return the average age of them.
    # Returns nil if no issues or pulls are provided.
    def average_age_for(issues_or_pulls)
      return nil if issues_or_pulls.empty?

      ages = issues_or_pulls.map {|iop| time_ago_in_seconds(iop['created_at'])}
      raw_average = ages.reduce(:+) / ages.length

      seconds_in_a_year = 31_556_926
      seconds_in_a_month = 2_629_743
      seconds_in_a_week = 604_800
      seconds_in_a_day = 86_400

      years = raw_average / seconds_in_a_year
      years_remainder = raw_average % seconds_in_a_year

      months = years_remainder / seconds_in_a_month
      months_remainder = years_remainder % seconds_in_a_month

      weeks = months_remainder / seconds_in_a_week
      weeks_remainder = months_remainder % seconds_in_a_week

      days = weeks_remainder / seconds_in_a_day

      values = [
        [years, "year"],
        [months, "month"],
        [weeks, "week"],
        [days, "day"],
      ].reject {|(v, k)| v == 0}.map{ |(v,k)|
        k = k + 's' if v != 1
        [v, k]
      }

      most_significant = values[0, 2].map {|x| x.join(" ")}

      if most_significant.length < 2
        value = most_significant.first
      else
        value = most_significant.join(" and ")
      end

      "approximately #{value}"
    end

    # Given an Array of issues or pulls, return the creation date of the oldest.
    # Returns nil if no issues or pulls are provided.
    def oldest_for(issues_or_pulls)
      return nil if issues_or_pulls.empty?

      issues_or_pulls.sort_by {|x| DateTime.parse(x['created_at']) }.first
    end

    # Given an issue or PR, returns the date it was created.
    def date_for(issue_or_pull)
      DateTime.parse(issue_or_pull['created_at'])
    end

  private
    # Takes an Array of labels, and returns amodified list that includes links
    # to each label.
    def with_label_links(labels, repository)
      labels.map do |label, num_issues|
        label_link = "https://github.com/#{repository}/issues?q=" + CGI.escape("is:open is:issue label:\"#{label}\"")

        [label, {'link' => label_link, 'total' => num_issues}]
      end.to_h
    end

    # Returns how many seconds ago a date (as a String) was.
    def time_ago_in_seconds(x)
      DateTime.now.strftime("%s").to_i - DateTime.parse(x).strftime("%s").to_i
    end<commit_msg>Track newest issue/PR in Analyzer.<commit_after>require 'contracts'
require 'ostruct'
require 'date'
require 'json'

module HowIs
  ##
  # Represents a completed analysis of the repository being analyzed.
  class Analysis < OpenStruct
  end

  class Analyzer
    include Contracts::Core

    ##
    # Raised when attempting to export to an unsupported format.
    class UnsupportedImportFormat < StandardError
      def initialize(format)
        super("Unsupported import format: #{format}")
      end
    end

    ##
    # Generates and returns an analysis.
    Contract Fetcher::Results, C::KeywordArgs[analysis_class: C::Optional[Class]] => Analysis
    def call(data, analysis_class: Analysis)
      issues = data.issues
      pulls = data.pulls

      analysis_class.new(
        issues_url: "https://github.com/#{data.repository}/issues",
        pulls_url: "https://github.com/#{data.repository}/pulls",

        repository: data.repository,

        number_of_issues:  issues.length,
        number_of_pulls:   pulls.length,

        issues_with_label: with_label_links(num_with_label(issues), data.repository),
        issues_with_no_label: {'link' => nil, 'total' => num_with_no_label(issues)},

        average_issue_age: average_age_for(issues),
        average_pull_age:  average_age_for(pulls),

        oldest_issue: issue_or_pull_to_hash(oldest_for(issues)),
        oldest_pull: issue_or_pull_to_hash(oldest_for(pulls)),

        newest_issue: issue_or_pull_to_hash(newest_for(issues)),
        newest_pull: issue_or_pull_to_hash(newest_for(pulls)),
      )
    end

    ##
    # Generates an analysis from a JSON report.
    def from_file(file)
      extension = file.split('.').last
      raise UnsupportedImportFormat, extension unless extension == 'json'

      hash = JSON.parse(open(file).read)
      hash = hash.map do |k, v|
        v = DateTime.parse(v) if k.end_with?('_date')

        [k, v]
      end.to_h

      %w[oldest_issue oldest_pull].each do |key|
        hash[key]['date'] = DateTime.parse(hash[key]['date'])
      end

      Analysis.new(hash)
    end

    # Given an Array of issues or pulls, return a Hash specifying how many
    # issues or pulls use each label.
    def num_with_label(issues_or_pulls)
      # Returned hash maps labels to frequency.
      # E.g., given 10 issues/pulls with label "label1" and 5 with label "label2",
      # {
      #   "label1" => 10,
      #   "label2" => 5
      # }

      hash = Hash.new(0)
      issues_or_pulls.each do |iop|
        next unless iop['labels']

        iop['labels'].each do |label|
          hash[label['name']] += 1
        end
      end
      hash
    end

    # Returns the number of issues with no label.
    def num_with_no_label(issues)
      issues.select { |x| x['labels'].empty? }.length
    end

    # Given an Array of dates, average the timestamps and return the date that
    # represents.
    def average_date_for(issues_or_pulls)
      timestamps = issues_or_pulls.map { |iop| Date.parse(iop['created_at']).strftime('%s').to_i }
      average_timestamp = timestamps.reduce(:+) / issues_or_pulls.length

      DateTime.strptime(average_timestamp.to_s, '%s')
    end

    # Given an Array of issues or pulls, return the average age of them.
    # Returns nil if no issues or pulls are provided.
    def average_age_for(issues_or_pulls)
      return nil if issues_or_pulls.empty?

      ages = issues_or_pulls.map {|iop| time_ago_in_seconds(iop['created_at'])}
      raw_average = ages.reduce(:+) / ages.length

      seconds_in_a_year = 31_556_926
      seconds_in_a_month = 2_629_743
      seconds_in_a_week = 604_800
      seconds_in_a_day = 86_400

      years = raw_average / seconds_in_a_year
      years_remainder = raw_average % seconds_in_a_year

      months = years_remainder / seconds_in_a_month
      months_remainder = years_remainder % seconds_in_a_month

      weeks = months_remainder / seconds_in_a_week
      weeks_remainder = months_remainder % seconds_in_a_week

      days = weeks_remainder / seconds_in_a_day

      values = [
        [years, "year"],
        [months, "month"],
        [weeks, "week"],
        [days, "day"],
      ].reject {|(v, k)| v == 0}.map{ |(v,k)|
        k = k + 's' if v != 1
        [v, k]
      }

      most_significant = values[0, 2].map {|x| x.join(" ")}

      if most_significant.length < 2
        value = most_significant.first
      else
        value = most_significant.join(" and ")
      end

      "approximately #{value}"
    end

    def sort_iops_by_created_at(issues_or_pulls)
      issues_or_pulls.sort_by {|x| DateTime.parse(x['created_at']) }.first
    end

    # Given an Array of issues or pulls, return the oldest.
    # Returns nil if no issues or pulls are provided.
    def oldest_for(issues_or_pulls)
      return nil if issues_or_pulls.empty?

      sort_iops_by_created_at(issues_or_pulls).first
    end

    # Given an Array of issues or pulls, return the newest.
    # Returns nil if no issues or pulls are provided.
    def newest_for(issues_or_pulls)
      return nil if issues_or_pulls.empty?

      sort_iops_by_created_at(issues_or_pulls).last
    end

    # Given an issue or PR, returns the date it was created.
    def date_for(issue_or_pull)
      DateTime.parse(issue_or_pull['created_at'])
    end

  private
    # Takes an Array of labels, and returns amodified list that includes links
    # to each label.
    def with_label_links(labels, repository)
      labels.map do |label, num_issues|
        label_link = "https://github.com/#{repository}/issues?q=" + CGI.escape("is:open is:issue label:\"#{label}\"")

        [label, {'link' => label_link, 'total' => num_issues}]
      end.to_h
    end

    # Returns how many seconds ago a date (as a String) was.
    def time_ago_in_seconds(x)
      DateTime.now.strftime("%s").to_i - DateTime.parse(x).strftime("%s").to_i
    end
{'fraction_non_alphanumeric': 0.1046047378227309, 'fraction_numerical': 0.007630201401827699, 'mean_word_length': 2.794008751262201, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6502012', 'n_tokens_mistral': 3769, 'n_tokens_neox': 3599, 'n_words': 912}