<commit_before>require 'open-uri'

class Evidence
  # This class collects and caches all evidence gathered so far on a project.
  # If parallel execution is possible, this class locks/unlocks so
  # parallel writing doesn't cause any harm.

  def initialize(project)
    @project = project
    @changes = {}
    @cached_data = {}
  end

  attr_reader :project

  # We need to use the project_homepage_url and repo_url often,
  # so we'll create helpers to access them quickly.
  def project_homepage_url
    @project[:project_homepage_url]
  end

  def repo_url
    @project[:repo_url]
  end

  # Don't download more than this number of bytes per file;
  # this helps counter easy DoS attacks.
  MAXREAD = 10 * (2**20)

  # Get contents of given URL and return it (cached).
  # TODO: Handle exceptions - turn into nothing useful.
  # TODO: Lock for parallel access. Possibly return while still reading.
  # TODO: Timeout on reads.
  def get(url)
    unless @cached_data.key?(url)
      open(url, 'rb') do |file|
        @cached_data[url] = file.read(MAXREAD)
      end
    end
    @cached_data[url]
  end
end<commit_msg>Tweak evidence - don't need more than 1MB download<commit_after>require 'open-uri'

class Evidence
  # This class collects and caches all evidence gathered so far on a project.
  # If parallel execution is possible, this class locks/unlocks so
  # parallel writing doesn't cause any harm.

  def initialize(project)
    @project = project # ActiveRecord. Detectives should NOT change this.
    @cached_data = {}
  end

  attr_reader :project

  # We need to use the project_homepage_url and repo_url often,
  # so we'll create helpers to access them quickly.
  def project_homepage_url
    @project[:project_homepage_url]
  end

  def repo_url
    @project[:repo_url]
  end

  # Don't download more than this number of bytes per file;
  # this helps counter easy DoS attacks.
  MAXREAD = 1 * (2**20)

  # Get contents of given URL and return it (cached).
  # TODO: Handle exceptions - turn into nothing useful.
  # TODO: Lock for parallel access. Possibly return while still reading.
  # TODO: Timeout on reads.
  def get(url)
    unless @cached_data.key?(url)
      open(url, 'rb') do |file|
        @cached_data[url] = file.read(MAXREAD)
      end
    end
    @cached_data[url]
  end
end
{'fraction_non_alphanumeric': 0.07838891294932872, 'fraction_numerical': 0.0043308791684712, 'mean_word_length': 3.2777777777777777, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7418095', 'n_tokens_mistral': 720, 'n_tokens_neox': 692, 'n_words': 271}