<commit_before># frozen_string_literal: true
module S3Object
  class File
    include ActiveModel::Model

    def self.select(bucket:, prefix:)
      s3 = Aws::S3::Client.new
      resp = s3.list_objects_v2(bucket: bucket, prefix: prefix)
      resp.contents.select do |content|
        content.key =~ /\A#{prefix}([\s\S]*)\z/ &&
          !Regexp.last_match(1).include?('/')
      end.map do |file|
        new(file)<commit_msg>Refactor<commit_after># frozen_string_literal: true
module S3Object
  class File
    include ActiveModel::Model

    def self.select(bucket:, prefix:)
      select_child_contents_by_prefix(
        bucket: bucket,
        prefix: prefix
      ).map do |file|
        new(file)
      end
    end

    def self.select_child_contents_by_prefix(bucket:, prefix:)
      s3 = Aws::S3::Client.new
      resp = s3.list_objects_v2(bucket: bucket, prefix: prefix)
      resp.contents.select do |content|
        content.key =~ /\A#{prefix}([\s\S]*)\z/ &&
          !Regexp.last_match(1).include?('/')
{'fraction_non_alphanumeric': 0.13837095191364082, 'fraction_numerical': 0.011776251226692836, 'mean_word_length': 2.8202247191011236, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2106165', 'n_tokens_mistral': 346, 'n_tokens_neox': 336, 'n_words': 67}