<commit_before>require "#{Rails.root.join('lib', 'modules', 'aws_interface.rb')}"
include AwsInterface
require 'pry'

namespace :stories do

  desc 'send md5 files to aws'
  task :md5, [:type] => :environment do |t,args|
    Story.find_each do |story|
      filename = story.url.scan(/([\w\d\-\.\s]+)(?:\.ogg)/).flatten.join
      md5 = Digest::MD5.new
      hash = md5.update filename
      aws = AwsInterface::Md5Putter.new(hash, filename)
      if aws.response.exists?
        story.update!(md5_url: aws.response.public_url)
      end
    end
  end

  desc 'display the current environment of rake'
  task :current_environment do
    puts "You are running rake task in #{Rails.env} environment"
  end
end<commit_msg>removing gem not available in production [ci skip]<commit_after>require "#{Rails.root.join('lib', 'modules', 'aws_interface.rb')}"
include AwsInterface

namespace :stories do

  desc 'send md5 files to aws'
  task :md5, [:type] => :environment do |t,args|
    Story.find_each do |story|
      filename = story.url.scan(/([\w\d\-\.\s]+)(?:\.ogg)/).flatten.join
      md5 = Digest::MD5.new
      hash = md5.update filename
      aws = AwsInterface::Md5Putter.new(hash, filename)
      if aws.response.exists?
        story.update!(md5_url: aws.response.public_url)
      end
    end
  end

  desc 'display the current environment of rake'
  task :current_environment do
    puts "You are running rake task in #{Rails.env} environment"
  end
end
{'fraction_non_alphanumeric': 0.13552361396303902, 'fraction_numerical': 0.009582477754962354, 'mean_word_length': 3.3, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1269322', 'n_tokens_mistral': 496, 'n_tokens_neox': 501, 'n_words': 132}