<commit_before>require 'nokogiri'
require 'hashie'
require 'net/http'
require 'open-uri'

module Caboodle
  
  begin
    require 'memcached'
    CACHE = Memcached.new
    puts "Running with memcache"
  rescue
    puts "Running without memcache"
  end
  
  def self.round_time(integer, factor)
    return integer if(integer % factor == 0)
    return integer - (integer % factor)
  end
  
  def self.scrape url
    begin
      if defined?(CACHE)
        begin
          puts "Scraping with cache optimisation"
          timeout = 60*60*1000
          response = CACHE.get("#{round_time(Time.new.to_i, timeout)}:#{url}") rescue nil
          response ||= open(url).read
          CACHE.set("#{round_time(Time.new.to_i, timeout)}:#{url}", response)
          CACHE.set("0:#{url}", response)
        rescue<commit_msg>use memcached for scraping<commit_after>require 'nokogiri'
require 'hashie'
require 'net/http'
require 'open-uri'

module Caboodle
  
  begin
    require 'memcached'
    CACHE = Memcached.new(ENV["MEMCACHE_SERVERS"])
    puts "Running with memcache"
  rescue
    puts "Running without memcache"
  end
  
  def self.round_time(integer, factor)
    return integer if(integer % factor == 0)
    return integer - (integer % factor)
  end
  
  def self.scrape url
    begin
      if defined?(CACHE)
        begin
          puts "Scraping with cache optimisation"
          timeout = 60*60*1000
          response = CACHE.get("#{round_time(Time.new.to_i, timeout)}:#{url}") rescue nil
          response ||= open(url).read
          CACHE.set("#{round_time(Time.new.to_i, timeout)}:#{url}", response)
          CACHE.set("0:#{url}", response)
        rescue
{'fraction_non_alphanumeric': 0.11778846153846154, 'fraction_numerical': 0.01201923076923077, 'mean_word_length': 2.784090909090909, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4295613', 'n_tokens_mistral': 542, 'n_tokens_neox': 531, 'n_words': 132}