<commit_before>require 'faraday'
require 'faraday_middleware'

class HttpFile
  attr_reader :path
  def initialize(client, path, all)
    @client = client
    @path = path
    @all = all
  end
  def work_to_do?(suffix)
    res = @path.end_with?(suffix)
    return false if res

    res = @all.include?(flag_path(suffix))
    return !res
  end
  def flag_path(suffix)
    "#{@path}.#{suffix}"
  end
  def get
    response = @client.get("/files/#{@path}")
    return response.body
  end
  def mark_as_copied(suffix)
    res = @client.post("/files/#{@path}", {filename: @path, suffix: suffix})
    if res.status != 200
      raise 'problems while markme'
    end
  end
  def to_s
    "HttpFile(#{@path})"
  end
  def modification_time
    m = Regexp.new(".*PANO_(....)(..)(..)_(..)(..).*").match(@path)
    m = m || Regexp.new(".*VID_(....)(..)(..)_(..)(..).*").match(@path)
    if m
      res = Time.new(m[1], m[2], m[3], m[4], m[5])
      puts "time from filename: #{res}"
      return res
    end
    res = Time.now
    puts "falling back to #{res}"
    return res
  end
end

class HttpStorage
  def initialize(id, path)
    @id = id
    @path = path[0...-1]
    @client = Faraday.new(url: "http://#{@id}:4567") do |faraday|
      faraday.request :url_encoded
      faraday.response :logger
      faraday.response :json, :content_type => 'application/json'
      faraday.adapter Faraday.default_adapter
    end
  end
  def glob(pattern)
    puts "globbing '#{pattern}'"
    response = @client.get '/index'
    all = response.body
    puts "total # of files: #{all.size}"
    filtered = all.select {|f|
      File.fnmatch(pattern, f, File::FNM_EXTGLOB) && !f.include?('/.thumbnails/')<commit_msg>Implement fallback for images with time in the filename<commit_after>require 'faraday'
require 'faraday_middleware'

class HttpFile
  attr_reader :path
  def initialize(client, path, all)
    @client = client
    @path = path
    @all = all
  end
  def work_to_do?(suffix)
    res = @path.end_with?(suffix)
    return false if res

    res = @all.include?(flag_path(suffix))
    return !res
  end
  def flag_path(suffix)
    "#{@path}.#{suffix}"
  end
  def get
    response = @client.get("/files/#{@path}")
    return response.body
  end
  def mark_as_copied(suffix)
    res = @client.post("/files/#{@path}", {filename: @path, suffix: suffix})
    if res.status != 200
      raise 'problems while markme'
    end
  end
  def to_s
    "HttpFile(#{@path})"
  end
  def modification_time
    m = Regexp.new(".*PANO_(....)(..)(..)_(..)(..).*").match(@path)
    m = m || Regexp.new(".*VID_(....)(..)(..)_(..)(..).*").match(@path)
    m = m || Regexp.new(".*IMG_(....)(..)(..)_(..)(..).*").match(@path)
    if m
      res = Time.new(m[1], m[2], m[3], m[4], m[5])
      puts "time from filename: #{res}"
      return res
    end
    res = Time.now
    puts "falling back to #{res}"
    return res
  end
end

class HttpStorage
  def initialize(id, path)
    @id = id
    @path = path[0...-1]
    @client = Faraday.new(url: "http://#{@id}:4567") do |faraday|
      faraday.request :url_encoded
      faraday.response :logger
      faraday.response :json, :content_type => 'application/json'
      faraday.adapter Faraday.default_adapter
    end
  end
  def glob(pattern)
    puts "globbing '#{pattern}'"
    response = @client.get '/index'
    all = response.body
    puts "total # of files: #{all.size}"
    filtered = all.select {|f|
      File.fnmatch(pattern, f, File::FNM_EXTGLOB) && !f.include?('/.thumbnails/')
{'fraction_non_alphanumeric': 0.18036529680365296, 'fraction_numerical': 0.007990867579908675, 'mean_word_length': 2.7688172043010755, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7122123', 'n_tokens_mistral': 1297, 'n_tokens_neox': 1275, 'n_words': 302}