<commit_before>require 'rbbt/util/misc'
require 'rbbt/util/tsv'

module Path
  attr_accessor :pkg_module, :datadir

  def self.find_files_back_to(path, target, subdir)
    return [] if path.nil?
    raise "Path #{ path } not in directory #{ subdir }" if not Misc.in_directory? path, subdir

    pkg_module = path.pkg_module

    files = []
    while path != subdir
      path = File.dirname(path)
      path.extend Path
      path.pkg_module = pkg_module
      path.datadir    = path.datadir
      if path[target].exists? 
        files << path[target]
      end
    end

    files
  end

  def self.path(string, datadir = nil, pkg_module = nil)
    string.extend Path
    string.datadir = datadir
    string.pkg_module = case
                        when pkg_module.nil?
                          nil
                        when String === pkg_module
                          Misc.string2const pkg_module
                        else
                          pkg_module
                        end
    string
  end

  def method_missing(name, *args, &block)
    new = File.join(self.dup, name.to_s)
    new.extend Path
    new.pkg_module = pkg_module
    new.datadir    = datadir
    new
  end

  def [](name)
    new = File.join(self.dup, name.to_s)
    new.extend Path
    new.pkg_module = pkg_module
    new.datadir    = datadir
    new
  end

  def namespace
    return nil if self.nil? or self.empty? or (not datadir.nil? and (self == datadir or File.dirname(self) == datadir))
    if File.directory? self
      File.basename(self)
    else
      File.basename(File.dirname(self))
    end
  end

  def identifier_files
    if datadir.nil?
      path = File.join(File.dirname(self), 'identifiers')
      path.extend Path
      path.pkg_module = pkg_module
      if path.exists?
        [path]
      else
        []
      end
    else
      identifier_files = Path.find_files_back_to(self, 'identifiers', datadir)
      return identifier_files.collect{|f| Path.path(f, datadir, pkg_module)}
    end
  end

  def tsv(key = nil, options = {})
    if options.empty? and Hash === key
      options, key = key, nil
    end

    produce
    #TSV.new self, key, options.merge(:datadir => datadir, :namespace => namespace)
    TSV.new self, key, options
  end

  def index(options = {})
    produce
    TSV.index self, options
  end

  def open(options = {})
    produce
    Open.open(self, options)
  end

  def read(options = {})
    produce
    Open.read(self, options)
  end

  def fields(sep = nil, header_hash = nil)
    produce
    TSV.parse_header(self.open, sep, header_hash)[1].collect{|f| f.extend TSV::Field; f.namespace = namespace ;f}
  end

  def all_fields(sep = nil, header_hash = nil)
    produce
    key_field, fields = TSV.parse_header(self.open, sep, header_hash).values_at(0, 1).flatten.collect{|f| f.extend TSV::Field; f.namespace = namespace; f}
  end

  def fields_in_namespace(sep = nil, header_hash = nil)
    produce
    TSV.parse_header(self.open, sep, header_hash)[1].collect{|f| f.extend TSV::Field; f.namespace = namespace ;f}.select{|f| f.namespace == namespace}
  end

  def all_namespace_fields(namespace, sep = /\t/, header_hash = "#")
    produce
    key_field, fields = TSV.parse_header(self.open, sep, header_hash).values_at(0, 1).flatten.collect{|f| f.extend TSV::Field; f.namespace = namespace; f}.select{|f| f.namespace == namespace}
  end

  def filename
    self.to_s
  end 

  def exists?
    begin
      produce
      true
    rescue
      false
    end
  end

  def produce
    return self if File.exists? self

    raise "No especified pkg_data for file #{ self }, cannot produce" if pkg_data.nil?

    Log.debug("Trying to produce '#{ self }'")
    file, producer = pkg_module.reclaim self

    raise "File #{self} has not been claimed, cannot produce" if file.nil? or producer.nil?

    pkg_module.produce(self, producer[:get], producer[:subdir], producer[:sharedir])

    self
  end
end


<commit_msg>do not set datadir and namespace from Path<commit_after>require 'rbbt/util/misc'
require 'rbbt/util/tsv'

module Path
  attr_accessor :pkg_module, :datadir

  def self.find_files_back_to(path, target, subdir)
    return [] if path.nil?
    raise "Path #{ path } not in directory #{ subdir }" if not Misc.in_directory? path, subdir

    pkg_module = path.pkg_module

    files = []
    while path != subdir
      path = File.dirname(path)
      path.extend Path
      path.pkg_module = pkg_module
      path.datadir    = path.datadir
      if path[target].exists? 
        files << path[target]
      end
    end

    files
  end

  def self.path(string, datadir = nil, pkg_module = nil)
    string.extend Path
    string.datadir = datadir
    string.pkg_module = case
                        when pkg_module.nil?
                          nil
                        when String === pkg_module
                          Misc.string2const pkg_module
                        else
                          pkg_module
                        end
    string
  end

  def method_missing(name, *args, &block)
    new = File.join(self.dup, name.to_s)
    new.extend Path
    new.pkg_module = pkg_module
    new.datadir    = datadir
    new
  end

  def [](name)
    new = File.join(self.dup, name.to_s)
    new.extend Path
    new.pkg_module = pkg_module
    new.datadir    = datadir
    new
  end

  def namespace
    return nil if self.nil? or self.empty? or (not datadir.nil? and (self == datadir or File.dirname(self) == datadir))
    if File.directory? self
      File.basename(self)
    else
      File.basename(File.dirname(self))
    end
  end

  def identifier_files
    if datadir.nil?
      path = File.join(File.dirname(self), 'identifiers')
      path.extend Path
      path.pkg_module = pkg_module
      if path.exists?
        [path]
      else
        []
      end
    else
      identifier_files = Path.find_files_back_to(self, 'identifiers', datadir)
      return identifier_files.collect{|f| Path.path(f, datadir, pkg_module)}
    end
  end

  def tsv(key = nil, options = {})
    if options.empty? and Hash === key
      options, key = key, nil
    end

    produce
    TSV.new self, key, options
  end

  def index(options = {})
    produce
    TSV.index self, options
  end

  def open(options = {})
    produce
    Open.open(self, options)
  end

  def read(options = {})
    produce
    Open.read(self, options)
  end

  def fields(sep = nil, header_hash = nil)
    produce
    TSV.parse_header(self.open, sep, header_hash)[1].collect{|f| f.extend TSV::Field; f.namespace = namespace ;f}
  end

  def all_fields(sep = nil, header_hash = nil)
    produce
    key_field, fields = TSV.parse_header(self.open, sep, header_hash).values_at(0, 1).flatten.collect{|f| f.extend TSV::Field; f.namespace = namespace; f}
  end

  def fields_in_namespace(sep = nil, header_hash = nil)
    produce
    TSV.parse_header(self.open, sep, header_hash)[1].collect{|f| f.extend TSV::Field; f.namespace = namespace ;f}.select{|f| f.namespace == namespace}
  end

  def all_namespace_fields(namespace, sep = /\t/, header_hash = "#")
    produce
    key_field, fields = TSV.parse_header(self.open, sep, header_hash).values_at(0, 1).flatten.collect{|f| f.extend TSV::Field; f.namespace = namespace; f}.select{|f| f.namespace == namespace}
  end

  def filename
    self.to_s
  end 

  def exists?
    begin
      produce
      true
    rescue
      false
    end
  end

  def produce
    return self if File.exists? self

    raise "No especified pkg_data for file #{ self }, cannot produce" if pkg_data.nil?

    Log.debug("Trying to produce '#{ self }'")
    file, producer = pkg_module.reclaim self

    raise "File #{self} has not been claimed, cannot produce" if file.nil? or producer.nil?

    pkg_module.produce(self, producer[:get], producer[:subdir], producer[:sharedir])

    self
  end
end



{'fraction_non_alphanumeric': 0.10475340894609404, 'fraction_numerical': 0.001784121320249777, 'mean_word_length': 2.4196078431372547, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3354980', 'n_tokens_mistral': 2648, 'n_tokens_neox': 2552, 'n_words': 666}