<commit_before>module Gitlab
  ## 
  # Class that represents a simplified path to a file or directory
  #
  # This is IO-operations safe class, that does similar job to 
  # Ruby's Pathname but without the risk of accessing filesystem.
  #
  # TODO, better support for '../' and './'
  #
  class StringPath
    attr_reader :path, :universe

    def initialize(path, universe, metadata = [])
      @path = prepare(path)
      @universe = universe.map { |entry| prepare(entry) }
      @universe << './' unless @universe.include?('./')
      @metadata = metadata
    end

    def to_s
      @path
    end

    def absolute?
      @path.start_with?('/')
    end

    def relative?
      !absolute?
    end

    def directory?
      @path.end_with?('/')
    end

    def file?
      !directory?
    end

    def has_parent?
      @universe.include?(@path.sub(basename, ''))
    end

    def parent
      return nil unless has_parent?
      new(@path.sub(basename, ''))
    end

    def basename
      directory? ? name + ::File::SEPARATOR : name
    end

    def name
      @path.split(::File::SEPARATOR).last
    end

    def has_descendants?
      descendants.any?
    end

    def descendants
      return [] unless directory?
      children = @universe.select { |entry| entry =~ /^#{@path}.+/ }
      children.map { |path| new(path) }
    end

    def children
      return [] unless directory?
      return @children if @children
      children = @universe.select { |entry| entry =~ %r{^#{@path}[^/]+/?$} }
      @children = children.map { |path| new(path) }
    end

    def directories
      return [] unless directory?
      children.select(&:directory?)
    end

    def directories!
      has_parent? ? directories.prepend(new(@path + '../')) : directories
    end

    def files
      return [] unless directory?
      children.select(&:file?)
    end

    def metadata
      index = @universe.index(@path)
      @metadata[index]
    end

    def ==(other)
      @path == other.path && @universe == other.universe
    end

    def inspect
      "#{self.class.name}: #{@path}"
    end

    private

    def new(path)
      self.class.new(path, @universe)
    end

    def prepare(path)
      return path if path =~ %r{^(/|\.|\.\.)}
      path.dup.prepend('./')
    end
  end
end<commit_msg>Add path sanitization to `StringPath`<commit_after>module Gitlab
  ## 
  # Class that represents a simplified path to a file or directory
  #
  # This is IO-operations safe class, that does similar job to 
  # Ruby's Pathname but without the risk of accessing filesystem.
  #
  class StringPath
    attr_reader :path, :universe

    def initialize(path, universe, metadata = [])
      @path = sanitize(path)
      @universe = universe.map { |entry| sanitize(entry) }
      @metadata = metadata
    end

    def to_s
      @path
    end

    def absolute?
      @path.start_with?('/')
    end

    def relative?
      !absolute?
    end

    def directory?
      @path.end_with?('/')
    end

    def file?
      !directory?
    end

    def has_parent?
      @universe.include?(@path.sub(basename, ''))
    end

    def parent
      return nil unless has_parent?
      new(@path.sub(basename, ''))
    end

    def basename
      directory? ? name + ::File::SEPARATOR : name
    end

    def name
      @path.split(::File::SEPARATOR).last
    end

    def has_descendants?
      descendants.any?
    end

    def descendants
      return [] unless directory?
      select { |entry| entry =~ /^#{@path}.+/ }
    end

    def children
      return [] unless directory?
      return @children if @children

      @children = select do |entry|
        self.class.child?(@path, entry)
      end
    end

    def directories
      return [] unless directory?
      children.select(&:directory?)
    end

    def directories!
      has_parent? ? directories.prepend(new(@path + '../')) : directories
    end

    def files
      return [] unless directory?
      children.select(&:file?)
    end

    def metadata
      index = @universe.index(@path)
      @metadata[index]
    end

    def ==(other)
      @path == other.path && @universe == other.universe
    end

    def inspect
      "#{self.class.name}: #{@path}"
    end

    private

    def new(path)
      self.class.new(path, @universe)
    end

    def select
      selected = @universe.select { |entry| yield entry }
      selected.map { |path| new(path) }
    end

    def sanitize(path)
      self.class.sanitize(path)
    end

    def self.sanitize(path)
      # It looks like Pathname#new doesn't touch a file system,
      # neither Pathname#cleanpath does, so it is, hopefully, filesystem safe

      clean = Pathname.new(path).cleanpath.to_s
      raise ArgumentError, 'Invalid path' if clean.start_with?('../')
      clean + (path.end_with?('/') ? '/' : '')
    end

    def self.child?(path, entry)
      entry =~ %r{^#{path}[^/\s]+/?$}
    end
  end
end
{'fraction_non_alphanumeric': 0.12535612535612536, 'fraction_numerical': 0.0, 'mean_word_length': 2.2061317677756036, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4706966', 'n_tokens_mistral': 1573, 'n_tokens_neox': 1501, 'n_words': 418}