<commit_before>abstract class VirtualFile
  property filename : String
  property fullpath : String
  property fulldirectorypath : String
  property directory : String
  property size : UInt64
  property content : String

  def initialize
    @filename = nil
    @fullpath = nil
    @fulldirectorypath = nil
    @directory = nil
    @size = nil
    @content = nil
  end

  # Yields each line of the file
  def each_line
    @content.each_line do |line|
      yield line
    end
  end

  def to_s(io)
    io << @content
  end
end

# A virtual file
class EvalFile < VirtualFile
  def initialize(content)
    @filename = "virtual-#{Time.new.epoch_ms}.charly"
    @fullpath = "/virtual/#{@filename}"
    @fulldirectorypath = "/virtual"
    @directory = "virtual"
    @size = content.size.to_u64
    @content = content
  end
end

# A real file that exists on the filesystem
class RealFile < VirtualFile
  def initialize(path)
    @filename = File.basename path
    @fullpath = File.expand_path path
    @fulldirectorypath = File.dirname @fullpath
    @directory = File.dirname path
    @size = File.size path
    @content = File.read path
  end
end<commit_msg>Removed unused properties<commit_after>abstract class VirtualFile
  property filename : String
  property fullpath : String
  property fulldirectorypath : String
  property content : String

  def initialize
    @filename = nil
    @fullpath = nil
    @fulldirectorypath = nil
    @content = nil
  end

  # Yields each line of the file
  def each_line
    @content.each_line do |line|
      yield line
    end
  end

  def to_s(io)
    io << @content
  end
end

# A virtual file
class EvalFile < VirtualFile
  def initialize(content)
    @filename = "virtual-#{Time.new.epoch_ms}.charly"
    @fullpath = ENV["PWD"] + "/#{@filename}"
    @fulldirectorypath = ENV["PWD"]
    @content = content
  end
end

# A real file that exists on the filesystem
class RealFile < VirtualFile
  def initialize(path)
    @filename = File.basename path
    @fullpath = File.expand_path path
    @fulldirectorypath = File.dirname @fullpath
    @content = File.read path
  end
end
{'fraction_non_alphanumeric': 0.08183538315988648, 'fraction_numerical': 0.001892147587511826, 'mean_word_length': 3.1067961165048543, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '612825', 'n_tokens_mistral': 663, 'n_tokens_neox': 649, 'n_words': 206}