<commit_before># encoding: utf-8
#
# author: <NAME>
# author: <NAME>

require "train/file/local"
require "train/file/remote"
require "train/extras/stat"

module Train
  class File # rubocop:disable Metrics/ClassLength
    def initialize(backend, path, follow_symlink = true)
      @backend = backend
      @path = path || ""
      @follow_symlink = follow_symlink

      sanitize_filename(path)
    end

    # This method gets override by particular os class.
    def sanitize_filename(_path)
      nil
    end

    # interface methods: these fields should be implemented by every
    # backend File
    DATA_FIELDS = %w{
      exist? mode owner group uid gid content mtime size selinux_label path
    }.freeze

    DATA_FIELDS.each do |m|
      define_method m.to_sym do
        raise NotImplementedError, "File must implement the #{m}() method."
      end
    end

    def to_json
      res = Hash[DATA_FIELDS.map { |x| [x, method(x).call] }]
      # additional fields provided as input
      res["type"] = type
      res["follow_symlink"] = @follow_symlink
      res
    end

    def type
      :unknown
    end

    def source
      if @follow_symlink<commit_msg>Remove multiply defined method warnings.<commit_after># encoding: utf-8
#
# author: <NAME>
# author: <NAME>

require "train/file/local"
require "train/file/remote"
require "train/extras/stat"

module Train
  class File # rubocop:disable Metrics/ClassLength
    def initialize(backend, path, follow_symlink = true)
      @backend = backend
      @path = path || ""
      @follow_symlink = follow_symlink

      sanitize_filename(path)
    end

    # This method gets override by particular os class.
    def sanitize_filename(_path)
      nil
    end

    # interface methods: these fields should be implemented by every
    # backend File
    DATA_FIELDS = %w{
      exist? mode owner group uid gid content mtime size selinux_label path
    }.freeze

    DATA_FIELDS.each do |m|
      next if m == "path"

      define_method m do
        raise NotImplementedError, "File must implement the #{m}() method."
      end
    end

    def to_json
      res = Hash[DATA_FIELDS.map { |x| [x, method(x).call] }]
      # additional fields provided as input
      res["type"] = type
      res["follow_symlink"] = @follow_symlink
      res
    end

    def type
      :unknown
    end

    def source
      if @follow_symlink
{'fraction_non_alphanumeric': 0.0865182696346073, 'fraction_numerical': 0.0008399832003359933, 'mean_word_length': 2.6477794793261866, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6672497', 'n_tokens_mistral': 742, 'n_tokens_neox': 723, 'n_words': 229}