<commit_before>module Photish
  module Render
    class ChangeManifest
      def initialize(output_dir, worker_index, version_hash)
        @output_dir = output_dir
        @worker_index = worker_index
        @version_hash = version_hash
        @dirty = false
        @cache = {}
      end

      def record(key, file_path = nil)
        @dirty = true
        db[key] = checksum_of_file(file_path || key)
      end

      def changed?(key, file_path = nil)
        checksum_of_file(file_path || key) != old_checksum_of_file(key)
      end

      def flush_to_disk
        return unless dirty
        File.open(worker_db_file, 'w') { |f| f.write((db || {}).to_yaml) }
      end

      def preload
        db
      end

      def self.concat_db_files(output_dir, workers)
        changes = (1..workers).inject({}) do |changes, worker_index|
          file = worker_db_file(output_dir, worker_index)
          changes.merge!(YAML.load_file(file)) if File.exist?(file)
          changes
        end
        File.open(db_file(output_dir), 'w') { |f| f.write(changes.to_yaml) }
      end

      def self.db_file(output_dir)
        File.join(output_dir, '.changes.yml')
      end

      def self.worker_db_file(output_dir, index)
        File.join(output_dir, ".changes.#{index}.yml")
      end

      private

      attr_reader :output_dir,
                  :dirty,
                  :cache,
                  :version_hash,
                  :worker_index

      def checksum_of_file(file_path)
        cache.fetch(file_path.hash) do |key|
          cache[key] = version_hash.to_s +
                       Digest::MD5.file(file_path).hexdigest
        end
      end

      def old_checksum_of_file(file_path)
        db[file_path]
      end

      def db
        return @db if @db
        @db = File.exist?(db_file) ? YAML.load_file(db_file) : {}
      end

      def db_file
        self.class.db_file(output_dir)
      end

      def worker_db_file
        self.class.worker_db_file(output_dir, worker_index)<commit_msg>ensuring change from each worker is merged<commit_after>module Photish
  module Render
    class ChangeManifest
      def initialize(output_dir, worker_index, version_hash)
        @output_dir = output_dir
        @worker_index = worker_index
        @version_hash = version_hash
        @dirty = false
        @cache = {}
        @worker_db = {}
      end

      def record(key, file_path = nil)
        @dirty = true
        worker_db[key] = checksum_of_file(file_path || key)
      end

      def changed?(key, file_path = nil)
        checksum_of_file(file_path || key) != db[key] &&
          checksum_of_file(file_path || key) != worker_db[key]
      end

      def flush_to_disk
        return unless dirty
        File.open(worker_db_file, 'w') { |f| f.write(worker_db.to_yaml) }
      end

      def preload
        db
      end

      def self.concat_db_files(output_dir, workers)
        changes = (1..workers).inject({}) do |changes, worker_index|
          file = worker_db_file(output_dir, worker_index)
          changes.merge!(YAML.load_file(file)) if File.exist?(file)
          changes
        end
        File.open(db_file(output_dir), 'w') { |f| f.write(changes.to_yaml) }
      end

      def self.db_file(output_dir)
        File.join(output_dir, '.changes.yml')
      end

      def self.worker_db_file(output_dir, index)
        File.join(output_dir, ".changes.#{index}.yml")
      end

      private

      attr_reader :output_dir,
                  :dirty,
                  :cache,
                  :version_hash,
                  :worker_index,
                  :worker_db

      def checksum_of_file(file_path)
        cache.fetch(file_path.hash) do |key|
          cache[key] = version_hash.to_s +
                       Digest::MD5.file(file_path).hexdigest
        end
      end

      def db
        return @db if @db
        @db = File.exist?(db_file) ? YAML.load_file(db_file) : {}
      end

      def db_file
        self.class.db_file(output_dir)
      end

      def worker_db_file
        self.class.worker_db_file(output_dir, worker_index)
{'fraction_non_alphanumeric': 0.09697855750487329, 'fraction_numerical': 0.0009746588693957114, 'mean_word_length': 1.9489942528735633, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4025461', 'n_tokens_mistral': 1334, 'n_tokens_neox': 1296, 'n_words': 253}