<commit_before>require 'json'
require 'pstore'
require 'fileutils'

module Nutella

  module Persist

    # This module exposes a single method to retrieve a JSONStore
    def Persist.getJsonStore(file_name)
      dir = file_name[0..file_name.rindex('/')]
      file = file_name[file_name.rindex('/')..file_name.length-1]
      new_dir = dir + Nutella.run_id
      FileUtils.mkdir_p new_dir
      JSONStore.new(new_dir + file)
    end

  end

# JSONStore provides the same functionality as PStore, except it uses JSON
# to dump objects instead of Marshal.
# Example use:
#   store = nutella.persist.getJsonStore("json_store/json_test.json")
#   # Write
#   store.transaction { store["key"]="value" }
#   # Read
#   value = store.transaction { store["key"] }
#   puts value # prints "value"
#   # Dump the whole store
#   hash = store.transaction { store.to_h }
#   p hash # prints {"key" => "value"}

  class JSONStore < PStore

    def dump(table)
      table.to_json
    end

    def load(content)
      JSON.parse(content)
    end

    # Dumps the whole store to hash
    # example:
    # store = JSONStore.new("my_file.json")
    # hash = store.transaction { store.to_h }
    def to_h
      @table
    end

    def merge!( hash )
      @table.merge!(hash)
    end


    def marshal_dump_supports_canonical_option?
      false
    end

    EMPTY_MARSHAL_DATA = {}.to_json
    EMPTY_MARSHAL_CHECKSUM = Digest::MD5.digest(EMPTY_MARSHAL_DATA)
    def empty_marshal_data
      EMPTY_MARSHAL_DATA
    end
    def empty_marshal_checksum
      EMPTY_MARSHAL_CHECKSUM
    end
  end


end

<commit_msg>Added multithreading capability on nutella.persist<commit_after>require 'json'
require 'pstore'
require 'fileutils'
require 'thread'

module Nutella

  module Persist

    # This module exposes a single method to retrieve a JSONStore
    def Persist.getJsonStore(file_name)
      dir = file_name[0..file_name.rindex('/')]
      file = file_name[file_name.rindex('/')..file_name.length-1]
      new_dir = dir + Nutella.run_id
      FileUtils.mkdir_p new_dir
      JSONStore.new(new_dir + file)
    end

  end

# JSONStore provides the same functionality as PStore, except it uses JSON
# to dump objects instead of Marshal.
# Example use:
#   store = nutella.persist.getJsonStore("json_store/json_test.json")
#   # Write
#   store.transaction { store["key"]="value" }
#   # Read
#   value = store.transaction { store["key"] }
#   puts value # prints "value"
#   # Dump the whole store
#   hash = store.transaction { store.to_h }
#   p hash # prints {"key" => "value"}

  class JSONStore < PStore

    def initialize(path)
      super
      @semaphore = Mutex.new
    end

    def dump(table)
      table.to_json
    end

    def load(content)
      JSON.parse(content)
    end

    def transaction(read_only=false, &block)
      @semaphore.synchronize {
        # access shared resource
        super
      }
    end

    # Dumps the whole store to hash
    # example:
    # store = JSONStore.new("my_file.json")
    # hash = store.transaction { store.to_h }
    def to_h
      @table
    end

    def merge!( hash )
      @table.merge!(hash)
    end


    def marshal_dump_supports_canonical_option?
      false
    end

    EMPTY_MARSHAL_DATA = {}.to_json
    EMPTY_MARSHAL_CHECKSUM = Digest::MD5.digest(EMPTY_MARSHAL_DATA)
    def empty_marshal_data
      EMPTY_MARSHAL_DATA
    end
    def empty_marshal_checksum
      EMPTY_MARSHAL_CHECKSUM
    end
  end


end
{'fraction_non_alphanumeric': 0.09366869037294015, 'fraction_numerical': 0.0017346053772766695, 'mean_word_length': 2.8063806380638066, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2236213', 'n_tokens_mistral': 1203, 'n_tokens_neox': 1145, 'n_words': 279}