<commit_before>module Sequel
  module Plugins
    # Sequel's built in Serialization plugin allows you to keep serialized
    # ruby objects in the database, while giving you deserialized objects
    # when you call an accessor.
    #
    # This plugin works by keeping the serialized value in the values, and
    # adding a @deserialized_values hash.  The reader method for serialized columns
    # will check the @deserialized_values for the value, return it if present,
    # or deserialized the entry in @values and return it.  The writer method will
    # set the @deserialized_values entry.  This plugin adds a before_save hook
    # that serializes all @deserialized_values to @values.
    #
    # You can specify the serialization format as a pair of serializer/deserializer
    # callable objects.  You can also specify the serialization format as a single
    # symbol, if such a symbol has a registered serializer/deserializer pair in the
    # plugin.  By default, the plugin registers the :marshal, :yaml, and :json
    # serialization formats.  To register your own serialization formats, use
    # Sequel::Plugins::Serialization.register_format.
    # If you use yaml or json format, you need to require the libraries, Sequel
    # does not do the requiring for you.
    #
    # You can specify the columns to serialize when loading the plugin, or later
    # using the serialize_attributes class method.
    #
    # Because of how this plugin works, it must be used inside each model class
    # that needs serialization, after any set_dataset method calls in that class.
    # Otherwise, it is possible that the default column accessors will take
    # precedence.
    #
    # == Example
    #
    #   require 'sequel'
    #   # Require json, as the plugin doesn't require it for you.
    #   require 'json'
    #
    #   # Register custom serializer/deserializer pair
    #   Sequel::Plugins::Serialization.register_format(:reverse,
    #     lambda{|v| v.reverse},
    #     lambda{|v| v.reverse})
    #
    #   class User < Sequel::Model
    #     # Built-in format support when loading the plugin
    #     plugin :serialization, :json, :permissions
    #
    #     # Built-in format support after loading the plugin using serialize_attributes
    #     plugin :serialization
    #     serialize_attributes :marshal, :permissions
    #
    #     # Use custom registered serialization format just like built-in format
    #     serialize_attributes :reverse, :password
    #
    #     # Use a custom serializer/deserializer pair without registering
    #     serialize_attributes [lambda{|v| v.reverse}, lambda{|v| v.reverse}], :password
    #   end
    #   user = User.create
    #   user.permissions = { :global => 'read-only' }
    #   user.save
    module Serialization
      # The default serializers supported by the serialization module.
      # Use register_format to add serializers to this hash.
      REGISTERED_FORMATS = {}

      # Set up the column readers to do deserialization and the column writers
      # to save the value in deserialized_values.
      def self.apply(model, *args)
        model.instance_eval do
          @deserialization_map = {}
          @serialization_map = {}
        end
      end
      
      # Automatically call serialize_attributes with the format and columns unless
      # no columns were provided.
      def self.configure(model, format=nil, *columns)
        model.serialize_attributes(format, *columns) unless columns.empty?
      end

      # Register a serializer/deserializer pair with a format symbol, to allow
      # models to pick this format by name.  Both serializer and deserializer
      # should be callable objects.
      def self.register_format(format, serializer, deserializer)
        REGISTERED_FORMATS[format] = [serializer, deserializer]
      end
      register_format(:marshal, lambda{|v| [Marshal.dump(v)].pack('m')}, lambda{|v| Marshal.load(v.unpack('m')[0]) rescue Marshal.load(v)})
      register_format(:yaml, lambda{|v| v.to_yaml}, lambda{|v| YAML.load(v)})
      register_format(:json, lambda{|v| v.to_json}, lambda{|v| Sequel.parse_json(v)})<commit_msg>Try to raise more accurate error message in serialization failure<commit_after>module Sequel
  module Plugins
    # Sequel's built in Serialization plugin allows you to keep serialized
    # ruby objects in the database, while giving you deserialized objects
    # when you call an accessor.
    #
    # This plugin works by keeping the serialized value in the values, and
    # adding a @deserialized_values hash.  The reader method for serialized columns
    # will check the @deserialized_values for the value, return it if present,
    # or deserialized the entry in @values and return it.  The writer method will
    # set the @deserialized_values entry.  This plugin adds a before_save hook
    # that serializes all @deserialized_values to @values.
    #
    # You can specify the serialization format as a pair of serializer/deserializer
    # callable objects.  You can also specify the serialization format as a single
    # symbol, if such a symbol has a registered serializer/deserializer pair in the
    # plugin.  By default, the plugin registers the :marshal, :yaml, and :json
    # serialization formats.  To register your own serialization formats, use
    # Sequel::Plugins::Serialization.register_format.
    # If you use yaml or json format, you need to require the libraries, Sequel
    # does not do the requiring for you.
    #
    # You can specify the columns to serialize when loading the plugin, or later
    # using the serialize_attributes class method.
    #
    # Because of how this plugin works, it must be used inside each model class
    # that needs serialization, after any set_dataset method calls in that class.
    # Otherwise, it is possible that the default column accessors will take
    # precedence.
    #
    # == Example
    #
    #   require 'sequel'
    #   # Require json, as the plugin doesn't require it for you.
    #   require 'json'
    #
    #   # Register custom serializer/deserializer pair
    #   Sequel::Plugins::Serialization.register_format(:reverse,
    #     lambda{|v| v.reverse},
    #     lambda{|v| v.reverse})
    #
    #   class User < Sequel::Model
    #     # Built-in format support when loading the plugin
    #     plugin :serialization, :json, :permissions
    #
    #     # Built-in format support after loading the plugin using serialize_attributes
    #     plugin :serialization
    #     serialize_attributes :marshal, :permissions
    #
    #     # Use custom registered serialization format just like built-in format
    #     serialize_attributes :reverse, :password
    #
    #     # Use a custom serializer/deserializer pair without registering
    #     serialize_attributes [lambda{|v| v.reverse}, lambda{|v| v.reverse}], :password
    #   end
    #   user = User.create
    #   user.permissions = { :global => 'read-only' }
    #   user.save
    module Serialization
      # The default serializers supported by the serialization module.
      # Use register_format to add serializers to this hash.
      REGISTERED_FORMATS = {}

      # Set up the column readers to do deserialization and the column writers
      # to save the value in deserialized_values.
      def self.apply(model, *args)
        model.instance_eval do
          @deserialization_map = {}
          @serialization_map = {}
        end
      end
      
      # Automatically call serialize_attributes with the format and columns unless
      # no columns were provided.
      def self.configure(model, format=nil, *columns)
        model.serialize_attributes(format, *columns) unless columns.empty?
      end

      # Register a serializer/deserializer pair with a format symbol, to allow
      # models to pick this format by name.  Both serializer and deserializer
      # should be callable objects.
      def self.register_format(format, serializer, deserializer)
        REGISTERED_FORMATS[format] = [serializer, deserializer]
      end
      register_format(:marshal, lambda{|v| [Marshal.dump(v)].pack('m')},
        lambda do |v|
          begin
            Marshal.load(v.unpack('m')[0])
          rescue => e
            begin
              # Backwards compatibility for unpacked marshal output.
              Marshal.load(v)
            rescue
              raise e
            end
          end
        end)
      register_format(:yaml, lambda{|v| v.to_yaml}, lambda{|v| YAML.load(v)})
      register_format(:json, lambda{|v| v.to_json}, lambda{|v| Sequel.parse_json(v)})
{'fraction_non_alphanumeric': 0.07374665581016633, 'fraction_numerical': 0.00023263929277654997, 'mean_word_length': 2.8349687778768957, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '911554', 'n_tokens_mistral': 2336, 'n_tokens_neox': 2283, 'n_words': 875}