<commit_before>require 'rdf'
require 'enumerator'
require 'mongo'

module RDF
  class Statement
    ##
    # Creates a BSON representation of the statement.
    # @return [Hash]
    def to_mongo
      self.to_hash.inject({}) do |hash, (place_in_statement, entity)|
        hash.merge(RDF::Mongo::Conversion.to_mongo(entity, place_in_statement))
      end
    end

    ##
    # Create BSON for a statement representation. Note that if the statement has no graph name,
    # a value of `false` will be used to indicate the default context
    #
    # @param [RDF::Statement] statement
    # @return [Hash] Generated BSON representation of statement.
    def self.from_mongo(statement)
      RDF::Statement.new(
        subject:    RDF::Mongo::Conversion.from_mongo(statement['s'], statement['st'], statement['sl']),
        predicate:  RDF::Mongo::Conversion.from_mongo(statement['p'], statement['pt'], statement['pl']),
        object:     RDF::Mongo::Conversion.from_mongo(statement['o'], statement['ot'], statement['ol']),
        graph_name: RDF::Mongo::Conversion.from_mongo(statement['c'], statement['ct'], statement['cl']))
    end
  end

  module Mongo
    autoload :VERSION, "rdf/mongo/version"

    class Conversion
      ##
      # Translate an RDF::Value type to BSON key/value pairs.
      #
      # @param [RDF::Value, Symbol, false, nil] value
      #   URI, BNode or Literal. May also be a Variable or Symbol to indicate
      #   a pattern for a named graph, or `false` to indicate the default graph.
      #   A value of `nil` indicates a pattern that matches any value.
      # @param [:subject, :predicate, :object, :graph_name] place_in_statement
      #   Position within statement.
      # @return [Hash] BSON representation of the statement
      def self.to_mongo(value, place_in_statement)
        case value
        when RDF::URI
          v, k = value.to_s, :u
        when RDF::Literal
          if value.has_language?
            v, k, ll = value.value, :ll, value.language.to_s
          elsif value.has_datatype?
            v, k, ll = value.value, :lt, value.datatype.to_s
          else
            v, k, ll = value.value, :l, nil
          end
        when RDF::Node
          v, k = value.id.to_s, :n
        when RDF::Query::Variable, Symbol
          # Returns anything other than the default context
          v, k = nil, {"$ne" => :default}
        when false
          # Used for the default context
          v, k = false, :default
        when nil
          v, k = nil, nil
        else
          v, k = value.to_s, :u
        end
        v = nil if v == ''

        case place_in_statement
        when :subject
          t, k1, lt = :st, :s, :sl
        when :predicate
          t, k1, lt = :pt, :p, :pl
        when :object
          t, k1, lt = :ot, :o, :ol
        when :graph_name
          t, k1, lt = :ct, :c, :cl
        end
        h = {k1 => v, t => k, lt => ll}
        h.delete_if {|kk,_| h[kk].nil?}
      end

      ##
      # Translate an BSON positional reference to an RDF Value.
      #
      # @return [RDF::Value]
      def self.from_mongo(value, value_type = :u, literal_extra = nil)
        case value_type
        when :u
          RDF::URI.intern(value)
        when :ll
          RDF::Literal.new(value, language: literal_extra.to_sym)
        when :lt
          RDF::Literal.new(value, datatype: RDF::URI.intern(literal_extra))
        when :l
          RDF::Literal.new(value)
        when :n
          @nodes ||= {}
          @nodes[value] ||= RDF::Node.new(value)
        when :default
          nil # The default context returns as nil, although it's queried as false.
        end
      end
    end

    class Repository < ::RDF::Repository
      # The Mongo database instance
      # @return [Mongo::DB]
      attr_reader :client

      # The collection used for storing quads
      # @return [Mongo::Collection]
      attr_reader :collection

      ##
      # Initializes this repository instance.
      #
      # @overload initialize(options = {}, &block)
      #   @param  [Hash{Symbol => Object}] options
      #   @option options [String, #to_s] :title (nil)
      #   @option options [URI, #to_s]    :uri (nil)
      #     URI in the form `mongodb://host:port/db`. The URI should also identify the collection use, but appending a `collection` path component such as `mongodb://host:port/db/collection`, this ensures that the collection will be maintained if cloned. See [Mongo::Client options](https://docs.mongodb.org/ecosystem/tutorial/ruby-driver-tutorial-2-0/#uri-options-conversions) for more information on Mongo URIs.
      #
      # @overload initialize(options = {}, &block)
      #   @param  [Hash{Symbol => Object}] options
      #     See [Mongo::Client options](https://docs.mongodb.org/ecosystem/tutorial/ruby-driver-tutorial-2-0/#uri-options-conversions) for more information on Mongo Client options.
      #   @option options [String, #to_s] :title (nil)
      #   @option options [String] :host
      #     a single address or an array of addresses, which may contain a port designation
      #   @option options [Integer] :port (27017) applied to host address(es)
      #   @option options [String] :database ('quadb')
      #   @option options [String] :collection ('quads')
      #
      # @yield  [repository]
      # @yieldparam [Repository] repository
      def initialize(options = {}, &block)
        collection = nil
        if options[:uri]
          options = options.dup
          uri = RDF::URI(options.delete(:uri))
          _, db, coll = uri.path.split('/')
          collection = coll || options.delete(:collection)
          db ||= "quadb"
          uri.path = "/#{db}" if coll
          @client = ::Mongo::Client.new(uri.to_s, options)
        else
          warn "[DEPRECATION] RDF::Mongo::Repository#initialize expects a uri argument. Called from #{Gem.location_of_caller.join(':')}" unless options.empty?
          options[:database] ||= options.delete(:db) # 1.x compat
          options[:database] ||= 'quadb'
          hosts = Array(options[:host] || 'localhost')
          hosts.map! {|h| "#{h}:#{options[:port]}"} if options[:port]
          @client = ::Mongo::Client.new(hosts, options)
        end

        @collection = @client[options.delete(:collection) || 'quads']
        @collection.indexes.create_many([
          {key: {s: 1}},
          {key: {p: 1}},
          {key: {o: "hashed"}},
          {key: {c: 1}},
          {key: {s: 1, p: 1}},
          #{key: {s: 1, o: "hashed"}}, # Muti-key hashed indexes not allowed
          #{key: {p: 1, o: "hashed"}}, # Muti-key hashed indexes not allowed
        ])
        super(options, &block)
      end

      # @see RDF::Mutable#insert_statement
      def supports?(feature)
        case feature.to_sym
          when :graph_name   then true
          when :atomic_write then true
          when :validity     then @options.fetch(:with_validity, true)
          else false
        end
      end

      def apply_changeset(changeset)
        ops = []

        changeset.deletes.each do |d|
          ops << { delete_one: { filter: statement_to_delete(d)} }
        end

        changeset.inserts.each do |i|
          ops << { update_one: { filter: statement_to_insert(i), update: statement_to_insert(i), upsert: true} }
        end

        @collection.bulk_write(ops, ordered: true)
      end

      def insert_statement(statement)
        st_mongo = statement_to_insert(statement)
        @collection.update_one(st_mongo, st_mongo, upsert: true)
      end

      # @see RDF::Mutable#delete_statement
      def delete_statement(statement)
        st_mongo = statement_to_delete(statement)
        @collection.delete_one(st_mongo)
      end

      ##<commit_msg>Only use ordered bulk write if necessary<commit_after>require 'rdf'
require 'enumerator'
require 'mongo'

module RDF
  class Statement
    ##
    # Creates a BSON representation of the statement.
    # @return [Hash]
    def to_mongo
      self.to_hash.inject({}) do |hash, (place_in_statement, entity)|
        hash.merge(RDF::Mongo::Conversion.to_mongo(entity, place_in_statement))
      end
    end

    ##
    # Create BSON for a statement representation. Note that if the statement has no graph name,
    # a value of `false` will be used to indicate the default context
    #
    # @param [RDF::Statement] statement
    # @return [Hash] Generated BSON representation of statement.
    def self.from_mongo(statement)
      RDF::Statement.new(
        subject:    RDF::Mongo::Conversion.from_mongo(statement['s'], statement['st'], statement['sl']),
        predicate:  RDF::Mongo::Conversion.from_mongo(statement['p'], statement['pt'], statement['pl']),
        object:     RDF::Mongo::Conversion.from_mongo(statement['o'], statement['ot'], statement['ol']),
        graph_name: RDF::Mongo::Conversion.from_mongo(statement['c'], statement['ct'], statement['cl']))
    end
  end

  module Mongo
    autoload :VERSION, "rdf/mongo/version"

    class Conversion
      ##
      # Translate an RDF::Value type to BSON key/value pairs.
      #
      # @param [RDF::Value, Symbol, false, nil] value
      #   URI, BNode or Literal. May also be a Variable or Symbol to indicate
      #   a pattern for a named graph, or `false` to indicate the default graph.
      #   A value of `nil` indicates a pattern that matches any value.
      # @param [:subject, :predicate, :object, :graph_name] place_in_statement
      #   Position within statement.
      # @return [Hash] BSON representation of the statement
      def self.to_mongo(value, place_in_statement)
        case value
        when RDF::URI
          v, k = value.to_s, :u
        when RDF::Literal
          if value.has_language?
            v, k, ll = value.value, :ll, value.language.to_s
          elsif value.has_datatype?
            v, k, ll = value.value, :lt, value.datatype.to_s
          else
            v, k, ll = value.value, :l, nil
          end
        when RDF::Node
          v, k = value.id.to_s, :n
        when RDF::Query::Variable, Symbol
          # Returns anything other than the default context
          v, k = nil, {"$ne" => :default}
        when false
          # Used for the default context
          v, k = false, :default
        when nil
          v, k = nil, nil
        else
          v, k = value.to_s, :u
        end
        v = nil if v == ''

        case place_in_statement
        when :subject
          t, k1, lt = :st, :s, :sl
        when :predicate
          t, k1, lt = :pt, :p, :pl
        when :object
          t, k1, lt = :ot, :o, :ol
        when :graph_name
          t, k1, lt = :ct, :c, :cl
        end
        h = {k1 => v, t => k, lt => ll}
        h.delete_if {|kk,_| h[kk].nil?}
      end

      ##
      # Translate an BSON positional reference to an RDF Value.
      #
      # @return [RDF::Value]
      def self.from_mongo(value, value_type = :u, literal_extra = nil)
        case value_type
        when :u
          RDF::URI.intern(value)
        when :ll
          RDF::Literal.new(value, language: literal_extra.to_sym)
        when :lt
          RDF::Literal.new(value, datatype: RDF::URI.intern(literal_extra))
        when :l
          RDF::Literal.new(value)
        when :n
          @nodes ||= {}
          @nodes[value] ||= RDF::Node.new(value)
        when :default
          nil # The default context returns as nil, although it's queried as false.
        end
      end
    end

    class Repository < ::RDF::Repository
      # The Mongo database instance
      # @return [Mongo::DB]
      attr_reader :client

      # The collection used for storing quads
      # @return [Mongo::Collection]
      attr_reader :collection

      ##
      # Initializes this repository instance.
      #
      # @overload initialize(options = {}, &block)
      #   @param  [Hash{Symbol => Object}] options
      #   @option options [String, #to_s] :title (nil)
      #   @option options [URI, #to_s]    :uri (nil)
      #     URI in the form `mongodb://host:port/db`. The URI should also identify the collection use, but appending a `collection` path component such as `mongodb://host:port/db/collection`, this ensures that the collection will be maintained if cloned. See [Mongo::Client options](https://docs.mongodb.org/ecosystem/tutorial/ruby-driver-tutorial-2-0/#uri-options-conversions) for more information on Mongo URIs.
      #
      # @overload initialize(options = {}, &block)
      #   @param  [Hash{Symbol => Object}] options
      #     See [Mongo::Client options](https://docs.mongodb.org/ecosystem/tutorial/ruby-driver-tutorial-2-0/#uri-options-conversions) for more information on Mongo Client options.
      #   @option options [String, #to_s] :title (nil)
      #   @option options [String] :host
      #     a single address or an array of addresses, which may contain a port designation
      #   @option options [Integer] :port (27017) applied to host address(es)
      #   @option options [String] :database ('quadb')
      #   @option options [String] :collection ('quads')
      #
      # @yield  [repository]
      # @yieldparam [Repository] repository
      def initialize(options = {}, &block)
        collection = nil
        if options[:uri]
          options = options.dup
          uri = RDF::URI(options.delete(:uri))
          _, db, coll = uri.path.split('/')
          collection = coll || options.delete(:collection)
          db ||= "quadb"
          uri.path = "/#{db}" if coll
          @client = ::Mongo::Client.new(uri.to_s, options)
        else
          warn "[DEPRECATION] RDF::Mongo::Repository#initialize expects a uri argument. Called from #{Gem.location_of_caller.join(':')}" unless options.empty?
          options[:database] ||= options.delete(:db) # 1.x compat
          options[:database] ||= 'quadb'
          hosts = Array(options[:host] || 'localhost')
          hosts.map! {|h| "#{h}:#{options[:port]}"} if options[:port]
          @client = ::Mongo::Client.new(hosts, options)
        end

        @collection = @client[options.delete(:collection) || 'quads']
        @collection.indexes.create_many([
          {key: {s: 1}},
          {key: {p: 1}},
          {key: {o: "hashed"}},
          {key: {c: 1}},
          {key: {s: 1, p: 1}},
          #{key: {s: 1, o: "hashed"}}, # Muti-key hashed indexes not allowed
          #{key: {p: 1, o: "hashed"}}, # Muti-key hashed indexes not allowed
        ])
        super(options, &block)
      end

      # @see RDF::Mutable#insert_statement
      def supports?(feature)
        case feature.to_sym
          when :graph_name   then true
          when :atomic_write then true
          when :validity     then @options.fetch(:with_validity, true)
          else false
        end
      end

      def apply_changeset(changeset)
        ops = []

        changeset.deletes.each do |d|
          ops << { delete_one: { filter: statement_to_delete(d)} }
        end

        changeset.inserts.each do |i|
          ops << { update_one: { filter: statement_to_insert(i), update: statement_to_insert(i), upsert: true} }
        end

        # Only used an ordered write if we have both deletes and inserts
        ordered = ! (changeset.inserts.empty? or changeset.deletes.empty?)
        @collection.bulk_write(ops, ordered: ordered)
      end

      def insert_statement(statement)
        st_mongo = statement_to_insert(statement)
        @collection.update_one(st_mongo, st_mongo, upsert: true)
      end

      # @see RDF::Mutable#delete_statement
      def delete_statement(statement)
        st_mongo = statement_to_delete(statement)
        @collection.delete_one(st_mongo)
      end

      ##
{'fraction_non_alphanumeric': 0.12292464878671776, 'fraction_numerical': 0.00280970625798212, 'mean_word_length': 2.2887442251154977, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6912020', 'n_tokens_mistral': 4852, 'n_tokens_neox': 4729, 'n_words': 1358}