<commit_before>require 'bel'
require 'bel_parser/expression/model'
require 'bel/nanopub/util'
require 'mongo'
require_relative 'api'
require_relative 'mongo_facet'

module OpenBEL
  module Nanopub

    class Nanopub
      include API
      include Mongo

      def initialize(options = {})
        host = options[:host]
        port = options[:port]
        db   = options[:database]
        @db  = MongoClient.new(
          host,
          port,
          :op_timeout => nil,
          :pool_size  => 30
        ).db(db)

        # Authenticate user if provided.
        username = options[:username]
        password = options[:password]
        if username && password
          auth_db = options[:authentication_database] || db
          @db.authenticate(username, password, nil, auth_db)
        end

        @collection      = @db[:nanopub]
        @nanopub_facets = NanopubFacets.new(options)

        # ensure all indexes are created and maintained
        ensure_all_indexes
      end

      def create_nanopub(nanopub)
        # insert nanopub; acknowledge journal
        if nanopub.respond_to?(:each_pair)
          _id = @collection.insert(nanopub, :w => 1, :j => true)

          # remove nanopub_facets after insert to facets
          remove_nanopub_facets(_id)
          _id
        elsif nanopub.respond_to?(:each)
          @collection.insert(nanopub.to_a, :w => 1, :j => true)
        else
          raise "nanopub type #{nanopub.class} cannot be inserted into Mongo."
        end
      end

      def find_nanopub_by_id(value)
        @collection.find_one(to_id(value))
      end

      def find_nanopub(filters = [], offset = 0, length = 0, facet = false, facet_value_limit = -1)
        if includes_fts_search?(filters)
          text_search = get_fts_search(filters)
          nanopub_aggregate(text_search, filters, offset, length, facet, facet_value_limit)
        else
          nanopub_query(filters, offset, length, facet, facet_value_limit)
        end
      end

      def find_dataset_nanopub(dataset, filters = [], offset = 0, length = 0, facet = false, facet_value_limit = -1)
        query_hash = to_query(filters)
        query_hash[:$and] ||= []
        query_hash[:$and].unshift({
          :_dataset => dataset[:identifier]
        })

        query_opts = query_options(
          query_hash,
          :skip  => offset,
          :limit => length,
          :sort  => [
            [:bel_statement, :asc]
          ]
        )

        results = {
          :cursor => @collection.find(query_hash, query_opts)
        }
        if facet
          facets_cursor = @nanopub_facets.find_facets(query_hash, filters, facet_value_limit)
          results[:facets] = facets_cursor.to_a
        end

        results
      end

      def find_all_namespace_references
        references = @collection.aggregate(
          [
            {
              :$project => {"references.namespaces" => 1}
            },
            {
              :$unwind => "$references.namespaces"
            },
            {
              :$project => {
                keyword: "$references.namespaces.keyword",
                type:    "$references.namespaces.type",
                domain:  "$references.namespaces.domain"
              }
            },
            {
              :$group => {
                _id: "$keyword",
                type: {
                  :$addToSet => "$type"
                },
                domain: {
                  :$addToSet => "$domain"
                }
              }
            },
            {
              :$unwind => "$type"
            },
            {
              :$unwind => "$domain"
            },
            {
              :$project => {
                keyword: "$_id", type: "$type", domain: "$domain", _id: 0
              }
            }
          ],
          {
            allowDiskUse: true,
            cursor: {}
          }
        )

        union = []
        remap = {}
        references.each do |obj|
          namespace =
            BELParser::Expression::Model::Namespace.new(
              *obj.values_at('keyword', 'type', 'domain')
            )
          union, new_remap =
            BEL::Nanopub.union_namespace_references(union, [namespace], 'incr')
          remap.merge!(new_remap)
        end

        remap
      end

      def find_all_annotation_references
        references = @collection.aggregate(
          [
            {
              :$project => {"references.annotations" => 1}
            },
            {
              :$unwind => "$references.annotations"
            },
            {
              :$project => {
                keyword: "$references.annotations.keyword",<commit_msg>fixes #119<commit_after>require 'bel'
require 'bel_parser/expression/model'
require 'bel/nanopub/util'
require 'mongo'
require_relative 'api'
require_relative 'mongo_facet'

module OpenBEL
  module Nanopub

    class Nanopub
      include API
      include Mongo

      def initialize(options = {})
        host = options[:host]
        port = options[:port]
        db   = options[:database]
        @db  = MongoClient.new(
          host,
          port,
          :op_timeout => nil,
          :pool_size  => 30
        ).db(db)

        # Authenticate user if provided.
        username = options[:username]
        password = options[:password]
        if username && password
          auth_db = options[:authentication_database] || db
          @db.authenticate(username, password, nil, auth_db)
        end

        @collection      = @db[:nanopub]
        @nanopub_facets = NanopubFacets.new(options)

        # ensure all indexes are created and maintained
        ensure_all_indexes
      end

      def create_nanopub(nanopub)
        # insert nanopub; acknowledge journal
        if nanopub.respond_to?(:each_pair)
          _id = @collection.insert(nanopub, :w => 1, :j => true)

          # remove nanopub_facets after insert to facets
          remove_nanopub_facets(_id)
          _id
        elsif nanopub.respond_to?(:each)
          @collection.insert(nanopub.to_a, :w => 1, :j => true)
        else
          raise "nanopub type #{nanopub.class} cannot be inserted into Mongo."
        end
      end

      def find_nanopub_by_id(value)
        @collection.find_one(to_id(value))
      end

      def find_nanopub(filters = [], offset = 0, length = 0, facet = false, facet_value_limit = -1)
        if includes_fts_search?(filters)
          text_search = get_fts_search(filters)
          nanopub_aggregate(text_search, filters, offset, length, facet, facet_value_limit)
        else
          nanopub_query(filters, offset, length, facet, facet_value_limit)
        end
      end

      def find_dataset_nanopub(dataset, filters = [], offset = 0, length = 0, facet = false, facet_value_limit = -1)
        query_hash = to_query(filters)
        query_hash[:$and] ||= []
        query_hash[:$and].unshift({
          :_dataset => dataset[:identifier]
        })

        query_opts = query_options(
          query_hash,
          :skip  => offset,
          :limit => length,
          :sort  => [
            [:bel_statement, :asc]
          ]
        )

        results = {
          :cursor => @collection.find(query_hash, query_opts)
        }
        if facet
          facets_cursor = @nanopub_facets.find_facets(query_hash, filters, facet_value_limit)
          results[:facets] = facets_cursor.to_a
        end

        results
      end

      def find_all_namespace_references
        references = @collection.aggregate(
          [
            {
              :$project => {"references.namespaces" => 1}
            },
            {
              :$unwind => "$references.namespaces"
            },
            {
              :$project => {
                keyword: "$references.namespaces.keyword",
                type:    "$references.namespaces.type",
                domain:  "$references.namespaces.domain"
              }
            },
            {
              :$group => {
                _id: "$keyword",
                type: {
                  :$addToSet => "$type"
                },
                domain: {
                  :$addToSet => "$domain"
                }
              }
            },
            {
              :$unwind => "$type"
            },
            {
              :$unwind => "$domain"
            },
            {
              :$project => {
                keyword: "$_id", type: "$type", domain: "$domain", _id: 0
              }
            }
          ],
          {
            allowDiskUse: true,
            cursor: {}
          }
        )

        union = []
        remap = {}
        references.each do |obj|
          keyword = obj['keyword']
          type = obj['type']
          domain = obj['domain']
          uri = url = nil
          if type == :url
              url = domain
          elsif type == :uri
              uri = domain
          end
          args = [keyword, uri, url]
          namespace = BELParser::Expression::Model::Namespace.new(*args)
          union, new_remap =
            BEL::Nanopub.union_namespace_references(union, [namespace], 'incr')
          remap.merge!(new_remap)
        end

        remap
      end

      def find_all_annotation_references
        references = @collection.aggregate(
          [
            {
              :$project => {"references.annotations" => 1}
            },
            {
              :$unwind => "$references.annotations"
            },
            {
              :$project => {
                keyword: "$references.annotations.keyword",
{'fraction_non_alphanumeric': 0.1075738660008323, 'fraction_numerical': 0.0030170620058260506, 'mean_word_length': 1.4756631470512491, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3375803', 'n_tokens_mistral': 2794, 'n_tokens_neox': 2636, 'n_words': 539}