<commit_before>#    Copyright (C) 2011-2013 University of Southampton
#    Copyright (C) 2011-2013 <NAME>
#    Copyright (C) 2011-2013 <NAME>
#    Copyright (C) 2011-2013 <NAME>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License, version 3,
#    as published by the Free Software Foundation.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
import logging
import types

class ObjectStoreQuery:
    """ Handles JSON queries to an objectstore. """

    exact_types = [types.IntType, types.LongType, types.BooleanType, types.FloatType, types.StringType, types.UnicodeType]
    operator_map = {
        "$gt": ">",
        "$lt": "<",
        "$ge": ">=",
        "$le": "<="
    }
    logical_map = {
        "$or": "UNION",
        "$and": "INTERSECT",
    }

    def __init__(self):
        logging.debug("ObjectStoreQuery __init__")

        self.table_counter = 0

# SELECT DISTINCT wb_v_latest_triples.triple_order, wb_v_latest_triples.subject, wb_v_latest_triples.predicate, wb_v_latest_triples.obj_value, wb_v_latest_triples.obj_type, wb_v_latest_triples.obj_lang, wb_v_latest_triples.obj_datatype
# 
# FROM wb_v_latest_triples
# 
# WHERE wb_v_latest_triples.subject IN
#     (
#       SELECT DISTINCT subj_2.subject FROM wb_v_latest_triples AS subj_2
#          WHERE (subj_2.predicate = 'boxes'
# 	        AND subj_2.obj_value = ALL
# 	            (SELECT DISTINCT subj_1.subject FROM wb_v_latest_triples AS subj_1
#                         WHERE (subj_1.predicate = 'box' AND subj_1.obj_value = 'box1')
#                      INTERSECT
#                      SELECT DISTINCT subj_1.subject FROM wb_v_latest_triples AS subj_1
#                         WHERE (subj_1.predicate = 'box' AND subj_1.obj_value = 'box2')
#                     )
#                )
#     INTERSECT SELECT DISTINCT subj_2.subject FROM wb_v_latest_triples AS subj_2
#          WHERE (subj_2.predicate = 'boxes'
# 	        AND subj_2.obj_value = ALL
# 	            (SELECT DISTINCT subj_1.subject FROM wb_v_latest_triples AS subj_1
#                         WHERE (subj_1.predicate = 'box' AND subj_1.obj_value = 'box2')
#                         -- UNION / INTERSECT
#                     )
#                )  
#     )
# 

    def process_predicates(self, q):
        """ Process a query and return the query fragment (without surrounding SELECT / FROM etc) about the predicates.
            This function is recursable.
        """
        logging.debug("ObjectStoreQuery process_predicates: {0}".format(q))

        subqueries = []
        params = []

        def gen_subquery(pred, vals, is_subquery = False, operator = "="):

            #self.table_counter += 1
            #table = "sub_{0}".format(self.table_counter)


            # single value
            if is_subquery:
                value_querypart = vals
                params_querypart = []
            else:
                value_querypart = "%s"
                params_querypart = ["{0}".format(vals)] # format is to convert to a string before querying


            if pred == "@id":
                query, params = self.create_subquery(subject = value_querypart, subject_operator = operator, inner_params = params_querypart)

#                query = "SELECT DISTINCT {0}.subject FROM wb_v_latest_triples AS {0} WHERE ({0}.subject {2} {1})".format(table, value_querypart, operator)
#                params = []
            else:
                query, params = self.create_subquery(predicate = pred, obj = value_querypart, obj_operator = operator, inner_params = params_querypart)
#                query = "SELECT DISTINCT {0}.subject FROM wb_v_latest_triples AS {0} WHERE ({0}.predicate = %s AND {0}.obj_value {2} {1})".format(table, value_querypart, operator)
#                params = [pred]

            #params.extend(params_querypart)
            return query, params


        for predicate, val in q.items():

            if predicate in self.logical_map:
                # when the predicate matches a logical operator
                # e.g. {"$or": [ {"type": "foobar"}, {"type": "baz"} ] }<commit_msg>Fix dealing with predicates that might be none<commit_after>#    Copyright (C) 2011-2013 University of Southampton
#    Copyright (C) 2011-2013 <NAME>
#    Copyright (C) 2011-2013 <NAME>
#    Copyright (C) 2011-2013 <NAME>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License, version 3,
#    as published by the Free Software Foundation.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
import logging
import types

class ObjectStoreQuery:
    """ Handles JSON queries to an objectstore. """

    exact_types = [types.IntType, types.LongType, types.BooleanType, types.FloatType, types.StringType, types.UnicodeType]
    operator_map = {
        "$gt": ">",
        "$lt": "<",
        "$ge": ">=",
        "$le": "<="
    }
    logical_map = {
        "$or": "UNION",
        "$and": "INTERSECT",
    }

    def __init__(self):
        logging.debug("ObjectStoreQuery __init__")

        self.table_counter = 0

# SELECT DISTINCT wb_v_latest_triples.triple_order, wb_v_latest_triples.subject, wb_v_latest_triples.predicate, wb_v_latest_triples.obj_value, wb_v_latest_triples.obj_type, wb_v_latest_triples.obj_lang, wb_v_latest_triples.obj_datatype
# 
# FROM wb_v_latest_triples
# 
# WHERE wb_v_latest_triples.subject IN
#     (
#       SELECT DISTINCT subj_2.subject FROM wb_v_latest_triples AS subj_2
#          WHERE (subj_2.predicate = 'boxes'
# 	        AND subj_2.obj_value = ALL
# 	            (SELECT DISTINCT subj_1.subject FROM wb_v_latest_triples AS subj_1
#                         WHERE (subj_1.predicate = 'box' AND subj_1.obj_value = 'box1')
#                      INTERSECT
#                      SELECT DISTINCT subj_1.subject FROM wb_v_latest_triples AS subj_1
#                         WHERE (subj_1.predicate = 'box' AND subj_1.obj_value = 'box2')
#                     )
#                )
#     INTERSECT SELECT DISTINCT subj_2.subject FROM wb_v_latest_triples AS subj_2
#          WHERE (subj_2.predicate = 'boxes'
# 	        AND subj_2.obj_value = ALL
# 	            (SELECT DISTINCT subj_1.subject FROM wb_v_latest_triples AS subj_1
#                         WHERE (subj_1.predicate = 'box' AND subj_1.obj_value = 'box2')
#                         -- UNION / INTERSECT
#                     )
#                )  
#     )
# 

    def process_predicates(self, q):
        """ Process a query and return the query fragment (without surrounding SELECT / FROM etc) about the predicates.
            This function is recursable.
        """
        logging.debug("ObjectStoreQuery process_predicates: {0}".format(q))

        subqueries = []
        params = []

        def gen_subquery(pred, vals, is_subquery = False, operator = "="):

            #self.table_counter += 1
            #table = "sub_{0}".format(self.table_counter)


            # single value
            if is_subquery:
                value_querypart = vals
                params_querypart = []
            else:
                value_querypart = "%s"
                params_querypart = ["{0}".format(vals)] # format is to convert to a string before querying


            if pred == "@id":
                query, params = self.create_subquery(subject = value_querypart, subject_operator = operator, inner_params = params_querypart)

#                query = "SELECT DISTINCT {0}.subject FROM wb_v_latest_triples AS {0} WHERE ({0}.subject {2} {1})".format(table, value_querypart, operator)
#                params = []
            else:
                query, params = self.create_subquery(predicate = pred, obj = value_querypart, obj_operator = operator, inner_params = params_querypart)
#                query = "SELECT DISTINCT {0}.subject FROM wb_v_latest_triples AS {0} WHERE ({0}.predicate = %s AND {0}.obj_value {2} {1})".format(table, value_querypart, operator)
#                params = [pred]

            #params.extend(params_querypart)
            return query, params


        if q is None:
            return "", params # this means we can look at length of wheres in to_sql to determine if this is an empty query

        for predicate, val in q.items():

            if predicate in self.logical_map:
                # when the predicate matches a logical operator
                # e.g. {"$or": [ {"type": "foobar"}, {"type": "baz"} ] }
{'fraction_non_alphanumeric': 0.09783315276273023, 'fraction_numerical': 0.015601300108342361, 'mean_word_length': 2.2710843373493974, 'pattern_counts': {'":': 20, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '640405', 'n_tokens_mistral': 2933, 'n_tokens_neox': 2759, 'n_words': 770}