<commit_before>"""
Copyright (c) 2011, 2012, Regents of the University of California
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions 
are met:

 - Redistributions of source code must retain the above copyright
   notice, this list of conditions and the following disclaimer.
 - Redistributions in binary form must reproduce the above copyright
   notice, this list of conditions and the following disclaimer in the
   documentation and/or other materials provided with the
   distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 
THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
OF THE POSSIBILITY OF SUCH DAMAGE.
"""
"""
@author <NAME> <<EMAIL>>
"""

import os
import operator
import time
import datetime
import inspect
import logging
import collections
import itertools
import re

from twisted.internet import defer
from twisted.python import log

from smap import operators
from smap.util import build_recursive, is_string, flatten, SetDict 
from smap import sjson as json
from smap.contrib import dtutil

from smap.archiver.data import escape_string
from smap.archiver.querydata import extract_data
from smap.archiver import querygen as qg
from smap.archiver import data, stream, help, ast, consumers

import ply
import ply.lex as lex
import ply.yacc as yacc
import copy

tokens = (
    'HAS', 'AND', 'OR', 'NOT', 'LIKE', 'DISTINCT',
    'TAGS', 'SELECT', 'WHERE',
    'QSTRING','NUMBER', 'LVALUE', 'IN', 'DATA', 'DELETE',
    'SET', 'TILDE', 'BEFORE', 'AFTER', 'NOW', 'LIMIT', 'STREAMLIMIT',
    'APPLY', 'TO', 'AS', 'GROUP', 'BY', 'HELP', 'ALL', 
    'LTE', 'GTE', 'NE'
    )

precedence = (
    ('left', 'LIKE', 'TILDE', 'IN'),
    ('left', 'AND'),
    ('left', 'OR'),
    ('right', '='),
    ('right', 'NOT'),
    ('left', ','),
    ('left', '+'),
    ('left', '-'),
    ('left', '*'),
    ('left', '^'),
    )

reserved = {
    'where' : 'WHERE',
    'distinct' : 'DISTINCT',
    'select' : 'SELECT',
    'delete' : 'DELETE',
    'set' : 'SET',
    'tags' : 'TAGS',
    'has' : 'HAS',
    'and' : 'AND',
    'or' : 'OR',
    'not' : 'NOT',
    'like' : 'LIKE',
    'data': 'DATA',
    'in' : 'IN',
    'before' : 'BEFORE',
    'after' : 'AFTER',
    'now' : 'NOW',
    'limit' : 'LIMIT',
    'streamlimit' : 'STREAMLIMIT',
    '~' : 'TILDE',
    'apply' : 'APPLY',
    'to' : 'TO',
    'as' : 'AS',
    'group' : 'GROUP',
    'by' : 'BY',
    'help' : 'HELP',
    'all' : 'ALL',
    }
literals = '()[]*^.,<>=+-/'

time_units = re.compile('^(d|days?|h|hours?|m|minutes?|s|seconds?)$')
def get_timeunit(t):
    if not time_units.match(t):
        raise qg.QueryException("Invalid timeunit: %s" % t)
    if t.startswith('d'): return 'days'
    elif t.startswith('h'): return 'hours'
    elif t.startswith('m'): return 'minutes'
    elif t.startswith('s'): return 'seconds'

def t_CMP(t):
    r'(<=)|(>=)|(!=)'
    t.type = {
        '<=' : 'LTE',
        '>=' : 'GTE',
        '!=' : 'NE'}[t.value]
    return t

def t_QSTRING(t):
    r'("[^"\\]*?(\\.[^"\\]*?)*?")|(\'[^\'\\]*?(\\.[^\'\\]*?)*?\')'    
    if t.value[0] == '"':
        t.value = t.value[1:-1].replace('\\"', '"')
    elif t.value[0] == "'":
        t.value = t.value[1:-1].replace("\\'", "'")
    return t

def t_LVALUE(t):
    r'[a-zA-Z\~\$\_][a-zA-Z0-9\/\%_\-]*'
    t.type = reserved.get(t.value, 'LVALUE')
    return t

def t_NUMBER(t):
    r'([+-]?([0-9]*\.)?[0-9]+)'
    if '.' in t.value:
        try:
            t.value = float(t.value)
        except ValueError:
            print "Invalid floating point number", t.value
            t.value = 0
    else:
        try:
            t.value = int(t.value)
        except ValueError:
            print "Integer value too large %d", t.value
            t.value = 0
        
    return t
is_number = lambda x: isinstance(x, int) or isinstance(x, float)

t_ignore = " \t"
def t_newline(t):
    r'[\n\r]+'
    t.lexer.lineno += t.value.count("\n")

def t_error(t):
    print("Illegal character '%s'" % t.value[0])
    t.lexer.skip(1)

smapql_lex = lex.lex()

# precedence = ('and')
names = {}

## Extractors
##  extractors are chained onto postgres operators and transform the
##  result for presentation to the client.

def ext_default(x):
    return map(operator.itemgetter(0), x)

def ext_non_null(x):
    return filter(None, ext_default(x))

def ext_deletor(x):
    data.del_streams(map(operator.itemgetter(0), x))
    return map(operator.itemgetter(1), x)

def ext_plural(tags, vals):
    return [build_recursive(dict(zip(tags, v)), suppress=[]) for v in vals]

def ext_recursive(vals):
    return [build_recursive(x[0], suppress=[]) for x in vals]

def ext_set(tag_list, x):
    """Extractor for the set operation

    Returns time series uuid and modified tags
    """
    rv = [{'uuid': v[0]} for v in x]
    for rv_i, v in itertools.izip(rv, x):
        rv_i.update(v[1])
    return [build_recursive(x, suppress=[]) for x in rv]
        

TIMEZONE_PATTERNS = [
    "%m/%d/%Y",
    "%m/%d/%Y %H:%M",
    "%Y-%m-%dT%H:%M:%S",
    ]
def parse_time(ts):
    for pat in TIMEZONE_PATTERNS:
        try:
            return dtutil.strptime_tz(ts, pat, tzstr='America/Los_Angeles')
        except ValueError:
            continue
    raise ValueError("Invalid time string:" + ts)

def make_select_rv(t, sel, wherestmt='true'):
    return sel.extract, ("""SELECT %s FROM stream s, subscription sub
              WHERE (%s) AND (%s) AND
              sub.id = s.subscription_id""" % 
                         (sel.select, 
                          wherestmt,
                          qg.build_authcheck(t.parser.request)))

def build_setstring(setvals, wherevals):
    #  set tags 
    regex = None
    regex_tag = None
    for stmt in wherevals:
        if stmt.op == ast.Statement.OP_REGEX:
            if regex == None:
                regex = stmt
            else:
                raise qg.QueryException("Too many regexes in set.  Only one supported!")

    if regex == None:
        new_tags = ' || '.join(map(lambda (t,v): "hstore(%s, %s)" % 
                                   (escape_string(t), escape_string(v)),
                                   setvals))
    else:
        new_tags = ' || '.join(map(lambda (t,v): "hstore(%s, regexp_replace(metadata -> %s, '^.*%s.*$', %s))" % 
                                   (escape_string(t), 
                                    regex.args[0],
                                    regex.args[1][1:-1].replace('\\\\', '\\'),
                                    escape_string(v).replace('\\\\', '\\')),
                                   setvals))
        regex_tag = regex.args[0][1:-1]

    return new_tags, regex_tag


def make_set_rv(t):
    # build the query for a set expression
    if len(t) > 3:
        # if there is a where clause, grab it
        where_clause = t[4]
    else:
        # if no where clause, just do something fast that matches
        # everything
        where_clause = ast.Statement(ast.Statement.OP_UUID)
    noop = "noop" in t.parser.request.args

    # build a list of what tags we're updating, as well as the
    # name of the regex tag.  We're currently not smart enough to
    # deal with more than one regex match in a set statement, due
    # to issues with back references.
    new_tags, regex_tag = build_setstring(t[2], where_clause)
    tag_list = [v[0] for v in t[2]]
    if regex_tag: tag_list.append(regex_tag)

    # if we're in noop mode, we need to tweak the SQL
    if not noop: 
        command = "UPDATE stream SET metadata = metadata || "
        from_stmt = ""
    else:
        command = "SELECT uuid, "
        from_stmt = "FROM stream"

    # build the first part of the query that depends on if this is a noop
    q = "%(command)s %(hstore_expression)s %(from_stmt)s WHERE ID IN" % {
        "command": command,
        "hstore_expression": new_tags,
        "from_stmt": from_stmt }

    # the where caluse always looks the same
    q += "(SELECT s.id FROM stream s, subscription sub " + \
        "WHERE (" + where_clause.render() + ") AND s.subscription_id = sub.id AND " + \
        qg.build_authcheck(t.parser.request, forceprivate=True)  + ") " 

    if not noop:
        # return the list of modifications.  if this was a noop we
        # get this for free.
        q += " RETURNING uuid, slice(metadata, ARRAY[" + \
            ','.join((escape_string(v) for v in tag_list)) + "])"
    return lambda x: ext_set(tag_list, x), q

def make_delete_rv(t):
    noop = "noop" in t.parser.request.args
    extractor = ext_deletor if not noop else ext_default

    # a new delete inner statement enforces that we only delete
    # things which we have the key for.
    if len(t) > 2 and t[2] == 'where':
        # delete the whole stream, gone.  this also deletes the
        # data in the backend readingdb.
        if not noop:
            statement = "DELETE FROM stream"
            returning = "RETURNING id, uuid"
        else:
            statement = "SELECT uuid FROM stream"
            returning = ""
        return extractor, \
            """%(statement)s WHERE id IN (
                 SELECT s.id FROM stream s, subscription sub 
                 WHERE (%(restrict)s) AND s.subscription_id = sub.id AND 
                 (%(auth)s)
               ) %(returning)s
            """ % { 
            'restrict': t[3].render(),
            'auth': qg.build_authcheck(t.parser.request, forceprivate=True),
            'statement': statement,
            'returning': returning
            }
    else:
        # to make the return value right, we need to tweak the
        # where clause.  we want to only select streams where
        # there's actually something to delete, since all of the
        # streams the where-clause hits will get returned.
        tag_clause = ast.Statement(ast.Statement.OP_OR,
                                   *[ast.Statement(ast.Statement.OP_HAS, tag) 
                                     for tag in t[2]])
        if len(t) > 3: 
            where_clause = ast.Statement(ast.Statement.OP_AND, tag_clause, t[4])
        else:
            # if there's no where clause, we only look at streams
            # with the right tag.
            where_clause = tag_clause
        # this alters the tags but doesn't touch the data
        del_tags = ', '.join(map(escape_string, t[2]))
        if not noop:
            statement = "UPDATE stream SET metadata = metadata - ARRAY[" + \
                del_tags + "]"
            returning = "RETURNING id, uuid"
        else:
            statement = "SELECT uuid FROM stream"
            returning = ""

        q = statement + " WHERE id IN " + \
            "(SELECT s.id FROM stream s, subscription sub " + \
            "WHERE (" + where_clause.render() + ") AND s.subscription_id = sub.id AND " + \
            qg.build_authcheck(t.parser.request, forceprivate=True)  + ")" + returning
        return extractor, q


# top-level statement dispatching
def p_query(t):
    """query : SELECT selector WHERE statement<commit_msg>Fix data loss bug with tag deletion<commit_after>"""
Copyright (c) 2011, 2012, Regents of the University of California
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions 
are met:

 - Redistributions of source code must retain the above copyright
   notice, this list of conditions and the following disclaimer.
 - Redistributions in binary form must reproduce the above copyright
   notice, this list of conditions and the following disclaimer in the
   documentation and/or other materials provided with the
   distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 
THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 
STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 
OF THE POSSIBILITY OF SUCH DAMAGE.
"""
"""
@author <NAME> <<EMAIL>>
"""

import os
import operator
import time
import datetime
import inspect
import logging
import collections
import itertools
import re

from twisted.internet import defer
from twisted.python import log

from smap import operators
from smap.util import build_recursive, is_string, flatten, SetDict 
from smap import sjson as json
from smap.contrib import dtutil

from smap.archiver.data import escape_string
from smap.archiver.querydata import extract_data
from smap.archiver import querygen as qg
from smap.archiver import data, stream, help, ast, consumers

import ply
import ply.lex as lex
import ply.yacc as yacc
import copy

tokens = (
    'HAS', 'AND', 'OR', 'NOT', 'LIKE', 'DISTINCT',
    'TAGS', 'SELECT', 'WHERE',
    'QSTRING','NUMBER', 'LVALUE', 'IN', 'DATA', 'DELETE',
    'SET', 'TILDE', 'BEFORE', 'AFTER', 'NOW', 'LIMIT', 'STREAMLIMIT',
    'APPLY', 'TO', 'AS', 'GROUP', 'BY', 'HELP', 'ALL', 
    'LTE', 'GTE', 'NE'
    )

precedence = (
    ('left', 'LIKE', 'TILDE', 'IN'),
    ('left', 'AND'),
    ('left', 'OR'),
    ('right', '='),
    ('right', 'NOT'),
    ('left', ','),
    ('left', '+'),
    ('left', '-'),
    ('left', '*'),
    ('left', '^'),
    )

reserved = {
    'where' : 'WHERE',
    'distinct' : 'DISTINCT',
    'select' : 'SELECT',
    'delete' : 'DELETE',
    'set' : 'SET',
    'tags' : 'TAGS',
    'has' : 'HAS',
    'and' : 'AND',
    'or' : 'OR',
    'not' : 'NOT',
    'like' : 'LIKE',
    'data': 'DATA',
    'in' : 'IN',
    'before' : 'BEFORE',
    'after' : 'AFTER',
    'now' : 'NOW',
    'limit' : 'LIMIT',
    'streamlimit' : 'STREAMLIMIT',
    '~' : 'TILDE',
    'apply' : 'APPLY',
    'to' : 'TO',
    'as' : 'AS',
    'group' : 'GROUP',
    'by' : 'BY',
    'help' : 'HELP',
    'all' : 'ALL',
    }
literals = '()[]*^.,<>=+-/'

time_units = re.compile('^(d|days?|h|hours?|m|minutes?|s|seconds?)$')
def get_timeunit(t):
    if not time_units.match(t):
        raise qg.QueryException("Invalid timeunit: %s" % t)
    if t.startswith('d'): return 'days'
    elif t.startswith('h'): return 'hours'
    elif t.startswith('m'): return 'minutes'
    elif t.startswith('s'): return 'seconds'

def t_CMP(t):
    r'(<=)|(>=)|(!=)'
    t.type = {
        '<=' : 'LTE',
        '>=' : 'GTE',
        '!=' : 'NE'}[t.value]
    return t

def t_QSTRING(t):
    r'("[^"\\]*?(\\.[^"\\]*?)*?")|(\'[^\'\\]*?(\\.[^\'\\]*?)*?\')'    
    if t.value[0] == '"':
        t.value = t.value[1:-1].replace('\\"', '"')
    elif t.value[0] == "'":
        t.value = t.value[1:-1].replace("\\'", "'")
    return t

def t_LVALUE(t):
    r'[a-zA-Z\~\$\_][a-zA-Z0-9\/\%_\-]*'
    t.type = reserved.get(t.value, 'LVALUE')
    return t

def t_NUMBER(t):
    r'([+-]?([0-9]*\.)?[0-9]+)'
    if '.' in t.value:
        try:
            t.value = float(t.value)
        except ValueError:
            print "Invalid floating point number", t.value
            t.value = 0
    else:
        try:
            t.value = int(t.value)
        except ValueError:
            print "Integer value too large %d", t.value
            t.value = 0
        
    return t
is_number = lambda x: isinstance(x, int) or isinstance(x, float)

t_ignore = " \t"
def t_newline(t):
    r'[\n\r]+'
    t.lexer.lineno += t.value.count("\n")

def t_error(t):
    print("Illegal character '%s'" % t.value[0])
    t.lexer.skip(1)

smapql_lex = lex.lex()

# precedence = ('and')
names = {}

## Extractors
##  extractors are chained onto postgres operators and transform the
##  result for presentation to the client.

def ext_default(x):
    print x
    return map(operator.itemgetter(0), x)

def ext_non_null(x):
    return filter(None, ext_default(x))

def ext_deletor(x):
    data.del_streams(map(operator.itemgetter(0), x))
    return map(operator.itemgetter(1), x)

def ext_tag_deletor(x):
    return map(operator.itemgetter(1), x)

def ext_plural(tags, vals):
    return [build_recursive(dict(zip(tags, v)), suppress=[]) for v in vals]

def ext_recursive(vals):
    return [build_recursive(x[0], suppress=[]) for x in vals]

def ext_set(tag_list, x):
    """Extractor for the set operation

    Returns time series uuid and modified tags
    """
    rv = [{'uuid': v[0]} for v in x]
    for rv_i, v in itertools.izip(rv, x):
        rv_i.update(v[1])
    return [build_recursive(x, suppress=[]) for x in rv]
        

TIMEZONE_PATTERNS = [
    "%m/%d/%Y",
    "%m/%d/%Y %H:%M",
    "%Y-%m-%dT%H:%M:%S",
    ]
def parse_time(ts):
    for pat in TIMEZONE_PATTERNS:
        try:
            return dtutil.strptime_tz(ts, pat, tzstr='America/Los_Angeles')
        except ValueError:
            continue
    raise ValueError("Invalid time string:" + ts)

def make_select_rv(t, sel, wherestmt='true'):
    return sel.extract, ("""SELECT %s FROM stream s, subscription sub
              WHERE (%s) AND (%s) AND
              sub.id = s.subscription_id""" % 
                         (sel.select, 
                          wherestmt,
                          qg.build_authcheck(t.parser.request)))

def build_setstring(setvals, wherevals):
    #  set tags 
    regex = None
    regex_tag = None
    for stmt in wherevals:
        if stmt.op == ast.Statement.OP_REGEX:
            if regex == None:
                regex = stmt
            else:
                raise qg.QueryException("Too many regexes in set.  Only one supported!")

    if regex == None:
        new_tags = ' || '.join(map(lambda (t,v): "hstore(%s, %s)" % 
                                   (escape_string(t), escape_string(v)),
                                   setvals))
    else:
        new_tags = ' || '.join(map(lambda (t,v): "hstore(%s, regexp_replace(metadata -> %s, '^.*%s.*$', %s))" % 
                                   (escape_string(t), 
                                    regex.args[0],
                                    regex.args[1][1:-1].replace('\\\\', '\\'),
                                    escape_string(v).replace('\\\\', '\\')),
                                   setvals))
        regex_tag = regex.args[0][1:-1]

    return new_tags, regex_tag


def make_set_rv(t):
    # build the query for a set expression
    if len(t) > 3:
        # if there is a where clause, grab it
        where_clause = t[4]
    else:
        # if no where clause, just do something fast that matches
        # everything
        where_clause = ast.Statement(ast.Statement.OP_UUID)
    noop = "noop" in t.parser.request.args

    # build a list of what tags we're updating, as well as the
    # name of the regex tag.  We're currently not smart enough to
    # deal with more than one regex match in a set statement, due
    # to issues with back references.
    new_tags, regex_tag = build_setstring(t[2], where_clause)
    tag_list = [v[0] for v in t[2]]
    if regex_tag: tag_list.append(regex_tag)

    # if we're in noop mode, we need to tweak the SQL
    if not noop: 
        command = "UPDATE stream SET metadata = metadata || "
        from_stmt = ""
    else:
        command = "SELECT uuid, "
        from_stmt = "FROM stream"

    # build the first part of the query that depends on if this is a noop
    q = "%(command)s %(hstore_expression)s %(from_stmt)s WHERE ID IN" % {
        "command": command,
        "hstore_expression": new_tags,
        "from_stmt": from_stmt }

    # the where caluse always looks the same
    q += "(SELECT s.id FROM stream s, subscription sub " + \
        "WHERE (" + where_clause.render() + ") AND s.subscription_id = sub.id AND " + \
        qg.build_authcheck(t.parser.request, forceprivate=True)  + ") " 

    if not noop:
        # return the list of modifications.  if this was a noop we
        # get this for free.
        q += " RETURNING uuid, slice(metadata, ARRAY[" + \
            ','.join((escape_string(v) for v in tag_list)) + "])"
    return lambda x: ext_set(tag_list, x), q

def make_delete_rv(t):
    noop = "noop" in t.parser.request.args

    # a new delete inner statement enforces that we only delete
    # things which we have the key for.
    if len(t) > 2 and t[2] == 'where':
        # delete the whole stream, gone.  this also deletes the
        # data in the backend readingdb.
        if not noop:
            statement = "DELETE FROM stream"
            returning = "RETURNING id, uuid"
        else:
            statement = "SELECT uuid FROM stream"
            returning = ""
        return ext_deletor if not noop else ext_default, \
            """%(statement)s WHERE id IN (
                 SELECT s.id FROM stream s, subscription sub 
                 WHERE (%(restrict)s) AND s.subscription_id = sub.id AND 
                 (%(auth)s)
               ) %(returning)s
            """ % { 
            'restrict': t[3].render(),
            'auth': qg.build_authcheck(t.parser.request, forceprivate=True),
            'statement': statement,
            'returning': returning
            }
    else:
        # to make the return value right, we need to tweak the
        # where clause.  we want to only select streams where
        # there's actually something to delete, since all of the
        # streams the where-clause hits will get returned.
        tag_clause = ast.Statement(ast.Statement.OP_OR,
                                   *[ast.Statement(ast.Statement.OP_HAS, tag) 
                                     for tag in t[2]])
        if len(t) > 3: 
            where_clause = ast.Statement(ast.Statement.OP_AND, tag_clause, t[4])
        else:
            # if there's no where clause, we only look at streams
            # with the right tag.
            where_clause = tag_clause
        # this alters the tags but doesn't touch the data
        del_tags = ', '.join(map(escape_string, t[2]))
        if not noop:
            statement = "UPDATE stream SET metadata = metadata - ARRAY[" + \
                del_tags + "]"
            returning = "RETURNING id, uuid"
        else:
            statement = "SELECT id, uuid FROM stream"
            returning = ""

        q = statement + " WHERE id IN " + \
            "(SELECT s.id FROM stream s, subscription sub " + \
            "WHERE (" + where_clause.render() + ") AND s.subscription_id = sub.id AND " + \
            qg.build_authcheck(t.parser.request, forceprivate=True)  + ")" + returning
        return ext_tag_deletor, q


# top-level statement dispatching
def p_query(t):
    """query : SELECT selector WHERE statement
{'fraction_non_alphanumeric': 0.13911968980357067, 'fraction_numerical': 0.004218330563722357, 'mean_word_length': 2.3009845288326303, 'pattern_counts': {'":': 8, '<': 15, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5544851', 'n_tokens_mistral': 7702, 'n_tokens_neox': 7466, 'n_words': 2295}