<commit_before># coding: utf-8

'''
MAGE Query Language
'''

from pyparsing import Forward, Group, Suppress, Optional, ZeroOrMore, Combine, \
    Word, alphas, nums, QuotedString, CaselessLiteral, FollowedBy
from ref.models.instances import ComponentInstance, ComponentInstanceField, ComponentInstanceRelation
from django.db.models.query import Prefetch

def __build_grammar():
    expr = Forward()

    k_select = CaselessLiteral("SELECT")
    k_from = CaselessLiteral("FROM")
    k_where = CaselessLiteral("WHERE")
    k_and = CaselessLiteral("AND")
    k_instances = CaselessLiteral("INSTANCES")
    qs = QuotedString("'", escQuote="''")

    identifier = Combine(Word(alphas + "_", exact=1) + Optional(Word(nums + alphas + "_")))("identifier")
    navigation = Group(identifier + ZeroOrMore(Suppress(".") + identifier))("navigation")

    filter_predicate = Group(navigation + Suppress("=") + (qs('value') | (Suppress('(') + expr('subquery') + Suppress(')'))))('predicate')
    where_clause = Group(Suppress(k_where) + filter_predicate + ZeroOrMore(Suppress(k_and) + filter_predicate))('where')

    # Pre filters
    impl = Optional(Suppress(CaselessLiteral("implementation"))) + qs('impl')
    cic = Suppress(CaselessLiteral("offer")) + qs('cic')
    lc = Suppress(CaselessLiteral("lc")) + qs('lc')
    envt = Suppress(CaselessLiteral("environment")) + qs('envt')
    pre_filter = Optional(envt) + Optional(lc) + Optional(cic) + Optional(impl) + FollowedBy(k_instances)

    # Dict query (only select some elements and navigate)
    nl_expr = Group(navigation + ZeroOrMore(Suppress(',') + navigation) + FollowedBy(k_from))('selector')

    # The sum of all fears
    select = Group(Suppress(k_select) + Optional(nl_expr + Suppress(k_from)) + pre_filter + Suppress(k_instances) + Optional(where_clause) + Optional(CaselessLiteral('WITH COMPUTATIONS')('compute')))('select')

    expr << select
    return expr

__grammar = __build_grammar()


def run(query):
    expr = __grammar.parseString(query)
    return __run(expr)
    #return  __grammar.parseString(query)

def __run(q):
    if q.select != None:
        return __select_compo(q.select)

def __select_compo(q):

    rs = ComponentInstance.objects.filter(deleted=False)

    if q.lc:
        rs = rs.filter(instanciates__implements__name=q.lc)
    if q.cic:
        rs = rs.filter(instanciates__name=q.cic)
    if q.impl:
        rs = rs.filter(description__name=q.impl)
    if q.envt:
        rs = rs.filter(environments__name=q.envt)

    if q.where:
        for predicate in q.where:
            ## Special keys begin with '_', normal keys without '_' are CI attributes
            if len(predicate.navigation) == 1 and predicate.navigation[0].startswith('mage_'):
                key = predicate.navigation[0]
                if key == "mage_type":
                    rs = rs.filter(description__name=predicate.value)
                if key == "mage_id":
                    rs = rs.filter(id=predicate.value)
                if key == "mage_envt":
                    rs = rs.filter(environments__name=predicate.value)
                if key == "mage_backup":
                    rs = rs.filter(include_in_envt_backup= (predicate.value.upper() == 'TRUE') )


                continue

            ## Key analysis: last part is always a simple field, others are relationship fields
            r = {}<commit_msg>MQL: special mage field keys are now case insensitive<commit_after># coding: utf-8

'''
MAGE Query Language
'''

from pyparsing import Forward, Group, Suppress, Optional, ZeroOrMore, Combine, \
    Word, alphas, nums, QuotedString, CaselessLiteral, FollowedBy
from ref.models.instances import ComponentInstance, ComponentInstanceField, ComponentInstanceRelation
from django.db.models.query import Prefetch

def __build_grammar():
    expr = Forward()

    k_select = CaselessLiteral("SELECT")
    k_from = CaselessLiteral("FROM")
    k_where = CaselessLiteral("WHERE")
    k_and = CaselessLiteral("AND")
    k_instances = CaselessLiteral("INSTANCES")
    qs = QuotedString("'", escQuote="''")

    identifier = Combine(Word(alphas + "_", exact=1) + Optional(Word(nums + alphas + "_")))("identifier")
    navigation = Group(identifier + ZeroOrMore(Suppress(".") + identifier))("navigation")

    filter_predicate = Group(navigation + Suppress("=") + (qs('value') | (Suppress('(') + expr('subquery') + Suppress(')'))))('predicate')
    where_clause = Group(Suppress(k_where) + filter_predicate + ZeroOrMore(Suppress(k_and) + filter_predicate))('where')

    # Pre filters
    impl = Optional(Suppress(CaselessLiteral("implementation"))) + qs('impl')
    cic = Suppress(CaselessLiteral("offer")) + qs('cic')
    lc = Suppress(CaselessLiteral("lc")) + qs('lc')
    envt = Suppress(CaselessLiteral("environment")) + qs('envt')
    pre_filter = Optional(envt) + Optional(lc) + Optional(cic) + Optional(impl) + FollowedBy(k_instances)

    # Dict query (only select some elements and navigate)
    nl_expr = Group(navigation + ZeroOrMore(Suppress(',') + navigation) + FollowedBy(k_from))('selector')

    # The sum of all fears
    select = Group(Suppress(k_select) + Optional(nl_expr + Suppress(k_from)) + pre_filter + Suppress(k_instances) + Optional(where_clause) + Optional(CaselessLiteral('WITH COMPUTATIONS')('compute')))('select')

    expr << select
    return expr

__grammar = __build_grammar()


def run(query):
    expr = __grammar.parseString(query)
    return __run(expr)
    #return  __grammar.parseString(query)

def __run(q):
    if q.select != None:
        return __select_compo(q.select)

def __select_compo(q):

    rs = ComponentInstance.objects.filter(deleted=False)

    if q.lc:
        rs = rs.filter(instanciates__implements__name=q.lc)
    if q.cic:
        rs = rs.filter(instanciates__name=q.cic)
    if q.impl:
        rs = rs.filter(description__name=q.impl)
    if q.envt:
        rs = rs.filter(environments__name=q.envt)

    if q.where:
        for predicate in q.where:
            ## Special keys begin with 'mage_', normal keys without 'mage_' are CI attributes
            if len(predicate.navigation) == 1 and predicate.navigation[0].lower().startswith('mage_'):
                key = predicate.navigation[0]
                if key.lower() == "mage_type":
                    rs = rs.filter(description__name=predicate.value)
                if key.lower() == "mage_id":
                    rs = rs.filter(id=predicate.value)
                if key.lower() == "mage_envt":
                    rs = rs.filter(environments__name=predicate.value)
                if key.lower() == "mage_backup":
                    rs = rs.filter(include_in_envt_backup=(predicate.value.upper() == 'TRUE'))

                continue

            ## Key analysis: last part is always a simple field, others are relationship fields
            r = {}
{'fraction_non_alphanumeric': 0.13331384300540858, 'fraction_numerical': 0.00146177459435755, 'mean_word_length': 3.405666452028332, 'pattern_counts': {'":': 8, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5662787', 'n_tokens_mistral': 2170, 'n_tokens_neox': 2081, 'n_words': 445}