<commit_before>from pyparsing import Word, Suppress, alphas, Literal, Group, Combine, opAssoc, alphanums
from pyparsing import Regex, operatorPrecedence, oneOf, ZeroOrMore, nums, Optional, delimitedList
from django.conf import settings
import redis

# By default, PyParsing treats \n as whitespace and ignores it
# In our grammer, \n is significant, so tell PyParsing not to ignore it
# ParserElement.setDefaultWhitespaceChars(" \t")
#
#
# rule ::= "FOR Tenant WHEN"+property +"[< > = <= >=]+X+"DO"+action
#
# condition ::= property +"[< > = <= >=]+X
# condition_list ::= condition
#                     | condition_list AND condition
#                     | condition_list OR condition
# FOR Tenant WHEN"+ condition_list +"DO"+action
#
# FOR Tenant WHEN"+ condition AND condition AND condition OR condition etc.+"DO"+action
#
# TODO: Parse = TRUE or = False or condicion number. Check to convert to float or convert to boolean.


def get_redis_connection():
    return redis.Redis(connection_pool=settings.REDIS_CON_POOL)


def parse_group_tenants(tokens):
    r = get_redis_connection()
    data = r.lrange(tokens[0], 0, -1)
    return data


def parse(input_string):
    # TODO Raise an exception if not metrics or not action registered
    # TODO Raise an exception if group of tenants don't exist.
    r = get_redis_connection()

    # Support words to construct the grammar.
    word = Word(alphas)
    when = Suppress(Literal("WHEN"))
    literal_for = Suppress(Literal("FOR"))
    boolean_condition = oneOf("AND OR")
    # Condition part
    param = Word(alphanums+"_") + Suppress(Literal("=")) + Word(alphanums+"_")
    metrics_workload = r.keys("metric:*")
    services = map(lambda x: "".join(x.split(":")[1]), metrics_workload)
    services_options = oneOf(services)
    operand = oneOf("< > == != <= >=")
    number = Regex(r"[+-]?\d+(:?\.\d*)?(:?[eE][+-]?\d+)?")
    condition = Group(services_options + operand("operand") + number("limit_value"))
    condition_list = operatorPrecedence(condition, [
                                ("AND", 2, opAssoc.LEFT, ),
                                ("OR", 2, opAssoc.LEFT, ),
                                ])
    # For tenant or group of tenants
    group_id = Word(nums)
    container = Group(Literal("CONTAINER")("type") + Suppress(":") + Combine(Word(alphanums) + Literal("/") + Word(alphanums+"_-")))
    obj = Group(Literal("OBJECT")("type") + Suppress(":") + Combine(Word(alphanums)+Literal("/") + Word(alphanums+"_-")+Literal("/") + Word(alphanums+"_-.")))
    tenant = Group(Literal("TENANT")("type") + Suppress(":") + Combine(Word(alphanums)))
    tenant_group = Combine(Literal("G:") + group_id)
    tenant_group_list = tenant_group + ZeroOrMore(Suppress("AND") + tenant_group)
    tenant_list = tenant + ZeroOrMore(Suppress("AND") + tenant)
    container_list = container + ZeroOrMore(Suppress("AND") + container)
    obj_list = obj + ZeroOrMore(Suppress("AND")+obj)
    target = Group(delimitedList(tenant) ^ delimitedList(obj) ^ delimitedList(container) ^ delimitedList(tenant_group))
    # Group(tenant_list ^ tenant_group_list ^ container_list ^ obj_list)
    # Action part
    action = oneOf("SET DELETE")
    sfilters_list = r.keys("dsl_filter:*")
    sfilter = map(lambda x: "".join(x.split(":")[1]), sfilters_list)
    with_params = Suppress(Literal("WITH"))
    do = Suppress(Literal("DO"))
    params_list = delimitedList(param)
    server_execution = oneOf("PROXY OBJECT")
    # TRANSIENT
    transient = Literal("TRANSIENT")
    action = Group(action("action") + oneOf(sfilter)("filter") +
                   Optional(with_params + params_list("params") +
                            Optional(Suppress("ON")+server_execution("server_execution"))) +
                   Optional(transient("transient")))
    
    action_list = Group(delimitedList(action))
    # Object types
    operand_object = oneOf("< > == != <= >=")
    object_parameter = oneOf("OBJECT_TYPE OBJECT_SIZE")
    object_type = Group(Literal("OBJECT_TYPE")("type") + Literal("=") + word(alphanums)("object_value"))("object_type")
    object_size = Group(Literal("OBJECT_SIZE")("type") + operand_object("operand") + number("object_value"))("object_size")
    object_list = Group(object_type ^ object_size ^ object_type + "," + object_size ^ object_size + "," + object_type)
    to = Suppress("TO")
    
    # Functions post-parsed
    convertToDict = lambda tokens: dict(zip(*[iter(tokens)]*2))
    remove_repeated_elements = lambda tokens: [list(set(tokens[0]))]
    
    params_list.setParseAction(convertToDict)
    target.setParseAction(remove_repeated_elements)
    tenant_group.setParseAction(parse_group_tenants)
    
    # Final rule structure
    rule_parse = literal_for + target("target") + \
                 Optional(when + condition_list("condition_list")) + do + \
                 action_list("action_list") + Optional(to + object_list("object_list"))
                
    # Parse the rule
    parsed_rule = rule_parse.parseString(input_string)
    
    # Pos-parsed validation
    has_condition_list = True
    if not parsed_rule.condition_list:
        has_condition_list = False
        
    for action in parsed_rule.action_list:
        if action.params:
            filter_info = r.hgetall("dsl_filter:"+str(action.filter))
            if "valid_parameters" in filter_info.keys():
                params = eval(filter_info["valid_parameters"])
                result = set(action.params.keys()).intersection(params.keys())
                if len(result) == len(action.params.keys()):
                    # TODO Check params types.
                    return has_condition_list, parsed_rule
                else:
                    raise Exception
            else:
                raise Exception
            
    return has_condition_list, parsed_rule

                
# condition_list, rule_parsed = parse("FOR TENANT:T1 WHEN get_ops_tenant > 5 DO SET cache TRANSIENT")
# print condition_list, rule_parsed


# rules ="""FOR OBJECT:4f0279da74ef4584a29dc72c835fe2c9/2/2 AND OBJECT:4f0279da74ef4584a29dc72c835fe2c9/2/2 DO SET compression WITH bw=2 ON OBJECT, SET uonetrace WITH bw=2 ON PROXY """.splitlines()
# # rules = """\
# #     FOR 4f0279da74ef4584a29dc72c835fe2c9 WHEN througput < 3 OR slowdown == 1 AND througput == 5 OR througput == 6 DO SET compression WITH param1=2
# #     FOR G:1 WHEN slowdown > 3 OR slowdown > 3 AND slowdown == 5 OR slowdown <= 6 DO SET compression WITH param1=2, param2=3
# #     FOR G:4 AND G:4 WHEN slowdown > 3 AND slowdown > 50 DO SET compression WITH""".splitlines()
# #
# for rule in rules:
#      _, parsed_rule = parse(rule)
#      print parsed_rule.target[1][1].split('/', 3)
#      print parsed_rule.action_list
#      print "object", parsed_rule.object_list
#      for target in parsed_rule.target:
#         if target.type is "OBJECT":
#             print target[1]
#     print 'as_list', stats.asList()
#     print stats
#     print 'subject', stats.subject
#     print "group", stats.subject.tenant_group_list
#     try:
#         stats = parse(rule)
#     except:
#         print 'This rule ***'+rule+'  *** could not be parsed'
#     else:
#         print stats.asList()
<commit_msg>Active Storage Orchestration DSL<commit_after>from pyparsing import Word, alphas, Literal, Group, Combine, opAssoc, alphanums, OneOrMore
from pyparsing import Regex, operatorPrecedence, oneOf, ZeroOrMore, nums, Optional, delimitedList

def parse(input_string):

    """ Support words to construct the grammar """
    WORD = Word(alphanums+"_-.^[]")
    SLASH = Literal("/")
    LBRKT = Literal("{")
    RBRKT = Literal("}")
    COLON = Literal(":")
    MARK = Literal("'")
    COMMA = Literal(",")
    
    WHEN = Literal("WHEN").suppress()
    FOR = Literal("FOR").suppress()
    MOVE = Literal("MOVE").suppress()
    COPY = Literal("COPY").suppress()
    GROUP = Literal("GROUP").suppress()
    DELETE = Literal("DELETE").suppress()
    DO = Literal("DO").suppress()
    AS = Literal("AS").suppress()
    ON = Literal("ON").suppress()
    BY = Literal("BY").suppress()
    
    condition = oneOf("AND OR")
    action = oneOf("SET DELETE")("action")
    operands = oneOf("< > == != <= >=")


    """ 
        Active Storage Orchestation
    """
    
    """ Single object format in Swift """
    obj = Combine(WORD + SLASH + WORD + Optional(SLASH + WORD))
    """ List of objects delimited by comma """
    objects = Group(delimitedList(obj))

    filters = map(lambda x: "".join(x), {'USER_NAME', 'ROLES'}) # Object Metadata
    filter_list = oneOf(filters)("filter")
    
    storlets = map(lambda x: "".join(x), {'COMPRESSION', 'FILTER_CONTENT', 'TRANSCODER', 'GREP'}) # Storlets
    storlets_list = oneOf(storlets)("storlet")
    json = Combine(MARK + WORD + MARK + COLON + MARK + WORD + MARK + Optional(COMMA))
    metadata = Combine(LBRKT + OneOrMore(json) + RBRKT)("metadata")
    action = Group(action + storlets_list + Optional(BY + filter_list + metadata) + Optional(metadata))
    actions = Group(delimitedList(action))
        
    """ Final rule structure """
    rule_parse = FOR + objects("objects") + DO + actions("actions")
    """ Parse the rule """
    parsed_rule = rule_parse.parseString(input_string)
    
    return parsed_rule


""" ----------------------------------- TEST DSL PARSER ------------------------------------ """
       
rule_parsed = parse("FOR data/out/sample_3mb.csv, data/sampe3_gb.csv DO SET COMPRESSION BY USER_NAME {'josep':'gz','raul':'zip'}, SET GREP BY ROLES {'admin':'^r'}")

print "Active storage orchestration test:"
for obj in rule_parsed.objects:
    print " Object ID: " + str(obj)

print "\nStorlets applied:"
for action in rule_parsed.actions:
    print " Action: " + action.action
    print " Storlet: " + action.storlet
    print " Filter: " + action.filter
    print " Metadata: " + action.metadata
    print " -------------------------------------"


{'fraction_non_alphanumeric': 0.15012106537530268, 'fraction_numerical': 0.010391444713478611, 'mean_word_length': 3.175652906486942, 'pattern_counts': {'":': 7, '<': 15, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2974688', 'n_tokens_mistral': 3173, 'n_tokens_neox': 3075, 'n_words': 710}