<commit_before># encoding: utf-8

import tornado.ioloop
import tornado.web
import sys
import getopt
import time
import re
import ply.yacc as yacc
from sa_lex import tokens
from treetagger import TreeTagger
from treetagger_wordnet import TreetaggerToWordnet


class MainHandler(tornado.web.RequestHandler):
    """ """
    
    def initialize(self, rules, chunks, language):
        """ """
        self.rules = rules
        self.chunks = chunks
        self.language = language
        self.tt = TreeTagger(encoding='latin-1', language=language)
        self.to_wordnet = TreetaggerToWordnet()

    def get(self):
        """ """
        # Start operation time
        time_start = time.time()
        # Prepare the response DS
        response = {}
        # Check input text
        text = self.get_argument("text")
        text = text.replace("\n", "")
        text = text.encode("utf-8")
        response["raw_text"] = text
        # Perform PosTaggin onto the text
        aux = ""

        # Replace chunks symbols
        for chunk in self.chunks:
            text = text.replace(chunk, " # ")

        # Mapping treetagger postaging to wordnet postagging
        for tag in self.tt.tag(text):
            lemma = tag[2]
            if lemma == u"<unknown>":
                lemma = tag[0]
            aux += "%s.%s " % (lemma, self.to_wordnet.wordnet_morph_category(self.language, tag[1]))
        text = aux

        # Replace postagging for the entities
        aux = ""
        chunks = text.split(" ")
        for chunk in chunks:
            sub_chunks = chunk.split(".")
            if sub_chunks[0] in entities or sub_chunks[0]  in inverters:
                aux += "%s " % sub_chunks[0]
            else:
                aux += "%s " % chunk
        text = aux
        response["text"] = text


        # Split input text
        chunks = text.split('#.None')
        triggered_rule_cont = 0
        response["matches"] = {}

        for chunk in chunks:

            # Apply rules and replace matchs
            rule_triggered = False
            while True:

                # Appling all rules over the text
                rule_triggered = False
                for rule in compiled_rules:
                    for match in compiled_rules[rule]["regex"].finditer(chunk):
                        rule_triggered = True
                        response["matches"][triggered_rule_cont] = {}
                        response["matches"][triggered_rule_cont]["rule_triggered"] = rule
                        response["matches"][triggered_rule_cont]["score"] = compiled_rules[rule]["score"]
                        response["matches"][triggered_rule_cont]["match"] = match.group()

                        # Replace chunk with "***"" filling
                        aux = ""
                        for e in match.group().split(" "):
                            aux += "*" * len(e) + " "
                        aux = aux[0:len(aux)-2]

                        # Perfom replace
                        chunk = chunk.replace(match.group(), aux)
                        triggered_rule_cont += 1
                # Break rule matching (precedence behavior)
                if not rule_triggered:
                    break
        # Return response
        response["elipsed_time"] = time.time() - time_start

        # Return result
        self.write(response)

# Variables to manage values in the parser process
start = 'rule'
regex = ''
compiled_rules = {}

def p_rule(p):
    'rule : BEGINRULE expression THEN SCORE ENDRULE'
    if p[1] in compiled_rules:
        print("Error: Rule #ID duplicated.")
        sys.exit(0)
    compiled_rules[p[1]] = {}
    compiled_rules[p[1]]["regex"] = regex
    compiled_rules[p[1]]["score"] = float(p[4])
    print(regex.pattern)


def p_expresion_simple_one(p):
    'expression : QUALIFICATOR ENTITY'
    global regex
    if p[1] == "+":
        regex = re.compile("%s(\s\S+){0,3}\s%s" % (combined_positives, combined_entities))
    else:
        regex = re.compile("%s(\s\S+){0,3}\s%s" % (combined_negatives, combined_entities))

def p_expresion_simple_two(p):
    'expression : ENTITY QUALIFICATOR'
    global regex
    print(combined_entities)
    if p[2] == "+":
        regex = re.compile("%s(\s\S+){0,3}\s%s" % (combined_entities, combined_positives))
    else:
        regex = re.compile("%s(\s\S+){0,3}\s%s" % (combined_entities, combined_negatives))

def p_expresion_simple_swaping_one(p):
    'expression : SWAP QUALIFICATOR ENTITY'
    global regex
    if p[2] == "+":
         regex = re.compile("%s(\s\S+){0,3}\s%s(\s\S+){0,3}\s%s" % (combined_inverters, combined_positives, combined_entities))
    else:
         regex = re.compile("%s(\s\S+){0,3}\s%s(\s\S+){0,3}\s%s" % (combined_inverters, combined_negatives, combined_entities))

def p_expresion_simple_swaping_two(p):
    'expression : ENTITY SWAP QUALIFICATOR'
    global regex
    if p[3] == "+":
         regex = re.compile("%s(\s\S+){0,3}\s%s(\s\S+){0,3}\s%s" % (combined_entities, combined_inverters, combined_positives))
    else:
         regex = re.compile("%s(\s\S+){0,3}\s%s(\s\S+){0,3}\s%s" % (combined_entities, combined_inverters, combined_negatives))

def p_expresion_adhoc_one(p):
    'expression : ENTITY IDENTIFICATOR'
    global regex
    regex = re.compile("%s(\s\S+){0,3}\s(%s)" % (combined_entities, p[2]))

def p_expresion_adhoc_two(p):
    'expression : IDENTIFICATOR ENTITY'
    global regex
    regex = re.compile("%s(\s\S+){0,3}\s(%s)" % (p[1], combined_entities))



# Error rule for syntax errors
def p_error(p):
    print ">>> %s" % p
    print "Syntax error in input!"
    sys.exit(0)

def load_dict(file):
    """
    Load files from disk into variables
    """
    aux = []
    for line in open(file, "r"):
        line = line.replace("\r\n", "")
        aux.append(line)
    return aux



# Check if parameters are valid
try:
    opts, args = getopt.getopt(sys.argv[1:], "l:p:", ["language=","port="])
    assert(len(opts) == 2)
except:
    print 'sentiment_Service.py -l <language> -p <port>'
    sys.exit()


for o, a in opts:
    if o == "-l":
        language = a
        if language not in ["spanish", "english"]:
            print 'Valid languages: spanish, english'
            sys.exit()
    elif o == "-p":
        port = a<commit_msg>Fix windows text file carriage returns<commit_after># encoding: utf-8

import tornado.ioloop
import tornado.web
import sys
import getopt
import time
import re
import ply.yacc as yacc
from sa_lex import tokens
from treetagger import TreeTagger
from treetagger_wordnet import TreetaggerToWordnet


class MainHandler(tornado.web.RequestHandler):
    """ """
    
    def initialize(self, rules, chunks, language):
        """ """
        self.rules = rules
        self.chunks = chunks
        self.language = language
        self.tt = TreeTagger(encoding='latin-1', language=language)
        self.to_wordnet = TreetaggerToWordnet()

    def get(self):
        """ """
        # Start operation time
        time_start = time.time()
        # Prepare the response DS
        response = {}
        # Check input text
        text = self.get_argument("text")
        text = text.replace("\n", "")
        text = text.encode("utf-8")
        response["raw_text"] = text
        # Perform PosTaggin onto the text
        aux = ""

        # Replace chunks symbols
        for chunk in self.chunks:
            text = text.replace(chunk, " # ")

        # Mapping treetagger postaging to wordnet postagging
        for tag in self.tt.tag(text):
            lemma = tag[2]
            if lemma == u"<unknown>":
                lemma = tag[0]
            aux += "%s.%s " % (lemma, self.to_wordnet.wordnet_morph_category(self.language, tag[1]))
        text = aux

        # Replace postagging for the entities
        aux = ""
        chunks = text.split(" ")
        for chunk in chunks:
            sub_chunks = chunk.split(".")
            if sub_chunks[0] in entities or sub_chunks[0]  in inverters:
                aux += "%s " % sub_chunks[0]
            else:
                aux += "%s " % chunk
        text = aux
        response["text"] = text


        # Split input text
        chunks = text.split('#.None')
        triggered_rule_cont = 0
        response["matches"] = {}

        for chunk in chunks:

            # Apply rules and replace matchs
            rule_triggered = False
            while True:

                # Appling all rules over the text
                rule_triggered = False
                for rule in compiled_rules:
                    for match in compiled_rules[rule]["regex"].finditer(chunk):
                        rule_triggered = True
                        response["matches"][triggered_rule_cont] = {}
                        response["matches"][triggered_rule_cont]["rule_triggered"] = rule
                        response["matches"][triggered_rule_cont]["score"] = compiled_rules[rule]["score"]
                        response["matches"][triggered_rule_cont]["match"] = match.group()

                        # Replace chunk with "***"" filling
                        aux = ""
                        for e in match.group().split(" "):
                            aux += "*" * len(e) + " "
                        aux = aux[0:len(aux)-2]

                        # Perfom replace
                        chunk = chunk.replace(match.group(), aux)
                        triggered_rule_cont += 1
                # Break rule matching (precedence behavior)
                if not rule_triggered:
                    break
        # Return response
        response["elipsed_time"] = time.time() - time_start

        # Return result
        self.write(response)

# Variables to manage values in the parser process
start = 'rule'
regex = ''
compiled_rules = {}

def p_rule(p):
    'rule : BEGINRULE expression THEN SCORE ENDRULE'
    if p[1] in compiled_rules:
        print("Error: Rule #ID duplicated.")
        sys.exit(0)
    compiled_rules[p[1]] = {}
    compiled_rules[p[1]]["regex"] = regex
    compiled_rules[p[1]]["score"] = float(p[4])
    print(regex.pattern)


def p_expresion_simple_one(p):
    'expression : QUALIFICATOR ENTITY'
    global regex
    if p[1] == "+":
        regex = re.compile("%s(\s\S+){0,3}\s%s" % (combined_positives, combined_entities))
    else:
        regex = re.compile("%s(\s\S+){0,3}\s%s" % (combined_negatives, combined_entities))

def p_expresion_simple_two(p):
    'expression : ENTITY QUALIFICATOR'
    global regex
    print(combined_entities)
    if p[2] == "+":
        regex = re.compile("%s(\s\S+){0,3}\s%s" % (combined_entities, combined_positives))
    else:
        regex = re.compile("%s(\s\S+){0,3}\s%s" % (combined_entities, combined_negatives))

def p_expresion_simple_swaping_one(p):
    'expression : SWAP QUALIFICATOR ENTITY'
    global regex
    if p[2] == "+":
         regex = re.compile("%s(\s\S+){0,3}\s%s(\s\S+){0,3}\s%s" % (combined_inverters, combined_positives, combined_entities))
    else:
         regex = re.compile("%s(\s\S+){0,3}\s%s(\s\S+){0,3}\s%s" % (combined_inverters, combined_negatives, combined_entities))

def p_expresion_simple_swaping_two(p):
    'expression : ENTITY SWAP QUALIFICATOR'
    global regex
    if p[3] == "+":
         regex = re.compile("%s(\s\S+){0,3}\s%s(\s\S+){0,3}\s%s" % (combined_entities, combined_inverters, combined_positives))
    else:
         regex = re.compile("%s(\s\S+){0,3}\s%s(\s\S+){0,3}\s%s" % (combined_entities, combined_inverters, combined_negatives))

def p_expresion_adhoc_one(p):
    'expression : ENTITY IDENTIFICATOR'
    global regex
    regex = re.compile("%s(\s\S+){0,3}\s(%s)" % (combined_entities, p[2]))

def p_expresion_adhoc_two(p):
    'expression : IDENTIFICATOR ENTITY'
    global regex
    regex = re.compile("%s(\s\S+){0,3}\s(%s)" % (p[1], combined_entities))



# Error rule for syntax errors
def p_error(p):
    print ">>> %s" % p
    print "Syntax error in input!"
    sys.exit(0)

def load_dict(file):
    """
    Load files from disk into variables
    """
    aux = []
    for line in open(file, "r"):
        line = line.replace("\r\n", "")
        line = line.replace("\n", "")
        aux.append(line)
    return aux



# Check if parameters are valid
try:
    opts, args = getopt.getopt(sys.argv[1:], "l:p:", ["language=","port="])
    assert(len(opts) == 2)
except:
    print 'sentiment_Service.py -l <language> -p <port>'
    sys.exit()


for o, a in opts:
    if o == "-l":
        language = a
        if language not in ["spanish", "english"]:
            print 'Valid languages: spanish, english'
            sys.exit()
    elif o == "-p":
        port = a
{'fraction_non_alphanumeric': 0.1384870761204648, 'fraction_numerical': 0.008853055094458936, 'mean_word_length': 2.1693386773547094, 'pattern_counts': {'":': 14, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6096369', 'n_tokens_mistral': 4083, 'n_tokens_neox': 3968, 'n_words': 898}