<commit_before>import re
import json
import redis

from twisted.python import log
from twisted.internet.protocol import Protocol, ReconnectingClientFactory
from twisted.internet.task import LoopingCall

import binascii
from smpp.pdu import unpack_pdu
from smpp.pdu_builder import (BindTransceiver,
                                DeliverSMResp,
                                SubmitSM,
                                SubmitMulti,
                                EnquireLink,
                                QuerySM
                                )
from smpp.pdu_inspector import (MultipartMessage,
                                detect_multipart,
                                multipart_key
                                )

from vumi.utils import get_deploy_int


# TODO this will move to pdu_inspector in python-smpp
ESME_command_status_map = {
    "ESME_ROK"              : "No Error",
    "ESME_RINVMSGLEN"       : "Message Length is invalid",
    "ESME_RINVCMDLEN"       : "Command Length is invalid",
    "ESME_RINVCMDID"        : "Invalid Command ID",
    "ESME_RINVBNDSTS"       : "Incorrect BIND Status for given command",
    "ESME_RALYBND"          : "ESME Already in Bound State",
    "ESME_RINVPRTFLG"       : "Invalid Priority Flag",
    "ESME_RINVREGDLVFLG"    : "Invalid Registered Delivery Flag",
    "ESME_RSYSERR"          : "System Error",
    "ESME_RINVSRCADR"       : "Invalid Source Address",
    "ESME_RINVDSTADR"       : "Invalid Dest Addr",
    "ESME_RINVMSGID"        : "Message ID is invalid",
    "ESME_RBINDFAIL"        : "Bind Failed",
    "ESME_RINVPASWD"        : "Invalid Password",
    "ESME_RINVSYSID"        : "Invalid System ID",
    "ESME_RCANCELFAIL"      : "Cancel SM Failed",
    "ESME_RREPLACEFAIL"     : "Replace SM Failed",
    "ESME_RMSGQFUL"         : "Message Queue Full",
    "ESME_RINVSERTYP"       : "Invalid Service Type",
    "ESME_RINVNUMDESTS"     : "Invalid number of destinations",
    "ESME_RINVDLNAME"       : "Invalid Distribution List name",
    "ESME_RINVDESTFLAG"     : "Destination flag is invalid (submit_multi)",
    "ESME_RINVSUBREP"       : "Invalid 'submit with replace' request (i.e. submit_sm with replace_if_present_flag set)",
    "ESME_RINVESMCLASS"     : "Invalid esm_class field data",
    "ESME_RCNTSUBDL"        : "Cannot Submit to Distribution List",
    "ESME_RSUBMITFAIL"      : "submit_sm or submit_multi failed",
    "ESME_RINVSRCTON"       : "Invalid Source address TON",
    "ESME_RINVSRCNPI"       : "Invalid Source address NPI",
    "ESME_RINVDSTTON"       : "Invalid Destination address TON",
    "ESME_RINVDSTNPI"       : "Invalid Destination address NPI",
    "ESME_RINVSYSTYP"       : "Invalid system_type field",
    "ESME_RINVREPFLAG"      : "Invalid replace_if_present flag",
    "ESME_RINVNUMMSGS"      : "Invalid number of messages",
    "ESME_RTHROTTLED"       : "Throttling error (ESME has exceeded allowed message limits)",
    "ESME_RINVSCHED"        : "Invalid Scheduled Delivery Time",
    "ESME_RINVEXPIRY"       : "Invalid message validity period (Expiry time)",
    "ESME_RINVDFTMSGID"     : "Predefined Message Invalid or Not Found",
    "ESME_RX_T_APPN"        : "ESME Receiver Temporary App Error Code",
    "ESME_RX_P_APPN"        : "ESME Receiver Permanent App Error Code",
    "ESME_RX_R_APPN"        : "ESME Receiver Reject Message Error Code",
    "ESME_RQUERYFAIL"       : "query_sm request failed",
    "ESME_RINVOPTPARSTREAM" : "Error in the optional part of the PDU Body.",
    "ESME_ROPTPARNOTALLWD"  : "Optional Parameter not allowed",
    "ESME_RINVPARLEN"       : "Invalid Parameter Length.",
    "ESME_RMISSINGOPTPARAM" : "Expected Optional Parameter missing",
    "ESME_RINVOPTPARAMVAL"  : "Invalid Optional Parameter Value",
    "ESME_RDELIVERYFAILURE" : "Delivery Failure (used for data_sm_resp)",
    "ESME_RUNKNOWNERR"      : "Unknown Error",
}


class EsmeTransceiver(Protocol):

    def __init__(self, seq, config, vumi_options):
        self.build_maps()
        self.name = 'Proto' + str(seq)
        log.msg('__init__', self.name)
        self.defaults = {}
        self.state = 'CLOSED'
        log.msg(self.name, 'STATE :', self.state)
        self.seq = seq
        self.config = config
        self.vumi_options = vumi_options
        self.inc = int(self.config['smpp_increment'])
        self.datastream = ''
        self.__connect_callback = None
        self.__submit_sm_resp_callback = None
        self.__delivery_report_callback = None
        self.__deliver_sm_callback = None
        self.error_handlers = {}

        self.r_server = redis.Redis("localhost",
                db=get_deploy_int(self.vumi_options['vhost']))
        log.msg("Connected to Redis")
        self.r_prefix = "%s@%s:%s" % (
                self.config['system_id'],
                self.config['host'],
                self.config['port'])
        log.msg("r_prefix = %s" % self.r_prefix)

    def build_maps(self):
        self.ESME_command_status_dispatch_map = {
            "ESME_ROK"              : self.command_status_dispatch_ok,
            #"ESME_RINVMSGLEN"       : ,
            #"ESME_RINVCMDLEN"       : ,
            #"ESME_RINVCMDID"        : ,
            #"ESME_RINVBNDSTS"       : ,
            #"ESME_RALYBND"          : ,
            #"ESME_RINVPRTFLG"       : ,
            #"ESME_RINVREGDLVFLG"    : ,
            "ESME_RSYSERR"          : self.command_status_dispatch_permafault,
            #"ESME_RINVSRCADR"       : ,
            #"ESME_RINVDSTADR"       : ,
            #"ESME_RINVMSGID"        : ,
            #"ESME_RBINDFAIL"        : ,
            "ESME_RINVPASWD"        : self.command_status_dispatch_permafault,
            "ESME_RINVSYSID"        : self.command_status_dispatch_permafault,
            #"ESME_RCANCELFAIL"      : ,
            #"ESME_RREPLACEFAIL"     : ,
            #"ESME_RMSGQFUL"         : ,
            #"ESME_RINVSERTYP"       : ,
            #"ESME_RINVNUMDESTS"     : ,
            #"ESME_RINVDLNAME"       : ,
            #"ESME_RINVDESTFLAG"     : ,
            #"ESME_RINVSUBREP"       : ,
            #"ESME_RINVESMCLASS"     : ,
            #"ESME_RCNTSUBDL"        : ,
            #"ESME_RSUBMITFAIL"      : ,
            #"ESME_RINVSRCTON"       : ,
            #"ESME_RINVSRCNPI"       : ,
            #"ESME_RINVDSTTON"       : ,
            #"ESME_RINVDSTNPI"       : ,
            #"ESME_RINVSYSTYP"       : ,
            #"ESME_RINVREPFLAG"      : ,
            #"ESME_RINVNUMMSGS"      : ,
            #"ESME_RTHROTTLED"       : ,
            #"ESME_RINVSCHED"        : ,
            #"ESME_RINVEXPIRY"       : ,
            #"ESME_RINVDFTMSGID"     : ,
            #"ESME_RX_T_APPN"        : ,
            #"ESME_RX_P_APPN"        : ,
            #"ESME_RX_R_APPN"        : ,
            #"ESME_RQUERYFAIL"       : ,
            #"ESME_RINVOPTPARSTREAM" : ,
            #"ESME_ROPTPARNOTALLWD"  : ,
            #"ESME_RINVPARLEN"       : ,
            #"ESME_RMISSINGOPTPARAM" : ,
            #"ESME_RINVOPTPARAMVAL"  : ,
            #"ESME_RDELIVERYFAILURE" : ,
            #"ESME_RUNKNOWNERR"      : ,
        }

    def command_status_dispatch(self, pdu):
        method = self.ESME_command_status_dispatch_map.get(pdu['header']['command_status'], self.command_status_dispatch_ok)
        return method(pdu)

    def command_status_dispatch_ok(self, pdu):
        return self.error_handlers.get("command_status_dispatch_ok")

    def command_status_dispatch_permafault(self, pdu):
        return self.error_handlers.get("command_status_dispatch_permafault")

    # TODO this is currently unused ... i think
    def set_handler(self, handler):
        self.handler = handler

    def update_error_handlers(self, handler_dict={}):
        self.error_handlers.update(handler_dict)

    def getSeq(self):
        return self.seq[0]

    def incSeq(self):
        self.seq[0] += self.inc

    def popData(self):
        data = None
        if(len(self.datastream) >= 16):
            command_length = int(binascii.b2a_hex(self.datastream[0:4]), 16)
            if(len(self.datastream) >= command_):
                data = self.datastream[0:command_length]
                self.datastream = self.datastream[command_length:]
        return data

    def handleData(self, data):
        pdu = unpack_pdu(data)
        log.msg('INCOMING <<<<', pdu)
        error_handler = self.command_status_dispatch(pdu)
        if error_handler:
            error_handler(pdu)
        if pdu['header']['command_id'] == 'bind_transceiver_resp':
            self.handle_bind_transceiver_resp(pdu)
        if pdu['header']['command_id'] == 'submit_sm_resp':
            self.handle_submit_sm_resp(pdu)
        if pdu['header']['command_id'] == 'submit_multi_resp':
            self.handle_submit_multi_resp(pdu)
        if pdu['header']['command_id'] == 'deliver_sm':
            self.handle_deliver_sm(pdu)
        if pdu['header']['command_id'] == 'enquire_link_resp':
            self.handle_enquire_link_resp(pdu)
        log.msg(self.name, 'STATE :', self.state)
<commit_msg>typo fix<commit_after>import re
import json
import redis

from twisted.python import log
from twisted.internet.protocol import Protocol, ReconnectingClientFactory
from twisted.internet.task import LoopingCall

import binascii
from smpp.pdu import unpack_pdu
from smpp.pdu_builder import (BindTransceiver,
                                DeliverSMResp,
                                SubmitSM,
                                SubmitMulti,
                                EnquireLink,
                                QuerySM
                                )
from smpp.pdu_inspector import (MultipartMessage,
                                detect_multipart,
                                multipart_key
                                )

from vumi.utils import get_deploy_int


# TODO this will move to pdu_inspector in python-smpp
ESME_command_status_map = {
    "ESME_ROK"              : "No Error",
    "ESME_RINVMSGLEN"       : "Message Length is invalid",
    "ESME_RINVCMDLEN"       : "Command Length is invalid",
    "ESME_RINVCMDID"        : "Invalid Command ID",
    "ESME_RINVBNDSTS"       : "Incorrect BIND Status for given command",
    "ESME_RALYBND"          : "ESME Already in Bound State",
    "ESME_RINVPRTFLG"       : "Invalid Priority Flag",
    "ESME_RINVREGDLVFLG"    : "Invalid Registered Delivery Flag",
    "ESME_RSYSERR"          : "System Error",
    "ESME_RINVSRCADR"       : "Invalid Source Address",
    "ESME_RINVDSTADR"       : "Invalid Dest Addr",
    "ESME_RINVMSGID"        : "Message ID is invalid",
    "ESME_RBINDFAIL"        : "Bind Failed",
    "ESME_RINVPASWD"        : "Invalid Password",
    "ESME_RINVSYSID"        : "Invalid System ID",
    "ESME_RCANCELFAIL"      : "Cancel SM Failed",
    "ESME_RREPLACEFAIL"     : "Replace SM Failed",
    "ESME_RMSGQFUL"         : "Message Queue Full",
    "ESME_RINVSERTYP"       : "Invalid Service Type",
    "ESME_RINVNUMDESTS"     : "Invalid number of destinations",
    "ESME_RINVDLNAME"       : "Invalid Distribution List name",
    "ESME_RINVDESTFLAG"     : "Destination flag is invalid (submit_multi)",
    "ESME_RINVSUBREP"       : "Invalid 'submit with replace' request (i.e. submit_sm with replace_if_present_flag set)",
    "ESME_RINVESMCLASS"     : "Invalid esm_class field data",
    "ESME_RCNTSUBDL"        : "Cannot Submit to Distribution List",
    "ESME_RSUBMITFAIL"      : "submit_sm or submit_multi failed",
    "ESME_RINVSRCTON"       : "Invalid Source address TON",
    "ESME_RINVSRCNPI"       : "Invalid Source address NPI",
    "ESME_RINVDSTTON"       : "Invalid Destination address TON",
    "ESME_RINVDSTNPI"       : "Invalid Destination address NPI",
    "ESME_RINVSYSTYP"       : "Invalid system_type field",
    "ESME_RINVREPFLAG"      : "Invalid replace_if_present flag",
    "ESME_RINVNUMMSGS"      : "Invalid number of messages",
    "ESME_RTHROTTLED"       : "Throttling error (ESME has exceeded allowed message limits)",
    "ESME_RINVSCHED"        : "Invalid Scheduled Delivery Time",
    "ESME_RINVEXPIRY"       : "Invalid message validity period (Expiry time)",
    "ESME_RINVDFTMSGID"     : "Predefined Message Invalid or Not Found",
    "ESME_RX_T_APPN"        : "ESME Receiver Temporary App Error Code",
    "ESME_RX_P_APPN"        : "ESME Receiver Permanent App Error Code",
    "ESME_RX_R_APPN"        : "ESME Receiver Reject Message Error Code",
    "ESME_RQUERYFAIL"       : "query_sm request failed",
    "ESME_RINVOPTPARSTREAM" : "Error in the optional part of the PDU Body.",
    "ESME_ROPTPARNOTALLWD"  : "Optional Parameter not allowed",
    "ESME_RINVPARLEN"       : "Invalid Parameter Length.",
    "ESME_RMISSINGOPTPARAM" : "Expected Optional Parameter missing",
    "ESME_RINVOPTPARAMVAL"  : "Invalid Optional Parameter Value",
    "ESME_RDELIVERYFAILURE" : "Delivery Failure (used for data_sm_resp)",
    "ESME_RUNKNOWNERR"      : "Unknown Error",
}


class EsmeTransceiver(Protocol):

    def __init__(self, seq, config, vumi_options):
        self.build_maps()
        self.name = 'Proto' + str(seq)
        log.msg('__init__', self.name)
        self.defaults = {}
        self.state = 'CLOSED'
        log.msg(self.name, 'STATE :', self.state)
        self.seq = seq
        self.config = config
        self.vumi_options = vumi_options
        self.inc = int(self.config['smpp_increment'])
        self.datastream = ''
        self.__connect_callback = None
        self.__submit_sm_resp_callback = None
        self.__delivery_report_callback = None
        self.__deliver_sm_callback = None
        self.error_handlers = {}

        self.r_server = redis.Redis("localhost",
                db=get_deploy_int(self.vumi_options['vhost']))
        log.msg("Connected to Redis")
        self.r_prefix = "%s@%s:%s" % (
                self.config['system_id'],
                self.config['host'],
                self.config['port'])
        log.msg("r_prefix = %s" % self.r_prefix)

    def build_maps(self):
        self.ESME_command_status_dispatch_map = {
            "ESME_ROK"              : self.command_status_dispatch_ok,
            #"ESME_RINVMSGLEN"       : ,
            #"ESME_RINVCMDLEN"       : ,
            #"ESME_RINVCMDID"        : ,
            #"ESME_RINVBNDSTS"       : ,
            #"ESME_RALYBND"          : ,
            #"ESME_RINVPRTFLG"       : ,
            #"ESME_RINVREGDLVFLG"    : ,
            "ESME_RSYSERR"          : self.command_status_dispatch_permafault,
            #"ESME_RINVSRCADR"       : ,
            #"ESME_RINVDSTADR"       : ,
            #"ESME_RINVMSGID"        : ,
            #"ESME_RBINDFAIL"        : ,
            "ESME_RINVPASWD"        : self.command_status_dispatch_permafault,
            "ESME_RINVSYSID"        : self.command_status_dispatch_permafault,
            #"ESME_RCANCELFAIL"      : ,
            #"ESME_RREPLACEFAIL"     : ,
            #"ESME_RMSGQFUL"         : ,
            #"ESME_RINVSERTYP"       : ,
            #"ESME_RINVNUMDESTS"     : ,
            #"ESME_RINVDLNAME"       : ,
            #"ESME_RINVDESTFLAG"     : ,
            #"ESME_RINVSUBREP"       : ,
            #"ESME_RINVESMCLASS"     : ,
            #"ESME_RCNTSUBDL"        : ,
            #"ESME_RSUBMITFAIL"      : ,
            #"ESME_RINVSRCTON"       : ,
            #"ESME_RINVSRCNPI"       : ,
            #"ESME_RINVDSTTON"       : ,
            #"ESME_RINVDSTNPI"       : ,
            #"ESME_RINVSYSTYP"       : ,
            #"ESME_RINVREPFLAG"      : ,
            #"ESME_RINVNUMMSGS"      : ,
            #"ESME_RTHROTTLED"       : ,
            #"ESME_RINVSCHED"        : ,
            #"ESME_RINVEXPIRY"       : ,
            #"ESME_RINVDFTMSGID"     : ,
            #"ESME_RX_T_APPN"        : ,
            #"ESME_RX_P_APPN"        : ,
            #"ESME_RX_R_APPN"        : ,
            #"ESME_RQUERYFAIL"       : ,
            #"ESME_RINVOPTPARSTREAM" : ,
            #"ESME_ROPTPARNOTALLWD"  : ,
            #"ESME_RINVPARLEN"       : ,
            #"ESME_RMISSINGOPTPARAM" : ,
            #"ESME_RINVOPTPARAMVAL"  : ,
            #"ESME_RDELIVERYFAILURE" : ,
            #"ESME_RUNKNOWNERR"      : ,
        }

    def command_status_dispatch(self, pdu):
        method = self.ESME_command_status_dispatch_map.get(pdu['header']['command_status'], self.command_status_dispatch_ok)
        return method(pdu)

    def command_status_dispatch_ok(self, pdu):
        return self.error_handlers.get("command_status_dispatch_ok")

    def command_status_dispatch_permafault(self, pdu):
        return self.error_handlers.get("command_status_dispatch_permafault")

    # TODO this is currently unused ... i think
    def set_handler(self, handler):
        self.handler = handler

    def update_error_handlers(self, handler_dict={}):
        self.error_handlers.update(handler_dict)

    def getSeq(self):
        return self.seq[0]

    def incSeq(self):
        self.seq[0] += self.inc

    def popData(self):
        data = None
        if(len(self.datastream) >= 16):
            command_length = int(binascii.b2a_hex(self.datastream[0:4]), 16)
            if(len(self.datastream) >= command_length):
                data = self.datastream[0:command_length]
                self.datastream = self.datastream[command_length:]
        return data

    def handleData(self, data):
        pdu = unpack_pdu(data)
        log.msg('INCOMING <<<<', pdu)
        error_handler = self.command_status_dispatch(pdu)
        if error_handler:
            error_handler(pdu)
        if pdu['header']['command_id'] == 'bind_transceiver_resp':
            self.handle_bind_transceiver_resp(pdu)
        if pdu['header']['command_id'] == 'submit_sm_resp':
            self.handle_submit_sm_resp(pdu)
        if pdu['header']['command_id'] == 'submit_multi_resp':
            self.handle_submit_multi_resp(pdu)
        if pdu['header']['command_id'] == 'deliver_sm':
            self.handle_deliver_sm(pdu)
        if pdu['header']['command_id'] == 'enquire_link_resp':
            self.handle_enquire_link_resp(pdu)
        log.msg(self.name, 'STATE :', self.state)

{'fraction_non_alphanumeric': 0.10454269470662914, 'fraction_numerical': 0.0011039355301650383, 'mean_word_length': 2.05016835016835, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2333006', 'n_tokens_mistral': 6232, 'n_tokens_neox': 6069, 'n_words': 1005}