<commit_before>"""
WrappedCursor for a generic database connection proxy

"""

import re
import os
import sys
import warnings
from pandalogger.PandaLogger import PandaLogger
from config import panda_config

warnings.filterwarnings('ignore')

# logger
_logger = PandaLogger().getLogger('WrappedCursor')

# proxy
class WrappedCursor(object):

    # constructor
    def __init__(self, connection):
        # connection object
        self.conn = connection
        # cursor object
        self.cur = self.conn.cursor()
        # backend
        self.backend = panda_config.backend
        # statement
        self.statement = None


    # __iter__
    def __iter__(self):
        return iter(self.cur)


    # serialize
    def __str__(self):
        return 'WrappedCursor[%(conn)s]' % ({'conn': self.conn})

    # initialize
    def initialize(self):
        hostname = None
        if self.backend == 'oracle':
            # get hostname
            self.execute("SELECT SYS_CONTEXT('USERENV','HOST') FROM dual")
            res = self.fetchone()
            if res != None:
                hostname = res[0]
            # set TZ
            self.execute("ALTER SESSION SET TIME_ZONE='UTC'")
            # set DATE format
            self.execute("ALTER SESSION SET NLS_DATE_FORMAT='YYYY/MM/DD HH24:MI:SS'")
        else:
            # get hostname
            self.execute("SELECT SUBSTRING_INDEX(USER(),'@',-1)")
            res = self.fetchone()
            if res != None:
                hostname = res[0]
            # set TZ
            self.execute("SET @@SESSION.TIME_ZONE = '+00:00'")
            # set DATE format
            #self.execute("SET @@SESSION.DATETIME_FORMAT='%%Y/%%m/%%d %%H:%%i:%%s'")
            # disable autocommit
            self.execute("SET autocommit=0")
        return hostname
    # execute query on cursor
    def execute(self, sql, varDict=None, cur=None  # , returningInto=None
                ):
        if varDict is None:
            varDict = {}
        if cur is None:
            cur = self.cur
        ret = None
        if self.backend == 'oracle':
            # schema names
            sql = re.sub('ATLAS_PANDA\.',     panda_config.schemaPANDA + '.',     sql)
            sql = re.sub('ATLAS_PANDAMETA\.', panda_config.schemaMETA + '.',      sql)
            sql = re.sub('ATLAS_GRISLI\.',    panda_config.schemaGRISLI + '.',    sql)
            sql = re.sub('ATLAS_PANDAARCH\.', panda_config.schemaPANDAARCH + '.', sql)

            # remove `
            sql = re.sub('`','',sql)
            ret = cur.execute(sql, varDict)
        elif self.backend == 'mysql':
            print "DEBUG execute : original SQL     %s " % sql
            print "DEBUG execute : original varDict %s " % varDict
            # CURRENT_DATE interval
            sql = re.sub("CURRENT_DATE\s*-\s*(\d+|:[^\s\)]+)", "DATE_SUB(CURRENT_DATE,INTERVAL \g<1> DAY)", sql)
            # SYSDATE interval
            sql = re.sub("SYSDATE\s*-\s*(\d+|:[^\s\)]+)", "DATE_SUB(SYSDATE,INTERVAL \g<1> DAY)", sql)
            # SYSDATE
            sql = re.sub('SYSDATE', 'SYSDATE()', sql)
            # EMPTY_CLOB()
            sql = re.sub('EMPTY_CLOB\(\)', "''", sql)
            # ROWNUM
            sql = re.sub("(?i)(AND)*\s*ROWNUM\s*<=\s*(\d+)", " LIMIT \g<2>", sql)
            sql = re.sub("(?i)(WHERE)\s*LIMIT\s*(\d+)", " LIMIT \g<2>" , sql)
            # NOWAIT
            sql = re.sub('NOWAIT', "", sql)
            # RETURNING INTO
            returningInto = None
            m = re.search("RETURNING ([^\s]+) INTO ([^\s]+)", sql, re.I)
            if m is not None:
                returningInto = [{'returning': m.group(1), 'into': m.group(2)}]
                self._returningIntoMySQLpre(returningInto, varDict, cur)
                sql = re.sub(m.group(0), '', sql)
            # Addressing sequence
            if "INSERT" in sql:
                sql = re.sub('[a-zA-Z\._]+\.nextval','NULL',sql)
            # schema names
            sql = re.sub('ATLAS_PANDA\.',     panda_config.schemaPANDA + '.',     sql)
            sql = re.sub('ATLAS_PANDAMETA\.', panda_config.schemaMETA + '.',      sql)
            sql = re.sub('ATLAS_GRISLI\.',    panda_config.schemaGRISLI + '.',    sql)
            sql = re.sub('ATLAS_PANDAARCH\.', panda_config.schemaPANDAARCH + '.', sql)
            # bind variables
            newVarDict = {}
            # make sure that :prodDBlockToken will not be replaced by %(prodDBlock)sToken
            keys = sorted(varDict.keys(), key=lambda s:-len(str(s)))<commit_msg>Fix 2 regexes for ROWNUM to meet all cases<commit_after>"""
WrappedCursor for a generic database connection proxy

"""

import re
import os
import sys
import warnings
from pandalogger.PandaLogger import PandaLogger
from config import panda_config

warnings.filterwarnings('ignore')

# logger
_logger = PandaLogger().getLogger('WrappedCursor')

# proxy
class WrappedCursor(object):

    # constructor
    def __init__(self, connection):
        # connection object
        self.conn = connection
        # cursor object
        self.cur = self.conn.cursor()
        # backend
        self.backend = panda_config.backend
        # statement
        self.statement = None


    # __iter__
    def __iter__(self):
        return iter(self.cur)


    # serialize
    def __str__(self):
        return 'WrappedCursor[%(conn)s]' % ({'conn': self.conn})

    # initialize
    def initialize(self):
        hostname = None
        if self.backend == 'oracle':
            # get hostname
            self.execute("SELECT SYS_CONTEXT('USERENV','HOST') FROM dual")
            res = self.fetchone()
            if res != None:
                hostname = res[0]
            # set TZ
            self.execute("ALTER SESSION SET TIME_ZONE='UTC'")
            # set DATE format
            self.execute("ALTER SESSION SET NLS_DATE_FORMAT='YYYY/MM/DD HH24:MI:SS'")
        else:
            # get hostname
            self.execute("SELECT SUBSTRING_INDEX(USER(),'@',-1)")
            res = self.fetchone()
            if res != None:
                hostname = res[0]
            # set TZ
            self.execute("SET @@SESSION.TIME_ZONE = '+00:00'")
            # set DATE format
            #self.execute("SET @@SESSION.DATETIME_FORMAT='%%Y/%%m/%%d %%H:%%i:%%s'")
            # disable autocommit
            self.execute("SET autocommit=0")
        return hostname
    # execute query on cursor
    def execute(self, sql, varDict=None, cur=None  # , returningInto=None
                ):
        if varDict is None:
            varDict = {}
        if cur is None:
            cur = self.cur
        ret = None
        if self.backend == 'oracle':
            # schema names
            sql = re.sub('ATLAS_PANDA\.',     panda_config.schemaPANDA + '.',     sql)
            sql = re.sub('ATLAS_PANDAMETA\.', panda_config.schemaMETA + '.',      sql)
            sql = re.sub('ATLAS_GRISLI\.',    panda_config.schemaGRISLI + '.',    sql)
            sql = re.sub('ATLAS_PANDAARCH\.', panda_config.schemaPANDAARCH + '.', sql)

            # remove `
            sql = re.sub('`','',sql)
            ret = cur.execute(sql, varDict)
        elif self.backend == 'mysql':
            print "DEBUG execute : original SQL     %s " % sql
            print "DEBUG execute : original varDict %s " % varDict
            # CURRENT_DATE interval
            sql = re.sub("CURRENT_DATE\s*-\s*(\d+|:[^\s\)]+)", "DATE_SUB(CURRENT_DATE,INTERVAL \g<1> DAY)", sql)
            # SYSDATE interval
            sql = re.sub("SYSDATE\s*-\s*(\d+|:[^\s\)]+)", "DATE_SUB(SYSDATE,INTERVAL \g<1> DAY)", sql)
            # SYSDATE
            sql = re.sub('SYSDATE', 'SYSDATE()', sql)
            # EMPTY_CLOB()
            sql = re.sub('EMPTY_CLOB\(\)', "''", sql)
            # ROWNUM
            sql = re.sub("(?i)(AND)*\s*ROWNUM.*(\d+)", " LIMIT \g<2>", sql)
            sql = re.sub("(?i)(WHERE)\s*ROWNUM\s*(\d+)", " LIMIT \g<2>" , sql)
            # NOWAIT
            sql = re.sub('NOWAIT', "", sql)
            # RETURNING INTO
            returningInto = None
            m = re.search("RETURNING ([^\s]+) INTO ([^\s]+)", sql, re.I)
            if m is not None:
                returningInto = [{'returning': m.group(1), 'into': m.group(2)}]
                self._returningIntoMySQLpre(returningInto, varDict, cur)
                sql = re.sub(m.group(0), '', sql)
            # Addressing sequence
            if "INSERT" in sql:
                sql = re.sub('[a-zA-Z\._]+\.nextval','NULL',sql)
            # schema names
            sql = re.sub('ATLAS_PANDA\.',     panda_config.schemaPANDA + '.',     sql)
            sql = re.sub('ATLAS_PANDAMETA\.', panda_config.schemaMETA + '.',      sql)
            sql = re.sub('ATLAS_GRISLI\.',    panda_config.schemaGRISLI + '.',    sql)
            sql = re.sub('ATLAS_PANDAARCH\.', panda_config.schemaPANDAARCH + '.', sql)
            # bind variables
            newVarDict = {}
            # make sure that :prodDBlockToken will not be replaced by %(prodDBlock)sToken
            keys = sorted(varDict.keys(), key=lambda s:-len(str(s)))
{'fraction_non_alphanumeric': 0.146883946267342, 'fraction_numerical': 0.0038537767011671436, 'mean_word_length': 1.8491217063989962, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3002519', 'n_tokens_mistral': 2915, 'n_tokens_neox': 2834, 'n_words': 611}