<commit_before>import csv, os, re, sqlite3, types, unicodedata, zipfile
from ftplib import *

def jarow(s1,s2):
    try:
        #s1 = s1.upper()
        #s2 = s2.upper()
        if s1==s2:
            return 1.0
        #if s1=="" or s2=="":
        #    return 0.0
        short, long = len(s1)>len(s2) and [s2, s1] or [s1, s2]
        for l in range(0, min(5, len(short))):
          if short[l] != long[l]:
            break
        mtch = ""
        mtch2=[]
        dist = len(long)/2-1
        m = 0.0

        for i, x in enumerate(long):
          jx, jy = (lambda x,y: x==y and (x,y+1) or (x,y))(max(0, i-dist), min(len(short), i+dist))
          for j in range(jx, jy):
            if j<len(short) and x == short[j]:
              m+=1
              mtch+=x
              mtch2.extend([[j,x]])
              short=short[:j]+"*"+short[min(len(short), j+1):]
              break
        mtch2 = "".join(x[1] for x in sorted(mtch2))
        t = 0.0

        for i in range(0, len(mtch)):      
          if mtch[i]!=mtch2[i]:
            t += 0.5

        d = 0.1 
        # this is the jaro-distance 
        if m==0:
          d_j = 0
        else:
          d_j = 1/3.0 * ((m/len(short)) + (m/len(long)) + ((m - t)/m))
        return d_j + (l * d * (1 - d_j))
    except:
        print "Jaro-Winkler exception thrown on comparison between " + s1 + " and " + s2
        return 0

def tblExist(c, table):
	    return c.execute("SELECT count(*) FROM sqlite_master WHERE tbl_name='%s'" % table).fetchone()[0]>0

class uQvals:
    def __init__(self):
        self.dList=[]
    def step(self, value):
        self.dList.append(value.upper())
    def finalize(self):
        out = list(set(self.dList))
        return len(out)>1 and "|".join(out) or ""

def uniVert(data):
    def uni(data):
        try: return unicode(data, "utf-8")
        except: return unicode("")
    return [[uni(y) for y in x] for x in data]


def quickSQL(c, data, table="", header=False, typescan=50, typeList = []):
    if table=="":
        table = "debug%d" % len([x[0] for x in c.execute("SELECT tbl_name FROM sqlite_master WHERE type='table' order by tbl_name") if len(re.findall(r"debug[0-9]+", x[0]))>0])
    else:
        if c.execute("SELECT count(*) FROM sqlite_master WHERE tbl_name='%s'" % table).fetchone()[0]>0:
            return

    tList = []
    for i,x in enumerate(data[1]):
        if str(typeList).upper().find("%s " % data[0][i].upper())<0:
            cType = {types.StringType:"VARCHAR", types.UnicodeType:"VARCHAR", types.IntType:"INTEGER", types.FloatType: "REAL"}[type(x)]
            if type(typescan)==types.IntType and cType=="VARCHAR":
                least = 2
                ints = 1
                for j in range(1, min(typescan+1, len(data))):
                    if type(data[j][i])==types.StringType or type(data[j][i])==types.UnicodeType:
                        if re.sub(r"[-,.]", "", data[j][i]).isdigit():
                            if len(re.findall(r"[.]", data[j][i]))==0:   pass
                            elif len(re.findall(r"[.]", data[j][i]))==1: ints = 0
                            else: least = 0; break
                        else: least = 0; break
                cType = {0:"VARCHAR", 1:"INTEGER", 2:"REAL"}[max(least-ints, 0)]
            if header:
                tList.append("%s %s" % (data[0][i], cType))
            else:
                tList.append("v%d %s" % (i, cType))
        else:
            tList.extend([y for y in typeList if y.upper().find("%s " % data[0][i].upper())==0])

    c.execute("CREATE TABLE IF NOT EXISTS %s (%s)" % (table, ", ".join(tList)))
    if header==False:
        c.executemany("INSERT INTO %s VALUES (%s)" % (table, ", ".join(["?"]*len(data[0]))), data)
    else:
        c.executemany("INSERT INTO %s VALUES (%s)" % (table, ", ".join(["?"]*len(data[0]))), data[1:])


####################################
def get_ctypes(x):
    return { types.StringType:"VARCHAR",
             types.UnicodeType:"VARCHAR",
             types.IntType:"INTEGER",
             types.FloatType: "REAL"
	   }[type(x)]

def text_type(datatype):
    return type(datatype)==types.StringType or type(datatype)==types.UnicodeType


# The naming here is unfortunate with respect to how the
# program logic works.
def is_real(data):
    lengthall = len(re.findall(r"[.]", data))
    #print "length: ", lengthall
    return lengthall


# This function won't handle valid floating point notation.
def is_all_digits(data):
    return re.sub(r"[-,.]", "", data).isdigit()


def get_ctype(typescan, data, i):

	least = 2
	ints = 1
	for j in range(1, min(typescan+1, len(data))):
	    if type(data[j][i])==types.StringType or type(data[j][i])==types.UnicodeType:
		if re.sub(r"[-,.]", "", data[j][i]).isdigit():

		    lengthall = is_real(data[j][i])

		    if lengthall == 0:
		        pass
		    elif lengthall == 1:
			ints = 0
		    else:
			least = 0;
			# This break is unfortunate, prevents an easy refactoring of
			# the conditional logic. General principal: try not to break
			# out of loops from deeply nested conditionals.
			break
		else:
		    least = 0;
		    break

        key = max(least-ints, 0)
	print "key: ", key
	value =  {0:"VARCHAR", 1:"INTEGER", 2:"REAL"}[key]
	print "value: ", value<commit_msg>big cleanup in get_ctype function<commit_after>import csv, os, re, sqlite3, types, unicodedata, zipfile
from ftplib import *

def jarow(s1,s2):
    try:
        #s1 = s1.upper()
        #s2 = s2.upper()
        if s1==s2:
            return 1.0
        #if s1=="" or s2=="":
        #    return 0.0
        short, long = len(s1)>len(s2) and [s2, s1] or [s1, s2]
        for l in range(0, min(5, len(short))):
          if short[l] != long[l]:
            break
        mtch = ""
        mtch2=[]
        dist = len(long)/2-1
        m = 0.0

        for i, x in enumerate(long):
          jx, jy = (lambda x,y: x==y and (x,y+1) or (x,y))(max(0, i-dist), min(len(short), i+dist))
          for j in range(jx, jy):
            if j<len(short) and x == short[j]:
              m+=1
              mtch+=x
              mtch2.extend([[j,x]])
              short=short[:j]+"*"+short[min(len(short), j+1):]
              break
        mtch2 = "".join(x[1] for x in sorted(mtch2))
        t = 0.0

        for i in range(0, len(mtch)):      
          if mtch[i]!=mtch2[i]:
            t += 0.5

        d = 0.1 
        # this is the jaro-distance 
        if m==0:
          d_j = 0
        else:
          d_j = 1/3.0 * ((m/len(short)) + (m/len(long)) + ((m - t)/m))
        return d_j + (l * d * (1 - d_j))
    except:
        print "Jaro-Winkler exception thrown on comparison between " + s1 + " and " + s2
        return 0

def tblExist(c, table):
	    return c.execute("SELECT count(*) FROM sqlite_master WHERE tbl_name='%s'" % table).fetchone()[0]>0

class uQvals:
    def __init__(self):
        self.dList=[]
    def step(self, value):
        self.dList.append(value.upper())
    def finalize(self):
        out = list(set(self.dList))
        return len(out)>1 and "|".join(out) or ""

def uniVert(data):
    def uni(data):
        try: return unicode(data, "utf-8")
        except: return unicode("")
    return [[uni(y) for y in x] for x in data]


def quickSQL(c, data, table="", header=False, typescan=50, typeList = []):
    if table=="":
        table = "debug%d" % len([x[0] for x in c.execute("SELECT tbl_name FROM sqlite_master WHERE type='table' order by tbl_name") if len(re.findall(r"debug[0-9]+", x[0]))>0])
    else:
        if c.execute("SELECT count(*) FROM sqlite_master WHERE tbl_name='%s'" % table).fetchone()[0]>0:
            return

    tList = []
    for i,x in enumerate(data[1]):
        if str(typeList).upper().find("%s " % data[0][i].upper())<0:
            cType = {types.StringType:"VARCHAR", types.UnicodeType:"VARCHAR", types.IntType:"INTEGER", types.FloatType: "REAL"}[type(x)]
            if type(typescan)==types.IntType and cType=="VARCHAR":
                least = 2
                ints = 1
                for j in range(1, min(typescan+1, len(data))):
                    if type(data[j][i])==types.StringType or type(data[j][i])==types.UnicodeType:
                        if re.sub(r"[-,.]", "", data[j][i]).isdigit():
                            if len(re.findall(r"[.]", data[j][i]))==0:   pass
                            elif len(re.findall(r"[.]", data[j][i]))==1: ints = 0
                            else: least = 0; break
                        else: least = 0; break
                cType = {0:"VARCHAR", 1:"INTEGER", 2:"REAL"}[max(least-ints, 0)]
            if header:
                tList.append("%s %s" % (data[0][i], cType))
            else:
                tList.append("v%d %s" % (i, cType))
        else:
            tList.extend([y for y in typeList if y.upper().find("%s " % data[0][i].upper())==0])

    c.execute("CREATE TABLE IF NOT EXISTS %s (%s)" % (table, ", ".join(tList)))
    if header==False:
        c.executemany("INSERT INTO %s VALUES (%s)" % (table, ", ".join(["?"]*len(data[0]))), data)
    else:
        c.executemany("INSERT INTO %s VALUES (%s)" % (table, ", ".join(["?"]*len(data[0]))), data[1:])


####################################
def get_ctypes(x):
    return { types.StringType:"VARCHAR",
             types.UnicodeType:"VARCHAR",
             types.IntType:"INTEGER",
             types.FloatType: "REAL"
	   }[type(x)]

def text_type(datatype):
    return type(datatype)==types.StringType or type(datatype)==types.UnicodeType


# The naming here is unfortunate with respect to how the
# program logic works.
def is_real(data):
    lengthall = len(re.findall(r"[.]", data))
    #print "length: ", lengthall
    return lengthall


# This function won't handle valid floating point notation.
def is_all_digits(data):
    return re.sub(r"[-,.]", "", data).isdigit()


def get_ctype(typescan, data, i):

	least = 2
	ints = 1
	length = min(typescan+1, len(data))

	for j in range(1, length):

	    if text_type(data[j][i]):

		if re.sub(r"[-,.]", "", data[j][i]).isdigit():

		    lengthall = is_real(data[j][i])

		    if lengthall == 0:
		        pass
		    elif lengthall == 1:
			ints = 0
		    else:
			least = 0;
			# This break is unfortunate, prevents an easy refactoring of
			# the conditional logic. General principal: try not to break
			# out of loops from deeply nested conditionals.
			break
		else:
		    least = 0;
		    break

        key = max(least-ints, 0)
	print "key: ", key
	value =  {0:"VARCHAR", 1:"INTEGER", 2:"REAL"}[key]
	print "value: ", value
{'fraction_non_alphanumeric': 0.1802391799544419, 'fraction_numerical': 0.01917236142748671, 'mean_word_length': 2.1680697534576066, 'pattern_counts': {'":': 6, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3770526', 'n_tokens_mistral': 3889, 'n_tokens_neox': 3638, 'n_words': 923}