<commit_before>import re, os
import logging
import inspect
from lxml import etree
from datetime import datetime

from django.db import connection
from django.http import HttpResponse, HttpResponseBadRequest

from transformers.csv import format_csv


MAX_MYSQL_TABLE_NAME_LENGTH = 64
MAX_PREFIX_LENGTH= 7
MAX_LENGTH = MAX_MYSQL_TABLE_NAME_LENGTH - MAX_PREFIX_LENGTH

def format_table_name(name, version=None, domain_name=None, prefix="schema_"):
    """Get rid of the leading 'http://dev.commcarehq.org/' or whatever host 
       at the start of the xmlns, to generate a table."""
    # NOTE: we may actually want these namespaces to make it to our table 
    # names eventually, though they are too long at the moment.
    r = re.match('http://[a-zA-Z\.]+/(?P<tail>.*)', name)
    if r:
        tail = r.group('tail')
        if tail: 
            # table prefix is appended after sanitation because
            # sanitation truncates to MAX_LENGTH minus len(prefix)
            name = tail
    if version:
        name = "%s_%s" % ( name, version )
    if domain_name:
        prefix = "%s%s_" % (prefix, domain_name)
    
    # this magical -5 is because postgres uses _pkey as an auto-name for the primary
    # key, and if it's > than 64 characters truncates it.  Technically we could get
    # away with just doing -1, but this way the pkey's look like pkeys in the database
    # This is a quick fix made on 6/17 and this code should really be cleaned better
    return ("%s%s" % (prefix, sanitize(name, MAX_MYSQL_TABLE_NAME_LENGTH - len(prefix) - 4))).lower()

def table_exists( table_name):
    """Returns whether a table exists."""
    query = "select * from " + table_name + " limit 1";
    cursor = connection.cursor()
    try:
        cursor.execute(query)
    except:
        return False<commit_msg>fix -4 to be -5 for _pkey accommodation.<commit_after>import re, os
import logging
import inspect
from lxml import etree
from datetime import datetime

from django.db import connection
from django.http import HttpResponse, HttpResponseBadRequest

from transformers.csv import format_csv


MAX_MYSQL_TABLE_NAME_LENGTH = 64
MAX_PREFIX_LENGTH= 7
MAX_LENGTH = MAX_MYSQL_TABLE_NAME_LENGTH - MAX_PREFIX_LENGTH

def format_table_name(name, version=None, domain_name=None, prefix="schema_"):
    """Get rid of the leading 'http://dev.commcarehq.org/' or whatever host 
       at the start of the xmlns, to generate a table."""
    # NOTE: we may actually want these namespaces to make it to our table 
    # names eventually, though they are too long at the moment.
    r = re.match('http://[a-zA-Z\.]+/(?P<tail>.*)', name)
    if r:
        tail = r.group('tail')
        if tail: 
            # table prefix is appended after sanitation because
            # sanitation truncates to MAX_LENGTH minus len(prefix)
            name = tail
    if version:
        name = "%s_%s" % ( name, version )
    if domain_name:
        prefix = "%s%s_" % (prefix, domain_name)
    
    # this magical -5 is because postgres uses _pkey as an auto-name for the primary
    # key, and if it's > than 64 characters truncates it.  Technically we could get
    # away with just doing -1, but this way the pkey's look like pkeys in the database
    # This is a quick fix made on 6/17 and this code should really be cleaned better
    return ("%s%s" % (prefix, sanitize(name, MAX_MYSQL_TABLE_NAME_LENGTH - len(prefix) - 5))).lower()

def table_exists( table_name):
    """Returns whether a table exists."""
    query = "select * from " + table_name + " limit 1";
    cursor = connection.cursor()
    try:
        cursor.execute(query)
    except:
        return False
{'fraction_non_alphanumeric': 0.09061045715849986, 'fraction_numerical': 0.0071174377224199285, 'mean_word_length': 3.091825307950728, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5086478', 'n_tokens_mistral': 1102, 'n_tokens_neox': 1061, 'n_words': 416}