<commit_before>#!/usr/bin/python

import os, sys, re

# Path to eng checkout root directory. To use this as a git pre-commit hook,
# create a symlink to this file in .git/hooks with the name pre-commit
basepath = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + os.sep
ascommithook = False

prunelist = ('hsqldb19b3',
             'hsqldb',
             'jetty716',
             'proj_gen',
             'jni_md.h',
             'jni.h',
             'org_voltdb_jni_ExecutionEngine.h',
             'org_voltcore_utils_DBBPool.h',
             'org_voltcore_utils_DBBPool_DBBContainer.h',
             'simplejson',
             'projectfile',
             'deploymentfile',
             'xml',
             'helloworld',
             'jaxb',
             'pmsg')

def verifyLicense(f, content, approvedLicensesJavaC, approvedLicensesPython):
    if f.endswith('.py'):
        if not content.startswith("#"):
            if content.lstrip().startswith("#"):
                print "ERROR: \"%s\" contains whitespace before initial comment." % f
                return 1
            else:
                print "ERROR: \"%s\" does not begin with a comment." % f
                return 1

        # skip hashbang
        if content.startswith("#!"):
            (ignore, content) = content.split("\n", 1)
            content = content.lstrip()

        # skip python coding magic
        if content.startswith("# -*-"):
            (ignore, content) = content.split("\n", 1)
            content = content.lstrip()

        # verify license
        for license in approvedLicensesPython:
            if content.startswith(license):
                return 0
        print "ERROR: \"%s\" does not start with an approved license." % f
    else:
        if not content.startswith("/*"):
            if content.lstrip().startswith("/*"):
                print "ERROR: \"%s\" contains whitespace before initial comment." % f
            else:
                print "ERROR: \"%s\" does not begin with a comment." % f
            return 1
        for license in approvedLicensesJavaC:
            if content.startswith(license):
                return 0
        print "ERROR: \"%s\" does not start with an approved license." % f
    return 1

def verifyTrailingWhitespace(f, content):
    if re.search(r'[\t\f\v ]\n', content):
        print("ERROR: \"%s\" contains trailing whitespace." % (f))
        return 1
    return 0

def verifyTabs(f, content):
    num = content.count('\t')
    if num  > 0:
        print("ERROR: \"%s\" contains %d tabs." % (f, num))
        return 1
    return 0

def verifySprintf(f, content):
    num = content.count('sprintf')
    if num > 0:
        print("ERROR: \"%s\" contains %d calls to sprintf(). Use snprintf()." % (f, num))
        return 1
    return 0

def verifyGetStringChars(f, content):
    if not (f.endswith('.cpp') or f.endswith('.c') or f.endswith('.h') or f.endswith('.hpp')):
        return 0
    num = content.count('GetStringChars')
    num += content.count('GetStringUTFChars')
    if num > 0:
        print("ERROR: \"%s\" contains %d calls to GetStringChars/GetStringUTFChars." % (f, num))
        return 1
    return 0

def readFile(filename):
    "read a file into a string"
    FH=open(filename, 'r')
    fileString = FH.read()
    FH.close()
    return fileString

def processFile(f, approvedLicensesJavaC, approvedLicensesPython):
    for suffix in ('.java', '.cpp', '.cc', '.h', '.hpp', '.py'):
        if f.endswith(suffix):
            break
    else:
        return 0
    content = readFile(f)
    retval = verifyLicense(f, content,  approvedLicensesJavaC, approvedLicensesPython)
    if retval != 0:
        return retval
    retval = verifyTabs(f, content)
    if retval != 0:
        return retval
    retval = verifyTrailingWhitespace(f, content)
    if (retval != 0):
        return retval
    retval = verifySprintf(f, content)
    if (retval != 0):<commit_msg>Improve the error message to explain why GetStringChars/GetStringUTFChars is no acceptable.<commit_after>#!/usr/bin/python

import os, sys, re

# Path to eng checkout root directory. To use this as a git pre-commit hook,
# create a symlink to this file in .git/hooks with the name pre-commit
basepath = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + os.sep
ascommithook = False

prunelist = ('hsqldb19b3',
             'hsqldb',
             'jetty716',
             'proj_gen',
             'jni_md.h',
             'jni.h',
             'org_voltdb_jni_ExecutionEngine.h',
             'org_voltcore_utils_DBBPool.h',
             'org_voltcore_utils_DBBPool_DBBContainer.h',
             'simplejson',
             'projectfile',
             'deploymentfile',
             'xml',
             'helloworld',
             'jaxb',
             'pmsg')

def verifyLicense(f, content, approvedLicensesJavaC, approvedLicensesPython):
    if f.endswith('.py'):
        if not content.startswith("#"):
            if content.lstrip().startswith("#"):
                print "ERROR: \"%s\" contains whitespace before initial comment." % f
                return 1
            else:
                print "ERROR: \"%s\" does not begin with a comment." % f
                return 1

        # skip hashbang
        if content.startswith("#!"):
            (ignore, content) = content.split("\n", 1)
            content = content.lstrip()

        # skip python coding magic
        if content.startswith("# -*-"):
            (ignore, content) = content.split("\n", 1)
            content = content.lstrip()

        # verify license
        for license in approvedLicensesPython:
            if content.startswith(license):
                return 0
        print "ERROR: \"%s\" does not start with an approved license." % f
    else:
        if not content.startswith("/*"):
            if content.lstrip().startswith("/*"):
                print "ERROR: \"%s\" contains whitespace before initial comment." % f
            else:
                print "ERROR: \"%s\" does not begin with a comment." % f
            return 1
        for license in approvedLicensesJavaC:
            if content.startswith(license):
                return 0
        print "ERROR: \"%s\" does not start with an approved license." % f
    return 1

def verifyTrailingWhitespace(f, content):
    if re.search(r'[\t\f\v ]\n', content):
        print("ERROR: \"%s\" contains trailing whitespace." % (f))
        return 1
    return 0

def verifyTabs(f, content):
    num = content.count('\t')
    if num  > 0:
        print("ERROR: \"%s\" contains %d tabs." % (f, num))
        return 1
    return 0

def verifySprintf(f, content):
    num = content.count('sprintf')
    if num > 0:
        print("ERROR: \"%s\" contains %d calls to sprintf(). Use snprintf()." % (f, num))
        return 1
    return 0

def verifyGetStringChars(f, content):
    if not (f.endswith('.cpp') or f.endswith('.c') or f.endswith('.h') or f.endswith('.hpp')):
        return 0
    num = content.count('GetStringChars')
    num += content.count('GetStringUTFChars')
    if num > 0:
        print("ERROR: \"%s\" contains %d calls to GetStringChars/GetStringUTFChars. These methods return invalid UTF-8 code points for some characters. You should do the encoding in Java and pass the string to native code as a byte array." % (f, num))
        return 1
    return 0

def readFile(filename):
    "read a file into a string"
    FH=open(filename, 'r')
    fileString = FH.read()
    FH.close()
    return fileString

def processFile(f, approvedLicensesJavaC, approvedLicensesPython):
    for suffix in ('.java', '.cpp', '.cc', '.h', '.hpp', '.py'):
        if f.endswith(suffix):
            break
    else:
        return 0
    content = readFile(f)
    retval = verifyLicense(f, content,  approvedLicensesJavaC, approvedLicensesPython)
    if retval != 0:
        return retval
    retval = verifyTabs(f, content)
    if retval != 0:
        return retval
    retval = verifyTrailingWhitespace(f, content)
    if (retval != 0):
        return retval
    retval = verifySprintf(f, content)
    if (retval != 0):
{'fraction_non_alphanumeric': 0.12624953720844131, 'fraction_numerical': 0.00777489818585709, 'mean_word_length': 2.0813688212927755, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4589419', 'n_tokens_mistral': 2488, 'n_tokens_neox': 2380, 'n_words': 721}