<commit_before>import sys

from jcc import cpp

options = {
    'include': ('lib/tika-app-1.6-SNAPSHOT.jar','lib/org.eclipse.osgi.jar','lib/log4j.properties.jar', 'lib/translator.microsoft.properties.jar', 'lib/microsoft-translator-java-api-0.6.2.jar', 'lib/json-simple-1.1.jar'),
    'jar': ('lib/tika-parsers-1.6-SNAPSHOT.jar', 'lib/tika-core-1.6-SNAPSHOT.jar', 'lib/tika-translate-1.6-SNAPSHOT.jar'),
    'package': ('org.xml.sax', 'java.lang', 'java.util', 'java.text', 'java.io'),
    'python': 'tika',
    'version': '1.6-SNAPSHOT',
    'module': 'parser',
    'reserved': ('asm',),
    'classes': ('java.io.File', 'java.io.FileInputStream', 'java.io.ByteArrayInputStream', 'java.lang.System','java.lang.Runtime','java.util.Arrays','java.util.Collections','java.util.HashMap','java.util.HashSet','java.util.TreeSet','java.lang.IllegalStateException','java.lang.IndexOutOfBoundsException','java.util.NoSuchElementException','java.text.SimpleDateFormat','java.text.DecimalFormat','java.text.Collator','java.io.StringReader','java.io.DataInputStream'),
}

import sys
import os.path

jcc_args = []

for dir in sys.path:
    probe_dir = os.path.join(dir, 'jcc')
    if os.path.exists(probe_dir):
        print "Found jcc library at: %s" % probe_dir
        jcc_args = [os.path.join(probe_dir, 'nonexistent-argv-0')]
        break<commit_msg>- took out translator jar properties didn't work anyways<commit_after>import sys

from jcc import cpp

options = {
    'include': ('lib/tika-app-1.6-SNAPSHOT.jar','lib/org.eclipse.osgi.jar','lib/log4j.properties.jar', 'lib/microsoft-translator-java-api-0.6.2.jar', 'lib/json-simple-1.1.jar'),
    'jar': ('lib/tika-parsers-1.6-SNAPSHOT.jar', 'lib/tika-core-1.6-SNAPSHOT.jar', 'lib/tika-translate-1.6-SNAPSHOT.jar'),
    'package': ('org.xml.sax', 'java.lang', 'java.util', 'java.text', 'java.io'),
    'python': 'tika',
    'version': '1.6-SNAPSHOT',
    'module': 'parser',
    'reserved': ('asm',),
    'classes': ('java.io.File', 'java.io.FileInputStream', 'java.io.ByteArrayInputStream', 'java.lang.System','java.lang.Runtime','java.util.Arrays','java.util.Collections','java.util.HashMap','java.util.HashSet','java.util.TreeSet','java.lang.IllegalStateException','java.lang.IndexOutOfBoundsException','java.util.NoSuchElementException','java.text.SimpleDateFormat','java.text.DecimalFormat','java.text.Collator','java.io.StringReader','java.io.DataInputStream'),
}

import sys
import os.path

jcc_args = []

for dir in sys.path:
    probe_dir = os.path.join(dir, 'jcc')
    if os.path.exists(probe_dir):
        print "Found jcc library at: %s" % probe_dir
        jcc_args = [os.path.join(probe_dir, 'nonexistent-argv-0')]
        break
{'fraction_non_alphanumeric': 0.20454545454545456, 'fraction_numerical': 0.012667660208643815, 'mean_word_length': 4.582120582120582, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7598045', 'n_tokens_mistral': 996, 'n_tokens_neox': 1007, 'n_words': 126}