<commit_before>DUMP_BASENAME = 'wikidata-{date}-all.json.gz'
DUMP_LOCATION = '/public/dumps/public/wikidatawiki/entities/'
DUMP_LINK = '/data/project/sqid/projects/dumpfiles/wikidatawiki/json-{date}/{date}.json.gz'
GRID_SUBMIT = '/usr/bin/jsub'
GRID_MEMORY = '-mem'
GRID_NAME = '-N'
GRID_ONCE = '-once'
DUMP_PROCESS_MEMORY = '20g'
STATISTICS_PROCESS_MEMORY = '2g'
JAVA_BASEDIR = '/data/project/sqid/projects'
JAR = 'org.wikidata.sqid.helper-0.10.0.jar'
JAVA_MEMORY = '4g'
JAVA_ARGS = ['-Xmx{memory}'.format(memory=JAVA_MEMORY),
             '-jar', JAR,
             '-a', 'sqid', '-n']
RESULTS_LOCATION = '/data/project/sqid/projects/results/wikidatawiki-{date}'
RESULTS_NAMES = ['classes.json', 'properties.json', 'statistics.json']<commit_msg>Bump heap size in helper<commit_after>DUMP_BASENAME = 'wikidata-{date}-all.json.gz'
DUMP_LOCATION = '/public/dumps/public/wikidatawiki/entities/'
DUMP_LINK = '/data/project/sqid/projects/dumpfiles/wikidatawiki/json-{date}/{date}.json.gz'
GRID_SUBMIT = '/usr/bin/jsub'
GRID_MEMORY = '-mem'
GRID_NAME = '-N'
GRID_ONCE = '-once'
DUMP_PROCESS_MEMORY = '21g'
STATISTICS_PROCESS_MEMORY = '2g'
JAVA_BASEDIR = '/data/project/sqid/projects'
JAR = 'org.wikidata.sqid.helper-0.10.0.jar'
JAVA_MEMORY = '5g'
JAVA_ARGS = ['-Xmx{memory}'.format(memory=JAVA_MEMORY),
             '-jar', JAR,
             '-a', 'sqid', '-n']
RESULTS_LOCATION = '/data/project/sqid/projects/results/wikidatawiki-{date}'
RESULTS_NAMES = ['classes.json', 'properties.json', 'statistics.json']
{'fraction_non_alphanumeric': 0.1823020625415835, 'fraction_numerical': 0.010645375914836993, 'mean_word_length': 4.675471698113207, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4095805', 'n_tokens_mistral': 634, 'n_tokens_neox': 611, 'n_words': 77}