<commit_before>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4 encoding=utf-8

from django.core.management import execute_manager
import sys, os

filedir = os.path.dirname(__file__)

# remove pyc files
for root, dirs, files in os.walk(filedir):
    for file in files:
        if any(file.endswith(e) for e in ['pyc', 'pyo', 'py.class']):
            os.remove(os.path.join(root, file))


sys.path.append(os.path.join(filedir))
sys.path.append(os.path.join(filedir,'submodules'))
submodules_list = os.listdir(os.path.join(filedir, 'submodules'))
for d in submodules_list:
    if d == "__init__.py" or d == '.' or d == '..':
        continue
    sys.path.append(os.path.join(filedir,'submodules',d))

try:<commit_msg>make submodules precede virtualenv packages in path<commit_after>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4 encoding=utf-8

from django.core.management import execute_manager
import sys, os

filedir = os.path.dirname(__file__)

# remove pyc files
for root, dirs, files in os.walk(filedir):
    for file in files:
        if any(file.endswith(e) for e in ['pyc', 'pyo', 'py.class']):
            os.remove(os.path.join(root, file))


submodules_list = os.listdir(os.path.join(filedir, 'submodules'))
submodule_paths = []
for d in submodules_list:
    if d == "__init__.py" or d == '.' or d == '..':
        continue
    submodule_paths.append(os.path.join(filedir,'submodules',d))

sys.path.append(os.path.join(filedir,'submodules'))
sys.path = [filedir] + submodule_paths + sys.path

try:
{'fraction_non_alphanumeric': 0.15521796565389695, 'fraction_numerical': 0.005284015852047556, 'mean_word_length': 3.918831168831169, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3537640', 'n_tokens_mistral': 563, 'n_tokens_neox': 546, 'n_words': 129}