<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

from __future__ import print_function

import ast
import logging
import multiprocessing
import os
import sys
from multiprocessing.pool import ThreadPool
from subprocess import PIPE, STDOUT, Popen
from datetime import datetime

# append root of the python code tree to sys.apth so that imports are working
#   alternative: add path to riotam_backend to the PYTHONPATH environment variable, but this includes one more step
#   which could be forget
CUR_DIR = os.path.abspath(os.path.dirname(__file__))
PROJECT_ROOT_DIR = os.path.normpath(os.path.join(CUR_DIR, "..", "..", ".."))
sys.path.append(PROJECT_ROOT_DIR)

from BuildTaskStatistic import BuildTaskStatistic
from riotam_backend.common.MyDatabase import MyDatabase

LOGFILE = os.path.join(PROJECT_ROOT_DIR, "log", "prepare_all_log.txt")
USING_CACHE = False

db = MyDatabase()
stat = BuildTaskStatistic()


def main():

    thread_count = multiprocessing.cpu_count()

    print("preparing build tasks...")
    tasks = get_build_tasks()[:2]

    print("got %s tasks" % len(tasks))
<commit_msg>set usingcache to true for testing........<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

from __future__ import print_function

import ast
import logging
import multiprocessing
import os
import sys
from multiprocessing.pool import ThreadPool
from subprocess import PIPE, STDOUT, Popen
from datetime import datetime

# append root of the python code tree to sys.apth so that imports are working
#   alternative: add path to riotam_backend to the PYTHONPATH environment variable, but this includes one more step
#   which could be forget
CUR_DIR = os.path.abspath(os.path.dirname(__file__))
PROJECT_ROOT_DIR = os.path.normpath(os.path.join(CUR_DIR, "..", "..", ".."))
sys.path.append(PROJECT_ROOT_DIR)

from BuildTaskStatistic import BuildTaskStatistic
from riotam_backend.common.MyDatabase import MyDatabase

LOGFILE = os.path.join(PROJECT_ROOT_DIR, "log", "prepare_all_log.txt")
USING_CACHE = True

db = MyDatabase()
stat = BuildTaskStatistic()


def main():

    thread_count = multiprocessing.cpu_count()

    print("preparing build tasks...")
    tasks = get_build_tasks()[:2]

    print("got %s tasks" % len(tasks))

{'fraction_non_alphanumeric': 0.10357142857142858, 'fraction_numerical': 0.0017857142857142857, 'mean_word_length': 4.805699481865285, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2637891', 'n_tokens_mistral': 716, 'n_tokens_neox': 697, 'n_words': 209}