<commit_before>import ConfigParser, argparse, os, sys, oerplib
import requests
import requests.auth
import subprocess
import shutil
import time
import re
import socket

import ufload

def _home():
    if sys.platform == "win32" and 'USERPROFILE' in os.environ:
        return os.environ['USERPROFILE']
    return os.environ['HOME']

_logs = []
args=[]
#Create log file (if it does not exist, else append to existing file)
filename = '%s/uf_%s.log' % (args.local, time.strftime('%Y%m%d%H%M%S'))
def _progress(p):
    global _logs, args
    p = time.strftime('%H:%M:%S') + ': ' + p
    print >> sys.stderr, p
    _logs.append(p)

    if args.local:
        #Create directory if necessary
        try:
            os.stat(args.local)
        except:
            os.mkdir(args.local)
        #Create log file (if it does not exist, else append to existing file)
        filename = '%s/uf_%s.log' % (args.local, time.strftime('%Y%m%d'))
        #Write logs to file
        with open(filename, 'ab') as file:
            #file.write('\n'.join(_logs))
            file.write('%s\n' % p)

ufload.progress = _progress

def _ocToDir(oc):
    x = oc.lower()
    if x == 'oca':
        return 'OCA_Backups'
    elif x == 'ocb':
        return 'OCB_Backups'
    elif x == 'ocg':
        return 'OCG_Backups'
    elif x == 'ocp':
        return 'OCP_Backups'
    else:
        # no OC abbrev, assume this is a real directory name
        return oc


def _required(args, req):<commit_msg>corection<commit_after>import ConfigParser, argparse, os, sys, oerplib
import requests
import requests.auth
import subprocess
import shutil
import time
import re
import socket

import ufload

def _home():
    if sys.platform == "win32" and 'USERPROFILE' in os.environ:
        return os.environ['USERPROFILE']
    return os.environ['HOME']

_logs = []
args=[]
starttime = time.strftime('%Y%m%d%H%M%S')
def _progress(p):
    global _logs, args
    p = time.strftime('%H:%M:%S') + ': ' + p
    print >> sys.stderr, p
    _logs.append(p)

    if args.local:
        #Create directory if necessary
        try:
            os.stat(args.local)
        except:
            os.mkdir(args.local)
        #Create log file (if it does not exist, else append to existing file)
        filename = '%s/uf_%s.log' % (args.local, starttime)
        #Write logs to file
        with open(filename, 'ab') as file:
            #file.write('\n'.join(_logs))
            file.write('%s\n' % p)

ufload.progress = _progress

def _ocToDir(oc):
    x = oc.lower()
    if x == 'oca':
        return 'OCA_Backups'
    elif x == 'ocb':
        return 'OCB_Backups'
    elif x == 'ocg':
        return 'OCG_Backups'
    elif x == 'ocp':
        return 'OCP_Backups'
    else:
        # no OC abbrev, assume this is a real directory name
        return oc


def _required(args, req):
{'fraction_non_alphanumeric': 0.1304809052333805, 'fraction_numerical': 0.0014144271570014145, 'mean_word_length': 2.4882860665844637, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2085680', 'n_tokens_mistral': 969, 'n_tokens_neox': 957, 'n_words': 278}