<jupyter_start><jupyter_text>Output<jupyter_code>if os.path.isdir('takehome'):
    shutil.rmtree('takehome')
time.sleep(2)
os.mkdir('takehome')
os.mkdir('takehome/data')
os.mkdir('takehome/ConSav')
os.mkdir('takehome/censored_estimates')
# a. choose batch size
batch = 50
exts = ['pdf','tex','xls','txt']

# b. copy to sub-folder
files = []
for ext in exts:
    files.extend(glob.glob(f'output/*.{ext}'))
for i,file in enumerate(files):
    if i==0 or i%batch==0:
        dirnow = f'takehome/{i+1}_{i+batch}'
        os.mkdir(dirnow)
    dst = dirnow + '/' + file.split('\\')[1]
    shutil.copyfile(file,dst)
    
# c. consav copy to sub-folder
files = []
for ext in exts:
    files.extend(glob.glob(f'output/ConSav/*.{ext}'))
    
for i,file in enumerate(files):
    if i==0 or i%batch==0:
        dirnow = f'takehome/ConSav/{i+1}_{i+batch}'
        os.mkdir(dirnow)
    dst = dirnow + '/' + file.split('\\')[1]
    shutil.copyfile(file,dst)
    
# d. censored_estiamted copy to sub-folder
files = []
for ext in exts:
    files.extend(glob.glob(f'censored_estimates/*.{ext}'))
    
for i,file in enumerate(files):
    if i==0 or i%batch==0:
        dirnow = f'takehome/censored_estimates/{i+1}_{i+batch}'
        os.mkdir(dirnow)
    dst = dirnow + '/' + file.split('\\')[1]
    shutil.copyfile(file,dst)
    
# e. data
shutil.copyfile('data/a_mean_p1.csv','takehome/data/a_mean_p1.txt')
shutil.copyfile('data/a_mean_p100.csv','takehome/data/a_mean_p100.txt')
shutil.copyfile('data/a_p50_p1.csv','takehome/data/a_p50_p1.txt')
shutil.copyfile('data/a_p50_p100.csv','takehome/data/a_p50_p100.txt')<jupyter_output><empty_output><jupyter_text>Code<jupyter_code>if os.path.isdir('takehome_code'):
    shutil.rmtree('takehome_code')
time.sleep(2)    
os.mkdir('takehome_code')
# a. data
files = glob.glob('data/*.xlsx')
for i,file in enumerate(files):
    dst = 'takehome_code' + '/data_' + file.split('\\')[1]
    shutil.copyfile(file,dst)
    
shutil.copyfile('data/a_mean_p1.csv','takehome_code/data_a_mean_p1__csv.txt')
shutil.copyfile('data/a_mean_p100.csv','takehome_code/data_a_mean_p100__csv.txt')
shutil.copyfile('data/a_p50_p1.csv','takehome_code/data_a_p50_p1__csv.txt')
shutil.copyfile('data/a_p50_p100.csv','takehome_code/data_a_p50_p100__csv.txt')
    
# b. code
for (dirpath,dirnames,filenames) in os.walk(os.getcwd()):
    
    # i. dir
    if dirpath == 'K:\\Workdata\\706248\\LRT':
        dirpathshort = ''
    else:
        dirpathshort = dirpath.split('K:\\Workdata\\706248\\LRT\\')[1]        
    
    if not dirpathshort in ['','ABB','ConSav','ConSav\cfuncs','ConSav\cfuncs\includes','LRT']:
        continue
        
    dirpathshort = dirpathshort.replace('\\','__')
        
    # ii. file
    for filename in filenames:
        
        body = filename.split('.')[0]
        ext = filename.split('.')[1]
        
        if not ext in ['sas','do','m','cpp','ipynb','md','py']:
            continue
            
        if body in ['make_takehome']:
            continue
            
        src = dirpath + '\\' + filename
        dst = 'takehome_code/' +  dirpathshort + '__' + body + '__' + ext + '.txt'
        
        shutil.copyfile(src,dst)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18099406064395124, 'fraction_numerical': 0.02500781494216943, 'mean_word_length': 3.25531914893617, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '233811', 'license': 'MIT', 'n_tokens_mistral': 1318, 'n_tokens_neox': 1258, 'n_words': 174, 'path': 'takehome.ipynb', 'repo_name': 'JeppeDruedahl/LRT', 'size': '3199'}