<commit_before>#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# changing author of this commit
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
__author__= '<NAME>'

import sys
import os
import os.path
import argparse
import glob
import gzip

def debug_here():
	# break at line 13 or 15
	return

def header_row():
	return ['placeholder']

def get_files(dir,name):
	flist=[]
	fpath=os.path.join(dir,name)
	fpath=glob.glob(fpath)
	for file in fpath:
		flist.append(file)	
	for f in next(os.walk(dir))[1]:
		folder=os.path.join(dir,f)
		fpath=os.path.join(folder,name)
		fpath=glob.glob(fpath)
		for file in fpath:
			flist.append(file)	
	return flist

def get_fid(fname):
	fid=''
	id=['','']
	for p in fname.split('_'):
		if p[0]=='A':
			id[0]=p
		elif p[0]=='P':
			id[1]=p
	fid='_'.join(id)
	return fid

def get_key(h):
	info=h.split(':')
	return info[5]+'_'+info[6]

def row_to_string(fid,key,data):
	row='\t'.join([fid,key])
	row=row+'\t'+ str(data)
	return row

def totals_to_string(totals):
	txt=''
	for k in totals.keys():
		txt=txt+str(k) + '\t' + str(totals[k]) + '\t'<commit_msg>Adding a valid index tag<commit_after>#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# changing author of this commit
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
__author__= '<NAME>'

import sys
import os
import os.path
import argparse
import glob
import gzip

def debug_here():
	# break at line 13 or 15
	return

def header_row():
	return ['placeholder']

def get_files(dir,name):
	flist=[]
	fpath=os.path.join(dir,name)
	fpath=glob.glob(fpath)
	for file in fpath:
		flist.append(file)	
	for f in next(os.walk(dir))[1]:
		folder=os.path.join(dir,f)
		fpath=os.path.join(folder,name)
		fpath=glob.glob(fpath)
		for file in fpath:
			flist.append(file)	
	return flist

def get_fid(fname):
	fid=''
	id=['','']
	for p in fname.split('_'):
		if p[0]=='A':
			id[0]=p
		elif p[0]=='P':
			id[1]=p
	for n in range(2):
		id[n]=id[n].split('.')[0]
	fid='_'.join(id)
	return fid

def get_key(h):
	info=h.split(':')
	return info[5]+'_'+info[6]

def row_to_string(fid,key,data):
	row='\t'.join([fid,key])
	row=row+'\t'+ str(data)
	return row

def totals_to_string(totals):
	txt=''
	for k in totals.keys():
		txt=txt+str(k) + '\t' + str(totals[k]) + '\t'
{'fraction_non_alphanumeric': 0.269010989010989, 'fraction_numerical': 0.01054945054945055, 'mean_word_length': 5.101876675603217, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6923976', 'n_tokens_mistral': 982, 'n_tokens_neox': 929, 'n_words': 202}