<commit_before>import os 
import argparse

"""
        Generate dev set in such a way that 

   		1) if user is picked for train then all chunks of the user fall into train
   		2) if in oversampled mode appropriate over sampled input for the correct users should be in train set
   		3) test set should not be in oversampling 
   		4) maintain ratio of positive - negative users

"""
join = lambda x,y: os.path.join(x,y)
isdir = lambda x: os.path.isdir(x)
isfile = lambda y: os.path.isfile(y)


def get_usernames(path):
	# get user names from <path>/<positive folder>/<chunk1-2>

	subdirs = [ join(path,subdir) for subdir in os.listdir(path) if isdir(join(path,subdir))]
	print 
	positive_dir = [ subdir for subdir in subdirs if 'positive' in subdir ] [0]
	target_folder = positive_dir+'/'+'chunk1-2'

	if not isdir(target_folder):
		print 'positive users subfolder does not have chunk1-2 !!'
		exit(1)

	return [ u.split('.')[0] for u in os.listdir(target_folder) if isfile(join(target_folder,u)) ]

if __name__ == '__main__':

	parser = argparse.ArgumentParser(description='USAGE :   python deveset_generator.py --path . --oversampling yes --fparam 4 ')
	parser.add_argument('-p','--path',help='path to train data folder',required=True)
	parser.add_argument('-o','--oversampling',help='if oversampling say yes else no',required=True)
	parser.add_argument('-f','--fparam',help='f parameter')
	args= vars(parser.parse_args())

	print get_usernames(args['path'])




	<commit_msg>new arg for get usernames and the script<commit_after>import os 
import argparse

"""
        Generate dev set in such a way that 

   		1) if user is picked for train then all chunks of the user fall into train
   		2) if in oversampled mode appropriate over sampled input for the correct users should be in train set
   		3) test set should not be in oversampling 
   		4) maintain ratio of positive - negative users

"""
join = lambda x,y: os.path.join(x,y)
isdir = lambda x: os.path.isdir(x)
isfile = lambda y: os.path.isfile(y)


def get_usernames(path,sign):
	# get user names from <path>/<positive folder>/<chunk1-2>

	subdirs = [ join(path,subdir) for subdir in os.listdir(path) if isdir(join(path,subdir))]
	required_dir = [ subdir for subdir in subdirs if sign in subdir ] [0]
	target_folder = required_dir+'/'+'chunk1-2'

	if not isdir(target_folder):
		print sign+'users subfolder does not have chunk1-2 !!'
		exit(1)

	return [ u.split('.')[0] for u in os.listdir(target_folder) if isfile(join(target_folder,u)) ]

if __name__ == '__main__':

	parser = argparse.ArgumentParser(description='USAGE :   python deveset_generator.py --path . --oversampling yes --fparam 4 ')
	parser.add_argument('-p','--path',help='path to train data folder',required=True)
	parser.add_argument('-sp','--splitpercentage',help='size of the devset in percentage',required=True)
	parser.add_argument('-o','--oversampling',help='if oversampling say yes else no',required=True)
	parser.add_argument('-f','--fparam',help='f parameter')
	args= vars(parser.parse_args())

	print get_usernames(args['path'],'positive')

	print get_usernames(args['path'],'negative')




	
{'fraction_non_alphanumeric': 0.12858966177409062, 'fraction_numerical': 0.008934269304403318, 'mean_word_length': 4.433275563258232, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2441728', 'n_tokens_mistral': 1091, 'n_tokens_neox': 1042, 'n_words': 317}