<commit_before>part of coUserver;

Map getStreetEntities(String tsid)
{
	if(tsid == null)
		return null;
	
	if(tsid.startsWith("G"))
		tsid = tsid.replaceFirst("G", "L");
	Map entities = null;
	File file = new File('./streetEntities/$tsid');
	if(file.existsSync())
		entities = JSON.decode(file.readAsStringSync());
	
	return entities;
}
saveStreetData(Map params)
{
	String tsid = params['tsid'];
	if(tsid.startsWith("G"))
		tsid = tsid.replaceFirst("G", "L");
	
	List entities = JSON.decode(params['entities']);
	File file = new File('./streetEntities/$tsid');
	if(file.existsSync())
	{
		Map oldFile = JSON.decode(file.readAsStringSync());
		//backup the older file and replace it with this new file
    	File backup = new File('./streetEntities/$tsid.bak');
    	if(backup.existsSync())
    	{
    		Map oldData = JSON.decode(backup.readAsStringSync());
    		List backups = oldData['backups'];
    		backups.add({new DateTime.now().toIso8601String():oldFile});
    		backup.writeAsStringSync(JSON.encode({'backups':backups}));
    	}
    	else
    	{
    		backup.createSync(recursive:true);
	    	Map oldData = {'backups':[{new DateTime.now().toIso8601String():oldFile}]};
	    	backup.writeAsStringSync(JSON.encode(oldData));
    	}
    }
	else
		file.createSync(recursive:true);
	
	file.writeAsStringSync(JSON.encode({'entities':entities}));
	
	
	//save a list of finished and partially finished streets
	File finished = new File('./streetEntities/finished.json');
	if(!finished.existsSync())
		_createFinishedFile();
	Map finishedMap = JSON.decode(finished.readAsStringSync());
	int required = params['required'];
	int complete = params['complete'];
	bool streetFinished = (required-complete == 0) ? true : false;
	finishedMap[tsid] = {"entitiesRequired":params['required'],
	                     "entitiesComplete":params['complete'],
	                     "streetFinished":streetFinished};
	finished.writeAsStringSync(JSON.encode(finishedMap));
}

void _createFinishedFile()
{
	File finished = new File('./streetEntities/finished.json');
	finished.createSync(recursive:true);
	//insert any streets that were finished before this file was created
	Directory streetEntities = new Directory('./streetEntities');
	Map finishedMap = {};
	for(FileSystemEntity entity in streetEntities.listSync(recursive:true))
	{
		String filename = entity.path.substring(entity.path.lastIndexOf('/')+1);
		if(!filename.contains('.'))
		{
			//we'll assume it's incomplete
			finishedMap[filename] = {"entitiesRequired":0,
				                     "entitiesComplete":0,
            	                     "streetFinished":false};
		}
	}
	finished.writeAsStringSync(JSON.encode(finishedMap));
}<commit_msg>accept as strings instead of ints so that the client doesn't yell<commit_after>part of coUserver;

Map getStreetEntities(String tsid)
{
	if(tsid == null)
		return null;
	
	if(tsid.startsWith("G"))
		tsid = tsid.replaceFirst("G", "L");
	Map entities = null;
	File file = new File('./streetEntities/$tsid');
	if(file.existsSync())
		entities = JSON.decode(file.readAsStringSync());
	
	return entities;
}
saveStreetData(Map params)
{
	String tsid = params['tsid'];
	if(tsid.startsWith("G"))
		tsid = tsid.replaceFirst("G", "L");
	
	List entities = JSON.decode(params['entities']);
	File file = new File('./streetEntities/$tsid');
	if(file.existsSync())
	{
		Map oldFile = JSON.decode(file.readAsStringSync());
		//backup the older file and replace it with this new file
    	File backup = new File('./streetEntities/$tsid.bak');
    	if(backup.existsSync())
    	{
    		Map oldData = JSON.decode(backup.readAsStringSync());
    		List backups = oldData['backups'];
    		backups.add({new DateTime.now().toIso8601String():oldFile});
    		backup.writeAsStringSync(JSON.encode({'backups':backups}));
    	}
    	else
    	{
    		backup.createSync(recursive:true);
	    	Map oldData = {'backups':[{new DateTime.now().toIso8601String():oldFile}]};
	    	backup.writeAsStringSync(JSON.encode(oldData));
    	}
    }
	else
		file.createSync(recursive:true);
	
	file.writeAsStringSync(JSON.encode({'entities':entities}));
	
	
	//save a list of finished and partially finished streets
	File finished = new File('./streetEntities/finished.json');
	if(!finished.existsSync())
		_createFinishedFile();
	Map finishedMap = JSON.decode(finished.readAsStringSync());
	int required = int.parse(params['required']);
	int complete = int.parse(params['complete']);
	bool streetFinished = (required-complete == 0) ? true : false;
	finishedMap[tsid] = {"entitiesRequired":params['required'],
	                     "entitiesComplete":params['complete'],
	                     "streetFinished":streetFinished};
	finished.writeAsStringSync(JSON.encode(finishedMap));
}

void _createFinishedFile()
{
	File finished = new File('./streetEntities/finished.json');
	finished.createSync(recursive:true);
	//insert any streets that were finished before this file was created
	Directory streetEntities = new Directory('./streetEntities');
	Map finishedMap = {};
	for(FileSystemEntity entity in streetEntities.listSync(recursive:true))
	{
		String filename = entity.path.substring(entity.path.lastIndexOf('/')+1);
		if(!filename.contains('.'))
		{
			//we'll assume it's incomplete
			finishedMap[filename] = {"entitiesRequired":0,
				                     "entitiesComplete":0,
            	                     "streetFinished":false};
		}
	}
	finished.writeAsStringSync(JSON.encode(finishedMap));
}
{'fraction_non_alphanumeric': 0.14816849816849817, 'fraction_numerical': 0.004395604395604396, 'mean_word_length': 4.056481481481481, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2160270', 'n_tokens_mistral': 1786, 'n_tokens_neox': 1688, 'n_words': 325}