<commit_before>import codecs
import os

def test(cursor):
	cursor.execute("""SELECT `Reporting Entity_Original Name`,r.name,r.RICname,r2.RICname
	 from flow 
	 LEFT OUTER JOIN entity_names_cleaning as r ON `Reporting Entity_Original Name`=r.original_name COLLATE NOCASE
	 LEFT OUTER JOIN RICentities r2 ON r2.RICname=r.RICname
	 WHERE r2.RICname is null
	 GROUP BY `Reporting Entity_Original Name`""")
	missing_reportings_in_RICnames= list(cursor)
	print "%s missing reporting in RICnames"%len(missing_reportings_in_RICnames)
	with codecs.open(os.path.join("..","out_data","missings_reporting_in_ricnames.csv"),"w",encoding="UTF8") as f:
		f.write('"original partner name","modified name","RICname in table2","RICname in table3"\n')
		for r in missing_reportings_in_RICnames:
			f.write('"'+'","'.join(unicode(_) for _ in r)+'"\n')
	

	cursor.execute("""SELECT `Partner Entity_Original Name`,p.name,p.RICname,p2.RICname
	 from flow 
	 LEFT OUTER JOIN entity_names_cleaning as p ON trim(`Partner Entity_Original Name`)=p.original_name COLLATE NOCASE
	 LEFT OUTER JOIN RICentities p2 ON p2.RICname=p.RICname
	 WHERE p2.RICname is null
	 GROUP BY `Partner Entity_Original Name`""")
	missing_partners_in_RICnames= list(cursor)
	print "%s missing partners in RICnames"%len(missing_partners_in_RICnames)
	with codecs.open(os.path.join("..","out_data","missings_partners_in_ricnames.csv"),"w",encoding="UTF8") as f:
		f.write('"original partner name","modified name","RICname in table2","RICname in table3"\n')
		for r in missing_partners_in_RICnames:
			f.write('"'+'","'.join(unicode(_) for _ in r)+'"\n')
		print "missings written in out_data"

	return True<commit_msg>update test on RICname<commit_after>import codecs
import os

def test(cursor):
	cursor.execute("""SELECT reporting,r.RICname,r2.RICname
	 from flows
	 LEFT OUTER JOIN entity_names as r ON reporting=r.original_name COLLATE NOCASE
	 LEFT OUTER JOIN RICentities r2 ON r2.RICname=r.RICname
	 WHERE r2.RICname is null
	 GROUP BY reporting""")
	missing_reportings_in_RICnames= list(cursor)
	print "%s missing reporting in RICnames"%len(missing_reportings_in_RICnames)
	with codecs.open(os.path.join("..","out_data","missings_reporting_in_ricnames.csv"),"w",encoding="UTF8") as f:
		f.write('"original partner name","modified name","RICname in table2","RICname in table3"\n')
		for r in missing_reportings_in_RICnames:
			f.write('"'+'","'.join(unicode(_) for _ in r)+'"\n')


	cursor.execute("""SELECT partner,p.RICname,p2.RICname
	 from flows
	 LEFT OUTER JOIN entity_names as p ON trim(partner)=p.original_name COLLATE NOCASE
	 LEFT OUTER JOIN RICentities p2 ON p2.RICname=p.RICname
	 WHERE p2.RICname is null
	 GROUP BY partner""")
	missing_partners_in_RICnames= list(cursor)
	print "%s missing partners in RICnames"%len(missing_partners_in_RICnames)
	with codecs.open(os.path.join("..","out_data","missings_partners_in_ricnames.csv"),"w",encoding="UTF8") as f:
		f.write('"original partner name","modified name","RICname in table2","RICname in table3"\n')
		for r in missing_partners_in_RICnames:
			f.write('"'+'","'.join(unicode(_) for _ in r)+'"\n')
		print "missings written in out_data"

	return True
{'fraction_non_alphanumeric': 0.13005050505050506, 'fraction_numerical': 0.008838383838383838, 'mean_word_length': 6.153498871331829, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7229183', 'n_tokens_mistral': 1189, 'n_tokens_neox': 1120, 'n_words': 264}