<commit_before>from copy import deepcopy
import re


def clean_nodes_no_names(tag, data):
	if not isinstance(tag, tuple):
		for each in data:
			if each['k'] != [] and each['v'] != []:
				if tag in each['k'] and 'name' not in each['k']:
					each['removed'] = 'true'
					tagValueData = dict(zip(each['k'], each['v']))
					if tagValueData.get('amenity') == 'atm':
						each['removed'] = 'false'
			yield each
	else:
		for each in data:
			if each['k'] != [] and each['v'] != []:
				if tag[0] in each['k'] and tag[1] in each['v'] and 'name' not in each['k']:
					each['removed'] = 'true'
			yield each


def expand_country_name(tag, name, data):
	for each in data:
		if tag in each['k'] and each['v'][each['k'].index(tag)] == 'IN':
			each['v'][each['k'].index(tag)] = name
		yield each


def detect_area_from_way(data):
	for each in data:
		if int(each['refs'][0]) == int(each['refs'][-1]):
			each['type'] = 'area'
			print(each)
		yield each


def fixAddress(data):
	for each in data:
		tempK = list()
		for tag in each['k']:
			if tag.split(':')[0] == 'addr':
				tempK.append(tag.split(':')[1])
			else:
				tempK.append(tag)
			each['k'] = deepcopy(tempK)
		yield each


def removeHindiNames(data):
	hi = r"\w+:hi$"
	for each in data:<commit_msg>Squashed the AREA marking bug<commit_after>from copy import deepcopy
import re


def clean_nodes_no_names(tag, data):
	if not isinstance(tag, tuple):
		for each in data:
			if each['k'] != [] and each['v'] != []:
				if tag in each['k'] and 'name' not in each['k']:
					each['removed'] = 'true'
					tagValueData = dict(zip(each['k'], each['v']))
					if tagValueData.get('amenity') == 'atm':
						each['removed'] = 'false'
			yield each
	else:
		for each in data:
			if each['k'] != [] and each['v'] != []:
				if tag[0] in each['k'] and tag[1] in each['v'] and 'name' not in each['k']:
					each['removed'] = 'true'
			yield each


def expand_country_name(tag, name, data):
	for each in data:
		if tag in each['k'] and each['v'][each['k'].index(tag)] == 'IN':
			each['v'][each['k'].index(tag)] = name
		yield each


def detect_area_from_way(data):
	for each in data:
		if int(each['refs'][0]) == int(each['refs'][-1]):
			each['type'] = 'area'
		yield each


def fixAddress(data):
	for each in data:
		tempK = list()
		for tag in each['k']:
			if tag.split(':')[0] == 'addr':
				tempK.append(tag.split(':')[1])
			else:
				tempK.append(tag)
			each['k'] = deepcopy(tempK)
		yield each


def removeHindiNames(data):
	hi = r"\w+:hi$"
	for each in data:
{'fraction_non_alphanumeric': 0.20294468762435336, 'fraction_numerical': 0.00477516912057302, 'mean_word_length': 3.6990654205607476, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '482482', 'n_tokens_mistral': 1093, 'n_tokens_neox': 954, 'n_words': 264}