<commit_before># coding: utf-8
from __future__ import unicode_literals

swears = """
shit
fuck
shitting
fucking
fucked
shits
fucks
fuckers
pussy
bitch
bitches
asshole
assholes
cunt
cunts
pussies
shitty
gay
"""

swears = set([x for x in swears.splitlines() if len(x)])

bad_swears = """
faggot
faggots
nigger
niggers
"""

bad_swears = set([x for x in bad_swears.splitlines() if len(x)])
swears.update(bad_swears)

swears_fr = """
tapette
pédale
nique
niquer
pute
suceboule
raton
baise
baiser
black
blacks
"""
swears_fr = set([x for x in swears_fr.splitlines() if len(x)])


custom_ipa = """
eg ˈɛɡ
thx θˈæŋks
xx ˈɛksˈɛks
selfies sˈɛlfiːz
dms dˈiːɛmz
"""

custom_ipa = [tuple(x.split()) for x in custom_ipa.splitlines() if len(x)]



ukraine = ['show',
 'demand',
 'sbseurovision',
 'azerbaijan',
 'separatist',
 'hungary',
 'trouble',
 'giant',
 'pray',
 'point',
 'reuters',
 'pretty',<commit_msg>added more french swears<commit_after># coding: utf-8
from __future__ import unicode_literals

swears = """
shit
fuck
shitting
fucking
fucked
shits
fucks
fuckers
pussy
bitch
bitches
asshole
assholes
cunt
cunts
pussies
shitty
gay
"""

swears = set([x for x in swears.splitlines() if len(x)])

bad_swears = """
faggot
faggots
nigger
niggers
"""

bad_swears = set([x for x in bad_swears.splitlines() if len(x)])
swears.update(bad_swears)

swears_fr = """
tapette
pédale
nique
niquer
pute
suceboule
raton
baise
baiser
black
blacks
bite
bites
putes
pute
niké
nické
niker
nicker
chienne
chiennes
"""

swears_fr = set([x for x in swears_fr.splitlines() if len(x)])


custom_ipa = """
eg ˈɛɡ
thx θˈæŋks
xx ˈɛksˈɛks
selfies sˈɛlfiːz
dms dˈiːɛmz
"""

custom_ipa = [tuple(x.split()) for x in custom_ipa.splitlines() if len(x)]



ukraine = ['show',
 'demand',
 'sbseurovision',
 'azerbaijan',
 'separatist',
 'hungary',
 'trouble',
 'giant',
 'pray',
 'point',
 'reuters',
 'pretty',
{'fraction_non_alphanumeric': 0.12627073301230604, 'fraction_numerical': 0.001070090957731407, 'mean_word_length': 5.131147540983607, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4004886', 'n_tokens_mistral': 841, 'n_tokens_neox': 848, 'n_words': 212}