<jupyter_start><jupyter_text>Python Cleaning Up Text Files<jupyter_code>with open('data\\Veidenbaums.txt') as fin:
    mylines = fin.readlines()
# MAC ONLY and LINUX
with open('data/Veidenbaums.txt', encoding='utf-8') as fin:
    mylines = fin.readlines()
# Windows
with open('data/Veidenbaums.txt', encoding='utf-8') as fin:
    mylines = fin.readlines()
len(mylines)
# Universal Operating System module library
import os
fname = "Veidenbaums.txt"
fpath = os.path.join(os.getcwd(), 'data', fname)
fpath
import os
newpath = os.path.join(os.getcwd(), 'data', 'cleaned.txt')
newpath
badwords = ['***']
mylines[:5]
cleanlines = []
for line in mylines:
    if '***' not in line:
        cleanlines.append(line)
len(cleanlines)
cleanlines2 = [line for line in mylines if '***' not in line]
len(cleanlines2)
badwords = ['***', "reallybadword"]
# universal way to clean lines of any bad words
with open(fpath, encoding='utf-8') as oldfile, open(newpath, 'w', encoding='utf-8') as newfile:
    for line in oldfile:
        if not any(badword in line for badword in badwords):
            newfile.write(line)

newpath
with open(newpath, encoding='utf-8') as f:
    txt=f.read()
len(txt)
txt[:100],txt[-100:]<jupyter_output><empty_output><jupyter_text>Exercise 1. Saskaitīt tekstā sastopamo vārdu biežumu2. Izdrukāt unikālo vārdu skaitu3. Izdrukāt 20. biežāk sastopamo vārdu sarakstu4. Kādi ir tekstā visretāk sastopamie vārdi?<jupyter_code># using string replace function to clean text of bad data
badchars = ',;:!?."\'-'
for char in badchars:
    txt = txt.replace(char, '') # suggestion to use regular expressions for heavier tasks
len(txt)
txt[:100]
# <NAME>'s extra whitespace cleaner
# ' '.join(mystring.split())
# without arguments will use common separators https://docs.python.org/3/library/stdtypes.html
words = txt.split()
len(words)
mydict = {}
for word in words:
    word = word.lower()
    if word in mydict.keys():
        mydict[word] += 1
    else:
        mydict[word] = 1
len(mydict)
mycount = list(mydict.items())
type(mycount)
mycount[0]
type(mycount[0])
cwords = [word.title() for word in words]
len(cwords)
vdict = { word: cwords.count(word) for word in set(cwords)}
len(vdict)
list(vdict.keys())[:5]
max(vdict, key = lambda k: vdict[k])
vdict['Un']
sortedWords = sorted(vdict.items(), key = lambda x: x[1])
len(sortedWords)
type(vdict.items())
sortedWords.reverse()
sortedWords[:10]
sortedWords[:50]
# we can count with Counter no need to make our own dictionary!
from collections import Counter
cnt = Counter(cwords)
type(cnt)
cnt.most_common(50)
cnt.most_common(10)
# Reading files from URL
# https://docs.python.org/3.7/library/urllib.request.html#module-urllib.request<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14248895434462444, 'fraction_numerical': 0.014727540500736377, 'mean_word_length': 4.855603448275862, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '262186', 'license': 'MIT', 'n_tokens_mistral': 1025, 'n_tokens_neox': 987, 'n_words': 264, 'path': '.ipynb_checkpoints/Python Cleaning Up Text Files-checkpoint.ipynb', 'repo_name': 'ValRCS/RCS_Data_Analysis_Python_2019', 'size': '2723'}