<jupyter_start><jupyter_text>Basic Word Count<jupyter_code>%%writefile wordcount.py

#!/usr/bin/python3
from mrjob.job import MRJob

class MyJob(MRJob):

    def mapper(self, _, line):
        line = line.strip()
        words = line.split()
        for word in words:
            yield word, 1

    def reducer(self, word, values):
        yield word, sum(values)
        
if __name__ == '__main__':
    MyJob.run()
!python wordcount.py /data/dataset/text/small.txt<jupyter_output>No configs found; falling back on auto-configuration
No configs specified for inline runner
Creating temp directory /tmp/wordcount.hadoop.20220308.101317.576209
Running step 1 of 1...
job output is in /tmp/wordcount.hadoop.20220308.101317.576209/output
Streaming final output from /tmp/wordcount.hadoop.20220308.101317.576209/output...
"oder"	1
"schnelllebig"	1
"manuellen"	1
"Datenverarbeitung"	1
"und"	2
"um"	1
"umfassenden"	1
"beispielsweise"	1
"gross,"	1
"herk\u00f6mmlichen"	1
"Data"	1
"Datafizierung"	1
"steht"	1
"strukturiert"	1
"aus"	1
"auszuwerten."	1
"sind,"	1
"stammende"	1
"Prozess"	1
"Sprachraum"	1
"Der"	1
"Methoden"	1
"mit"	2
"bezeichnet"	1
"welche"	1
"zu"	4
"dem"	2
"Begriff"	1
"Big"	1
"Datenmengen,"	1
"Zusammenhang"	1
"der"	2
"in"	1
"komplex,"	1
"engem"	1
"englischen"	1
"schwach"	1
"sie"	1
Removing temp directory /tmp/wordcount.hadoop.20220308.101317.576209...<jupyter_text>Optional - Word Count with Combiners<jupyter_code>%%writefile wordcountCombiner.py

#!/usr/bin/python3
from mrjob.job import MRJob

class MyJobCombiner(MRJob):

    def mapper(self, _, line):
        line = line.strip()
        words = line.split()
        for word in words:
            yield word, 1
        
    def combiner(self, word, counts):
        yield word, sum(counts)

    def reducer(self, word, counts):
        yield word, sum(counts)

if __name__ == '__main__':
    MyJobCombiner.run()
!python wordcountCombiner.py /data/dataset/text/small.txt<jupyter_output>No configs found; falling back on auto-configuration
No configs specified for inline runner
Creating temp directory /tmp/wordcountCombiner.hadoop.20220308.101321.111933
Running step 1 of 1...
job output is in /tmp/wordcountCombiner.hadoop.20220308.101321.111933/output
Streaming final output from /tmp/wordcountCombiner.hadoop.20220308.101321.111933/output...
"manuellen"	1
"herk\u00f6mmlichen"	1
"Datenmengen,"	1
"stammende"	1
"sie"	1
"sind,"	1
"Zusammenhang"	1
"engem"	1
"englischen"	1
"Big"	1
"Data"	1
"schwach"	1
"Sprachraum"	1
"schnelllebig"	1
"Der"	1
"Methoden"	1
"Datenverarbeitung"	1
"in"	1
"komplex,"	1
"aus"	1
"auszuwerten."	1
"um"	1
"umfassenden"	1
"und"	2
"welche"	1
"steht"	1
"strukturiert"	1
"beispielsweise"	1
"Begriff"	1
"Datafizierung"	1
"zu"	4
"Prozess"	1
"bezeichnet"	1
"gross,"	1
"dem"	2
"der"	2
"mit"	2
"oder"	1
Removing temp directory /tmp/wordcountCombiner.hadoop.20220308.101321.111933...
{'fraction_non_alphanumeric': 0.12604748603351956, 'fraction_numerical': 0.08729050279329609, 'mean_word_length': 4.0, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '562410', 'license': 'MIT', 'n_tokens_mistral': 1317, 'n_tokens_neox': 1212, 'n_words': 308, 'path': 'V3/v3_exercises_material/solutions/2_WordCount/1_WordCount_Solution.ipynb', 'repo_name': 'fred1234/BDLC_FS22', 'size': '2864'}