<jupyter_start><jupyter_text>![alt text](https://4.bp.blogspot.com/-gbL5nZDkpFQ/XScFYwoTEII/AAAAAAAAAGY/CcVb_HDLwvs2Brv5T4vSsUcz7O4r2Q79ACK4BGAYYCw/s1600/kk3-header00-beta.png)[<NAME>](http://www.linkedin.com/in/prithwis)  Map Reduce ConceptHere we demonstrate the principles of Map-Reduce WITHOUT the need for any complex software like Hadoop or Spark  Create a map.py and red(uce).py file<jupyter_code>%%writefile map.py
#!/usr/bin/env python

import sys

# input comes from STDIN (standard input)
for line in sys.stdin:
    # remove leading and trailing whitespace
    line = line.strip()
    # split the line into words
    words = line.split()
    # increase counters
    for word in words:
        # write the results to STDOUT (standard output);
        # what we output here will be the input for the
        # Reduce step, i.e. the input for reducer.py
        #
        # tab-delimited; the trivial word count is 1
        print (word, 1)
%%writefile red.py
#!/usr/bin/env python

from operator import itemgetter
import sys

current_word = None
current_count = 0
word = None

# input comes from STDIN
for line in sys.stdin:
    # remove leading and trailing whitespace
    line = line.strip()
    

    # parse the input we got from mapper.py
    word, count = line.split(' ')
    # convert count (currently a string) to int
    count = int(count)

    # this IF-switch only works because Hadoop sorts map output
    # by key (here: word) before it is passed to the reducer
    if current_word == word:
        current_count += count
    else:
        if current_word:
            # write result to STDOUT
            print (current_word, current_count)
        current_count = count
        current_word = word

# do not forget to output the last word if needed!
if current_word == word:
    print (current_word, current_count)
!ls -al
# Removing carriage returns ... not required anymore
#!sed 's/\r$//' map.py > map1.py 
#!sed 's/\r$//' red.py > red1.py
# make files executable
!chmod u+rwx /content/map.py
!chmod u+rwx /content/red.py
!ls -al<jupyter_output>total 24
drwxr-xr-x 1 root root 4096 Aug 26 06:07 .
drwxr-xr-x 1 root root 4096 Aug 26 06:06 ..
drwxr-xr-x 4 root root 4096 Aug 13 13:34 .config
-rwxr--r-- 1 root root  525 Aug 26 06:07 map.py
-rwxr--r-- 1 root root  870 Aug 26 06:07 red.py
drwxr-xr-x 1 root root 4096 Aug 13 13:35 sample_data<jupyter_text>3 steps > MAP > SORT > REDUCE<jupyter_code># Step 0 > just PRINT the input
!echo "the king beneath the mountain the king of carven stone"
# Step 1 > MAP the data
!echo "the king beneath the mountain the king of carven stone" | ./map.py
# Step 2 > SORT the MAP output
!echo "the king beneath the mountain the king of carven stone" | ./map.py | sort
# Step 3 > REDUCE the SORT output
!echo "the king beneath the mountain the king of carven stone" | ./map.py | sort | ./red.py<jupyter_output>beneath 1
carven 1
king 2
mountain 1
of 1
stone 1
the 3<jupyter_text>Use an input file<jupyter_code>%%writefile hobbit.txt
The King beneath the mountains
The King of carven stone
The lord of silver fountains
Shall come into his own
His crown shall be upholding
His harp shall be restrung
His halls shall echo golden
To songs of yore re-sung
The woods shall wave on mountains
And grass beneath the sun
His wealth shall flow in fountains
And the rivers golden run
The streams shall run in gladness
The lakes shall shine and burn
All sorrow fail and sadness
At the mountain kings return
The King beneath the mountains
The King of carven stone
The lord of silver fountains
Shall come into his own
His crown shall be upholding
His harp shall be restrung
His halls shall echo golden
To songs of yore re-sung
The woods shall wave on mountains
And grass beneath the sun
His wealth shall flow in fountains
And the rivers golden run
The streams shall run in gladness
The lakes shall shine and burn
All sorrow fail and sadness
At the mountain kings return
The King beneath the mountains
The King of carven stone
The lord of silver fountains
Shall come into his own
His crown shall be upholding
His harp shall be restrung
His halls shall echo golden
To songs of yore re-sung
The woods shall wave on mountains
And grass beneath the sun
His wealth shall flow in fountains
And the rivers golden run
The streams shall run in gladness
The lakes shall shine and burn
All sorrow fail and sadness
At the mountain kings return
All sorrow fail and sadness
At the mountain kings return
#!cat hobbit.txt
!cat hobbit.txt | ./map.py | sort | ./red.py<jupyter_output>All 4
and 7
And 6
At 4
be 6
beneath 6
burn 3
carven 3
come 3
crown 3
echo 3
fail 4
flow 3
fountains 6
gladness 3
golden 6
grass 3
halls 3
harp 3
his 3
His 12
in 6
into 3
King 6
kings 4
lakes 3
lord 3
mountain 4
mountains 6
of 9
on 3
own 3
restrung 3
re-sung 3
return 4
rivers 3
run 6
sadness 4
shall 21
Shall 3
shine 3
silver 3
songs 3
sorrow 4
stone 3
streams 3
sun 3
the 13
The 18
To 3
upholding 3
wave 3
wealth 3
woods 3
yore 3
{'fraction_non_alphanumeric': 0.06014580801944107, 'fraction_numerical': 0.03240178209801539, 'mean_word_length': 3.4216651745747537, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 20, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '369803', 'license': 'MIT', 'n_tokens_mistral': 1767, 'n_tokens_neox': 1594, 'n_words': 754, 'path': 'Basic_WordCount_Concept.ipynb', 'repo_name': 'ShahrukhGouhar/BDSN', 'size': '4949'}