<commit_before>---
layout: default
title: WordCount Python
category: MapReduce
type: MAPREDUCE
---

### Mapper


`#!/usr/bin/python
import sys
def mapper():
    for line in sys.stdin:
        data = line.strip().split(" ")
        words = data
        for word in words:	
            print "{0}\t{1}".format(word,"1")
mapper()
`

### Reducer

`#!/usr/bin/python
import sys
wordCount = {}
def reducer():    
    for line in sys.stdin:        
        line = line.strip()
        word, count = line.split("\t",1)
        try:
            count = int(count)
        except ValueError:
            continue
        try:
            wordCount[word] = wordCount[word] + count
        except:
            wordCount[word] =count 
    for word in wordCount.keys():
        print "{0}\t{1}".format(word,wordCount[word])
reducer()
`



<commit_msg>addin explanation to wordcount python<commit_after>---
layout: default
title: WordCount Python
category: MapReduce
type: MAPREDUCE
---

These snippets of Python code are an example
of how a MapReduce job will be developed in Python.
The code demonstrates the wordcount application of 
hadoop. It is possible to feed in a text file to the 
system using this mapper and reducer and the output 
will be the count of the words in the text file.

### Mapper


`#!/usr/bin/python
import sys
def mapper():
    for line in sys.stdin:
        data = line.strip().split(" ")
        words = data
        for word in words:	
            print "{0}\t{1}".format(word,"1")
mapper()
`

### Reducer

`#!/usr/bin/python
import sys
wordCount = {}
def reducer():    
    for line in sys.stdin:        
        line = line.strip()
        word, count = line.split("\t",1)
        try:
            count = int(count)
        except ValueError:
            continue
        try:
            wordCount[word] = wordCount[word] + count
        except:
            wordCount[word] =count 
    for word in wordCount.keys():
        print "{0}\t{1}".format(word,wordCount[word])
reducer()
`




{'fraction_non_alphanumeric': 0.12125748502994012, 'fraction_numerical': 0.005988023952095809, 'mean_word_length': 2.1875993640699525, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5620173', 'n_tokens_mistral': 633, 'n_tokens_neox': 615, 'n_words': 195}