<commit_before>#!/usr/bin/env python

import sys

# -------------------------------------------------------------------------
# This mapper code will input a <date word, value> input file, and move date into 
# the value field for output
#
# Note, this program is wittern in a simple style and does not full advantage of Python
# data structures, but I believe it is more readable.
# 
# Note there is NO error checking of the input, it is assume to be correct
# meaning no extra spaces, missing inputs or counts, etc..
#
# See https://docs.python.org/2/tutorial/index.html for details  and python  tutorials
# -------------------------------------------------------------------------

for line in sys.stdin:
	line = line.strip() # strip out carriage return
	key_value = line.spilit(",") # split line, into key and value, returns list
	key_in = key_value[0].split(" ") # key is first item in list
	value_in = key_value[1] # value is 2nd item

	# print key_in
	if len(key_in) >= 2: # if this entry has <date word> in key
		date = key[0] # now get date from key field
		word = key_in[1] 
		value_out = date + " " + value_in # concatenate date, blank, and value_in

		print( '%s\t%s' % (word, value_out) )  #print a string, tab, and string
	else: # key is only <word> so just pass it through
		print( '%s\t%s' % (key_in[0], value_in) )  #print a string tab and string

# Note that Hadoop expects a tab to separate key value
# but this program assumes the input file has a ',' separating key value<commit_msg>Change spilit to be spilit.<commit_after>#!/usr/bin/env python

import sys

# -------------------------------------------------------------------------
# This mapper code will input a <date word, value> input file, and move date into 
# the value field for output
#
# Note, this program is wittern in a simple style and does not full advantage of Python
# data structures, but I believe it is more readable.
# 
# Note there is NO error checking of the input, it is assume to be correct
# meaning no extra spaces, missing inputs or counts, etc..
#
# See https://docs.python.org/2/tutorial/index.html for details  and python  tutorials
# -------------------------------------------------------------------------

for line in sys.stdin:
	line = line.strip() # strip out carriage return
	key_value = line.split(",") # split line, into key and value, returns list
	key_in = key_value[0].split(" ") # key is first item in list
	value_in = key_value[1] # value is 2nd item

	# print key_in
	if len(key_in) >= 2: # if this entry has <date word> in key
		date = key[0] # now get date from key field
		word = key_in[1] 
		value_out = date + " " + value_in # concatenate date, blank, and value_in

		print( '%s\t%s' % (word, value_out) )  #print a string, tab, and string
	else: # key is only <word> so just pass it through
		print( '%s\t%s' % (key_in[0], value_in) )  #print a string tab and string

# Note that Hadoop expects a tab to separate key value
# but this program assumes the input file has a ',' separating key value
{'fraction_non_alphanumeric': 0.18437603442568687, 'fraction_numerical': 0.005296259516716319, 'mean_word_length': 2.449771689497717, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4981480', 'n_tokens_mistral': 903, 'n_tokens_neox': 869, 'n_words': 397}