<commit_before>/*
 * Copyright 2013 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.springframework.xd.examples.hadoop;

import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang.StringEscapeUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

public class HashtagCount {

	public static class TokenizerMapper extends
			Mapper<Object, Text, Text, LongWritable> {

		final static Pattern TAG_PATTERN = Pattern.compile("\"hashtags\":\\[([^\\]]*)");
		private final static LongWritable ONE = new LongWritable(1L);
		private Text word = new Text();

		public void map(Object key, Text value, Context context)
				throws IOException, InterruptedException {
			Matcher matcher = TAG_PATTERN.matcher(value.toString());
			while (matcher.find()) {
				String found = matcher.group();
				String cleanedString = found.replaceFirst("\"hashtags\":\\[\\{\"text\":\"", "");
				String superPolished = cleanedString.split("\",\"")[0];

				String useMe = superPolished;
				if (superPolished.startsWith("\\u")) {
					useMe = StringEscapeUtils.unescapeJava(superPolished);
				}
				useMe = useMe.split("\"")[0];

				word.set(useMe.toLowerCase());
				context.write(word, ONE);
			}
		}

	}

	public static class LongSumReducer extends
			Reducer<Text, LongWritable, Text, LongWritable> {<commit_msg>Change hashtag mapper to use JSON ObjectMapper<commit_after>/*
 * Copyright 2013 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.springframework.xd.examples.hadoop;

import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;

public class HashtagCount {

	public static class TokenizerMapper extends
			Mapper<Object, Text, Text, LongWritable> {

		private static final ObjectMapper jsonMapper = new ObjectMapper(new JsonFactory());
		private final static LongWritable ONE = new LongWritable(1L);

		public void map(Object key, Text value, Context context)
				throws IOException, InterruptedException {

			Map<String, Object> tweet = jsonMapper.readValue(value.toString(),
					new TypeReference<HashMap<String, Object>>() {
					});
			Map<String, Object> entities = (Map<String, Object>) tweet.get("entities");
			List<Map<String, Object>> hashTagEntries = null;

			if (entities != null) {
				hashTagEntries = (List<Map<String, Object>>) entities.get("hashtags");
			}

			if (hashTagEntries != null && hashTagEntries.size() > 0) {
				for (Map<String, Object> hashTagEntry : hashTagEntries) {
					String hashTag = hashTagEntry.get("text").toString();
					context.write(new Text(hashTag), ONE);
				}
			}
		}

	}

	public static class LongSumReducer extends
			Reducer<Text, LongWritable, Text, LongWritable> {
{'fraction_non_alphanumeric': 0.11090458488228006, 'fraction_numerical': 0.004337050805452293, 'mean_word_length': 4.478506787330317, 'pattern_counts': {'":': 3, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2502232', 'n_tokens_mistral': 1510, 'n_tokens_neox': 1375, 'n_words': 406}