<commit_before>package org.hackreduce.models;

import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.logging.Logger;

import org.apache.hadoop.io.Text;

/**
 * Record for a topic from the Freebase topic dump.
 * Each line in the file represents a single topic as described here:
 * http://wiki.freebase.com/wiki/Data_dumps#Simple_Topic_Dump
 * <p>
 * The dumps themselves are available from: <ul>
 * <li>http://download.freebase.com/datadumps/latest/freebase-simple-topic-dump.tsv.bz2 (updated weekly)</li>
 * <li>http://aws.amazon.com/datasets/8247878934976180 (updated occasionally)</li>
 * 
 * The simple topic dump is 1.4 GB compressed and 5.4 GB uncompressed (November 2012).
 * 
 * Note that unless you are using Hadoop 0.23.x or 2.x the bzip2 codec isn't
 * splittable, so you'll need to decompression the dump before processing.
 * 
 * @author <NAME> <<EMAIL>>
 * 
 */
public class FreebaseTopicRecord {

	Logger LOG = Logger.getLogger(FreebaseTopicRecord.class.getName());
	
	private String mid;
	private String name;
	private String[] ids;
	private String[] wp_ids;
	private String[] fb_types;
	private String blurb;
	

	/**
	 * Construct a FreebaseTopicRecord object from a single line of text
	 * as read from the simple topic dump.
	 * 
	 * @param text the line of text 
	 * @throws IllegalArgumentException
	 */
	public FreebaseTopicRecord(Text text) throws IllegalArgumentException {
		this(text.toString());
	}
	
	/**
	 * Construct a FreebaseTopicRecord object from a single line of text
	 * as read from the simple topic dump.
	 * 
	 * @param string the line of text 
	 * @throws IllegalArgumentException
	 */
	public FreebaseTopicRecord(String string) throws IllegalArgumentException {
		String[] pieces = string.split("\t");
		mid = pieces[0];
		name = unescape(pieces[1]);
		ids = split(pieces[2]);
		wp_ids = split(pieces[3]);
		fb_types = split(pieces[4]);
		blurb = unescape(pieces[5]);
	}

	private static String unescape(String s) {
		// \\N is a special signal value representing a null
		if ("\\N".equals(s)) {
			return null;
		}
		// tabs, newlines, and backslashes are the only other escaped characters
		return s.replaceAll("\\t","\t").replaceAll("\\n","\n").replaceAll("\\\\", "\\");
	}

	private static String escape(String s) {
		// \\N is a special signal value representing a null
		if (s == null) {
			return "\\N";
		}
		// Escape tabs, newlines, and backslashes
		return s.replaceAll("\\","\\\\").replaceAll("\t","\\t").replaceAll("\n","\\n");
	}

	// Split a comma separated string of ids
	private static String[] split(String s) {
		if ("\\N".equals(s)) {
			return new String[0];
		}
		return s.split(",");
	}


	/**
	 * @return the primary id (MID) of the Freebase topic which this<commit_msg>Correct minor doc typo<commit_after>package org.hackreduce.models;

import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.logging.Logger;

import org.apache.hadoop.io.Text;

/**
 * Record for a topic from the Freebase topic dump.
 * Each line in the file represents a single topic as described here:
 * http://wiki.freebase.com/wiki/Data_dumps#Simple_Topic_Dump
 * <p>
 * The dumps themselves are available from: <ul>
 * <li>http://download.freebase.com/datadumps/latest/freebase-simple-topic-dump.tsv.bz2 (updated weekly)</li>
 * <li>http://aws.amazon.com/datasets/8247878934976180 (updated occasionally)</li>
 * 
 * The simple topic dump is 1.4 GB compressed and 5.4 GB uncompressed (November 2012).
 * 
 * Note that unless you are using Hadoop 0.23.x or 2.x the bzip2 codec isn't
 * splittable, so you'll need to decompression the dump before processing.
 * 
 * @author <NAME> <<EMAIL>>
 * 
 */
public class FreebaseTopicRecord {

	Logger LOG = Logger.getLogger(FreebaseTopicRecord.class.getName());
	
	private String mid;
	private String name;
	private String[] ids;
	private String[] wp_ids;
	private String[] fb_types;
	private String blurb;
	

	/**
	 * Construct a FreebaseTopicRecord object from a single line of text
	 * as read from the simple topic dump.
	 * 
	 * @param text the line of text 
	 * @throws IllegalArgumentException
	 */
	public FreebaseTopicRecord(Text text) throws IllegalArgumentException {
		this(text.toString());
	}
	
	/**
	 * Construct a FreebaseTopicRecord object from a single line of text
	 * as read from the simple topic dump.
	 * 
	 * @param string the line of text 
	 * @throws IllegalArgumentException
	 */
	public FreebaseTopicRecord(String string) throws IllegalArgumentException {
		String[] pieces = string.split("\t");
		mid = pieces[0];
		name = unescape(pieces[1]);
		ids = split(pieces[2]);
		wp_ids = split(pieces[3]);
		fb_types = split(pieces[4]);
		blurb = unescape(pieces[5]);
	}

	private static String unescape(String s) {
		// \N is a special signal value representing a null
		if ("\\N".equals(s)) {
			return null;
		}
		// tabs, newlines, and backslashes are the only other escaped characters
		return s.replaceAll("\\t","\t").replaceAll("\\n","\n").replaceAll("\\\\", "\\");
	}

	private static String escape(String s) {
		// \N is a special signal value representing a null
		if (s == null) {
			return "\\N";
		}
		// Escape tabs, newlines, and backslashes
		return s.replaceAll("\\","\\\\").replaceAll("\t","\\t").replaceAll("\n","\\n");
	}

	// Split a comma separated string of ids
	private static String[] split(String s) {
		if ("\\N".equals(s)) {
			return new String[0];
		}
		return s.split(",");
	}


	/**
	 * @return the primary id (MID) of the Freebase topic which this
{'fraction_non_alphanumeric': 0.13465346534653466, 'fraction_numerical': 0.01332133213321332, 'mean_word_length': 3.66890756302521, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7148950', 'n_tokens_mistral': 1890, 'n_tokens_neox': 1692, 'n_words': 552}