<commit_before>package ch.tkuhn.memetools;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang3.tuple.Pair;
import org.supercsv.io.CsvListReader;
import org.supercsv.io.CsvListWriter;

import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterException;

public class CalculateCommData {

	@Parameter(description = "chronologically-sorted-input-file", required = true)
	private List<String> parameters = new ArrayList<String>();

	private File inputFile;

	@Parameter(names = "-o", description = "Output file for community data")
	private File outputFile;

	@Parameter(names = "-t", description = "File with terms", required = true)
	private File termsFile;

	@Parameter(names = "-tcol", description = "Index or name of column to read terms (if term file is in CSV format)")
	private String termCol = "TERM";

	@Parameter(names = "-c", description = "Only use first c terms")
	private int termCount = -1;

	@Parameter(names = "-m", description = "Community map file", required = true)
	private File communityMapFile;

	private File logFile;

	public static final void main(String[] args) {
		CalculateCommData obj = new CalculateCommData();
		JCommander jc = new JCommander(obj);
		try {
			jc.parse(args);
		} catch (ParameterException ex) {
			jc.usage();
			System.exit(1);
		}
		if (obj.parameters.size() != 1) {
			System.err.println("ERROR: Exactly one main argument is needed");
			jc.usage();
			System.exit(1);
		}
		obj.inputFile = new File(obj.parameters.get(0));
		obj.run();
	}

	private Map<String,String> communityMap;
	private List<String> communitySequence;
	private Map<String,Integer> commFreq;
	private List<String> terms;
	private Map<String,Long> timeSum;

	private Map<String,Integer> termFreq;
	private Map<String,Map<String,Integer>> termCommFreq;

	private Map<String,List<Pair<String,Float>>> commTopMemes;

	public void run() {
		init();
		try {
			readTerms();
			readCommunityMap();
			readData();
			analyzeData();
			writeOutput();
		} catch (Throwable th) {
			log(th);
			System.exit(1);
		}
		log("Finished");
	}

	private void init() {
		logFile = new File(MemeUtils.getLogDir(), getOutputFileName() + ".log");
		log("==========");

		terms = new ArrayList<String>();
		communityMap = new HashMap<String,String>();
		communitySequence = new ArrayList<String>();
		commFreq = new HashMap<String,Integer>();<commit_msg>Forgot to initialize map...<commit_after>package ch.tkuhn.memetools;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang3.tuple.Pair;
import org.supercsv.io.CsvListReader;
import org.supercsv.io.CsvListWriter;

import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterException;

public class CalculateCommData {

	@Parameter(description = "chronologically-sorted-input-file", required = true)
	private List<String> parameters = new ArrayList<String>();

	private File inputFile;

	@Parameter(names = "-o", description = "Output file for community data")
	private File outputFile;

	@Parameter(names = "-t", description = "File with terms", required = true)
	private File termsFile;

	@Parameter(names = "-tcol", description = "Index or name of column to read terms (if term file is in CSV format)")
	private String termCol = "TERM";

	@Parameter(names = "-c", description = "Only use first c terms")
	private int termCount = -1;

	@Parameter(names = "-m", description = "Community map file", required = true)
	private File communityMapFile;

	private File logFile;

	public static final void main(String[] args) {
		CalculateCommData obj = new CalculateCommData();
		JCommander jc = new JCommander(obj);
		try {
			jc.parse(args);
		} catch (ParameterException ex) {
			jc.usage();
			System.exit(1);
		}
		if (obj.parameters.size() != 1) {
			System.err.println("ERROR: Exactly one main argument is needed");
			jc.usage();
			System.exit(1);
		}
		obj.inputFile = new File(obj.parameters.get(0));
		obj.run();
	}

	private Map<String,String> communityMap;
	private List<String> communitySequence;
	private Map<String,Integer> commFreq;
	private List<String> terms;
	private Map<String,Long> timeSum;

	private Map<String,Integer> termFreq;
	private Map<String,Map<String,Integer>> termCommFreq;

	private Map<String,List<Pair<String,Float>>> commTopMemes;

	public void run() {
		init();
		try {
			readTerms();
			readCommunityMap();
			readData();
			analyzeData();
			writeOutput();
		} catch (Throwable th) {
			log(th);
			System.exit(1);
		}
		log("Finished");
	}

	private void init() {
		logFile = new File(MemeUtils.getLogDir(), getOutputFileName() + ".log");
		log("==========");

		terms = new ArrayList<String>();
		communityMap = new HashMap<String,String>();
		communitySequence = new ArrayList<String>();
		commFreq = new HashMap<String,Integer>();
		timeSum = new HashMap<String,Long>();
{'fraction_non_alphanumeric': 0.13603209336250913, 'fraction_numerical': 0.002552881108679796, 'mean_word_length': 4.891514500537057, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7483857', 'n_tokens_mistral': 1860, 'n_tokens_neox': 1711, 'n_words': 447}