<commit_before>package j2s;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;

public class GenerateSwiftQueryString {

	private static HashMap<String, Integer> frequency = new HashMap<String, Integer>();
	private static HashSet<String> filterKeys;
	private static final int NUM_FREQ_RETURN = 1;

	static {
		filterKeys = new HashSet<String>();
		filterKeys.add(" ");
		filterKeys.add("public");
		filterKeys.add("string");
		filterKeys.add("\n");
		filterKeys.add(".");
		filterKeys.add(",");
		filterKeys.add(":");
		filterKeys.add(";");
		filterKeys.add("");
		filterKeys.add("{");
		filterKeys.add("}");
		filterKeys.add("[");
		filterKeys.add("]");
		filterKeys.add("?");
		filterKeys.add("-");
		filterKeys.add(">");
		filterKeys.add("<");
		filterKeys.add("=");
		filterKeys.add("!");
		filterKeys.add("\\");
		filterKeys.add("\"");
	}

	public ArrayList<String> executeFrequencyAnalysis(String filepath) {
		BufferedReader br = null;
		try {
			String sCurrentLine;

			br = new BufferedReader(new FileReader(filepath));
			while ((sCurrentLine = br.readLine()) != null) {
				if (!sCurrentLine.trim().equals("")) {
					StringTokenizer st = new StringTokenizer(sCurrentLine);
					while (st.hasMoreTokens()) {
						parseToken(st.nextToken());
					}
				}
			}<commit_msg>% change on both sides<commit_after>package j2s;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;

public class GenerateSwiftQueryString {

	private static HashMap<String, Integer> frequency = new HashMap<String, Integer>();
	private static HashSet<String> filterKeys;
	private static final int NUM_FREQ_RETURN = 1;

	static {
		filterKeys = new HashSet<String>();
		filterKeys.add(" ");
		filterKeys.add("public");
		filterKeys.add("string");
		filterKeys.add("\n");
		filterKeys.add(".");
		filterKeys.add(",");
		filterKeys.add(":");
		filterKeys.add(";");
		filterKeys.add("");
		filterKeys.add("{");
		filterKeys.add("}");
		filterKeys.add("[");
		filterKeys.add("]");
		filterKeys.add("?");
		filterKeys.add("-");
		filterKeys.add(">");
		filterKeys.add("<");
		filterKeys.add("=");
		filterKeys.add("!");
		filterKeys.add("\\");
		filterKeys.add("\"");
		filterKeys.add("%");
	}

	public ArrayList<String> executeFrequencyAnalysis(String filepath) {
		BufferedReader br = null;
		try {
			String sCurrentLine;

			br = new BufferedReader(new FileReader(filepath));
			while ((sCurrentLine = br.readLine()) != null) {
				if (!sCurrentLine.trim().equals("")) {
					StringTokenizer st = new StringTokenizer(sCurrentLine);
					while (st.hasMoreTokens()) {
						parseToken(st.nextToken());
					}
				}
			}
{'fraction_non_alphanumeric': 0.1691449814126394, 'fraction_numerical': 0.0012391573729863693, 'mean_word_length': 4.6948853615520285, 'pattern_counts': {'":': 2, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6703802', 'n_tokens_mistral': 1155, 'n_tokens_neox': 1043, 'n_words': 207}