<commit_before>package org.libbun;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;

import java.text.DecimalFormat;

public class Main {
	public final static String  ProgName  = "libbun";
	public final static String  CodeName  = "yokohama";
	public final static int     MajorVersion = 2;
	public final static int     MinerVersion = 0;
	public final static int     PatchLevel   = 0;
	public final static String  Version = "2.0";
	public final static String  Copyright = "Copyright (c) 2013-2014, Konoha project authors";
	public final static String  License = "BSD-Style Open Source";

	// -l konoha.peg
	private static String LanguagePeg = "lib/peg/konoha.peg"; // default

	// -d driver
	private static String DriverName = "debug";  // default

	// -i
	private static boolean ShellMode = false;

	// -o
	private static String OutputFileName = null;

	//
	private static String InputFileName = null;

	// --parse-only
	public static boolean ParseOnly = false;

	// --verbose
	public static boolean EnableVerbose = false;

	// --verbose:ast
	public static boolean EnableVerboseAst = false;

	// --profile
	public static boolean ProfileMode = false;

	// --verbose:peg
	public static boolean PegDebuggerMode = false;

	// --disable-memo
	public static boolean NonMemoPegMode = false;

	private static void parseCommandArgument(String[] args) {
		int index = 0;
		while (index < args.length) {
			String argument = args[index];
			if (!argument.startsWith("-")) {
				break;
			}
			index = index + 1;
			if (argument.equals("--peg") && (index < args.length)) {
				LanguagePeg = args[index];
				PegDebuggerMode = true;
				index = index + 1;
			}
			else if (argument.equals("-l") && (index < args.length)) {
				LanguagePeg = args[index];
				index = index + 1;
			}
			else if ((argument.equals("-d") || argument.equals("--driver")) && (index < args.length)) {
				DriverName = args[index];
				index = index + 1;
			}
			else if ((argument.equals("-o") || argument.equals("--out")) && (index < args.length)) {
				OutputFileName = args[index];
				index = index + 1;
			}
			else if (argument.equals("-i")) {
				ShellMode = true;
			}
			else if (argument.equals("--disable-memo")) {
				NonMemoPegMode = true;
			}
			else if (argument.equals("--profile")) {
				ProfileMode = true;
			}
			else if (argument.equals("--parse-only")) {
				ParseOnly = true;
			}
			else if(argument.startsWith("--verbose")) {
				if(argument.equals("--verbose:ast")) {
					EnableVerboseAst = true;
				}
//				else if(argument.equals("--verbose:peg")) {
//					pegDebugger = true;
//				}
				else {
					EnableVerbose = true;
					EnableVerboseAst = true;
				}
			}
			else {
				ShowUsage("unknown option: " + argument);
			}
		}
		if (index < args.length) {
			InputFileName = args[index];
		}
		else {
			ShellMode = true;
		}
	}

	public final static void ShowUsage(String Message) {
		System.out.println(ProgName + " :");
		System.out.println("  --peg|-l  FILE          Language file");
		System.out.println("  --driver|-d  NAME       Driver");
		System.out.println("  --out|-o  FILE          Output filename");
		System.out.println("  --verbose               Printing Debug infomation");
		System.out.println("  --profile               Show memory usage and parse time");
		Main._Exit(0, Message);
	}

	private final static UniMap<Class<?>> driverMap = new UniMap<Class<?>>();
	static {
		driverMap.put("py", PythonDriver.class);
		driverMap.put("python", PythonDriver.class);
		driverMap.put("ll", LLVMDriver.class);
		driverMap.put("llvm", LLVMDriver.class);
		driverMap.put("jvm", JvmDriver.class);
		driverMap.put("jvm-debug", JvmDriver.DebuggableJvmDriver.class);
		driverMap.put("jvm-indy", JvmIndyDriver.class);
		driverMap.put("jvm-indy-debug", JvmIndyDriver.DebuggableJvmIndyDriver.class);
		driverMap.put("debug-gui", DebugGuiDriver.class);
	}

	private static BunDriver loadDriver(String driverName) {
		if(PegDebuggerMode) {
			return new Debugger();
		}
		try {
			return (BunDriver) driverMap.get(driverName, PythonDriver.class).newInstance();
		}
		catch(Throwable t) {
			System.err.println("cannot load driver: " + driverName);
			System.err.println("instead load python driver");
		}<commit_msg>remove unused code<commit_after>package org.libbun;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;

import java.text.DecimalFormat;

public class Main {
	public final static String  ProgName  = "libbun";
	public final static String  CodeName  = "yokohama";
	public final static int     MajorVersion = 2;
	public final static int     MinerVersion = 0;
	public final static int     PatchLevel   = 0;
	public final static String  Version = "2.0";
	public final static String  Copyright = "Copyright (c) 2013-2014, Konoha project authors";
	public final static String  License = "BSD-Style Open Source";

	// -l konoha.peg
	private static String LanguagePeg = "lib/peg/konoha.peg"; // default

	// -d driver
	private static String DriverName = "debug";  // default

	// -i
	private static boolean ShellMode = false;

	// -o
	private static String OutputFileName = null;

	//
	private static String InputFileName = null;

	// --parse-only
	public static boolean ParseOnly = false;

	// --verbose
	public static boolean EnableVerbose = false;

	// --verbose:ast
	public static boolean EnableVerboseAst = false;

	// --profile
	public static boolean ProfileMode = false;

	// --verbose:peg
	public static boolean PegDebuggerMode = false;

	// --disable-memo
	public static boolean NonMemoPegMode = false;

	private static void parseCommandArgument(String[] args) {
		int index = 0;
		while (index < args.length) {
			String argument = args[index];
			if (!argument.startsWith("-")) {
				break;
			}
			index = index + 1;
			if (argument.equals("--peg") && (index < args.length)) {
				LanguagePeg = args[index];
				PegDebuggerMode = true;
				index = index + 1;
			}
			else if (argument.equals("-l") && (index < args.length)) {
				LanguagePeg = args[index];
				index = index + 1;
			}
			else if ((argument.equals("-d") || argument.equals("--driver")) && (index < args.length)) {
				DriverName = args[index];
				index = index + 1;
			}
			else if ((argument.equals("-o") || argument.equals("--out")) && (index < args.length)) {
				OutputFileName = args[index];
				index = index + 1;
			}
			else if (argument.equals("-i")) {
				ShellMode = true;
			}
			else if (argument.equals("--disable-memo")) {
				NonMemoPegMode = true;
			}
			else if (argument.equals("--profile")) {
				ProfileMode = true;
			}
			else if (argument.equals("--parse-only")) {
				ParseOnly = true;
			}
			else if(argument.startsWith("--verbose")) {
				if(argument.equals("--verbose:ast")) {
					EnableVerboseAst = true;
				}
//				else if(argument.equals("--verbose:peg")) {
//					pegDebugger = true;
//				}
				else {
					EnableVerbose = true;
					EnableVerboseAst = true;
				}
			}
			else {
				ShowUsage("unknown option: " + argument);
			}
		}
		if (index < args.length) {
			InputFileName = args[index];
		}
		else {
			ShellMode = true;
		}
	}

	public final static void ShowUsage(String Message) {
		System.out.println(ProgName + " :");
		System.out.println("  --peg|-l  FILE          Language file");
		System.out.println("  --driver|-d  NAME       Driver");
		System.out.println("  --out|-o  FILE          Output filename");
		System.out.println("  --verbose               Printing Debug infomation");
		System.out.println("  --profile               Show memory usage and parse time");
		Main._Exit(0, Message);
	}

	private final static UniMap<Class<?>> driverMap = new UniMap<Class<?>>();
	static {
		driverMap.put("py", PythonDriver.class);
		driverMap.put("python", PythonDriver.class);
		driverMap.put("ll", LLVMDriver.class);
		driverMap.put("llvm", LLVMDriver.class);
		driverMap.put("jvm", JvmDriver.class);
		driverMap.put("jvm-debug", JvmDriver.DebuggableJvmDriver.class);
		driverMap.put("jvm-indy", JvmIndyDriver.class);
		driverMap.put("jvm-indy-debug", JvmIndyDriver.DebuggableJvmIndyDriver.class);
	}

	private static BunDriver loadDriver(String driverName) {
		if(PegDebuggerMode) {
			return new Debugger();
		}
		try {
			return (BunDriver) driverMap.get(driverName, PythonDriver.class).newInstance();
		}
		catch(Throwable t) {
			System.err.println("cannot load driver: " + driverName);
			System.err.println("instead load python driver");
		}
{'fraction_non_alphanumeric': 0.14142450142450141, 'fraction_numerical': 0.004558404558404558, 'mean_word_length': 3.2396135265700483, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '889424', 'n_tokens_mistral': 3191, 'n_tokens_neox': 2786, 'n_words': 684}