<commit_before>package edu.cmu.minorthird.text;

import edu.cmu.minorthird.text.gui.TextBaseViewer;
import edu.cmu.minorthird.text.learn.AnnotatorLearner;
import edu.cmu.minorthird.text.util.SimpleTextLoader;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;

import org.apache.log4j.*;

/**
 * Loads data files in a configurable way.
 *
 * To use this, put data.properties on the classpath,
 * and use it to define wcohen.scriptdir and wcohen.datadir.
 * Then FancyLoader.loadFoo("bar") will look in scriptdir
 * for a script named bar, execute it with dataDir bound to
 * the data directory, and return the result as an object of type 'Foo'.
 *
 * @author <NAME>
*/
public class FancyLoader
{
	private static Logger log = Logger.getLogger(FancyLoader.class);

	/** Property defining location of raw data */
	public static final String DATADIR_PROP = "edu.cmu.minorthird.dataDir";
	/** Property defining location of labels added to data */
	public static final String LABELDIR_PROP = "edu.cmu.minorthird.labelDir";
	/** Property defining location of scripts for loading data */
	public static final String SCRIPTDIR_PROP = "edu.cmu.minorthird.scriptDir";

	private static Properties props = new Properties();
	static {
		try {
			InputStream in = FancyLoader.class.getClassLoader().getResourceAsStream("data.properties");
			if (in != null) {
				props.load(in);
				log.info("loaded properties from stream "+in);
			} else {
				log.error("can't find data.properties");
				throw new IllegalStateException("can't get resource data.properties");
			}
		} catch (IOException e) {
			throw new IllegalStateException("error getting wcohen.properties: "+e);
		}
	};

  /** use the bsh script to load a textbase */
	public static TextBase loadTextBase(String script) {
		Object o = loadObject(script);
		if (o instanceof TextBase) return (TextBase)o;
		else if (o instanceof TextLabels) return ((TextLabels)o).getTextBase();
		else throw new IllegalStateException("script "+script+" doesn't load a text base");
	}

  /**
   * 1) call with BSH script which returns TextLabels
   * 2) call with a file name stem (eg "myData")
   *      load TextBase using either dir or .base (eg: myData/* or myData.base)
   *        - If both file myData.base and directory myData/ exist, loads from file myData.base
   *        - If a single file then load as DOC_PER_LINE
   *      load Labels either from .labels or out of data files (eg: myData.labels)
   *        - if myData.labels is found load from there throwing out xml tags
   *
   * @param script
   * @return TextLabels object with the annotations
   */
	public static TextLabels loadTextLabels(String script) 
	{

    //is this a script, ie: the file is real
		String scriptDir = getProperty(SCRIPTDIR_PROP);
    File f = new File(new File(scriptDir), script);

    if (f.exists() && !f.isDirectory()) {
			Object o = loadObject(script);
			if (o==null) 
				throw new IllegalArgumentException("script "+script+" from dir "+scriptDir+" returns null");
			if (!(o instanceof TextLabels)) 
				throw new IllegalArgumentException("script "+script+" from dir "+scriptDir+" doesn't return a TextLabels");
			return (TextLabels)o;
		}

    //look for other files - 1st does it have labels?
    File lablesFile = new File(script + ".labels");
    File dataFile = new File(script + ".base");
    File dataDir = new File(script);

    if (dataFile.exists())
      return SimpleTextLoader.load(dataFile, lablesFile.exists());

    if (dataDir.exists())
      return SimpleTextLoader.load(dataDir, lablesFile.exists());

    log.error("no data found! script=" + script);
    return null;
	}

	public static AnnotatorLearner loadAnnotatorLearner(String script) {
		return (AnnotatorLearner)loadObject(script);
	}

	public static Object loadObject(String script)
	{
		String dataDir = getProperty(DATADIR_PROP);
		String labelDir = getProperty(LABELDIR_PROP);
		String scriptDir = getProperty(SCRIPTDIR_PROP);
		log.info("dataDir: "+dataDir+" labelDir: "+labelDir+" scriptDir: "+scriptDir);
		try {
			bsh.Interpreter interpreter = new bsh.Interpreter();
			interpreter.eval("File dataDir = new File(\""+dataDir+"\");");
			interpreter.eval("File labelDir = new File(\""+labelDir+"\");");
			File f =  new File(new File(scriptDir),script);
			if (!f.exists()) throw new IllegalArgumentException("can't find file "+f.getAbsolutePath());
			log.info("loading object defined by "+f.getAbsolutePath());
			return interpreter.source(f.getAbsolutePath());
			} catch (Exception e) {
				log.error(e, e);
				return null;
			}
		}
		
	private static String getProperty(String prop) {
		String v = System.getProperty(prop);
		return v!=null ? v : props.getProperty(prop);
	}

	static public void main(String[] args) {
		Object o = FancyLoader.loadObject(args[0]);
		System.out.println("loaded "+o);
		if (o instanceof TextLabels) {
			TextBaseViewer.view((TextLabels) o );
		}
	}
}<commit_msg>made getProperty public for use by RepositoryLoader<commit_after>package edu.cmu.minorthird.text;

import edu.cmu.minorthird.text.gui.TextBaseViewer;
import edu.cmu.minorthird.text.learn.AnnotatorLearner;
import edu.cmu.minorthird.text.util.SimpleTextLoader;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;

import org.apache.log4j.*;

/**
 * Loads data files in a configurable way.
 *
 * To use this, put data.properties on the classpath,
 * and use it to define wcohen.scriptdir and wcohen.datadir.
 * Then FancyLoader.loadFoo("bar") will look in scriptdir
 * for a script named bar, execute it with dataDir bound to
 * the data directory, and return the result as an object of type 'Foo'.
 *
 * @author <NAME>
*/
public class FancyLoader
{
	private static Logger log = Logger.getLogger(FancyLoader.class);

	/** Property defining location of raw data */
	public static final String DATADIR_PROP = "edu.cmu.minorthird.dataDir";
	/** Property defining location of labels added to data */
	public static final String LABELDIR_PROP = "edu.cmu.minorthird.labelDir";
	/** Property defining location of scripts for loading data */
	public static final String SCRIPTDIR_PROP = "edu.cmu.minorthird.scriptDir";

	private static Properties props = new Properties();
	static {
		try {
			InputStream in = FancyLoader.class.getClassLoader().getResourceAsStream("data.properties");
			if (in != null) {
				props.load(in);
				log.info("loaded properties from stream "+in);
			} else {
				log.error("can't find data.properties");
				throw new IllegalStateException("can't get resource data.properties");
			}
		} catch (IOException e) {
			throw new IllegalStateException("error getting wcohen.properties: "+e);
		}
	};

  /** use the bsh script to load a textbase */
	public static TextBase loadTextBase(String script) {
		Object o = loadObject(script);
		if (o instanceof TextBase) return (TextBase)o;
		else if (o instanceof TextLabels) return ((TextLabels)o).getTextBase();
		else throw new IllegalStateException("script "+script+" doesn't load a text base");
	}

  /**
   * 1) call with BSH script which returns TextLabels
   * 2) call with a file name stem (eg "myData")
   *      load TextBase using either dir or .base (eg: myData/* or myData.base)
   *        - If both file myData.base and directory myData/ exist, loads from file myData.base
   *        - If a single file then load as DOC_PER_LINE
   *      load Labels either from .labels or out of data files (eg: myData.labels)
   *        - if myData.labels is found load from there throwing out xml tags
   *
   * @param script
   * @return TextLabels object with the annotations
   */
	public static TextLabels loadTextLabels(String script)
	{

    //is this a script, ie: the file is real
		String scriptDir = getProperty(SCRIPTDIR_PROP);
    File f = new File(new File(scriptDir), script);

    if (f.exists() && !f.isDirectory()) {
			Object o = loadObject(script);
			if (o==null)
				throw new IllegalArgumentException("script "+script+" from dir "+scriptDir+" returns null");
			if (!(o instanceof TextLabels))
				throw new IllegalArgumentException("script "+script+" from dir "+scriptDir+" doesn't return a TextLabels");
			return (TextLabels)o;
		}

    //look for other files - 1st does it have labels?
    File lablesFile = new File(script + ".labels");
    File dataFile = new File(script + ".base");
    File dataDir = new File(script);

    if (dataFile.exists())
      return SimpleTextLoader.load(dataFile, lablesFile.exists());

    if (dataDir.exists())
      return SimpleTextLoader.load(dataDir, lablesFile.exists());

    log.error("no data found! script=" + script);
    return null;
	}

	public static AnnotatorLearner loadAnnotatorLearner(String script) {
		return (AnnotatorLearner)loadObject(script);
	}

	public static Object loadObject(String script)
	{
		String dataDir = getProperty(DATADIR_PROP);
		String labelDir = getProperty(LABELDIR_PROP);
		String scriptDir = getProperty(SCRIPTDIR_PROP);
		log.info("dataDir: "+dataDir+" labelDir: "+labelDir+" scriptDir: "+scriptDir);
		try {
			bsh.Interpreter interpreter = new bsh.Interpreter();
			interpreter.eval("File dataDir = new File(\""+dataDir+"\");");
			interpreter.eval("File labelDir = new File(\""+labelDir+"\");");
			File f =  new File(new File(scriptDir),script);
			if (!f.exists()) throw new IllegalArgumentException("can't find file "+f.getAbsolutePath());
			log.info("loading object defined by "+f.getAbsolutePath());
			return interpreter.source(f.getAbsolutePath());
			} catch (Exception e) {
				log.error(e, e);
				return null;
			}
		}

	public static String getProperty(String prop) {
		String v = System.getProperty(prop);
		return v!=null ? v : props.getProperty(prop);
	}

	static public void main(String[] args) {
		Object o = FancyLoader.loadObject(args[0]);
		System.out.println("loaded "+o);
		if (o instanceof TextLabels) {
			TextBaseViewer.view((TextLabels) o );
		}
	}
}
{'fraction_non_alphanumeric': 0.11788291900561347, 'fraction_numerical': 0.0010024057738572574, 'mean_word_length': 3.9885, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4431349', 'n_tokens_mistral': 3217, 'n_tokens_neox': 2997, 'n_words': 946}