<commit_before>package edu.cmu.minorthird.text;

import org.apache.log4j.Logger;

import java.io.*;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import edu.cmu.minorthird.Loader;

/** Loads the contents of a TextBase from a file.
 *
 *
 * @author <NAME>
*/

public class TextBaseLoader implements Loader
{
	private static Logger log = Logger.getLogger(TextBaseLoader.class);

	private class StackEntry {
		public int index;
		public String markupTag;
		public StackEntry(int index,String markupTag) {
			this.index=index; this.markupTag=markupTag;
		}
	}
	private class CharSpan {
		public int lo,hi;
		String type;
		public CharSpan(int lo,int hi,String type) {
			this.lo=lo; this.hi=hi; this.type = type;
		}
	}

	private int closurePolicy = TextLabelsLoader.CLOSE_ALL_TYPES;
	/** Set the closure policy.
	 * @param policy one of TextLabelsLoader.CLOSE_ALL_TYPES,
	 * TextLabelsLoader.CLOSE_TYPES_IN_LABELED_DOCS, TextLabelsLoader.DONT_CLOSE_TYPES
	 */
	public void setClosurePolicy(int policy) { this.closurePolicy = policy; }

	// saves labels associated with last set of files loaded
	private MutableTextLabels labels;

	/**
   * Load from either a file (one document per line) or a directory (one document per file)
   * Directory is assumed to be tagged files
   * Single file assumed not to be tagged
   */
	public void loadFile(TextBase base,File file) throws IOException,FileNotFoundException
	{
		if (file.isDirectory())
      loadTaggedFiles(base,file);
		else {
			loadLines(base,file);
			labels = new BasicTextLabels(base);
		}
	}

	/** Get all markup suggested by XML/SGML tags from the files loaded by loadTaggedFiles. */
	public MutableTextLabels getFileMarkup() { return labels; }

	/** Load files from a directory, stripping out any XML/SGML tags. */
	public void loadTaggedFiles(TextBase base,File dir) throws IOException,FileNotFoundException
	{
		labels = new BasicTextLabels(base);
		Pattern markupPattern = Pattern.compile("</?([^ ><]+)( [^<>]+)?>");

		File[] files = dir.listFiles();
		if (files==null) throw new IllegalArgumentException("can't list directory "+dir.getName());

		for (int i=0; i<files.length; i++) {

			// skip CVS directories
			if ("CVS".equals(files[i].getName())) continue;

      loadTaggedFile(files[i], markupPattern, base);

    }
	}

  public void loadTaggedFile(File file, Pattern markupPattern, TextBase base) throws IOException
  {
    if (labels == null)
      labels = new BasicTextLabels(base);

    if (markupPattern == null)
      markupPattern = Pattern.compile("</?([^ ><]+)( [^<>]+)?>");

    // stack of open tags
    ArrayList stack = new ArrayList();
    // list of constructed spans
    List spanList = new ArrayList();
    // file name used as ID
    String id = file.getName();
    // holds a string representation of the file with xml tags removed
    StringBuffer buf = new StringBuffer("");

    LineNumberReader in = new LineNumberReader(new FileReader(file));
    String line;
    while ((line = in.readLine())!=null) {
      int currentChar = 0;
      Matcher matcher = markupPattern.matcher(line);
      while (matcher.find()) {
        String tag = matcher.group(1);
        boolean isOpenTag = !matcher.group().startsWith("</");
        log.debug("matcher.group='"+matcher.group()+"'");
        log.debug("found '"+tag+"' tag ,open="+isOpenTag+", at "+matcher.start()+" in:\n"+line);
        //copy stuff up to tag into buffer
        buf.append( line.substring(currentChar, matcher.start()) );
        currentChar = matcher.end();
        if (isOpenTag) {
          stack.add( new StackEntry(buf.length(), tag) );
        } else {
          // pop the corresponding open off the stack
          StackEntry entry = null;
          for (int j=stack.size()-1; j>=0; j--) {
            entry = (StackEntry)stack.get(j);
            if (tag.equals(entry.markupTag)) {
              stack.remove(j);
              break;
            }
          }
          if (entry==null)
            throw new IllegalStateException(id+"@"+in.getLineNumber()+": close '"+tag+"' tag with no open");
          if (!tag.equals(entry.markupTag))
            throw new IllegalStateException(id+"@"+in.getLineNumber()+": close '"+tag+"' tag paired with open '"
                                            +entry.markupTag+"'");
          log.debug("adding a "+tag+" span from "+entry.index+" to "+buf.length()
                    +": '"+buf.substring(entry.index)+"'");
          spanList.add( new CharSpan(entry.index, buf.length(), tag) );
        }
      }
      // append stuff from end of last tag to end of line into the buffer
      buf.append( line.substring(currentChar, line.length()) );
      buf.append( "\n" );
    }
    // add the document to the textbase
    base.loadDocument(id, buf.toString() );
    // add the markup to the labels
    Set types = new TreeSet();
    for (Iterator j=spanList.iterator(); j.hasNext(); ) {
      CharSpan charSpan = (CharSpan)j.next();
      types.add( charSpan.type );
      Span approxSpan = base.documentSpan(id).charIndexSubSpan(charSpan.lo, charSpan.hi);
      log.debug("approximating "+charSpan.type+" span '"
                +buf.toString().substring(charSpan.lo,charSpan.hi)
                +"' with token span '"+approxSpan);
      labels.addToType( approxSpan, charSpan.type );
    }
    new TextLabelsLoader().closeLabels( labels, closurePolicy );
  }

  //
	// loadLines code<commit_msg>Close file in loadTaggedFile<commit_after>package edu.cmu.minorthird.text;

import org.apache.log4j.Logger;

import java.io.*;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import edu.cmu.minorthird.Loader;

/** Loads the contents of a TextBase from a file.
 *
 *
 * @author <NAME>
*/

public class TextBaseLoader implements Loader
{
	private static Logger log = Logger.getLogger(TextBaseLoader.class);

	private class StackEntry {
		public int index;
		public String markupTag;
		public StackEntry(int index,String markupTag) {
			this.index=index; this.markupTag=markupTag;
		}
	}
	private class CharSpan {
		public int lo,hi;
		String type;
		public CharSpan(int lo,int hi,String type) {
			this.lo=lo; this.hi=hi; this.type = type;
		}
	}

	private int closurePolicy = TextLabelsLoader.CLOSE_ALL_TYPES;
	/** Set the closure policy.
	 * @param policy one of TextLabelsLoader.CLOSE_ALL_TYPES,
	 * TextLabelsLoader.CLOSE_TYPES_IN_LABELED_DOCS, TextLabelsLoader.DONT_CLOSE_TYPES
	 */
	public void setClosurePolicy(int policy) { this.closurePolicy = policy; }

	// saves labels associated with last set of files loaded
	private MutableTextLabels labels;

	/**
   * Load from either a file (one document per line) or a directory (one document per file)
   * Directory is assumed to be tagged files
   * Single file assumed not to be tagged
   */
	public void loadFile(TextBase base,File file) throws IOException,FileNotFoundException
	{
		if (file.isDirectory())
      loadTaggedFiles(base,file);
		else {
			loadLines(base,file);
			labels = new BasicTextLabels(base);
		}
	}

	/** Get all markup suggested by XML/SGML tags from the files loaded by loadTaggedFiles. */
	public MutableTextLabels getFileMarkup() { return labels; }

	/** Load files from a directory, stripping out any XML/SGML tags. */
	public void loadTaggedFiles(TextBase base,File dir) throws IOException,FileNotFoundException
	{
		labels = new BasicTextLabels(base);
		Pattern markupPattern = Pattern.compile("</?([^ ><]+)( [^<>]+)?>");

		File[] files = dir.listFiles();
		if (files==null) throw new IllegalArgumentException("can't list directory "+dir.getName());

		for (int i=0; i<files.length; i++) {

			// skip CVS directories
			if ("CVS".equals(files[i].getName())) continue;

      loadTaggedFile(files[i], markupPattern, base);

    }
	}

  public void loadTaggedFile(File file, Pattern markupPattern, TextBase base) throws IOException
  {
    if (labels == null)
      labels = new BasicTextLabels(base);

    if (markupPattern == null)
      markupPattern = Pattern.compile("</?([^ ><]+)( [^<>]+)?>");

    // stack of open tags
    ArrayList stack = new ArrayList();
    // list of constructed spans
    List spanList = new ArrayList();
    // file name used as ID
    String id = file.getName();
    // holds a string representation of the file with xml tags removed
    StringBuffer buf = new StringBuffer("");

    LineNumberReader in = new LineNumberReader(new FileReader(file));
    String line;
    while ((line = in.readLine())!=null) {
      int currentChar = 0;
      Matcher matcher = markupPattern.matcher(line);
      while (matcher.find()) {
        String tag = matcher.group(1);
        boolean isOpenTag = !matcher.group().startsWith("</");
        log.debug("matcher.group='"+matcher.group()+"'");
        log.debug("found '"+tag+"' tag ,open="+isOpenTag+", at "+matcher.start()+" in:\n"+line);
        //copy stuff up to tag into buffer
        buf.append( line.substring(currentChar, matcher.start()) );
        currentChar = matcher.end();
        if (isOpenTag) {
          stack.add( new StackEntry(buf.length(), tag) );
        } else {
          // pop the corresponding open off the stack
          StackEntry entry = null;
          for (int j=stack.size()-1; j>=0; j--) {
            entry = (StackEntry)stack.get(j);
            if (tag.equals(entry.markupTag)) {
              stack.remove(j);
              break;
            }
          }
          if (entry==null)
            throw new IllegalStateException(id+"@"+in.getLineNumber()+": close '"+tag+"' tag with no open");
          if (!tag.equals(entry.markupTag))
            throw new IllegalStateException(id+"@"+in.getLineNumber()+": close '"+tag+"' tag paired with open '"
                                            +entry.markupTag+"'");
          log.debug("adding a "+tag+" span from "+entry.index+" to "+buf.length()
                    +": '"+buf.substring(entry.index)+"'");
          spanList.add( new CharSpan(entry.index, buf.length(), tag) );
        }
      }
      // append stuff from end of last tag to end of line into the buffer
      buf.append( line.substring(currentChar, line.length()) );
      buf.append( "\n" );
    }
    in.close();
    // add the document to the textbase
    base.loadDocument(id, buf.toString() );
    // add the markup to the labels
    Set types = new TreeSet();
    for (Iterator j=spanList.iterator(); j.hasNext(); ) {
      CharSpan charSpan = (CharSpan)j.next();
      types.add( charSpan.type );
      Span approxSpan = base.documentSpan(id).charIndexSubSpan(charSpan.lo, charSpan.hi);
      log.debug("approximating "+charSpan.type+" span '"
                +buf.toString().substring(charSpan.lo,charSpan.hi)
                +"' with token span '"+approxSpan);
      labels.addToType( approxSpan, charSpan.type );
    }
    new TextLabelsLoader().closeLabels( labels, closurePolicy );
  }

  //
	// loadLines code
{'fraction_non_alphanumeric': 0.13332112443915392, 'fraction_numerical': 0.001098800476146873, 'mean_word_length': 2.9803206997084546, 'pattern_counts': {'":': 6, '<': 21, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2266560', 'n_tokens_mistral': 3363, 'n_tokens_neox': 3236, 'n_words': 897}