<commit_before>package mstparser.io;

import mstparser.DependencyInstance;
import mstparser.Util;

import java.io.*;
import java.util.*;

public class CONLLReader extends DependencyReader {


    public DependencyInstance getNext() throws IOException {

	ArrayList<String[]> lineList = new ArrayList<String[]>();

	String line = inputReader.readLine();
	while (line != null && !line.equals("")) {
	    lineList.add(line.split("\t"));
	    line = inputReader.readLine();
	}

	int length = lineList.size();

	if(length == 0) {
	    inputReader.close();
	    return null;
	}

	String[] forms = new String[length+1];
	String[] lemmas = new String[length+1];
	String[] cpos = new String[length+1];
	String[] pos = new String[length+1];
	String[][] feats = new String[length+1][];
	String[] deprels = new String[length+1];
	int[] heads = new int[length+1];

	forms[0] = "<root>";
	lemmas[0] = "<root-LEMMA>";
	cpos[0] = "<root-CPOS>";
	pos[0] = "<root-POS>";
	String[] rootFeat = {"<root-XFACT>"};
	feats[0] = rootFeat;
	deprels[0] = "<no-type>";
	heads[0] = -1;

	for(int i = 0; i < length; i++) {
	    String[] info = lineList.get(i);
	    forms[i+1] = normalize(info[1]);
	    lemmas[i+1] = normalize(info[2]);
	    cpos[i+1] = info[3];
	    pos[i+1] = info[4];
	    feats[i+1] = info[5].split("|");
	    deprels[i+1] = labeled ? info[7] : "<no-type>";
	    heads[i+1] = Integer.parseInt(info[6]);
	}


	return new DependencyInstance(forms, lemmas, cpos, pos, feats, deprels, heads);

    }


    protected boolean fileContainsLabels (String file) throws IOException {
	BufferedReader in = new BufferedReader(new FileReader(file));
	String line = in.readLine();
	in.close();<commit_msg>Fixed bug in how feature declarations were split --needed to escape | as \|.<commit_after>package mstparser.io;

import mstparser.DependencyInstance;
import mstparser.Util;

import java.io.*;
import java.util.*;

public class CONLLReader extends DependencyReader {


    public DependencyInstance getNext() throws IOException {

	ArrayList<String[]> lineList = new ArrayList<String[]>();

	String line = inputReader.readLine();
	while (line != null && !line.equals("")) {
	    lineList.add(line.split("\t"));
	    line = inputReader.readLine();
	}

	int length = lineList.size();

	if(length == 0) {
	    inputReader.close();
	    return null;
	}

	String[] forms = new String[length+1];
	String[] lemmas = new String[length+1];
	String[] cpos = new String[length+1];
	String[] pos = new String[length+1];
	String[][] feats = new String[length+1][];
	String[] deprels = new String[length+1];
	int[] heads = new int[length+1];

	forms[0] = "<root>";
	lemmas[0] = "<root-LEMMA>";
	cpos[0] = "<root-CPOS>";
	pos[0] = "<root-POS>";
	deprels[0] = "<no-type>";
	heads[0] = -1;

	for(int i = 0; i < length; i++) {
	    String[] info = lineList.get(i);
	    forms[i+1] = normalize(info[1]);
	    lemmas[i+1] = normalize(info[2]);
	    cpos[i+1] = info[3];
	    pos[i+1] = info[4];
	    feats[i+1] = info[5].split("\\|");
	    deprels[i+1] = labeled ? info[7] : "<no-type>";
	    heads[i+1] = Integer.parseInt(info[6]);
	}

	feats[0] = new String[feats[1].length];
	for (int i = 0; i< feats[1].length; i++)
	    feats[0][i] = "<root-feat>"+i;

	return new DependencyInstance(forms, lemmas, cpos, pos, feats, deprels, heads);

    }


    protected boolean fileContainsLabels (String file) throws IOException {
	BufferedReader in = new BufferedReader(new FileReader(file));
	String line = in.readLine();
	in.close();
{'fraction_non_alphanumeric': 0.18715964459730583, 'fraction_numerical': 0.018916595012897677, 'mean_word_length': 4.080058224163028, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2463425', 'n_tokens_mistral': 1319, 'n_tokens_neox': 1236, 'n_words': 292}