<commit_before>package org.grobid.core.engines;

import org.grobid.core.GrobidModels;
import org.grobid.core.data.Affiliation;
import org.grobid.core.exceptions.GrobidException;
import org.grobid.core.features.FeaturesVectorAffiliationAddress;
import org.grobid.core.lexicon.Lexicon;
import org.grobid.core.utilities.OffsetPosition;
import org.grobid.core.utilities.TextUtilities;

import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;

/**
 * @author <NAME>
 */
public class AffiliationAddressParser extends AbstractParser {
    public Lexicon lexicon = Lexicon.getInstance();

    public AffiliationAddressParser() {
        super(GrobidModels.AFFIILIATON_ADDRESS);
    }

    public ArrayList<Affiliation> processing(String input) {
        try {
            ArrayList<String> affiliationBlocks = new ArrayList<String>();
            input = input.trim();

            input = TextUtilities.dehyphenize(input);
            StringTokenizer st = new StringTokenizer(input, " \n\t" + TextUtilities.fullPunctuations, true);

            ArrayList<String> tokenizations = new ArrayList<String>();
            while (st.hasMoreTokens()) {
                String tok = st.nextToken();
                if (tok.length() == 0) continue;
                if (tok.equals("\n")) {
                    tokenizations.add(" ");
                } else {
                    tokenizations.add(tok);
                }
                if (!tok.equals(" ")) {
                    if (tok.equals("\n")) {
                        affiliationBlocks.add("@newline");
                    }
                    affiliationBlocks.add(tok + " <affiliation>");
                }
            }


            List<List<OffsetPosition>> placesPositions = new ArrayList<List<OffsetPosition>>();
            placesPositions.add(lexicon.inCityNames(input));

            String header = FeaturesVectorAffiliationAddress.addFeaturesAffiliationAddress(affiliationBlocks, placesPositions);

            // add context
            st = new StringTokenizer(header, "\n");

            feedTaggerAndParse(st);

            StringBuilder res = new StringBuilder();
            for (int i = 0; i < tagger.size(); i++) {
                for (int j = 0; j < tagger.xsize(); j++) {
                    res.append(tagger.x(i, j)).append("\t");
                }

                res.append("<affiliation>" + "\t");
                res.append(tagger.y2(i));
                res.append("\n");
            }
            return resultBuilder(res.toString(), tokenizations, false); // don't use pre-labels
        } catch (Exception e) {
            throw new GrobidException("An exception occurred while running Grobid.", e);
        }
    }

    /**
     * Post processing of extracted field affiliation and address.
     * Here the input string to be processed comes from a previous parser: the segmentation
     * can be kept and we filter in all tokens labelled <address> or <affiliation>.
     * We also need to keep the original tokenization information to recreate the exact
     * initial string.
     */
    public ArrayList<Affiliation> processReflow(String result,
                                                ArrayList<String> tokenizations) throws Exception {
        try {
            ArrayList<String> affiliationBlocks = new ArrayList<String>();
            ArrayList<String> subTokenizations = new ArrayList<String>();

            filterAffiliationAddress(result, tokenizations, affiliationBlocks, subTokenizations);

            //System.out.println(affiliationBlocks.toString());
            //System.out.println(subTokenizations.toString());

            return processingReflow(affiliationBlocks, subTokenizations);
        } catch (Exception e) {
//			e.printStackTrace();
            throw new GrobidException("An exception occured while running Grobid.", e);
        }
//		return null;
    }


    private void filterAffiliationAddress(String result,
                                          ArrayList<String> tokenizations,
                                          ArrayList<String> affiliationBlocks,
                                          ArrayList<String> subTokenizations) {
        StringTokenizer st = new StringTokenizer(result, "\n");
        //System.out.println(result);
        //System.out.println(tokenizations.toString());
        String lastLabel = null;
        int lineCount = 0;
        int p = 0;
        while (st.hasMoreTokens()) {
            String toke = tokenizations.get(p);
            List<String> tokes = new ArrayList<String>();
            while ((toke.equals(" ") || toke.equals("\n") || (toke.length() == 0)) && ((p+1) < tokenizations.size())) {
                p++;
                if (toke.length() == 0) {
                    toke = tokenizations.get(p);
                    continue;
                }
                tokes.add(toke);
                if (toke.equals("\n")) {
                    affiliationBlocks.add("@newline");
                }
                toke = tokenizations.get(p);
            }

            String line = st.nextToken();
            if (line.length() == 0) {
                continue;
            }

            if (line.trim().length() == 0) {
                affiliationBlocks.add("\n");
                //subTokenizations.add(" ");
                lastLabel = null;<commit_msg>Correct bug in the tokenization of the affiliation/address block.<commit_after>package org.grobid.core.engines;

import org.grobid.core.GrobidModels;
import org.grobid.core.data.Affiliation;
import org.grobid.core.exceptions.GrobidException;
import org.grobid.core.features.FeaturesVectorAffiliationAddress;
import org.grobid.core.lexicon.Lexicon;
import org.grobid.core.utilities.OffsetPosition;
import org.grobid.core.utilities.TextUtilities;

import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;

/**
 * @author <NAME>
 */
public class AffiliationAddressParser extends AbstractParser {
    public Lexicon lexicon = Lexicon.getInstance();

    public AffiliationAddressParser() {
        super(GrobidModels.AFFIILIATON_ADDRESS);
    }

    public ArrayList<Affiliation> processing(String input) {
        try {
            ArrayList<String> affiliationBlocks = new ArrayList<String>();
            input = input.trim();

            input = TextUtilities.dehyphenize(input);
            StringTokenizer st = new StringTokenizer(input, " \n\t" + TextUtilities.fullPunctuations, true);

            ArrayList<String> tokenizations = new ArrayList<String>();
            while (st.hasMoreTokens()) {
                String tok = st.nextToken();
                if (tok.length() == 0) continue;
                if (tok.equals("\n")) {
                    tokenizations.add(" ");
                } else {
                    tokenizations.add(tok);
                }
                if (!tok.equals(" ")) {
                    if (tok.equals("\n")) {
                        affiliationBlocks.add("@newline");
                    }
                    affiliationBlocks.add(tok + " <affiliation>");
                }
            }


            List<List<OffsetPosition>> placesPositions = new ArrayList<List<OffsetPosition>>();
            placesPositions.add(lexicon.inCityNames(input));

            String header = FeaturesVectorAffiliationAddress.addFeaturesAffiliationAddress(affiliationBlocks, placesPositions);

            // add context
            st = new StringTokenizer(header, "\n");

            feedTaggerAndParse(st);

            StringBuilder res = new StringBuilder();
            for (int i = 0; i < tagger.size(); i++) {
                for (int j = 0; j < tagger.xsize(); j++) {
                    res.append(tagger.x(i, j)).append("\t");
                }

                res.append("<affiliation>" + "\t");
                res.append(tagger.y2(i));
                res.append("\n");
            }
            return resultBuilder(res.toString(), tokenizations, false); // don't use pre-labels
        } catch (Exception e) {
            throw new GrobidException("An exception occurred while running Grobid.", e);
        }
    }

    /**
     * Post processing of extracted field affiliation and address.
     * Here the input string to be processed comes from a previous parser: the segmentation
     * can be kept and we filter in all tokens labelled <address> or <affiliation>.
     * We also need to keep the original tokenization information to recreate the exact
     * initial string.
     */
    public ArrayList<Affiliation> processReflow(String result,
                                                ArrayList<String> tokenizations) throws Exception {
        try {
            ArrayList<String> affiliationBlocks = new ArrayList<String>();
            ArrayList<String> subTokenizations = new ArrayList<String>();

            filterAffiliationAddress(result, tokenizations, affiliationBlocks, subTokenizations);

            //System.out.println(affiliationBlocks.toString());
            //System.out.println(subTokenizations.toString());

            return processingReflow(affiliationBlocks, subTokenizations);
        } catch (Exception e) {
//			e.printStackTrace();
            throw new GrobidException("An exception occured while running Grobid.", e);
        }
//		return null;
    }


    private void filterAffiliationAddress(String result,
                                          ArrayList<String> tokenizations,
                                          ArrayList<String> affiliationBlocks,
                                          ArrayList<String> subTokenizations) {
        StringTokenizer st = new StringTokenizer(result, "\n");
        //System.out.println(result);
        //System.out.println(tokenizations.toString());
        String lastLabel = null;
        int lineCount = 0;
        int p = 0;
        while (st.hasMoreTokens() && (p < tokenizations.size())) {
            String toke = tokenizations.get(p);
            List<String> tokes = new ArrayList<String>();
            while ((toke.equals(" ") || toke.equals("\n") || (toke.length() == 0)) && ((p+1) < tokenizations.size())) {
                p++;
                if (toke.length() == 0) {
                    toke = tokenizations.get(p);
                    continue;
                }
                tokes.add(toke);
                if (toke.equals("\n")) {
                    affiliationBlocks.add("@newline");
                }
                toke = tokenizations.get(p);
            }

            String line = st.nextToken();
            if (line.length() == 0) {
                continue;
            }

            if (line.trim().length() == 0) {
                affiliationBlocks.add("\n");
                //subTokenizations.add(" ");
                lastLabel = null;
{'fraction_non_alphanumeric': 0.11401073477697575, 'fraction_numerical': 0.0020359059781602813, 'mean_word_length': 1.8635400105988342, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1568733', 'n_tokens_mistral': 2763, 'n_tokens_neox': 2613, 'n_words': 609}