<commit_before>package org.grobid.core.engines;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;
import org.grobid.core.GrobidModels;
import org.grobid.core.data.BibDataSet;
import org.grobid.core.data.BiblioItem;
import org.grobid.core.data.Date;
import org.grobid.core.document.Document;
import org.grobid.core.exceptions.GrobidException;
import org.grobid.core.features.FeaturesVectorCitation;
import org.grobid.core.lexicon.Lexicon;
import org.grobid.core.utilities.Consolidation;
import org.grobid.core.utilities.GrobidProperties;
import org.grobid.core.utilities.OffsetPosition;
import org.grobid.core.utilities.TextUtilities;

/**
 * @author <NAME>
 */
public class CitationParser extends AbstractParser {
	private AuthorParser authorParser = null;
	private DateParser dateParser = null;
	private Consolidation consolidator = null;

	private File tmpPath = null;
	private String pathXML = null;

	public Lexicon lexicon = Lexicon.getInstance();

	public CitationParser() {
		super(GrobidModels.CITATION);
		tmpPath = GrobidProperties.getTempPath();
	}

	public String taggerRun(String ress) throws Exception {
		// clear internal context
		StringTokenizer st = new StringTokenizer(ress, "\n");
		feedTaggerAndParse(st);

		StringBuilder res = new StringBuilder();
		for (int i = 0; i < tagger.size(); i++) {
			for (int j = 0; j < tagger.xsize(); j++) {
				res.append(tagger.x(i, j)).append("\t");
			}
			res.append(tagger.y2(i));
			res.append("\n");
		}

		return res.toString();
	}

	public BiblioItem processing(String input, boolean consolidate) {
		BiblioItem resCitation;
		if (StringUtils.isBlank(input)) {
			return null;
		}

		try {
			ArrayList<String> citationBlocks = new ArrayList<String>();

			input = TextUtilities.dehyphenize(input);
			input = input.replace("\n", " ");
			input = input.replaceAll("\\p{Cntrl}"," ").trim();
			StringTokenizer st = new StringTokenizer(input,
					TextUtilities.fullPunctuations, true);

			if (st.countTokens() == 0)
				return null;

			ArrayList<String> tokenizations = new ArrayList<String>();
			while (st.hasMoreTokens()) {
				String tok = st.nextToken();
				tokenizations.add(tok);
				if (!tok.equals(" ")) {
					citationBlocks.add(tok + " <citation>");
				}
			}
			citationBlocks.add("\n");

			List<List<OffsetPosition>> journalsPositions = new ArrayList<List<OffsetPosition>>();
			List<List<OffsetPosition>> abbrevJournalsPositions = new ArrayList<List<OffsetPosition>>();
			List<List<OffsetPosition>> conferencesPositions = new ArrayList<List<OffsetPosition>>();
			List<List<OffsetPosition>> publishersPositions = new ArrayList<List<OffsetPosition>>();

			journalsPositions.add(lexicon.inJournalNames(input));
			abbrevJournalsPositions.add(lexicon.inAbbrevJournalNames(input));
			conferencesPositions.add(lexicon.inConferenceNames(input));
			publishersPositions.add(lexicon.inPublisherNames(input));

			String ress = FeaturesVectorCitation.addFeaturesCitation(
					citationBlocks, journalsPositions, abbrevJournalsPositions,
					conferencesPositions, publishersPositions);
			String res = taggerRun(ress);<commit_msg>Code quality update.<commit_after>package org.grobid.core.engines;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;
import org.grobid.core.GrobidModels;
import org.grobid.core.data.BibDataSet;
import org.grobid.core.data.BiblioItem;
import org.grobid.core.data.Date;
import org.grobid.core.document.Document;
import org.grobid.core.exceptions.GrobidException;
import org.grobid.core.features.FeaturesVectorCitation;
import org.grobid.core.lexicon.Lexicon;
import org.grobid.core.utilities.Consolidation;
import org.grobid.core.utilities.GrobidProperties;
import org.grobid.core.utilities.OffsetPosition;
import org.grobid.core.utilities.TextUtilities;

/**
 * @author <NAME>
 */
public class CitationParser extends AbstractParser {
	private AuthorParser authorParser = null;
	private DateParser dateParser = null;
	private Consolidation consolidator = null;

	private File tmpPath = null;
	private String pathXML = null;

	public Lexicon lexicon = Lexicon.getInstance();

	public CitationParser() {
		super(GrobidModels.CITATION);
		tmpPath = GrobidProperties.getTempPath();
	}

	public String taggerRun(String ress) throws Exception {
		// clear internal context
		StringTokenizer st = new StringTokenizer(ress, "\n");
		feedTaggerAndParse(st);

		StringBuilder res = new StringBuilder();
		for (int i = 0; i < tagger.size(); i++) {
			for (int j = 0; j < tagger.xsize(); j++) {
				res.append(tagger.x(i, j)).append("\t");
			}
			res.append(tagger.y2(i));
			res.append("\n");
		}

		return res.toString();
	}

	public BiblioItem processing(String input, boolean consolidate) {
		BiblioItem resCitation;
		if (StringUtils.isBlank(input)) {
			return null;
		}

		try {
			ArrayList<String> citationBlocks = new ArrayList<String>();

			input = TextUtilities.dehyphenize(input);
			input = input.replace("\n", " ");
			input = input.replaceAll("\\p{Cntrl}"," ").trim();
			StringTokenizer st = new StringTokenizer(input,
					TextUtilities.fullPunctuations, true);

			if (st.countTokens() == 0)
				return null;

			ArrayList<String> tokenizations = new ArrayList<String>();
			while (st.hasMoreTokens()) {
				final String tok = st.nextToken();
				tokenizations.add(tok);
				if (!tok.equals(" ")) {
					citationBlocks.add(tok + " <citation>");
				}
			}
			citationBlocks.add("\n");

			List<List<OffsetPosition>> journalsPositions = new ArrayList<List<OffsetPosition>>();
			List<List<OffsetPosition>> abbrevJournalsPositions = new ArrayList<List<OffsetPosition>>();
			List<List<OffsetPosition>> conferencesPositions = new ArrayList<List<OffsetPosition>>();
			List<List<OffsetPosition>> publishersPositions = new ArrayList<List<OffsetPosition>>();

			journalsPositions.add(lexicon.inJournalNames(input));
			abbrevJournalsPositions.add(lexicon.inAbbrevJournalNames(input));
			conferencesPositions.add(lexicon.inConferenceNames(input));
			publishersPositions.add(lexicon.inPublisherNames(input));

			String ress = FeaturesVectorCitation.addFeaturesCitation(
					citationBlocks, journalsPositions, abbrevJournalsPositions,
					conferencesPositions, publishersPositions);
			String res = taggerRun(ress);
{'fraction_non_alphanumeric': 0.12713529856387, 'fraction_numerical': 0.0015117157974300832, 'mean_word_length': 5.097695852534562, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4333770', 'n_tokens_mistral': 2308, 'n_tokens_neox': 2019, 'n_words': 398}