<commit_before>package pl.edu.icm.cermine.bibref.extraction.model;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import pl.edu.icm.cermine.structure.model.BxLine;
import pl.edu.icm.cermine.structure.model.BxZone;

/**
 * Stores BxDocument's bibliographic references lines with their labels.
 *
 * @author <NAME> (<EMAIL>)
 */
public class BxDocumentBibReferences {

    private static final int MAX_TITLE_LENGTH = 30;
    
    /** A list of references' lines */
    private List<BxLine> lines = new ArrayList<BxLine>();

    /** A map associating references' line with their zones */
    private Map<BxLine, BxZone> lineZones = new HashMap<BxLine, BxZone>();

    /** A map associating references' lines with their labels */
    private Map<BxLine, BibReferenceLineLabel> lineLabels = new HashMap<BxLine, BibReferenceLineLabel>();

    public void addZone(BxZone zone) {
        for (BxLine line : zone.getLines()) {
            String normalized = line.toText().toLowerCase().replaceAll("[^a-z]", "");
            if (line.toText().length() < MAX_TITLE_LENGTH && zone.getLines().indexOf(line) == 0 && 
                    (normalized.startsWith("refer") || normalized.startsWith("biblio")
                    || normalized.startsWith("acknowled"))) {
                continue;
            }
            if (line.toText().replaceAll("[\\s\\u00A0]", "").isEmpty()) {
                continue;
            }
                   
            lines.add(line);
            lineZones.put(line, zone);
        }
    }

    public List<BxLine> getLines() {
        return lines;
    }

    public BxZone getZone(BxLine line) {
        return lineZones.get(line);
    }

    public BibReferenceLineLabel getLabel(BxLine line) {
        return lineLabels.get(line);<commit_msg>lines before "references" line are removed<commit_after>package pl.edu.icm.cermine.bibref.extraction.model;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import pl.edu.icm.cermine.structure.model.BxLine;
import pl.edu.icm.cermine.structure.model.BxZone;

/**
 * Stores BxDocument's bibliographic references lines with their labels.
 *
 * @author <NAME> (<EMAIL>)
 */
public class BxDocumentBibReferences {

    private static final int MAX_TITLE_LENGTH = 30;
    
    /** A list of references' lines */
    private List<BxLine> lines = new ArrayList<BxLine>();

    /** A map associating references' line with their zones */
    private Map<BxLine, BxZone> lineZones = new HashMap<BxLine, BxZone>();

    /** A map associating references' lines with their labels */
    private Map<BxLine, BibReferenceLineLabel> lineLabels = new HashMap<BxLine, BibReferenceLineLabel>();

    public void addZone(BxZone zone) {
        for (BxLine line : zone.getLines()) {
            String normalized = line.toText().toLowerCase().replaceAll("[^a-z]", "");
            if (line.toText().length() < MAX_TITLE_LENGTH && zone.getLines().indexOf(line) == 0 && 
                    (normalized.startsWith("refer") || normalized.startsWith("biblio")
                    || normalized.startsWith("acknowled") || normalized.startsWith("conclus"))) {
                lines.clear();
                lineZones.clear();
                continue;
            }
            if (line.toText().replaceAll("[\\s\\u00A0]", "").isEmpty()) {
                continue;
            }
                   
            lines.add(line);
            lineZones.put(line, zone);
        }
    }

    public List<BxLine> getLines() {
        return lines;
    }

    public BxZone getZone(BxLine line) {
        return lineZones.get(line);
    }

    public BibReferenceLineLabel getLabel(BxLine line) {
        return lineLabels.get(line);
{'fraction_non_alphanumeric': 0.12914346327234155, 'fraction_numerical': 0.0031821797931583136, 'mean_word_length': 2.5821462488129154, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6445356', 'n_tokens_mistral': 1151, 'n_tokens_neox': 1104, 'n_words': 243}