<commit_before>package org.grobid.core.layout;

import org.grobid.core.utilities.UnicodeUtil;
import org.grobid.core.engines.label.TaggingLabel;
import java.util.ArrayList;
import java.util.List;

/**
 * Class for representing the layout information associated to a PDF object.
 *
 * @author <NAME>
 */
public class LayoutToken implements Comparable<LayoutToken> {
    private String text = null;
    public double y = -1.0;
    public double x = -1.0;
    public double width = 0.0;
    public double height = 0.0;
    private String font = null;
    private boolean bold = false;
    private boolean italic = false;
    private String colorFont = null;
    public double fontSize = 0.0;
    private boolean rotation = false;
    private int page = -1;
    private boolean newLineAfter;
    private int blockPtr;
	private int offset = 0;
	
	/**
	 * All TaggingLabel accumulated for this token
	 */
	private ArrayList<TaggingLabel> labels = null;

    public LayoutToken() {
    }

    public LayoutToken(String text) {
        this.text = text;
    }

    public LayoutToken(LayoutToken token) {
        this.text = token.text;
        this.y = token.y;
        this.x = token.x;
        this.width = token.width;
        this.height = token.height;
        this.font = token.font;
        this.bold = token.bold;
        this.italic = token.italic;
        this.colorFont = token.colorFont;
        this.fontSize = token.fontSize;
        this.rotation = token.rotation;
        this.page = token.page;
        this.newLineAfter = token.newLineAfter;
        this.blockPtr = token.blockPtr;
        this.offset = token.offset;
    }
    
    public LayoutToken(String text, TaggingLabel label) {
    	this(text);
    	this.addLabel(label);
    }

    public void setFont(String f) {
        font = f;
    }

    public String getFont() {
        return font;
    }

    public void setText(String f) {
        //text = f.replaceAll("\n", "");
		text = f;
    }<commit_msg>Ensure that TaggingLabels in LayoutToken are propagated in case of subtokenization<commit_after>package org.grobid.core.layout;

import org.grobid.core.utilities.UnicodeUtil;
import org.grobid.core.engines.label.TaggingLabel;
import java.util.ArrayList;
import java.util.List;

/**
 * Class for representing the layout information associated to a PDF object.
 *
 * @author <NAME>
 */
public class LayoutToken implements Comparable<LayoutToken> {
    private String text = null;
    public double y = -1.0;
    public double x = -1.0;
    public double width = 0.0;
    public double height = 0.0;
    private String font = null;
    private boolean bold = false;
    private boolean italic = false;
    private String colorFont = null;
    public double fontSize = 0.0;
    private boolean rotation = false;
    private int page = -1;
    private boolean newLineAfter;
    private int blockPtr;
	private int offset = 0;
	
	/**
	 * All TaggingLabel accumulated for this token
	 */
	private ArrayList<TaggingLabel> labels = null;

    public LayoutToken() {
    }

    public LayoutToken(String text) {
        this.text = text;
    }

    public LayoutToken(LayoutToken token) {
        this.text = token.text;
        this.y = token.y;
        this.x = token.x;
        this.width = token.width;
        this.height = token.height;
        this.font = token.font;
        this.bold = token.bold;
        this.italic = token.italic;
        this.colorFont = token.colorFont;
        this.fontSize = token.fontSize;
        this.rotation = token.rotation;
        this.page = token.page;
        this.newLineAfter = token.newLineAfter;
        this.blockPtr = token.blockPtr;
        this.offset = token.offset;

        // deep copy of the TaggingLabel list
        if (token.labels != null) {
            this.labels = new ArrayList<TaggingLabel>();
            for(TaggingLabel l : token.labels) {
                this.labels.add(l);
            }
        }
    }
    
    public LayoutToken(String text, TaggingLabel label) {
    	this(text);
    	this.addLabel(label);
    }

    public void setFont(String f) {
        font = f;
    }

    public String getFont() {
        return font;
    }

    public void setText(String f) {
        //text = f.replaceAll("\n", "");
		text = f;
    }
{'fraction_non_alphanumeric': 0.10103141115799344, 'fraction_numerical': 0.005625879043600563, 'mean_word_length': 2.3103180760279285, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7150495', 'n_tokens_mistral': 1287, 'n_tokens_neox': 1255, 'n_words': 350}