<commit_before>package pl.edu.icm.cermine.bibref.transformers;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.jdom.Element;
import org.jdom.Text;
import pl.edu.icm.cermine.bibref.model.BibEntry;
import pl.edu.icm.cermine.exception.TransformationException;
import pl.edu.icm.cermine.tools.transformers.ModelToModelConverter;

/**
 * @author <NAME>
 */
public class NLMElementToBibEntryConverter implements ModelToModelConverter<Element, BibEntry> {

    private static final Map<String, String> NLM_TO_BIBENTRY = new HashMap<String, String>();

    static {
        NLM_TO_BIBENTRY.put("article-title",    BibEntry.FIELD_TITLE);
        NLM_TO_BIBENTRY.put("named-content",    BibEntry.FIELD_CONTENTS);
        NLM_TO_BIBENTRY.put("edition",          BibEntry.FIELD_EDITION);
        NLM_TO_BIBENTRY.put("publisher-name",   BibEntry.FIELD_PUBLISHER);
        NLM_TO_BIBENTRY.put("publisher-loc",    BibEntry.FIELD_LOCATION);
        NLM_TO_BIBENTRY.put("series",           BibEntry.FIELD_SERIES);
        NLM_TO_BIBENTRY.put("source",           BibEntry.FIELD_JOURNAL);
        NLM_TO_BIBENTRY.put("uri",              BibEntry.FIELD_URL);
        NLM_TO_BIBENTRY.put("volume",           BibEntry.FIELD_VOLUME);
        NLM_TO_BIBENTRY.put("year",             BibEntry.FIELD_YEAR);
        NLM_TO_BIBENTRY.put("issue",            BibEntry.FIELD_NUMBER);
    }
    
    @Override
    public BibEntry convert(Element source, Object... hints) throws TransformationException {
        BibEntry bibEntry = new BibEntry(BibEntry.TYPE_ARTICLE);
        String text = source.getValue().trim().replaceAll("\\s+", " ");
        bibEntry.setText(text);
      
        Map<Element, Integer> fIndex = new HashMap<Element, Integer>();
        Map<Element, Integer> lIndex = new HashMap<Element, Integer>();
        
        List content = source.getContent();
        int index = 0;
        for (Object child : content) {
            if (child instanceof Text) {
                String chText = ((Text) child).getTextNormalize();
                int oldLength = text.length();
                text = text.substring(chText.length()).trim();
                index += (oldLength - text.length());
            } else if (child instanceof Element) {
                Element elChild = (Element) child;
                fIndex.put(elChild, index);
                String chText = elChild.getValue().trim().replaceAll("\\s+", " ");
                lIndex.put(elChild, index + chText.length() - 1);
                int oldLength = text.length();
                text = text.substring(chText.length()).trim();
                index += (oldLength - text.length());
            } else {<commit_msg>last index calculation corrected<commit_after>package pl.edu.icm.cermine.bibref.transformers;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.jdom.Element;
import org.jdom.Text;
import pl.edu.icm.cermine.bibref.model.BibEntry;
import pl.edu.icm.cermine.exception.TransformationException;
import pl.edu.icm.cermine.tools.transformers.ModelToModelConverter;

/**
 * @author <NAME>
 */
public class NLMElementToBibEntryConverter implements ModelToModelConverter<Element, BibEntry> {

    private static final Map<String, String> NLM_TO_BIBENTRY = new HashMap<String, String>();

    static {
        NLM_TO_BIBENTRY.put("article-title",    BibEntry.FIELD_TITLE);
        NLM_TO_BIBENTRY.put("named-content",    BibEntry.FIELD_CONTENTS);
        NLM_TO_BIBENTRY.put("edition",          BibEntry.FIELD_EDITION);
        NLM_TO_BIBENTRY.put("publisher-name",   BibEntry.FIELD_PUBLISHER);
        NLM_TO_BIBENTRY.put("publisher-loc",    BibEntry.FIELD_LOCATION);
        NLM_TO_BIBENTRY.put("series",           BibEntry.FIELD_SERIES);
        NLM_TO_BIBENTRY.put("source",           BibEntry.FIELD_JOURNAL);
        NLM_TO_BIBENTRY.put("uri",              BibEntry.FIELD_URL);
        NLM_TO_BIBENTRY.put("volume",           BibEntry.FIELD_VOLUME);
        NLM_TO_BIBENTRY.put("year",             BibEntry.FIELD_YEAR);
        NLM_TO_BIBENTRY.put("issue",            BibEntry.FIELD_NUMBER);
    }
    
    @Override
    public BibEntry convert(Element source, Object... hints) throws TransformationException {
        BibEntry bibEntry = new BibEntry(BibEntry.TYPE_ARTICLE);
        String text = source.getValue().trim().replaceAll("\\s+", " ");
        bibEntry.setText(text);
      
        Map<Element, Integer> fIndex = new HashMap<Element, Integer>();
        Map<Element, Integer> lIndex = new HashMap<Element, Integer>();
        
        List content = source.getContent();
        int index = 0;
        for (Object child : content) {
            if (child instanceof Text) {
                String chText = ((Text) child).getTextNormalize();
                int oldLength = text.length();
                text = text.substring(chText.length()).trim();
                index += (oldLength - text.length());
            } else if (child instanceof Element) {
                Element elChild = (Element) child;
                fIndex.put(elChild, index);
                String chText = elChild.getValue().trim().replaceAll("\\s+", " ");
                lIndex.put(elChild, index + chText.length());
                int oldLength = text.length();
                text = text.substring(chText.length()).trim();
                index += (oldLength - text.length());
            } else {
{'fraction_non_alphanumeric': 0.12294930875576036, 'fraction_numerical': 0.0005529953917050691, 'mean_word_length': 2.53485342019544, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4159562', 'n_tokens_mistral': 1668, 'n_tokens_neox': 1590, 'n_words': 282}