<commit_before>package nl.mpi.kinnate.gedcomimport;

import java.util.ArrayList;

/**
 * Document : GedcomLineStructure Created on : Jul 27, 2012, 6:03:36 PM
 *
 * @author <NAME>
 */
public class GedcomLineStructure extends ImportLineStructure {

    public GedcomLineStructure(String lineString, ArrayList<String> gedcomLevelStrings) throws ImportException {
        super(lineString, gedcomLevelStrings);
        isFileHeader = lineString.startsWith("0 HEAD");
        String[] lineParts = lineString.split(" ", 3);
        gedcomLevel = Integer.parseInt(lineParts[0]);
        if (isFileHeader) {
            currentID = lineParts[1];
            addFieldEntry(lineParts[1], null);
        } else if (!isFileHeader && gedcomLevel == 0) {
            currentID = lineParts[1];
//            currentName = lineParts[2];
            if (lineParts.length > 2) {
                setType(lineParts[2]);
                if (!currentID.startsWith("@") || !currentID.endsWith("@")) {
                    throw new ImportException("Incorrect gedcom identifier format: " + currentID);
                }
            }
        } else {
            if (lineParts.length > 2) {
                addFieldEntry(lineParts[1], lineParts[2]);
            } else {
                addFieldEntry(lineParts[1], "");
            }
        }
        while (gedcomLevelStrings.size() > gedcomLevel) {
            gedcomLevelStrings.remove(gedcomLevelStrings.size() - 1);
        }
        gedcomLevelStrings.add(lineParts[1]);
    }

    private void setType(String getdomType) {
        if (getdomType.equals("NOTE")) {
            entityType = "Gedcom Note";
        } else if (getdomType.equals("FAM")) {
            entityType = "Gedcom Family Group";
        } else if (getdomType.equals("INDI")) {
            // do not set a type for individuals
        } else if (getdomType.equals("OBJE")) {
            entityType = "Resource File";
        } else if (getdomType.equals("REPO")) {
            entityType = "Repository";
        } else if (getdomType.equals("SUBN")) {
            entityType = "Submission";
        } else if (getdomType.equals("SOUR")) {
            entityType = "Source";
        } else if (getdomType.equals("SUBM")) {
            entityType = "Submitter";
        } else {
            entityType = getdomType;
        }
    }

    @Override
    public String getProfileForEntityType(String individualTypeProfile, String otherTypeProfile) {
        if (entityType == null && !isFileHeader) {
            return individualTypeProfile;
        } else {
            return otherTypeProfile;
        }
    }

    @Override
    public String getEscapedLineContents() throws ImportException {
        return getLineContents().replace("@@", "@");
    }

    @Override
    public boolean isContinueLine() {
        return hasCurrentField() && getCurrentField().currentName.equals("CONC");<commit_msg>Corrected some of the import code for the single file export format. Made the identifier creation code on import to be more consistent in its behaviour on re-import. refs #2248 #2097 #2258 #2259<commit_after>package nl.mpi.kinnate.gedcomimport;

import java.util.ArrayList;

/**
 * Document : GedcomLineStructure Created on : Jul 27, 2012, 6:03:36 PM
 *
 * @author <NAME>
 */
public class GedcomLineStructure extends ImportLineStructure {

    public GedcomLineStructure(String lineString, ArrayList<String> gedcomLevelStrings) throws ImportException {
        super(lineString, gedcomLevelStrings);
        isFileHeader = lineString.startsWith("0 HEAD");
        String[] lineParts = lineString.split(" ", 3);
        gedcomLevel = Integer.parseInt(lineParts[0]);
        if (isFileHeader) {
            currentID = lineParts[1];
            addFieldEntry(lineParts[1], null);
        } else if (!isFileHeader && gedcomLevel == 0) {
            currentID = lineParts[1];
//            currentName = lineParts[2];
            if (lineParts.length > 2) {
                setType(lineParts[2]);
                if (!currentID.startsWith("@") || !currentID.endsWith("@")) {
                    throw new ImportException("Incorrect gedcom identifier format: " + currentID);
                }
            }
        } else {
            if (lineParts.length > 2) {
                addFieldEntry(lineParts[1], lineParts[2]);
            } else {
                addFieldEntry(lineParts[1], "");
            }
        }
        while (gedcomLevelStrings.size() > gedcomLevel) {
            gedcomLevelStrings.remove(gedcomLevelStrings.size() - 1);
        }
        gedcomLevelStrings.add(lineParts[1]);
    }

    private void setType(String getdomType) {
        if (getdomType.equals("NOTE")) {
            entityType = "Gedcom Note";
        } else if (getdomType.equals("FAM")) {
            entityType = "Gedcom Family Group";
        } else if (getdomType.equals("INDI")) {
            // do not set a type for individuals
        } else if (getdomType.equals("OBJE")) {
            entityType = "Resource File";
        } else if (getdomType.equals("REPO")) {
            entityType = "Repository";
        } else if (getdomType.equals("SUBN")) {
            entityType = "Submission";
        } else if (getdomType.equals("SOUR")) {
            entityType = "Source";
        } else if (getdomType.equals("SUBM")) {
            entityType = "Submitter";
        } else {
            entityType = getdomType;
        }
    }

    @Override
    public String getProfileForEntityType(String individualTypeProfile, String otherTypeProfile) {
        if (entityType == null && !isFileHeader) {
            return individualTypeProfile;
        } else {
            return otherTypeProfile;
        }
    }

    @Override
    public String getEscapedLineContents() throws ImportException {
        return getLineContents().replace("@@", "@"); // the testged file does not have any instances of &gt; or &lt; so we do not replace these (.replace("&gt;", ">").replace("&lt;", "<");)
    }

    @Override
    public boolean isContinueLine() {
        return hasCurrentField() && getCurrentField().currentName.equals("CONC");
{'fraction_non_alphanumeric': 0.11976439790575916, 'fraction_numerical': 0.011452879581151832, 'mean_word_length': 2.2689839572192514, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1090964', 'n_tokens_mistral': 1724, 'n_tokens_neox': 1670, 'n_words': 385}