<commit_before>package org.opencb.biodata.models.variant;

import htsjdk.variant.variantcontext.Allele;
import htsjdk.variant.variantcontext.VariantContext;
import htsjdk.variant.vcf.VCFConstants;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.opencb.biodata.models.core.Region;
import org.opencb.biodata.models.variant.avro.*;
import org.opencb.biodata.models.variant.protobuf.VariantProto;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;
import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Variant builder.
 * <p>
 * Builder for a single-study single-file variant.
 */
public class VariantBuilder {

    // Known INFO gags
    private static final String CIPOS_INFO = "CIPOS";
    private static final String CIEND_INFO = "CIEND";
    private static final String SVINSSEQ_INFO = "SVINSSEQ";
    private static final String LEFT_SVINSSEQ_INFO = "LEFT_SVINSSEQ";
    private static final String RIGHT_SVINSSEQ_INFO = "RIGHT_SVINSSEQ";
    private static final String END_INFO = "END";

    // Known FORMAT tags
    private static final String COPY_NUMBER_FORMAT = "CN";

    // Known symbolic alternates
    // TODO: Support alternates like DEL:ME, INS:ME, ...
    private static final String CNV_PREFIX_ALT = "<CN";
    private static final String CNV_ALT = "<CNV>";
    private static final String DUP_ALT = "<DUP>";
    private static final String DUP_TANDEM_ALT = "<DUP:TANDEM>";
    private static final String DEL_ALT = "<DEL>";
    private static final String INV_ALT = "<INV>";
    private static final String INS_ALT = "<INS>";
    private static final Pattern CNV_ALT_PATTERN = Pattern.compile("<CN([0-9]+)>");

    private static final Set<String> VALID_NTS = new HashSet<>(Arrays.asList("A", "C", "G", "T", "N"));
    protected static final String VARIANT_STRING_FORMAT
            = "(chr)"
            + ":[(cipos_left)<](start)[<(cipos_right)]" + "[-[(ciend_left)<](end)[<(ciend_right)]]"
            + "[:(ref)]"
            + ":[(alt)|(left_ins_seq)...(right_ins_seq)]";

    private static final EnumSet<VariantType> SV_TYPES;
    // Variant types where the reference is incomplete.
    private static final EnumSet<VariantType> INCOMPLETE_REFERENCE_TYPES;
    protected static final String DUP_ALT_EXTENDED = "<DUP:";
    protected static final String DEL_ALT_EXTENDED = "<DEL:";
    protected static final String INV_ALT_EXTENDED = "<INV:";
    protected static final String INS_ALT_EXTENDED = "<INS:";
    private static final Pattern BREAKEND_MATED_PATTERN = Pattern.compile("(.*)([\\[\\]])(.+):(\\p{Digit}+)([\\[\\]])(.*)");

    protected static Logger logger = LoggerFactory.getLogger(VariantBuilder.class);

    static {
        SV_TYPES = EnumSet.copyOf(Variant.SV_SUBTYPES);
        SV_TYPES.add(VariantType.SV);
        SV_TYPES.add(VariantType.SYMBOLIC);

        INCOMPLETE_REFERENCE_TYPES = EnumSet.copyOf(SV_TYPES);
        INCOMPLETE_REFERENCE_TYPES.remove(VariantType.INSERTION);
        INCOMPLETE_REFERENCE_TYPES.add(VariantType.NO_VARIATION);
    }

    private String id;
    private List<String> names;
    private String chromosome;
    private Integer start;
    private Integer end;
    private Integer length;
    private String reference;
    private ArrayList<String> alternates;
    private VariantType type;
    private StructuralVariation sv;
    private String strand = "+";
    private String studyId;
    private String fileId;
    private LinkedHashMap<String, Integer> samplesPosition;
    private List<List<String>> samplesData;
    private Map<String, String> attributes;
    private String call;
    private List<String> format;

    private String variantString;

    public VariantBuilder() {
    }

    /**
     * Creates a variant parsing a string.
     *
     * Format : (chr):[(cipos_left)<](start)[<(cipos_right)][-[(ciend_left)<](end)[<(ciend_right)]][:(ref)]:(alt)
     *
     * @param variantString Variant string
     * @throws IllegalArgumentException if the variant does not match with the pattern
     */
    public VariantBuilder(String variantString) {
        this();
        this.variantString = variantString;
        if (variantString != null && !variantString.isEmpty()) {
            String[] fields;
            int idx;
            // Symbolic alternates may use ':' within the alternate. If ends with ">", is a symbolic variant
            // Get last index of '<'. Start and end may use '<' for imprecise positions.
            if (variantString.endsWith(">")) {
                idx = variantString.lastIndexOf("<");
                String[] split = variantString.substring(0, idx - 1).split(":", -1);
                fields = new String[split.length + 1];
                for (int i = 0; i < split.length; i++) {
                    fields[i] = split[i];
                }
                fields[fields.length - 1] = variantString.substring(idx);
            } else {
                idx = StringUtils.indexOfAny(variantString, '[', ']');
                if (idx != StringUtils.INDEX_NOT_FOUND) {
                    fields = variantString.split(":", 4);
                } else {
                    fields = variantString.split(":", -1);
                }
            }
            if (fields.length == 3) {
                setChromosome(fields[0]);
                parseAlternate(fields[2]);
                setReference("");

                // Structural variant (except <INS>) needs start-end coords
                if (fields[1].contains("-")) {
                    String[] coordinatesParts = fields[1].split("-");
                    parseStart(coordinatesParts[0], variantString);
                    parseEnd(coordinatesParts[1], variantString);

                    // Short variant or <INS>, no reference specified
                } else {
                    parseStart(fields[1], variantString);
                }
            } else if (fields.length == 4) {
                setChromosome(fields[0]);
                setReference(fields[2]);<commit_msg>models: Improve VariantBuilder parse string for multiallelic symbolic and BND<commit_after>package org.opencb.biodata.models.variant;

import htsjdk.variant.variantcontext.Allele;
import htsjdk.variant.variantcontext.VariantContext;
import htsjdk.variant.vcf.VCFConstants;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.opencb.biodata.models.core.Region;
import org.opencb.biodata.models.variant.avro.*;
import org.opencb.biodata.models.variant.protobuf.VariantProto;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;
import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Variant builder.
 * <p>
 * Builder for a single-study single-file variant.
 */
public class VariantBuilder {

    // Known INFO gags
    private static final String CIPOS_INFO = "CIPOS";
    private static final String CIEND_INFO = "CIEND";
    private static final String SVINSSEQ_INFO = "SVINSSEQ";
    private static final String LEFT_SVINSSEQ_INFO = "LEFT_SVINSSEQ";
    private static final String RIGHT_SVINSSEQ_INFO = "RIGHT_SVINSSEQ";
    private static final String END_INFO = "END";

    // Known FORMAT tags
    private static final String COPY_NUMBER_FORMAT = "CN";

    // Known symbolic alternates
    // TODO: Support alternates like DEL:ME, INS:ME, ...
    private static final String CNV_PREFIX_ALT = "<CN";
    private static final String CNV_ALT = "<CNV>";
    private static final String DUP_ALT = "<DUP>";
    private static final String DUP_TANDEM_ALT = "<DUP:TANDEM>";
    private static final String DEL_ALT = "<DEL>";
    private static final String INV_ALT = "<INV>";
    private static final String INS_ALT = "<INS>";
    private static final Pattern CNV_ALT_PATTERN = Pattern.compile("<CN([0-9]+)>");

    private static final Set<String> VALID_NTS = new HashSet<>(Arrays.asList("A", "C", "G", "T", "N"));
    protected static final String VARIANT_STRING_FORMAT
            = "(chr)"
            + ":[(cipos_left)<](start)[<(cipos_right)]" + "[-[(ciend_left)<](end)[<(ciend_right)]]"
            + "[:(ref)]"
            + ":[(alt)|(left_ins_seq)...(right_ins_seq)]";

    private static final EnumSet<VariantType> SV_TYPES;
    // Variant types where the reference is incomplete.
    private static final EnumSet<VariantType> INCOMPLETE_REFERENCE_TYPES;
    protected static final String DUP_ALT_EXTENDED = "<DUP:";
    protected static final String DEL_ALT_EXTENDED = "<DEL:";
    protected static final String INV_ALT_EXTENDED = "<INV:";
    protected static final String INS_ALT_EXTENDED = "<INS:";
    private static final Pattern BREAKEND_MATED_PATTERN = Pattern.compile("(.*)([\\[\\]])(.+):(\\p{Digit}+)([\\[\\]])(.*)");

    protected static Logger logger = LoggerFactory.getLogger(VariantBuilder.class);

    static {
        SV_TYPES = EnumSet.copyOf(Variant.SV_SUBTYPES);
        SV_TYPES.add(VariantType.SV);
        SV_TYPES.add(VariantType.SYMBOLIC);

        INCOMPLETE_REFERENCE_TYPES = EnumSet.copyOf(SV_TYPES);
        INCOMPLETE_REFERENCE_TYPES.remove(VariantType.INSERTION);
        INCOMPLETE_REFERENCE_TYPES.add(VariantType.NO_VARIATION);
    }

    private String id;
    private List<String> names;
    private String chromosome;
    private Integer start;
    private Integer end;
    private Integer length;
    private String reference;
    private ArrayList<String> alternates;
    private VariantType type;
    private StructuralVariation sv;
    private String strand = "+";
    private String studyId;
    private String fileId;
    private LinkedHashMap<String, Integer> samplesPosition;
    private List<List<String>> samplesData;
    private Map<String, String> attributes;
    private String call;
    private List<String> format;

    private String variantString;

    public VariantBuilder() {
    }

    /**
     * Creates a variant parsing a string.
     *
     * Format : (chr):[(cipos_left)<](start)[<(cipos_right)][-[(ciend_left)<](end)[<(ciend_right)]][:(ref)]:(alt)
     *
     * @param variantString Variant string
     * @throws IllegalArgumentException if the variant does not match with the pattern
     */
    public VariantBuilder(String variantString) {
        this();
        this.variantString = variantString;
        if (variantString != null && !variantString.isEmpty()) {
            String[] fields;

            // Symbolic and breakend variants may use ':' within the alternate.
            //  If contains '>', is a symbolic variant
            //  If contains ']' or "[", is a breakend
            // Split in 4 segments. If reference (field[2]) contains a '<', '>', ']', '[', the reference was missing,
            // so it has to split in 3 segments.
            // Get last index of '<'. Start and end may use '<' for imprecise positions.
            if (StringUtils.containsAny(variantString, '>', ']', '[')) {
                fields = variantString.split(":", 4);
                if (fields.length == 4 && StringUtils.containsAny(fields[2], '<', '>', ']', '[')) {
                    fields = variantString.split(":", 3);
                }
            } else {
                fields = variantString.split(":", -1);
            }
            if (fields.length == 3) {
                setChromosome(fields[0]);
                parseAlternate(fields[2]);
                setReference("");

                // Structural variant (except <INS>) needs start-end coords
                if (fields[1].contains("-")) {
                    String[] coordinatesParts = fields[1].split("-");
                    parseStart(coordinatesParts[0], variantString);
                    parseEnd(coordinatesParts[1], variantString);

                    // Short variant or <INS>, no reference specified
                } else {
                    parseStart(fields[1], variantString);
                }
            } else if (fields.length == 4) {
                setChromosome(fields[0]);
                setReference(fields[2]);
{'fraction_non_alphanumeric': 0.12268480576954598, 'fraction_numerical': 0.003933781347320112, 'mean_word_length': 2.781530833591571, 'pattern_counts': {'":': 10, '<': 79, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6723990', 'n_tokens_mistral': 3645, 'n_tokens_neox': 3492, 'n_words': 875}