<commit_before>#!/usr/bin/env python

# native Python imports
import os.path
import sys
import sqlite3
import numpy as np
from itertools import repeat
import json
import shutil
import uuid

# third-party imports
import cyvcf as vcf

# gemini modules
import version
from ped import default_ped_fields, load_ped_file
import gene_table
import infotag
import database
import annotations
import func_impact
import severe_impact
import popgen
import structural_variants as svs
from gemini_constants import *
from compression import pack_blob
from gemini.config import read_gemini_config

def get_phred_lik(gt_phred_likelihoods, dtype=np.int32, empty_val=-1):
    """
    Force each sample to have 3 GL's (0/0, 0/1, 1/1).
    If no samples have GL's, then we just return None to save space.
    """
    out = []
    all_empty = True
    empty_line = [empty_val] * 3
    for row in gt_phred_likelihoods:
        # we only try to use the correct PL's if it already has size 3
        if row is not None and len(row) == 3:
            out.append([int(v) if v is not None else empty_val for v in row])
            all_empty = False
        else:
            out.append(empty_line)
    if all_empty:
        return None
    return np.array(out, dtype=dtype)

class GeminiLoader(object):
    """
    Object for creating and populating a gemini
    database and auxillary data files.
    """
    def __init__(self, args, buffer_size=10000):
        self.args = args
        self.seen_multi = False

        # create the gemini database
        self._create_db()
        # create a reader for the VCF file
        self.vcf_reader = self._get_vcf_reader()
        # load sample information

        if not self.args.no_genotypes and not self.args.no_load_genotypes:
            # load the sample info from the VCF file.<commit_msg>Catch issues where phred_likelihoods are not correctly specified in VCF as a list<commit_after>#!/usr/bin/env python

# native Python imports
import os.path
import sys
import sqlite3
import numpy as np
from itertools import repeat
import json
import shutil
import uuid

# third-party imports
import cyvcf as vcf

# gemini modules
import version
from ped import default_ped_fields, load_ped_file
import gene_table
import infotag
import database
import annotations
import func_impact
import severe_impact
import popgen
import structural_variants as svs
from gemini_constants import *
from compression import pack_blob
from gemini.config import read_gemini_config

def get_phred_lik(gt_phred_likelihoods, dtype=np.int32, empty_val=-1):
    """
    Force each sample to have 3 GL's (0/0, 0/1, 1/1).
    If no samples have GL's, then we just return None to save space.
    """
    out = []
    all_empty = True
    empty_line = [empty_val] * 3
    for row in gt_phred_likelihoods:
        # we only try to use the correct PL's if it already has size 3
        if row is not None and isinstance(row, (list, tuple)) and len(row) == 3:
            out.append([int(v) if v is not None else empty_val for v in row])
            all_empty = False
        else:
            out.append(empty_line)
    if all_empty:
        return None
    return np.array(out, dtype=dtype)

class GeminiLoader(object):
    """
    Object for creating and populating a gemini
    database and auxillary data files.
    """
    def __init__(self, args, buffer_size=10000):
        self.args = args
        self.seen_multi = False

        # create the gemini database
        self._create_db()
        # create a reader for the VCF file
        self.vcf_reader = self._get_vcf_reader()
        # load sample information

        if not self.args.no_genotypes and not self.args.no_load_genotypes:
            # load the sample info from the VCF file.
{'fraction_non_alphanumeric': 0.06321993035092419, 'fraction_numerical': 0.01017948031074203, 'mean_word_length': 2.9513227513227513, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2822359', 'n_tokens_mistral': 1153, 'n_tokens_neox': 1112, 'n_words': 423}