<commit_before># -*- coding: utf-8 -*-
import csv
import datetime


def parse_panel(csv_stream):
    """Parse user submitted panel."""
    reader = csv.DictReader(csv_stream, delimiter=';', quoting=csv.QUOTE_NONE)
    genes = []
    for gene_row in reader:
        transcripts_raw = gene_row.get('Disease_associated_transcript')
        if transcripts_raw:
            transcripts_list = [tx.split(':', 1)[-1].strip() for tx in transcripts_raw.split(',')]
        else:
            transcripts_list = []

        models_raw = gene_row.get('Genetic_disease_model')
        models_list = [model.strip() for model in models_raw.split(',')] if models_raw else []

        panel_gene = dict(
            symbol=gene_row['HGNC_symbol'].strip() if gene_row.get('HGNC_symbol') else None,
            hgnc_id=(int(gene_row['HGNC_IDnumber'].strip()) if gene_row.get('HGNC_IDnumber')
                     else None),
            disease_associated_transcripts=transcripts_list,
            reduced_penetrance=True if gene_row.get('Reduced_penetrance') else None,
            mosaicism=True if gene_row.get('Mosaicism') else None,
            inheritance_models=models_list,
        )
        genes.append(panel_gene)

    return genes


def build_panel(adapter, institute_id, panel_name, display_name, version, panel_genes):
    """Build a new (updated) gene panel."""
    for panel_gene in panel_genes:
        gene_id = panel_gene['hgnc_id'] or panel_gene['symbol']
        hgnc_gene = adapter.hgnc_gene(gene_id)
        if hgnc_gene is None:
            raise ValueError("can't find gene: {}".format(gene_id))
        if panel_gene['symbol'] and (panel_gene['symbol'] != hgnc_gene['hgnc_symbol']):
            raise ValueError("HGNC symbol not matching: {} vs. {}"
                             .format(panel_gene['symbol'], hgnc_gene['hgnc_symbol']))
        panel_gene['symbol'] = hgnc_gene['hgnc_symbol']

    gene_panel = dict(
        institute=institute_id,
        panel_name=panel_name,
        display_name=display_name,
        version=version,
        genes=panel_genes,
        date=datetime.datetime.now(),
    )
    return gene_panel<commit_msg>fill in correct hgnc id for new panel<commit_after># -*- coding: utf-8 -*-
import csv
import datetime


def parse_panel(csv_stream):
    """Parse user submitted panel."""
    reader = csv.DictReader(csv_stream, delimiter=';', quoting=csv.QUOTE_NONE)
    genes = []
    for gene_row in reader:
        transcripts_raw = gene_row.get('Disease_associated_transcript')
        if transcripts_raw:
            transcripts_list = [tx.split(':', 1)[-1].strip() for tx in transcripts_raw.split(',')]
        else:
            transcripts_list = []

        models_raw = gene_row.get('Genetic_disease_model')
        models_list = [model.strip() for model in models_raw.split(',')] if models_raw else []

        panel_gene = dict(
            symbol=gene_row['HGNC_symbol'].strip() if gene_row.get('HGNC_symbol') else None,
            hgnc_id=(int(gene_row['HGNC_IDnumber'].strip()) if gene_row.get('HGNC_IDnumber')
                     else None),
            disease_associated_transcripts=transcripts_list,
            reduced_penetrance=True if gene_row.get('Reduced_penetrance') else None,
            mosaicism=True if gene_row.get('Mosaicism') else None,
            inheritance_models=models_list,
        )
        genes.append(panel_gene)

    return genes


def build_panel(adapter, institute_id, panel_name, display_name, version, panel_genes):
    """Build a new (updated) gene panel."""
    for panel_gene in panel_genes:
        gene_id = panel_gene['hgnc_id'] or panel_gene['symbol']
        hgnc_gene = adapter.hgnc_gene(gene_id)
        if hgnc_gene is None:
            raise ValueError("can't find gene: {}".format(gene_id))
        if panel_gene['symbol'] and (panel_gene['symbol'] != hgnc_gene['hgnc_symbol']):
            raise ValueError("HGNC symbol not matching: {} vs. {}"
                             .format(panel_gene['symbol'], hgnc_gene['hgnc_symbol']))
        panel_gene['symbol'] = hgnc_gene['hgnc_symbol']
        panel_gene['hgnc_id'] = hgnc_gene['hgnc_id']

    gene_panel = dict(
        institute=institute_id,
        panel_name=panel_name,
        display_name=display_name,
        version=version,
        genes=panel_genes,
        date=datetime.datetime.now(),
    )
    return gene_panel
{'fraction_non_alphanumeric': 0.1194882339501942, 'fraction_numerical': 0.0013708019191226869, 'mean_word_length': 2.905441570026762, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '87995', 'n_tokens_mistral': 1457, 'n_tokens_neox': 1350, 'n_words': 268}