<commit_before>"""
Created on June 6, 2016

@author: <NAME> (<EMAIL>)
"""

import os, sys
import pandas as pd 
import yaml
from string import Template
from collections import OrderedDict

#===============
# Django imports
#---------------
from .models import Sequencing, SublibraryInformation
from django.conf import settings
from django.shortcuts import get_object_or_404

#==================================================
# upload, parse and populate Sublibrary Information
#--------------------------------------------------
def parse_smartchipapp_file(csv_infile):
    """parse the result file of SmartChipApp."""
    df = pd.read_excel(csv_infile, sheetname=0)
    ## filter out the cells whose Spot_Well value is not NaN
    df = df[~df['Spot_Well'].isnull()]
    df = df.sort_values(by='Sample')
    ## change the column names to match the filed names of the model
    df.columns = [c.lower() for c in df.columns]
    return df

def create_sublibrary_information(s):
    """create SublibraryInformation instance from given pandas Series."""
    d = s.to_dict()
    instance = SublibraryInformation(**d)
    return instance

def bulk_create_sublibrary(library, df):
    """add sublirary information in df for given library instance."""
    num_sublibraries = df.size/len(df.columns)

    ## delete previous sublibraryinformation_set
    library.sublibraryinformation_set.all().delete()

    instances = []
    for instance in df.apply(create_sublibrary_information, axis=1):
        instance.library_id = library.pk
        instances.append(instance)

    library.sublibraryinformation_set.bulk_create(instances)
    return num_sublibraries


#=================
# history manager
#-----------------
class HistoryManager(object):

    """
    An api for simple_history app.
    """

    @staticmethod
    def print_history(object, history_type=None):
        print '=' * 100
        print "Object\tID\tDate\tAction\tUser"
        print '=' * 100
        if history_type is None:
            histories = object.history.all()
        else:
            histories = object.history.filter(history_type=history_type)

        for h in histories:
            print "\t".join([
                str(h.instance),
                str(h.instance.id),
                str(h.history_date),
                h.get_history_type_display(),
                str(h.history_user),
                ])
            print '-' * 100

#======================
# generate sample sheet
#----------------------
def generate_samplesheet(pk):
    """generate samplesheet for the given Sequencing using template in SC-180."""
    samplesheet = SampleSheet(pk)
    sheet_name = samplesheet.sheet_name
    ofilename = os.path.join(settings.MEDIA_ROOT, sheet_name)
    samplesheet.write_header(ofilename)
    samplesheet.write_data(ofilename)
    return sheet_name, os.path.abspath(ofilename)

class SampleSheet(object):

    """
    Sequencing SampleSheet.
    """

    def __init__(self, pk):
        self._sequencing = get_object_or_404(Sequencing, pk=pk)
        self._sheet_name = str(self._sequencing) + '_samplesheet.csv'
        self._header = os.path.join(settings.BASE_DIR,
            "templates/template_samplesheet_header.html")
        self._colnames = [
        'Sample_ID',
        'Sample_Name',
        'Sample_Plate',
        'Sample_Well',
        'I7_Index_ID',
        'index',
        'I5_Index_ID',
        'index2',
        'Sample_Project',
        'Description'
        ]

    @property
    def sequencing(self):
        return self._sequencing

    @property
    def sheet_name(self):
        return self._sheet_name

    def write_header(self, ofilename):
        """write the header section of the sequencing SampleSheet."""
        with open(self._header, 'r') as tempstr:
            s = Template(tempstr.read())
            d = {
            'sequencing_instrument': self._sequencing.get_sequencing_instrument_display(),
            'submission_date': self._sequencing.submission_date,
            'pool_id': self._sequencing.library.pool_id,
            'read1_length': self._sequencing.read1_length,
            'read2_length': self._sequencing.read2_length,
            }

            # Sequencing may have no SequencingDetail
            try:
                d['flow_cell_id'] = self._sequencing.sequencingdetail.flow_cell_id
            except:
                d['flow_cell_id'] = None

            s = s.safe_substitute(**d)
            ofile = open(ofilename, 'w')
            ofile.write(s)
            ofile.close()

    def write_data(self, ofilename):
        """write the data section of the sequencing SampleSheet."""
        data_table = self._mk_data_table()

        # reorder the columns
        data_table = data_table[self._colnames]
        data_table.to_csv(ofilename, mode='a', index=False)

    def _mk_data_table(self):
        """make data table for data section of the samplesheet template."""
        def _map_to_template(s):
            d = s.to_dict()
            # This is the relation between columns in the template samplesheet
            # and the actual columns in df from LIMS.
            res = {
            'Sample_ID': '-'.join([
            str(self._sequencing.library.sample),
            str(self._sequencing.library.pool_id),
            'R'+ str(d['row']),
            'C'+ str(d['column'])
            ]),
            'Sample_Name': '',
            'Sample_Plate': 'R' + str(d['row']) + '_C' + str(d['column']),
            'Sample_Well': 'R' + str(d['row']) + '_C' + str(d['img_col']),
            'I7_Index_ID': d['index_i7'],
            'index': d['primer_i7'],
            'I5_Index_ID': d['index_i5'],
            'index2': d['primer_i5'],
            #'Description': 'CC=<cell call number>;EC=<experimental condition letter>',
            'Description': 'CC=' + d['pick_met'] + ';' + 'EC=' + d['spot_class'],
            }
            return res

        sample_project = '' #','.join(sequencing.library.projects.names())
        newl = []
        oldl = list(self._sequencing.library.sublibraryinformation_set.values())
        df = pd.DataFrame(oldl)
        for d in df.apply(_map_to_template, axis=1):
            d['Sample_Project'] = sample_project
            newl.append(d)
        return pd.DataFrame(newl)


#=============================
# generate GSC submission form
#-----------------------------
def generate_gsc_form(pk, metadata):
    """generate the GSC submission form for the given library."""
    gsc_form = GSCForm(pk)
    pool_df = gsc_form.meta_df
    sample_df = gsc_form.data_df
    header1 = gsc_form.meta_header
    header2 = gsc_form.data_header
    form_name = gsc_form.form_name
    ofilename = os.path.join(settings.MEDIA_ROOT, form_name)

    workbook = Submission(pool_df, sample_df, ofilename)
    workbook.set_column_width()
    workbook.write_address_box(metadata)
    workbook.write_pool_header(header1)
    workbook.write_sample_header(header2)
    workbook.close()
    return form_name, os.path.abspath(ofilename)

class GSCForm(object):

    """
    GSC sequencing submission form.
    """

    def __init__(self, pk):
        self._sequencing = get_object_or_404(Sequencing, pk=pk)
        self._library = self._sequencing.library
        self._libconst = self._library.libraryconstructioninformation
        self._libquant = self._library.libraryquantificationandstorage
        self._sample = self._library.sample
        self._sample_addinfo = self._sample.additionalsampleinformation
        self._form_name = str(self._sequencing) + '_gsc_submission_form.xlsx'
        self._meta_header = os.path.join(settings.BASE_DIR,
            "templates/template_gsc_meta_header.html")
        self._data_header = os.path.join(settings.BASE_DIR,
            "templates/template_gsc_data_header.html")
        self._meta_colnames = [
        'POOL ID',
        'Tube Label',
        'Taxonomy ID',
        'DNA Volume (uL)',
        'DNA Concentration (nM)',
        'Storage Medium',
        'Quantification Method',
        'Library Type',
        'Library Construction Method',
        'Size Range (bp)',
        'Average Size (bp)',
        'Number of libraries in pool',
        'Read Type',
        'Read Length',
        'Sequencing Goal',
        'Reference genome for alignment',
        'Format for data dissemination',
        'Additional comments',
        ]
        self._data_colnames = [
        'Sub-Library ID',
        'Tube Label',
        'Taxonomy ID',
        'Anonymous Patient ID',
        'Strain',
        'Disease Condition/Health Status',
        'Sex',
        'Sample Collection Date',
        'Anatomic Site',
        'Anatomic Sub-Site',
        'Developmental Stage',
        'Tissue Type',
        'Cell Type',
        'Cell Line ID',
        'Pathology/Disease Name (for diseased sample only)',
        'Additional Pathology Information',
        'Grade',
        'Stage',
        'Tumor content (%)',
        'Pathology Occurrence',
        'Treatment Status',
        'Family Information',
        'DNA Volume (uL)',
        'DNA Concentration (nM)',
        'Storage Medium',
        'Quantification Method',
        'Library Type',
        'Library Construction Method',
        'Size Range (bp)',
        'Average Size (bp)',
        "Indexed? If the libraries are indexed, provide the index sequence from 5' to 3'",
        'Index Read Type (select from drop down list)',
        'Dual Indices for LIMS Upload',
        'No. of cells/IP',
        'Crosslinking Method',
        'Crosslinking Time',
        'Sonication Time',
        'Antibody Used',
        'Antibody catalogue #',
        'Antibody Vendor',
        'Amount of Antibody Used',
        'I7_Index_ID',
        'index',
        'I5_Index_ID',
        'index2'
        ]
        self._meta_df = self._get_meta_df()
        self._data_df = self._get_data_df()

    @property
    def sequencing(self):
        return self._sequencing

    @property
    def form_name(self):
        return self._form_name

    @property
    def meta_header(self):
        return yaml.load(open(self._meta_header), Loader = YODLoader)

    @property
    def data_header(self):
        return yaml.load(open(self._data_header), Loader = YODLoader)

    @property
    def meta_df(self):
        return self._meta_df

    @property
    def data_df(self):
        return self._data_df

    def _get_meta_df(self):
        """return a dataframe of metadata information for self._sequencing."""
        # it's ordered based on the self._meta_colnames.
        values = [
        '_'.join([self._library.sample.sample_id, self._library.pool_id]),
        ' '.join([self._library.pool_id, self._library.sample.sample_id]),
        self._library.sample.taxonomy_id,
        self._libquant.dna_volume,
        self._libquant.dna_concentration_nm,
        self._libquant.storage_medium,
        self._libquant.quantification_method,
        self._libconst.library_type,
        self._libconst.library_construction_method,
        self._libquant.size_range,
        self._libquant.average_size,
        self._library.num_sublibraries,
        self._sequencing.get_read_type_display(),
        self._sequencing.read1_length,
        self._sequencing.sequencing_goal,
        "N/A",
        self._sequencing.format_for_data_submission,
        "",
        ]

        # to avoid the "ValueError: If using all scalar values,
        # you must must pass an index".
        data = {k:[v] for k, v in zip(self._meta_colnames, values)}
        df = pd.DataFrame(data)

        # reorder the columns
        df = df[self._meta_colnames]
        return df

    def _get_data_df(self):
        """return a dataframe of sublibrary information for the given library."""
        sample_columns = {
        'Taxonomy ID': self._sample.taxonomy_id,
        'Anonymous Patient ID': self._sample.anonymous_patient_id,
        'Strain': self._sample.strain,
        'Disease Condition/Health Status': self._sample_addinfo.disease_condition_health_status,
        'Sex': self._sample_addinfo.get_sex_display(),
        'Anatomic Site': self._sample_addinfo.anatomic_site,
        'Anatomic Sub-Site': self._sample_addinfo.anatomic_sub_site,
        'Developmental Stage':self._sample_addinfo.developmental_stage,
        'Tissue Type': self._sample_addinfo.get_tissue_type_display(),
        'Cell Type': self._sample_addinfo.cell_type,
        'Cell Line ID': self._sample.cell_line_id,
        'Pathology/Disease Name (for diseased sample only)': self._sample_addinfo.pathology_disease_name,
        'Additional Pathology Information': self._sample_addinfo.additional_pathology_info,
        'Grade': self._sample_addinfo.grade,
        'Stage': self._sample_addinfo.stage,
        'Tumor content (%)': self._sample_addinfo.tumour_content,
        'Pathology Occurrence': self._sample_addinfo.get_pathology_occurrence_display(),
        'Treatment Status': self._sample_addinfo.get_treatment_status_display(),
        'Family Information': self._sample_addinfo.family_information,
        }

        library_columns = {
        'Tube Label': 'NA', #self.library.library_tube_label,
        'Sample Collection Date': self._library.librarysampledetail.sample_spot_date,
        'DNA Volume (uL)': self._libquant.dna_volume,
        'DNA Concentration (nM)': self._libquant.dna_concentration_nm,
        'Storage Medium': self._libquant.storage_medium,
        'Quantification Method': self._libquant.quantification_method,
        'Library Type': self._libconst.library_type,
        'Library Construction Method': self._libconst.library_construction_method,
        'Size Range (bp)': self._libquant.size_range,
        'Average Size (bp)': self._libquant.average_size,
        }<commit_msg>Changed the value of these columns in "Analyte Information (For unpooled Libraries only)" to blank: DNA Volume (ul), DNA Concentration (nm), Storage Medium, Quantification Method.<commit_after>"""
Created on June 6, 2016

@author: <NAME> (<EMAIL>)
"""

import os, sys
import pandas as pd 
import yaml
from string import Template
from collections import OrderedDict

#===============
# Django imports
#---------------
from .models import Sequencing, SublibraryInformation
from django.conf import settings
from django.shortcuts import get_object_or_404

#==================================================
# upload, parse and populate Sublibrary Information
#--------------------------------------------------
def parse_smartchipapp_file(csv_infile):
    """parse the result file of SmartChipApp."""
    df = pd.read_excel(csv_infile, sheetname=0)
    ## filter out the cells whose Spot_Well value is not NaN
    df = df[~df['Spot_Well'].isnull()]
    df = df.sort_values(by='Sample')
    ## change the column names to match the filed names of the model
    df.columns = [c.lower() for c in df.columns]
    return df

def create_sublibrary_information(s):
    """create SublibraryInformation instance from given pandas Series."""
    d = s.to_dict()
    instance = SublibraryInformation(**d)
    return instance

def bulk_create_sublibrary(library, df):
    """add sublirary information in df for given library instance."""
    num_sublibraries = df.size/len(df.columns)

    ## delete previous sublibraryinformation_set
    library.sublibraryinformation_set.all().delete()

    instances = []
    for instance in df.apply(create_sublibrary_information, axis=1):
        instance.library_id = library.pk
        instances.append(instance)

    library.sublibraryinformation_set.bulk_create(instances)
    return num_sublibraries


#=================
# history manager
#-----------------
class HistoryManager(object):

    """
    An api for simple_history app.
    """

    @staticmethod
    def print_history(object, history_type=None):
        print '=' * 100
        print "Object\tID\tDate\tAction\tUser"
        print '=' * 100
        if history_type is None:
            histories = object.history.all()
        else:
            histories = object.history.filter(history_type=history_type)

        for h in histories:
            print "\t".join([
                str(h.instance),
                str(h.instance.id),
                str(h.history_date),
                h.get_history_type_display(),
                str(h.history_user),
                ])
            print '-' * 100

#======================
# generate sample sheet
#----------------------
def generate_samplesheet(pk):
    """generate samplesheet for the given Sequencing using template in SC-180."""
    samplesheet = SampleSheet(pk)
    sheet_name = samplesheet.sheet_name
    ofilename = os.path.join(settings.MEDIA_ROOT, sheet_name)
    samplesheet.write_header(ofilename)
    samplesheet.write_data(ofilename)
    return sheet_name, os.path.abspath(ofilename)

class SampleSheet(object):

    """
    Sequencing SampleSheet.
    """

    def __init__(self, pk):
        self._sequencing = get_object_or_404(Sequencing, pk=pk)
        self._sheet_name = str(self._sequencing) + '_samplesheet.csv'
        self._header = os.path.join(settings.BASE_DIR,
            "templates/template_samplesheet_header.html")
        self._colnames = [
        'Sample_ID',
        'Sample_Name',
        'Sample_Plate',
        'Sample_Well',
        'I7_Index_ID',
        'index',
        'I5_Index_ID',
        'index2',
        'Sample_Project',
        'Description'
        ]

    @property
    def sequencing(self):
        return self._sequencing

    @property
    def sheet_name(self):
        return self._sheet_name

    def write_header(self, ofilename):
        """write the header section of the sequencing SampleSheet."""
        with open(self._header, 'r') as tempstr:
            s = Template(tempstr.read())
            d = {
            'sequencing_instrument': self._sequencing.get_sequencing_instrument_display(),
            'submission_date': self._sequencing.submission_date,
            'pool_id': self._sequencing.library.pool_id,
            'read1_length': self._sequencing.read1_length,
            'read2_length': self._sequencing.read2_length,
            }

            # Sequencing may have no SequencingDetail
            try:
                d['flow_cell_id'] = self._sequencing.sequencingdetail.flow_cell_id
            except:
                d['flow_cell_id'] = None

            s = s.safe_substitute(**d)
            ofile = open(ofilename, 'w')
            ofile.write(s)
            ofile.close()

    def write_data(self, ofilename):
        """write the data section of the sequencing SampleSheet."""
        data_table = self._mk_data_table()

        # reorder the columns
        data_table = data_table[self._colnames]
        data_table.to_csv(ofilename, mode='a', index=False)

    def _mk_data_table(self):
        """make data table for data section of the samplesheet template."""
        def _map_to_template(s):
            d = s.to_dict()
            # This is the relation between columns in the template samplesheet
            # and the actual columns in df from LIMS.
            res = {
            'Sample_ID': '-'.join([
            str(self._sequencing.library.sample),
            str(self._sequencing.library.pool_id),
            'R'+ str(d['row']),
            'C'+ str(d['column'])
            ]),
            'Sample_Name': '',
            'Sample_Plate': 'R' + str(d['row']) + '_C' + str(d['column']),
            'Sample_Well': 'R' + str(d['row']) + '_C' + str(d['img_col']),
            'I7_Index_ID': d['index_i7'],
            'index': d['primer_i7'],
            'I5_Index_ID': d['index_i5'],
            'index2': d['primer_i5'],
            #'Description': 'CC=<cell call number>;EC=<experimental condition letter>',
            'Description': 'CC=' + d['pick_met'] + ';' + 'EC=' + d['spot_class'],
            }
            return res

        sample_project = '' #','.join(sequencing.library.projects.names())
        newl = []
        oldl = list(self._sequencing.library.sublibraryinformation_set.values())
        df = pd.DataFrame(oldl)
        for d in df.apply(_map_to_template, axis=1):
            d['Sample_Project'] = sample_project
            newl.append(d)
        return pd.DataFrame(newl)


#=============================
# generate GSC submission form
#-----------------------------
def generate_gsc_form(pk, metadata):
    """generate the GSC submission form for the given library."""
    gsc_form = GSCForm(pk)
    pool_df = gsc_form.meta_df
    sample_df = gsc_form.data_df
    header1 = gsc_form.meta_header
    header2 = gsc_form.data_header
    form_name = gsc_form.form_name
    ofilename = os.path.join(settings.MEDIA_ROOT, form_name)

    workbook = Submission(pool_df, sample_df, ofilename)
    workbook.set_column_width()
    workbook.write_address_box(metadata)
    workbook.write_pool_header(header1)
    workbook.write_sample_header(header2)
    workbook.close()
    return form_name, os.path.abspath(ofilename)

class GSCForm(object):

    """
    GSC sequencing submission form.
    """

    def __init__(self, pk):
        self._sequencing = get_object_or_404(Sequencing, pk=pk)
        self._library = self._sequencing.library
        self._libconst = self._library.libraryconstructioninformation
        self._libquant = self._library.libraryquantificationandstorage
        self._sample = self._library.sample
        self._sample_addinfo = self._sample.additionalsampleinformation
        self._form_name = str(self._sequencing) + '_gsc_submission_form.xlsx'
        self._meta_header = os.path.join(settings.BASE_DIR,
            "templates/template_gsc_meta_header.html")
        self._data_header = os.path.join(settings.BASE_DIR,
            "templates/template_gsc_data_header.html")
        self._meta_colnames = [
        'POOL ID',
        'Tube Label',
        'Taxonomy ID',
        'DNA Volume (uL)',
        'DNA Concentration (nM)',
        'Storage Medium',
        'Quantification Method',
        'Library Type',
        'Library Construction Method',
        'Size Range (bp)',
        'Average Size (bp)',
        'Number of libraries in pool',
        'Read Type',
        'Read Length',
        'Sequencing Goal',
        'Reference genome for alignment',
        'Format for data dissemination',
        'Additional comments',
        ]
        self._data_colnames = [
        'Sub-Library ID',
        'Tube Label',
        'Taxonomy ID',
        'Anonymous Patient ID',
        'Strain',
        'Disease Condition/Health Status',
        'Sex',
        'Sample Collection Date',
        'Anatomic Site',
        'Anatomic Sub-Site',
        'Developmental Stage',
        'Tissue Type',
        'Cell Type',
        'Cell Line ID',
        'Pathology/Disease Name (for diseased sample only)',
        'Additional Pathology Information',
        'Grade',
        'Stage',
        'Tumor content (%)',
        'Pathology Occurrence',
        'Treatment Status',
        'Family Information',
        'DNA Volume (uL)',
        'DNA Concentration (nM)',
        'Storage Medium',
        'Quantification Method',
        'Library Type',
        'Library Construction Method',
        'Size Range (bp)',
        'Average Size (bp)',
        "Indexed? If the libraries are indexed, provide the index sequence from 5' to 3'",
        'Index Read Type (select from drop down list)',
        'Dual Indices for LIMS Upload',
        'No. of cells/IP',
        'Crosslinking Method',
        'Crosslinking Time',
        'Sonication Time',
        'Antibody Used',
        'Antibody catalogue #',
        'Antibody Vendor',
        'Amount of Antibody Used',
        'I7_Index_ID',
        'index',
        'I5_Index_ID',
        'index2'
        ]
        self._meta_df = self._get_meta_df()
        self._data_df = self._get_data_df()

    @property
    def sequencing(self):
        return self._sequencing

    @property
    def form_name(self):
        return self._form_name

    @property
    def meta_header(self):
        return yaml.load(open(self._meta_header), Loader = YODLoader)

    @property
    def data_header(self):
        return yaml.load(open(self._data_header), Loader = YODLoader)

    @property
    def meta_df(self):
        return self._meta_df

    @property
    def data_df(self):
        return self._data_df

    def _get_meta_df(self):
        """return a dataframe of metadata information for self._sequencing."""
        # it's ordered based on the self._meta_colnames.
        values = [
        '_'.join([self._library.sample.sample_id, self._library.pool_id]),
        ' '.join([self._library.pool_id, self._library.sample.sample_id]),
        self._library.sample.taxonomy_id,
        self._libquant.dna_volume,
        self._libquant.dna_concentration_nm,
        self._libquant.storage_medium,
        self._libquant.quantification_method,
        self._libconst.library_type,
        self._libconst.library_construction_method,
        self._libquant.size_range,
        self._libquant.average_size,
        self._library.num_sublibraries,
        self._sequencing.get_read_type_display(),
        self._sequencing.read1_length,
        self._sequencing.sequencing_goal,
        "N/A",
        self._sequencing.format_for_data_submission,
        "",
        ]

        # to avoid the "ValueError: If using all scalar values,
        # you must must pass an index".
        data = {k:[v] for k, v in zip(self._meta_colnames, values)}
        df = pd.DataFrame(data)

        # reorder the columns
        df = df[self._meta_colnames]
        return df

    def _get_data_df(self):
        """return a dataframe of sublibrary information for the given library."""
        sample_columns = {
        'Taxonomy ID': self._sample.taxonomy_id,
        'Anonymous Patient ID': self._sample.anonymous_patient_id,
        'Strain': self._sample.strain,
        'Disease Condition/Health Status': self._sample_addinfo.disease_condition_health_status,
        'Sex': self._sample_addinfo.get_sex_display(),
        'Anatomic Site': self._sample_addinfo.anatomic_site,
        'Anatomic Sub-Site': self._sample_addinfo.anatomic_sub_site,
        'Developmental Stage':self._sample_addinfo.developmental_stage,
        'Tissue Type': self._sample_addinfo.get_tissue_type_display(),
        'Cell Type': self._sample_addinfo.cell_type,
        'Cell Line ID': self._sample.cell_line_id,
        'Pathology/Disease Name (for diseased sample only)': self._sample_addinfo.pathology_disease_name,
        'Additional Pathology Information': self._sample_addinfo.additional_pathology_info,
        'Grade': self._sample_addinfo.grade,
        'Stage': self._sample_addinfo.stage,
        'Tumor content (%)': self._sample_addinfo.tumour_content,
        'Pathology Occurrence': self._sample_addinfo.get_pathology_occurrence_display(),
        'Treatment Status': self._sample_addinfo.get_treatment_status_display(),
        'Family Information': self._sample_addinfo.family_information,
        }

        library_columns = {
        'Tube Label': 'NA', #self.library.library_tube_label,
        'Sample Collection Date': self._library.librarysampledetail.sample_spot_date,
        'DNA Volume (uL)': "", #self._libquant.dna_volume,
        'DNA Concentration (nM)': "", #self._libquant.dna_concentration_nm,
        'Storage Medium': "", #self._libquant.storage_medium,
        'Quantification Method': "", #self._libquant.quantification_method,
        'Library Type': self._libconst.library_type,
        'Library Construction Method': self._libconst.library_construction_method,
        'Size Range (bp)': self._libquant.size_range,
        'Average Size (bp)': self._libquant.average_size,
        }
{'fraction_non_alphanumeric': 0.12591280456944545, 'fraction_numerical': 0.003831971657870002, 'mean_word_length': 2.493685274059106, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '186889', 'n_tokens_mistral': 8295, 'n_tokens_neox': 7969, 'n_words': 1750}