<jupyter_start><jupyter_text>Common libs and constants for Ag1000g data analysis<jupyter_code>from scipy.spatial.distance import pdist, cdist
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import numpy as np
import itertools
import scipy
import allel
import zarr
import sys
import os
allel.__version__, zarr.__version__
# ag1k data files
# newer zarr does not have sample metadata
# HAPS_ZARR = '../../../data/ag1k/ag1000g.phase2.ar1.haplotypes/'
HAPS_ZARR = '../../../data/ag1k/haplotypes/'
HAPS_X_ZARR = '../../../data/ag1k/haplotypes_X/'
SAMPLE_META = '../../../data/ag1k/phase2_samples.meta.txt'
# panel
PANEL = '../../data/panel_extended_info.csv'
# mosquito amplicons
AMPLS = [str(i) for i in range(62)]
# outputs
WD = '../../../data/phylo_ampl_ag1k_hap/'
AMPL_HAP_ZARR = os.path.join(WD, 'ampl_hap')
WSP_VAR_FILE = 'data/1_wsp_var.csv'
! mkdir -p {AMPL_HAP_ZARR}
! mkdir -p 'data'
# read amplicon data
panel = pd.read_csv(PANEL)
# get insert coordinates - variation in primers is not picked up by sequencing
panel['start_insert'] = panel.start + panel.F.str.len()
panel['end_insert'] = panel.end - panel.R.str.len()
# remove plasmodium primers from consideration
panel_mosquito = panel[panel['Primary_ID'].isin(AMPLS)].set_index('Primary_ID')
panel_mosquito.columns
# read samples metadata
samples = pd.read_csv(SAMPLE_META, delimiter='\t')
samples.columns
# species
ms_sp = {
 'M':'coluzzii',
 'S':'gambiae',
 'M/S':'hybrid'}
samples['species'] = samples.m_s.replace(ms_sp)
# add unknown species for missing M/S values - applicable to Kenya
samples['species'] = samples.species.fillna('unknown')
samples.species.value_counts()
# populations
populations = {
    'AOcol': 'Angola coluzzii',
    'GHcol': 'Ghana coluzzii',
    'BFcol': 'Burkina coluzzii',
    'CIcol': 'Côte d\'Ivoire coluzzii',
    'GNcol': 'Guinea colizzii',
    'CMgam': 'Cameroon coluzzii',
    'GHgam': 'Ghana gambiae',
    'BFgam': 'Burkina gambiae',
    'GNgam': 'Guinea gambiae',
    'GW': 'Guinea-Bissau',
    'GM': 'Gambia',
    'GAgam': 'Gabon gambiae',
    'UGgam': 'Uganda gambiae',
    'FRgam': 'Mayotte gambiae',
    'GQgam': 'Equatorial Guinea gambiae',
    'KE': 'Kenya',
}
samples['pop_long'] = samples.population.replace(populations)
samples.pop_long.value_counts()
# population colours
reds = sns.color_palette('Reds', 5)
blues = sns.color_palette('Blues', 4)
greens = sns.color_palette('Greens', 2)
browns = sns.color_palette('YlOrBr', 4)
purples = sns.color_palette('Purples', 2)
greys = sns.color_palette('Greys', 3)
pop_colors = {
    'Angola coluzzii': reds[4],
    'Ghana coluzzii': reds[3],
    'Burkina coluzzii': reds[2],
    'Côte d\'Ivoire coluzzii': reds[1],
    'Guinea coluzzii': reds[0],
    'Cameroon colizzii': blues[3],
    'Ghana gambiae': blues[2],
    'Burkina gambiae': blues[1],
    'Guinea gambiae': blues[0],
    'Guinea-Bissau': browns[1],
    'Gambia': browns[2],
    'Gabon gambiae': greens[1],
    'Uganda gambiae': greens[0],
    'Mayotte gambiae': purples[1],
    'Equatorial Guinea gambiae': purples[0],
    'Kenya': greys[1],
}<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16371823737536184, 'fraction_numerical': 0.013509166934705693, 'mean_word_length': 4.485008818342152, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '528222', 'license': 'MIT', 'n_tokens_mistral': 1295, 'n_tokens_neox': 1209, 'n_words': 264, 'path': 'work/6_ag1k_extraction/common.ipynb', 'repo_name': 'malariagen/ano-spp', 'size': '3109'}