<jupyter_start><jupyter_text>get info<jupyter_code>utlas_to_intros = defaultdict(list)
shapefile_code_to_name = {}
for file in tqdm.tqdm(os.listdir("../results/MCC_files_all/")):
    if file.endswith(".csv"):
        lin = file.strip(".csv").split("_")[0]
        with open(f"../results/MCC_files_all/{file}") as f:
            data = csv.DictReader(f)
            for l in data:
                if l['end_utla'] != l['start_utla']:
                    if l['end_code'] in ltla_code_to_utla_code:
                        e_code = ltla_code_to_utla_code[l['end_code']]
                    else:
                        e_code = l['end_code']
                    if l['start_code'] in ltla_code_to_utla_code:
                        s_code = ltla_code_to_utla_code[l['start_code']]
                    else:
                        s_code = l['start_code']
                    
                    if e_code != s_code:
                        utlas_to_intros[e_code].append(l['end_real_date'])

utla_intro_counts = defaultdict(dict)
for utla, dates in utlas_to_intros.items():
    utla_intro_counts[utla] = Counter(dates)
ltla_code_to_utla_code[""]
#these are exports from England - into Wales or Scotland
with open("../results/introductions_per_utla.csv", 'w') as fw:
    headers = ["day", "utla", "utla_code", "number_introductions"]
    writer=csv.DictWriter(fw, fieldnames=headers)
    writer.writeheader()
    for code, day_dict in utla_intro_counts.items():
        if code != "":
            for day, count in day_dict.items():
                write_dict = {}
                write_dict["day"] = day
                write_dict["utla"] = code_to_name[code]
                write_dict['utla_code'] = code
                write_dict["number_introductions"] = count

                writer.writerow(write_dict)<jupyter_output><empty_output><jupyter_text>Plot map<jupyter_code>london_codes = set()
for i in england_utla.loc[england_utla["FILE_NAME"] == "GREATER_LONDON_AUTHORITY"]["CODE"]:
    london_codes.add(i)
to_merge = []
for i in england_utla["CODE"]:
    if i in london_codes:
        to_merge.append("E13000001")
    elif i in ltla_code_to_utla_code:
        to_merge.append(ltla_code_to_utla_code[i])
    else:
        to_merge.append(i)
#     elif i == "E06000060":
#         to_merge.append("E10000002")

        
england_utla["new_codes"] = to_merge
        
with_new_codes = england_utla.dissolve(by="new_codes")
with_new_codes["CODE"] = with_new_codes.index
with_new_codes.to_file("UTLA_shapes_final")
with_new_codes
with_new_codes.loc[with_new_codes.index == "E06000008"]
df_dict = defaultdict(list)
code_to_total = {}
with open("../results/introductions_per_utla.csv") as f:
    data = csv.DictReader(f)
    for l in data:
        total = float(l['number_introductions'])
        code = l['utla_code']
        if "E13000001" in code:
            code = "E13000001"
        
        if code in code_to_total:
            code_to_total[code] += total
        else:
            code_to_total[code] = total
        
for code, total in code_to_total.items(): 
    df_dict["CODE"].append(code)
    df_dict["intro_counts"].append((total))
        
df = pd.DataFrame(df_dict)
together = with_new_codes.merge(df, how='outer')

together.to_csv("total_intro_counts.csv")
fig, ax = plt.subplots(1,1,figsize=(5,5))
together["geometry"] = together.geometry.simplify(1000)
matplotlib.rcParams.update({'font.size': 15})

cmap = matplotlib.colors.LinearSegmentedColormap.from_list("", ["#484C7E","#FE6362", "#FFD544"])

together.plot(ax=ax,column="intro_counts",cmap=cmap, missing_kwds={"color":"lightgrey","label":"Data omitted"}, legend=True)

ax.axis('off')

new_ticks = []

# norm = matplotlib.colors.Normalize(vmin=0, vmax=5)
# bar = fig.colorbar(cm.ScalarMappable(norm=norm, cmap=cmap), ax=ax)
# for i in bar.get_ticks():
#     new_ticks.append(int(10**(i)))
# bar.set_ticks(bar.get_ticks())
# bar.set_ticklabels(new_ticks)



plt.savefig("../figures/map_number_introductions.pdf", bbox_inches="tight")

plt.show()<jupyter_output><empty_output><jupyter_text>New utlas sampled in per week<jupyter_code>colours = {
"0_67":"#e2ecf2",
"0_92":"#ffd544",
"0_190":"#fe6362",
"0_322":"#484c7e",
"0_386":"#bbbee9",
"1_51": "#98c1d9",
"2_337": "#293241"  
}

names = {
"0_67":"Lineage IV",
"0_92":"Lineage I",
"0_190":"Lineage II",
"0_322":"Lineage VI",
"0_386":"Lineage VII",
"1_51": "Lineage V",
"2_337": "Lineage III" 
}

colours_renamed = {
"Lineage I":"#ffd544",
"Lineage II":"#fe6362",
"Lineage III":"#293241",
"Lineage V":"#98c1d9",
"Lineage IV":"#e2ecf2",
"Lineage VI":"#484c7e",
"Lineage VII":"#bbbee9"
}

large_lins = ["0_92", "0_190", "2_337", "0_67", "1_51", "0_322", "0_386"]
lineage_to_taxa = defaultdict(list)

for intro in large_lins:
    num_file = intro.split("_")[0]
    rel_file = f'15K_00{num_file}.median_tree.tsv'
    lineage = intro.split("_")[1]
    with open(os.path.join("../input_files/median_import_tree/",rel_file)) as f:
        data = csv.DictReader(f, delimiter="\t")
        for l in data:
            if l['lineage'] == lineage:
                value = l['taxa'].replace(" ","").split(";")
                lineage_to_taxa[intro] = value
        
seq_to_date = {}
for lin, taxa in lineage_to_taxa.items():
    for tax in taxa:
        seq_to_date[tax.split("|")[0]] = dt.datetime.strptime(tax.split("|")[1], "%Y-%m-%d").date()
        

seq_to_loc = {}
with open(GEOGRAPHICAL METADATA) as f:
    data = csv.DictReader(f)
    for l in tqdm.tqdm(data):
        if l['sequence_name'] in seq_to_date:
            if "|" in l['utla'] or l['utla'] == "":
                utla = "ambiguous"
            else:
                utla = l['utla']
            seq_to_loc[f'{l["sequence_name"]}|{seq_to_date[l["sequence_name"]]}'] = utla


lin_week_loc = defaultdict(dict)

for lineage, taxa in lineage_to_taxa.items():
    if lineage in large_lins:
        week_to_adm2 = defaultdict(set)
        for seq in taxa:
            week = Week.fromdate(seq_to_date[seq.split("|")[0]])
            if seq in seq_to_loc:
                loc = seq_to_loc[seq]
                if loc != "ambiguous":
                    week_to_adm2[week].add(loc)
            else:
                print(seq)
                
        lin_week_loc[lineage] = week_to_adm2
lin_to_cumulative = defaultdict(dict)

for lineage in large_lins:
# for lineage, week_dict in lin_week_loc.items():
    week_dict = lin_week_loc[lineage]
    already_in = set()
    cum_locs = defaultdict(set)
    diff_locs = {}
    
    for week, loc_set in sorted(week_dict.items()):
        new_lst = []
        last_week = len(already_in)
        for loc in loc_set:
            already_in.add(loc)

        for i in already_in:
            new_lst.append(i)
            
        cum_locs[week.startdate()] = len(new_lst)
        

    lin_to_cumulative[lineage] = cum_locs


fig, ax = plt.subplots(1,figsize=(5,5))
loc = plticker.MultipleLocator(base=7.0)

for lin, lin_dict in lin_to_cumulative.items():

    x = list(lin_dict.keys())
    y = list(lin_dict.values())
    plt.plot(x,y, color=colours[lin], label=f'Transmission {names[lin]}')
    

plt.legend()
plt.xticks(rotation=90, size=12)
plt.yticks(size=12)
plt.xlabel("Week commencing", size=12)
plt.ylabel("Cumulative number of adm2s with samples", size=12)

ax.xaxis.set_major_locator(loc)


plt.savefig(f"../figures/new_adm2s_absolute_time.pdf", bbox_inches="tight")
    
plt.show()


relative = defaultdict(dict)
for lin, lin_dict in lin_to_cumulative.items():
    new_dict = {}
    count = 0
    for i,v in lin_dict.items():
        count += 1
        new_dict[count] = v
        
    relative[lin] = new_dict
fig, ax = plt.subplots(1,figsize=(5,5))

for lin, lin_dict in relative.items():
    
    x = list(lin_dict.keys())
    y = list(lin_dict.values())
    plt.plot(x,y, color=colours[lin], label=f'Transmission {names[lin]}')
    
    
plt.legend()
plt.xticks(size=12)
plt.yticks(size=12)
plt.xlabel("Weeks since first sample", size=12)
plt.ylabel("Cumulative number of adm2s with samples", size=12)
plt.savefig(f"../figures/new_utlas_relative_time.pdf", bbox_inches = "tight")


plt.show()
locations = []
with open("../results/MCC_files_all/1_51_mcc_data.csv") as f:
    data = csv.DictReader(f)
    print(data.fieldnames)
    for l in data:
        locations.append(l['start_utla'])
        
    
location_counts = Counter(locations)
sorted_dict = {k: v for k, v in sorted(location_counts.items(), key=lambda item: item[1], reverse=True)}
sorted_dict<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15067527891955373, 'fraction_numerical': 0.028655314151497357, 'mean_word_length': 3.0981713185755533, 'pattern_counts': {'":': 27, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '127322', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 3406, 'n_tokens_neox': 3166, 'n_words': 537, 'path': 'analysis/UK_continuous_phylogeography/scripts/7c. Getting intros into all UTLAs.ipynb', 'repo_name': 'COG-UK/Delta-analysis', 'size': '8515'}