<jupyter_start><jupyter_text>Plot five largest lineages on mapDirection is anti-clockwise<jupyter_code>dates = []
for edge, edgedict in edge_dict_0_67.items():
    dates.append(edgedict["start_dec_date"])
    dates.append(edgedict["end_dec_date"])
for edge, edgedict in edge_dict_0_92.items():
    dates.append(edgedict["start_dec_date"])
    dates.append(edgedict["end_dec_date"])
for edge, edgedict in edge_dict_0_190.items():
    dates.append(edgedict["start_dec_date"])
    dates.append(edgedict["end_dec_date"])
for edge, edgedict in edge_dict_0_322.items():
    dates.append(edgedict["start_dec_date"])
    dates.append(edgedict["end_dec_date"])
for edge, edgedict in edge_dict_0_386.items():
    dates.append(edgedict["start_dec_date"])
    dates.append(edgedict["end_dec_date"])
for edge, edgedict in edge_dict_1_51.items():
    dates.append(edgedict["start_dec_date"])
    dates.append(edgedict["end_dec_date"])
for edge, edgedict in edge_dict_2_337.items():
    dates.append(edgedict["start_dec_date"])
    dates.append(edgedict["end_dec_date"])
    
norm = matplotlib.colors.Normalize(vmin=min(dates), vmax=max(dates))

cmap = matplotlib.colors.LinearSegmentedColormap.from_list("", ["#484C7E","#FE6362", "#FFD544"])
colour_dict = {}
for date in dates:
    colour_dict[date] = cmap(norm(date))
def make_plot(edge_dict, lineage_name, colour_dict):
    
        
    fig, ax = plt.subplots(1,1, figsize=(5,5))
    matplotlib.rcParams.update({'font.size': 8})
    england["geometry"] = england.geometry.simplify(1000)

    england.plot(ax=ax, color="whitesmoke", edgecolor="lightgrey")

    count = 0
    
    for edge, edgedict in tqdm.tqdm(edge_dict.items()):
        
        count += 1
        
        point_dict = defaultdict(list)

        A = edgedict["start_long"],edgedict["start_lat"]
        B = edgedict["end_long"],edgedict["end_lat"]
        start_date = edgedict["start_dec_date"]
        end_date = edgedict["end_dec_date"]

        kw = dict(color="slategrey",linewidth=0.3)

        liney = mpatches.FancyArrowPatch(A, B, arrowstyle="-",
                                 connectionstyle="arc3,rad=.5", **kw)

        ax.add_patch(liney)

        ax.scatter(A[0], A[1], s=3, color=colour_dict[start_date], alpha=1.0, zorder=5)
        ax.scatter(B[0], B[1], s=3, color=colour_dict[end_date], alpha=1.0, zorder=6)

    if lineage_name == "Lineage VII":

        bar = fig.colorbar(cm.ScalarMappable(norm=norm, cmap=cmap), ax=ax)
        new_ticks = []

        for i in bar.get_ticks():
            new_ticks.append(true_date(i))
        bar.set_ticks(bar.get_ticks())
        bar.set_ticklabels(new_ticks)

    ax.axis('off')
#     plt.title(lineage_name)
    plt.savefig(f"../figures/{lineage_name}_geographical_spread.eps", bbox_inches='tight')
    plt.savefig(f"../figures/{lineage_name}_geographical_spread.pdf", bbox_inches='tight')

#     plt.show()
make_plot(edge_dict_0_67, names["0_67"], colour_dict)
make_plot(edge_dict_0_92, names["0_92"], colour_dict)
make_plot(edge_dict_0_190, names["0_190"], colour_dict)
make_plot(edge_dict_0_322, names["0_322"], colour_dict)
make_plot(edge_dict_0_386, names["0_386"], colour_dict)
make_plot(edge_dict_1_51, names["1_51"], colour_dict)
make_plot(edge_dict_2_337, names["2_337"], colour_dict)<jupyter_output>/usr/local/lib/python3.7/site-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  
100%|██████████| 3038/3038 [02:52<00:00, 17.59it/s]
/usr/local/lib/python3.7/site-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  
100%|██████████| 3358/3358 [03:44<00:00, 14.94it/s]
/usr/local/lib/python3.7/site-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col[...]<jupyter_text>Plot against movements<jupyter_code>date_to_normalised_mvmt = {}
with open("../../Data/uk_movement.csv") as f:
    data = csv.DictReader(f)
    for l in data:
        date = dt.datetime.strptime(l['date'], "%Y-%m-%d").date()
        date_to_normalised_mvmt[date] = float(l["movement_normalised"])
person_movements = {k:v for k,v in sorted(date_to_normalised_mvmt.items())}
virus_movements = {}
between_pc_mvmts = {}
for edgedict in tqdm.tqdm(lin_dicts.values()):
    for i, edge in edgedict.items():
            
        date = dt.datetime.strptime(edge['start_real_date'], "%Y-%m-%d").date()
        
        if date not in virus_movements:
            virus_movements[date] = 1
        else:
            virus_movements[date] += 1
        
        if edge["start_pc"] != edge["end_pc"]:
            if date not in between_pc_mvmts:
                between_pc_mvmts[date] = 1
            else:
                between_pc_mvmts[date] += 1
                
first_date = min(virus_movements)
last_date = max(virus_movements)

day = dt.timedelta(1)
date = first_date
for i in range((last_date - first_date).days):
    new_date = date + day
    if new_date not in virus_movements:
        virus_movements[new_date] = 0
    if new_date not in between_pc_mvmts:
        between_pc_mvmts[new_date] = 0
    date = new_date
    
virus_movement_ordered = {k:v for k,v in sorted(virus_movements.items())}
between_pc_movement_ordered = {k:v for k,v in sorted(between_pc_mvmts.items())}
df_dict = defaultdict(list)
for k,v in virus_movement_ordered.items():
    df_dict["Date"].append(k)
    df_dict["Movements"].append(v)
    df_dict["Between_PC"].append(between_pc_movement_ordered[k])
    
df = pd.DataFrame(df_dict)
df['7 day rolling'] = df.Movements.rolling(7).mean()
df['7 day rolling between pc'] = df.Between_PC.rolling(7).mean()

new_last_date = dt.date(2021,6,1)
first_mvmt = dt.date(2021,1,1)

relevant_person_movements = {}
for k,v in person_movements.items():
    if k >= first_mvmt:
        relevant_person_movements[k] = v
        
relevant_person_movements2 = {}
for k,v in person_movements.items():
    if k >= first_date and k <= new_last_date:
        relevant_person_movements2[k] = v


relevant_virus_movements = {}
for k,v in zip(df["Date"], df["7 day rolling"]):
    if k >= first_date and k <= new_last_date:
        relevant_virus_movements[k] = v
        
relevant_pc_movements = {}
for k,v in zip(df["Date"], df["7 day rolling between pc"]):
    if k >= first_date and k <= new_last_date:
        relevant_pc_movements[k] = v
        
#removed last week for sampling lag
matplotlib.rcParams.update({'font.size': 10})
loc = plticker.MultipleLocator(base=14.0)

key_dates = [dt.date(2021,3,8), dt.date(2021,3,29), dt.date(2021,4,12), dt.date(2021,5,17)]

fig, ax = plt.subplots(1,1,figsize=(8,5))
ax2 = ax.twinx()

line1 = ax.plot(relevant_virus_movements.keys(),relevant_virus_movements.values(), color="#484c7e", linewidth=3,label="Lineages through time")
line2 = ax2.plot(relevant_person_movements.keys(), relevant_person_movements.values(), color="#ffd544",linewidth=3, label="Human movements")
ax2.scatter(relevant_person_movements.keys(), relevant_person_movements.values(), s=75, color="#ffd544")

line3 = ax.plot(relevant_pc_movements.keys(), relevant_pc_movements.values(), color="#bbbee9", linewidth=3, label="Between postcode virus movements")


for date in key_dates[1:]:
    ax.axvline(date, color="slategrey", linestyle="dotted")

ax.axvline(dt.date(2021,3,8), color="slategrey", linestyle="dotted", label="Stages of lockdown easing")

ax.set_ylabel("Number of virus movements\n(7 day rolling average)")
ax2.set_ylabel("Number of human movements normalised")
ax.set_xlabel("Date")

ax.set_xlim(first_mvmt-dt.timedelta(2),new_last_date+dt.timedelta(2))

ax.tick_params(axis='x', labelrotation = 90)

lines, labels = ax.get_legend_handles_labels()
lines2, labels2 = ax2.get_legend_handles_labels()
ax2.legend(lines + lines2, labels + labels2, loc=0)

ax.xaxis.set_major_locator(loc)

plt.savefig("../figures/human_and_virus_movements.pdf", bbox_inches='tight')

plt.show()
fracs = {}
for date,between, all_mv in zip(df["Date"],df["7 day rolling between pc"], df["7 day rolling"]):
    if date >= first_date and date <= new_last_date:
        if all_mv != 0:
            fracs[date] = between/all_mv
    
fig, ax = plt.subplots(1,1,figsize=(10,10))
loc = plticker.MultipleLocator(base=7.0)

plt.plot(fracs.keys(), fracs.values(), linewidth=5, color="#fe6362")

plt.xlabel("Day")
plt.ylabel("Fraction of all edges that are between postcode movements\n(7 day average)")
ax.xaxis.set_major_locator(loc)
ax.tick_params(axis='x', labelrotation = 90)

plt.ylim(0,1)
# plt.savefig("../")

plt.show()<jupyter_output><empty_output><jupyter_text>Source sink bits<jupyter_code>all_utlas = set()
for edgedict in lin_dicts.values():
    for i, edge in edgedict.items():
        all_utlas.add(edge['start_utla'])
        all_utlas.add(edge['end_utla'])
london = "GREATER_LONDON_AUTHORITY"
manchester_lst = ["SALFORD_DISTRICT_(B)","'TRAFFORD_DISTRICT_(B)", "STOCKPORT_DISTRICT_(B)", "OLDHAM_DISTRICT_(B)", "BOLTON_DISTRICT_(B)",
     "TAMESIDE_DISTRICT_(B)", "BURY_DISTRICT_(B)", "ROCHDALE_DISTRICT_(B)", "WIGAN_DISTRICT_(B)", "MANCHESTER_DISTRICT_(B)"]
manchester = set()
for i in manchester_lst:
    manchester.add(i)

out_of_london = 0
out_of_manc = 0

for edgedict in lin_dicts.values():
    for i, edge in edgedict.items():
        if edge["start_utla"] != edge["end_utla"]:
            if edge["start_utla"] in manchester and edge["end_utla"] not in manchester:
                out_of_manc += 1
            elif edge["start_utla"] == london:
                out_of_london += 1

print(out_of_manc)
print(out_of_london)
print(out_of_manc/out_of_london)
out_of_manc = 0
in_manc = 0

# for edgedict in lin_dicts.values():
for i, edge in edgedict_0_92.items():
    if edge["start_utla"] in manchester and edge["end_utla"] not in manchester:
        out_of_manc += 1
    elif edge["start_utla"] in manchester and edge["end_utla"] in manchester:
        in_manc += 1

print(out_of_manc)
print(in_manc)

print(in_manc/out_of_manc)<jupyter_output>5206
18476
3.548981943910872<jupyter_text>Distance of movements<jupyter_code>lin_to_movements = defaultdict(list)
for file in os.listdir("../results/movements/"):
    if file.endswith(".csv"):
        name = "_".join(file.strip(".csv").split("_")[0:2])
        if name in large_lins:
            new_name = names[name]
            with open(os.path.join("../results/movements/", file)) as f:
                data = csv.DictReader(f)
                for l in tqdm.tqdm(data):
                      lin_to_movements[new_name].append(float(l['distance']))
names_list = ["Lineage I", "Lineage II", "Lineage III", "Lineage IV", "Lineage V", "Lineage VI", "Lineage VII"]
fig, ax = plt.subplots(1,7,figsize=(25,5))
matplotlib.rcParams.update({'font.size': 15})

count = 0
for lin in names_list:
#     if lin == "lineage I":
#         ylim_upper = 1000
#     else:
    ylim_upper = 150
    ax[count].hist([x for x in lin_to_movements[lin] if x>50], bins=30, width=10,color=colours_renamed[lin])
    ax[count].set_title(f'Transmission \n{lin}')
    ax[count].set_xlim(50,400)
    ax[count].set_ylim(0,ylim_upper)
    count += 1

fig.add_subplot(111,frameon=False)
plt.tick_params(labelcolor='none', top=False, bottom=False, left=False, right=False)
plt.grid(False)
plt.xlabel("Distance (km)")
plt.ylabel("Frequency")

    
plt.savefig("../figures/viral_movement_histograms.pdf", bbox_inches="tight")
plt.savefig("../figures/viral_movement_histograms.png", bbox_inches="tight")

plt.show()<jupyter_output><empty_output><jupyter_text>Movements over time<jupyter_code>date_to_movements = defaultdict(list)
for file in os.listdir("../results/movements/"):
    if file.endswith(".csv"):
        with open(os.path.join("../results/movements/",file)) as f:
            data = csv.DictReader(f)
            for l in tqdm.tqdm(data):
                date = dt.datetime.strptime(l['date'], "%Y-%m-%d").date()
                date_to_movements[date].append(float(l['distance']))
sorted_mvmts = {k:v for k,v in sorted(date_to_movements.items(), key=lambda item: item[0])}
month_mvmts = defaultdict(list)
all_month_mvmts = defaultdict(list)
for day, dists in sorted_mvmts.items():
    to_add = []
    for i in dists:
        if i >= 50:
            to_add.append(i)
    
    month_mvmts[day.month].extend(to_add)
    all_month_mvmts[day.month].extend(dists)
    
def ecdf(a):
    x, counts = np.unique(a, return_counts=True)
    cusum = np.cumsum(counts)
    return x, cusum / cusum[-1]
month_dict={12:"December", 1:"January", 2:"February", 3:"March", 4:"April", 5:"May", 6:"June"}
fig, ax = plt.subplots(1,1, figsize=(15,15))

for month, data in month_mvmts.items():
    
    if month != 11:
                
        x,y = ecdf(data)

        reverse = []
        for i in y:
            reverse.append(1-i)

        ax.plot(x, reverse, label=month_dict[month])
        
ax.legend()

plt.show()
first_date = dt.date(2021,4,12)
second_date = dt.date(2021,5,17)

lockdown_mvmts = defaultdict(list)

for date, mvmts in sorted_mvmts.items():
#     if date < first_date:
#         lockdown_mvmts["Before"].extend(mvmts)
#     elif date >= first_date and date < second_date:
#         lockdown_mvmts["Between"].extend(mvmts)
#     else:
#         lockdown_mvmts["After"].extend(mvmts)
    if date < first_date:
        for i in mvmts:
            if i > 25:
                lockdown_mvmts["Before"].append(i)
    elif date >= first_date and date < second_date:
        for i in mvmts:
            if i > 25:
                lockdown_mvmts["Between"].append(i)
    elif date >= second_date:
        for i in mvmts:
            if i > 25:
                lockdown_mvmts["After"].append(i)
    else:
        print(date,mvmts)

    
fig, ax = plt.subplots(1,1, figsize=(10,10))
matplotlib.rcParams.update({'font.size': 15})

colours = {"Before":"#484c7e", "Between":"#ffd544", "After":"#fe6362"}

for period, data in lockdown_mvmts.items():
    
                
        x,y = ecdf(data)

        reverse = []
        for i in y:
            reverse.append(1-i)

        ax.plot(x, reverse, label=period, color=colours[period]) 
        
ax.legend()
ax.set_xlabel("Km")
ax.set_ylabel("Proportion")

plt.savefig("../figures/movement_length_over_time.pdf", bbox_inches='tight')
plt.savefig("../figures/movement_length_over_time.png", bbox_inches='tight')


plt.show()
props = {}
for month, dists in all_month_mvmts.items():
    below = 0
    above = 0
    for i in dists:
        if i < 50:
            below += 1
        else:
            above += 1
    props[month] = above/below
props<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1474930176885224, 'fraction_numerical': 0.02859422795584519, 'mean_word_length': 3.8701424870466323, 'pattern_counts': {'":': 5, '<': 30, '<?xml version=': 0, '>': 32, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '413075', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 5928, 'n_tokens_neox': 5567, 'n_words': 1006, 'path': 'analysis/UK_continuous_phylogeography/scripts/7a.Continuous Phylo figures.ipynb', 'repo_name': 'COG-UK/Delta-analysis', 'size': '15038'}