<jupyter_start><jupyter_text>Format data<jupyter_code>ref_dict, graph_dict = uf.clean_data(cohorts_dict, reference_dict)
print(graph_dict['UK Biobank']['Ethnicity'].keys())<jupyter_output>dict_keys(['Ethnicity', 'values', 'percent', 'reletive', 'reletive_colours', 'reletive_representative_or_not', 'reletive_rep_threhsold', 'reference standardised', 'missing', 'ref percent', 'description text', 'reletive text', 'abs text'])<jupyter_text>Testing bar split plot<jupyter_code>source = ColumnDataSource(data = graph_dict['UK Biobank']['Ethnicity'])
    
p = figure(
y_range = list(source.data['Ethnicity']), 
title = 'Ethnicity', 
x_range = (0,15),
toolbar_location= None
)

p.hbar(
y = 'Ethnicity',
right = 'percent', 
height = 0.9, 
color = '#003667',
line_alpha = 0,
source = source
)


 
p.hbar(
y = 'Ethnicity',
right = 'ref percent', 
height = 0.9,
fill_alpha = 0,
line_color = '#a0a0a0', 
line_width = 4,
line_alpha = 1,
source = source
)


hover2 = HoverTool(tooltips = [
('Ethnicity', '@Ethnicity'),
('Raw values', "@{values}"),
('Percent/%', "@{percent}{0.0}"),
('UK population percent/%', '@{ref percent}{0.0}')
],
       mode = 'mouse', name= 'data plot')

p.yaxis.major_label_text_font_size = '10pt' 
p.yaxis.major_label_text_font = 'helvetica'
p.yaxis.major_label_text_color = '#a0a0a0'
p.yaxis.major_tick_line_color = None  # turn off y-axis major ticks
p.yaxis.minor_tick_line_color = None
p.xaxis.major_tick_line_color = None  # turn off y-axis major ticks
p.xaxis.minor_tick_line_color = None
p.yaxis.axis_line_color = None
p.xaxis.axis_line_color = None
p.xaxis.major_label_text_font_size = '0pt'
p.xaxis.major_tick_line_color = None
p.xgrid.grid_line_color = None
p.ygrid.grid_line_color = None
p.outline_line_width = 0
p.background_fill_color = '#f5f5f5'
p.background_fill_alpha = 0.9
p.title.text_color = '#a0a0a0'
p.title.text_font_size = '24pt'
p.title.text_font = "helvetica"
p.add_tools(hover2)

q = figure(
y_range = list(source.data['Ethnicity']),  
x_range = (75,110),
toolbar_location= None
)

q.hbar(
y = 'Ethnicity',
right = 'percent', 
height = 0.9, 
color = '#003667',
legend_label = 'UK Biobank percent',
line_alpha = 0,
source = source
)


 
q.hbar(
y = 'Ethnicity',
right = 'ref percent', 
height = 0.9,
fill_alpha = 0,
line_color = '#a0a0a0', 
line_width = 4,
line_alpha = 1,
legend_label = 'UK Population Ratio',
source = source
)



hover3 = HoverTool(tooltips = [
('Ethnicity', '@Ethnicity'),
('Raw values', "@{values}"),
('Percent/%', "@{percent}{0.0}"),
('UK population percent/%', '@{ref percent}{0.0}')
],
       mode = 'mouse', name= 'data plot')

q.yaxis.major_label_text_font_size = '0pt' 
q.yaxis.major_tick_line_color = None  # turn off y-axis major ticks
q.yaxis.minor_tick_line_color = None
q.xaxis.major_tick_line_color = None  # turn off y-axis major ticks
q.xaxis.minor_tick_line_color = None
q.yaxis.axis_line_color = None
q.xaxis.axis_line_color = None
q.xaxis.major_label_text_font_size = '0pt'
q.xaxis.major_tick_line_color = None
q.xgrid.grid_line_color = None
q.ygrid.grid_line_color = None
q.outline_line_width = 0
q.background_fill_color = '#f5f5f5'
q.background_fill_alpha = 0.9
q.legend.location = 'top_right'
q.title.text_color = '#a0a0a0'
q.title.text_font_size = '24pt'
q.title.text_font = "helvetica"
q.legend.label_text_font = "helvetica"
q.legend.label_text_color = "#a0a0a0"
q.add_tools(hover2)

final = gridplot([[p,q]])
show(final)<jupyter_output><empty_output><jupyter_text>testing dot log plot<jupyter_code>dot_dict = graph_dict['UK Biobank']['Ethnicity']
dot_dict['log'] = [math.log(i) for i in dot_dict['percent']]
dot_dict['ref log'] = [math.log(i) for i in dot_dict['ref percent']]
dot_dict['lab_cords'] =[math.log(i) for i in [1,10,25,50,100]]
dot_dict['lab_cords_y'] = [6]*len(dot_dict['Ethnicity'])
dot_dict['label_perc'] =  ['1%','10%','25%','50%','100%']
dot_dict['new_y'] = [1,2,3,4,5]
dot_dict['label_x'] = [-1.7] * 5
print(dot_dict)
source = ColumnDataSource(data = dot_dict)

r = figure(title = 'Ethnicity -log values',x_range=(-1.7,max(source.data['log'])*1.1),y_range=(0.5,6.2))
r.segment('log','new_y','ref log','new_y', color = '#555555',line_width = 3,source = source)
r.circle(x = 'ref log',y = 'new_y', color = '#a0a0a0',size = 10,legend_label = 'UK Population',source = source)
r.circle(x = 'log',y = 'new_y', color = '#003667',size = 10 ,legend_label = 'UK Biobank',source = source)



logone = Span(location = math.log(1),dimension = 'height', line_color = '#555555',line_alpha =0.2, line_width = 3)
log10 = Span(location = math.log(10),dimension = 'height', line_color = '#555555',line_alpha =0.2, line_width = 3)
log25 = Span(location = math.log(25),dimension = 'height', line_color = '#555555',line_alpha =0.2, line_width = 3)
log50 = Span(location = math.log(50),dimension = 'height', line_color = '#555555',line_alpha =0.2, line_width = 3)
log100 = Span(location = math.log(100),dimension = 'height', line_color = '#555555',line_alpha =0.2, line_width = 3)

box1 = BoxAnnotation(top = 1.5, bottom =2.5, fill_color = '#000000',fill_alpha = 0.2)
box2 = BoxAnnotation(top = 3.5, bottom =4.5, fill_color = '#000000',fill_alpha = 0.2)

hover4 = HoverTool(tooltips = [
('Ethnicity', '@Ethnicity'),
('Raw values', "@{values}"),
('Percent/%', "@{percent}{0.0}"),
('UK population percent/%', '@{ref percent}{0.0}')
],
       mode = 'mouse', name= 'data plot')

labels = LabelSet(
        x='lab_cords', 
        y='lab_cords_y', 
        text='label_perc',
        text_align='right', 
        text_font ='helvetica',
        text_color = 'grey',
        source=source
    )

labels2 = LabelSet(
        x='label_x', 
        y='new_y', 
        text='Ethnicity',
        text_align='left', 
        text_font ='helvetica',
        text_color = 'grey',
        source=source
    )

r.yaxis.major_label_text_font_size = '0pt' 
r.yaxis.major_tick_line_color = None
r.yaxis.minor_tick_line_color = None
r.xaxis.major_tick_line_color = None  # turn off y-axis major ticks
r.xaxis.minor_tick_line_color = None
r.yaxis.axis_line_color = None
r.xaxis.axis_line_color = None
r.xaxis.major_label_text_font_size = '0pt'
r.xaxis.major_tick_line_color = None
r.xgrid.grid_line_color = None
r.ygrid.grid_line_color = None
r.outline_line_width = 0
r.background_fill_color = '#f5f5f5'
r.background_fill_alpha = 0.9

r.title.text_color = '#a0a0a0'
r.title.text_font_size = '24pt'
r.title.text_font = "helvetica"
r.add_layout(box1)
r.add_layout(box2)
r.add_layout(logone)
r.add_layout(log10)
r.add_layout(log25)
r.add_layout(log50)
r.add_layout(log100)
r.add_layout(labels2)
r.add_tools(hover4)
r.add_layout(labels)
r.legend.location = 'top_left'
r.legend.label_text_font = "helvetica"
r.legend.label_text_color = "#a0a0a0"
output_file('plots/ethnicitylogs.html')
show(r)<jupyter_output><empty_output><jupyter_text>Split dotplot<jupyter_code>dot_dict['label_x'] =[0]*5
dot_dict.keys()
def dot_plot(source,x_range,plot_width):
    if x_range[0] < 0:
        line_val = x_range[1]
        title = 'Ethnicity'
        place = 'right'
        other_place = 'left'
        line_end = 0
    else: 
        line_val = x_range[0]
        title = ''
        place = 'left'
        line_end = 100
        other_place = 'right'
    
    if line_end == 0:
        line_width = 1.5
    else:
        line_width = 3
    
    r = figure(title = title,x_range=x_range,y_range=(0.5,6.2),plot_width = plot_width)
    r.segment('percent','new_y','ref percent','new_y', color = '#555555',line_width = 3,source = source)
    r.circle(x = 'ref percent',y = 'new_y', color = '#a0a0a0',size = 10,legend_label = 'UK Population',source = source)
    r.circle(x = 'percent',y = 'new_y', color = '#003667',size = 10 ,legend_label = 'UK Biobank',source = source)
    
    line = Span(location = line_val,dimension = 'height', line_color = '#555555',line_alpha =0.2, line_width = 3)
    end_line = Span(location = line_end,dimension = 'height', line_color = '#555555',line_alpha =0.2, line_width = line_width)

    box1 = BoxAnnotation(top = 1.5, bottom =2.5, fill_color = '#000000',fill_alpha = 0.1)
    box2 = BoxAnnotation(top = 3.5, bottom =4.5, fill_color = '#000000',fill_alpha = 0.1)

    hover4 = HoverTool(tooltips = [
    ('Ethnicity', '@Ethnicity'),
    ('Raw values', "@{values}"),
    ('Percent/%', "@{percent}{0.0}"),
    ('UK population percent/%', '@{ref percent}{0.0}')
    ],
           mode = 'mouse', name= 'data plot')

    label = Label(x=line_val, y=6, 
                     text=str(line_val) + '%', render_mode='canvas',text_align = place,
                     border_line_alpha=0,
                     background_fill_alpha=0,
                     text_font = 'helvetica', 
                     text_color = '#a0a0a0'
                    )
    
    label_end = Label(x=line_end, y=6, 
                     text=str(line_end) + '%', render_mode='canvas',text_align = other_place,
                     border_line_alpha=0,
                     background_fill_alpha=0,
                     text_font = 'helvetica', 
                     text_color = '#a0a0a0'
                    )

    labels2 = LabelSet(
            x='label_x', 
            y='new_y', 
            text='Ethnicity',
            text_align='right', 
            text_font ='helvetica',
            text_color = 'grey',
            source=source
        )

    r.yaxis.major_label_text_font_size = '0pt' 
    r.yaxis.major_tick_line_color = None
    r.yaxis.minor_tick_line_color = None
    r.xaxis.major_tick_line_color = None  # turn off y-axis major ticks
    r.xaxis.minor_tick_line_color = None
    r.yaxis.axis_line_color = None
    r.xaxis.axis_line_color = None
    r.xaxis.major_label_text_font_size = '0pt'
    r.xaxis.major_tick_line_color = None
    r.xgrid.grid_line_color = None
    r.ygrid.grid_line_color = None
    r.outline_line_width = 0
    r.background_fill_color = '#f5f5f5'
    r.background_fill_alpha = 0.9
    r.min_border = 0
    r.title.text_color = '#a0a0a0'
    r.title.text_font_size = '24pt'
    r.title.text_font = "helvetica"
    r.add_layout(box1)
    r.add_layout(box2)
    r.add_layout(line)
    r.add_layout(end_line)
    if x_range[0] < 0:
        r.add_layout(labels2)
    r.add_tools(hover4)
    
    r.add_layout(label)
    r.add_layout(label_end)
    if x_range[0] >0:
        r.legend.location = 'top_right'
        r.legend.label_text_font = "helvetica"
        r.legend.label_text_color = "#a0a0a0"
    else:
        r.legend.glyph_height = 0
        r.legend.glyph_width = 0
        r.legend.label_text_font_size = '0pt'
        r.legend.background_fill_alpha = 0 
        r.legend.border_line_alpha = 0
    return(r)
source = ColumnDataSource(data = dot_dict)
left_range = (-4.75,10)
right_range = (85,100)
total_length = left_range[1] - left_range[0]+right_range[1] - right_range[0]
name_width = round(abs(left_range[0])/total_length*600)
left_plot_width = round((left_range[1])/total_length*(600 - name_width))
right_plot_width = round((right_range[1] - right_range[0])/total_length*(600 - name_width))
left = dot_plot(source,left_range,left_plot_width + name_width)
right = dot_plot(source,right_range,right_plot_width)
full_plot = gridplot([[left,right]])
output_file('plots/split.html')
show(full_plot)
print(left_plot_width)
print(right_plot_width)
print(name_width)<jupyter_output>169
254
96<jupyter_text>Boxy sanky<jupyter_code>box_dict = graph_dict['UK Biobank']['Ethnicity']
perc = box_dict['percent']
ref_p = box_dict['ref percent']
box_dict['y_coords'] = [[80 if i== 0 else sum(perc[:i]) ,
                         sum(perc[:i+1]),
                         sum(ref_p[:i+1]),
                         80 if i == 0 else sum(ref_p[:i])] for i in range(len(perc))]

box_dict['x_coords'] = [[0,0,100,100] for i in range(len(perc))]
box_dict['colours'] = ["#dbeaff","#9dd8e7","#006272","#ffe699","#fec200"]
print(box_dict['y_coords'])
print(box_dict['x_coords'])
source = ColumnDataSource(data = box_dict)
q = figure(title = 'Ethnicity')
colours = ["#cd8845","#9dd8e7","#fec200","#ffe699","#006272"]

q.patches(xs='x_coords', ys='y_coords',color='colours',source = source)
hover4 = HoverTool(tooltips = [
    ('Ethnicity', '@Ethnicity'),
    ('Raw values', "@{values}"),
    ('Percent/%', "@{percent}{0.0}"),
    ('UK population percent/%', '@{ref percent}{0.0}')
    ],
           mode = 'mouse', name= 'data plot')

q.add_tools(hover4)

source = ColumnDataSource(data = box_dict)
q = figure(title = 'Ethnicity',x_range=(-10,110))
colours = ["#003667","#ed6b00","#87f5fb","#a882dd","#721817"]

q.patches(xs='x_coords', ys='y_coords',color='colours',legend_field = 'Ethnicity',line_color = 'white',line_width =1.5,source = source)
    
   
    
perc_lab_cords = np.array([i[0] for i in source.data['y_coords']] + [100])
perc_x_lab_cords = np.array([0] * len(perc_lab_cords))
y_labels = [str(i)+'%' for i in perc_lab_cords]
perc_lab_cords = perc_lab_cords - 0.5


ref_p_lab_cords = np.array([i[3] for i in source.data['y_coords']] + [100])
ref_p_x_lab_cords = np.array([100] * len(perc_lab_cords))
ref_y_labels = [str(i)+'%' for i in ref_p_lab_cords]
ref_p_lab_cords = ref_p_lab_cords -0.5

hover4 = HoverTool(tooltips = [
    ('Ethnicity', '@Ethnicity'),
    ('Raw values', "@{values}"),
    ('Percent/%', "@{percent}{0.0}"),
    ('UK population percent/%', '@{ref percent}{0.0}')
    ],
           mode = 'mouse', name= 'data plot')

for i in range(len(perc_lab_cords)):
    label = Label(x = perc_x_lab_cords[i],y= perc_lab_cords[i],text = y_labels[i], render_mode='canvas',text_align = 'right',
        border_line_alpha=0,background_fill_alpha=0,text_font = 'helvetica', text_color = '#a0a0a0',
                  text_font_size = '10pt')
    
    label2 = Label(
        x = ref_p_x_lab_cords[i],
        y = ref_p_lab_cords[i],
        text = ref_y_labels[i],
        render_mode='canvas',
        text_align = 'left',
        border_line_alpha=0,
        background_fill_alpha=0,
        text_font = 'helvetica', 
        text_color = '#a0a0a0',
        text_font_size = '10pt')
    
    q.add_layout(label)
    q.add_layout(label2)
    
dataset_lab = Label(x = 20, y = 100, text = 'UK Biobank', render_mode='canvas',text_align = 'right',
        border_line_alpha=0,background_fill_alpha=0,text_font = 'helvetica', text_color = '#a0a0a0')
ref_lab = Label(x = 100, y = 100, text = 'UK Population', render_mode='canvas',text_align = 'right',
        border_line_alpha=0,background_fill_alpha=0,text_font = 'helvetica', text_color = '#a0a0a0')

q.yaxis.major_label_text_font_size = '0pt' 
q.yaxis.major_tick_line_color = None
q.yaxis.minor_tick_line_color = None
q.xaxis.major_tick_line_color = None  # turn off y-axis major ticks
q.xaxis.minor_tick_line_color = None
q.yaxis.axis_line_color = None
q.xaxis.axis_line_color = None
q.xaxis.major_label_text_font_size = '0pt'
q.xaxis.major_tick_line_color = None
q.xgrid.grid_line_color = None
q.ygrid.grid_line_color = None
q.outline_line_width = 0
q.background_fill_color = '#f2f6fe'
q.background_fill_alpha = 1
q.title.text_color = '#a0a0a0'
q.title.text_font_size = '24pt'
q.title.text_font = "helvetica"
q.legend.location = (46,24)
q.legend.label_text_font = "helvetica"
q.legend.label_text_color = "#a0a0a0"
q.add_layout(dataset_lab)
q.add_layout(ref_lab)
q.add_tools(hover4)
output_file('plots/boxysanky.html')
show(q)


print(perc_lab_cords)
print(perc_lab_cords -5)<jupyter_output>[79.5 94.3 95.8 98.  98.6 99.5]
[74.5 89.3 90.8 93.  93.6 94.5]<jupyter_text>multi pie plot<jupyter_code>pie_dict = graph_dict['UK Biobank']['Ethnicity']
pie_dict['angles'] = [i/sum(pie_dict['percent']) * 2*math.pi for i in pie_dict['percent']]
pie_dict['start_angle'] = [0] + [sum(pie_dict['angles'][:i+1]) for i in range(len(pie_dict['angles']) -1)]
pie_dict['end_angle'] = [sum(pie_dict['angles'][:i+1]) for i in range(len(pie_dict['angles']))]

pie_dict['ref_angles'] = [i/sum(pie_dict['percent']) * 2*math.pi for i in pie_dict['ref percent']]
pie_dict['ref_start_angle'] = [0] + [sum(pie_dict['ref_angles'][:i+1]) for i in range(len(pie_dict['ref_angles']) -1)]
pie_dict['ref_end_angle'] = [sum(pie_dict['ref_angles'][:i+1]) for i in range(len(pie_dict['ref_angles']))]
pie_dict['colours'] = colours
print(pie_dict['start_angle'])
source = ColumnDataSource(data = pie_dict)
s = figure(title = 'Ethnicity',x_range = (-0.6,0.8),y_range =(-0.6,0.6) )
s.annular_wedge(
    x =0,
    y=0,
    inner_radius = 0.31, 
    outer_radius = 0.5,
    start_angle = 'start_angle',
    end_angle ='end_angle',
    color = 'colours',
    legend_field = 'Ethnicity',
    source = source
)

s.annular_wedge(
    x =0,
    y=0,
    inner_radius = 0.1, 
    outer_radius = 0.29,
    start_angle = 'ref_start_angle',
    end_angle ='ref_end_angle',
    color = 'colours',
    source = source
)

hover4 = HoverTool(tooltips = [
    ('Ethnicity', '@Ethnicity'),
    ('Raw values', "@{values}"),
    ('Percent/%', "@{percent}{0.0}"),
    ('UK population percent/%', '@{ref percent}{0.0}')
    ],
           mode = 'mouse', name= 'data plot')
dataset_lab = Label(x = 0.6, y = 0, text = 'UK Biobank', render_mode='canvas',text_align = 'right',
        border_line_alpha=0,background_fill_alpha=0,text_font = 'helvetica', text_color = '#a0a0a0')
ref_lab = Label(x = 0.25, y = 0, text = 'UK Population', render_mode='canvas',text_align = 'right',
        border_line_alpha=0,background_fill_alpha=0,text_font = 'helvetica', text_color = '#a0a0a0')

s.yaxis.major_label_text_font_size = '0pt' 
s.yaxis.major_tick_line_color = None
s.yaxis.minor_tick_line_color = None
s.xaxis.major_tick_line_color = None  # turn off y-axis major ticks
s.xaxis.minor_tick_line_color = None
s.yaxis.axis_line_color = None
s.xaxis.axis_line_color = None
s.xaxis.major_label_text_font_size = '0pt'
s.xaxis.major_tick_line_color = None
s.xgrid.grid_line_color = None
s.ygrid.grid_line_color = None
s.outline_line_width = 0
s.background_fill_color = '#f5f5f5'
s.background_fill_alpha = 0.9
s.title.text_color = '#a0a0a0'
s.title.text_font_size = '24pt'
s.title.text_font = "helvetica"
s.legend.label_text_font = "helvetica"
s.legend.label_text_color = "#a0a0a0"
s.add_layout(dataset_lab)
s.add_layout(ref_lab)
s.add_tools(hover4)
output_file('plots/donuts.html')
show(s)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16153507807345502, 'fraction_numerical': 0.03848691444908731, 'mean_word_length': 4.183528070675406, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '149252', 'license': 'MIT', 'n_tokens_mistral': 7615, 'n_tokens_neox': 7162, 'n_words': 1204, 'path': 'representation_labels/testing_ethnicity_plots.ipynb', 'repo_name': 'wellcometrust/nutrition-labels', 'size': '18188'}