<jupyter_start><jupyter_text>Configuration<jupyter_code>%%output backend='bokeh'
%%opts Table.gates [width=1200]
%%opts Table.summary (row_headers=False) [width=1200]
configLabel="pipeline.config"
gates=config_objects(configLabel)
num_gates = len(gates)
summary=config_summary(configLabel, headers)
num_gates = len(summary)

gatesummary = [v for v in summary.values()]
di = {summary.get(element)[0]:str(summary.get(element)[0]).zfill(2)+"_"+summary.get(element)[4] for i,element in enumerate(summary)}
summaryTable=hv.Table(gatesummary,kdims=['Population','Parent','XMarker','YMarker','phenotype','startx', 'endx', 'starty', 'endy'], group='summary', label='Summary')

sortedTable=summaryTable.sort('Population')
sortedTable<jupyter_output><empty_output><jupyter_text>Marker and Axis Tables<jupyter_code>%%output backend='bokeh'
axis_popIndexDict = defaultdict(list)
popBounds={}
axises=[]
composite_axis=0
last_xmarker=""
last_ymarker=""
last_parent=0
gatesconfig=[]
for i in range(len(gates)):
    pop="pop"+str(i+1)
    config=gates.get(pop)
    xmarker=str(headers[config[1]-1])
    ymarker=str(headers[config[2]-1])
    startx=int((float(config[3])/200)*4096)
    starty=int((float(config[5])/200)*4096)
    endx=int((float(config[4])/200)*4096)
    endy=int((float(config[6])/200)*4096)
    parent=int(config[7])
    ctype=int(config[8])
    popBounds.update({pop:[xmarker, ymarker, startx,starty,endx,endy,ctype,"pop"+str(parent)]})
    key="axis"+str(composite_axis).zfill(2) 
    if (xmarker != last_xmarker) or (ymarker != last_ymarker) or (parent != last_parent):
        composite_axis=composite_axis+1
        key="axis"+str(composite_axis).zfill(2)
        axises.append([xmarker,ymarker,key,"pop"+str(parent)])
    axis_popIndexDict[key].append(pop)
    last_xmarker=xmarker
    last_ymarker=ymarker
    last_parent=parent
    gatesconfig.append([pop,xmarker,ymarker,parent])

num_axises = len(axises)
markerTable=hv.Table(markers,kdims=['Markers'])
axis_popTable=hv.Table(axis_popIndexDict, kdims=['Axis Index'], vdims=['sub populations'])
markerTable+axis_popTable.sort('Axis Index')<jupyter_output><empty_output><jupyter_text>Batch Population Statistics<jupyter_code>hv.notebook_extension('matplotlib')<jupyter_output><empty_output><jupyter_text>Population Percentage and Events Tables<jupyter_code>batchpercent_df = pd.read_csv('Gated/Batch_percentages.txt', sep='\t', index_col=0)
batchpercent_df=batchpercent_df.rename(di).round(2)
batchevents_df = pd.read_csv('Gated/Batch_events.txt', sep='\t', index_col=0)
batchevents_df=batchevents_df.rename(di)
batchnorm_df = colNormalize(batchevents_df)
display(md("#### Events table"))
with pd.option_context('display.max_columns', None):
    display(batchevents_df)
display(md("#### Population Percentage (by Parent's size) table"))
with pd.option_context('display.max_columns', None):
    display(batchpercent_df)
display(md("#### Population Percentage (by Singlets size) table"))
with pd.option_context('display.max_columns', None):
    display(batchnorm_df)
#percentdf=batchpercent_df.reindex(sorted(batchpercent_df.columns), axis=1).transpose()
percentdf=batchnorm_df.transpose()<jupyter_output><empty_output><jupyter_text>Combined Percent/Events Dataframe<jupyter_code>%%output backend="bokeh"
%%opts Table [width=1000]

p_df=pd.DataFrame(batchpercent_df.unstack())
p_df.columns=['Percent']

e_df=pd.DataFrame(batchevents_df.unstack())
e_df.columns=['Events']

c_df=pd.concat([p_df,e_df],axis=1, join='outer').reset_index()
c_df.columns=['Sample','Population','Percent','Events']
c_df=c_df.replace({"Population":di})<jupyter_output><empty_output><jupyter_text>Box Plots<jupyter_code>%%output backend="bokeh" size=200
%%opts BoxWhisker [xrotation=45]
percentBoxPlot=hv.BoxWhisker(c_df, kdims=['Population'],vdims='Percent').relabel('Population Percent Box Plot')
eventsBoxPlot=hv.BoxWhisker(c_df, kdims=['Population'],vdims='Events').relabel('Population Events Box Plot')
percentBoxPlot+eventsBoxPlot
%%output backend="bokeh"
%%opts Scatter [width=1200 height=600 scaling_method='width' scaling_factor=0.1 size_index=2 show_grid=True tools=['hover']] 
%%opts Scatter (color=Cycle('Category20') alpha=0.8 line_color='k')
%%opts NdOverlay [legend_position='bottom' show_frame=False]
cdf_table=hv.Table(c_df,kdims=['Sample','Population'],vdims=['Percent','Events'])
cdf_scatter = cdf_table.to.scatter('Population', ['Percent','Events'])
cdf_plot=cdf_scatter.overlay('Sample').relabel("Cross Sample Population Percent Comparison with Events Scaling")
(percentBoxPlot*cdf_plot).relabel("All-sample Population Percentage Boxplot with Relative Events Scaling")
#bdf_scatter.overlay('Sample')<jupyter_output><empty_output><jupyter_text>Composite 2D Dot-Plots Gated Populations<jupyter_code>hv.opts("RGB [width=600, height=600, bgcolor='#D3D3D3', fontsize={'title':8, 'xlabel':15, 'ylabel':15, 'ticks':10}]")
hv.opts("Points.cent (color='purple' marker='+' size=10)")
poplist=natural_sort(di.values())
size=600
popdfPlots = hv.HoloMap({(sample, poplist[j]): datashade(hv.Points(dfArray[k][1].loc[dfArray[k][1][gate[0]]==0], kdims=[gate[1], gate[2]]), width=size, height=size, x_range=(0,4096), y_range=(0,4096), dynamic=False, link_inputs=False, cmap=cmap_pop[j])
                    for k, sample in enumerate(sample_labels) for j, gate in enumerate(gatesconfig)}, kdims=['Sample', 'Pop'])
alldfPlots = hv.HoloMap({(sample, poplist[j]): datashade(hv.Points(dfArray[k][1], kdims=[gate[1], gate[2]]), width=size, height=size, x_range=(0,4096), y_range=(0,4096), dynamic=False, link_inputs=False, cmap=cmap_all)
                    for k, sample in enumerate(sample_labels) for j, gate in enumerate(gatesconfig)}, kdims=['Sample', 'Pop'])
parentdfPlots = hv.HoloMap({(sample, poplist[j]): datashade(hv.Points(dfArray[k][1].loc[(dfArray[k][1]["pop"+str(gate[3])]==0) & (dfArray[k][1][gate[0]]==1)], kdims=[gate[1], gate[2]]), width=size, height=size, x_range=(0,4096), y_range=(0,4096), dynamic=False, link_inputs=False, cmap=cmap_parent)
                    for k, sample in enumerate(sample_labels) for j, gate in enumerate(gatesconfig)}, kdims=['Sample', 'Pop'])
boundarydfPlots = hv.HoloMap({(sample, poplist[j]): hv.Bounds((popBounds.get(gate[0])[2], popBounds.get(gate[0])[3], popBounds.get(gate[0])[4], popBounds.get(gate[0])[5])).opts(style=dict(line_color=cmap_pop[j][0],color=cmap_pop[j][0]))
                    for k, sample in enumerate(sample_labels) for j, gate in enumerate(gatesconfig)}, kdims=['Sample', 'Pop'])
# centroiddfPlots = hv.HoloMap({(sample, j+1): hv.Points(centDict.get(sample+"_"+reclustermap.get(gate[0])), kdims=[gate[1], gate[2]], group="cent")
#                     for k, sample in enumerate(sample_labels) for j, gate in enumerate(gatesconfig)}, kdims=['Sample', 'Pop'])

def outputSampleGates(sample):
    newName=sample
    filename=export_path+"/"+newName
    hv.output(hv.NdLayout(combineddfPlots[newName,:]).cols(1), backend='matplotlib', size=200, fig='png', filename=filename)
    
    return filename

def outputPopGates(pop):
    filename=export_path+"/"+pop.replace(" ", "")
    length=len(combineddfPlots[:,pop])
    hv.output(hv.NdLayout(combineddfPlots[:,pop]).cols(6), backend='matplotlib', size=100, fig='png', filename=filename)
    
    return filename

def outputPopBokehGates(pop):
    
    
    filename=export_path+"/"+pop.replace(" ", "")
    length=len(combineddfPlots[:,pop])
    temp=hv.NdLayout(combineddfPlots[:,pop]).cols(length)
    
    return temp
#%%time
combineddfPlots=alldfPlots*parentdfPlots*popdfPlots*boundarydfPlots
scomp_pool = Pool(processes=min(10,numOfCores))
for pop in poplist:
    scomp_pool.apply_async(outputPopGates, args=[pop])
scomp_pool.close()
scomp_pool.join()
def composite_pop(pop):
    pop="func_"+pop.replace(" ", "").replace("-","n").replace("+","p")
    
    script='''<script>{}_show=true; 
    function {}_toggle() {{
    if ({}_show){{
    $('.{}').hide();
    }} else {{
    $('.{}').show();
    }}
    {}_show = !{}_show
    }} 
    $( document ).ready({}_toggle);
    </script>'''.format(pop, pop, pop, pop, pop, pop, pop, pop)
    return script

def composite_gates():
    pop="gates"
    
    script='''<script>{}_show=true; 
    function {}_toggle() {{
    if ({}_show){{
    $('.{}').hide();
    }} else {{
    $('.{}').show();
    }}
    {}_show = !{}_show
    }} 
    $( document ).ready({}_toggle);
    </script>'''.format(pop, pop, pop, pop, pop, pop, pop, pop)
    return script

def display_pop(pop):
    pop=pop.replace(" ", "")
    script="<div class=\""+"func_"+pop.replace("-","n").replace("+","p")+"\" >"+html_row(pop)+"</div>"
    return script

def buttons_pop(pop):
    pop=pop.replace(" ", "")
    popfunction = "func_"+pop.replace("-","n").replace("+","p")
    script='''<input type="submit" formaction="javascript:{}_toggle()" value="{}">'''.format(popfunction,pop)
    return script
def button_gates():
    script='''<input type="submit" formaction="javascript:{}_toggle()" value="{}">'''.format("gates","gates")
    return script
display(md("### Click to toggle show/hide Population or Gating-hierarchy Composite Plots"))
buttonsHtml="<form>"
for i, pop in enumerate(poplist):
    if (i%10) != 0:
        buttonsHtml=buttonsHtml+buttons_pop(pop)
    else:
        buttonsHtml=buttonsHtml+"</form><form>"+buttons_pop(pop)
buttonsHtml=buttonsHtml+"</form>"+"<form>"+button_gates()+"</form>"
HTML(buttonsHtml)
pophtml="".join(display_pop(pop) for pop in poplist)
display(HTML(pophtml))
#%%time
scomp_pool = Pool(processes=min(10,numOfCores))
for sample in sample_labels:
    scomp_pool.apply_async(outputSampleGates, args=[sample])
scomp_pool.close()
scomp_pool.join()
gateshtml="".join(html_row(sample) for sample in natsorted(sample_labels))
gateshtml="<div class=\"gates\">"+gateshtml+"</div>"
display(HTML(gateshtml))
HTML("".join(composite_pop(pop) for pop in natsorted(poplist))+composite_gates())<jupyter_output><empty_output><jupyter_text>Crosslinked tSNE and Tabular View of Batch Population Percentage<jupyter_code>tsne_data_array=percentdf.values.astype(np.float64)
data_tsne = tsne.fit_transform(np.copy(tsne_data_array))
dfn=pd.DataFrame(data_tsne, columns=['tsne-x','tsne-y'], index=percentdf.index).round(4)
results=pd.concat([percentdf,dfn],axis=1)
results.insert(0,"file",percentdf.index)
colnames=list(results)[0:-2]
# with pd.option_context('display.max_columns', None, 'display.max_rows', 20):
#     display(results)
from bokeh.models import HoverTool
hover = HoverTool(tooltips="""
    <div>
        <div>
            <rotate style="font-size: 17px; font-weight: bold;">@file</rotate>
            <rotate style="font-size: 15px; color: #966;">[$index]</rotate>
        </div>
        <div>
            <img
                src="./PNG/@file.png" height="300" width="100"
                style="float: left; margin: 0px 0px 0px 0px;"
                border="2"
            ></img>
        </div>
         
    </div>
    """
)
from bokeh.models.widgets import DataTable, TableColumn, HTMLTemplateFormatter
template="""
<div style="background:<%= 
    (function colorfromint(){
        if(value >= 95){
            return("#200000")}
        else if(value >= 80){
            return("#600000")}
        else if(value >= 70){
            return("#800000")}
        else if(value >= 60){
            return("#A00000")}
        else if(value >= 50){
            return("#FF0000")}
        else if(value >= 40){
            return("#CC3333")}
        else if(value >= 30){
            return("#FF9966")}
        else if(value >= 15){
            return("#FFCC99")}
        else if(value >= 3){
            return("#FFFFCC")}
        
        }()) %>; 
    color:<%= 
    (function colorfromint(){
        if(value >= 30){
            return("white")}
        else{return("black")}
        }()) %>"> 
<%= value %></div>
"""
formater =  HTMLTemplateFormatter(template=template)

def apply_format(plot, element):
    construct=plot.handles['plot']
    for i, column in enumerate(construct.columns[1:-2]):
        column.formatter=formater
    
%%output backend='bokeh'
%%opts Scatter.tSNE (size=5 nonselection_color='grey' cmap='Reds') [bgcolor='#D3D3D3' color_index=2 width=500 height=500 tools=['hover','box_select','poly_select','reset']] 
%%opts Layout [shared_datasource=True]
%%opts Table.tSNE (row_headers=False) [width=1600 height=500]
labels=[kd for i, kd in enumerate(colnames[4:len(colnames)])]
holomap = hv.HoloMap({(kd): hv.Scatter(results, kdims=['tsne-x','tsne-y'],vdims=[kd, 'file'], group="tSNE") for i, kd in enumerate(colnames[4:len(colnames)])}, kdims='Population')
tSNEtable = hv.Table(results, group="tSNE")
display(md("####      Click on box selection tool to enable tSNE events selection"))
hv.Layout(tSNEtable.opts(plot=dict(finalize_hooks=[apply_format]))+holomap.layout().cols(3)).cols(1)
HTML('''<script>
code_show=true; 
function code_toggle() {
 if (code_show){
 $('div.input').hide();
 } else {
 $('div.input').show();
 }
 code_show = !code_show
} 
$( document ).ready(code_toggle);
</script>
<form action="javascript:code_toggle()"><input type="submit" value="Click here to toggle on/off the raw code."></form>''')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18676895032659882, 'fraction_numerical': 0.022026431718061675, 'mean_word_length': 5.7871134020618555, 'pattern_counts': {'":': 1, '<': 71, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '299356', 'license': 'MIT', 'n_tokens_mistral': 4880, 'n_tokens_neox': 4658, 'n_words': 722, 'path': 'Notebooks/StaticCompositePlots.ipynb', 'repo_name': 'JCVenterInstitute/DAFi-gating', 'size': '13166'}