<jupyter_start><jupyter_text>Exploration Data and Design Line Notebook Notebook to to Upload data, view it, change NA values, explore it in a GUI, modify it, summary statistics (pivot), then use for design lines as a combined tab  How to use NotebookThe packages must be loaded first, then upload file and run output. This will run all cells below.Current order of cells is as follows: -  This notebook will only work fully if you have the following columns 'Depth [m], Location, lat and lon'; -  Show Null values as a heatmap and replace common typos and blanks to NaN for plot use; -  Create a GUI to explore the data that can be used to remake the csv; -  Create excel like table that can be modified further for one time use (allows experimentation of table without saving it); -  Press widget to proceed if modified, leave if not; -  Saves current df into the notebook to be used later, if original file has been modified, modifed version will be used, if unmodified, original will be used; -  Pivot table of data to see summary statistics; -  Custom tab template to view data; -  Map has not been tested (no coordinate data to use), but will filter locations based on filters used once implemented;Current issues with notebook -  Unable to let user choose category and purpose columns - locked to Depth and Location currently; -  Slow processing of data if >10000 points, requires a different scatter plot generator or reduction of visuals available; -  No static labels on map - possible to implement. Tasks to implement: -  Add custom filter for category and purpose, done, but slow; -  Create app for all users to use.<jupyter_code># Standard Data Science Helpers
#data reading
from IPython.display import Javascript
import numpy as np
import pandas as pd
import os
import qgrid
#plots
import plotly.express as px
import plotly.offline as pyoff
import plotly.graph_objs as go
import plotly.io as pio
import seaborn as sns
import matplotlib.pyplot as plt
#widgets
import ipywidgets as widgets
from ipywidgets import interact, SelectMultiple, interact_manual
#maps
import geopandas as gpd
#remove warnings
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)

#allow plots to work offline
from plotly.offline import iplot, init_notebook_mode, plot, download_plotlyjs
init_notebook_mode(connected=True)
pyoff.init_notebook_mode(connected=True)

import cufflinks as cf
cf.go_offline(connected=True)
cf.set_config_file(colorscale='plotly', world_readable=True)

# Show all code cells outputs
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = 'all'

#Upload file
import io
from ipywidgets import FileUpload
upload = widgets.FileUpload(
    accept='.csv, .xlsx',  # Accepted file extension e.g. '.txt', '.pdf', 'image/*', 'image/*,.pdf'
    multiple=False  # True to accept multiple files upload else False
    )

display(upload)

def run_all(ev):
    display(Javascript('IPython.notebook.execute_cells_below()'))

button = widgets.Button(description="Press this")
button.on_click(run_all)
display(button)
#Take upload file and turn into a dataframe
input_file = list(upload.value.values())[0]
content = input_file['content']
content = io.StringIO(content.decode('utf-8'))
df = pd.read_csv(content)
#df.head() #check data
missing_values=['??','na','N/A'] #add other repitive values if needed
df=df.replace(missing_values,np.NaN)
qgrid_widget = qgrid.show_grid(df, show_toolbar=True, grid_options={'forceFitColumns': False, 'defaultColumnWidth': 100})
qgrid_widget
def run_all(ev):
    display(Javascript('IPython.notebook.execute_cells_below()'))

button = widgets.Button(description="Use Modified")
button.on_click(run_all)
display(button)
df = qgrid_widget.get_changed_df()
#df.to_csv('C:/Users/abc/Desktop/file_name.csv') #add folder location and file name
#error? try this df.to_csv('file_name.csv', encoding='utf-8')
#Pivot Table for sumary stats
from pivottablejs import pivot_ui
pivot_ui(df,outfile_path='pivottablejs.html')
ALL = 'ALL'
def unique_sorted_values_plus_ALL(array):
    unique = array.unique().tolist()
    unique.sort()
    unique.insert(0, ALL)
    return unique
#Testing scatter plot
stats_tab = widgets.Output()
EDA_tab = widgets.Output()
trendline_tab = widgets.Output()
map_tab = widgets.Output()

dropdown_year = widgets.Dropdown(options = unique_sorted_values_plus_ALL(df.Units), description='Units:')
dropdown_purpose = widgets.Dropdown(options = unique_sorted_values_plus_ALL(df.Location), description='Location:')

def common_filtering(year, purpose):
    stats_tab.clear_output()
    EDA_tab.clear_output()
    trendline_tab.clear_output()
    map_tab.clear_output()
    
    if (year == ALL) & (purpose == ALL):
        common_filter = df
    elif (year == ALL):
        common_filter = df[df.Location == purpose]
    elif (purpose == ALL):
        common_filter = df[df.Units == year]
    else:
        common_filter = df[(df.Units == year) & (df.Location == purpose)]


    with stats_tab:
        @interact
        def test7(Value=list(df.select_dtypes('number').columns)[1:]):
            g = common_filter.groupby('Units')
            print(g[Value].describe())
    
    with EDA_tab:
        @interact_manual
        def EDA(x=list(df.select_dtypes('number').columns)[0:],
                y=list(df.select_dtypes('number').columns)[1:],
                theme=list(cf.themes.THEMES.keys()),
                colorscale=list(cf.colors._scales_names.keys())):

            scatter = common_filter.iplot(kind='scatter', x=x, y=y, mode='markers', categories='Units', subplots=False, 
                     xTitle=x.title(), yTitle=y.title(),
                     title=f'{y.title()} vs {x.title()}',
                    theme=theme, colorscale=colorscale, asFigure=True)
            scatter['layout']['yaxis']['autorange'] = "reversed"
            scatter['layout']['xaxis'] = dict(side='top')
            scatter.update_layout(autosize=True)
            scatter.iplot()
        
            box = common_filter.pivot(columns='Units', values=y).iplot(
            kind='box', theme=theme, colorscale=colorscale,
            yTitle=y.title(),
            title=f'{y.title()} vs Unit')

    with trendline_tab:
        @interact_manual
        def Trend(x=list(df.select_dtypes('number').columns)[0:],
                  y=list(df.select_dtypes('number').columns)[1:],
                  category1=list(df.select_dtypes('object').columns)[0:],                     
                  template=list(pio.templates.keys())):

            fig = px.scatter(common_filter, x=x, y=y, color=category1, opacity=0.5,
                             trendline="lowess", trendline_color_override="red",
                             marginal_y="box", render_mode='auto', template=template)
            #alternatively use box or violin for marginals
            #Below for cursor options/crosshairs
            fig.update_xaxes(showgrid=True, zeroline=True, rangeslider_visible=True, showticklabels=True,
                 showspikes=True, spikemode='across', spikesnap='cursor', showline=True,
                 spikecolor="grey",spikethickness=1, spikedash='solid')
            fig.update_yaxes(showspikes=True, spikedash='solid',spikemode='across', 
                spikecolor="grey",spikesnap="cursor",spikethickness=1)
            #fig.update_traces(hovertemplate=None) #changes hoverdata to simple
            fig.update_layout(spikedistance=1000, hoverdistance=10)
            #below for drawing
            fig.update_layout(xaxis_title=x.title(), yaxis_title=y.title(), title=f'{y.title()} vs {x.title()}',
                             legend_traceorder="reversed")        
            fig.update_layout(
            dragmode='drawopenpath',
            newshape=dict(line_color='cyan'))
            fig.show(config={'modeBarButtonsToAdd':['drawline',
                                        'drawopenpath',
                                        'drawclosedpath',
                                        'drawcircle',
                                        'drawrect',
                                        'eraseshape'
                                       ]})
            #to print data from trend use 'print(fig.data[1].y)'
            #to remove fill of freehand, remove theme or change to plotly
            #to print data from trend use 'print(fig.data[1].y)'
            
            box2 = px.box(data_frame=common_filter, x=x, y=y, color=
            'Units', facet_row=None, facet_col=None, notched=False, template=template)
            box2.update_xaxes(showgrid=True, zeroline=True, rangeslider_visible=True, showticklabels=True,
                 showspikes=True, spikemode='across', spikesnap='cursor', showline=True,
                 spikecolor="grey",spikethickness=1, spikedash='solid')
            box2.update_yaxes(showspikes=True, spikedash='solid',spikemode='across', 
                spikecolor="grey",spikesnap="cursor",spikethickness=1)
            #fig.update_traces(hovertemplate=None) #changes hoverdata to simple
            box2.update_layout(spikedistance=1000, hoverdistance=500)
            #below for drawing
            box2.update_layout(legend_traceorder="reversed",
            dragmode='drawopenpath',
            newshape=dict(line_color='cyan'),
            title_text='')
            box2.show(config={'modeBarButtonsToAdd':['drawline',
                                        'drawopenpath',
                                        'drawclosedpath',
                                        'drawcircle',
                                        'drawrect',
                                        'eraseshape'
                                       ]})            

    with map_tab:
        #map_table = common_filter['Location', 'lon','lat'].unique()
        fig = px.scatter_mapbox(common_filter, lat="lat", lon="lon", hover_name="Location",
                        color_discrete_sequence=["fuchsia"], zoom=1, height=600, center=go.layout.mapbox.Center(lat=55, lon=3))
        fig.update_layout(mapbox_style="white-bg", autosize=True, mapbox_layers=[{
            "below": 'traces', "sourcetype": "raster", "sourceattribution": "United States Geological Survey", "source": [
                "https://basemap.nationalmap.gov/arcgis/rest/services/USGSImageryOnly/MapServer/tile/{z}/{y}/{x}"]}
        ])
        
        fig.show()
        
        display(common_filter['Location'].unique)
        #display(map_table)
        #display df unique location values
def dropdown_year_eventhandler(change):
    common_filtering(change.new, dropdown_purpose.value)
    
def dropdown_purpose_eventhandler(change):
    common_filtering(dropdown_year.value, change.new)
    
dropdown_year.observe(dropdown_year_eventhandler, names='value')
dropdown_purpose.observe(dropdown_purpose_eventhandler, names='value')

item_layout = widgets.Layout(margin='0 0 50px 0')
input_widgets = widgets.HBox([dropdown_year, dropdown_purpose], layout=item_layout)
tab = widgets.Tab([stats_tab, EDA_tab, trendline_tab, map_tab], layout=item_layout)
tab.set_title(0, 'Statistics')
tab.set_title(1, 'EDA')
tab.set_title(2, 'Trend Lines')
tab.set_title(3, 'Maps')

%%html
<style>
.jupyter-widgets.widget-tab > .p-TabBar .p-TabBar-tab {
    flex:0 1 300px
    
}
</style>
dashboard = widgets.VBox([input_widgets, tab])
display(dashboard)
import pkg_resources
import types
def get_imports():
    for name, val in globals().items():
        if isinstance(val, types.ModuleType):
            # Split ensures you get root package, 
            # not just imported function
            name = val.__name__.split(".")[0]

        elif isinstance(val, type):
            name = val.__module__.split(".")[0]

        # Some packages are weird and have different
        # imported names vs. system/pip names. Unfortunately,
        # there is no systematic way to get pip names from
        # a package's imported name. You'll have to add
        # exceptions to this list manually!
        poorly_named_packages = {
            "PIL": "Pillow",
            "sklearn": "scikit-learn"
        }
        if name in poorly_named_packages.keys():
            name = poorly_named_packages[name]

        yield name
imports = list(set(get_imports()))

# The only way I found to get the version of the root package
# from only the name of the package is to cross-check the names 
# of installed packages vs. imported packages
requirements = []
for m in pkg_resources.working_set:
    if m.project_name in imports and m.project_name!="pip":
        requirements.append((m.project_name, m.version))

for r in requirements:
    print("{}=={}".format(*r))<jupyter_output>seaborn==0.11.1
plotly==5.1.0
pandas==1.2.0
numpy==1.19.5
matplotlib==3.3.4
ipywidgets==7.6.3
geopandas==0.9.0
qgrid==1.3.0
cufflinks==0.17.3
{'fraction_non_alphanumeric': 0.11112846544318626, 'fraction_numerical': 0.007887543928153065, 'mean_word_length': 2.9943855271366187, 'pattern_counts': {'":': 7, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '314535', 'license': 'MIT', 'n_tokens_mistral': 3864, 'n_tokens_neox': 3724, 'n_words': 942, 'path': 'Scatter_Geotech.ipynb', 'repo_name': 'Crimzo1/Geotechnical-Data-Analysis', 'size': '12805'}