<jupyter_start><jupyter_text>Three functions here:oneHot ---> takes columns with CSV data and moves them to a OHE dataframeplotableOHE ---> takes a column with CSV data and a "measure" column of numeric data and gives count and average of that measure for each category in the columnmultiplePlotOHE ---> same as plotableOHE but can take and return multople "measure" columns<jupyter_code>import pandas as pd
import numpy as np
from sklearn.preprocessing import MultiLabelBinarizer
import altair as alt
alt.data_transformers.enable('default', max_rows=None)
data = pd.read_csv("InitialClean.csv")
#INPUT: string name of column
#OUTPUT: dataframe with every unique entry as a one-hot encoding
#Notes: entries must be seperated by ";"
#       output will be in same order as dataframe passed in
#            and can be appended if desired
#       null entries will be categorized with a "1" in the "nan" column
#CHANGE 'DATA' TO WHATEVER THE MAIN DATAFRAME IS NAMED
def oneHot(columnName):
    tempData = pd.DataFrame(data[columnName].str.split(";"))
    tempData = tempData[columnName].tolist()
    for i in range(len(tempData)):
        if type(tempData[i])==float:
            tempData[i]=[str(tempData[i])]
    mlb = MultiLabelBinarizer()
    mlb.fit(tempData)
    oneHot = pd.DataFrame(mlb.transform(tempData), columns = mlb.classes_)
    return oneHot
currentLanguages = oneHot("LanguageWorkedWith")
#now you can do analysis. For example: 
currentLanguages.Python.mean()
currentLanguages.describe()
#CHANGE 'DATA' TO WHATEVER THE MAIN DATAFRAME IS NAMED IN YOUR WORKSHEET
#for this function aggName and oheColumn are both single strings. 
#support for multiple aggregation columns is below
def plotableOHE(aggName, oheColumn):
    #use oneHot to oneHot the column of interest
    oheFrame = oneHot(oheColumn)
    #add the aggregate column (ie salary). 
    oheFrame[aggName] = data[aggName]
    #remove rows where the aggregate value is null
    oheFrame = oheFrame[oheFrame[aggName].notnull()]
    #get the counts of each label
    ones = np.ones(len(oheFrame[aggName]))
    countColumn = np.dot(ones,oheFrame)
    #get the total of each label
    outputFrame = pd.DataFrame(oheFrame[aggName].dot(oheFrame))
    outputFrame.insert(loc=0,column=oheColumn, value=outputFrame.index)
    outputFrame.reset_index(drop=True, inplace=True)
    countsName = oheColumn + "Counts"
    outputFrame.insert(loc=1,column=countsName, value=countColumn)
    #the aggName row will have squared in the dot product- we just drop it
    outputFrame=outputFrame[outputFrame[oheColumn]!=aggName]
    averageName = aggName+"Average"
    outputFrame[averageName] = outputFrame[aggName]/outputFrame[countsName]
    outputFrame.rename(columns = {aggName:aggName+"Sum"}, inplace = True)
    return outputFrame
#example
plotLangSalary = plotableOHE("ConvertedSalary", "LanguageWorkedWith")
plotLangSalary.head()
#aggNames is a list of numerical column inputs. Can only work with one OHE column
def multiplePlotOHE(aggNames, oheColumn):
    outputFrame = plotableOHE(aggNames[0], oheColumn)
    for agg in aggNames[1:]:
        newOutputFrame = plotableOHE(agg, oheColumn)
        outputFrame = outputFrame.join(newOutputFrame.set_index(oheColumn), on=oheColumn, rsuffix='_other')
        dropColumn = oheColumn + "Counts_other"
        outputFrame.drop(dropColumn, axis=1, inplace=True)
    return outputFrame
        
#example
plotTwo = multiplePlotOHE(["ConvertedSalary","CompanySizeQuantSmall"], "LanguageWorkedWith")
plotTwo.head(20)
example_chart = alt.Chart(plotLangSalary, title='Salary By Programming Language').mark_bar(opacity=0.7, tooltip=True).encode(
    x='LanguageWorkedWith',
    y=alt.Y('ConvertedSalaryAverage:Q', stack=None, title='Average Salary ($)'),
    #color='variable'
)
example_chart
#anotherExample:
#example
plotLangSalary = plotableOHE("ConvertedSalary", "UndergradMajor")
plotLangSalary.head()
example_chart2 = alt.Chart(plotLangSalary, title='Salary By Programming Language').mark_bar(opacity=0.7, tooltip=True).encode(
    x='UndergradMajor',
    y=alt.Y('ConvertedSalaryAverage:Q', stack=None, title='Average Salary ($)'),
    #color='variable'
)
example_chart2
#SEE HOW FAST THAT WAS!!!???<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10236406619385342, 'fraction_numerical': 0.003309692671394799, 'mean_word_length': 5.070301291248207, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '283345', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1334, 'n_tokens_neox': 1303, 'n_words': 364, 'path': 'CleanOHEforCharts.ipynb', 'repo_name': 'CMU-IDS-2022/final-project-zebra', 'size': '4230'}