<jupyter_start><jupyter_text>Visualización de datos sysarmy y análisis exploratorio<jupyter_code>import io
import matplotlib
import matplotlib.pyplot as plt
import numpy
import pandas as pd
import seaborn

seaborn.set_context('talk')
url = 'https://cs.famaf.unc.edu.ar/~mteruel/datasets/diplodatos/sysarmy_survey_2020_processed.csv'
df = pd.read_csv(url)
df[:3]<jupyter_output><empty_output><jupyter_text>Análisis Descriptivo<jupyter_code>relevant_columns = ['tools_programming_languages', 'salary_monthly_NETO']
# Convert the comma-separated string of languages to a list of string.
# Remove 'ninguno de los anteriores' option, spaces and training commas.
def split_languages(languages_str):
  if not isinstance(languages_str, str):
    return []
  # Remove 'other' option
  languages_str = languages_str.lower()\
    .replace('ninguno de los anteriores', '')
  # Split string into list of items
  # Remove spaces and commas for each item
  return [lang.strip().replace(',', '')
          for lang in languages_str.split()]

# Create a new column with the list of languages
df.loc[:, 'cured_programming_languages'] = df.tools_programming_languages\
    .apply(split_languages)
if 'cured_programming_languages' not in relevant_columns:
    relevant_columns.append('cured_programming_languages') 

# Duplicate each row of df for each programming language
# mentioned in the response.
# We only include in df_lang the columns we are going to analyze later, so we
# don't duplicate innecesary information.
df_lang = df.cured_programming_languages\
    .apply(pd.Series).stack()\
    .reset_index(level=-1, drop=True).to_frame()\
    .join(df[relevant_columns])\
    .rename(columns={0: 'programming_language'})
# Horrible programming style! But a lot of data science code can be written with
# as concatenations of functions (pipelines), and there's no elegant way of
# doing that on Python.
df_lang[:5]
language_count = df_lang.programming_language.value_counts()\
    .reset_index()\
    .rename(columns={'index': 'language', 'programming_language': 'frequency'})
language_count[:10]
#hasta aca nos dieron ellos
#barplot vamos a plotear los primeros 20
languages_20 = language_count[:19]
import seaborn as sns
sns.barplot(x="frequency", y="language", data=languages_20)

#cuales son los lenguajes asociados a los mejores sueldos
plt.figure(figsize=(40,20))
sns.boxplot(x="salary_monthly_NETO", y="programming_language", data=df_lang)
#definitivamente hay q limpiar estos datos
#vamos a filtrar los lenguajes que estan orientados a front-end programming 
front_end = ['basic','vb','vb5','vb6', 'vba','vbscript','visual','.net', 'net', 'asp', 'c#']
df_lang.head()
web=df_lang[df_lang['programming_language'].isin(['basic','vb','vb5','vb6', 'vba','vbscript','visual','.net', 'net', 'asp', 'c#'])]
web.head()
sns.boxplot(x="salary_monthly_NETO", y="programming_language", data=web)

web_design=['react','elm','dart','ruby','kotlin','typescript','javascript', 'html', 'css', 'php']
web_d=df_lang[df_lang['programming_language'].isin(['react','elm','dart','ruby','kotlin','typescript','javascript', 'html', 'css', 'php'])]

web_d.describe() 
sns.boxenplot(x="salary_monthly_NETO", data=web_d)
sns.boxplot(x="salary_monthly_NETO", y="programming_language", data=web_d)
#existen muchos valores extremos
#veamos si mejora la visualización utilizando boxeplot en vez de boxplot
plt.figure(figsize=(12, 8))
sns.boxenplot(x="salary_monthly_NETO", y="programming_language", data=web_d)<jupyter_output><empty_output><jupyter_text>1er paso: eliminar react y elm<jupyter_code>#1er paso: eliminar elm y react
web_d1=df_lang[df_lang['programming_language'].isin(['dart','ruby','kotlin','typescript','javascript', 'html', 'css', 'php'])]
plt.figure(figsize=(12, 8))
sns.boxenplot(x="salary_monthly_NETO", y="programming_language", data=web_d1)<jupyter_output><empty_output><jupyter_text>2do paso: eliminar valores extremos con PERCENTILES<jupyter_code>#2do paso: eliminar valores extremos 
web_d1['salary_monthly_NETO'].quantile([.95, .98, .99, .995, .998])
fig, axes = plt.subplots(figsize=(12, 12), nrows=3)
max_limit = web_d1['salary_monthly_NETO'].quantile(.98)
data = web_d1[web_d1['salary_monthly_NETO'] < max_limit]['salary_monthly_NETO']
sns.histplot(x=data, ax=axes[0])
sns.boxplot(x=data, ax=axes[1])
sns.boxenplot(x=data, ax=axes[2])
web_d1.describe()<jupyter_output><empty_output><jupyter_text>2.1 Eliminar valores extremos usando RANGOS INTERCUARTILICOS<jupyter_code>def clean_outliers(dataset, column_name):
    """Returns dataset removing the outlier rows from column @column_name."""
    interesting_col = dataset[column_name]
    # Here we can remove the outliers from both ends, or even add more restrictions.
    mask_outlier = (
        numpy.abs(interesting_col - interesting_col.mean()) <= (2.5 * interesting_col.std()))
    return dataset[mask_outlier]

salary_col='salary_monthly_NETO'

fig, axes = plt.subplots(figsize=(12, 12), nrows=3)
max_limit = web_d1[salary_col].quantile(.98)
data = clean_outliers(web_d1, salary_col)[salary_col]
sns.histplot(x=data, ax=axes[0])
sns.boxplot(x=data, ax=axes[1])
sns.boxenplot(x=data, ax=axes[2])<jupyter_output><empty_output><jupyter_text>Exploramos subconjunto de ciencia de datos<jupyter_code>'python',
'r',
'dax', DATA ANALYSIS EXPRESSIONS
'regex'
'db',
'esql',
'mongodb',
'nosql',
'oracle',
'pl-sql','pl/sql', 'plsql',
'sql',
'bi',
'adabas'
ds_data=df_lang[df_lang['programming_language'].isin(['python','r','mongodb','sql','nosql','bi','pl-sql','pl/sql', 'plsql', 'db', 'esql', 'oracle', 'adabas'])]

plt.figure(figsize=(12, 8))
sns.boxplot(x="salary_monthly_NETO", y="programming_language", data=ds_data)
#eliminar los lenguajes q tienen muy pocos casos
#pasar a boxenplot
ds_data1=df_lang[df_lang['programming_language'].isin(['python','r','sql','pl-sql','pl/sql', 'plsql', 'esql', 'oracle'])]
plt.figure(figsize=(12, 8))
sns.boxenplot(x="salary_monthly_NETO", y="programming_language", data=ds_data1)<jupyter_output><empty_output><jupyter_text>Eliminar outliers con PERCENTILES<jupyter_code>fig, axes = plt.subplots(figsize=(12, 12), nrows=3)
max_limit = ds_data1['salary_monthly_NETO'].quantile(.98)
data = ds_data1[ds_data1['salary_monthly_NETO'] < max_limit]['salary_monthly_NETO']
sns.histplot(x=data, ax=axes[0])
sns.boxplot(x=data, ax=axes[1])
sns.boxenplot(x=data, ax=axes[2])<jupyter_output><empty_output><jupyter_text>Eliminar outliers con RANGO INTERCUARTILICO<jupyter_code>fig, axes = plt.subplots(figsize=(12, 12), nrows=3)
max_limit = ds_data1[salary_col].quantile(.98)
data = clean_outliers(ds_data1, salary_col)[salary_col]
sns.histplot(x=data, ax=axes[0])
sns.boxplot(x=data, ax=axes[1])
sns.boxenplot(x=data, ax=axes[2])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.153125, 'fraction_numerical': 0.016964285714285713, 'mean_word_length': 7.949400798934754, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 33, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '513801', 'license': 'MIT', 'n_tokens_mistral': 2634, 'n_tokens_neox': 2497, 'n_words': 482, 'path': 'diplo_vis_punto1.ipynb', 'repo_name': 'data-datum/AnalisisyVisualizacion', 'size': '6720'}