<jupyter_start><jupyter_text>Exploratory Data Analysis posiblemente sea necesario instalar wquantiles con la siguiente línea:<jupyter_code>!pip install wquantiles
import pandas as pd
import numpy as np
from scipy.stats import trim_mean
from statsmodels import robust
import wquantiles

import seaborn as sns
import matplotlib.pylab as plt
BASE_URL = "https://raw.githubusercontent.com/lmcanavals/avd/main/data/"
AIRLINE_STATS_CSV   = f"{BASE_URL}airline_stats.csv"
AIRPORT_DELAYS_CSV  = f"{BASE_URL}dfw_airline.csv"
KC_TAX_CSV          = f"{BASE_URL}kc_tax.csv.gz"
LC_LOANS_CSV        = f"{BASE_URL}lc_loans.csv"
SP500_DATA_CSV      = f"{BASE_URL}sp500_data.csv.gz"
SP500_SECTORS_CSV   = f"{BASE_URL}sp500_sectors.csv"
STATE_CSV           = f"{BASE_URL}state.csv"<jupyter_output><empty_output><jupyter_text>Estimación de ubicación<jupyter_code>state = pd.read_csv(STATE_CSV)
state.head()
state['Population'].mean()
trim_mean(state['Population'], 0.1)
state['Population'].median()
state['Murder.Rate'].mean()
np.average(state['Murder.Rate'], weights=state['Population'])<jupyter_output><empty_output><jupyter_text>Estimaciones de Variabilidad<jupyter_code>state['Population'].std()
state['Population'].quantile(.75) - state['Population'].quantile(.25)
robust.scale.mad(state['Population'])
abs(state['Population'] - state['Population'].median()).median() / 0.6744897501960817<jupyter_output><empty_output><jupyter_text>Percentiles y Boxplots<jupyter_code>percentages = [.05, .25, .5, .75, .95]
state['Murder.Rate'].quantile(percentages)
df = pd.DataFrame(state['Murder.Rate'].quantile(percentages))
df.index = [f'{p * 100}' for p in percentages]
df.transpose()
ax = (state['Population']/1_000_000).plot.box(figsize=(3, 4))
ax.set_ylabel('Población (millones)')

plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Tablas de frecuencia e Histogramas<jupyter_code>binnedPopulation = pd.cut(state['Population'], 10)
binnedPopulation.value_counts()
binnedPopulation.name = 'binnedPopulation'
df = pd.concat([state, binnedPopulation], axis = 1)
df = df.sort_values(by='Population')

groups = []
for group, subset in df.groupby(by='binnedPopulation'):
    groups.append({
        'BinRange': group,
        'Count': len(subset),
        'States': ','.join(subset.Abbreviation)
    })
pd.DataFrame(groups)
ax = (state['Population'] / 1_000_000).plot.hist(figsize=(4, 4))
ax.set_xlabel('Población (millones)')

plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Estimaciones de densidad<jupyter_code>ax = state['Murder.Rate'].plot.hist(density=True, xlim=[0, 12],
                                    bins=range(1, 12), figsize=(4, 4))
state['Murder.Rate'].plot.density(ax=ax)
ax.set_xlabel('Murder Rate (per 100000)')

plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Data binaria y categórica<jupyter_code>!head $AIRPORT_DELAYS_CSV
dfw = pd.read_csv(AIRPORT_DELAYS_CSV)
print(100 * dfw / dfw.values.sum())
ax = dfw.transpose().plot.bar(figsize=(4,4), legend=False)
ax.set_xlabel('Cause of delay')
ax.set_ylabel('Count')

plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Correlation<jupyter_code>sp500_sym = pd.read_csv(SP500_SECTORS_CSV)
sp500_px = pd.read_csv(SP500_DATA_CSV, index_col=0)
telecomSymbols = sp500_sym[sp500_sym['sector'] == 'telecommunications_services']['symbol']

telecom = sp500_px.loc[sp500_px.index >= '2012-07-01', telecomSymbols]
telecom.corr()
telecom
etfs = sp500_px.loc[sp500_px.index > '2012-07-01',
                    sp500_sym[sp500_sym['sector'] == 'etf']['symbol']]
etfs.head()
fig, ax = plt.subplots(figsize=(5, 4))
ax = sns.heatmap(etfs.corr(), vmin=-1, vmax=1,
                 cmap=sns.diverging_palette(20, 220, as_cmap=True),
                 ax=ax)

plt.tight_layout()
plt.show()
from matplotlib.collections import EllipseCollection
from matplotlib.colors import Normalize

def plot_corr_ellipses(data, figsize=None, **kwargs):
    ''' https://stackoverflow.com/a/34558488 '''
    M = np.array(data)
    if not M.ndim == 2:
        raise ValueError('data must be a 2D array')
    fig, ax = plt.subplots(1, 1, figsize=figsize, subplot_kw={'aspect':'equal'})
    ax.set_xlim(-0.5, M.shape[1] - 0.5)
    ax.set_ylim(-0.5, M.shape[0] - 0.5)
    ax.invert_yaxis()

    # xy locations of each ellipse center
    xy = np.indices(M.shape)[::-1].reshape(2, -1).T

    # set the relative sizes of the major/minor axes according to the strength
    # of the positive/negative correlation
    w = np.ones_like(M).ravel() + 0.01
    h = 1 - np.abs(M).ravel() - 0.01
    a = 45 * np.sign(M).ravel()

    ec = EllipseCollection(widths=w, heights=h, angles=a, units='x', offsets=xy,
                           norm=Normalize(vmin=-1, vmax=1),
                           transOffset=ax.transData, array=M.ravel(), **kwargs)
    ax.add_collection(ec)

    # if data is a DataFrame, use the row/column names as tick labels
    if isinstance(data, pd.DataFrame):
        ax.set_xticks(np.arange(M.shape[1]))
        ax.set_xticklabels(data.columns, rotation=90)
        ax.set_yticks(np.arange(M.shape[0]))
        ax.set_yticklabels(data.index)

    return ec

m = plot_corr_ellipses(etfs.corr(), figsize=(5, 4), cmap='bwr_r')
cb = fig.colorbar(m)
cb.set_label('Correlation coefficient')

plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>Scatterplots<jupyter_code>ax = telecom.plot.scatter(x='T', y='VZ', figsize=(4, 4), marker='$\u25EF$')
ax.set_xlabel('ATT (T)')
ax.set_ylabel('Verizon (VZ)')
ax.axhline(0, color='grey', lw=1)
ax.axvline(0, color='grey', lw=1)

plt.tight_layout()
plt.show()
ax = telecom.plot.scatter(x='T', y='VZ', figsize=(4, 4),
                          marker='$\u25EF$', alpha=0.1)
ax.set_xlabel('ATT (T)')
ax.set_ylabel('Verizon (VZ)')
ax.axhline(0, color='grey', lw=1)
ax.axvline(0, color='grey', lw=1)

plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>2 o más variables Hexbin<jupyter_code>kc_tax = pd.read_csv(KC_TAX_CSV)
kc_tax0 = kc_tax.loc[(kc_tax['TaxAssessedValue'] < 750000) &
                     (kc_tax['SqFtTotLiving'] > 100) &
                     (kc_tax['SqFtTotLiving'] < 3500), :]
print(kc_tax.shape, kc_tax0.shape)
ax = kc_tax0.plot.hexbin(x='SqFtTotLiving', y='TaxAssessedValue',
                         gridsize=30, sharex=False, figsize=(5, 4))
ax.set_xlabel('Finished Square Feet')
ax.set_ylabel('Tax Assessed Value')

plt.tight_layout()
plt.show()
fig, ax = plt.subplots(figsize=(4, 4))
ax = sns.kdeplot(kc_tax0['SqFtTotLiving'], kc_tax0['TaxAssessedValue'], ax=ax)
ax.set_xlabel('Finished Square Feet')
ax.set_ylabel('Tax Assessed Value')

plt.tight_layout()
plt.show()<jupyter_output><empty_output><jupyter_text>2 Variables Categóricas<jupyter_code>lc_loans = pd.read_csv(LC_LOANS_CSV)
crosstab = lc_loans.pivot_table(index='grade', columns='status',
                                aggfunc=lambda x: len(x), margins=True)
crosstab
df = crosstab.copy().loc['A':'G', :]
df.loc[:, 'Charged Off':'Late'] =df.loc[:, 'Charged Off':'Late'].div(df['All'], axis=0)
df['All'] = df['All'] / sum(df['All'])
perc_crosstab = df
perc_crosstab<jupyter_output><empty_output><jupyter_text>Violinplot para Data Categórica y Numérica<jupyter_code>airline_stats = pd.read_csv(AIRLINE_STATS_CSV)
airline_stats.head()
ax = airline_stats.boxplot(by='airline', column='pct_carrier_delay',
                           figsize=(5, 5))
ax.set_xlabel('')
ax.set_ylabel('% diario de vuelos demorados')
plt.suptitle('')

plt.tight_layout()
plt.show()
fig, ax = plt.subplots(figsize=(5, 5))
sns.violinplot(airline_stats['airline'], airline_stats['pct_carrier_delay'],
               ax=ax, inner='quartile', color='white')
ax.set_xlabel('')
ax.set_ylabel('% diario de vuelos demorados')

plt.tight_layout()
plt.show()<jupyter_output>/usr/local/lib/python3.6/dist-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  FutureWarning<jupyter_text>Visualización de múltiples variables<jupyter_code>zip_codes = [98188, 98105, 98108, 98126]
kc_tax_zip = kc_tax0.loc[kc_tax0['ZipCode'].isin(zip_codes), :]
kc_tax_zip
def hexbin(x, y, color, **kwargs):
    cmap = sns.light_palette(color, as_cmap=True)
    plt.hexbin(x, y, gridsize=25, cmap=cmap, **kwargs)

g = sns.FacetGrid(kc_tax_zip, col='ZipCode', col_wrap=2)
g.map(hexbin, 'SqFtTotLiving', 'TaxAssessedValue',
      extent=[0, 3500, 0, 700000])
g.set_axis_labels('Finished Square Feet', 'Tax Assessed Value')
g.set_titles('Zip code {col_name: .0f}')

plt.tight_layout()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15958419008453278, 'fraction_numerical': 0.03255654557916381, 'mean_word_length': 5.088317107093185, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 55, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '412790', 'license': 'CC0-1.0', 'n_tokens_mistral': 3690, 'n_tokens_neox': 3375, 'n_words': 552, 'path': 'notebooks/aa_data_exploration.ipynb', 'repo_name': 'lmcanavals/analytics_visualization', 'size': '8754'}