<jupyter_start><jupyter_text>AutoCSV DevWNixalo –– 2018/6/2-3<jupyter_code>%matplotlib inline
%reload_ext autoreload
%autoreload 2
from pathlib import Path
import numpy as np
import pandas as pd
import os
import matplotlib.pyplot as plt

from cifar_utils import count_files
PATH = Path('data/cifar10')<jupyter_output><empty_output><jupyter_text>Dataset Population<jupyter_code>with open(PATH/'labels.txt') as f:
    text = f.read()
# print(text)
cats = text.split('\n')[:-1]
cats
cat_pops = {cat:count_files(PATH/'train'/cat) for cat in cats}
cat_pops
cols = dict()
cols['cat'],cols['pop'] = zip(*cat_pops.items())
df = pd.DataFrame(cols)
df.head()
df.plot.bar(x=df[df.columns[0]], ylim=(0,1.2*df[df.columns[1]].values.max()), 
            yerr=max(df.mean()[0]*0.005, df.std()[0]), alpha=.8)
df.mean()[0], df.std()[0]<jupyter_output><empty_output><jupyter_text>Uneven data set:<jupyter_code>def plot_pops(df, print_ms=True):
    if print_ms: print(f"{df.mean()[0]:.2f} {df.std()[0]:.2f}")
    df.plot.bar(x=df[df.columns[0]], ylim=(0,1.2*df[df.columns[1]].values.max()), 
            yerr=max(df.mean()[0]*0.005, df.std()[0]), alpha=.8)
    df.mean()[0], df.std()[0]
df_backup = df.copy()
# df.at[2,'cat'] # equals 'bird'
df.at[2, 'pop'] /= 2 # 4250 -> 2125
df.at[2,'pop'] # for editing df values see: https://stackoverflow.com/a/13842286
plot_pops(df)
df.std()[0]<jupyter_output><empty_output><jupyter_text>AutoCSV<jupyter_code># generate a csv from a dataset
def generate_csv(path, labelmap=None, folder='train'):
    """Infers a csv from directory structure. 
       `labelmap` is a dictionary mapping class folders to class names.
       Class names are taken from class folder names if no mapping is provided.
       For Single-Category Classification.
    """
    # find categories
    catfolders = [f for f in os.listdir(path/folder) if (path/folder/f).is_dir()]
    if labelmap is None: labelmap = {cf:cf for cf in catfolders}

    rows = []

    for cat in catfolders:
        catpath = path/folder/cat
        fpaths  = list(map(lambda x: cat+'/'+x, os.listdir(catpath)))
        rows.extend(list(zip(fpaths,[labelmap[cat] for i in range(len(fpaths))])))

    df = pd.DataFrame(rows, columns=['file','class'])
    return df
%%time
csv = generate_csv(PATH)
csv.sample(n=10)<jupyter_output><empty_output><jupyter_text>CSV-based Pop Counting<jupyter_code>def pops_from_df(df, catdx=1, colnames=['cat','pop']):
    """Extracts category populations from a DataFrame.
       If `colnames=None`: returns a dictionary, otherwise
       a dataframe with `colnames` columns.
    """
    catcol = df.columns[catdx] # prevsly: y = df.columns[ydx]
    cats = df[catcol].unique()
    pops = [df[df[catcol]==cat].count()[0] for cat in cats] # prevsly: y -> catcol
    cat_pops = {cat:pop for cat,pop in zip(cats,pops)}
    
    if colnames:
        cat_pops = list(zip(cats,pops))
        cat_pops = pd.DataFrame(cat_pops, columns=colnames)
    else:
        cat_pops = {cat:pop for cat,pop in zip(cats,pops)}
    
    return cat_pops
pops_from_df(csv, colnames=None)
plot_pops(pops_from_df(csv))<jupyter_output>4250.00 0.00<jupyter_text>CSV samplingRetreive a portion of the csv<jupyter_code>def csv_subset(df, catdx=1, p=0.5):
    """Returns a percetnage of the original dataset, sampled uniformly by category."""
    if type(p)==int: p /= df.count()
    
    catcol = df.columns[catdx]
    cats   = df[catcol].unique()
    df_slice  = df[catcol]
    keep_idxs = np.array([], dtype=np.int64)
    
    for cat in cats:
        cat_idxs = np.where(df_slice==cat)[0]
        n = max(1, int(len(cat_idxs)*p))
        keep_idxs = np.concatenate((keep_idxs, np.random.choice(cat_idxs, size=n, replace=False)))
    
    return df.iloc[keep_idxs]
full_df  = generate_csv(PATH)
cat_pops = pops_from_df(full_df)

full_df.count()[0]
new_csv = csv_subset(full_df)
new_csv.count()[0]
pops_from_df(new_csv)
new_csv.head()
np.unique(new_csv[new_csv['class']=='cat'].as_matrix()[:,0]).shape # check: no repeats<jupyter_output><empty_output><jupyter_text>Dataset Smoothing"Smooth" out category populations to within a specified standard deviation.<jupyter_code>def smooth_csv_dataset(df, eps=0.1, full_df=None, catdx=1):
    """'Smooths' out a dataset by adding copied samples.
    
        For use with single-label classification (2-column) CSVs."""
    
    # result DF and sampling DF
    new_df  = df.copy()
    full_df = df if full_df is None else full_df
    # get category column name
    catcol = df.columns[catdx]
    
    # get category populations & calculate desired range
    cat_pops = pops_from_df(df)
    c_totals = cat_pops.as_matrix(columns=['pop'])
    sd       = eps * c_totals.mean()
    # Normalize category sizes
    c_norm     = c_totals/c_totals.max()
    new_mean   = c_totals.max() - sd
    new_totals = (2*sd * c_norm + (new_mean - sd)).astype(int)
    
    # Increase category sizes by differences
    diffs     = new_totals - c_totals
    cats      = cat_pops['cat'].values
    copy_idxs = []
    
    for i,cat in enumerate(cats):
        diff         = diffs[i]
        cat_idxs     = np.where(full_df[catcol]==cat)[0]
        full_cat_pop = len(cat_idxs)
        
        # if the difference is more than Nx greater, copy the whole category N times
        if diff > full_cat_pop:
            n_copy = int(diff) // full_cat_pop
            diff  -= n_copy * full_cat_pop
            for i in range(n_copy): copy_idxs.extend(cat_idxs)
        copy_idxs.extend(np.random.choice(cat_idxs, size=diff, replace=False))
    
    copy_rows = full_df.iloc[copy_idxs]
    new_df    = new_df.append(copy_rows)
    
    return new_df
def basic_plot2(c_totals=c_totals, sd=sd, oldmean=None, catlist=None):
    sa,mean = c_totals.std(),c_totals.mean() if oldmean is None else oldmean
    plt.bar(x=range(len(c_totals)),height=c_totals, alpha=0.4, color='k');
    plt.axhline(y=mean,c='r'); plt.axhline(y=mean+sd,c='k'); plt.axhline(y=mean-sd,c='k');
    if catlist is not None: plt.xticks(range(len(c_totals)), catlist, rotation=90)
    print(mean, (sd, sa))
    print(c_totals)<jupyter_output><empty_output><jupyter_text>Makes this:<jupyter_code>df = generate_csv(PATH)

skewed_df = df.sample(n=5000)
drop_idxs = np.where(skewed_df['class']=='airplane')[0]
drop_idxs = np.random.choice(drop_idxs, size=len(drop_idxs)//2, replace=False)
skewed_df.drop(skewed_df.index[drop_idxs], inplace=True)

cat_pops = pops_from_df(skewed_df)
c_totals = cat_pops.as_matrix(columns=['pop'])
catlist  = cat_pops.as_matrix(columns=['cat'])

eps = 0.1; sd = c_totals.mean()*eps

basic_plot2(c_totals=c_totals[:,0],sd=sd,catlist=catlist[:,0])<jupyter_output>476.5 (47.650000000000006, 81.13599694340361)
[517 513 497 477 510 500 514 236 487 514]<jupyter_text>Become this:<jupyter_code>smoothed_df = smooth_csv_dataset(skewed_df, eps=eps, full_df=df)

cat_pops = pops_from_df(smoothed_df)
c_totals = cat_pops.as_matrix(columns=['pop'])
catlist  = cat_pops.as_matrix(columns=['cat'])

basic_plot2(c_totals=c_totals[:,0], sd=sd, catlist=catlist[:,0])<jupyter_output>509.1 (47.650000000000006, 14.895972610071489)
[517 516 513 509 515 513 516 465 511 516]<jupyter_text>Looking at it another way:<jupyter_code>basic_plot2(c_totals=c_totals[:,0], sd=sd, catlist=catlist[:,0], oldmean=c_totals.max()-sd)<jupyter_output>469.35 (47.650000000000006, 14.895972610071489)
[517 516 513 509 515 513 516 465 511 516]<jupyter_text>Manual prototyping: New algorithm:```set max to max category.calculate new mean as 1 σδ below max.normalize dataset to 1 σδ of new mean.```normalization:- todo<jupyter_code>def basic_plot():
    mean,sa = c_totals.mean(),c_totals.std()
    plt.bar(x=range(len(c_totals)),height=c_totals, alpha=0.4, color='k');
    plt.axhline(y=mean,c='r'); plt.axhline(y=mean+sd,c='k'); plt.axhline(y=mean-sd,c='k');
    print(mean, (sd, sa))
    print(c_totals)
df = generate_csv(PATH)<jupyter_output><empty_output><jupyter_text>Getting a 'skewed' dataset from the original DataFrame<jupyter_code>skewed_df = df.sample(n=5000)
counts = pops_from_df(skewed_df); counts.mean()[0], counts.std()[0]
drop_idxs = np.where(skewed_df['class']=='dog')[0]
drop_idxs.shape
drop_idxs = np.random.choice(drop_idxs, size=len(drop_idxs)//2, replace=False)
drop_idxs.shape
skewed_df.drop(skewed_df.index[drop_idxs], inplace=True)
skewed_df.count()[0], np.where(skewed_df['class']=='dog')[0].shape[0]
plot_pops(pops_from_df(skewed_df))<jupyter_output>473.40 75.34<jupyter_text>In this case with an ε of 10%, we'd want a standard deviation of about 47. It's currently at 75.Modifying the algorithm to work on dataframes instead of integer arrays:<jupyter_code>eps = 0.1

cat_pops = pops_from_df(skewed_df)
c_totals = cat_pops.as_matrix(columns=['pop'])<jupyter_output><empty_output><jupyter_text>NumPy and Pandas seem to disagree on the standard deviation. Looks like by default they're [calculated subtly differently](https://stackoverflow.com/a/24984205).<jupyter_code>c_totals.mean(), c_totals.std()
sd = eps * c_totals.mean() # desired standard deviation<jupyter_output><empty_output><jupyter_text>Normalize category sizes:<jupyter_code># normalize category sizes
c_normd = c_totals/c_totals.max()# - c_totals.min()/c_totals.max()
mean = c_totals.max() - sd
new_totals = (2*sd * c_normd + (mean - sd)).astype(int)

#display
print(c_totals.mean(),mean);print(c_totals.std(),new_totals.std()); list(zip(c_totals, new_totals))
c_normd
c_totals.max()
c_totals[:,0]<jupyter_output><empty_output><jupyter_text>Looks like things are working to spec:<jupyter_code>def basic_plot2(c_totals=c_totals, sd=sd, oldmean=None):
    sa,mean = c_totals.std(),c_totals.mean() if oldmean is None else oldmean
    plt.bar(x=range(len(c_totals)),height=c_totals, alpha=0.4, color='k');
    plt.axhline(y=mean,c='r'); plt.axhline(y=mean+sd,c='k'); plt.axhline(y=mean-sd,c='k');
    print(mean, (sd, sa))
    print(c_totals)<jupyter_output><empty_output><jupyter_text>Showing the new distribution with original mean flanked by desired standard deviation:<jupyter_code>basic_plot2(c_totals=new_totals[:,0], sd=sd, oldmean=mean)<jupyter_output>477.65999999999997 (47.34, 12.867012085173464)
[521 521 525 522 517 522 515 478 518 513]<jupyter_text>Showing the new mean flanked by desired standard deviation:<jupyter_code>basic_plot2(c_totals=new_totals[:,0], sd=sd)<jupyter_output>515.2 (47.34, 12.867012085173464)
[521 521 525 522 517 522 515 478 518 513]<jupyter_text>Looks like the algorithm mechanics are working as they should. Now to do the dataframe resample bit.<jupyter_code>diffs = new_totals - c_totals
diffs[:,0]<jupyter_output><empty_output><jupyter_text>Adding to a single category:<jupyter_code>cat_pops['cat'][0]
cat_pops
# cats = cat_pops['cat'].values
idx = 0; cat = cat_pops['cat'].values[idx]
cat<jupyter_output><empty_output><jupyter_text>New Totals says the 'deer' category should have 15 elements added to it, bringing a total of 506 to 521 (I'm a bit paranoid about numerical stability):<jupyter_code>cat_pops.iloc[idx]['pop'], new_totals[idx][0]<jupyter_output><empty_output><jupyter_text>Grab a random sample of 15 elements from the original dataset for 'deer':<jupyter_code>copy_idxs = np.random.choice(np.where(df['class'] == cat)[0], size=diffs[idx], replace=False)
copy_idxs
copyrows = df.iloc[copy_idxs]
copyrows<jupyter_output><empty_output><jupyter_text>Holy shit it works.<jupyter_code>tmpdf = skewed_df.copy()
tmpdf.describe()
tmpdf.append(copyrows).describe()
tmpdf = tmpdf.append(copyrows)
tmpdf.iloc[-16:]<jupyter_output><empty_output><jupyter_text>Looks good. Now for all categories.<jupyter_code>cats = cat_pops['cat'].values
copy_idxs = []
for i,cat in enumerate(cats):
    diff = diffs[i]
    copy_idxs.extend(np.random.choice(np.where(df['class'] == cat)[0], size=diff, replace=False))
len(copy_idxs)
new_totals.sum() - c_totals.sum()
copyrows = df.iloc[copy_idxs]
copyrows.count()
copyrows.sample(n=5)<jupyter_output><empty_output><jupyter_text>Moment of truth: adding it back together and counting the result:<jupyter_code>tmpdf = skewed_df.copy()
tmpdf.count()
print(skewed_df.count()[0])
skewed_df = skewed_df.append(copyrows)
print(skewed_df.count()[0])
tmpdf.count()
plot_pops(pops_from_df(skewed_df))<jupyter_output>515.20 13.56<jupyter_text>What the dataset looks like now:<jupyter_code>basic_plot2(c_totals=pops_from_df(skewed_df).as_matrix(columns=['pop'])[:,0], sd=sd)<jupyter_output>515.2 (47.34, 12.867012085173464)
[521 521 525 522 517 522 515 478 518 513]<jupyter_text>And what it looks like with the original mean superimposed:<jupyter_code>basic_plot2(c_totals=pops_from_df(skewed_df).as_matrix(columns=['pop'])[:,0], sd=sd, oldmean=c_totals.mean())<jupyter_output>473.4 (47.34, 12.867012085173464)
[521 521 525 522 517 522 515 478 518 513]<jupyter_text>The dataset's class distribution has been smoothed out to within 1 desired standard deviation of the mean.Automated dataset smoothing:<jupyter_code>def smooth_csv_dataset(df, eps=0.1, full_df=None, catdx=1):
    """'Smooths' out a dataset by adding copied samples.
    
        For use with single-label classification (2-column) CSVs."""
    
    # result DF and sampling DF
    new_df  = df.copy()
    full_df = df if full_df is None else full_df
    # get category column name
    catcol = df.columns[catdx]
    
    # get category populations & calculate desired range
    cat_pops = pops_from_df(df)
    c_totals = cat_pops.as_matrix(columns=['pop'])
    sd       = eps * c_totals.mean()
    # Normalize category sizes
    c_norm     = c_totals/c_totals.max()
    new_mean   = c_totals.max() - sd
    new_totals = (2*sd * c_norm + (new_mean - sd)).astype(int)
    
    # Increase category sizes by differences
    diffs     = new_totals - c_totals
    cats      = cat_pops['cat'].values
    copy_idxs = []
    
    for i,cat in enumerate(cats):
        diff         = diffs[i]
        cat_idxs     = np.where(full_df[catcol]==cat)[0]
        full_cat_pop = len(cat_idxs)
        
        # if the difference is more than Nx greater, copy the whole category N times
        if diff > full_cat_pop:
            n_copy = int(diff) // full_cat_pop
            diff  -= n_copy * full_cat_pop
#             import pdb;pdb.set_trace()
            for i in range(n_copy): copy_idxs.extend(cat_idxs)
        copy_idxs.extend(np.random.choice(cat_idxs, size=diff, replace=False))
    
    copy_rows = full_df.iloc[copy_idxs]
    new_df    = new_df.append(copy_rows)
    
    return new_df
int(np.array([1]))
tmp = []
arr = [1,2,3]
ndarr = np.array(arr)
# tmp.extend(arr)
# tmp.extend(ndarr)
tmp.extend(np.array([1,2,3]))
tmp<jupyter_output><empty_output><jupyter_text>Testing:<jupyter_code>df = generate_csv(PATH)
skewed_df = df.sample(n=5000)
counts = pops_from_df(skewed_df); counts.mean()[0], counts.std()[0]

drop_idxs = np.where(skewed_df['class']=='dog')[0]
drop_idxs = np.random.choice(drop_idxs, size=len(drop_idxs)//2, replace=False)

skewed_df.drop(skewed_df.index[drop_idxs], inplace=True)
skewed_df.count()[0], np.where(skewed_df['class']=='dog')[0].shape[0]
cat_pops = pops_from_df(skewed_df); c_totals = cat_pops.as_matrix(columns=['pop']); sd=c_totals.mean()*eps
basic_plot2(c_totals=c_totals[:,0], sd=sd)<jupyter_output>475.5 (47.550000000000004, 80.82109873046765)
[515 245 452 494 485 495 505 547 532 485]<jupyter_text>Without a 'full' base dataset:<jupyter_code>eps * c_totals.mean()
c_totals.max()
smooth_df = smooth_csv_dataset(skewed_df)
smooth_df.count()[0]
oldmean
c_totals.mean()<jupyter_output><empty_output><jupyter_text>I got a little confused looking at these plots, and realized I wasn't actually displaying or calculating the 'old' mean: more of a 'pseudomean'. The 'oldmean' parameter to the plot ('new_mean' inside the smoother function) *is* the pseudomean.Although it's what I meant, I'm not actually asking for the dataset to be within 1 desired standard deviation from the mean. I'm asking that *no element of the dataset be less than 2 desired standard deviations from the maximum.*That also means nothing is further than ±1 standard deviation from the new *actual* mean. So I'm in effect imposing 2 constraints: 1. Nothing is further than 1 stdev from the (actual) mean, 2. and the minumum is within 2 stdevs of the maximum.<jupyter_code># Smoothed dataset is now
cat_pops = pops_from_df(smooth_df); c_totals = cat_pops.as_matrix(columns=['pop'])
basic_plot2(c_totals=c_totals[:,0], sd=sd, oldmean=c_totals.max()-sd)
cat_pops = pops_from_df(smooth_df); c_totals = cat_pops.as_matrix(columns=['pop'])
basic_plot2(c_totals=c_totals[:,0], sd=sd)<jupyter_output>534.1 (47.550000000000004, 14.088647912415158)
[541 494 530 537 536 537 539 547 544 536]<jupyter_text>With a 'full' base dataset:<jupyter_code>%time smooth_df = smooth_csv_dataset(skewed_df, full_df=df)
# entire dataset is within 2 stdevs of the maximum
cat_pops = pops_from_df(smooth_df); c_totals = cat_pops.as_matrix(columns=['pop'])
basic_plot2(c_totals=c_totals[:,0], sd=sd, oldmean=c_totals.max()-sd)
# entire dataset is within 1 stdev of the mean
cat_pops = pops_from_df(smooth_df); c_totals = cat_pops.as_matrix(columns=['pop'])
basic_plot2(c_totals=c_totals[:,0], sd=sd)
smooth_df.describe()
smooth_df.count()[0], skewed_df.count()[0]<jupyter_output><empty_output><jupyter_text>Scrap<jupyter_code>cat_pops.iloc[idx]['pop'], new_totals[idx][0]<jupyter_output><empty_output><jupyter_text>Grab a random sample of 15 elements from the original dataset for 'deer':<jupyter_code>copy_idxs = np.random.choice(np.where(df['class'] == cat)[0], size=diffs[idx], replace=False)
copy_idxs
copyrows = df.iloc[copy_idxs]
# randomly sample from dataset to fill the difference
adds = []
diffs = new_totals - c_totals
for i,diff in enumerate(diffs):
    add = np.random.choice(df[catcol]==#TODO)
c_totals[np.argmin(c_totals)], c_normd[np.argmin(c_totals)]
# normalize wrt max
c_normd = c_totals/c_totals.max()

# set floor as 2sd below max

cmin = c_totals[np.argmin(c_totals)]
cnormin = c_normd[np.argmin(c_totals)]

2*sd * cnormin<jupyter_output><empty_output><jupyter_text>Too many values are being reduced. My algorithm isn't perfect for the job. *But*: what if I just take the max of the result? What does the mean & stdv look like then?<jupyter_code>maxs = np.max((c_totals, new_totals), axis=0)
print(maxs.mean(), maxs.std())
list(zip(*maxs))[0]<jupyter_output>491.9 24.570103784884587<jupyter_text>No values have increased beyond the original max, and all<jupyter_code>maxs.min() - maxs.mean(), maxs.max() - maxs.mean()
c_totals.mean(), mean
new_totals
## prototyping: dummy dataset variation
c_totals = np.random.randint(1000, 3000, size=10)
# calculate mean, actual & desired standard deviation
# mean = c_totals.mean()
# sa   = c_totals.std()
sd   = eps * mean

# normalize category sizes
c_normd = c_totals/c_totals.mean() - c_totals.min()/c_totals.max()
mean = c_totals.max() - sd
new_totals = (2*sd * c_normd + (mean - sd)).astype(int)

# randomly sample from dataset to fill the difference
adds = []
diffs = new_totals - c_totals
for i,diff in enumerate(diffs):
    add = np.random.choice(df[catcol]==#TODO)
cat_pops.std()
c_totals
cat_pops
def plot_pops(df, print_ms=True):
    if print_ms: print(f"{df.mean()[0]:.2f} {df.std()[0]:.2f}")
    df.plot.bar(x=df[df.columns[0]], ylim=(0,1.2*df[df.columns[1]].values.max()), 
            yerr=max(df.mean()[0]*0.005, df.std()[0]), alpha=.8)
    df.mean()[0], df.std()[0]
cat_pops.mean()
cat_pops.std()
cat_pops.as_matrix(columns=['pop']).std()
cat_pops.as_matrix(columns=['pop'])
cat_pops.as_matrix(columns=['pop']).mean(), cat_pops.as_matrix(columns=['pop']).std()
cat_pops['pop'].sum()
cat_pops['pop'].mean(), cat_pops['pop'].std()
c_totals.sum()
## prototyping: dummy dataset variation
c_totals = np.random.randint(1000, 3000, size=10)
# calculate mean, actual & desired standard deviation
# mean = c_totals.mean()
# sa   = c_totals.std()
sd   = eps * mean

# normalize category sizes
c_normd = c_totals/c_totals.mean() - c_totals.min()/c_totals.max()
mean = c_totals.max() - sd
new_totals = (2*sd * c_normd + (mean - sd)).astype(int)

# randomly sample from dataset to fill the difference
adds = []
diffs = new_totals - c_totals
for i,diff in enumerate(diffs):
    add = np.random.choice(df[catcol]==)
tmp = []
tmp.append(df.iloc[5:10])
tmp
pd.DataFrame(tmp)
eps = 0.1

c_totals = np.random.randint(1000, 3000, size=10)
mean = c_totals.mean()
sa   = c_totals.std()
sd   = eps * mean

basic_plot()

c_normd = c_totals/c_totals.mean() - c_totals.min()/c_totals.max()
mean = c_totals.max() - sd
c_totals = (2*sd * c_normd + (mean - sd))

basic_plot()<jupyter_output>1936.1 (193.61, 608.770145457216)
[2648 1644 1261 1311 1215 2895 2227 2026 1475 2659]
2732.4879792746115 (193.61, 121.75402909144326)
[2874.86797927 2674.06797927 2597.46797927 2607.46797927 2588.26797927
 2924.26797927 2790.66797927 2750.46797927 2640.26797927 2877.06797927]<jupyter_text>First time I got the normalization algorithm working:<jupyter_code>eps = 0.1

c_totals = np.random.randint(1000, 3000, size=10)
mean = c_totals.mean()
sa   = c_totals.std()
sd   = eps * mean

basic_plot()
c_totals / c_totals.mean() - (c_totals.min()/c_totals.max())
c_normd = c_totals/c_totals.mean() - c_totals.min()/c_totals.max()
c_normd
new_mean = c_totals.max() - sd
new_mean
new_totals = (2*sd * c_normd + (new_mean - sd))
new_totals
c_totals = new_totals
mean = new_mean

basic_plot()
c_norm = c_totals / np.linalg.norm(c_totals, ord=np.inf)
c_range= c_norm * sd*2
c_range
c_range.max()<jupyter_output><empty_output><jupyter_text>New algorithm:```set max to max category.calculate new mean as 1 σδ below max.normalize dataset to 1 σδ of new mean.```normalization:- % distance to new mean = % distance to original mean, with max distance set by σδ.    - This means that categories further from the original mean than the max is from it will be clamped to new mean - σδ.<jupyter_code>c_max = c_totals.max()
new_mean = c_max - sd

c_norm = c_totals/np.linalg.norm(c_totals, ord=np.inf)
c_total = c_norm * sd + new_mean

# for i,c_tot in enumerate(c_totals):
#     c_totals[i] = max(new_mean-sd, min())
#     if c_tot < mean:
#         c_totals[i] = max(new_mean-sd, c_tot*(c_tot/mean)/(mean/(new_mean+sd)))
#     elif c_tot > mean:
#         c_totals[i] = max(new_mean-sd, c_tot*(c_tot/c_max)/(c_max/(new_mean+sd)))

mean = new_mean
sa   = c_totals.std()
sd   = eps*mean

basic_plot()
c_max = c_totals.max()
new_mean = c_max - sd
for i,c_tot in enumerate(c_totals):
#     c_totals[i] = max(new_mean-sd, min())
    if c_tot < mean:
        c_totals[i] = max(new_mean-sd, c_tot*(c_tot/mean)/(mean/(new_mean+sd)))
    elif c_tot > mean:
        c_totals[i] = max(new_mean-sd, c_tot*(c_tot/c_max)/(c_max/(new_mean+sd)))

mean = new_mean
sa   = c_totals.std()
sd   = eps*mean

basic_plot()
diffs
# cat_pops = pops_from_df(csv, colnames=None)
cat_pops = pops_from_df(csv)
cat_pops.as_matrix(columns=['pop'])
totals = cat_pops.as_matrix(columns=['pop'])
totals.std()
cat_pops
for cat in cat_pops.as_matrix(columns=['cat']):
    print(cat)
df['class']==cat[0]
c_max = 150
c_tot = c_max
sd = 20
new_mean = c_max - sd

c_tot*(c_tot/c_max)/(c_max/(new_mean+sd))
c_max
new_mean + sd
c_totals.mean()
mean
basic_plot()
(70/100)/(100/150)
(150/100)/(100/150)<jupyter_output><empty_output><jupyter_text>Standard Deviation formula:So, I want σ to be `eps` percent of the total dataset. The main factor in this eq is the difference from the mean. And it's squared, so there's a high penalty on being off-mean. What makes this not-simple is σ and μ affect each other, so there's a differential equation somewhere that has to be solved.1st algorithm idea:```σ: standard deviation; α: actual; δ: desiredk = σδ/2While the σα is greater than σδ:    For each category:        if that category is below the mean by more than 1 σδ:            add k σδ of random copies to that category    recalculate mean    recalculate σδ    If no categories are below the mean by more than 1 σδ:        k = k/2```I'm concerned about updates. The 'perfect' numerical solution is to recalculate the mean/stdv every time `k` is added, and to keep `k` at 1. This is also a worst-case scenario for time. Setting `k=σδ/2` is my attempt at striking a balance between 'resolution' and speed.<jupyter_code>eps = 0.1

c_totals = np.random.randint(1000, 3000, size=10)
mean = c_totals.mean()
sa   = c_totals.std()
sd   = eps * mean

k = sd/2

iters = 0

basic_plot()<jupyter_output>2191.6 (219.16, 501.9958565566055)
[2187 1255 1965 2302 2380 2358 2208 2954 2836 1471]<jupyter_text>this initial version of the algorithm stops progressing when the minimum total is within 1 desired standard deviation of the mean. However, if any totals are more than a desired stdv from the mean, the actual stdv will be greater than that desired. This algorithm will fail to converge whenever this condition exists.<jupyter_code># while sa > sd:
k_OK = False
for i,c_tot in enumerate(c_totals):
    if mean - c_tot > sd:
        c_totals[i] += k
        k_OK = True
        
# update
mean = c_totals.mean()
sa   = c_totals.std()
sd   = eps * mean

# check resolution change
if not k_OK: k /= 2

iters += 1; print(f'iterations: {iters}')
basic_plot()
k
k
cat_pops
data = np.array([1000, 1500, 700, 1200, 1100, 900])
data.mean(), data.std()
ε = 0.1
σ = data.mean() * ε; σ
pops_from_df(df.sample(n=1000))
n = 100

catdx=1
catcol = df.columns[catdx]
cats = df[catcol].unique()

df_slice = df[catcol]

keep_idxs = np.array([], dtype=np.int64) ## np.dtype(int) --> dtype('int64')
for cat in cats:
    keep_idxs = np.concatenate((keep_idxs, np.random.choice(np.where(df_slice==cat)[0], size=n, replace=False)))
np.where(df_slice==cat)[0]
len(np.where(df_slice==cat)[0])
np.array([], dtype='int64'), np.array([], dtype=np.int64)
keep_idxs.shape
df.iloc[keep_idxs].head(n=10)
df.iloc[keep_idxs].sample(n=10)
np.dtype(type(3))
keep_idxs[:100]
np.concatenate(keep_idxs, np.random.choice(np.where(df_slice==cat)[0], size=n, replace=False))
np.random.choice(np.where(df_slice==cat)[0], size=n, replace=False)
keep_idxs = np.array([])
x = np.array([1,2,3,])
x.shape
np.concatenate((keep_idxs, x))
catdx=1
catcol = df.columns[catdx]
cats = df[catcol].unique()
for cat in cats:
#     pops = [df[df[y]==cat].count()[0] for cat in cats]
    keepidx = np.where(df[df[catcol]==cat])
    break
df_slice = df[catcol]
keepidxs = np.random.choice(np.where(df_slice==cat)[0], size = 20, replace=False)
np.where(df_slice==cat)[0]
df_slice
keepidx = np.where(df[df[catcol]==cat])
keepidx[0].shape
keepidx = df[df[catcol]==cat].as_matrix()[:,0]
keepidx.shape
keepidx[:n]
keepidx = np.where(df[df[catcol]==cat])
keepidx[0][:10]
df.iloc[keepidx].head()
list(zip(cats,pops))
list(zip(cats,pops))
rows=[]
for cat in catfolders:
    catpath = path/folder/cat
    fpaths  = list(map(lambda x: cat+'/'+x, os.listdir(catpath)))
    rows.extend(list(zip(fpaths,[labelmap[cat] for i in range(len(fpaths))])))

df = pd.DataFrame(rows, columns=['file','class'])
rows[0]
cols['cat']
list(zip(cols['cat'],cols['pop']))
dictionary=cat_pops

cols = dict()
cols['cat'],cols['pop'] = zip(*dictionary.items())
# pd.DataFrame([cols['cat'],cols['pop']],columns=['cat','pop'])
cols
pd.DataFrame(cols)
cols['cat']
cols = dict()
cols['cat'],cols['pop'] = zip(*cat_pops.items())
cols
df=csv;xdx=0;ydx=1

x,y = df.columns[xdx],df.columns[ydx]
cats = df[y].unique()
print(cats)
pops = [df[df[y]==cat].count()[0] for cat in cats]
csv[csv['class']=='cat'].count()
csv.columns[1]
%%time
labelmap = None
folder='train'
path = PATH
# find categories
catfolders = [f for f in os.listdir(path/folder) if (path/folder/f).is_dir()]
if labelmap is None: labelmap = {cf:cf for cf in catfolders}

df = pd.DataFrame(columns=['file_path','class'])
rows = []

for cat in catfolders:
    catpath = path/folder/cat
    fpaths = list(map(lambda x: cat+'/'+x , os.listdir(catpath)))
#     fpaths  = list(map(str, catpath.iterdir()))
    rows.extend(list(zip(fpaths,[cat for i in range(len(fpaths))])))
    break
# rows
len(rows)
df = pd.DataFrame(rows, columns=['file_path','class'])
df.head()
n=10
# fpaths[:n], [cat for i in range(n)]

list(zip(fpaths[:n],[cat for i in range(n)]))
n = 2000
%%time
rows = []
rows1 = list(zip(fpaths[:n], [cat for i in range(n)]));
rows2 = list(zip(fpaths[n:2*n], [cat for i in range(n)]));
rows.append(rows1)
rows.append(rows2)
rows
%%time
rows = []
rows1 = list(zip(fpaths[:n], [cat for i in range(n)]));
rows2 = list(zip(fpaths[n:2*n], [cat for i in range(n)]));
rows.extend(rows1)
rows.extend(rows2)
rows
rows
df = pd.DataFrame(list(zip(fpaths[:n],[cat for i in range(n)])), columns=['file_path','class'])
df
df = pd.DataFrame(rows[0],rows[1], columns=['file_path','class'])
df
files = generate_csv(PATH)
cat = 'cat'
path = PATH
folder = 'train'
files_np = np.array(files)
catpath = path/folder/cat;
catpath
list(map(str, catpath.iterdir()))[:20]
cat_pops_np = np.array([[k,v] for k,v in cat_pops.items()])
cat_pops_np[:,1]
from collections import defaultdict
cat_pops_dict = defaultdict(lambda x: [])
for k,v in cat_pops.items():
    cat_pops_dict[k] = v
cat_pops_dict
cat_pops_dict = {'cat':k,'pop':v for k,v in cat_pops.items()}
df = pd.DataFrame({'cat':[cat for cat in cat_pops.keys()],'pop':[pop for pop in cat_pops.values()]})
df.plot.bar()
df = pd.DataFrame({'cat':})
cat_pops_np[:,0]
df = pd.DataFrame({'cat':cat_pops_np[:,0],'pop':cat_pops_np[:,1]})
df.
df.plot.bar()
def csv_smooth_data(path, threshold):<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1497755407427561, 'fraction_numerical': 0.04618419262685349, 'mean_word_length': 5.096827700601286, 'pattern_counts': {'":': 0, '<': 148, '<?xml version=': 0, '>': 155, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 6, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 1, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '40', 'dir': 'jupyter-structured-clean-dedup', 'id': '569010', 'license': 'MIT', 'n_tokens_mistral': 12495, 'n_tokens_neox': 11466, 'n_words': 2461, 'path': 'research/autocsv-dev.ipynb', 'repo_name': 'WNoxchi/Kawkasos', 'size': '29404'}