<jupyter_start><jupyter_text>TablesIn this notebook, we give all the values that are provided in the different tables.**Requirements**:- You need to run notebook `6-herding` to get the file `time_series_ranks,csv` in the folder `tmp`. In other words, you need to **run the first 6 cells of `6-herding`**.- However, to run notebook `6-herding`, you need to **run the first 7 cells of `4-zscores`** first.**Benchmark time**: This notebook has been run on a Dell Latitude (ElementaryOS 0.4.1 Loki, i7-7600U, 16GB RAM).Here are the anchors to the different Tables that are created by this notebook:- [Table 1 - Dataset Size](table1)- [Table 2 - Dataset size after matching](table2)- [Table 3 - Number beers per paired-treatment](table3)- [Table 4 - Brewery location before and after matching](table4)- [Table 5 - Counts and Probabilities](table5)- [Table 5 - Update version](table5_upd)- [Table 6 - Standardized 5th rating](table6)<jupyter_code>import os
os.chdir('..')
# Helpers functions
from python.helpers import parse
import python.get_old_file as get_old_file

# Libraries for preparing data
import ast
import json
import gzip
import numpy as np
import pandas as pd
from datetime import datetime

# Libraries for plotting
import seaborn as sns
import matplotlib.pyplot as plt
from matplotlib.ticker import FuncFormatter
import matplotlib

# Folders
data_folder = '../data/'
fig_folder = '../figures/'

# For the Python notebook
%matplotlib inline
%reload_ext autoreload
%autoreload 2

# Check that folders exist
if not os.path.exists(data_folder + 'tmp'):
    os.makedirs(data_folder + 'tmp')
    
# Other
labels = {'ba': 'BeerAdvocate', 'rb': 'RateBeer'}<jupyter_output><empty_output><jupyter_text>Table 1 - Dataset size<jupyter_code>dataset_size = {}

for key in ['rb', 'ba']:
    
    dataset_size[key] = {}
    
    brew = pd.read_csv(data_folder + key + '/breweries.csv')
    dataset_size[key]['#breweries'] = len(brew)
    
    beers = pd.read_csv(data_folder + key + '/beers.csv')
    dataset_size[key]['#beers'] = len(beers)
    dataset_size[key]['#beers (>=5 ratings)'] = len(beers[beers['nbr_ratings']>=5])
    dataset_size[key]['#beers (>=10 ratings)'] = len(beers[beers['nbr_ratings']>=10])
    dataset_size[key]['#beers (>=20 ratings)'] = len(beers[beers['nbr_ratings']>=20])
        
    users = pd.read_csv(data_folder + key + '/users.csv')
    dataset_size[key]['#users'] = len(users)
    dataset_size[key]['#users (>=10 ratings)'] = len(users[users['nbr_ratings']>=10])
    dataset_size[key]['#users (>=100 ratings)'] = len(users[users['nbr_ratings']>=100])

    dataset_size[key]['#ratings'] = np.sum(beers['nbr_ratings'])

for key in ['rb', 'ba']:
    print(labels[key])
    for k in dataset_size[key].keys():
        print('  {}: {}'.format(k, dataset_size[key][k]))
    print()<jupyter_output>RateBeer
  #breweries: 24189
  #beers: 442081
  #beers (>=5 ratings): 166043
  #beers (>=10 ratings): 104062
  #beers (>=20 ratings): 60451
  #users: 70174
  #users (>=10 ratings): 17744
  #users (>=100 ratings): 6419
  #ratings: 7122074

BeerAdvocate
  #breweries: 16758
  #beers: 280823
  #beers (>=5 ratings): 96156
  #beers (>=10 ratings): 61193
  #beers (>=20 ratings): 38533
  #users: 153704
  #users (>=10 ratings): 48595
  #users (>=100 ratings): 14488
  #ratings: 8393032<jupyter_text>Table 2 - Dataset size after matching<jupyter_code>dataset_size = {}

beers = pd.read_csv(data_folder + 'matched/beers.csv', header=[0,1])

for nbr in [0, 5, 10, 20]:
    dataset_size[nbr] = {}
    
    df = beers[(beers['ba']['nbr_ratings'] >= nbr) & (beers['rb']['nbr_ratings'] >= nbr)]
    
    dataset_size[nbr]['#breweries'] = len(np.unique(df['ba']['brewery_name']))
    dataset_size[nbr]['#beers'] = len(df)
    
    dataset_size[nbr]['#ratings on BA'] = np.sum(df['ba']['nbr_ratings'])
    dataset_size[nbr]['#ratings on RB'] = np.sum(df['rb']['nbr_ratings'])

for nbr in dataset_size.keys():
    print('Min. nbr of ratings per beer: {}'.format(nbr))
    for key in dataset_size[nbr].keys():
        print('  {}: {}'.format(key, dataset_size[nbr][key]))
    print()<jupyter_output>Min. nbr of ratings per beer: 0
  #breweries: 6084
  #beers: 45640
  #ratings on BA: 955968
  #ratings on RB: 1020638

Min. nbr of ratings per beer: 5
  #breweries: 2561
  #beers: 12890
  #ratings on BA: 873944
  #ratings on RB: 761496

Min. nbr of ratings per beer: 10
  #breweries: 1711
  #beers: 7424
  #ratings on BA: 812070
  #ratings on RB: 650642

Min. nbr of ratings per beer: 20
  #breweries: 1079
  #beers: 4051
  #ratings on BA: 732165
  #ratings on RB: 542961<jupyter_text>Table 3 - Number beers per paired-treatment<jupyter_code># Get the time series with ranks
df = pd.read_csv(data_folder + 'tmp/time_series_ranks.csv', header=[0,1])

# Transform strings into arrays
for i in df.index:
    for key1 in ['ba', 'rb']:
        for key2 in ['dates', 'ratings', 'z_scores', 'ranks']:
            arr = ast.literal_eval(df.loc[i][key1][key2])
            df.set_value(i, (key1, key2), arr)
            
# Compute the zscores in a dictionnary
diffs = {'ba': [], 'rb': []}

for key in diffs.keys():
    for i in df.index:
        row = df.iloc[i]
        diffs[key].append(row[key]['z_scores'][0])

thresholds = {}

for key in ['ba', 'rb']:
    thresholds[key] = {}
    thresholds[key]['low'] = np.percentile(diffs[key], 15)
    thresholds[key]['high'] = np.percentile(diffs[key], 85)
    
# Compute the paired-treatment groups
classes = {}

for i in df.index:    
    ba = diffs['ba'][i]
    rb = diffs['rb'][i]


    if ba > thresholds['ba']['high']:
        class_ = 'H'
    elif ba >= thresholds['ba']['low']:
        class_ = 'M'
    else:
        class_ = 'L'

    class_ += '-'

    if rb > thresholds['rb']['high']:
        class_ += 'H'
    elif rb >= thresholds['rb']['low']:
        class_ += 'M'
    else:
        class_ += 'L' 

    if class_ not in classes.keys():
        classes[class_] = 0

    classes[class_] += 1
    
print(classes)<jupyter_output>{'M-M': 6593, 'H-M': 1210, 'L-M': 1242, 'M-H': 1213, 'H-L': 138, 'M-L': 1225, 'L-L': 568, 'L-H': 116, 'H-H': 585}<jupyter_text>Table 4 - Brewery location before and after matching<jupyter_code>brew_matched = pd.read_csv(data_folder + 'matched/breweries.csv', header=[0,1])

loc = []
for i, l in enumerate(brew_matched['ba']['location']):
    if 'United States' in l:
        loc.append('United States')
    else:
        loc.append(l)
brew_matched.loc[:, (('ba', 'location'))] = loc
        
brew_ba = pd.read_csv(data_folder + 'ba/breweries.csv')

loc = []
for i, l in enumerate(brew_ba['location']):
    if 'United States' in l:
        loc.append('United States')
    else:
        loc.append(l)
        
brew_ba.loc[:, 'location'] = loc
        
brew_rb = pd.read_csv(data_folder + 'rb/breweries.csv')

loc = []
for i, l in enumerate(brew_rb['location']):
    if 'United States' in l:
        loc.append('United States')
    else:
        loc.append(l)
        
brew_rb.loc[:, 'location'] = loc
brew_matched['ba']['location'].value_counts().head(10)/len(brew_matched)*100
countries = ['United States', 'Germany', 'England', 'Canada', 'Italy', 'Belgium', 'France', 'Spain', 'Australia', 'Netherlands']

for c in countries:
    print('{}:'.format(c))
    count = np.sum(brew_matched['ba']['location'] == c)/len(brew_matched)*100
    print('  Matched: {:.1f}%'.format(count))
    count = np.sum(brew_ba['location'] == c)/len(brew_ba)*100
    print('  BA: {:.1f}%'.format(count))
    count = np.sum(brew_rb['location'] == c)/len(brew_rb)*100
    print('  RB: {:.1f}%'.format(count))
    print()<jupyter_output>United States:
  Matched: 47.8%
  BA: 44.4%
  RB: 28.6%

Germany:
  Matched: 6.4%
  BA: 8.5%
  RB: 8.3%

England:
  Matched: 5.8%
  BA: 5.9%
  RB: 8.8%

Canada:
  Matched: 4.9%
  BA: 4.6%
  RB: 3.7%

Italy:
  Matched: 2.7%
  BA: 2.2%
  RB: 4.3%

Belgium:
  Matched: 2.5%
  BA: 2.0%
  RB: 1.9%

France:
  Matched: 2.3%
  BA: 2.4%
  RB: 3.5%

Spain:
  Matched: 2.3%
  BA: 1.9%
  RB: 3.2%

Australia:
  Matched: 2.2%
  BA: 2.4%
  RB: 2.3%

Netherlands:
  Matched: 2.1%
  BA: 1.5%
  RB: 2.2%<jupyter_text>Table 5 - Counts and ProbabilitiesWhile working on the notebooks to make them clean, we saw that a wrong file was used to produce the results of Table 5. In the first cell, we provide a way to compute the old file to get the same results as in the paper. However, if you would like to have the results with the correct file, you can skip this first cell and continue with the next cells.**WARNING**: Since we have to draw a subset of the beers to compare the numbers, we can't reproduce exactly the results from Table 5. Therefore, you will have very similar results with very small differences. (*In other words, we forgot to keep the seed*.)<jupyter_code>%%time

# Purpose of this cell is to compute the old file to get 
# the same results as in the paper

get_old_file.run()
%%time

# Old file as in the paper (Comment or uncomment the three following lines of code)
ts = pd.read_csv(data_folder + 'tmp/time_series_with_text.csv', header=[0,1])

# Transform strings into arrays
for i in ts.index:
    for key1 in ['ba', 'rb']:
        for key2 in ['dates', 'ratings', 'z_scores']:
            arr = ast.literal_eval(ts.loc[i][key1][key2])
            ts.set_value(i, (key1, key2), arr)

df = pd.read_csv(data_folder + 'matched/beers.csv', header=[0,1])

brews = {'ba': None, 'rb': None}
brews['ba'] = pd.read_csv(data_folder + 'ba/breweries.csv')
brews['rb'] = pd.read_csv(data_folder + 'rb/breweries.csv')

diffs = {'ba': [], 'rb': []}
for i in ts.index:
    row = ts.iloc[i]
    for key in ['rb', 'ba']:
        diffs[key].append(row[key]['z_scores'][0])
        
thresholds = {}
for key in ['ba', 'rb']:
    thresholds[key] = {}
    thresholds[key]['low'] = np.percentile(diffs[key], 15)
    thresholds[key]['high'] = np.percentile(diffs[key], 85)
    
for key in ['ba', 'rb']:
    tmp = []
    nbr_rats = []
    
    for i in ts.index:      
        if diffs[key][i] > thresholds[key]['high']:
            tmp.append('H')
        elif diffs[key][i] >= thresholds[key]['low']:
            tmp.append('M')
        else:
            tmp.append('L')
            
        nbr_rats.append(len(ts.loc[i][key]['ratings']))

    ts.loc[:, ((key, 'class'))] = tmp
    ts.loc[:, ((key, 'nbr_ratings'))] = nbr_rats
    
ts = ts[(ts['ba']['nbr_ratings'] >= 5) & (ts['rb']['nbr_ratings'] >= 5)]
ts.index = range(len(ts))
    
dfs = {'ba': None, 'rb': None}

for key in dfs.keys():
    dfs[key] = ts[key][['beer_id', 'class']]

    dfs[key]['style'] = np.array(df[df[key]['beer_id'].isin(dfs[key]['beer_id'])][key]['style'])
    dfs[key]['brewery_name'] = np.array(df[df[key]['beer_id'].isin(dfs[key]['beer_id'])][key]['brewery_name'])

    locations = []
    for i in dfs[key].index:
        brew_name = dfs[key].loc[i]['brewery_name']

        subdf = brews[key][brews[key]['name'] == brew_name]['location']

        loc = subdf.loc[subdf.index[0]]

        if 'United States' in loc:
            loc = 'United States'

        locations.append(loc)

    dfs[key]['location'] = locations
    
dfs['rb']['style'] = dfs['ba']['style']<jupyter_output>CPU times: user 1min 39s, sys: 360 ms, total: 1min 39s
Wall time: 1min 42s<jupyter_text>Get the DF for the styles and print the table<jupyter_code>%%time

styles = dfs['ba']['style'].unique()

df_json = {}

classes = ['H-L', 'H-M', 'L-M']

for cl in classes:
    df_json[cl] = []
    df_json[cl[::-1]] = []
    
for cl in classes:
    idx1 = np.array((dfs['ba']['class'] == cl[0]) & (dfs['rb']['class'] == cl[-1]))
    subdf1 = dfs['ba'][idx1]
    subdf1.index = range(len(subdf1))
    
    idx2 = np.array((dfs['ba']['class'] == cl[-1]) & (dfs['rb']['class'] == cl[0]))
    subdf2 = dfs['ba'][idx2]
    subdf2.index = range(len(subdf2))

    nbr = min(len(subdf1), len(subdf2))
    
    subdf1 = subdf1.loc[np.random.choice(range(len(subdf1)), nbr, replace=False)]
    subdf1.index = range(nbr)
    
    subdf2 = subdf2.loc[np.random.choice(range(len(subdf2)), nbr, replace=False)]
    subdf2.index = range(nbr)    
    
    df_json[cl].append(nbr)
    df_json[cl[::-1]].append(nbr)

    for s in styles:
        subsubdf1 = subdf1[subdf1['style'] == s]
        df_json[cl].append(len(subsubdf1))
        
        subsubdf2 = subdf2[subdf2['style'] == s]
        df_json[cl[::-1]].append(len(subsubdf2))
        
styles = np.insert(styles, 0, 'Total')
df_json['styles'] = styles

df_styles = pd.DataFrame.from_dict(df_json)

df_styles.to_csv(data_folder + 'tmp/styles.csv', index=False)
dataframes = {}

for t in ['HM', 'ML', 'HL']:

    nbrs = df_styles[t[0] + '-' + t[1]] + df_styles[t[1] + '-' + t[0]]
    nbrs = nbrs[1:]
    
    indices = nbrs.nlargest(10).index
    
    small_df = df_styles.iloc[indices]

    df_json = {'style': {
        'name': list(small_df['styles'])
    },  'nbrs': {
        'BA': list(small_df[t[0] + '-' + t[1]]), 
        'RB': list(small_df[t[1] + '-' + t[0]])
    }, 'proba': {
        'BA': [],
        'RB': []
    }}

    for vba, vrb in zip(df_json['nbrs']['BA'], df_json['nbrs']['RB']):
        df_json['proba']['BA'].append(vba/(vba+vrb))
        df_json['proba']['RB'].append(vrb/(vba+vrb))
        
    dataframes[t] = pd.DataFrame.from_dict({(i, j): df_json[i][j]
                                 for i in df_json.keys()
                                 for j in df_json[i].keys()})
    
for t in dataframes:
    print(t)
    print(dataframes[t])
    print()<jupyter_output>HM
  nbrs         proba                                       style
    BA  RB        BA        RB                              name
0  105  88  0.544041  0.455959                      American IPA
1  100  81  0.552486  0.447514    American Double / Imperial IPA
2   35  30  0.538462  0.461538            Saison / Farmhouse Ale
3   32  32  0.500000  0.500000           American Pale Ale (APA)
4   27  36  0.428571  0.571429                 American Wild Ale
5   22  36  0.379310  0.620690  American Double / Imperial Stout
6   15  20  0.428571  0.571429                American Black Ale
7   14  18  0.437500  0.562500                   American Porter
8   12  13  0.480000  0.520000            Russian Imperial Stout
9   10  14  0.416667  0.583333                    American Stout

ML
  nbrs         proba                                     style
    BA  RB        BA        RB                            name
0   64  67  0.488550  0.511450                    American IPA
1   36  44  0.450000  0.[...]<jupyter_text>Get the DF for the styles and print the table<jupyter_code>%%time

countries = dfs['ba']['location'].unique()

df_json = {}

classes = ['H-L', 'H-M', 'L-M']

for cl in classes:
    df_json[cl] = []
    df_json[cl[::-1]] = []
    
for cl in classes:
    idx1 = np.array((dfs['ba']['class'] == cl[0]) & (dfs['rb']['class'] == cl[-1]))
    subdf1 = dfs['ba'][idx1]
    subdf1.index = range(len(subdf1))
    
    idx2 = np.array((dfs['ba']['class'] == cl[-1]) & (dfs['rb']['class'] == cl[0]))
    subdf2 = dfs['ba'][idx2]
    subdf2.index = range(len(subdf2))
    
    nbr = min(len(subdf1), len(subdf2))
    
    subdf1 = subdf1.loc[np.random.choice(range(len(subdf1)), nbr, replace=False)]
    subdf1.index = range(nbr)
    
    subdf2 = subdf2.loc[np.random.choice(range(len(subdf2)), nbr, replace=False)]
    subdf2.index = range(nbr)    
    
    df_json[cl].append(nbr)
    df_json[cl[::-1]].append(nbr)

    for c in countries:
        subsubdf1 = subdf1[subdf1['location'] == c]
        df_json[cl].append(len(subsubdf1))
        
        subsubdf2 = subdf2[subdf2['location'] == c]
        df_json[cl[::-1]].append(len(subsubdf2))
        
countries = np.insert(countries, 0, 'Total')
df_json['countries'] = countries

df_countries = pd.DataFrame.from_dict(df_json)

df_countries.to_csv(data_folder + 'tmp/countries.csv', index=False)
dataframes = {}

for t in ['HM', 'ML', 'HL']:

    nbrs = df_countries[t[0] + '-' + t[1]] + df_countries[t[1] + '-' + t[0]]
    nbrs = nbrs[1:]
    
    indices = nbrs.nlargest(10).index
    
    small_df = df_countries.iloc[indices]

    df_json = {'country': {
        'name': list(small_df['countries'])
    },  'nbrs': {
        'BA': list(small_df[t[0] + '-' + t[1]]), 
        'RB': list(small_df[t[1] + '-' + t[0]])
    }, 'proba': {
        'BA': [],
        'RB': []
    }}

    for vba, vrb in zip(df_json['nbrs']['BA'], df_json['nbrs']['RB']):
        df_json['proba']['BA'].append(vba/(vba+vrb))
        df_json['proba']['RB'].append(vrb/(vba+vrb))
        
    dataframes[t] = pd.DataFrame.from_dict({(i, j): df_json[i][j]
                                 for i in df_json.keys()
                                 for j in df_json[i].keys()})
    
for t in dataframes:
    print(t)
    print(dataframes[t])
    print()<jupyter_output>HM
         country nbrs          proba          
            name   BA   RB        BA        RB
0  United States  545  493  0.525048  0.474952
1         Canada   24   37  0.393443  0.606557
2        Belgium   29   31  0.483333  0.516667
3        England    7   13  0.350000  0.650000
4        Germany   10    7  0.588235  0.411765
5      Australia    3   14  0.176471  0.823529
6         Sweden   10    6  0.625000  0.375000
7          Italy    3    8  0.272727  0.727273
8        Denmark    4    7  0.363636  0.636364
9    New Zealand    0    9  0.000000  1.000000

ML
         country nbrs          proba          
            name   BA   RB        BA        RB
0  United States  445  449  0.497763  0.502237
1         Canada   84   56  0.600000  0.400000
2        Belgium   28   36  0.437500  0.562500
3        Germany   20   21  0.487805  0.512195
4      Australia   20   20  0.500000  0.500000
5        England    9   17  0.346154  0.653846
6    Netherlands    7   11  0.388889  0.611111
7     [...]<jupyter_text>Table 5 - Updated versionWe redo the Table 5 with the correct file to show you that despite using the wrong file, the conclusions do not change.<jupyter_code>%%time

# Correct file
ts = pd.read_csv(data_folder + 'tmp/time_series.csv', header=[0,1])

# Transform strings into arrays
for i in ts.index:
    for key1 in ['ba', 'rb']:
        for key2 in ['dates', 'ratings', 'z_scores']:
            arr = ast.literal_eval(ts.loc[i][key1][key2])
            ts.set_value(i, (key1, key2), arr)

df = pd.read_csv(data_folder + 'matched/beers.csv', header=[0,1])

brews = {'ba': None, 'rb': None}
brews['ba'] = pd.read_csv(data_folder + 'ba/breweries.csv')
brews['rb'] = pd.read_csv(data_folder + 'rb/breweries.csv')

diffs = {'ba': [], 'rb': []}
for i in ts.index:
    row = ts.iloc[i]
    for key in ['rb', 'ba']:
        diffs[key].append(row[key]['z_scores'][0])
        
thresholds = {}
for key in ['ba', 'rb']:
    thresholds[key] = {}
    thresholds[key]['low'] = np.percentile(diffs[key], 15)
    thresholds[key]['high'] = np.percentile(diffs[key], 85)
    
for key in ['ba', 'rb']:
    tmp = []
    nbr_rats = []
    
    for i in ts.index:      
        if diffs[key][i] > thresholds[key]['high']:
            tmp.append('H')
        elif diffs[key][i] >= thresholds[key]['low']:
            tmp.append('M')
        else:
            tmp.append('L')
            
        nbr_rats.append(len(ts.loc[i][key]['ratings']))

    ts.loc[:, ((key, 'class'))] = tmp
    ts.loc[:, ((key, 'nbr_ratings'))] = nbr_rats
    
ts = ts[(ts['ba']['nbr_ratings'] >= 5) & (ts['rb']['nbr_ratings'] >= 5)]
ts.index = range(len(ts))
    
dfs = {'ba': None, 'rb': None}

for key in dfs.keys():
    dfs[key] = ts[key][['beer_id', 'class']]

    dfs[key]['style'] = np.array(df[df[key]['beer_id'].isin(dfs[key]['beer_id'])][key]['style'])
    dfs[key]['brewery_name'] = np.array(df[df[key]['beer_id'].isin(dfs[key]['beer_id'])][key]['brewery_name'])

    locations = []
    for i in dfs[key].index:
        brew_name = dfs[key].loc[i]['brewery_name']

        subdf = brews[key][brews[key]['name'] == brew_name]['location']

        loc = subdf.loc[subdf.index[0]]

        if 'United States' in loc:
            loc = 'United States'

        locations.append(loc)

    dfs[key]['location'] = locations
    
dfs['rb']['style'] = dfs['ba']['style']<jupyter_output>CPU times: user 2min 13s, sys: 452 ms, total: 2min 14s
Wall time: 2min 14s<jupyter_text>Get the DF for the styles and print the table<jupyter_code>%%time

styles = dfs['ba']['style'].unique()

df_json = {}

classes = ['H-L', 'H-M', 'L-M']

for cl in classes:
    df_json[cl] = []
    df_json[cl[::-1]] = []
    
for cl in classes:
    idx1 = np.array((dfs['ba']['class'] == cl[0]) & (dfs['rb']['class'] == cl[-1]))
    subdf1 = dfs['ba'][idx1]
    subdf1.index = range(len(subdf1))
    
    idx2 = np.array((dfs['ba']['class'] == cl[-1]) & (dfs['rb']['class'] == cl[0]))
    subdf2 = dfs['ba'][idx2]
    subdf2.index = range(len(subdf2))

    nbr = min(len(subdf1), len(subdf2))
    
    subdf1 = subdf1.loc[np.random.choice(range(len(subdf1)), nbr, replace=False)]
    subdf1.index = range(nbr)
    
    subdf2 = subdf2.loc[np.random.choice(range(len(subdf2)), nbr, replace=False)]
    subdf2.index = range(nbr)    
    
    df_json[cl].append(nbr)
    df_json[cl[::-1]].append(nbr)

    for s in styles:
        subsubdf1 = subdf1[subdf1['style'] == s]
        df_json[cl].append(len(subsubdf1))
        
        subsubdf2 = subdf2[subdf2['style'] == s]
        df_json[cl[::-1]].append(len(subsubdf2))
        
styles = np.insert(styles, 0, 'Total')
df_json['styles'] = styles

df_styles = pd.DataFrame.from_dict(df_json)

df_styles.to_csv(data_folder + 'tmp/styles.csv', index=False)
dataframes = {}

for t in ['HM', 'ML', 'HL']:

    nbrs = df_styles[t[0] + '-' + t[1]] + df_styles[t[1] + '-' + t[0]]
    nbrs = nbrs[1:]
    
    indices = nbrs.nlargest(10).index
    
    small_df = df_styles.iloc[indices]

    df_json = {'style': {
        'name': list(small_df['styles'])
    },  'nbrs': {
        'BA': list(small_df[t[0] + '-' + t[1]]), 
        'RB': list(small_df[t[1] + '-' + t[0]])
    }, 'proba': {
        'BA': [],
        'RB': []
    }}

    for vba, vrb in zip(df_json['nbrs']['BA'], df_json['nbrs']['RB']):
        df_json['proba']['BA'].append(vba/(vba+vrb))
        df_json['proba']['RB'].append(vrb/(vba+vrb))
        
    dataframes[t] = pd.DataFrame.from_dict({(i, j): df_json[i][j]
                                 for i in df_json.keys()
                                 for j in df_json[i].keys()})
    
for t in dataframes:
    print(t)
    print(dataframes[t])
    print()<jupyter_output>HM
  nbrs          proba                                       style
    BA   RB        BA        RB                              name
0  164  199  0.451791  0.548209                      American IPA
1  156  117  0.571429  0.428571    American Double / Imperial IPA
2   83   77  0.518750  0.481250                 American Wild Ale
3   80   69  0.536913  0.463087            Saison / Farmhouse Ale
4   60   68  0.468750  0.531250           American Pale Ale (APA)
5   44   59  0.427184  0.572816  American Double / Imperial Stout
6   39   34  0.534247  0.465753                   American Porter
7   23   41  0.359375  0.640625                American Black Ale
8   22   30  0.423077  0.576923            Russian Imperial Stout
9   27   21  0.562500  0.437500          American Amber / Red Ale

ML
  nbrs          proba                                     style
    BA   RB        BA        RB                            name
0  111  133  0.454918  0.545082                    American IPA
1   75   [...]<jupyter_text>Get the DF for the styles and print the table<jupyter_code>%%time

countries = dfs['ba']['location'].unique()

df_json = {}

classes = ['H-L', 'H-M', 'L-M']

for cl in classes:
    df_json[cl] = []
    df_json[cl[::-1]] = []
    
for cl in classes:
    idx1 = np.array((dfs['ba']['class'] == cl[0]) & (dfs['rb']['class'] == cl[-1]))
    subdf1 = dfs['ba'][idx1]
    subdf1.index = range(len(subdf1))
    
    idx2 = np.array((dfs['ba']['class'] == cl[-1]) & (dfs['rb']['class'] == cl[0]))
    subdf2 = dfs['ba'][idx2]
    subdf2.index = range(len(subdf2))
    
    nbr = min(len(subdf1), len(subdf2))
    
    subdf1 = subdf1.loc[np.random.choice(range(len(subdf1)), nbr, replace=False)]
    subdf1.index = range(nbr)
    
    subdf2 = subdf2.loc[np.random.choice(range(len(subdf2)), nbr, replace=False)]
    subdf2.index = range(nbr)    
    
    df_json[cl].append(nbr)
    df_json[cl[::-1]].append(nbr)

    for c in countries:
        subsubdf1 = subdf1[subdf1['location'] == c]
        df_json[cl].append(len(subsubdf1))
        
        subsubdf2 = subdf2[subdf2['location'] == c]
        df_json[cl[::-1]].append(len(subsubdf2))
        
countries = np.insert(countries, 0, 'Total')
df_json['countries'] = countries

df_countries = pd.DataFrame.from_dict(df_json)

df_countries.to_csv(data_folder + 'tmp/countries.csv', index=False)
dataframes = {}

for t in ['HM', 'ML', 'HL']:

    nbrs = df_countries[t[0] + '-' + t[1]] + df_countries[t[1] + '-' + t[0]]
    nbrs = nbrs[1:]
    
    indices = nbrs.nlargest(10).index
    
    small_df = df_countries.iloc[indices]

    df_json = {'country': {
        'name': list(small_df['countries'])
    },  'nbrs': {
        'BA': list(small_df[t[0] + '-' + t[1]]), 
        'RB': list(small_df[t[1] + '-' + t[0]])
    }, 'proba': {
        'BA': [],
        'RB': []
    }}

    for vba, vrb in zip(df_json['nbrs']['BA'], df_json['nbrs']['RB']):
        df_json['proba']['BA'].append(vba/(vba+vrb))
        df_json['proba']['RB'].append(vrb/(vba+vrb))
        
    dataframes[t] = pd.DataFrame.from_dict({(i, j): df_json[i][j]
                                 for i in df_json.keys()
                                 for j in df_json[i].keys()})
    
for t in dataframes:
    print(t)
    print(dataframes[t])
    print()<jupyter_output>HM
         country  nbrs          proba          
            name    BA   RB        BA        RB
0  United States  1003  924  0.520498  0.479502
1         Canada    49   70  0.411765  0.588235
2        Belgium    33   34  0.492537  0.507463
3        England    21   34  0.381818  0.618182
4        Germany    22   10  0.687500  0.312500
5      Australia     9   19  0.321429  0.678571
6         Sweden    14   14  0.500000  0.500000
7          Italy     5   13  0.277778  0.722222
8        Denmark     6   10  0.375000  0.625000
9        Ireland     6    6  0.500000  0.500000

ML
         country nbrs          proba          
            name   BA   RB        BA        RB
0  United States  801  772  0.509218  0.490782
1         Canada  162  100  0.618321  0.381679
2        Belgium   36   52  0.409091  0.590909
3        Germany   38   41  0.481013  0.518987
4      Australia   26   40  0.393939  0.606061
5        England   16   35  0.313725  0.686275
6    Netherlands   15   25  0.375000  0.6[...]<jupyter_text>Table 6 - Standardized 5th rating<jupyter_code>%%time

df = pd.read_csv(data_folder + 'tmp/time_series_ranks.csv', header=[0,1])

# Transform strings into arrays
for i in df.index:
    for key1 in ['ba', 'rb']:
        for key2 in ['dates', 'ratings', 'z_scores', 'ranks']:
            arr = ast.literal_eval(df.loc[i][key1][key2])
            df.set_value(i, (key1, key2), arr)
            
diffs = {'ba': [], 'rb': []}

for key in diffs.keys():
    for i in df.index:
        row = df.iloc[i]
        diffs[key].append(row[key]['z_scores'][0])

thresholds = {}

for key in ['ba', 'rb']:
    thresholds[key] = {}
    thresholds[key]['low'] = np.percentile(diffs[key], 15)
    thresholds[key]['high'] = np.percentile(diffs[key], 85)
np.random.seed(10)

nbr_draws = 1000

min_nbr_rats = 5

for to_study in ['H-L', 'H-M', 'M-L']:
    
    size_ = np.infty
    
    for cl in [to_study, to_study[::-1]]:


        subdf = df[((df['ba']['class'] == cl[0]) & (df['rb']['class'] == cl[-1])) & 
                   (df['ba']['nbr_ratings'] >= min_nbr_rats) & (df['rb']['nbr_ratings'] >= min_nbr_rats)]    
        
        if size_ > len(subdf):
            size_ = len(subdf)
            
    nbrs = {'ba': {}, 'rb': {}}

    for cl in [to_study, to_study[::-1]]:

        cls = {'ba': cl[0], 'rb': cl[-1]}

        subdf = df[((df['ba']['class'] == cl[0]) & (df['rb']['class'] == cl[-1])) & 
                   (df['ba']['nbr_ratings'] >= min_nbr_rats) & (df['rb']['nbr_ratings'] >= min_nbr_rats)] 
        subdf.index = range(len(subdf))
        
        subdf = subdf.loc[np.random.choice(len(subdf), size_, replace=False)]
        
        for key in ['ba', 'rb']:

            nbrs[key][cls[key]] = {}

            ranks = np.array([r[min_nbr_rats-1] for r in subdf[key]['z_scores']])

            nbrs[key][cls[key]]['avg'] = np.mean(ranks)

            tmp = []
            # Go through each draw
            for d in range(nbr_draws):

                # Get the indices
                idx = np.random.randint(0, len(subdf[key]), len(subdf[key]))

                ranks_tmp = ranks[idx]

                tmp.append(np.mean(ranks_tmp))
            nbrs[key][cls[key]]['low'] = np.percentile(tmp, 2.5)
            nbrs[key][cls[key]]['high'] = np.percentile(tmp, 97.5)
        

    print('{}:'.format(to_study))
    for key in nbrs:
        print('  {}:'.format(key.upper()))
        for key2 in nbrs[key]:
            print('    {}: {:.3f} [{:.3f}, {:.3f}]'.format(key2, nbrs[key][key2]['avg'], nbrs[key][key2]['low'], nbrs[key][key2]['high']))
    print()<jupyter_output>H-L:
  BA:
    H: -0.032 [-0.193, 0.123]
    L: -0.392 [-0.572, -0.231]
  RB:
    L: -0.066 [-0.220, 0.099]
    H: 0.226 [0.045, 0.393]

H-M:
  BA:
    H: 0.270 [0.225, 0.314]
    M: 0.057 [0.006, 0.107]
  RB:
    M: 0.376 [0.334, 0.423]
    H: 0.498 [0.460, 0.542]

M-L:
  BA:
    M: -0.506 [-0.563, -0.450]
    L: -0.657 [-0.721, -0.596]
  RB:
    L: -0.414 [-0.467, -0.363]
    M: -0.229 [-0.279, -0.176]
{'fraction_non_alphanumeric': 0.17422153473670052, 'fraction_numerical': 0.06456385769049701, 'mean_word_length': 2.516911069063387, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 31, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '600441', 'license': 'MIT', 'n_tokens_mistral': 12535, 'n_tokens_neox': 11371, 'n_words': 2553, 'path': 'code/notebooks/8-tables.ipynb', 'repo_name': 'epfl-dlab/when_sheep_shop', 'size': '29738'}