<jupyter_start><jupyter_text>High<jupyter_code>high = pd.read_table("tsv/CoNLL-SIGMORPHON Shared Task 1 Results - high.tsv")
high.tail()
high.iloc[:-1].mean().sort_values(ascending=False).astype(float).to_frame().reset_index()<jupyter_output>/home/judit/miniconda3/lib/python3.7/site-packages/pandas/core/dtypes/cast.py:730: ComplexWarning: Casting complex values to real discards the imaginary part
  return arr.astype(dtype, copy=True)<jupyter_text>Medium<jupyter_code>medium = pd.read_table("tsv/CoNLL-SIGMORPHON Shared Task 1 Results - medium.tsv")
medium.iloc[:-1].mean().sort_values(ascending=False).to_frame().reset_index()<jupyter_output><empty_output><jupyter_text>Low<jupyter_code>low = pd.read_table("tsv/CoNLL-SIGMORPHON Shared Task 1 Results - low.tsv")
low.iloc[:-1].mean().sort_values(ascending=False).to_frame().reset_index()
r = high.iloc[:-1].mean()  
r[r.index.str.startswith('bme')].sort_index()
r = medium.iloc[:-1].mean()  
r[r.index.str.startswith('bme')].sort_index()
r = low.iloc[:-1].mean()  
r[r.index.str.startswith('bme')].sort_index()<jupyter_output><empty_output><jupyter_text>Training data stats<jupyter_code>DATA_DIR = "/home/judit/repo/external/conll2018/task1/all"
SURPRISE_DIR = "/home/judit/repo/external/conll2018/task1/surprise"

def load_stats(fn):
    maxlen_lemma = 0
    maxlen_infl = 0
    maxlen_tags = 0
    types_lemma = set()
    types_infl = set()
    types_tags = set()
    alphabet = set()
    cnt = 0
    sumlen_lemma = 0
    sumlen_infl = 0
    sumlen_tags = 0
    upper_cnt = 0
    with open(fn) as f:
        for l in f:
            cnt += 1
            lemma, infl, tags = l.rstrip('\n').split('\t')
            tags = tags.split(';')
            maxlen_lemma = max(maxlen_lemma, len(lemma))
            maxlen_infl = max(maxlen_infl, len(infl))
            maxlen_tags = max(maxlen_tags, len(tags))
            sumlen_lemma += len(lemma)
            sumlen_infl += len(infl)
            sumlen_tags += len(tags)
            alphabet |= set(lemma)
            alphabet |= set(infl)
            types_lemma.add(lemma)
            types_infl.add(infl)
            types_tags |= set(tags)
            upper_cnt += sum(c.isupper() for c in lemma)
    return {
        'lemma maxlen': maxlen_lemma,
        'inflected maxlen': maxlen_infl,
        'tags maxlen': maxlen_tags,
        'alphabet size': len(alphabet),
        'lemma types': len(types_lemma),
        'inflected types': len(types_infl),
        'tag types': len(types_tags),
        'lemma avglen' : sumlen_lemma / cnt,
        'infl avglen' : sumlen_infl / cnt,
        'tags avglen' : sumlen_tags / cnt,
        'upper ratio': upper_cnt / sumlen_lemma,
    }
        
stats = []
for fn in os.listdir(DATA_DIR):
    if fn.endswith('train-high'):
        stat = load_stats(os.path.join(DATA_DIR, fn))
        stat['language'] = '-'.join(fn.split('-')[:-2])
        stats.append(stat)
for fn in os.listdir(SURPRISE_DIR):
    if fn.endswith('train-high'):
        stat = load_stats(os.path.join(SURPRISE_DIR, fn))
        stat['language'] = '-'.join(fn.split('-')[:-2])
        stats.append(stat)
    
stats = pd.DataFrame(stats).set_index('language').sort_index()<jupyter_output><empty_output><jupyter_text>Analysis<jupyter_code>df = high.rename(columns={'Unnamed: 0': 'language'}).set_index('language')
print(df['bme-02'].quantile(.8))
good = df[df['bme-02'] >= df['bme-02'].quantile(.8)]
good<jupyter_output>99.0<jupyter_text>Overall stats for all languages<jupyter_code>stats.describe()<jupyter_output><empty_output><jupyter_text>Same stats for top 20% languages<jupyter_code>good[['bme-02']].join(stats)<jupyter_output><empty_output><jupyter_text>Bottom 20%<jupyter_code>print(df['bme-02'].quantile(.2))
bad = df[df['bme-02'] <= df['bme-02'].quantile(.2)]
bad<jupyter_output>90.83999999999999<jupyter_text>Stat table again<jupyter_code>stats.describe()<jupyter_output><empty_output><jupyter_text>Same stats for bottom 20%<jupyter_code>bad[['bme-02']].join(stats)<jupyter_output><empty_output><jupyter_text>bme-02 wins on these languages<jupyter_code>df.loc[df.idxmax(axis=1) == 'bme-02'][['bme-02', 'uzh-01', 'uzh-02']]<jupyter_output><empty_output><jupyter_text>bme-02 is worse than best-sigma on these language<jupyter_code>df.loc[df.max(axis=1) - df.std(axis=1) > df['bme-02']]<jupyter_output><empty_output><jupyter_text>bme-02 is worse than mean acc on these language<jupyter_code>df.loc[df.mean(axis=1) > df['bme-02']]<jupyter_output><empty_output><jupyter_text>bme-02 is better than mean + sigmaThe best teams are listed.<jupyter_code>d = df.loc[df.mean(axis=1) + df.std(axis=1) < df['bme-02']]
d[d.idxmax(axis=1).unique()]<jupyter_output><empty_output><jupyter_text>bme-02 is better than mean + 2\*sigma<jupyter_code>df.loc[df.mean(axis=1) + 2*2*df.std(axis=1) < df['bme-02']]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16580418132891742, 'fraction_numerical': 0.02483957772717864, 'mean_word_length': 4.257889009793254, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '466381', 'license': 'MIT', 'n_tokens_mistral': 1937, 'n_tokens_neox': 1861, 'n_words': 259, 'path': 'notebooks/task1_result_analysis.ipynb', 'repo_name': 'juditacs/sigmorphon2018', 'size': '4831'}