<jupyter_start><jupyter_text>Gutenberg Children Books `2018-12-08`**Tests: "LG English", agglomerative clustering, 2000/1000/500/50/20 clusters**.  Link Grammar 5.4.4., `test_grammar` updated 2018-10-19.  This notebook is shared as static [Gutenberg-Children-Books-2018-12-08.html](http://langlearn.singularitynet.io/data/clustering_2018/html/Gutenberg-Children-Books-2018-12-08.html).  The "All tests" table is shared as 'all_tests_table.txt' in [Gutenberg-Children-Books-2018-12-08](http://langlearn.singularitynet.io/data/clustering_2018/Gutenberg-Children-Books-2018-12-08) directory.    Basic settings<jupyter_code>import os, sys, time
module_path = os.path.abspath(os.path.join('..'))
if module_path not in sys.path: sys.path.append(module_path)
from src.grammar_learner.utl import UTC
from src.grammar_learner.read_files import check_dir, check_corpus
from src.grammar_learner.write_files import list2file
from src.grammar_learner.widgets import html_table
from src.grammar_learner.pqa_table import table_rows, params, wide_rows
tmpath = module_path + '/tmp/'
check_dir(tmpath, True, 'none')
start = time.time()
runs = (1,1)<jupyter_output><empty_output><jupyter_text>Corpus test settings<jupyter_code>corpus = 'Gutenberg-Children-Books-Caps'
dataset = 'LG-English'
kwargs = {
    'left_wall'     :   ''          ,
    'period'        :   False       ,
    'context'       :   1           ,
    'min_word_count':   31          ,   # ~7K words
    'word_space'    :   'sparse'    ,
    'clustering'    :   ['agglomerative', 'ward'],
    'clustering_metric' : ['silhouette', 'cosine'],
    'cluster_range' :   1000        ,
    'top_level'     :   0.01        ,
    'grammar_rules' :   2           ,
    'max_disjuncts' :   1000000     ,   # off
    'tmpath'        :   tmpath      , 
    'verbose'       :   'min'       ,
    'template_path' :   'poc-turtle',
    'linkage_limit' :   1000        }   # no tests
lines = [
    [45, corpus , 'LG-English'                 ,0,0, 'none'  ], 
    [46, corpus , 'LG-ANY-all-parses-agm-opt'  ,0,0, 'none'  ]]
rp = module_path + '/data/' + corpus + '/LG-E-clean/GCB-LG-English-clean.ull'
cp = rp  # corpus path = reference_path :: use 'gold' parses as test corpus
runs = (1,1)
out_dir = module_path + '/output/' + corpus + '-' + str(UTC())[:10]<jupyter_output><empty_output><jupyter_text>Tests: "LG English", no limits, 10/50/500/1000/2000 clusters<jupyter_code>%%capture
table = []
kwargs['cluster_range'] = 2000
line = [[2.1, corpus, dataset, 0, 0, 'none']]
a, _, header, log, rules = wide_rows(line, out_dir, cp, rp, runs, **kwargs)
table.extend(a)
display(html_table([header] + a))
%%capture
kwargs['cluster_range'] = 1000
line = [[2.2, corpus, dataset, 0, 0, 'none']]
a, _, header, log, rules = wide_rows(line, out_dir, cp, rp, runs, **kwargs)
table.extend(a)
display(html_table([header] + a))
%%capture
kwargs['cluster_range'] = 500
line = [[2.3, corpus, dataset, 0, 0, 'none']]
a, _, header, log, rules = wide_rows(line, out_dir, cp, rp, runs, **kwargs)
table.extend(a)
display(html_table([header] + a))
%%capture
kwargs['cluster_range'] = 100
line = [[2.4, corpus, dataset, 0, 0, 'none']]
a, _, header, log, rules = wide_rows(line, out_dir, cp, rp, runs, **kwargs)
table.extend(a)
display(html_table([header] + a))
%%capture
kwargs['cluster_range'] = 50
line = [[2.5, corpus, dataset, 0, 0, 'none']]
a, _, header, log, rules = wide_rows(line, out_dir, cp, rp, runs, **kwargs)
table.extend(a)
display(html_table([header] + a))
%%capture
kwargs['cluster_range'] = 20
line = [[2.6, corpus, dataset, 0, 0, 'none']]
a, _, header, log, rules = wide_rows(line, out_dir, cp, rp, runs, **kwargs)
table.extend(a)
display(html_table([header] + a))
%%capture
kwargs['cluster_range'] = 10
line = [[2.7, corpus, dataset, 0, 0, 'none']]
a, _, header, log, rules = wide_rows(line, out_dir, cp, rp, runs, **kwargs)
table.extend(a)
display(html_table([header] + a))<jupyter_output><empty_output><jupyter_text>All tests<jupyter_code>display(html_table([header] + table))<jupyter_output><empty_output><jupyter_text>_Interrupted after 51+ hours run with 10 clusters setting_<jupyter_code>print(UTC(), ':: finished, elapsed', str(round((time.time()-start)/3600.0, 1)), 'hours')
table_str = list2file(table, out_dir + '/all_tests_table.txt')
print('Results saved to', out_dir + '/all_tests_table.txt')<jupyter_output>2018-12-11 07:49:15 UTC :: finished, elapsed 72.8 hours
Results saved to /home/obaskov/94/language-learning/output/Gutenberg-Children-Books-Caps-2018-12-08/all_tests_table.txt
{'fraction_non_alphanumeric': 0.16626133097501658, 'fraction_numerical': 0.047092637629891666, 'mean_word_length': 4.2788798133022174, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '125209', 'license': 'MIT', 'n_tokens_mistral': 1841, 'n_tokens_neox': 1664, 'n_words': 357, 'path': 'notebooks/Gutenberg-Children-Books-LG-English-2018-12-08.ipynb', 'repo_name': 'akolonin/language-learning', 'size': '4523'}