<jupyter_start><jupyter_text>Gutenberg Children Books `2018-12-11`**Tests: "LG English", agglomerative clustering, min_word_count = 11, 3/5/7/10 Kclusters**. Link Grammar 5.5.1, `test_grammar` updated 2018-10-19.  This notebook **not** is shared as static [Gutenberg-Children-Books-2018-12-11.html](http://langlearn.singularitynet.io/data/clustering_2018/html/Gutenberg-Children-Books-2018-12-11.html).  Output data shared via [Gutenberg-Children-Books-2018-12-11](http://langlearn.singularitynet.io/data/clustering_2018/Gutenberg-Children-Books-2018-12-11) directory.  Basic settings<jupyter_code>import os, sys, time
module_path = os.path.abspath(os.path.join('..'))
if module_path not in sys.path: sys.path.append(module_path)
from src.grammar_learner.utl import UTC
from src.grammar_learner.read_files import check_dir, check_corpus
from src.grammar_learner.write_files import list2file
from src.grammar_learner.widgets import html_table
from src.grammar_learner.pqa_table import table_rows, params, wide_rows
tmpath = module_path + '/tmp/'
check_dir(tmpath, True, 'none')
start = time.time()
runs = (1,1)<jupyter_output><empty_output><jupyter_text>Corpus test settings<jupyter_code>corpus = 'GCB' # 2018-12-11 copy of 'Gutenberg-Children-Books-Caps'
dataset = 'LG-English'
kwargs = {
    'left_wall'     :   ''          ,
    'period'        :   False       ,
    'context'       :   1           ,
    'min_word_count':   11          ,
    'word_space'    :   'sparse'    ,
    'clustering'    :   ['agglomerative', 'ward'],
    'clustering_metric' : ['silhouette', 'cosine'],
    'cluster_range' :   3000        ,   # 3000, 5000, 7000, 10000
    'top_level'     :   0.01        ,
    'grammar_rules' :   2           ,
    'max_disjuncts' :   1000000     ,   # off
    'tmpath'        :   tmpath      , 
    'verbose'       :   'min'       ,
    'template_path' :   'poc-turtle',
    'linkage_limit' :   1000        }
rp = module_path + '/data/' + corpus + '/LG-E-clean/GCB-LG-English-clean.ull'
cp = rp  # corpus path = reference_path
runs = (1,1)
out_dir = module_path + '/output/' + corpus + '-' + str(UTC())[:10]
if check_corpus(rp, 'min'): print(UTC(), out_dir)<jupyter_output>2018-12-11 07:44:07 UTC /home/obaskov/94/language-learning/output/GCB-2018-12-11<jupyter_text>Tests: "LG English", min_word_count = 11, 3/5/7/10 Kclusters<jupyter_code>%%capture
table = []
kwargs['cluster_range'] = 3000
line = [[2.1, corpus, dataset, 0, 0, 'none']]
a, _, header, log, rules = wide_rows(line, out_dir, cp, rp, runs, **kwargs)
table.extend(a)
display(html_table([header] + table)); print(UTC())
%%capture
kwargs['cluster_range'] = 5000
line = [[2.2, corpus, dataset, 0, 0, 'none']]
a, _, header, log, rules = wide_rows(line, out_dir, cp, rp, runs, **kwargs)
table.extend(a)
display(html_table([header] + table)); print(UTC())
%%capture
kwargs['cluster_range'] = 7000
line = [[2.3, corpus, dataset, 0, 0, 'none']]
a, _, header, log, rules = wide_rows(line, out_dir, cp, rp, runs, **kwargs)
table.extend(a)
display(html_table([header] + table)); print(UTC())
%%capture
kwargs['cluster_range'] = 10000
line = [[2.3, corpus, dataset, 0, 0, 'none']]
a, _, header, log, rules = wide_rows(line, out_dir, cp, rp, runs, **kwargs)
table.extend(a)
display(html_table([header] + a)); print(UTC())<jupyter_output><empty_output><jupyter_text>All tests<jupyter_code>display(html_table([header] + table)); print(UTC())
print(UTC(), ':: finished, elapsed', str(round((time.time()-start)/3600.0, 1)), 'hours')
table_str = list2file(table, out_dir + '/all_tests_table.txt')
print('Results saved to', out_dir + '/all_tests_table.txt')<jupyter_output>2018-12-12 10:28:02 UTC :: finished, elapsed 26.7 hours
Results saved to /home/obaskov/94/language-learning/output/GCB-2018-12-11/all_tests_table.txt
{'fraction_non_alphanumeric': 0.16530018513620734, 'fraction_numerical': 0.05765670457550912, 'mean_word_length': 4.245492371705964, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '379451', 'license': 'MIT', 'n_tokens_mistral': 1562, 'n_tokens_neox': 1388, 'n_words': 279, 'path': 'notebooks/Gutenberg-Children-Books-LG-English-2018-12-11.ipynb', 'repo_name': 'akolonin/language-learning', 'size': '3781'}