<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Settings2&nbsp;&nbsp;Tests: "POC-English" corpus2.1&nbsp;&nbsp;"POC-English", ILE, no generalization2.2&nbsp;&nbsp;"POC-English", ILE + generalization3&nbsp;&nbsp;Tests: "Child Directed Speech" corpus ("CDS")3.1&nbsp;&nbsp;CDS "LG-E-clean"3.2&nbsp;&nbsp;CDS LG-E-5514&nbsp;&nbsp;"Gutenberg Children Books"  corpus  Iterative ILE Clustering `2018-02-28` checkThis notebook is shared as [Iterative-clustering-ILE-POCE-CDS-2019-02-27.html](http://langlearn.singularitynet.io/data/clustering_2019/html/Iterative-clustering-ILE-POCE-CDS-2019-02-27.html),  output data -- [Iterative-clustering-ILE-POCE-CDS-2019-02-27](http://langlearn.singularitynet.io/data/clustering_2019/Iterative-clustering-ILE-POCE-CDS-2019-02-27)**Looks like a tagging or parsing processing issue, OR input parses issues leading to wrong reading and tagging...***  Settings<jupyter_code>import os, sys, time
module_path = os.path.abspath(os.path.join('..'))
if module_path not in sys.path: sys.path.append(module_path)
from src.grammar_learner.utl import UTC, kwa
from src.grammar_learner.read_files import check_dir
from src.grammar_learner.widgets import html_table
from src.grammar_learner.pqa_table import params
from src.grammar_learner.incremental_clustering import iterate
tmpath = module_path + '/tmp/'
check_dir(tmpath, True, 'none')
table = []
start = time.time()
out_dir = module_path + \
    '/output/Iterative-Clustering-ILE-' + str(UTC())[:10]
print(UTC(), ':: out_dir:\n', out_dir)
corpus = 'GCB'
dataset = 'LG-E-clean'
input_path = module_path +'/data/'+ corpus +'/'+ dataset
ref_corpus = input_path
kwargs = {
    # Corpora: 
    'corpus'        : 'GCB'         ,
    'dataset'       : 'LG-E-clean'  ,
    'max_sentence_length':  10      ,
    # 'input_parses': input_path    , # paths are set by 'corpus' and 'dataset'
    'reference_path': ref_corpus    ,
    # 'corpus_path' : ref_corpus    , # corpus path = reference path 
    'module_path'   : module_path   , # language-learning dir (default)
    # Word space:
    'stop_words'    :   []          , # trash filter off
    'min_word_count':   1           ,
    'left_wall'     :   ''          ,
    'period'        :   False       ,
    'context'       :   2           , # disjunct-based word vector space
    'word_space'    :   'discrete'  , # "ILE"
    # Category learning:
    'clustering'    :   'group'     , # "ILE"
    'cluster_range' :   0           ,
    'cluster_criteria'  : 'silhouette',
    'clustering_metric' : ('silhouette', 'cosine'),
    'categories_generalization' : 'off',
    # Grammar rules induction:
    'grammar_rules'         : 2     , # disjunct-based link grammar rules
    'rules_generalization'  : 'off' ,
    'rules_merge'           : 0.8   ,
    'rules_aggregation'     : 0.2   ,
    'top_level'             : 0.01  ,
    # Etc...:
    'out_path'      :   out_dir     ,
    'output_grammar':   out_dir     ,
    'tmpath'        :   tmpath      , 
    'verbose'       :   'min'       ,
    'template_path' :   'poc-turtle',
    'linkage_limit' :   1000        ,
    'iterations'    :   12
}
if check_dir(input_path): print(UTC(), ':: input_path:\n', input_path)<jupyter_output>2019-02-27 17:18:19 UTC :: input_path:
 /home/obaskov/94/language-learning/data/GCB/LG-E-clean<jupyter_text>Tests: "POC-English" corpus  "POC-English", ILE, no generalization<jupyter_code>%%capture
kwargs['corpus'] = 'POC-English-Amb'
kwargs['dataset'] = 'MST-fixed-manually'
kwargs['reference_path'] = module_path + \
    '/data/POC-English-Amb/MST-fixed-manually/poc-english_ex-parses-gold.txt'
if 'corpus_path' in kwargs: del kwargs['corpus_path']
kwargs['rules_generalization'] = 'off'
kwargs['cluster_range']        =  0  # just numbering directories
if 'min_word_count' in kwargs: del kwargs['min_word_count']
t21, re21 = iterate(**kwargs)
display(html_table(t21))
print(re21['project_directory'][42:-12])<jupyter_output><empty_output><jupyter_text>"POC-English", ILE + generalization<jupyter_code>%%capture
kwargs['rules_generalization'] = 'fast'
kwargs['rules_aggregation'] = 0.2
kwargs['cluster_range']     =   2  # just numbering directories
t22, re22 = iterate(**kwargs)
display(html_table(t22))
print(re22['project_directory'][42:-12])
%%capture
kwargs['rules_aggregation'] = 0.1
kwargs['cluster_range']     =   1  # just numbering directories
t23, re23 = iterate(**kwargs)
display(html_table(t23)); print(re23['project_directory'][42:-12])<jupyter_output><empty_output><jupyter_text>Tests: "Child Directed Speech" corpus ("CDS")<jupyter_code>kwargs['corpus'] = 'CDS'
kwargs['reference_path'] = module_path + '/data/CDS/LG-E-clean'
if 'corpus_path' in kwargs: del kwargs['corpus_path']<jupyter_output><empty_output><jupyter_text>CDS "LG-E-clean"<jupyter_code>%%capture
kwargs['dataset'] = 'LG-E-clean'
kwargs['rules_generalization'] = 'off'
kwargs['cluster_range']        =  0  # just numbering directories
t31, re31 = iterate(**kwargs)
display(html_table(t31)); print(re31['project_directory'][42:-12])
%%capture
kwargs['rules_generalization'] = 'fast'
kwargs['rules_aggregation'] = 0.2
kwargs['cluster_range']     =   2  # just numbering directories
t32, re32 = iterate(**kwargs)
display(html_table(t32)); print(re32['project_directory'][42:-12])
%%capture
kwargs['rules_aggregation'] = 0.1
kwargs['cluster_range']     =   1  # just numbering directories
t33, re33 = iterate(**kwargs)
display(html_table(t33)); print(re33['project_directory'][42:-12])<jupyter_output><empty_output><jupyter_text>CDS LG-E-551<jupyter_code>%%capture
kwargs['dataset'] = 'LG-E-551'
kwargs['rules_generalization'] = 'off'
kwargs['cluster_range']        =  0  # just numbering directories
t35, re35 = iterate(**kwargs)
display(html_table(t35)); print(re35['project_directory'][42:-12])
%%capture
kwargs['rules_generalization'] = 'fast'
kwargs['rules_aggregation'] = 0.2  # 0.2: 2148⇒2091, 0.98⇒0.91
kwargs['cluster_range']     =   2  # just numbering directories
t36, re36 = iterate(**kwargs)
display(html_table(t36)); print(re36['project_directory'][42:-12])
%%capture
kwargs['rules_generalization'] = 'fast'
kwargs['rules_aggregation'] = 0.1
kwargs['cluster_range']     =   1  # just numbering directories
t37, re37 = iterate(**kwargs)
display(html_table(t37)); print(re37['project_directory'][42:-12])<jupyter_output><empty_output><jupyter_text>"Gutenberg Children Books"  corpus<jupyter_code>%%capture
kwargs['corpus'] = 'GCB'
kwargs['dataset'] = 'LG-E-clean'
if 'reference_path' in kwargs: del kwargs['reference_path']
if 'corpus_path' in kwargs: del kwargs['corpus_path']
kwargs['word_space'] = 'discrete'
kwargs['clustering'] = 'group'
kwargs['rules_generalization'] = 'off'
kwargs['cluster_range'] =  0  # just numbering directories
kwargs['min_word_count'] = 21
t40, r40 = iterate(**kwargs)
display(html_table(t40)); print(r40['project_directory'][42:-12])<jupyter_output><empty_output><jupyter_text>_Looks like a tagging or corpus processing issue... OR some issue in input parses leading to wrong reading and tagging?_<jupyter_code>%%capture
kwargs['corpus'] = 'GCB'
kwargs['dataset'] = 'LG-E-clean'  # 'LG-E-551-pq24878'
if 'reference_path' in kwargs: del kwargs['reference_path']
if 'corpus_path' in kwargs: del kwargs['corpus_path']
kwargs['word_space'] = 'discrete'
kwargs['clustering'] = 'group'
kwargs['rules_generalization'] = 'fast'
kwargs['cluster_range']     =   3  # just numbering directories
kwargs['min_word_count'] = 11
kwargs['rules_aggregation'] = 0.2
t43, r43 = iterate(**kwargs)
display(html_table(t43)); print(r43['project_directory'][42:-12])
%%capture
kwargs['corpus'] = 'GCB'
kwargs['dataset'] = 'LG-E-clean'  # 'LG-E-551-pq24878'
if 'reference_path' in kwargs: del kwargs['reference_path']
if 'corpus_path' in kwargs: del kwargs['corpus_path']
kwargs['word_space'] = 'discrete'
kwargs['clustering'] = 'group'
kwargs['rules_generalization'] = 'fast'
kwargs['cluster_range']     =   4  # just numbering directories
kwargs['min_word_count'] = 11
kwargs['rules_aggregation'] = 0.3
t44, r44 = iterate(**kwargs)
display(html_table(t44)); print(r44['project_directory'][42:-12])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15853508664126828, 'fraction_numerical': 0.038834951456310676, 'mean_word_length': 4.901377810007252, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '353258', 'license': 'MIT', 'n_tokens_mistral': 3092, 'n_tokens_neox': 2864, 'n_words': 518, 'path': 'notebooks/Iterative-clustering-ILE-POCE-CDS-2019-02-27.ipynb', 'repo_name': 'vsbogd/language-learning', 'size': '8137'}