<jupyter_start><jupyter_text>googlebooks-eng-all-5gram-20120701-np - 4mbgooglebooks-eng-all-5gram-20120701-xq 1mbdata_iterator = dd.read_csv(onegb,                            sep='\s+',                             names=['1','2','3','4','5','6','7','8','9','10','11','12','13','14'],                                dtype={                                    '1':'string',                                    '2':'string',                                    '3':'string',                                    '4':'string',                                    '5':'string',                                    '6':'string',                                    '7':'string',                                    '8':'string',                                    '9':'string',                                    '10':'string',                                    '11':'string',                                    '12':'string',                                    '13':'string',                                    '14':'string'},                            header=None, blocksize=64000000,error_bad_lines=False,                            encoding='utf-8',engine='python')<jupyter_code>#googlebooks-eng-all-5gram-20120701-np - 4mb
#googlebooks-eng-all-5gram-20120701-xq 1mb
dd = dd.read_csv(onegb,
                            sep='\s+',
                            header=None, blocksize=64000000,error_bad_lines=False,
                            encoding='utf-8',engine='python')
dd1995_2005 = dd[(dd[5]>1994)&(dd[5]<2006)]
type(dd1995_2005)
dd1995_2005.to_parquet('dd1995_2005')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14849624060150377, 'fraction_numerical': 0.08646616541353383, 'mean_word_length': 1.0343949044585987, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '410323', 'license': 'Apache-2.0', 'n_tokens_mistral': 508, 'n_tokens_neox': 399, 'n_words': 37, 'path': 'small_ds/.ipynb_checkpoints/usingDask-checkpoint.ipynb', 'repo_name': 'numberonewastefellow/projects', 'size': '1596'}