<jupyter_start><jupyter_text>"<NAME>"> "To re-create the dataset"- toc: false- branch: master- hidden: true- categories: [irish, kaggle, dataset]<jupyter_code>!curl https://www.rte.ie/radio/rnag/articles/eolas/2021/0712/1234521-podchraoltai/|grep 'il/Download</a>'|awk -F'href="' '{print $2}'|awk -F'"' '{print $1}'|sed -e 's/http:/https:/' > urls
!cat urls |while read i;do wget $(curl $i|grep '<enclosure'|awk -F'url="' '{print $2}'|awk -F'"' '{print $1}');done<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.23529411764705882, 'fraction_numerical': 0.038539553752535496, 'mean_word_length': 6.057142857142857, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '441416', 'license': 'Apache-2.0', 'n_tokens_mistral': 222, 'n_tokens_neox': 204, 'n_words': 41, 'path': '_notebooks/2021-08-01-rnag-podchraoltai.ipynb', 'repo_name': 'jimregan/notes', 'size': '504'}