<jupyter_start><jupyter_text>Fetching data using `PandasFetcher`Translating using pickle files.<jupyter_code>import sys

import rics

# Print relevant versions
print(f"{rics.__version__=}")
print(f"{sys.version=}")
!git log --pretty=oneline --abbrev-commit -1
from rics.utility.misc import get_local_or_remote
from rics.utility.logs import basic_config, logging

basic_config(level=logging.INFO, rics_level=logging.DEBUG)<jupyter_output><empty_output><jupyter_text>Make local Pickle filesWe'lll download data from https://datasets.imdbws.com and clean it to make sure all values are given (which means that for actors are dead and titles have stopped airing).<jupyter_code>sources = ["name.basics", "title.basics"]
from data import load_imdb

for dataset in sources:
    load_imdb(dataset)<jupyter_output>2022-03-17T21:29:58.348 [rics.utility.misc.get_local_or_remote:DEBUG] Local file path: '/home/dev/git/private-rics/jupyterlab/data-cache/name.basics.tsv.gz'.
2022-03-17T21:29:58.349 [rics.utility.misc.get_local_or_remote:DEBUG] Remote file path: 'https://datasets.imdbws.com/name.basics.tsv.gz'.
2022-03-17T21:29:58.447 [rics.utility.misc.get_local_or_remote:INFO] Fetching data from 'https://datasets.imdbws.com/name.basics.tsv.gz'..<jupyter_text>Create translator from config<jupyter_code>from rics.translation import Translator

translator = Translator.from_config("config.yaml")
print(f"Fetcher: {translator._fetcher}")
tmap = translator.store()
for source in tmap:
    translations = tmap[source]
    print(f"Translations for {source=};")
    for i, (idx, translation) in enumerate(tmap[source].items()):
        print(f"    {repr(idx)} -> {repr(translation)}")
        if i == 2:
            break<jupyter_output>Translations for source='name.basics';
    'nm0000001' -> 'nm0000001:<NAME> *1899†1987'
    'nm0000002' -> 'nm0000002:<NAME> *1924†2014'
    'nm0000004' -> 'nm0000004:<NAME> *1949†1982'
Translations for source='title.basics';
    'tt0025509' -> 'tt0025509:Les Misérables (original: Les misérables) *1934†1934'
    'tt0035803' -> 'tt0035803:The German Weekly Review (original: Die Deutsche Wochenschau) *1940†1945'
    'tt0038276' -> 'tt0038276:You Are an Artist (original: You Are an Artist) *1946†1955'
{'fraction_non_alphanumeric': 0.13426133812303548, 'fraction_numerical': 0.08307139649753031, 'mean_word_length': 5.552941176470588, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 21, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '352790', 'license': 'MIT', 'n_tokens_mistral': 929, 'n_tokens_neox': 796, 'n_words': 164, 'path': 'jupyterlab/demo/pickle-translation/PickleFetcher.ipynb', 'repo_name': 'rsundqvist/rics', 'size': '2246'}