<jupyter_start><jupyter_text><jupyter_code>!git clone https://github.com/huggingface/transformers
!pip install transformers
!pip install -r transformers/examples/requirements.txt
!python transformers/examples/run_generation.py --model_type=gpt2 --model_name_or_path=gpt2 --no_cuda
!python transformers/examples/run_language_modeling.py --output_dir=output --model_type=gpt2 --model_name_or_path=gpt2 --do_train --train_data_file=socrates.txt --do_eval --eval_data_file=socrates.txt --no_cuda --num_train_epochs 4
!python transformers/examples/run_generation.py --model_type=gpt2 --model_name_or_path=output --no_cuda<jupyter_output>02/15/2020 19:21:30 - INFO - transformers.tokenization_utils -   Model name 'output' not found in model shortcut name list (gpt2, gpt2-medium, gpt2-large, gpt2-xl, distilgpt2). Assuming 'output' is a path, a model identifier, or url to a directory containing tokenizer files.
02/15/2020 19:21:30 - INFO - transformers.tokenization_utils -   Didn't find file output/added_tokens.json. We won't load it.
02/15/2020 19:21:30 - INFO - transformers.tokenization_utils -   loading file output/vocab.json
02/15/2020 19:21:30 - INFO - transformers.tokenization_utils -   loading file output/merges.txt
02/15/2020 19:21:30 - INFO - transformers.tokenization_utils -   loading file None
02/15/2020 19:21:30 - INFO - transformers.tokenization_utils -   loading file output/special_tokens_map.json
02/15/2020 19:21:30 - INFO - transformers.tokenization_utils -   loading file output/tokenizer_config.json
02/15/2020 19:21:30 - INFO - transformers.configuration_utils -   loading configuration file output/config.[...]
{'fraction_non_alphanumeric': 0.10751374465485644, 'fraction_numerical': 0.07513744654856444, 'mean_word_length': 4.829181494661921, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '641797', 'license': 'MIT', 'n_tokens_mistral': 627, 'n_tokens_neox': 539, 'n_words': 121, 'path': 'transformers_test.ipynb', 'repo_name': 'xSakix/AI_colan_notebooks', 'size': '1637'}