<jupyter_start><jupyter_text><jupyter_code>!git clone https://github.com/nlpyang/BertSum.git
cd /content/BertSum/
!git clone https://github.com/thangarani/raw_stories.git
!wget http://nlp.stanford.edu/software/stanford-corenlp-latest.zip
!sudo apt install unzip
!unzip stanford-corenlp-latest.zip
!jar -tvf /content/BertSum/stanford-corenlp-4.0.0/stanford-corenlp-4.0.0.jar
%env CLASSPATH=/content/BertSum/stanford-corenlp-4.0.0/stanford-corenlp-4.0.0.jar
!pip3 install pytorch_pretrained_bert
!pip install tensorboardX
!pip install multiprocess
!git clone https://github.com/andersjo/pyrouge.git
cd pyrouge/tools/ROUGE-1.5.5
pip install perl
!export ROUGE_EVAL_HOME="/content/BertSum/pyrouge/tools/ROUGE-1.5.5/data/"
cd data/WordNet-2.0-Exceptions/
!apt-get install synaptic
!./buildExeptionDB.pl . exc WordNet-2.0.exc.db
cd /content/BertSum/pyrouge/tools/ROUGE-1.5.5/data
rm WordNet-2.0.exc.db
!ln -s WordNet-2.0-Exceptions/WordNet-2.0.exc.db WordNet-2.0.exc.db
!apt-get install libxml-dom-perl
!git clone https://github.com/bheinzerling/pyrouge.git
cd pyrouge
!python setup.py install
!pyrouge_set_rouge_path /content/BertSum/pyrouge/tools/ROUGE-1.5.5/
!python -m pyrouge.test
cd /content/BertSum/<jupyter_output><empty_output><jupyter_text>File "/content/BertSum/src/models/data_loader.py", line 31, in __init__    mask = 1 - (src == 0) -> change -> mask = ~ ( src == 0 )<jupyter_code>mkdir merged_stories_tokenized
cd /content/BertSum/src/
%%capture
!python preprocess.py -mode tokenize -raw_path ../raw_stories -save_path ../merged_stories_tokenized -log_file ../logs/cnndm.log 
cd /content/BertSum/merged_stories_tokenized/
!ls | wc -l
cd /content/BertSum/src/
!python preprocess.py -mode format_to_lines -raw_path ../merged_stories_tokenized -save_path ../json_data/cnndm -map_path ../urls -lower -log_file ../logs/cnndm.log
!cd /content/BertSum/
!ls
cd /content/BertSum/src/
!python preprocess.py -mode format_to_bert -raw_path ../json_data -save_path ../bert_data -oracle_mode greedy -n_cpus 4 -log_file ../logs/preprocess.log
!python train.py -mode train -encoder transformer -dropout 0.1 -bert_data_path ../bert_data/cnndm -model_path ../models/bert_transformer -lr 2e-3 -visible_gpus 0 -gpu_ranks 0 -world_size 1 -report_every 5000 -save_checkpoint_steps 25000 -batch_size 3000 -decay_method noam -train_steps 50000 -accum_count 2 -log_file ../logs/bert_transformer -use_interval true -warmup_steps 10000 -ff_size 2048 -inter_layers 2 -heads 8
cp /content/drive/My\ Drive/Colab\ Notebooks/bert_data/* /content/BertSum/bert_data/
cp /content/BertSum/models/bert_transformer/*.pt /content/drive/My\ Drive/Colab\ Notebooks/<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13556139692076605, 'fraction_numerical': 0.030041306796845663, 'mean_word_length': 4.381818181818182, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '383183', 'license': 'Apache-2.0', 'n_tokens_mistral': 1116, 'n_tokens_neox': 1030, 'n_words': 195, 'path': 'Copy_of_TextSummarization_bertsum.ipynb', 'repo_name': 'thangarani/bertsum', 'size': '2663'}