<jupyter_start><jupyter_text>Download & process list of DOIs in Sci-Hubhttps://twitter.com/Sci_Hub/status/843546352219017218Converts DOIs to lowercase as per https://git.io/v9GoT.<jupyter_code># Download DOI list from SCi-Hub
! wget --timestamping --no-if-modified-since \
    http://sci-hub.cc/downloads/doi.7z
# Extract doi.7z
! rm --force doi.log
! 7z x -bd doi.7z
import csv
import lzma
import os
dois = set()
with open('doi.log', encoding='cp1252') as read_file:
    for line in read_file:
        line = line.strip()
        doi = line.lower()
        dois.add(doi)

dois = sorted(dois)
f'{len(dois):,} distinct DOIs'
with lzma.open('scihub-dois.tsv.xz', 'wt') as write_file:
    for row in ['doi'] + dois:
        write_file.write(row + '\n')
if os.path.isfile('doi.log'):
    os.remove('doi.log')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15347721822541965, 'fraction_numerical': 0.03237410071942446, 'mean_word_length': 3.5628415300546448, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '297317', 'license': 'CC0-1.0', 'n_tokens_mistral': 354, 'n_tokens_neox': 325, 'n_words': 73, 'path': 'download/scihub-dois/1.retrieve.ipynb', 'repo_name': 'arielsvn/scihub', 'size': '834'}