<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
This script is for downloading and maintaining a local copy of all of the XML files of PLOS articles.
By default it doesn't rely on access to the PLOS's internal network (but can use it if available).
Diagram of relevant systems here: https://confluence.plos.org/confluence/display/CXP/How+allofPLOS+works
Workflow:
Check whether list of DOI files is complete
    * query Solr API for list of new articles (limited by date)
    * create list of missing DOIs, by comparing against existing list of DOIs or file names
Update by downloading articles content-repo if local store is not complete
Check for and download amended articles that have been issued amendments
Check for and download versions of record (VOR) for uncorrected proofs
Zip folder down, appending when able
Create log file for actions that can be referenced

TODO: add start date for beginning of time for article pubdates (2003-08-11), calculation for most recent pub date
"""

import argparse
import datetime
import errno
import gzip
import logging
import os
import shutil
import time
import tarfile
import zipfile

import lxml.etree as et
import requests
from tqdm import tqdm

from . import corpusdir, newarticledir

from .plos_regex import validate_doi
from .transformations import (BASE_URL_API, EXT_URL_TMP, INT_URL_TMP, URL_TMP, filename_to_doi,
                              doi_to_path)
from .article_class import Article

help_str = "This program downloads a zip file with all PLOS articles and checks for updates"

# Making sure DS.Store not included as file
ignore_func = shutil.ignore_patterns('.DS_Store')

# Some example URLs that may be useful
EXAMPLE_VOR_URL = ('http://solr-101.soma.plos.org:8011/solr/collection1/select?'<commit_msg>import uncorrected proof list from __init__<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
This script is for downloading and maintaining a local copy of all of the XML files of PLOS articles.
By default it doesn't rely on access to the PLOS's internal network (but can use it if available).
Diagram of relevant systems here: https://confluence.plos.org/confluence/display/CXP/How+allofPLOS+works
Workflow:
Check whether list of DOI files is complete
    * query Solr API for list of new articles (limited by date)
    * create list of missing DOIs, by comparing against existing list of DOIs or file names
Update by downloading articles content-repo if local store is not complete
Check for and download amended articles that have been issued amendments
Check for and download versions of record (VOR) for uncorrected proofs
Zip folder down, appending when able
Create log file for actions that can be referenced

TODO: add start date for beginning of time for article pubdates (2003-08-11), calculation for most recent pub date
"""

import argparse
import datetime
import errno
import gzip
import logging
import os
import shutil
import time
import tarfile
import zipfile

import lxml.etree as et
import requests
from tqdm import tqdm

from . import corpusdir, newarticledir, uncorrected_proofs_text_list

from .plos_regex import validate_doi
from .transformations import (BASE_URL_API, EXT_URL_TMP, INT_URL_TMP, URL_TMP, filename_to_doi,
                              doi_to_path)
from .article_class import Article

help_str = "This program downloads a zip file with all PLOS articles and checks for updates"

# Making sure DS.Store not included as file
ignore_func = shutil.ignore_patterns('.DS_Store')

# Some example URLs that may be useful
EXAMPLE_VOR_URL = ('http://solr-101.soma.plos.org:8011/solr/collection1/select?'
{'fraction_non_alphanumeric': 0.05482093663911846, 'fraction_numerical': 0.009917355371900827, 'mean_word_length': 4.4601503759398495, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1843735', 'n_tokens_mistral': 1071, 'n_tokens_neox': 994, 'n_words': 469}