<commit_before>#!/usr/bin/python2

"""
    Scrape article XMLs and package them for ingestion.

    Usage: repackage.py rhino_url prefix article_name

    Note:
        rhino_url - is the usl to the rhino SOA for the system you are interested in.
        prefix    - the doi prefix (for PLOS 10.1371)
        article_name - the first part of the article zip name.

   Ex. repackage.py --server="https://webprod.plosjournals.org/api"  "pone.0033205"
"""

from __future__ import print_function
from __future__ import with_statement
from cStringIO import StringIO
from optparse import OptionParser

import itertools
import json
import os
import re
import requests
import string
import sys
import zipfile


# ***********************************************
#      Text templates used in
#      repetative string creation.
# ***********************************************

# ***** Manifest DTD template *****
DTD_TEXT = """
<!ELEMENT manifest (articleBundle) >

<!ELEMENT articleBundle (article, object*) >

<!-- the article. 'main-entry' specifies the zip entry that contains the nlm
   - article xml; this must match one of the contained representations
   -->
<!ELEMENT article (representation+) >
<!ATTLIST article
    uri         CDATA          #REQUIRED
    main-entry  CDATA          #REQUIRED >

<!-- all included secondary objects (images, movies, data, etc) -->
<!ELEMENT object (representation+) >
<!ATTLIST object
    uri         CDATA          #REQUIRED
    strkImage   CDATA          #IMPLIED          >

<!-- a specific representation.
   - 'name' is the name (label) to store this representation under;
   - 'entry' specifies the entry in the zip that contains this representation
   -->
<!ELEMENT representation EMPTY >
<!ATTLIST representation
    name        CDATA          #REQUIRED
    entry       CDATA          #REQUIRED >
""".lstrip()

# ***** Skeletal Manifest Template *****
MANIFEST_TMPL = """<?xml version="1.1"  encoding="UTF-8"?>
<!DOCTYPE manifest SYSTEM "manifest.dtd">
<manifest>
  <articleBundle>
    {article}
    {objects}
  </articleBundle>
</manifest>
""".lstrip()

# ***** Manifest Object Entries *****
OBJECT_TMPL = """    <object uri="{uri}" {strkimage}>
        {reps}
     </object>"""

# ***** Object Representation Template *****
REPRESENTATION_TMPL = """<representation name="{name}" entry="{entry}" />"""

# ***** Manifest Article Template *****
ARTICLE_TMPL = """
     <article uri="{uri}" main-entry="{fname}">
        {reps}
     </article>"""

# ***** URI Template *****
URI_TMPL = """info:doi/{name}"""

# ***** DOI Template *****
DOI_TMPL = """{prefix}/journal.{name}"""

FETCH_ARTICLE_INFO_TMPL = '{server}/articles/{doi}'
FETCH_FILE_RHINO_TMPL = '{server}/assetfiles/{afid}'
FETCH_FILE_AMBRA_TMPL = '{server}/article/fetchObject.action?uri=info:doi/{doi}&representation={ext}'

DOI_PATTERN = re.compile(r'(\d+\.\d+/)?(journal\.)?(.*)\.([^.]*)')

# *****************************************************
def fetchManifestInfo(name, options):
    """
    Fetch the information necessary to build a manifest from Rhino.
    """
    (base_url, prefix) = (options.rhinoServer, options.prefix)
    doi = DOI_TMPL.format(prefix=prefix, name=name)
    uri = URI_TMPL.format(prefix=prefix, name=name)
    url = FETCH_ARTICLE_INFO_TMPL.format(server=base_url, doi=doi)
    response = requests.get(url, verify=False)

    # Load JSON into a Python object and use some values from it.
    article = json.loads(response.content)
    assets = dict((asset_name, asset_files.keys())<commit_msg>Fix article URI in MANIFEST<commit_after>#!/usr/bin/python2

"""
    Scrape article XMLs and package them for ingestion.

    Usage: repackage.py rhino_url prefix article_name

    Note:
        rhino_url - is the usl to the rhino SOA for the system you are interested in.
        prefix    - the doi prefix (for PLOS 10.1371)
        article_name - the first part of the article zip name.

   Ex. repackage.py --server="https://webprod.plosjournals.org/api"  "pone.0033205"
"""

from __future__ import print_function
from __future__ import with_statement
from cStringIO import StringIO
from optparse import OptionParser

import itertools
import json
import os
import re
import requests
import string
import sys
import zipfile


# ***********************************************
#      Text templates used in
#      repetative string creation.
# ***********************************************

# ***** Manifest DTD template *****
DTD_TEXT = """
<!ELEMENT manifest (articleBundle) >

<!ELEMENT articleBundle (article, object*) >

<!-- the article. 'main-entry' specifies the zip entry that contains the nlm
   - article xml; this must match one of the contained representations
   -->
<!ELEMENT article (representation+) >
<!ATTLIST article
    uri         CDATA          #REQUIRED
    main-entry  CDATA          #REQUIRED >

<!-- all included secondary objects (images, movies, data, etc) -->
<!ELEMENT object (representation+) >
<!ATTLIST object
    uri         CDATA          #REQUIRED
    strkImage   CDATA          #IMPLIED          >

<!-- a specific representation.
   - 'name' is the name (label) to store this representation under;
   - 'entry' specifies the entry in the zip that contains this representation
   -->
<!ELEMENT representation EMPTY >
<!ATTLIST representation
    name        CDATA          #REQUIRED
    entry       CDATA          #REQUIRED >
""".lstrip()

# ***** Skeletal Manifest Template *****
MANIFEST_TMPL = """<?xml version="1.1"  encoding="UTF-8"?>
<!DOCTYPE manifest SYSTEM "manifest.dtd">
<manifest>
  <articleBundle>
    {article}
    {objects}
  </articleBundle>
</manifest>
""".lstrip()

# ***** Manifest Object Entries *****
OBJECT_TMPL = """    <object uri="{uri}" {strkimage}>
        {reps}
     </object>"""

# ***** Object Representation Template *****
REPRESENTATION_TMPL = """<representation name="{name}" entry="{entry}" />"""

# ***** Manifest Article Template *****
ARTICLE_TMPL = """
     <article uri="{uri}" main-entry="{fname}">
        {reps}
     </article>"""

# ***** URI Template *****
URI_TMPL = """info:doi/{prefix}/journal.{name}"""

# ***** DOI Template *****
DOI_TMPL = """{prefix}/journal.{name}"""

FETCH_ARTICLE_INFO_TMPL = '{server}/articles/{doi}'
FETCH_FILE_RHINO_TMPL = '{server}/assetfiles/{afid}'
FETCH_FILE_AMBRA_TMPL = '{server}/article/fetchObject.action?uri=info:doi/{doi}&representation={ext}'

DOI_PATTERN = re.compile(r'(\d+\.\d+/)?(journal\.)?(.*)\.([^.]*)')

# *****************************************************
def fetchManifestInfo(name, options):
    """
    Fetch the information necessary to build a manifest from Rhino.
    """
    (base_url, prefix) = (options.rhinoServer, options.prefix)
    doi = DOI_TMPL.format(prefix=prefix, name=name)
    uri = URI_TMPL.format(prefix=prefix, name=name)
    url = FETCH_ARTICLE_INFO_TMPL.format(server=base_url, doi=doi)
    response = requests.get(url, verify=False)

    # Load JSON into a Python object and use some values from it.
    article = json.loads(response.content)
    assets = dict((asset_name, asset_files.keys())
{'fraction_non_alphanumeric': 0.17863694087766332, 'fraction_numerical': 0.0047975165796528855, 'mean_word_length': 3.6785478547854784, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 2, '>': 47, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2440863', 'n_tokens_mistral': 2275, 'n_tokens_neox': 2093, 'n_words': 587}