# Path: adsft/rules.py
# META_CONTENT = {
#     'xml': {
#         'fulltext': {
#             'xpath': ['//body',
#                       '//section[@type="body"]',
#                       '//journalarticle-body',
#                       '//bdy',
#                       '//app-group',
#                       '//section[not(@type="acknowledgments" or @type="dataAccess" or @type="dataAvailability" or @type="superSection")]'
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'acknowledgements': {
#             'xpath': ['//ack',
#                       '//section[@type="acknowledgments"]',
#                       '//subsection[@type="acknowledgement" '
#                       'or @type="acknowledgment"]'
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'dataset': {
#             'xpath': ['//named-content[@content-type="dataset"]'],
#             'type': 'list',
#             'info': 'xlink:href',
#             },
#         'facility': {
#             'xpath': ['//named-content[@content-type="facility"]'],
#             'type': 'list',
#             'info': 'xlink:href',
#         }
#     },
#     'teixml': {
#         'fulltext': {
#             'xpath': ['//body',
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'acknowledgements': {
#             'xpath': ['//div[@type="acknowledgement"]',
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#     },
#     'xmlelsevier': {
#         'fulltext': {
#             'xpath': ['//body',
#                       '//raw-text',
#                       '//appendices',
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'acknowledgements': {
#             'xpath': ['//acknowledgment',
#                       '//ack',
#                       '//section[@type="acknowledgments"]',
#                       '//subsection[@type="acknowledgement" '
#                       'or @type="acknowledgment"]',
#                       '//*[local-name()="acknowledgment"]'
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'dataset': {
#             'xpath': ['//named-content[@content-type="dataset"]'],
#             'type': 'list',
#             'info': 'xlink:href',
#         }
#     },
#     'html': {
#         'introduction': [
#             '//h2[contains(.,"ntroduction")]',
#             '//h3[contains(.,"ntroduction")]',
#             '//p[contains(.,"Abstract")]',
#             ],
#         'references': [
#             '//h2[contains(.,"References")]'
#         ],
#         'table': [
#             '//table'
#         ],
#         'table_links': [
#             '//a[contains(@href, "TABLE_NAME")]'
#         ],
#         'head': [
#             '//head'
#         ]
#     },
#     'txt': {'fulltext': ['']},
#     'ocr': {'fulltext': ['']},
#     'http': {'fulltext': ['']},
#     'pdf': {'fulltext': ['']},
#     'pdf-grobid': {'grobid_fulltext': ['']},
# }


import sys
import os
import json
import tempfile
import shutil
import gzip
            import traceback
from adsft.rules import META_CONTENT
from adsputils import setup_logging, load_config
    if payload_dictionary['file_format'] == 'pdf-grobid':
        full_text_output_file_path = os.path.join(bibcode_pair_tree_path, 'grobid_fulltext.xml')
    else:
        full_text_output_file_path = os.path.join(bibcode_pair_tree_path, 'fulltext.txt.gz')

    if 'UPDATE' in payload_dictionary and \
        payload_dictionary['UPDATE'] == 'FORCE_TO_SEND' and \
        os.path.exists(meta_output_file_path) and os.path.exists(full_text_output_file_path):
            # Data was already extracted and saved
            return

    if not os.path.exists(bibcode_pair_tree_path):
        try:
            os.makedirs(bibcode_pair_tree_path)
        except OSError:
            raise OSError

    # Write everything but the full text content to the meta.json
    meta_dict = {}

    for const in ('meta_path', 'ft_source', 'bibcode', 'provider', 'UPDATE', 'file_format', 'index_date', 'dataset', 'facility'):
        try:
            meta_dict[const] = payload_dictionary[const]
            logger.debug('Adding meta content: %s', const)
        except KeyError:
            #print('Missing meta content: {0}'.format(const))
            continue

    # Write the custom extractions of content to the meta.json
    logger.debug('Copying extra meta content')

# Path: adsft/rules.py
# META_CONTENT = {
#     'xml': {
#         'fulltext': {
#             'xpath': ['//body',
#                       '//section[@type="body"]',
#                       '//journalarticle-body',
#                       '//bdy',
#                       '//app-group',
#                       '//section[not(@type="acknowledgments" or @type="dataAccess" or @type="dataAvailability" or @type="superSection")]'
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'acknowledgements': {
#             'xpath': ['//ack',
#                       '//section[@type="acknowledgments"]',
#                       '//subsection[@type="acknowledgement" '
#                       'or @type="acknowledgment"]'
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'dataset': {
#             'xpath': ['//named-content[@content-type="dataset"]'],
#             'type': 'list',
#             'info': 'xlink:href',
#             },
#         'facility': {
#             'xpath': ['//named-content[@content-type="facility"]'],
#             'type': 'list',
#             'info': 'xlink:href',
#         }
#     },
#     'teixml': {
#         'fulltext': {
#             'xpath': ['//body',
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'acknowledgements': {
#             'xpath': ['//div[@type="acknowledgement"]',
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#     },
#     'xmlelsevier': {
#         'fulltext': {
#             'xpath': ['//body',
#                       '//raw-text',
#                       '//appendices',
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'acknowledgements': {
#             'xpath': ['//acknowledgment',
#                       '//ack',
#                       '//section[@type="acknowledgments"]',
#                       '//subsection[@type="acknowledgement" '
#                       'or @type="acknowledgment"]',
#                       '//*[local-name()="acknowledgment"]'
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'dataset': {
#             'xpath': ['//named-content[@content-type="dataset"]'],
#             'type': 'list',
#             'info': 'xlink:href',
#         }
#     },
#     'html': {
#         'introduction': [
#             '//h2[contains(.,"ntroduction")]',
#             '//h3[contains(.,"ntroduction")]',
#             '//p[contains(.,"Abstract")]',
#             ],
#         'references': [
#             '//h2[contains(.,"References")]'
#         ],
#         'table': [
#             '//table'
#         ],
#         'table_links': [
#             '//a[contains(@href, "TABLE_NAME")]'
#         ],
#         'head': [
#             '//head'
#         ]
#     },
#     'txt': {'fulltext': ['']},
#     'ocr': {'fulltext': ['']},
#     'http': {'fulltext': ['']},
#     'pdf': {'fulltext': ['']},
#     'pdf-grobid': {'grobid_fulltext': ['']},
# }


# Path: adsft/writer.py
import sys
import os
import json
import tempfile
import shutil
import gzip
            import traceback
from adsft.rules import META_CONTENT
from adsputils import setup_logging, load_config

    if payload_dictionary['file_format'] == 'pdf-grobid':
        full_text_output_file_path = os.path.join(bibcode_pair_tree_path, 'grobid_fulltext.xml')
    else:
        full_text_output_file_path = os.path.join(bibcode_pair_tree_path, 'fulltext.txt.gz')

    if 'UPDATE' in payload_dictionary and \
        payload_dictionary['UPDATE'] == 'FORCE_TO_SEND' and \
        os.path.exists(meta_output_file_path) and os.path.exists(full_text_output_file_path):
            # Data was already extracted and saved
            return

    if not os.path.exists(bibcode_pair_tree_path):
        try:
            os.makedirs(bibcode_pair_tree_path)
        except OSError:
            raise OSError

    # Write everything but the full text content to the meta.json
    meta_dict = {}

    for const in ('meta_path', 'ft_source', 'bibcode', 'provider', 'UPDATE', 'file_format', 'index_date', 'dataset', 'facility'):
        try:
            meta_dict[const] = payload_dictionary[const]
            logger.debug('Adding meta content: %s', const)
        except KeyError:
            #print('Missing meta content: {0}'.format(const))
            continue

    # Write the custom extractions of content to the meta.json
    logger.debug('Copying extra meta content')

    for meta_key_word in META_CONTENT[payload_dictionary['file_format']]: