# Path: adsft/rules.py
# META_CONTENT = {
#     'xml': {
#         'fulltext': {
#             'xpath': ['//body',
#                       '//section[@type="body"]',
#                       '//journalarticle-body',
#                       '//bdy',
#                       '//app-group',
#                       '//section[not(@type="acknowledgments" or @type="dataAccess" or @type="dataAvailability" or @type="superSection")]'
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'acknowledgements': {
#             'xpath': ['//ack',
#                       '//section[@type="acknowledgments"]',
#                       '//subsection[@type="acknowledgement" '
#                       'or @type="acknowledgment"]'
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'dataset': {
#             'xpath': ['//named-content[@content-type="dataset"]'],
#             'type': 'list',
#             'info': 'xlink:href',
#             },
#         'facility': {
#             'xpath': ['//named-content[@content-type="facility"]'],
#             'type': 'list',
#             'info': 'xlink:href',
#         }
#     },
#     'teixml': {
#         'fulltext': {
#             'xpath': ['//body',
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'acknowledgements': {
#             'xpath': ['//div[@type="acknowledgement"]',
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#     },
#     'xmlelsevier': {
#         'fulltext': {
#             'xpath': ['//body',
#                       '//raw-text',
#                       '//appendices',
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'acknowledgements': {
#             'xpath': ['//acknowledgment',
#                       '//ack',
#                       '//section[@type="acknowledgments"]',
#                       '//subsection[@type="acknowledgement" '
#                       'or @type="acknowledgment"]',
#                       '//*[local-name()="acknowledgment"]'
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'dataset': {
#             'xpath': ['//named-content[@content-type="dataset"]'],
#             'type': 'list',
#             'info': 'xlink:href',
#         }
#     },
#     'html': {
#         'introduction': [
#             '//h2[contains(.,"ntroduction")]',
#             '//h3[contains(.,"ntroduction")]',
#             '//p[contains(.,"Abstract")]',
#             ],
#         'references': [
#             '//h2[contains(.,"References")]'
#         ],
#         'table': [
#             '//table'
#         ],
#         'table_links': [
#             '//a[contains(@href, "TABLE_NAME")]'
#         ],
#         'head': [
#             '//head'
#         ]
#     },
#     'txt': {'fulltext': ['']},
#     'ocr': {'fulltext': ['']},
#     'http': {'fulltext': ['']},
#     'pdf': {'fulltext': ['']},
#     'pdf-grobid': {'grobid_fulltext': ['']},
# }


import os
import json
import gzip
from adsft.rules import META_CONTENT
from adsputils import setup_logging, load_config
    """
    Function that reads previously extracted data. It expects a json-type
    payload that has been converted into a Python dictionary.

    :param payload_dictionary: the complete extracted content and meta-data of
    the document payload
    :return: modified dictionary with recovered content or None if files do not exist
    """

    meta_output_file_path = payload_dictionary['meta_path']
    bibcode_pair_tree_path = os.path.dirname(meta_output_file_path)
    if payload_dictionary['file_format'] == "pdf-grobid":
        full_text_output_file_path = os.path.join(bibcode_pair_tree_path, 'grobid_fulltext.xml')
    else:
        full_text_output_file_path = os.path.join(bibcode_pair_tree_path, 'fulltext.txt.gz')

    content = {}
    if os.path.exists(meta_output_file_path):
        meta_dict = read_file(meta_output_file_path, json_format=True)
        for key, value in meta_dict.items():
            content[key] = value

        if os.path.exists(full_text_output_file_path):
            fulltext = read_file(full_text_output_file_path, json_format=False)
            content['fulltext'] = fulltext
        else:
            content['fulltext'] = ""

        # Read the custom extractions of content
        logger.debug('Copying extra meta content')

# Path: adsft/rules.py
# META_CONTENT = {
#     'xml': {
#         'fulltext': {
#             'xpath': ['//body',
#                       '//section[@type="body"]',
#                       '//journalarticle-body',
#                       '//bdy',
#                       '//app-group',
#                       '//section[not(@type="acknowledgments" or @type="dataAccess" or @type="dataAvailability" or @type="superSection")]'
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'acknowledgements': {
#             'xpath': ['//ack',
#                       '//section[@type="acknowledgments"]',
#                       '//subsection[@type="acknowledgement" '
#                       'or @type="acknowledgment"]'
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'dataset': {
#             'xpath': ['//named-content[@content-type="dataset"]'],
#             'type': 'list',
#             'info': 'xlink:href',
#             },
#         'facility': {
#             'xpath': ['//named-content[@content-type="facility"]'],
#             'type': 'list',
#             'info': 'xlink:href',
#         }
#     },
#     'teixml': {
#         'fulltext': {
#             'xpath': ['//body',
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'acknowledgements': {
#             'xpath': ['//div[@type="acknowledgement"]',
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#     },
#     'xmlelsevier': {
#         'fulltext': {
#             'xpath': ['//body',
#                       '//raw-text',
#                       '//appendices',
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'acknowledgements': {
#             'xpath': ['//acknowledgment',
#                       '//ack',
#                       '//section[@type="acknowledgments"]',
#                       '//subsection[@type="acknowledgement" '
#                       'or @type="acknowledgment"]',
#                       '//*[local-name()="acknowledgment"]'
#                       ],
#             'type': 'string',
#             'info': '',
#             },
#         'dataset': {
#             'xpath': ['//named-content[@content-type="dataset"]'],
#             'type': 'list',
#             'info': 'xlink:href',
#         }
#     },
#     'html': {
#         'introduction': [
#             '//h2[contains(.,"ntroduction")]',
#             '//h3[contains(.,"ntroduction")]',
#             '//p[contains(.,"Abstract")]',
#             ],
#         'references': [
#             '//h2[contains(.,"References")]'
#         ],
#         'table': [
#             '//table'
#         ],
#         'table_links': [
#             '//a[contains(@href, "TABLE_NAME")]'
#         ],
#         'head': [
#             '//head'
#         ]
#     },
#     'txt': {'fulltext': ['']},
#     'ocr': {'fulltext': ['']},
#     'http': {'fulltext': ['']},
#     'pdf': {'fulltext': ['']},
#     'pdf-grobid': {'grobid_fulltext': ['']},
# }


# Path: adsft/reader.py
import os
import json
import gzip
from adsft.rules import META_CONTENT
from adsputils import setup_logging, load_config

    """
    Function that reads previously extracted data. It expects a json-type
    payload that has been converted into a Python dictionary.

    :param payload_dictionary: the complete extracted content and meta-data of
    the document payload
    :return: modified dictionary with recovered content or None if files do not exist
    """

    meta_output_file_path = payload_dictionary['meta_path']
    bibcode_pair_tree_path = os.path.dirname(meta_output_file_path)
    if payload_dictionary['file_format'] == "pdf-grobid":
        full_text_output_file_path = os.path.join(bibcode_pair_tree_path, 'grobid_fulltext.xml')
    else:
        full_text_output_file_path = os.path.join(bibcode_pair_tree_path, 'fulltext.txt.gz')

    content = {}
    if os.path.exists(meta_output_file_path):
        meta_dict = read_file(meta_output_file_path, json_format=True)
        for key, value in meta_dict.items():
            content[key] = value

        if os.path.exists(full_text_output_file_path):
            fulltext = read_file(full_text_output_file_path, json_format=False)
            content['fulltext'] = fulltext
        else:
            content['fulltext'] = ""

        # Read the custom extractions of content
        logger.debug('Copying extra meta content')

        for meta_key_word in META_CONTENT[payload_dictionary['file_format']]: