<commit_before>"""Provide an api used to run and get statements from the sparser reading tool.
"""

from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str

__all__ = ['get_version', 'process_text', 'process_xml', 'process_nxml_file',
           'process_nxml_str', 'make_sparser_nxml_from_text', 'run_sparser',
           'process_json_dict']

import os
import json
import logging
import subprocess
import xml.etree.ElementTree as ET
import multiprocessing as mp

from indra.util import UnicodeXMLTreeBuilder as UTB

from .processor import SparserXMLProcessor, SparserJSONProcessor

logger = logging.getLogger('sparser')

sparser_path_var = 'SPARSERPATH'
sparser_path = os.environ.get(sparser_path_var)


def get_version():
    with open(os.path.join(sparser_path, 'version.txt'), 'r') as f:
        version = f.read()
    return version


def process_xml(xml_str):
    try:
        tree = ET.XML(xml_str, parser=UTB())
    except ET.ParseError as e:
        logger.error('Could not parse XML string')
        logger.error(e)
        return None
    sp = _process_elementtree(tree)
    return sp


def run_sparser(fname, output_fmt, outbuf=None):
    if not sparser_path or not os.path.exists(sparser_path):
        logger.error('Sparser executable not set in %s' % sparser_path_var)
        return None
    if output_fmt == 'xml':
        format_flag = '-x'
        suffix = '.xml'
    elif output_fmt == 'json':
        format_flag = '-j'
        suffix = '.json'
    else:
        logger.error('Unknown output format: %s' % output_fmt)
        return None
    sparser_exec_path = os.path.join(sparser_path, 'save-semantics.sh')
    output_path = fname.split('.')[0] + '-semantics' + suffix
    for fpath in [sparser_exec_path, fname]:
        if not os.path.exists(fpath):
            raise Exception("'%s' is not a valid path." % fpath)

    subprocess.call(
        [sparser_exec_path, format_flag, fname],
        stdout=outbuf,
        stdin=subprocess.PIPE
        )
    assert os.path.exists(output_path), 'No output file created by sparser.'
    return output_path


def convert_sparser_output(output_fname, output_fmt='json'):
    if output_fmt not in ['json', 'xml']:<commit_msg>Change to using check_output in sparser_api.<commit_after>"""Provide an api used to run and get statements from the sparser reading tool.
"""

from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str

__all__ = ['get_version', 'process_text', 'process_xml', 'process_nxml_file',
           'process_nxml_str', 'make_sparser_nxml_from_text', 'run_sparser',
           'process_json_dict']

import os
import json
import logging
import subprocess
import xml.etree.ElementTree as ET
import multiprocessing as mp

from indra.util import UnicodeXMLTreeBuilder as UTB

from .processor import SparserXMLProcessor, SparserJSONProcessor

logger = logging.getLogger('sparser')

sparser_path_var = 'SPARSERPATH'
sparser_path = os.environ.get(sparser_path_var)


def get_version():
    with open(os.path.join(sparser_path, 'version.txt'), 'r') as f:
        version = f.read()
    return version


def process_xml(xml_str):
    try:
        tree = ET.XML(xml_str, parser=UTB())
    except ET.ParseError as e:
        logger.error('Could not parse XML string')
        logger.error(e)
        return None
    sp = _process_elementtree(tree)
    return sp


def run_sparser(fname, output_fmt, outbuf=None):
    if not sparser_path or not os.path.exists(sparser_path):
        logger.error('Sparser executable not set in %s' % sparser_path_var)
        return None
    if output_fmt == 'xml':
        format_flag = '-x'
        suffix = '.xml'
    elif output_fmt == 'json':
        format_flag = '-j'
        suffix = '.json'
    else:
        logger.error('Unknown output format: %s' % output_fmt)
        return None
    sparser_exec_path = os.path.join(sparser_path, 'save-semantics.sh')
    output_path = fname.split('.')[0] + '-semantics' + suffix
    for fpath in [sparser_exec_path, fname]:
        if not os.path.exists(fpath):
            raise Exception("'%s' is not a valid path." % fpath)

    out_bts = subprocess.check_output([sparser_exec_path, format_flag, fname])
    outbuf.write(out_bts)
    outbuf.flush()
    assert os.path.exists(output_path), 'No output file created by sparser.'
    return output_path


def convert_sparser_output(output_fname, output_fmt='json'):
    if output_fmt not in ['json', 'xml']:
{'fraction_non_alphanumeric': 0.1046201688138605, 'fraction_numerical': 0.000444247001332741, 'mean_word_length': 3.4672619047619047, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1064347', 'n_tokens_mistral': 1498, 'n_tokens_neox': 1439, 'n_words': 384}