<commit_before>"""This module provides essential tools to run reading using indra's own
database. This may also be run as a script; for details run: 
`python read_pmids_db --help`
"""
from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str

import os
import sys
import re
import tempfile
import logging
import subprocess
import glob
import json
import random
import zlib
import pickle
import shutil
from argparse import ArgumentParser
from docutils.io import InputError
from datetime import datetime
from math import log10, floor
from os.path import join as pjoin
from os import path
from sqlalchemy.sql.expression import or_, and_, intersect

logger = logging.getLogger('read_db')
if __name__ == '__main__':
    parser = ArgumentParser(
        description='A tool to read and process content from the database.'
        )
    parser.add_argument(
        '-i', '--id_file',
        help=('A file containing a list of ids of the form <id_type>:<id>.'
              'Cannot be used in conjunction with -f/--file_file.')
        )
    parser.add_argument(
        '-f', '--file_file',
        help=('A file containing a list of files to be input into reach. These'
              'should be nxml or txt files. Cannot be used in conjunction with'
              '-i/--id_file. For safesed use, files should be given by'
              'absolute paths.')
        )
    parser.add_argument(
        '-r', '--readers',
        choices=['reach', 'sparser'],
        help='List of readers to be used.',
        nargs='+'
        )
    parser.add_argument(
        '-b', '--batch',
        help='Select the number of content entries to be read in each batch.',
        default=1000,
        type=int
        )
    parser.add_argument(
        '-n', '--num_procs',
        help='Select the number of processes to use.',
        type=int,
        default=1
        )
    parser.add_argument(
        '-s', '--sample',
        help='To read just a sample of the pmids in the file.',
        type=int
        )
    parser.add_argument(
        '-I', '--in_range',
        help='A range of pmids to be read in the form <start>:<end>.'
        )
    parser.add_argument(
        '-v', '--verbose',
        help='Include output from the readers.',
        action='store_true'
        )
    parser.add_argument(
        '-q', '--quiet',
        help='Suppress most output. Overrides -v and -d options.',
        action='store_true'
        )
    parser.add_argument(
        '-d', '--debug',
        help='Set the logging to debug level.',
        action='store_true'
        )
    parser.add_argument(
        '-m', '--messy',
        help='Do not clean up directories created while reading.',
        action='store_true'
        )
    parser.add_argument(
        '-p', '--pickle',
        help='Pickle all results and save in .pkl files.',
        action='store_true'
        )
    parser.add_argument(
        '--no_reading_upload',
        help='Choose not to upload the reading output to the database.',
        action='store_true'
        )
    parser.add_argument(
        '--no_statement_upload',
        help='Choose not to upload the statements to the databse.',
        action='store_true'
        )
    parser.add_argument(
        '--force_read',
        help=('Make the reader read all the content. Otherwise, the system '
              'will simply process existing readings where possible.'),
        action='store_true'
        )
    parser.add_argument(
        '--force_fulltext',
        help='Make the reader only read full text from the database.',
        action='store_true'
        )
    args = parser.parse_args()
    if args.debug and not args.quiet:
        logger.setLevel(logging.DEBUG)

from indra.util import unzip_string, zip_string
from indra.db import get_primary_db, formats, texttypes
from indra.tools.reading.read_pmids import get_mem_total
from indra.tools.get_version import get_version as get_indra_version
from indra.sources import reach, sparser


class ReadingError(Exception):
    pass


class ReachError(ReadingError):
    pass


class SparserError(ReadingError):
    pass<commit_msg>Add argument for no_read feature.<commit_after>"""This module provides essential tools to run reading using indra's own
database. This may also be run as a script; for details run: 
`python read_pmids_db --help`
"""
from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str

import os
import sys
import re
import tempfile
import logging
import subprocess
import glob
import json
import random
import zlib
import pickle
import shutil
from argparse import ArgumentParser
from docutils.io import InputError
from datetime import datetime
from math import log10, floor
from os.path import join as pjoin
from os import path
from sqlalchemy.sql.expression import or_, and_, intersect

logger = logging.getLogger('read_db')
if __name__ == '__main__':
    parser = ArgumentParser(
        description='A tool to read and process content from the database.'
        )
    parser.add_argument(
        '-i', '--id_file',
        help=('A file containing a list of ids of the form <id_type>:<id>.'
              'Cannot be used in conjunction with -f/--file_file.')
        )
    parser.add_argument(
        '-f', '--file_file',
        help=('A file containing a list of files to be input into reach. These'
              'should be nxml or txt files. Cannot be used in conjunction with'
              '-i/--id_file. For safesed use, files should be given by'
              'absolute paths.')
        )
    parser.add_argument(
        '-r', '--readers',
        choices=['reach', 'sparser'],
        help='List of readers to be used.',
        nargs='+'
        )
    parser.add_argument(
        '-b', '--batch',
        help='Select the number of content entries to be read in each batch.',
        default=1000,
        type=int
        )
    parser.add_argument(
        '-n', '--num_procs',
        help='Select the number of processes to use.',
        type=int,
        default=1
        )
    parser.add_argument(
        '-s', '--sample',
        help='To read just a sample of the pmids in the file.',
        type=int
        )
    parser.add_argument(
        '-I', '--in_range',
        help='A range of pmids to be read in the form <start>:<end>.'
        )
    parser.add_argument(
        '-v', '--verbose',
        help='Include output from the readers.',
        action='store_true'
        )
    parser.add_argument(
        '-q', '--quiet',
        help='Suppress most output. Overrides -v and -d options.',
        action='store_true'
        )
    parser.add_argument(
        '-d', '--debug',
        help='Set the logging to debug level.',
        action='store_true'
        )
    parser.add_argument(
        '-m', '--messy',
        help='Do not clean up directories created while reading.',
        action='store_true'
        )
    parser.add_argument(
        '-p', '--pickle',
        help='Pickle all results and save in .pkl files.',
        action='store_true'
        )
    parser.add_argument(
        '--no_reading_upload',
        help='Choose not to upload the reading output to the database.',
        action='store_true'
        )
    parser.add_argument(
        '--no_statement_upload',
        help='Choose not to upload the statements to the databse.',
        action='store_true'
        )
    parser.add_argument(
        '--force_read',
        help=('Make the reader read all the content. Otherwise, the system '
              'will simply process existing readings where possible.'),
        action='store_true'
        )
    parser.add_argument(
        '--force_fulltext',
        help='Make the reader only read full text from the database.',
        action='store_true'
        )
    parser.add_argument(
        '--no_read',
        help='Only create statements using existing reading results.',
        action='store_true'
        )
    args = parser.parse_args()
    if args.debug and not args.quiet:
        logger.setLevel(logging.DEBUG)

from indra.util import unzip_string, zip_string
from indra.db import get_primary_db, formats, texttypes
from indra.tools.reading.read_pmids import get_mem_total
from indra.tools.get_version import get_version as get_indra_version
from indra.sources import reach, sparser


class ReadingError(Exception):
    pass


class ReachError(ReadingError):
    pass


class SparserError(ReadingError):
    pass
{'fraction_non_alphanumeric': 0.10135373749264273, 'fraction_numerical': 0.00164802825191289, 'mean_word_length': 2.3541255428345833, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4798608', 'n_tokens_mistral': 2428, 'n_tokens_neox': 2381, 'n_words': 811}