# Path: adsft/tasks.py
# def task_check_if_extract(message):
# def task_extract(message):
#     def task_extract_grobid(message):
# def task_output_results(msg):
# def task_identify_facilities(message):
# 
# Path: adsft/utils.py
# class FileInputStream(object):
# class TextCleaner(object):
#     def __init__(self, input_stream):
#     def print_info(self):
#     def extract(self, force_extract=False, force_send=False):
#     def __init__(self, text):
#     def translate(self):
#     def decode(self):
#     def normalise(self):
#     def trimwords(self, maxlength=100):
#     def run(self, translate=True, decode=True, normalise=True, trim=True):
# def get_filenames(file_string):


import sys
import os
import tempfile
import argparse
import json
from builtins import zip
from builtins import str
from adsft import tasks, utils
from adsputils import setup_logging, load_config


    logger.debug('Setting variables')
    if 'max_queue_size' in kwargs:
        max_queue_size = kwargs['max_queue_size']
        logger.debug('Max queue size overridden: %d', kwargs['max_queue_size'])
    else:
        max_queue_size = 0

    if facility_ner and not force_extract:
        task_str = 'task_identify_facilities'
    else:
        task_str = 'task_check_if_extract'

    logger.info('Publishing records to: %s', task_str)

    i = 0
    total = len(records.payload)
    for record in records.payload:
        logger.debug('Publishing [%i/%i]: [%s]', i+1, total, record['bibcode'])

        if max_queue_size and i >= max_queue_size:
            logger.info('Max_queue_size reached, stopping. (Max queue size = %d)', max_queue_size)
            break

        if diagnose:
            print("[{}/{}] Calling '{}' with '{}'".format(i+1, total, task_str, str(record)))
        logger.debug("[%i/%i] Calling '%s' with '%s'", i+1, total, task_str, str(record))
        if i % 100000 == 0:
            logger.info("[%i/%i] Calling '%s'", i+1, total, task_str)

# Path: adsft/tasks.py
# def task_check_if_extract(message):
# def task_extract(message):
#     def task_extract_grobid(message):
# def task_output_results(msg):
# def task_identify_facilities(message):
# 
# Path: adsft/utils.py
# class FileInputStream(object):
# class TextCleaner(object):
#     def __init__(self, input_stream):
#     def print_info(self):
#     def extract(self, force_extract=False, force_send=False):
#     def __init__(self, text):
#     def translate(self):
#     def decode(self):
#     def normalise(self):
#     def trimwords(self, maxlength=100):
#     def run(self, translate=True, decode=True, normalise=True, trim=True):
# def get_filenames(file_string):


# Path: run.py
import sys
import os
import tempfile
import argparse
import json
from builtins import zip
from builtins import str
from adsft import tasks, utils
from adsputils import setup_logging, load_config



    logger.debug('Setting variables')
    if 'max_queue_size' in kwargs:
        max_queue_size = kwargs['max_queue_size']
        logger.debug('Max queue size overridden: %d', kwargs['max_queue_size'])
    else:
        max_queue_size = 0

    if facility_ner and not force_extract:
        task_str = 'task_identify_facilities'
    else:
        task_str = 'task_check_if_extract'

    logger.info('Publishing records to: %s', task_str)

    i = 0
    total = len(records.payload)
    for record in records.payload:
        logger.debug('Publishing [%i/%i]: [%s]', i+1, total, record['bibcode'])

        if max_queue_size and i >= max_queue_size:
            logger.info('Max_queue_size reached, stopping. (Max queue size = %d)', max_queue_size)
            break

        if diagnose:
            print("[{}/{}] Calling '{}' with '{}'".format(i+1, total, task_str, str(record)))
        logger.debug("[%i/%i] Calling '%s' with '%s'", i+1, total, task_str, str(record))
        if i % 100000 == 0:
            logger.info("[%i/%i] Calling '%s'", i+1, total, task_str)

        getattr(tasks, task_str).delay(record)