<commit_before>#!/usr/bin/env python2
# -*- coding: UTF-8 -*-

import os
import copy

import redis
import config

from flask import current_app

# Redis Queue
from redis import Redis
from rq import Queue

# other libraries for rdflib
from rdflib import Graph

from modules.qc.qc import qc
from middleware.blazegraph.reserve_id import write_reserve_id
from modules.ectyper.call_ectyper import call_ectyper_vf, call_ectyper_serotype
from modules.amr.amr import amr
from modules.amr.amr_to_dict import amr_to_dict
from middleware.display.beautify import beautify, display_subtyping
from middleware.graphers.datastruct_savvy import datastruct_savvy
from middleware.graphers.turtle_grapher import turtle_grapher
from middleware.graphers.turtle_utils import actual_filename
from modules.phylotyper import phylotyper
from middleware.models import Job

from modules.loggingFunctions import initialize_logging
import logging

# logging
initialize_logging()
logger = logging.getLogger(__name__)


# When naming queues, make sure you set a worker to listen to that queue
# we use the high priority queue for things that should be immediately
# returned to the user.
redis_url = config.REDIS_URL
redis_conn = redis.from_url(redis_url)
singles_q = Queue('singles', connection=redis_conn)
multiples_q = Queue('multiples', connection=redis_conn,
                    default_timeout=config.DEFAULT_TIMEOUT)
amr_q = Queue('amr', connection=redis_conn,
                    default_timeout=config.DEFAULT_TIMEOUT*2)
phylotyper_q = Queue('phylotyper', connection=redis_conn,
                    default_timeout=config.DEFAULT_TIMEOUT)
blazegraph_q = Queue('blazegraph', connection=redis_conn, default_timeout=config.DEFAULT_TIMEOUT)
if config.BACKLOG_ENABLED:
    # backlog queues
    backlog_singles_q = Queue('backlog_singles', connection=redis_conn)
    backlog_multiples_q = Queue(
        'backlog_multiples', connection=redis_conn, default_timeout=config.DEFAULT_TIMEOUT)
    backlog_amr_q = Queue(
        'backlog_amr', connection=redis_conn, default_timeout=config.DEFAULT_TIMEOUT*2)
    backlog_phylotyper_q = Queue('backlog_phylotyper', connection=redis_conn,
                        default_timeout=config.DEFAULT_TIMEOUT)

def _ectyper_pipeline_vf(query_file, single_dict, display_vf=True, pipeline=None, backlog=False):
    """
    Enqueue all the jobs required for VF.
    """
    # Dictionary of Job instances to return
    d = {}
    # Alias.
    job_id = pipeline.jobs['job_id'].rq_job
    if not backlog:<commit_msg>EDIT: bump phylotypers default_ttl as well<commit_after>#!/usr/bin/env python2
# -*- coding: UTF-8 -*-

import os
import copy

import redis
import config

from flask import current_app

# Redis Queue
from redis import Redis
from rq import Queue

# other libraries for rdflib
from rdflib import Graph

from modules.qc.qc import qc
from middleware.blazegraph.reserve_id import write_reserve_id
from modules.ectyper.call_ectyper import call_ectyper_vf, call_ectyper_serotype
from modules.amr.amr import amr
from modules.amr.amr_to_dict import amr_to_dict
from middleware.display.beautify import beautify, display_subtyping
from middleware.graphers.datastruct_savvy import datastruct_savvy
from middleware.graphers.turtle_grapher import turtle_grapher
from middleware.graphers.turtle_utils import actual_filename
from modules.phylotyper import phylotyper
from middleware.models import Job

from modules.loggingFunctions import initialize_logging
import logging

# logging
initialize_logging()
logger = logging.getLogger(__name__)


# When naming queues, make sure you set a worker to listen to that queue
# we use the high priority queue for things that should be immediately
# returned to the user.
redis_url = config.REDIS_URL
redis_conn = redis.from_url(redis_url)
singles_q = Queue('singles', connection=redis_conn)
multiples_q = Queue('multiples', connection=redis_conn,
                    default_timeout=config.DEFAULT_TIMEOUT)
amr_q = Queue('amr', connection=redis_conn,
                    default_timeout=config.DEFAULT_TIMEOUT*2)
phylotyper_q = Queue('phylotyper', connection=redis_conn,
                    default_timeout=config.DEFAULT_TIMEOUT*2)
blazegraph_q = Queue('blazegraph', connection=redis_conn, default_timeout=config.DEFAULT_TIMEOUT)
if config.BACKLOG_ENABLED:
    # backlog queues
    backlog_singles_q = Queue('backlog_singles', connection=redis_conn)
    backlog_multiples_q = Queue(
        'backlog_multiples', connection=redis_conn, default_timeout=config.DEFAULT_TIMEOUT)
    backlog_amr_q = Queue(
        'backlog_amr', connection=redis_conn, default_timeout=config.DEFAULT_TIMEOUT*2)
    backlog_phylotyper_q = Queue('backlog_phylotyper', connection=redis_conn,
                        default_timeout=config.DEFAULT_TIMEOUT*2)

def _ectyper_pipeline_vf(query_file, single_dict, display_vf=True, pipeline=None, backlog=False):
    """
    Enqueue all the jobs required for VF.
    """
    # Dictionary of Job instances to return
    d = {}
    # Alias.
    job_id = pipeline.jobs['job_id'].rq_job
    if not backlog:
{'fraction_non_alphanumeric': 0.07169662255579695, 'fraction_numerical': 0.0019751135690302193, 'mean_word_length': 4.9020979020979025, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2053206', 'n_tokens_mistral': 1654, 'n_tokens_neox': 1625, 'n_words': 384}