<commit_before>#Tests for this module are in tests.py in the controller app

from django.core.management.base import NoArgsCommand
from django.conf import settings
from django.utils import timezone

#from http://jamesmckay.net/2009/03/django-custom-managepy-commands-not-committing-transactions/
#Fix issue where db data in manage.py commands is not refreshed at all once they start running
from django.db import transaction

import requests
import urlparse
import time
import json
import logging
from statsd import statsd

import controller.util as util
from controller.models import Submission
from controller.models import GraderStatus, SubmissionState
import project_urls

log = logging.getLogger(__name__)

class Command(NoArgsCommand):

    def handle_noargs(self, **options):
        """
        Constant loop that pulls from queue and posts to grading controller
        """
        log.info(' [*] Pulling from xqueues...')

        #Define sessions for logging into xqueue and controller
        self.xqueue_session = util.xqueue_login()
        self.controller_session = util.controller_login()
        #Login, then setup endless query loop
        flag = True

        while flag:
            #Loop through each queue that is given in arguments
            for queue_name in settings.QUEUES_TO_PULL_FROM:
                #Check for new submissions on xqueue, and send to controller
                pull_from_single_queue(queue_name,self.controller_session,self.xqueue_session)

            #Check for finalized results from controller, and post back to xqueue
            transaction.commit_unless_managed()
            submissions_to_post = check_for_completed_submissions()
            for submission in list(submissions_to_post):
                post_one_submission_back_to_queue(submission, self.xqueue_session)
            transaction.commit_unless_managed()

            time.sleep(settings.TIME_BETWEEN_XQUEUE_PULLS)

def post_one_submission_back_to_queue(submission,xqueue_session):
    xqueue_header, xqueue_body = util.create_xqueue_header_and_body(submission)
    (success, msg) = util.post_results_to_xqueue(
        xqueue_session,
        json.dumps(xqueue_header),
        json.dumps(xqueue_body),
    )

    statsd.increment("open_ended_assessment.grading_controller.post_to_xqueue",
        tags=["success:{0}".format(success)])

    if success:
        log.debug("Successful post back to xqueue! Success: {0} Message: {1}".format(success,message))
        submission.posted_results_back_to_queue = True
        submission.save()
    else:
        log.warning("Could not post back.  Error: {0}".format(msg))<commit_msg>Patch pull from xqueue<commit_after>#Tests for this module are in tests.py in the controller app

from django.core.management.base import NoArgsCommand
from django.conf import settings
from django.utils import timezone

#from http://jamesmckay.net/2009/03/django-custom-managepy-commands-not-committing-transactions/
#Fix issue where db data in manage.py commands is not refreshed at all once they start running
from django.db import transaction

import requests
import urlparse
import time
import json
import logging
from statsd import statsd

import controller.util as util
from controller.models import Submission
from controller.models import GraderStatus, SubmissionState
import project_urls

log = logging.getLogger(__name__)

class Command(NoArgsCommand):

    def handle_noargs(self, **options):
        """
        Constant loop that pulls from queue and posts to grading controller
        """
        log.info(' [*] Pulling from xqueues...')

        #Define sessions for logging into xqueue and controller
        self.xqueue_session = util.xqueue_login()
        self.controller_session = util.controller_login()
        #Login, then setup endless query loop
        flag = True

        while flag:
            #Loop through each queue that is given in arguments
            for queue_name in settings.QUEUES_TO_PULL_FROM:
                #Check for new submissions on xqueue, and send to controller
                pull_from_single_queue(queue_name,self.controller_session,self.xqueue_session)

            #Check for finalized results from controller, and post back to xqueue
            transaction.commit_unless_managed()
            submissions_to_post = check_for_completed_submissions()
            for submission in list(submissions_to_post):
                post_one_submission_back_to_queue(submission, self.xqueue_session)
            transaction.commit_unless_managed()

            time.sleep(settings.TIME_BETWEEN_XQUEUE_PULLS)

def post_one_submission_back_to_queue(submission,xqueue_session):
    xqueue_header, xqueue_body = util.create_xqueue_header_and_body(submission)
    (success, msg) = util.post_results_to_xqueue(
        xqueue_session,
        json.dumps(xqueue_header),
        json.dumps(xqueue_body),
    )

    statsd.increment("open_ended_assessment.grading_controller.post_to_xqueue",
        tags=["success:{0}".format(success)])

    if success:
        log.debug("Successful post back to xqueue! Success: {0} Message: {1}".format(success,msg))
        submission.posted_results_back_to_queue = True
        submission.save()
    else:
        log.warning("Could not post back.  Error: {0}".format(msg))
{'fraction_non_alphanumeric': 0.071334214002642, 'fraction_numerical': 0.0037742970371768257, 'mean_word_length': 3.4240400667779634, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '365750', 'n_tokens_mistral': 1501, 'n_tokens_neox': 1456, 'n_words': 400}