<commit_before>from django.forms import ModelForm

from tastypie.resources import ModelResource, Resource
from tastypie.authorization import Authorization

from tastypie import fields
from tastypie.resources import ALL_WITH_RELATIONS, ALL
from tastypie.validation import FormValidation
from tastypie.bundle import Bundle

from apps.jobs.models import Task

from apps.dataviewer.api import AggregatorResource
from apps.dataviewer.api import FilterResource
from apps.dataviewer.api import CalculatorResource

from apps.dataviewer.models import Calculator

from apps.campaign.api import DatasetMetaResource

from apps.wsadmin.models import CassandraNode
from apps.wsadmin.models import GearmanNode

from gearman import GearmanClient
import pickle

class TaskUpdateResource(Resource):
    class Meta:
        resource_name = "task_update"    
        allowed_methods = ["post"]
    
    def obj_create(self, bundle, **kwargs):
        #raise Exception(bundle.data)
        obj = Task.objects.get(id=bundle.data["id"])
        try:
            obj.tasklet_count += bundle.data["increment"]
        except KeyError:
            try:
                obj.tasklet_count -= bundle.data["decrement"]
            except KeyError:
                pass
        if obj.tasklet_count < 0:
            obj.tasklet_count = 0
        obj.save()
        
        if obj.tasklet_count == 0:
            obj.done = True
            obj.save()
            try:
                new_task = obj.task_set.all()[0]
                new_dataset = DatasetMetaResource().get_via_uri(new_task.output_dataset, bundle.request)
                old_dataset = DatasetMetaResource().get_via_uri(new_task.input_dataset, bundle.request)
                new_dataset.highest_ts = old_dataset.highest_ts
                new_dataset.lowest_ts = old_dataset.lowest_ts
                new_dataset.save()
                
                
            except IndexError:
                #no more tasks to schedule
                return
            dat = {}
            dat["task"] = new_task.procedure_url
            dat["task_id"] = new_task.id
            dat["output_dataset"] = new_task.output_dataset
            dat["input_dataset"] = new_task.input_dataset
            dat["cassandra_nodes"] = CassandraNode.get_nodeip_list()
            client = GearmanClient(GearmanNode.get_nodeip_list())
            client.submit_job("pre_schedule", pickle.dumps(dat),background=True) 
            
            
            
class TaskResource(Resource):<commit_msg>api now measures number of tasks completed<commit_after>from django.forms import ModelForm

from tastypie.resources import ModelResource, Resource
from tastypie.authorization import Authorization

from tastypie import fields
from tastypie.resources import ALL_WITH_RELATIONS, ALL
from tastypie.validation import FormValidation
from tastypie.bundle import Bundle

from apps.jobs.models import Task

from apps.dataviewer.api import AggregatorResource
from apps.dataviewer.api import FilterResource
from apps.dataviewer.api import CalculatorResource

from apps.dataviewer.models import Calculator

from apps.campaign.api import DatasetMetaResource

from apps.wsadmin.models import CassandraNode
from apps.wsadmin.models import GearmanNode

from gearman import GearmanClient
from django_statsd.clients import statsd
import pickle

class TaskUpdateResource(Resource):
    class Meta:
        resource_name = "task_update"    
        allowed_methods = ["post"]
    
    def obj_create(self, bundle, **kwargs):
        #raise Exception(bundle.data)
        obj = Task.objects.get(id=bundle.data["id"])
        try:
            obj.tasklet_count += bundle.data["increment"]
        except KeyError:
            try:
                obj.tasklet_count -= bundle.data["decrement"]
            except KeyError:
                pass
        if obj.tasklet_count < 0:
            obj.tasklet_count = 0
        obj.save()
        statsd.incr('task_completed')

        
        if obj.tasklet_count == 0:
            obj.done = True
            obj.save()
            try:
                new_task = obj.task_set.all()[0]
                new_dataset = DatasetMetaResource().get_via_uri(new_task.output_dataset, bundle.request)
                old_dataset = DatasetMetaResource().get_via_uri(new_task.input_dataset, bundle.request)
                new_dataset.highest_ts = old_dataset.highest_ts
                new_dataset.lowest_ts = old_dataset.lowest_ts
                new_dataset.save()
                
                
            except IndexError:
                #no more tasks to schedule
                return
            dat = {}
            dat["task"] = new_task.procedure_url
            dat["task_id"] = new_task.id
            dat["output_dataset"] = new_task.output_dataset
            dat["input_dataset"] = new_task.input_dataset
            dat["cassandra_nodes"] = CassandraNode.get_nodeip_list()
            client = GearmanClient(GearmanNode.get_nodeip_list())
            client.submit_job("pre_schedule", pickle.dumps(dat),background=True) 
            
            
            
class TaskResource(Resource):
{'fraction_non_alphanumeric': 0.07514113295697879, 'fraction_numerical': 0.001557329180455519, 'mean_word_length': 2.3385315139701106, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3659888', 'n_tokens_mistral': 1441, 'n_tokens_neox': 1382, 'n_words': 302}