<commit_before>import json
import logging

from datetime import timedelta
from django.utils import timezone
from django.core.urlresolvers import reverse
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render
from django.views.decorators.csrf import csrf_protect
from django.contrib.auth.decorators import login_required
from rest_framework import serializers,generics
from django.forms.models import model_to_dict
from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework import status
from atlas.prodtask.models import RequestStatus
from atlas.prodtask.spdstodb import fill_template
from atlas.prodtask.views import create_steps_in_child_pattern, set_request_status, request_clone_slices, clone_slices
from ..prodtask.views import form_existed_step_list, form_step_in_page, create_request_for_pattern
from ..prodtask.forms import ProductionTrainForm, pattern_from_request, TRequestCreateCloneConfirmation, form_input_list_for_preview
from ..prodtask.models import TrainProductionLoad,TrainProduction,TRequest, InputRequestList, StepExecution, \
    ParentToChildRequest

_logger = logging.getLogger('prodtaskwebui')




def merge_pattern_train(train, requests):
    def form_output_lookup(outputs):
        result_dict = {}
        for slice in outputs:
            for output in slice[1]:
                result_dict[output] = slice[0]
        return result_dict
    outputs = train.output_by_slice
    outputs_lookup = form_output_lookup(outputs)
    dataset_slice = {}
    merge_requests = set()
    merged = 0
    for production_request in requests:
        slices = list(InputRequestList.objects.filter(request=production_request).order_by('id'))
        steps = list(StepExecution.objects.filter(request=production_request).order_by('slice'))
        if len(slices)!=len(steps):
            continue
        for index, step in enumerate(steps):
            dataset = slices[index].dataset_id
            current_dataset_outputs = dataset_slice.get(dataset,{})
            outputs = step.step_template.output_formats.split('.')
            if outputs[0] not in outputs_lookup:
                print train.id, production_request
            else:
                pattern_slice = outputs_lookup[outputs[0]]
                if pattern_slice not in current_dataset_outputs:
                    current_dataset_outputs[pattern_slice] = {'outputs':outputs,'count':1, 'requests':[production_request], 'slices':[slices[index].slice]}
                else:
                    new_outputs = [x for x in outputs if x not in current_dataset_outputs[pattern_slice]['outputs']]
                    if new_outputs:
                        merged += 1
                        current_dataset_outputs[pattern_slice]['count'] += 1
                        current_dataset_outputs[pattern_slice]['outputs'] = current_dataset_outputs[pattern_slice]['outputs'] + new_outputs
                        current_dataset_outputs[pattern_slice]['requests'].append(production_request)
                        current_dataset_outputs[pattern_slice]['slices'].append(slices[index].slice)
                        merge_requests.update(current_dataset_outputs[pattern_slice]['requests'])
                dataset_slice[dataset] = current_dataset_outputs
    return dataset_slice, merged, list(merge_requests)


def do_merge_requests(train_id, requests):
        train = TrainProduction.objects.get(id=train_id)
        dataset_slices, merged, merge_requests = merge_pattern_train(train,requests)
        dataset_to_merge = {}
        slices_to_skip = []
        for dataset in dataset_slices:
            for pattern_slice in dataset_slices[dataset]:
                if dataset_slices[dataset][pattern_slice]['count']>1:
                    dataset_to_merge.update({str(dataset_slices[dataset][pattern_slice]['requests'][0])+','+str(dataset_slices[dataset][pattern_slice]['slices'][0]):dataset_slices[dataset][pattern_slice]['outputs']})
                    for index, request_to_skip in enumerate(dataset_slices[dataset][pattern_slice]['requests'][1:],1):
                        slices_to_skip.append(str(request_to_skip)+','+str(dataset_slices[dataset][pattern_slice]['slices'][index]))
        for request in requests:
            if request not in merge_requests:
                print request,'approved'
                set_request_status('cron',request,'working','Automatic merged approve', 'Request was approved after merging')
                set_request_status('cron',request,'approved','Automatic merged approve', 'Request was approved during merging')
        if len(merge_requests)>0:
            base_request = TRequest.objects.get(reqid=merge_requests[0])
            merged_request = request_clone_slices(base_request.reqid, base_request.manager, 'Merged request for pattern %s'%str(train.pattern_request_id), base_request.ref_link,  [], base_request.project, False)
            for request in merge_requests:
                slices = list(InputRequestList.objects.filter(request=request))
                for slice in slices:
                    slice_index = str(request)+','+str(slice.slice)
                    if slice_index not in slices_to_skip:
                        new_slice = clone_slices(request,merged_request,[slice.slice],0,False)
                        step = StepExecution.objects.filter(slice=InputRequestList.objects.get(request=merged_request,slice=new_slice[0]))[0]
                        if slice_index in dataset_to_merge.keys():
                            step.step_template = fill_template(step.step_template.step,step.step_template.ctag,
                                                               step.step_template.priority,
                                                               '.'.join(dataset_to_merge[slice_index]))
                        step.status = 'Approved'
                        step.save()
                set_request_status('cron',request,'merged','Automatic merging', 'Request was merged to %s'%(str(merged_request)))
                relationship = ParentToChildRequest()
                relationship.status = 'active'
                relationship.parent_request = TRequest.objects.get(reqid=request)
                relationship.child_request = TRequest.objects.get(reqid=merged_request)
                relationship.train = None
                relationship.relation_type = 'MR'
                relationship.save()
                print request,'merged', merged_request
            set_request_status('cron',merged_request,'working','Automatic merged approve', 'Request was approved after merging')
            set_request_status('cron',merged_request,'approved','Automatic merged approve', 'Request was approved after merging')


@api_view(['POST'])
def merge_trains(request):
    data = json.loads(request.body)
    to_send = {}
    try:
        for train_id in data:
            train = TrainProduction.objects.get(id=train_id)
            dataset_slice, merged, temp = merge_pattern_train(train,data[train_id])
            to_send[train_id]  = dataset_slice
    except Exception, e:
        Response({"error": str(e)},status=status.HTTP_400_BAD_REQUEST)
    return Response({"load": to_send})

@api_view(['GET'])
def trains_to_merge(request):
    to_send = {}
    try:
        result = collect_trains(14)
        for entry in result.values():
            to_send[int(entry['train'].id)] = {'train_name':entry['train'].description,'request':map(int,entry['requests']),'request_count':len(entry['requests'])}
    except Exception, e:
        Response({"error": str(e)},status=status.HTTP_400_BAD_REQUEST)
    return Response({"trains": to_send})



def train_luanch(request):
    if request.method == 'GET':
        return render(request, 'prodtask/_train_merge.html', {
                'active_app': 'prodtask',
                'pre_form_text': 'Trains',
                'submit_url': 'prodtask:train_luanch',
                'parent_template': 'prodtask/_index.html',
            })



def collect_trains(days):
    min_request = RequestStatus.objects.filter(status='waiting', timestamp__gt=timezone.now()-timedelta(days=days)).order_by('id')[0].request_id
    requests = TRequest.objects.filter(request_type='GROUP',reqid__gte=min_request, cstatus='registered')<commit_msg>Add request merge for derivation<commit_after>import json
import logging

from datetime import timedelta
from django.utils import timezone
from django.core.urlresolvers import reverse
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render
from django.views.decorators.csrf import csrf_protect
from django.contrib.auth.decorators import login_required
from rest_framework import serializers,generics
from django.forms.models import model_to_dict
from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework import status
from atlas.prodtask.models import RequestStatus, HashTag, HashTagToRequest
from atlas.prodtask.spdstodb import fill_template
from atlas.prodtask.views import create_steps_in_child_pattern, set_request_status, request_clone_slices, clone_slices
from ..prodtask.views import form_existed_step_list, form_step_in_page, create_request_for_pattern
from ..prodtask.forms import ProductionTrainForm, pattern_from_request, TRequestCreateCloneConfirmation, form_input_list_for_preview
from ..prodtask.models import TrainProductionLoad,TrainProduction,TRequest, InputRequestList, StepExecution, \
    ParentToChildRequest

_logger = logging.getLogger('prodtaskwebui')




def merge_pattern_train(train, requests):
    def form_output_lookup(outputs):
        result_dict = {}
        for slice in outputs:
            for output in slice[1]:
                result_dict[output] = slice[0]
        return result_dict
    outputs = train.output_by_slice
    outputs_lookup = form_output_lookup(outputs)
    dataset_slice = {}
    merge_requests = set()
    merged = 0
    for production_request in requests:
        slices = list(InputRequestList.objects.filter(request=production_request).order_by('id'))
        steps = list(StepExecution.objects.filter(request=production_request).order_by('slice'))
        if len(slices)!=len(steps):
            continue
        for index, step in enumerate(steps):
            dataset = slices[index].dataset_id
            current_dataset_outputs = dataset_slice.get(dataset,{})
            outputs = step.step_template.output_formats.split('.')
            if outputs[0] not in outputs_lookup:
                pass
            else:
                pattern_slice = outputs_lookup[outputs[0]]
                if pattern_slice not in current_dataset_outputs:
                    current_dataset_outputs[pattern_slice] = {'outputs':outputs,'count':1, 'requests':[production_request], 'slices':[slices[index].slice]}
                else:
                    new_outputs = [x for x in outputs if x not in current_dataset_outputs[pattern_slice]['outputs']]
                    if new_outputs:
                        merged += 1
                        current_dataset_outputs[pattern_slice]['count'] += 1
                        current_dataset_outputs[pattern_slice]['outputs'] = current_dataset_outputs[pattern_slice]['outputs'] + new_outputs
                        current_dataset_outputs[pattern_slice]['requests'].append(production_request)
                        current_dataset_outputs[pattern_slice]['slices'].append(slices[index].slice)
                        merge_requests.update(current_dataset_outputs[pattern_slice]['requests'])
                dataset_slice[dataset] = current_dataset_outputs
    return dataset_slice, merged, list(merge_requests)


def do_merge_requests(train_id, requests):
        train = TrainProduction.objects.get(id=train_id)
        dataset_slices, merged, merge_requests = merge_pattern_train(train,requests)
        dataset_to_merge = {}
        slices_to_skip = []
        for dataset in dataset_slices:
            for pattern_slice in dataset_slices[dataset]:
                if dataset_slices[dataset][pattern_slice]['count']>1:
                    dataset_to_merge.update({str(dataset_slices[dataset][pattern_slice]['requests'][0])+','+str(dataset_slices[dataset][pattern_slice]['slices'][0]):dataset_slices[dataset][pattern_slice]['outputs']})
                    for index, request_to_skip in enumerate(dataset_slices[dataset][pattern_slice]['requests'][1:],1):
                        slices_to_skip.append(str(request_to_skip)+','+str(dataset_slices[dataset][pattern_slice]['slices'][index]))
        for request in requests:
            if request not in merge_requests:
                print request,'approved'
                set_request_status('cron',request,'working','Automatic merged approve', 'Request was approved after merging')
                set_request_status('cron',request,'approved','Automatic merged approve', 'Request was approved during merging')
        if len(merge_requests)>0:
            base_request = TRequest.objects.get(reqid=merge_requests[0])
            merged_request = request_clone_slices(base_request.reqid, base_request.manager, 'Merged request for pattern %s'%str(train.pattern_request_id), base_request.ref_link,  [], base_request.project, False)
            for request in merge_requests:
                slices = list(InputRequestList.objects.filter(request=request))
                for slice in slices:
                    slice_index = str(request)+','+str(slice.slice)
                    if slice_index not in slices_to_skip:
                        new_slice = clone_slices(request,merged_request,[slice.slice],0,False)
                        step = StepExecution.objects.filter(slice=InputRequestList.objects.get(request=merged_request,slice=new_slice[0]))[0]
                        if slice_index in dataset_to_merge.keys():
                            step.step_template = fill_template(step.step_template.step,step.step_template.ctag,
                                                               step.step_template.priority,
                                                               '.'.join(dataset_to_merge[slice_index]))
                        step.status = 'Approved'
                        step.save()
                set_request_status('cron',request,'merged','Automatic merging', 'Request was merged to %s'%(str(merged_request)))
                relationship = ParentToChildRequest()
                relationship.status = 'active'
                relationship.parent_request = TRequest.objects.get(reqid=request)
                relationship.child_request = TRequest.objects.get(reqid=merged_request)
                relationship.train = None
                relationship.relation_type = 'MR'
                relationship.save()
                print request,'merged', merged_request
            set_request_status('cron',merged_request,'working','Automatic merged approve', 'Request was approved after merging')
            set_request_status('cron',merged_request,'approved','Automatic merged approve', 'Request was approved after merging')


@api_view(['POST'])
def merge_trains(request):
    data = json.loads(request.body)
    to_send = {}
    try:
        for train_id in data:
            train = TrainProduction.objects.get(id=train_id)
            dataset_slice, merged, temp = merge_pattern_train(train,data[train_id])
            to_send[train_id]  = dataset_slice
    except Exception, e:
        Response({"error": str(e)},status=status.HTTP_400_BAD_REQUEST)
    return Response({"load": to_send})

@api_view(['GET'])
def trains_to_merge(request):
    to_send = {}
    try:
        result = collect_trains(14)
        for entry in result.values():
            to_send[int(entry['train'].id)] = {'train_name':entry['train'].description,'request':map(int,entry['requests']),'request_count':len(entry['requests'])}
    except Exception, e:
        return Response({"error": str(e)},status=status.HTTP_400_BAD_REQUEST)
    return Response({"trains": to_send})



def train_luanch(request):
    if request.method == 'GET':
        return render(request, 'prodtask/_train_merge.html', {
                'active_app': 'prodtask',
                'pre_form_text': 'Trains',
                'submit_url': 'prodtask:train_luanch',
                'parent_template': 'prodtask/_index.html',
            })


def patterns_to_show():
    requests = []
    if HashTag.objects.filter(hashtag__iexact='PatternToShowMerge').exists():
        patterns = list(HashTagToRequest.objects.filter(hashtag=HashTag.objects.filter(hashtag__iexact='PatternToShowMerge')[0]).values_list('request_id', flat=True))
        for pattern in patterns:
            trains = list(TrainProduction.objects.filter(pattern_request_id=pattern))
            requests += [x.request for x in trains if x.request]
    return requests


def collect_trains(days):
    min_request = RequestStatus.objects.filter(status='waiting', timestamp__gt=timezone.now()-timedelta(days=days)).order_by('id')[0].request_id
    requests = list(TRequest.objects.filter(request_type='GROUP',reqid__gte=min_request, cstatus='registered'))
    requests += list(TRequest.objects.filter(reqid__in=patterns_to_show()))
{'fraction_non_alphanumeric': 0.10037692084662221, 'fraction_numerical': 0.0031893302406494637, 'mean_word_length': 3.1022835394862036, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6692460', 'n_tokens_mistral': 4715, 'n_tokens_neox': 4565, 'n_words': 834}