<commit_before>import codecs
import csv
import os
from collections import OrderedDict
from datetime import timedelta, datetime

from StringIO import StringIO

import requests
from django.conf import settings
from django.core.mail import send_mail
from django.db.utils import IntegrityError
from django.forms.models import model_to_dict
from django.http.response import HttpResponse
from django.views.generic.base import View
from django.db.models import QuerySet
from django.shortcuts import reverse
from rest_framework.generics import GenericAPIView
from unicodecsv.py2 import UnicodeWriter

from dataloader.models import SamplingFeature, TimeSeriesResultValue, Unit, EquipmentModel, TimeSeriesResult, Result
from django.db.models.expressions import F
from django.utils.dateparse import parse_datetime
from rest_framework import exceptions
from rest_framework import status
from rest_framework.authentication import SessionAuthentication
from rest_framework.response import Response
from rest_framework.views import APIView

from dataloaderinterface.forms import SiteSensorForm, SensorDataForm
from dataloaderinterface.models import SiteSensor, SiteRegistration, SensorOutput, SensorMeasurement
from dataloaderservices.auth import UUIDAuthentication
from dataloaderservices.serializers import OrganizationSerializer

from leafpack.models import LeafPack

# TODO: Check user permissions to edit, add, or remove stuff with a permissions class.
# TODO: Use generic api views for create, edit, delete, and list.


class ModelVariablesApi(APIView):
    authentication_classes = (SessionAuthentication, )

    def get(self, request, format=None):
        if 'equipment_model_id' not in request.GET:
            return Response({'error': 'Equipment Model Id not received.'})

        equipment_model_id = request.GET['equipment_model_id']
        if equipment_model_id == '':
            return Response({'error': 'Empty Equipment Model Id received.'})

        equipment_model = EquipmentModel.objects.filter(pk=equipment_model_id).first()
        if not equipment_model:
            return Response({'error': 'Equipment Model not found.'})

        output = equipment_model.instrument_output_variables.values('variable', 'instrument_raw_output_unit')

        return Response(output)


class OutputVariablesApi(APIView):
    authentication_classes = (SessionAuthentication,)

    def get(self, request):
        output = SensorOutput.objects.for_filters()
        return Response(output)


class RegisterSensorApi(APIView):
    authentication_classes = (SessionAuthentication, )

    def post(self, request, format=None):
        form = SiteSensorForm(data=request.POST)

        if not form.is_valid():
            error_data = dict(form.errors)
            return Response(error_data, status=status.HTTP_206_PARTIAL_CONTENT)

        registration = form.cleaned_data['registration']
        sensor_output = form.cleaned_data['output_variable']
        height = form.cleaned_data['height']
        notes = form.cleaned_data['sensor_notes']

        site_sensor = SiteSensor.objects.create(registration=registration, sensor_output=sensor_output, height=height, sensor_notes=notes)
        return Response(model_to_dict(site_sensor, fields=[field.name for field in site_sensor._meta.fields]), status=status.HTTP_201_CREATED)


class EditSensorApi(APIView):
    authentication_classes = (SessionAuthentication, )

    def post(self, request, format=None):
        if 'id' not in request.POST:
            return Response({'id': 'No sensor id in the request.'}, status=status.HTTP_400_BAD_REQUEST)

        sensor = SiteSensor.objects.filter(pk=request.POST['id']).first()
        if not sensor:
            return Response({'id': 'Sensor not found.'}, status=status.HTTP_404_NOT_FOUND)

        form = SiteSensorForm(data=request.POST, instance=sensor)

        if not form.is_valid():
            error_data = dict(form.errors)
            return Response(error_data, status=status.HTTP_206_PARTIAL_CONTENT)

        sensor.sensor_output = form.cleaned_data['output_variable']
        sensor.height = form.cleaned_data['height']
        sensor.sensor_notes = form.cleaned_data['sensor_notes']
        sensor.save(update_fields=['sensor_output', 'height', 'sensor_notes'])
        return Response(model_to_dict(sensor, fields=[field.name for field in sensor._meta.fields]), status=status.HTTP_202_ACCEPTED)


class DeleteSensorApi(APIView):
    authentication_classes = (SessionAuthentication, )

    def post(self, request, format=None):
        if 'id' not in request.POST:
            return Response({'id': 'No sensor id in the request.'}, status=status.HTTP_400_BAD_REQUEST)

        sensor = SiteSensor.objects.filter(pk=request.POST['id']).first()
        if not sensor:
            return Response({'id': 'Sensor not found.'}, status=status.HTTP_404_NOT_FOUND)

        deleted = sensor.delete()
        return Response(deleted, status=status.HTTP_202_ACCEPTED)


class DeleteLeafpackApi(APIView):
    authentication_classes = (SessionAuthentication, )

    def post(self, request, format=None):
        if 'id' not in request.POST:
            return Response({'id': 'No leafpack id in the request.'}, status=status.HTTP_400_BAD_REQUEST)

        leafpack = LeafPack.objects.filter(pk=request.POST['id']).first()
        if not leafpack:
            return Response({'id': 'Leafpack not found.'}, status=status.HTTP_404_NOT_FOUND)

        deleted = leafpack.delete()
        return Response(deleted, status=status.HTTP_202_ACCEPTED)


class OrganizationApi(APIView):
    authentication_classes = (SessionAuthentication, )

    def post(self, request, format=None):
        organization_serializer = OrganizationSerializer(data=request.data)

        if organization_serializer.is_valid():
            organization_serializer.save()
            return Response(organization_serializer.data, status=status.HTTP_201_CREATED)

        error_data = dict(organization_serializer.errors)
        return Response(error_data, status=status.HTTP_206_PARTIAL_CONTENT)


class FollowSiteApi(APIView):
    authentication_classes = (SessionAuthentication,)

    def post(self, request, format=None):
        action = request.data['action']
        sampling_feature_code = request.data['sampling_feature_code']
        site = SiteRegistration.objects.get(sampling_feature_code=sampling_feature_code)

        if action == 'follow':
            request.user.followed_sites.add(site)
        elif action == 'unfollow':
            request.user.followed_sites.remove(site)

        return Response({}, status.HTTP_200_OK)


class SensorDataUploadView(APIView):
    authentication_classes = (SessionAuthentication,)
    header_row_indicators = ('Data Logger', 'Sampling Feature',
                             'Sensor', 'Variable', 'Result', 'Date and Time')

    def should_skip_row(self, row):
        if row[0].startswith(self.header_row_indicators):
            return True

    def build_results_dict(self, data_file):
        results = {'utc_offset': 0, 'site_uuid': '', 'results': {}}
        got_feature_uuid = False
        got_result_uuids = False
        got_UTC_offset = False

        for index, row in enumerate(csv.reader(data_file)):

            if row[0].startswith('Sampling Feature') and not got_feature_uuid:
                    results['site_uuid'] = row[0].replace(
                        'Sampling Feature UUID: ', '').replace(
                        'Sampling Feature: ', '')
                    got_feature_uuid = True

                    # oldest csv's from modular sensors have the result UUID's
                    # in the same row as the sampling feature UUID
                    # build dict with the rest of the columns
                    if row[1] != '' and not got_result_uuids:
                        results['results'] = {uuid:
                                              {'index': uuid_index,
                                               'values': []
                                               } for uuid_index, uuid in
                                              enumerate(row[1:], start=1)}
                        got_result_uuids = True

            elif row[0].startswith('Result UUID:') and not got_result_uuids:
                results['results'] = {uuid:
                                      {'index': uuid_index,
                                       'values': []
                                       } for uuid_index, uuid in
                                      enumerate(row[1:], start=1)}<commit_msg>fix #416<commit_after>import codecs
import csv
import os
from collections import OrderedDict
from datetime import timedelta, datetime

from StringIO import StringIO

import requests
from django.conf import settings
from django.core.mail import send_mail
from django.db.utils import IntegrityError
from django.forms.models import model_to_dict
from django.http.response import HttpResponse
from django.views.generic.base import View
from django.db.models import QuerySet
from django.shortcuts import reverse
from rest_framework.generics import GenericAPIView
from unicodecsv.py2 import UnicodeWriter

from dataloader.models import SamplingFeature, TimeSeriesResultValue, Unit, EquipmentModel, TimeSeriesResult, Result
from django.db.models.expressions import F
from django.utils.dateparse import parse_datetime
from rest_framework import exceptions
from rest_framework import status
from rest_framework.authentication import SessionAuthentication
from rest_framework.response import Response
from rest_framework.views import APIView

from dataloaderinterface.forms import SiteSensorForm, SensorDataForm
from dataloaderinterface.models import SiteSensor, SiteRegistration, SensorOutput, SensorMeasurement
from dataloaderservices.auth import UUIDAuthentication
from dataloaderservices.serializers import OrganizationSerializer

from leafpack.models import LeafPack

# TODO: Check user permissions to edit, add, or remove stuff with a permissions class.
# TODO: Use generic api views for create, edit, delete, and list.


class ModelVariablesApi(APIView):
    authentication_classes = (SessionAuthentication, )

    def get(self, request, format=None):
        if 'equipment_model_id' not in request.GET:
            return Response({'error': 'Equipment Model Id not received.'})

        equipment_model_id = request.GET['equipment_model_id']
        if equipment_model_id == '':
            return Response({'error': 'Empty Equipment Model Id received.'})

        equipment_model = EquipmentModel.objects.filter(pk=equipment_model_id).first()
        if not equipment_model:
            return Response({'error': 'Equipment Model not found.'})

        output = equipment_model.instrument_output_variables.values('variable', 'instrument_raw_output_unit')

        return Response(output)


class OutputVariablesApi(APIView):
    authentication_classes = (SessionAuthentication,)

    def get(self, request):
        output = SensorOutput.objects.for_filters()
        return Response(output)


class RegisterSensorApi(APIView):
    authentication_classes = (SessionAuthentication, )

    def post(self, request, format=None):
        form = SiteSensorForm(data=request.POST)

        if not form.is_valid():
            error_data = dict(form.errors)
            return Response(error_data, status=status.HTTP_206_PARTIAL_CONTENT)

        registration = form.cleaned_data['registration']
        sensor_output = form.cleaned_data['output_variable']
        height = form.cleaned_data['height']
        notes = form.cleaned_data['sensor_notes']

        site_sensor = SiteSensor.objects.create(registration=registration, sensor_output=sensor_output, height=height, sensor_notes=notes)
        return Response(model_to_dict(site_sensor, fields=[field.name for field in site_sensor._meta.fields]), status=status.HTTP_201_CREATED)


class EditSensorApi(APIView):
    authentication_classes = (SessionAuthentication, )

    def post(self, request, format=None):
        if 'id' not in request.POST:
            return Response({'id': 'No sensor id in the request.'}, status=status.HTTP_400_BAD_REQUEST)

        sensor = SiteSensor.objects.filter(pk=request.POST['id']).first()
        if not sensor:
            return Response({'id': 'Sensor not found.'}, status=status.HTTP_404_NOT_FOUND)

        form = SiteSensorForm(data=request.POST, instance=sensor)

        if not form.is_valid():
            error_data = dict(form.errors)
            return Response(error_data, status=status.HTTP_206_PARTIAL_CONTENT)

        sensor.sensor_output = form.cleaned_data['output_variable']
        sensor.height = form.cleaned_data['height']
        sensor.sensor_notes = form.cleaned_data['sensor_notes']
        sensor.save(update_fields=['sensor_output', 'height', 'sensor_notes'])
        return Response(model_to_dict(sensor, fields=[field.name for field in sensor._meta.fields]), status=status.HTTP_202_ACCEPTED)


class DeleteSensorApi(APIView):
    authentication_classes = (SessionAuthentication, )

    def post(self, request, format=None):
        if 'id' not in request.POST:
            return Response({'id': 'No sensor id in the request.'}, status=status.HTTP_400_BAD_REQUEST)

        sensor = SiteSensor.objects.filter(pk=request.POST['id']).first()
        if not sensor:
            return Response({'id': 'Sensor not found.'}, status=status.HTTP_404_NOT_FOUND)

        deleted = sensor.delete()
        return Response(deleted, status=status.HTTP_202_ACCEPTED)


class DeleteLeafpackApi(APIView):
    authentication_classes = (SessionAuthentication, )

    def post(self, request, format=None):
        if 'id' not in request.POST:
            return Response({'id': 'No leafpack id in the request.'}, status=status.HTTP_400_BAD_REQUEST)

        leafpack = LeafPack.objects.filter(pk=request.POST['id']).first()
        if not leafpack:
            return Response({'id': 'Leafpack not found.'}, status=status.HTTP_404_NOT_FOUND)

        deleted = leafpack.delete()
        return Response(deleted, status=status.HTTP_202_ACCEPTED)


class OrganizationApi(APIView):
    authentication_classes = (SessionAuthentication, )

    def post(self, request, format=None):
        organization_serializer = OrganizationSerializer(data=request.data)

        if organization_serializer.is_valid():
            organization_serializer.save()
            return Response(organization_serializer.data, status=status.HTTP_201_CREATED)

        error_data = dict(organization_serializer.errors)
        return Response(error_data, status=status.HTTP_206_PARTIAL_CONTENT)


class FollowSiteApi(APIView):
    authentication_classes = (SessionAuthentication,)

    def post(self, request, format=None):
        action = request.data['action']
        sampling_feature_code = request.data['sampling_feature_code']
        site = SiteRegistration.objects.get(sampling_feature_code=sampling_feature_code)

        if action == 'follow':
            request.user.followed_sites.add(site)
        elif action == 'unfollow':
            request.user.followed_sites.remove(site)

        return Response({}, status.HTTP_200_OK)


class SensorDataUploadView(APIView):
    authentication_classes = (SessionAuthentication,)
    header_row_indicators = ('Data Logger', 'Sampling Feature',
                             'Sensor', 'Variable', 'Result', 'Date and Time')

    def should_skip_row(self, row):
        if row[0].startswith(self.header_row_indicators):
            return True

    def build_results_dict(self, data_file):
        results = {'utc_offset': 0, 'site_uuid': '', 'results': {}}
        got_feature_uuid = False
        got_result_uuids = False
        got_UTC_offset = False

        for index, row in enumerate(csv.reader(data_file)):

            if row[0].startswith('Sampling Feature') and not got_feature_uuid:
                    results['site_uuid'] = row[0].replace(
                        'Sampling Feature UUID: ', '').replace(
                        'Sampling Feature: ', '')
                    got_feature_uuid = True

                    # oldest csv's from modular sensors have the result UUID's
                    # in the same row as the sampling feature UUID
                    # build dict with the rest of the columns
                    if len(row)>1:
                        if row[1] != '' and not got_result_uuids:
                            results['results'] = {uuid:
                                              {'index': uuid_index,
                                               'values': []
                                               } for uuid_index, uuid in
                                              enumerate(row[1:], start=1)}
                            got_result_uuids = True

            elif row[0].startswith('Result UUID:') and not got_result_uuids:
                results['results'] = {uuid:
                                      {'index': uuid_index,
                                       'values': []
                                       } for uuid_index, uuid in
                                      enumerate(row[1:], start=1)}
{'fraction_non_alphanumeric': 0.09031016791044776, 'fraction_numerical': 0.006763059701492537, 'mean_word_length': 2.961431870669746, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3199402', 'n_tokens_mistral': 4623, 'n_tokens_neox': 4449, 'n_words': 1092}