<commit_before>import json
import base64
import hashlib  # fix(clean): remove?
import zipfile
import datetime
from io import BytesIO


# external imports
from flask import request, abort, make_response
from sqlalchemy import not_
from sqlalchemy.orm.exc import NoResultFound
from flask_restful import Resource as ApiResource
from flask_login import current_user


# internal imports
from main.app import db
from main.users.models import User
from main.users.permissions import access_level, ACCESS_LEVEL_READ, ACCESS_LEVEL_WRITE
from main.util import parse_json_datetime
from main.resources.models import Resource, ResourceRevision, ResourceView, ControllerStatus, Thumbnail
from main.resources.resource_util import find_resource, read_resource, add_resource_revision, _create_file, update_sequence_value, \
    resource_type_number, _create_folders, create_sequence
from main.resources.file_conversion import convert_csv_to_xls, convert_xls_to_csv, convert_new_lines, compute_thumbnail
from main.users.auth import find_key  # fix(clean): remove?


class ResourceRecord(ApiResource):

    # get the current value or meta data of a resource
    def get(self, resource_path):
        resource_path = '/' + resource_path  # we always want resource paths to start with leading slash
        args = request.values
        result = {}

        # handle case of controller requesting about self
        if resource_path == '/self':
            if request.authorization:
                key = find_key(request.authorization.password)
            else:
                key = None
            if key and key.access_as_controller_id:
                try:
                    r = Resource.query.filter(Resource.id == key.access_as_controller_id).one()
                    resource_path = r.path()
                except NoResultFound:
                    abort(404)
            else:
                abort(403)

        # look up the resource record
        else:
            r = find_resource(resource_path)
            if not r:
                abort(404)  # fix(later): revisit to avoid leaking file existance
            if access_level(r.query_permissions()) < ACCESS_LEVEL_READ:
                abort(403)

        # if request meta-data
        if request.values.get('meta', False):
            result = r.as_dict(extended=True)
            result['path'] = resource_path

        # if request data
        else:

            # if folder, return contents list or zip of collection of files
            if 10 <= r.type < 20:

                # multi-file download
                if 'ids' in args and args.get('download', False):
                    ids = args['ids'].split(',')
                    return batch_download(r, ids)

                # contents list
                else:
                    recursive = request.values.get('recursive', False)
                    type_name = request.values.get('type', None)
                    if type_name:
                        type_number = resource_type_number(type_name)
                    else:
                        type_number = None
                    name_filter = request.values.get('filter', None)
                    extended = request.values.get('extended', False)
                    result = resource_list(r.id, recursive, type_number, name_filter, extended)

            # if sequence, return value(s)
            # fix(later): merge with file case?
            elif r.type == Resource.SEQUENCE:

                # get parameters
                text = request.values.get('text', '')
                download = request.values.get('download', False)
                count = int(request.values.get('count', 1))
                start_timestamp = request.values.get('start_timestamp', '')
                end_timestamp = request.values.get('end_timestamp', '')
                if start_timestamp:
                    try:
                        start_timestamp = parse_json_datetime(start_timestamp)
                    except ValueError:
                        abort(400, 'Invalid date/time.')
                if end_timestamp:
                    try:
                        end_timestamp = parse_json_datetime(end_timestamp)
                    except ValueError:
                        abort(400, 'Invalid date/time.')

                # if filters specified, assume we want a sequence of values
                if text or start_timestamp or end_timestamp or count > 1:

                    # get summary of values
                    if int(request.values.get('summary', False)):
                        return sequence_value_summary(r.id)

                    # get preliminary set of values
                    resource_revisions = ResourceRevision.query.filter(ResourceRevision.resource_id == r.id)

                    # apply filters (if any)
                    if text:
                        resource_revisions = resource_revisions.filter(text in ResourceRevision.data)
                    if start_timestamp:
                        resource_revisions = resource_revisions.filter(ResourceRevision.timestamp >= start_timestamp)
                    if end_timestamp:
                        resource_revisions = resource_revisions.filter(ResourceRevision.timestamp <= end_timestamp)
                    resource_revisions = resource_revisions.order_by('timestamp')
                    if resource_revisions.count() > count:
                        resource_revisions = resource_revisions[-count:]  # fix(later): is there a better/faster way to do this?

                    # return data
                    if download:
                        # timezone = r.root().system_attributes['timezone']  # fix(soon): use this instead of UTC
                        lines = ['utc_timestamp,value\n']
                        for rr in resource_revisions:
                            lines.append('%s,%s\n' % (rr.timestamp.strftime('%Y-%m-%d %H:%M:%S.%f'), rr.data))
                        result = make_response(''.join(lines))
                        result.headers['Content-Type'] = 'application/octet-stream'
                        result.headers['Content-Disposition'] = 'attachment; filename=' + r.name + '.csv'
                        return result
                    else:
                        epoch = datetime.datetime.utcfromtimestamp(0)  # fix(clean): merge with similar code for sequence viewer
                        # fix(clean): use some sort of unzip function
                        timestamps = [(rr.timestamp.replace(tzinfo=None) - epoch).total_seconds() for rr in resource_revisions]
                        values = [rr.data.decode() for rr in resource_revisions]
                        units = json.loads(r.system_attributes).get('units', None)
                        return {'name': r.name, 'path': resource_path, 'units': units, 'timestamps': timestamps, 'values': values}

                # if no filter assume just want current value
                # fix(later): should instead provide all values and have a separate way to get more recent value?
                else:
                    rev = request.values.get('rev')
                    if rev:
                        rev = int(rev)  # fix(soon): save int conversion
                    result = make_response(read_resource(r, revision_id=rev))
                    data_type = json.loads(r.system_attributes)['data_type']
                    if data_type == Resource.IMAGE_SEQUENCE:
                        result.headers['Content-Type'] = 'image/jpeg'
                    else:
                        result.headers['Content-Type'] = 'text/plain'

            # if file, return file data/contents
            else:
                data = read_resource(r)
                if not data:
                    abort(404)
                name = r.name
                if request.values.get('convert_to', request.values.get('convertTo', '')) == 'xls' and r.name.endswith('csv'):
                    data = convert_csv_to_xls(data)
                    name = name.replace('csv', 'xls')
                result = make_response(data)
                result.headers['Content-Type'] = 'application/octet-stream'
                if request.values.get('download', False):
                    result.headers['Content-Disposition'] = 'attachment; filename=' + name
        return result

    # delete a resource
    def delete(self, resource_path):
        r = find_resource('/' + resource_path)
        if not r:
            abort(404)  # fix(later): revisit to avoid leaking file existance
        if access_level(r.query_permissions()) < ACCESS_LEVEL_WRITE:<commit_msg>handle the case of requesting the latest value of sequence that has no values<commit_after>import json
import base64
import hashlib  # fix(clean): remove?
import zipfile
import datetime
from io import BytesIO


# external imports
from flask import request, abort, make_response
from sqlalchemy import not_
from sqlalchemy.orm.exc import NoResultFound
from flask_restful import Resource as ApiResource
from flask_login import current_user


# internal imports
from main.app import db
from main.users.models import User
from main.users.permissions import access_level, ACCESS_LEVEL_READ, ACCESS_LEVEL_WRITE
from main.util import parse_json_datetime
from main.resources.models import Resource, ResourceRevision, ResourceView, ControllerStatus, Thumbnail
from main.resources.resource_util import find_resource, read_resource, add_resource_revision, _create_file, update_sequence_value, \
    resource_type_number, _create_folders, create_sequence
from main.resources.file_conversion import convert_csv_to_xls, convert_xls_to_csv, convert_new_lines, compute_thumbnail
from main.users.auth import find_key  # fix(clean): remove?


class ResourceRecord(ApiResource):

    # get the current value or meta data of a resource
    def get(self, resource_path):
        resource_path = '/' + resource_path  # we always want resource paths to start with leading slash
        args = request.values
        result = {}

        # handle case of controller requesting about self
        if resource_path == '/self':
            if request.authorization:
                key = find_key(request.authorization.password)
            else:
                key = None
            if key and key.access_as_controller_id:
                try:
                    r = Resource.query.filter(Resource.id == key.access_as_controller_id).one()
                    resource_path = r.path()
                except NoResultFound:
                    abort(404)
            else:
                abort(403)

        # look up the resource record
        else:
            r = find_resource(resource_path)
            if not r:
                abort(404)  # fix(later): revisit to avoid leaking file existance
            if access_level(r.query_permissions()) < ACCESS_LEVEL_READ:
                abort(403)

        # if request meta-data
        if request.values.get('meta', False):
            result = r.as_dict(extended=True)
            result['path'] = resource_path

        # if request data
        else:

            # if folder, return contents list or zip of collection of files
            if 10 <= r.type < 20:

                # multi-file download
                if 'ids' in args and args.get('download', False):
                    ids = args['ids'].split(',')
                    return batch_download(r, ids)

                # contents list
                else:
                    recursive = request.values.get('recursive', False)
                    type_name = request.values.get('type', None)
                    if type_name:
                        type_number = resource_type_number(type_name)
                    else:
                        type_number = None
                    name_filter = request.values.get('filter', None)
                    extended = request.values.get('extended', False)
                    result = resource_list(r.id, recursive, type_number, name_filter, extended)

            # if sequence, return value(s)
            # fix(later): merge with file case?
            elif r.type == Resource.SEQUENCE:

                # get parameters
                text = request.values.get('text', '')
                download = request.values.get('download', False)
                count = int(request.values.get('count', 1))
                start_timestamp = request.values.get('start_timestamp', '')
                end_timestamp = request.values.get('end_timestamp', '')
                if start_timestamp:
                    try:
                        start_timestamp = parse_json_datetime(start_timestamp)
                    except ValueError:
                        abort(400, 'Invalid date/time.')
                if end_timestamp:
                    try:
                        end_timestamp = parse_json_datetime(end_timestamp)
                    except ValueError:
                        abort(400, 'Invalid date/time.')

                # if filters specified, assume we want a sequence of values
                if text or start_timestamp or end_timestamp or count > 1:

                    # get summary of values
                    if int(request.values.get('summary', False)):
                        return sequence_value_summary(r.id)

                    # get preliminary set of values
                    resource_revisions = ResourceRevision.query.filter(ResourceRevision.resource_id == r.id)

                    # apply filters (if any)
                    if text:
                        resource_revisions = resource_revisions.filter(text in ResourceRevision.data)
                    if start_timestamp:
                        resource_revisions = resource_revisions.filter(ResourceRevision.timestamp >= start_timestamp)
                    if end_timestamp:
                        resource_revisions = resource_revisions.filter(ResourceRevision.timestamp <= end_timestamp)
                    resource_revisions = resource_revisions.order_by('timestamp')
                    if resource_revisions.count() > count:
                        resource_revisions = resource_revisions[-count:]  # fix(later): is there a better/faster way to do this?

                    # return data
                    if download:
                        # timezone = r.root().system_attributes['timezone']  # fix(soon): use this instead of UTC
                        lines = ['utc_timestamp,value\n']
                        for rr in resource_revisions:
                            lines.append('%s,%s\n' % (rr.timestamp.strftime('%Y-%m-%d %H:%M:%S.%f'), rr.data))
                        result = make_response(''.join(lines))
                        result.headers['Content-Type'] = 'application/octet-stream'
                        result.headers['Content-Disposition'] = 'attachment; filename=' + r.name + '.csv'
                        return result
                    else:
                        epoch = datetime.datetime.utcfromtimestamp(0)  # fix(clean): merge with similar code for sequence viewer
                        # fix(clean): use some sort of unzip function
                        timestamps = [(rr.timestamp.replace(tzinfo=None) - epoch).total_seconds() for rr in resource_revisions]
                        values = [rr.data.decode() for rr in resource_revisions]
                        units = json.loads(r.system_attributes).get('units', None)
                        return {'name': r.name, 'path': resource_path, 'units': units, 'timestamps': timestamps, 'values': values}

                # if no filter assume just want current value
                # fix(later): should instead provide all values and have a separate way to get more recent value?
                else:
                    rev = request.values.get('rev')
                    if rev:
                        rev = int(rev)  # fix(soon): safe int conversion
                    value = read_resource(r, revision_id=rev)
                    if value is None:
                        value = ''  # if the sequence doesn't yet have any values, return an empty value (rather than a 400 or 404)
                    result = make_response(value)
                    data_type = json.loads(r.system_attributes)['data_type']
                    if data_type == Resource.IMAGE_SEQUENCE:
                        result.headers['Content-Type'] = 'image/jpeg'
                    else:
                        result.headers['Content-Type'] = 'text/plain'

            # if file, return file data/contents
            else:
                data = read_resource(r)
                if not data:
                    abort(404)
                name = r.name
                if request.values.get('convert_to', request.values.get('convertTo', '')) == 'xls' and r.name.endswith('csv'):
                    data = convert_csv_to_xls(data)
                    name = name.replace('csv', 'xls')
                result = make_response(data)
                result.headers['Content-Type'] = 'application/octet-stream'
                if request.values.get('download', False):
                    result.headers['Content-Disposition'] = 'attachment; filename=' + name
        return result

    # delete a resource
    def delete(self, resource_path):
        r = find_resource('/' + resource_path)
        if not r:
            abort(404)  # fix(later): revisit to avoid leaking file existance
        if access_level(r.query_permissions()) < ACCESS_LEVEL_WRITE:
{'fraction_non_alphanumeric': 0.08413351623228166, 'fraction_numerical': 0.00411522633744856, 'mean_word_length': 1.7381846635367761, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3267791', 'n_tokens_mistral': 4508, 'n_tokens_neox': 4133, 'n_words': 1147}