<commit_before>################################################################################
#
# (C) University of Southampton IT Innovation Centre, 2011
#
# Copyright in this library belongs to the University of Southampton
# University Road, Highfield, Southampton, UK, SO17 1BJ
#
# This software may not be used, sold, licensed, transferred, copied
# or reproduced in whole or in part in any manner or form or in or
# on any media by any person other than in accordance with the terms
# of the Licence Agreement supplied with the software, or otherwise
# without the prior written consent of the copyright owners.
#
# This software is distributed WITHOUT ANY WARRANTY, without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
# PURPOSE, except where stated in the Licence Agreement supplied with
# the software.
#
#    Created By :             <NAME>
#    Created Date :           2011-04-28
#    Created for Project :    Postmark
#
################################################################################
#
# Dependencies : none
#
################################################################################

from mserve.dataservice.models import *
from mserve.webdav.models import *
import logging
import datetime
import time
import os.path
import os
import urllib2
from copy import deepcopy
from datetime import datetime,timedelta,tzinfo
from django.http import *
from django.shortcuts import render_to_response
from django.template import RequestContext
from lxml import etree

def webdav(request,id):

    try:
        logging.info("%s id:%s %s" % (request.method, id, request.path_info));

        response = None
        try:
            dataservice = DataService.objects.get(id=id)
            dav_server = DavServer(dataservice,id)
            response = dav_server.handle(request)
        except DataService.DoesNotExist:
            #logging.info("%s Data Service %s doesnt exists" % (request.method, id));
            pass

        try:
            auth = Auth.objects.get(id=id)
            import dataservice.utils as utils
            base = utils.get_base_for_auth(auth)
            if utils.is_service(base):
                dataservice = DataService.objects.get(id=auth.base.id)
                dav_server = DavServer(dataservice,id)
                #logging.info("%s Auth:%s %s" % (request.method, dataservice, request.path_info));
                response = dav_server.handle(request)
        except Auth.DoesNotExist:
            pass

        #if response == None:
        #    logging.info("Response NOT FOUND")
        #    return HttpResponseNotFound()

        #logging.info("Response for %s : %s " % (request.method,response.status_code))

        if response['Content-Type'] == 'text/xml':
            pass#logging.info(response.content)
        else:
            pass#logging.info('<non-xml response data> %s ' % (response['Content-Type']))
        return response
    except Exception as e:
        logging.exception(e)

# Chunk Size - 50Mb
chunk_size=1024*1024*50

def fbuffer(f, length , chunk_size=chunk_size):

    #logging.info("reading %s in chunks of  %s "% (length,chunk_size))

    to_read = int(length)
    while to_read > 0 :
        chunk = f.read(chunk_size)
        to_read = to_read - chunk_size
        #logging.info("read chunk - %s to go "% (to_read))
        if not chunk:
            break
        yield chunk

class DavServer(object):

    SUPPORTED_PROPERTIES = [
        'creationdate',
        'displayname',
        'getcontentlength',
        'getcontenttype',
        'getetag',
        'getlastmodified',
        'resourcetype'
    ]

    def __init__(self,service,id):
        self.service = service
        self.id = id

    def handle(self, request):
        if request.method == 'COPY':
            return self._handle_copy(request)
        elif request.method == 'DELETE':
            return self._handle_delete(request)
        elif request.method == 'GET':
            return self._handle_get(request)
        elif request.method == 'HEAD':
            return self._handle_head(request)
        elif request.method == 'OPTIONS':
            return self._handle_options(request)
        elif request.method == 'MKCOL':
            return self._handle_mkcol(request)
        elif request.method == 'MOVE':
            return self._handle_move(request)
        elif request.method == 'POST':
            return self._handle_post(request)
        elif request.method == 'PROPFIND':
            return self._handle_propfind(request)
        elif request.method == 'PROPPATCH':
            return self._handle_proppatch(request)
        elif request.method == 'PUT':
            return self._handle_put(request)
        else:
            return HttpResponseBadRequest()

    def _handle_copy(self, request):
        # NOTE This assumes same-host destination
        # TODO Handle other host destination to allow remote copy
        dest_path = request.META['HTTP_DESTINATION'][
            request.META['HTTP_DESTINATION'].find(
                request.META['HTTP_HOST'])+len(request.META['HTTP_HOST']):]

        overwrite = True
        if request.META.has_key('HTTP_OVERWRITE') and request.META['HTTP_OVERWRITE'] == 'F':
            overwrite = False

        if dest_path == request.path_info and overwrite == False:
            return HttpResponseForbidden()

        return self.__do_copy(request, self.service, dest_path, overwrite)

    def _handle_delete(self, request):
        # TODO : Handle Recovery undelete
        logging.info("DELETE on %s" % self.service)

        request_uri = request.META['REQUEST_URI']
        if request_uri.find('#') != -1:
            logging.info("Bad Request, URI contains fragment %s " %request.META['REQUEST_URI'])
            return HttpResponseBadRequest()

        isFo,isFi,object = _get_resource_for_path(request.path_info,self.service,self.id)

        if isFo or isFi:
            object.delete()
            return HttpResponseNoContent()

        return HttpResponseNotFound()


    def _handle_get(self, request):
        logging.info("GET on %s" % self.service)
        response = self._handle_head(request)
        if response.status_code != 404:

            isFo,isFi,object = _get_resource_for_path(request.path_info,self.service,self.id)

            if isFi:
                return object.do("GET","file")

        else:
            return HttpResponseNotFound()

    def _handle_head(self, request):
        # TODO Configure response headers for HEAD request
        isFo,isFi,object = _get_resource_for_path(request.path_info,self.service,self.id)
        response = HttpResponse()
        if isFi:
            response = HttpResponse(mimetype=object.mimetype)
            response['Content-Length'] = object.file.size
            fstat = os.stat(object.file.path)
            response['Last-Modified'] = self.__get_localized_datetime(fstat.st_mtime).strftime("%a, %d %b %Y %H:%M:%S %Z")
            response['ETag'] = "%s-%s" % (hex(int(fstat.st_mtime))[2:], fstat.st_size)
            return response
        elif isFo:
            response['Content-Length'] = "4096"
            return response
        elif _get_path(request,self.id) == '/' :
            return response
        else:
            return HttpResponseNotFound()

    def _handle_mkcol(self, request):
        logging.info("MKCOL on %s" % self.service)

        # We don't support bodies for MKCOL requests
        if hasattr(request,"raw_post_data") and request.raw_post_data != '':
            return HttpResponseUnsupportedMediaType()

        isFo,isFi,object = _get_resource_for_path(request.path_info,self.service,self.id)

        if isFi:
            return HttpResponseMethodNotAllowed()

        if isFo:
            return HttpResponseMethodNotAllowed()


        is_folder,ancestors,name,fname = _get_path_info_request(request,self.id)

        foldername = ancestors.pop()
        
        ancestors_exist,parentfolder = self.__ancestors_exist(self.service, ancestors)

        if not ancestors_exist:
            return HttpResponseConflict()

        try:
            MFolder.objects.get(name=foldername,service=self.service,parent=parentfolder)
            return HttpResponseMethodNotAllowed()
        except MFolder.DoesNotExist:

            try:
                existingmfile = MFile.objects.get(service=self.service,name=foldername,folder=parentfolder)
                return HttpResponseMethodNotAllowed()
            except MFile.DoesNotExist:
                pass

            mfolder = MFolder(name=foldername,service=self.service,parent=parentfolder)
            mfolder.save()
            r = HttpResponseCreated()
            return r
            

    def _handle_move(self, request):
        logging.info("MOVE on %s" % self.service)

        # NOTE This assumes same-host destination
        # TODO Handle other host destination to allow remote move
        dest_path = request.META['HTTP_DESTINATION'][
            request.META['HTTP_DESTINATION'].find(
                request.META['HTTP_HOST'])+len(request.META['HTTP_HOST']):]

        overwrite = True
        if request.META['HTTP_OVERWRITE'] == 'F':
            overwrite = False

        # Move file(s)
        response = self.__do_move(request, self.service, dest_path, overwrite)

        return response

    def _handle_options(self, request):
        logging.info("OPTIONS on %s" % self.service)
        response = MultiValueHttpResponse()
        response['Content-Length'] = 0
        response.add_header_value('DAV', '1')
        #response.add_header_value('DAV', '<http://apache.org/dav/propset/fs/1>')

        response['MS-Author-Via'] = "DAV"
        
        isFo,isFi,object = _get_resource_for_path(request.path_info,self.service,self.id)

        if isFi:
            response['Allow'] = "COPY, DELETE, GET, HEAD, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT"
            response['Content-Type'] = object.mimetype
        elif isFo:
            response['Content-Type'] = 'httpd/unix-directory'
            response['Allow'] = "COPY, DELETE, GET, HEAD, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH"
        else:
            response['Content-Type'] = 'httpd/unix-directory'
            response['Allow'] = "COPY, DELETE, GET, HEAD, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH"

        return response

    def _handle_post(self, request):
        logging.info("POST on %s" % self.service)
        # POST handling is actually the same as PUT for a non-existent resource
        return self._handle_put(request)

    def _handle_propfind(self, request):
        logging.info("PROPFIND on %s" % self.service)
        # Handle Depth header
        depth = '1'
        if request.META.has_key('HTTP_DEPTH'):
            depth = request.META['HTTP_DEPTH']


        # First, find out what properties were requested
        props = []

        if hasattr(request,"raw_post_data") and request.raw_post_data != '':
            try:
                xmltree = etree.fromstring(request.raw_post_data)
                allprop_nodes = xmltree.xpath('*[local-name() = "allprop"]')
                if allprop_nodes:
                    for p in self.SUPPORTED_PROPERTIES:
                        props.append(DavProperty(name=p, ns='DAV:'))
                prop_nodes = xmltree.xpath('*[local-name() = "prop"]/*')
                for p in prop_nodes:
                    name = p.xpath('local-name()')
                    ns = p.xpath('namespace-uri()')
                    props.append(DavProperty(name=name, ns=ns))
            except:
                # Request body not valid XML
                return HttpResponseBadRequest()
        else:
            for p in self.SUPPORTED_PROPERTIES:
                props.append(DavProperty(name=p, ns='DAV:'))


        # Next, get all the files that we want to return properties for

        files = []
        mfiles = []

        isFo,isFi,object = _get_resource_for_path(request.path_info,self.service,self.id)
        real_path = request.path_info.split(self.id)[-1:][0]

        if isFi:
            finfo = self.__get_file_info(request, self.service, object, props=props)
            if finfo:
                files.append(finfo)

        elif isFo :
            finfo = self.__get_folder_info(request, self.service, object, props=props)
            files.append(finfo)

            if depth != '0':

                mfiles.extend(list(object.mfile_set.all()))
                for mfile in mfiles:
                    finfo = self.__get_file_info(request, self.service, mfile, props=props)
                    if finfo:
                        files.append(finfo)

                folders = list(MFolder.objects.filter(parent=object,service=self.service))
                for mfolder in folders:
                    finfo = self.__get_folder_info(request, self.service, mfolder, props=props)
                    files.append(finfo)
        elif real_path == '' or real_path == '/':
            finfo = self.__get_service_info( request, self.service, props)
            files.append(finfo)

            if depth != '0':
                service_mfiles = list(MFile.objects.filter(folder=None,service=self.service))
                for mfile in service_mfiles:
                    finfo = self.__get_file_info(request, self.service, mfile, props=props)
                    if finfo:
                        files.append(finfo)

                folders = list(MFolder.objects.filter(parent=None,service=self.service))
                for mfolder in folders:
                    finfo = self.__get_folder_info(request, self.service, mfolder, props=props)
                    files.append(finfo)
        else:
            return HttpResponseNotFound()
        
        # Finally, build the response using a django template
        response = render_to_response("multistatus.djt.xml",
            {'files': files, 'props': props},
            context_instance=RequestContext(request),
            mimetype='text/xml')


        if response.status_code == 200:
            response.status_code = 207
        return response

    def _handle_proppatch(self, request):
        logging.info("PROPPATCH on %s" % self.service)

        props = []
        if request.raw_post_data != '':
            try:
                xmltree = etree.fromstring(request.raw_post_data)
                set_prop_nodes = xmltree.xpath('*[local-name() = "set" or local-name() = "remove"]/*[local-name() = "prop"]/*')
                for p in set_prop_nodes:
                    ns = p.xpath('namespace-uri()')
                    name = p.xpath('local-name()')
                    # TODO: Handle other than text in value
                    value = ''.join(p.xpath('text()'))
                    action = p.xpath('local-name(../..)')
                    props.append(DavProperty(name=name, value=value, ns=ns, action=action))
            except:
                # Request body not valid XML
                return HttpResponseBadRequest()

        path = request.path_info
        # Set/remove properties as requested
        for p in props:
            if p.action == 'set':
                logging.info("PROPPATCH set '%s' on %s" % (p,path))
                self.__set_property(path=path, property=p)
            elif p.action == 'remove':
                logging.info("PROPPATCH remove '%s' on %s" % (p,path))
                self.__remove_property(path=path, property=p)
                p.removed = True


        files = []

        isFo,isFi,object = _get_resource_for_path(path,self.service,self.id)
        
        if isFi:
            finfo = self.__get_file_info(request, self.service, object, props=props)
            if finfo:
                files.append(finfo)

        response = render_to_response("multistatus.djt.xml", {'files': files, 'props': props},
            context_instance=RequestContext(request), mimetype='text/xml')

        # Default response is 200 for django
        if response.status_code == 200:
            # Needs to be 207 for multistatus response
            response.status_code = 207

        return response

    def __remove_property(self, path=None, property=None):
        if path and property:

            isFo,isFi,object = _get_resource_for_path(path,self.service,self.id)

            if isFi or isFo:
                try:
                    existing_dav_property = WebDavProperty.objects.get(base=object,name=property.name,ns=property.namespace)
                    existing_dav_property.delete()
                    
                except WebDavProperty.DoesNotExist:
                    logging.info("PROPPATCH Asked to remove non-existant property '%s' on %s" % (property,path))
                    pass

    def __set_property(self, path=None, property=None):
        if path and property:

            isFo,isFi,object = _get_resource_for_path(path,self.service,self.id)

            if isFi or isFo:
                try:
                    existing_dav_property = WebDavProperty.objects.get(base=object,name=property.name,ns=property.namespace)
                    existing_dav_property.set_value(property.value)
                    existing_dav_property.save()

                except WebDavProperty.DoesNotExist:
                    new_dav_property = WebDavProperty(base=object,name=property.name,ns=property.namespace)
                    new_dav_property.set_value(property.value)
                    new_dav_property.save()

    def _handle_put(self, request):
        logging.info("PUT on %s" % self.service)
        now = datetime.datetime.now()
        logging.info("PUT process started %s" % now)
        response = self.__handle_upload_service(request)
        then = datetime.datetime.now()
        logging.info("PUT process finished %s" % then)
        td = then - now
        secs = (td.microseconds + (td.seconds + td.days * 24.0 * 3600.0) * 10.0**6.0) / 10.0**6.0
        logging.info("PUT process took %s" % secs)
        return response

    def __mfile_exists(self, service, ancestors, filename):
        # Check Folders exists

        folder = None
        try:
            for foldername in ancestors:
                if ancestors[0] == foldername:
                    folder = service.mfolder_set.get(name=foldername)
                else:
                    folder = folder.mfolder_set.get(name=foldername)

        except MFolder.DoesNotExist:
            return False,None

        mfile = None
        try:<commit_msg>Updated webdav debug<commit_after>################################################################################
#
# (C) University of Southampton IT Innovation Centre, 2011
#
# Copyright in this library belongs to the University of Southampton
# University Road, Highfield, Southampton, UK, SO17 1BJ
#
# This software may not be used, sold, licensed, transferred, copied
# or reproduced in whole or in part in any manner or form or in or
# on any media by any person other than in accordance with the terms
# of the Licence Agreement supplied with the software, or otherwise
# without the prior written consent of the copyright owners.
#
# This software is distributed WITHOUT ANY WARRANTY, without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
# PURPOSE, except where stated in the Licence Agreement supplied with
# the software.
#
#    Created By :             <NAME>
#    Created Date :           2011-04-28
#    Created for Project :    Postmark
#
################################################################################
#
# Dependencies : none
#
################################################################################

from mserve.dataservice.models import *
from mserve.webdav.models import *
import logging
import datetime
import time
import os.path
import os
import urllib2
from copy import deepcopy
from datetime import datetime,timedelta,tzinfo
from django.http import *
from django.shortcuts import render_to_response
from django.template import RequestContext
from lxml import etree

def webdav(request,id):

    try:
        logging.info("%s id:%s %s" % (request.method, id, request.path_info));

        response = None
        try:
            dataservice = DataService.objects.get(id=id)
            dav_server = DavServer(dataservice,id)
            response = dav_server.handle(request)
        except DataService.DoesNotExist:
            #logging.info("%s Data Service %s doesnt exists" % (request.method, id));
            pass

        try:
            auth = Auth.objects.get(id=id)
            import dataservice.utils as utils
            base = utils.get_base_for_auth(auth)
            if utils.is_service(base):
                dataservice = DataService.objects.get(id=auth.base.id)
                dav_server = DavServer(dataservice,id)
                #logging.info("%s Auth:%s %s" % (request.method, dataservice, request.path_info));
                response = dav_server.handle(request)
        except Auth.DoesNotExist:
            pass

        #if response == None:
        #    logging.info("Response NOT FOUND")
        #    return HttpResponseNotFound()

        #logging.info("Response for %s : %s " % (request.method,response.status_code))

        if response['Content-Type'] == 'text/xml':
            pass#logging.info(response.content)
        else:
            pass#logging.info('<non-xml response data> %s ' % (response['Content-Type']))
        return response
    except Exception as e:
        logging.exception(e)

# Chunk Size - 50Mb
chunk_size=1024*1024*50

def fbuffer(f, length , chunk_size=chunk_size):

    #logging.info("reading %s in chunks of  %s "% (length,chunk_size))

    to_read = int(length)
    while to_read > 0 :
        chunk = f.read(chunk_size)
        to_read = to_read - chunk_size
        #logging.info("read chunk - %s to go "% (to_read))
        if not chunk:
            break
        yield chunk

class DavServer(object):

    SUPPORTED_PROPERTIES = [
        'creationdate',
        'displayname',
        'getcontentlength',
        'getcontenttype',
        'getetag',
        'getlastmodified',
        'resourcetype'
    ]

    def __init__(self,service,id):
        self.service = service
        self.id = id

    def handle(self, request):
        if request.method == 'COPY':
            return self._handle_copy(request)
        elif request.method == 'DELETE':
            return self._handle_delete(request)
        elif request.method == 'GET':
            return self._handle_get(request)
        elif request.method == 'HEAD':
            return self._handle_head(request)
        elif request.method == 'OPTIONS':
            return self._handle_options(request)
        elif request.method == 'MKCOL':
            return self._handle_mkcol(request)
        elif request.method == 'MOVE':
            return self._handle_move(request)
        elif request.method == 'POST':
            return self._handle_post(request)
        elif request.method == 'PROPFIND':
            return self._handle_propfind(request)
        elif request.method == 'PROPPATCH':
            return self._handle_proppatch(request)
        elif request.method == 'PUT':
            return self._handle_put(request)
        else:
            return HttpResponseBadRequest()

    def _handle_copy(self, request):
        # NOTE This assumes same-host destination
        # TODO Handle other host destination to allow remote copy
        dest_path = request.META['HTTP_DESTINATION'][
            request.META['HTTP_DESTINATION'].find(
                request.META['HTTP_HOST'])+len(request.META['HTTP_HOST']):]

        overwrite = True
        if request.META.has_key('HTTP_OVERWRITE') and request.META['HTTP_OVERWRITE'] == 'F':
            overwrite = False

        if dest_path == request.path_info and overwrite == False:
            return HttpResponseForbidden()

        return self.__do_copy(request, self.service, dest_path, overwrite)

    def _handle_delete(self, request):
        # TODO : Handle Recovery undelete
        logging.info("DELETE on %s" % self.service)

        request_uri = request.META['REQUEST_URI']
        if request_uri.find('#') != -1:
            logging.info("Bad Request, URI contains fragment %s " %request.META['REQUEST_URI'])
            return HttpResponseBadRequest()

        isFo,isFi,object = _get_resource_for_path(request.path_info,self.service,self.id)

        if isFo or isFi:
            object.delete()
            return HttpResponseNoContent()

        return HttpResponseNotFound()


    def _handle_get(self, request):
        logging.info("GET on %s" % self.service)
        response = self._handle_head(request)
        if response.status_code != 404:

            isFo,isFi,object = _get_resource_for_path(request.path_info,self.service,self.id)

            if isFi:
                return object.do("GET","file")

        else:
            return HttpResponseNotFound()

    def _handle_head(self, request):
        # TODO Configure response headers for HEAD request
        isFo,isFi,object = _get_resource_for_path(request.path_info,self.service,self.id)
        response = HttpResponse()
        if isFi:
            response = HttpResponse(mimetype=object.mimetype)
            response['Content-Length'] = object.file.size
            fstat = os.stat(object.file.path)
            response['Last-Modified'] = self.__get_localized_datetime(fstat.st_mtime).strftime("%a, %d %b %Y %H:%M:%S %Z")
            response['ETag'] = "%s-%s" % (hex(int(fstat.st_mtime))[2:], fstat.st_size)
            return response
        elif isFo:
            response['Content-Length'] = "4096"
            return response
        elif _get_path(request,self.id) == '/' :
            return response
        else:
            return HttpResponseNotFound()

    def _handle_mkcol(self, request):
        logging.info("MKCOL on %s" % self.service)

        # We don't support bodies for MKCOL requests
        if hasattr(request,"raw_post_data") and request.raw_post_data != '':
            return HttpResponseUnsupportedMediaType()

        isFo,isFi,object = _get_resource_for_path(request.path_info,self.service,self.id)

        if isFi:
            return HttpResponseMethodNotAllowed()

        if isFo:
            return HttpResponseMethodNotAllowed()


        is_folder,ancestors,name,fname = _get_path_info_request(request,self.id)

        foldername = ancestors.pop()
        
        ancestors_exist,parentfolder = self.__ancestors_exist(self.service, ancestors)

        if not ancestors_exist:
            return HttpResponseConflict()

        try:
            MFolder.objects.get(name=foldername,service=self.service,parent=parentfolder)
            return HttpResponseMethodNotAllowed()
        except MFolder.DoesNotExist:

            try:
                existingmfile = MFile.objects.get(service=self.service,name=foldername,folder=parentfolder)
                return HttpResponseMethodNotAllowed()
            except MFile.DoesNotExist:
                pass

            mfolder = MFolder(name=foldername,service=self.service,parent=parentfolder)
            mfolder.save()
            r = HttpResponseCreated()
            return r
            

    def _handle_move(self, request):
        logging.info("MOVE on %s" % self.service)

        # NOTE This assumes same-host destination
        # TODO Handle other host destination to allow remote move
        dest_path = request.META['HTTP_DESTINATION'][
            request.META['HTTP_DESTINATION'].find(
                request.META['HTTP_HOST'])+len(request.META['HTTP_HOST']):]

        overwrite = True
        if request.META['HTTP_OVERWRITE'] == 'F':
            overwrite = False

        # Move file(s)
        response = self.__do_move(request, self.service, dest_path, overwrite)

        return response

    def _handle_options(self, request):
        logging.info("OPTIONS on %s" % self.service)
        response = MultiValueHttpResponse()
        response['Content-Length'] = 0
        response.add_header_value('DAV', '1')
        #response.add_header_value('DAV', '<http://apache.org/dav/propset/fs/1>')

        response['MS-Author-Via'] = "DAV"
        
        isFo,isFi,object = _get_resource_for_path(request.path_info,self.service,self.id)

        if isFi:
            response['Allow'] = "COPY, DELETE, GET, HEAD, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT"
            response['Content-Type'] = object.mimetype
        elif isFo:
            response['Content-Type'] = 'httpd/unix-directory'
            response['Allow'] = "COPY, DELETE, GET, HEAD, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH"
        else:
            response['Content-Type'] = 'httpd/unix-directory'
            response['Allow'] = "COPY, DELETE, GET, HEAD, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH"

        return response

    def _handle_post(self, request):
        logging.info("POST on %s" % self.service)
        # POST handling is actually the same as PUT for a non-existent resource
        return self._handle_put(request)

    def _handle_propfind(self, request):
        logging.info("PROPFIND on %s" % self.service)
        # Handle Depth header
        depth = '1'
        if request.META.has_key('HTTP_DEPTH'):
            depth = request.META['HTTP_DEPTH']


        # First, find out what properties were requested
        props = []

        if hasattr(request,"raw_post_data") and request.raw_post_data != '':
            try:
                xmltree = etree.fromstring(request.raw_post_data)
                allprop_nodes = xmltree.xpath('*[local-name() = "allprop"]')
                if allprop_nodes:
                    for p in self.SUPPORTED_PROPERTIES:
                        props.append(DavProperty(name=p, ns='DAV:'))
                prop_nodes = xmltree.xpath('*[local-name() = "prop"]/*')
                for p in prop_nodes:
                    name = p.xpath('local-name()')
                    ns = p.xpath('namespace-uri()')
                    props.append(DavProperty(name=name, ns=ns))
            except:
                # Request body not valid XML
                return HttpResponseBadRequest()
        else:
            for p in self.SUPPORTED_PROPERTIES:
                props.append(DavProperty(name=p, ns='DAV:'))


        # Next, get all the files that we want to return properties for

        files = []
        mfiles = []

        isFo,isFi,object = _get_resource_for_path(request.path_info,self.service,self.id)
        real_path = request.path_info.split(self.id)[-1:][0]

        if isFi:
            finfo = self.__get_file_info(request, self.service, object, props=props)
            if finfo:
                files.append(finfo)

        elif isFo :
            finfo = self.__get_folder_info(request, self.service, object, props=props)
            files.append(finfo)

            if depth != '0':

                mfiles.extend(list(object.mfile_set.all()))
                for mfile in mfiles:
                    finfo = self.__get_file_info(request, self.service, mfile, props=props)
                    if finfo:
                        files.append(finfo)

                folders = list(MFolder.objects.filter(parent=object,service=self.service))
                for mfolder in folders:
                    finfo = self.__get_folder_info(request, self.service, mfolder, props=props)
                    files.append(finfo)
        elif real_path == '' or real_path == '/':
            finfo = self.__get_service_info( request, self.service, props)
            files.append(finfo)

            if depth != '0':
                service_mfiles = list(MFile.objects.filter(folder=None,service=self.service))
                for mfile in service_mfiles:
                    finfo = self.__get_file_info(request, self.service, mfile, props=props)
                    if finfo:
                        files.append(finfo)

                folders = list(MFolder.objects.filter(parent=None,service=self.service))
                for mfolder in folders:
                    finfo = self.__get_folder_info(request, self.service, mfolder, props=props)
                    files.append(finfo)
        else:
            return HttpResponseNotFound()
        
        # Finally, build the response using a django template
        response = render_to_response("multistatus.djt.xml",
            {'files': files, 'props': props},
            context_instance=RequestContext(request),
            mimetype='text/xml')


        if response.status_code == 200:
            response.status_code = 207
        return response

    def _handle_proppatch(self, request):
        logging.info("PROPPATCH on %s" % self.service)

        props = []
        if request.raw_post_data != '':
            try:
                xmltree = etree.fromstring(request.raw_post_data)
                set_prop_nodes = xmltree.xpath('*[local-name() = "set" or local-name() = "remove"]/*[local-name() = "prop"]/*')
                for p in set_prop_nodes:
                    ns = p.xpath('namespace-uri()')
                    name = p.xpath('local-name()')
                    # TODO: Handle other than text in value
                    value = ''.join(p.xpath('text()'))
                    action = p.xpath('local-name(../..)')
                    props.append(DavProperty(name=name, value=value, ns=ns, action=action))
            except:
                # Request body not valid XML
                return HttpResponseBadRequest()

        path = request.path_info
        # Set/remove properties as requested
        for p in props:
            if p.action == 'set':
                logging.info("PROPPATCH set '%s' on %s" % (p,path))
                self.__set_property(path=path, property=p)
            elif p.action == 'remove':
                logging.info("PROPPATCH remove '%s' on %s" % (p,path))
                self.__remove_property(path=path, property=p)
                p.removed = True


        files = []

        isFo,isFi,object = _get_resource_for_path(path,self.service,self.id)
        
        if isFi:
            finfo = self.__get_file_info(request, self.service, object, props=props)
            if finfo:
                files.append(finfo)

        response = render_to_response("multistatus.djt.xml", {'files': files, 'props': props},
            context_instance=RequestContext(request), mimetype='text/xml')

        # Default response is 200 for django
        if response.status_code == 200:
            # Needs to be 207 for multistatus response
            response.status_code = 207

        return response

    def __remove_property(self, path=None, property=None):
        if path and property:

            isFo,isFi,object = _get_resource_for_path(path,self.service,self.id)

            if isFi or isFo:
                try:
                    existing_dav_property = WebDavProperty.objects.get(base=object,name=property.name,ns=property.namespace)
                    existing_dav_property.delete()
                    
                except WebDavProperty.DoesNotExist:
                    logging.info("PROPPATCH Asked to remove non-existant property '%s' on %s" % (property,path))
                    pass

    def __set_property(self, path=None, property=None):
        if path and property:

            isFo,isFi,object = _get_resource_for_path(path,self.service,self.id)

            if isFi or isFo:
                try:
                    existing_dav_property = WebDavProperty.objects.get(base=object,name=property.name,ns=property.namespace)
                    existing_dav_property.set_value(property.value)
                    existing_dav_property.save()

                except WebDavProperty.DoesNotExist:
                    new_dav_property = WebDavProperty(base=object,name=property.name,ns=property.namespace)
                    new_dav_property.set_value(property.value)
                    new_dav_property.save()

    def _handle_put(self, request):
        logging.info("PUT on %s" % self.service)
        now = datetime.now()
        logging.info("PUT process started %s" % now)
        response = self.__handle_upload_service(request)
        then = datetime.now()
        logging.info("PUT process finished %s" % then)
        td = then - now
        secs = (td.microseconds + (td.seconds + td.days * 24.0 * 3600.0) * 10.0**6.0) / 10.0**6.0
        logging.info("PUT process took %s" % secs)
        return response

    def __mfile_exists(self, service, ancestors, filename):
        # Check Folders exists

        folder = None
        try:
            for foldername in ancestors:
                if ancestors[0] == foldername:
                    folder = service.mfolder_set.get(name=foldername)
                else:
                    folder = folder.mfolder_set.get(name=foldername)

        except MFolder.DoesNotExist:
            return False,None

        mfile = None
        try:
{'fraction_non_alphanumeric': 0.11399160232967628, 'fraction_numerical': 0.00449681701205472, 'mean_word_length': 2.1777567358181975, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 24}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1741925', 'n_tokens_mistral': 10649, 'n_tokens_neox': 10043, 'n_words': 2313}