<commit_before>import json
import base64
import os
import logging
import traceback
import time
from datetime import timedelta

from restless.dj import DjangoResource
from restless.resources import skip_prepare

from django.conf.urls import  url
from django.template import Context, Template
try:
    from django.utils.encoding import smart_text
except ImportError:
    from django.utils.encoding import smart_unicode as smart_text

from django.contrib import auth
from django.utils import timezone
from django.db.models import Q

from harvest.models import Job
from tablemanager.models import Publish,Workspace,Input,DataSource
from wmsmanager.models import WmsLayer
from harvest.jobstatemachine import JobStatemachine
from monitor.models import SlaveServer,PublishSyncStatus
from livelayermanager.models import Layer as LiveLayer

from borg_utils.hg_batch_push import try_set_push_owner, try_clear_push_owner, try_push_to_repository
from borg_utils.jobintervals import JobInterval
from borg_utils.borg_config import BorgConfiguration
from borg_utils.resource_status import ResourceStatus,ResourceAction
from harvest.jobstates import Completed

logger = logging.getLogger(__name__)

class BasicHttpAuthMixin(object):
    """
    :py:class:`restless.views.Endpoint` mixin providing user authentication
    based on HTTP Basic authentication.
    """

    def authenticate(self, request):
        if 'HTTP_AUTHORIZATION' in request.META:
            authdata = request.META['HTTP_AUTHORIZATION'].split()
            if len(authdata) == 2 and authdata[0].lower() == "basic":
                try:
                    raw = authdata[1].encode('ascii')
                    auth_parts = base64.b64decode(raw).split(b':')
                except:
                    return
                try:
                    uname, passwd = (smart_text(auth_parts[0]),
                        smart_text(auth_parts[1]))
                except DjangoUnicodeDecodeError:
                    return

                user = auth.authenticate(username=uname, password=<PASSWORD>)
                if user is not None and user.is_active:
                    # We don't user auth.login(request, user) because
                    # may be running without session
                    request.user = user
        return request.user.is_authenticated()


class JobResource(DjangoResource,BasicHttpAuthMixin):
    def is_authenticated(self):
        if self.request.user.is_authenticated():
            return True
        else:
            return self.authenticate(self.request)

    @staticmethod
    def urls():
        return [
            url(r'^jobs/$',JobResource.as_list(),name='create_job'),
        ]
     
    @skip_prepare
    def create(self):
        job_batch_id = JobInterval.Triggered.job_batch_id()
        resp = {"status":True}
        result = None
        for name in self.data.get('publishes') or []:
            resp[name] = {}
            try:
                result = JobStatemachine.create_job_by_name(name,JobInterval.Triggered,job_batch_id)
                if result[0]:
                    resp[name]["status"] = True
                    resp[name]["job_id"] = result[1]
                    resp[name]["message"] = "Succeed"
                else:
                    resp["status"] = False
                    resp[name]["status"] = False
                    resp[name]["message"] = result[1]
            except :
                msg = traceback.format_exc()
                logger.error(msg)
                resp["status"] = False
                resp[name]["status"] = False
                resp[name]["message"] = msg

        return resp

class MetaResource(DjangoResource,BasicHttpAuthMixin):
    def is_authenticated(self):
        if self.request.user.is_authenticated():
            return True
        else:
            return self.authenticate(self.request)

    @staticmethod
    def urls():
        return[
            url(r'^metajobs/$',MetaResource.as_list(),name='publish_meta'),
        ]
     
    @skip_prepare
    def create(self):
        resp = {"status":True}
        result = None
        try_set_push_owner("meta_resource")
        try:
            for layer in self.data.get('layers') or []:
                workspace,name = layer.split(":")
                resp[layer] = {}
                #get the workspace object
                try:
                    try:
                        workspace = Workspace.objects.get(name=workspace)
                    except Workspace.DoesNotExist:
                        #workspace does not exist
                        resp["status"] = False
                        resp[layer]["status"] = False
                        resp[layer]["message"] = "Workspace does not exist.".format(name)
                        continue
                    
                    try:
                        #try to locate it from publishs, and publish the meta data if found
                        pub = Publish.objects.get(workspace=workspace,name=name)
                        try:
                            pub.publish_meta_data()
                            resp[layer]["status"] = True
                            resp[layer]["message"] = "Succeed."
                        except:
                            msg = traceback.format_exc()
                            logger.error(msg)
                            resp["status"] = False
                            resp[layer]["status"] = False
                            resp[layer]["message"] = "Publish meta data failed!{}".format(msg)
                            continue
                    except Publish.DoesNotExist:
                        #not a publish object, try to locate it from live layers, and publish it if found
                        try:
                            livelayer = LiveLayer.objects.filter(datasource__workspace=workspace).get(Q(name=name) | Q(table=name))
                            try:
                                target_status = livelayer.next_status(ResourceAction.PUBLISH)
                                livelayer.status = target_status
                                livelayer.save(update_fields=["status","last_publish_time"])
                                resp[layer]["status"] = True
                                resp[layer]["message"] = "Succeed."
                            except :
                                msg = traceback.format_exc()
                                logger.error(msg)
                                resp["status"] = False
                                resp[layer]["status"] = False
                                resp[layer]["message"] = "Publish live layer failed!{}".format(msg)
                                continue
                        except LiveLayer.DoesNotExist:
                            #not a publish object, try to locate it from wms layers, and publish it if found
                            try:
                                wmslayer = WmsLayer.objects.get(server__workspace=workspace,kmi_name=name)
                                try:
                                    target_status = wmslayer.next_status(ResourceAction.PUBLISH)
                                    wmslayer.status = target_status
                                    wmslayer.save(update_fields=["status","last_publish_time"])
                                    resp[layer]["status"] = True
                                    resp[layer]["message"] = "Succeed."
                                except:
                                    msg = traceback.format_exc()
                                    logger.error(msg)
                                    resp["status"] = False
                                    resp[layer]["status"] = False
                                    resp[layer]["message"] = "Publish wms layer failed!{}".format(msg)
                                    continue
                            except WmsLayer.DoesNotExist:
                                #layer does not exist,
                                resp["status"] = False
                                resp[layer]["status"] = False
                                resp[layer]["message"] = "Does not exist.".format(name)
                                continue
                except :
                    msg = traceback.format_exc()
                    logger.error(msg)
                    resp["status"] = False
                    resp[layer]["status"] = False
                    resp[layer]["message"] = msg
                    continue
<commit_msg>Name is not unqiue in Workspace model<commit_after>import json
import base64
import os
import logging
import traceback
import time
from datetime import timedelta

from restless.dj import DjangoResource
from restless.resources import skip_prepare

from django.conf.urls import  url
from django.template import Context, Template
try:
    from django.utils.encoding import smart_text
except ImportError:
    from django.utils.encoding import smart_unicode as smart_text

from django.contrib import auth
from django.utils import timezone
from django.db.models import Q

from harvest.models import Job
from tablemanager.models import Publish,Workspace,Input,DataSource
from wmsmanager.models import WmsLayer
from harvest.jobstatemachine import JobStatemachine
from monitor.models import SlaveServer,PublishSyncStatus
from livelayermanager.models import Layer as LiveLayer

from borg_utils.hg_batch_push import try_set_push_owner, try_clear_push_owner, try_push_to_repository
from borg_utils.jobintervals import JobInterval
from borg_utils.borg_config import BorgConfiguration
from borg_utils.resource_status import ResourceStatus,ResourceAction
from harvest.jobstates import Completed

logger = logging.getLogger(__name__)

class BasicHttpAuthMixin(object):
    """
    :py:class:`restless.views.Endpoint` mixin providing user authentication
    based on HTTP Basic authentication.
    """

    def authenticate(self, request):
        if 'HTTP_AUTHORIZATION' in request.META:
            authdata = request.META['HTTP_AUTHORIZATION'].split()
            if len(authdata) == 2 and authdata[0].lower() == "basic":
                try:
                    raw = authdata[1].encode('ascii')
                    auth_parts = base64.b64decode(raw).split(b':')
                except:
                    return
                try:
                    uname, passwd = (smart_text(auth_parts[0]),
                        smart_text(auth_parts[1]))
                except DjangoUnicodeDecodeError:
                    return

                user = auth.authenticate(username=uname, password=<PASSWORD>)
                if user is not None and user.is_active:
                    # We don't user auth.login(request, user) because
                    # may be running without session
                    request.user = user
        return request.user.is_authenticated()


class JobResource(DjangoResource,BasicHttpAuthMixin):
    def is_authenticated(self):
        if self.request.user.is_authenticated():
            return True
        else:
            return self.authenticate(self.request)

    @staticmethod
    def urls():
        return [
            url(r'^jobs/$',JobResource.as_list(),name='create_job'),
        ]
     
    @skip_prepare
    def create(self):
        job_batch_id = JobInterval.Triggered.job_batch_id()
        resp = {"status":True}
        result = None
        for name in self.data.get('publishes') or []:
            resp[name] = {}
            try:
                result = JobStatemachine.create_job_by_name(name,JobInterval.Triggered,job_batch_id)
                if result[0]:
                    resp[name]["status"] = True
                    resp[name]["job_id"] = result[1]
                    resp[name]["message"] = "Succeed"
                else:
                    resp["status"] = False
                    resp[name]["status"] = False
                    resp[name]["message"] = result[1]
            except :
                msg = traceback.format_exc()
                logger.error(msg)
                resp["status"] = False
                resp[name]["status"] = False
                resp[name]["message"] = msg

        return resp

class MetaResource(DjangoResource,BasicHttpAuthMixin):
    def is_authenticated(self):
        if self.request.user.is_authenticated():
            return True
        else:
            return self.authenticate(self.request)

    @staticmethod
    def urls():
        return[
            url(r'^metajobs/$',MetaResource.as_list(),name='publish_meta'),
        ]
     
    @skip_prepare
    def create(self):
        resp = {"status":True}
        result = None
        try_set_push_owner("meta_resource")
        try:
            for layer in self.data.get('layers') or []:
                workspace,name = layer.split(":")
                resp[layer] = {}
                #get the workspace object
                try:
                    workspaces = Workspace.objects.filter(name=workspace)
                    if not len(workspaces):
                        #workspace does not exist
                        resp["status"] = False
                        resp[layer]["status"] = False
                        resp[layer]["message"] = "Workspace does not exist.".format(name)
                        continue
                    
                    try:
                        #try to locate it from publishs, and publish the meta data if found
                        pub = Publish.objects.get(workspace__in=workspaces,name=name)
                        try:
                            pub.publish_meta_data()
                            resp[layer]["status"] = True
                            resp[layer]["message"] = "Succeed."
                        except:
                            msg = traceback.format_exc()
                            logger.error(msg)
                            resp["status"] = False
                            resp[layer]["status"] = False
                            resp[layer]["message"] = "Publish meta data failed!{}".format(msg)
                            continue
                    except Publish.DoesNotExist:
                        #not a publish object, try to locate it from live layers, and publish it if found
                        try:
                            livelayer = LiveLayer.objects.filter(datasource__workspace__in=workspaces).get(Q(name=name) | Q(table=name))
                            try:
                                target_status = livelayer.next_status(ResourceAction.PUBLISH)
                                livelayer.status = target_status
                                livelayer.save(update_fields=["status","last_publish_time"])
                                resp[layer]["status"] = True
                                resp[layer]["message"] = "Succeed."
                            except :
                                msg = traceback.format_exc()
                                logger.error(msg)
                                resp["status"] = False
                                resp[layer]["status"] = False
                                resp[layer]["message"] = "Publish live layer failed!{}".format(msg)
                                continue
                        except LiveLayer.DoesNotExist:
                            #not a publish object, try to locate it from wms layers, and publish it if found
                            try:
                                wmslayer = WmsLayer.objects.get(server__workspace__in=workspaces,kmi_name=name)
                                try:
                                    target_status = wmslayer.next_status(ResourceAction.PUBLISH)
                                    wmslayer.status = target_status
                                    wmslayer.save(update_fields=["status","last_publish_time"])
                                    resp[layer]["status"] = True
                                    resp[layer]["message"] = "Succeed."
                                except:
                                    msg = traceback.format_exc()
                                    logger.error(msg)
                                    resp["status"] = False
                                    resp[layer]["status"] = False
                                    resp[layer]["message"] = "Publish wms layer failed!{}".format(msg)
                                    continue
                            except WmsLayer.DoesNotExist:
                                #layer does not exist,
                                resp["status"] = False
                                resp[layer]["status"] = False
                                resp[layer]["message"] = "Does not exist.".format(name)
                                continue
                except :
                    msg = traceback.format_exc()
                    logger.error(msg)
                    resp["status"] = False
                    resp[layer]["status"] = False
                    resp[layer]["message"] = msg
                    continue

{'fraction_non_alphanumeric': 0.08412800561633417, 'fraction_numerical': 0.0016380974667992746, 'mean_word_length': 1.388097233864208, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1320681', 'n_tokens_mistral': 4210, 'n_tokens_neox': 3969, 'n_words': 898}