<commit_before>from google.appengine.ext import ndb
from google.appengine.api import taskqueue
from google.appengine.datastore.datastore_query import Cursor
from google.appengine.api import search
from protorpc import messages
from search_helper import tokenize_autocomplete,SEARCH_QUERY_MODEL
from endpoints_proto_datastore.ndb import EndpointsModel
from iomodels.crmengine.tags import Tag,TagSchema
from iomodels.crmengine.opportunitystage import OpportunitystageSchema
from iograph import Node,Edge,InfoNodeListResponse
from iomodels.crmengine.documents import Document,DocumentListResponse
from iomodels.crmengine.notes import Note,TopicListResponse
from iomodels.crmengine.tasks import Task,TaskRequest,TaskListResponse
from iomodels.crmengine.events import Event,EventListResponse
from endpoints_helper import EndpointsHelper
import model
import iomessages
import datetime


class AccountSchema(messages.Message):
    id = messages.StringField(1)
    entityKey = messages.StringField(2)
    name = messages.StringField(3)

class ListRequest(messages.Message):
    limit = messages.IntegerField(1)
    pageToken = messages.StringField(2)

class OpportunityGetRequest(messages.Message):
    id = messages.IntegerField(1,required = True)
    topics = messages.MessageField(ListRequest, 2)
    tasks = messages.MessageField(ListRequest, 3)
    events = messages.MessageField(ListRequest, 4)
    documents = messages.MessageField(ListRequest, 5)

class OpportunityInsertRequest(messages.Message):
    name = messages.StringField(1)
    stage = messages.StringField(3)
    account = messages.StringField(4)
    contact = messages.StringField(5)
    access = messages.StringField(6)
    opportunity_type = messages.StringField(7)
    duration =  messages.IntegerField(8)
    duration_unit  = messages.StringField(9)
    currency = messages.StringField(10)
    amount_per_unit = messages.IntegerField(11)
    amount_total = messages.IntegerField(12)
    closed_date = messages.StringField(13)
    competitor = messages.StringField(14)
    description = messages.StringField(15)
    infonodes = messages.MessageField(iomessages.InfoNodeRequestSchema,16,repeated=True)


class OpportunitySchema(messages.Message):
    id = messages.StringField(1)
    entityKey = messages.StringField(2)
    name = messages.StringField(3)
    source = messages.StringField(4)
    current_stage = messages.MessageField(OpportunitystageSchema,5)
    stages = messages.MessageField(OpportunitystageSchema,6,repeated = True)
    infonodes = messages.MessageField(InfoNodeListResponse,7)
    topics = messages.MessageField(TopicListResponse,8)
    tasks = messages.MessageField(TaskListResponse,9)
    events = messages.MessageField(EventListResponse,10)
    documents = messages.MessageField(DocumentListResponse,11)
    tags = messages.MessageField(TagSchema,12, repeated = True)
    created_at = messages.StringField(13)
    updated_at = messages.StringField(14)
    access = messages.StringField(15)
    closed_date = messages.StringField(16)
    competitor = messages.StringField(17)
    reason_lost = messages.StringField(18)
    description = messages.StringField(19)
    opportunity_type = messages.StringField(20)
    duration =  messages.IntegerField(21)
    duration_unit  = messages.StringField(22)
    currency = messages.StringField(23)
    amount_per_unit = messages.IntegerField(24)
    amount_total = messages.IntegerField(25)
    account = messages.MessageField(AccountSchema,26)

class OpportunityListRequest(messages.Message):
    limit = messages.IntegerField(1)
    pageToken = messages.StringField(2)
    order = messages.StringField(3)
    tags = messages.StringField(4,repeated = True)
    owner = messages.StringField(5)
    stagename = messages.StringField(6)

class OpportunityListResponse(messages.Message):
    items = messages.MessageField(OpportunitySchema, 1, repeated=True)
    nextPageToken = messages.StringField(2)

class OpportunitySearchResult(messages.Message):
    id = messages.StringField(1)
    entityKey = messages.StringField(2)
    title = messages.StringField(3)
    amount = messages.IntegerField(4)
    account_name = messages.StringField(5)

class OpportunitySearchResults(messages.Message):
    items = messages.MessageField(OpportunitySearchResult, 1, repeated=True)
    nextPageToken = messages.StringField(2)

class Opportunity(EndpointsModel):
    owner = ndb.StringProperty()
    organization = ndb.KeyProperty()
    folder = ndb.StringProperty()
    account = ndb.KeyProperty()
    account_name = ndb.StringProperty()
    account_id = ndb.StringProperty()
    contact = ndb.KeyProperty()
    contact_name = ndb.StringProperty()
    contact_id = ndb.StringProperty()
    name = ndb.StringProperty()
    description = ndb.StringProperty()
    industry = ndb.StringProperty()
    closed_date = ndb.DateTimeProperty()
    reason_lost = ndb.StringProperty()
    created_at = ndb.DateTimeProperty(auto_now_add=True)
    updated_at = ndb.DateTimeProperty(auto_now=True)
    created_by = ndb.KeyProperty()
    last_modified_by = ndb.KeyProperty()
    address = ndb.StringProperty()
    stagename = ndb.StringProperty()
    stage_probability = ndb.IntegerProperty()
    competitor = ndb.StringProperty()
    source = ndb.StringProperty()
    opportunity_type = ndb.StringProperty()
    duration =  ndb.IntegerProperty()
    duration_unit  = ndb.StringProperty()
    currency = ndb.StringProperty()
    amount_per_unit = ndb.IntegerProperty()
    amount_total = ndb.IntegerProperty()
    # public or private
    access = ndb.StringProperty()



    def put(self, **kwargs):
        ndb.Model.put(self, **kwargs)
        self.put_index()
        self.set_perm()

    def set_perm(self):
        about_item = str(self.key.id())

        perm = model.Permission(about_kind='Opportunity',
                         about_item=about_item,
                         type = 'user',
                         role = 'owner',
                         value = self.owner)
        perm.put()



    def put_index(self,data=None):
        """ index the element at each"""
        empty_string = lambda x: x if x else ""
        organization = str(self.organization.id())
        title_autocomplete = ','.join(tokenize_autocomplete(self.name))
        if data:
            search_key = ['infos','opportunities','tags','collaborators']
            for key in search_key:
                if key not in data.keys():
                    data[key] = ""
            my_document = search.Document(
            doc_id = str(data['id']),
            fields=[
                search.TextField(name=u'type', value=u'Opportunity'),
                search.TextField(name='organization', value = empty_string(organization) ),
                search.TextField(name='access', value = empty_string(self.access) ),
                search.TextField(name='owner', value = empty_string(self.owner) ),
                search.TextField(name='collaborators', value = data['collaborators']  ),
                search.TextField(name='title', value = empty_string(self.name) ),
                search.TextField(name='stagename', value = empty_string(self.stagename) ),
                search.TextField(name='description', value = empty_string(self.description)),
                search.TextField(name='account_name', value = empty_string(self.account_name)),
                search.NumberField(name='amount_total', value = int(self.amount_total)),
                search.DateField(name='created_at', value = self.created_at),
                search.TextField(name='infos', value= data['infos']),
                search.TextField(name='tags', value= data['tags']),
                search.TextField(name='opportunities', value= data['opportunities']),
                search.TextField(name='title_autocomplete', value = empty_string(title_autocomplete)),
            ])
        else:
            my_document = search.Document(
            doc_id = str(self.key.id()),
            fields=[
                search.TextField(name=u'type', value=u'Opportunity'),
                search.TextField(name='organization', value = empty_string(organization) ),
                search.TextField(name='access', value = empty_string(self.access) ),
                search.TextField(name='owner', value = empty_string(self.owner) ),

                search.TextField(name='title', value = empty_string(self.name) ),
                search.TextField(name='stagename', value = empty_string(self.stagename) ),
                search.TextField(name='description', value = empty_string(self.description)),
                search.TextField(name='account_name', value = empty_string(self.account_name)),
                search.NumberField(name='amount_total', value = int(self.amount_total)),
                #search.DateField(name='closed_date', value = self.closed_date),
                search.DateField(name='created_at', value = self.created_at),
                #search.DateField(name='reason_lost', value = self.reason_lost),
                search.TextField(name='title_autocomplete', value = empty_string(title_autocomplete)),
            ])
        my_index = search.Index(name="GlobalIndex")
        my_index.put(my_document)

    @classmethod
    def get_schema(cls,user_from_email,request):
        opportunity = Opportunity.get_by_id(int(request.id))
        if opportunity is None:
            raise endpoints.NotFoundException('Opportunity not found.')
        if not Node.check_permission( user_from_email, opportunity ):
            raise endpoints.UnauthorizedException('You don\'t have permissions.')
        parents_edge_list = Edge.list(
                                    start_node = opportunity.key,
                                    kind = 'parents'
                                    )
        account_schema = None
        for parent in parents_edge_list['items']:
            if parent.end_node.kind() == 'Account':
                account = parent.end_node.get()
                account_schema = AccountSchema(
                                        id = str( account.key.id() ),
                                        entityKey = account.key.urlsafe(),
                                        name = account.name
                                        )
        #list of tags related to this account
        tag_list = Tag.list_by_parent(opportunity.key)
        # list of infonodes
        infonodes = Node.list_info_nodes(
                                        parent_key = opportunity.key,
                                        request = request
                                        )
        #list of topics related to this account
        topics = None
        if request.topics:
            topics = Note.list_by_parent(
                                        parent_key = opportunity.key,
                                        request = request
                                        )
        tasks = None
        if request.tasks:
            tasks = Task.list_by_parent(
                                        parent_key = opportunity.key,
                                        request = request
                                        )
        events = None
        if request.events:
            events = Event.list_by_parent(
                                        parent_key = opportunity.key,
                                        request = request
                                        )
        documents = None
        if request.documents:
            documents = Document.list_by_parent(
                                        parent_key = opportunity.key,
                                        request = request
                                        )
        opportunity_stage_edges = Edge.list(
                                                start_node = opportunity.key,
                                                kind = 'stages',
                                                limit = 1
                                                )
        current_stage_schema = None
        if len(opportunity_stage_edges['items'])>0:
            current_stage = opportunity_stage_edges['items'][0].end_node.get()
            current_stage_schema = OpportunitystageSchema(
                                                        name=current_stage.name,
                                                        probability= current_stage.probability,
                                                        stage_changed_at=opportunity_stage_edges['items'][0].created_at.isoformat()
                                                        )
        closed_date = None
        if opportunity.closed_date:
            closed_date = opportunity.closed_date.strftime("%Y-%m-%dT%H:%M:00.000")
        opportunity_schema = OpportunitySchema(
                                  id = str( opportunity.key.id() ),
                                  entityKey = opportunity.key.urlsafe(),
                                  access = opportunity.access,
                                  account = account_schema,
                                  name = opportunity.name,
                                  opportunity_type = opportunity.opportunity_type,
                                  duration = opportunity.duration,
                                  duration_unit = opportunity.duration_unit,
                                  amount_per_unit = opportunity.amount_per_unit,
                                  amount_total = opportunity.amount_total,
                                  currency = opportunity.currency,
                                  closed_date = closed_date,
                                  competitor = opportunity.competitor,
                                  reason_lost = opportunity.reason_lost,
                                  description = opportunity.description,
                                  source = opportunity.source,
                                  current_stage = current_stage_schema,
                                  tags = tag_list,
                                  topics = topics,
                                  tasks = tasks,
                                  events = events,
                                  documents = documents,
                                  infonodes = infonodes,
                                  created_at = opportunity.created_at.strftime("%Y-%m-%dT%H:%M:00.000"),
                                  updated_at = opportunity.updated_at.strftime("%Y-%m-%dT%H:%M:00.000")
                                )
        return  opportunity_schema
    @classmethod
    def list(cls,user_from_email,request):
        curs = Cursor(urlsafe=request.pageToken)
        if request.limit:
            limit = int(request.limit)
        else:
            limit = 10
        items = list()
        you_can_loop = True
        count = 0
        while you_can_loop:
            if request.order:
                ascending = True
                if request.order.startswith('-'):
                    order_by = request.order[1:]
                    ascending = False
                else:
                    order_by = request.order
                attr = cls._properties.get(order_by)
                if attr is None:
                    raise AttributeError('Order attribute %s not defined.' % (attr_name,))
                if ascending:
                    opportunities, next_curs, more =  cls.query().filter(cls.organization==user_from_email.organization).order(+attr).fetch_page(limit, start_cursor=curs)
                else:
                    opportunities, next_curs, more = cls.query().filter(cls.organization==user_from_email.organization).order(-attr).fetch_page(limit, start_cursor=curs)
            else:
                opportunities, next_curs, more = cls.query().filter(cls.organization==user_from_email.organization).fetch_page(limit, start_cursor=curs)
            for opportunity in opportunities:
                if count<= limit:
                    is_filtered = True
                    if request.tags and is_filtered:
                        end_node_set = [ndb.Key(urlsafe=tag_key) for tag_key in request.tags]
                        if not Edge.find(start_node=opportunity.key,kind='tags',end_node_set=end_node_set,operation='AND'):
                            is_filtered = False
                    if request.owner and opportunity.owner!=request.owner and is_filtered:
                        is_filtered = False
                    if request.stagename and opportunity.stagename!=request.stagename and is_filtered:
                        is_filtered = False
                    if is_filtered and Node.check_permission( user_from_email, opportunity ):
                        count = count + 1
                        #list of tags related to this opportunity
                        tag_list = Tag.list_by_parent(parent_key = opportunity.key)
                        closed_date = None
                        if opportunity.closed_date:
                            closed_date = opportunity.closed_date.strftime("%Y-%m-%dT%H:%M:00.000")
                        opportunity_schema = OpportunitySchema(
                                  id = str( opportunity.key.id() ),
                                  entityKey = opportunity.key.urlsafe(),
                                  name = opportunity.name,
                                  opportunity_type = opportunity.opportunity_type,
                                  duration = opportunity.duration,
                                  duration_unit = opportunity.duration_unit,
                                  amount_per_unit = opportunity.amount_per_unit,
                                  amount_total = opportunity.amount_total,
                                  currency = opportunity.currency,
                                  closed_date=closed_date,
                                  tags = tag_list,
                                  created_at = opportunity.created_at.strftime("%Y-%m-%dT%H:%M:00.000"),
                                  updated_at = opportunity.updated_at.strftime("%Y-%m-%dT%H:%M:00.000")
                                )
                        items.append(opportunity_schema)
            if (count == limit):
                you_can_loop = False
            if more and next_curs:
                curs = next_curs
            else:
                you_can_loop = False
        if next_curs and more:
            next_curs_url_safe = next_curs.urlsafe()<commit_msg>adding current_stage to opportunities.list api<commit_after>from google.appengine.ext import ndb
from google.appengine.api import taskqueue
from google.appengine.datastore.datastore_query import Cursor
from google.appengine.api import search
from protorpc import messages
from search_helper import tokenize_autocomplete,SEARCH_QUERY_MODEL
from endpoints_proto_datastore.ndb import EndpointsModel
from iomodels.crmengine.tags import Tag,TagSchema
from iomodels.crmengine.opportunitystage import OpportunitystageSchema
from iograph import Node,Edge,InfoNodeListResponse
from iomodels.crmengine.documents import Document,DocumentListResponse
from iomodels.crmengine.notes import Note,TopicListResponse
from iomodels.crmengine.tasks import Task,TaskRequest,TaskListResponse
from iomodels.crmengine.events import Event,EventListResponse
from endpoints_helper import EndpointsHelper
import model
import iomessages
import datetime


class AccountSchema(messages.Message):
    id = messages.StringField(1)
    entityKey = messages.StringField(2)
    name = messages.StringField(3)

class ListRequest(messages.Message):
    limit = messages.IntegerField(1)
    pageToken = messages.StringField(2)

class OpportunityGetRequest(messages.Message):
    id = messages.IntegerField(1,required = True)
    topics = messages.MessageField(ListRequest, 2)
    tasks = messages.MessageField(ListRequest, 3)
    events = messages.MessageField(ListRequest, 4)
    documents = messages.MessageField(ListRequest, 5)

class OpportunityInsertRequest(messages.Message):
    name = messages.StringField(1)
    stage = messages.StringField(3)
    account = messages.StringField(4)
    contact = messages.StringField(5)
    access = messages.StringField(6)
    opportunity_type = messages.StringField(7)
    duration =  messages.IntegerField(8)
    duration_unit  = messages.StringField(9)
    currency = messages.StringField(10)
    amount_per_unit = messages.IntegerField(11)
    amount_total = messages.IntegerField(12)
    closed_date = messages.StringField(13)
    competitor = messages.StringField(14)
    description = messages.StringField(15)
    infonodes = messages.MessageField(iomessages.InfoNodeRequestSchema,16,repeated=True)


class OpportunitySchema(messages.Message):
    id = messages.StringField(1)
    entityKey = messages.StringField(2)
    name = messages.StringField(3)
    source = messages.StringField(4)
    current_stage = messages.MessageField(OpportunitystageSchema,5)
    stages = messages.MessageField(OpportunitystageSchema,6,repeated = True)
    infonodes = messages.MessageField(InfoNodeListResponse,7)
    topics = messages.MessageField(TopicListResponse,8)
    tasks = messages.MessageField(TaskListResponse,9)
    events = messages.MessageField(EventListResponse,10)
    documents = messages.MessageField(DocumentListResponse,11)
    tags = messages.MessageField(TagSchema,12, repeated = True)
    created_at = messages.StringField(13)
    updated_at = messages.StringField(14)
    access = messages.StringField(15)
    closed_date = messages.StringField(16)
    competitor = messages.StringField(17)
    reason_lost = messages.StringField(18)
    description = messages.StringField(19)
    opportunity_type = messages.StringField(20)
    duration =  messages.IntegerField(21)
    duration_unit  = messages.StringField(22)
    currency = messages.StringField(23)
    amount_per_unit = messages.IntegerField(24)
    amount_total = messages.IntegerField(25)
    account = messages.MessageField(AccountSchema,26)

class OpportunityListRequest(messages.Message):
    limit = messages.IntegerField(1)
    pageToken = messages.StringField(2)
    order = messages.StringField(3)
    tags = messages.StringField(4,repeated = True)
    owner = messages.StringField(5)
    stagename = messages.StringField(6)

class OpportunityListResponse(messages.Message):
    items = messages.MessageField(OpportunitySchema, 1, repeated=True)
    nextPageToken = messages.StringField(2)

class OpportunitySearchResult(messages.Message):
    id = messages.StringField(1)
    entityKey = messages.StringField(2)
    title = messages.StringField(3)
    amount = messages.IntegerField(4)
    account_name = messages.StringField(5)

class OpportunitySearchResults(messages.Message):
    items = messages.MessageField(OpportunitySearchResult, 1, repeated=True)
    nextPageToken = messages.StringField(2)

class Opportunity(EndpointsModel):
    owner = ndb.StringProperty()
    organization = ndb.KeyProperty()
    folder = ndb.StringProperty()
    account = ndb.KeyProperty()
    account_name = ndb.StringProperty()
    account_id = ndb.StringProperty()
    contact = ndb.KeyProperty()
    contact_name = ndb.StringProperty()
    contact_id = ndb.StringProperty()
    name = ndb.StringProperty()
    description = ndb.StringProperty()
    industry = ndb.StringProperty()
    closed_date = ndb.DateTimeProperty()
    reason_lost = ndb.StringProperty()
    created_at = ndb.DateTimeProperty(auto_now_add=True)
    updated_at = ndb.DateTimeProperty(auto_now=True)
    created_by = ndb.KeyProperty()
    last_modified_by = ndb.KeyProperty()
    address = ndb.StringProperty()
    stagename = ndb.StringProperty()
    stage_probability = ndb.IntegerProperty()
    competitor = ndb.StringProperty()
    source = ndb.StringProperty()
    opportunity_type = ndb.StringProperty()
    duration =  ndb.IntegerProperty()
    duration_unit  = ndb.StringProperty()
    currency = ndb.StringProperty()
    amount_per_unit = ndb.IntegerProperty()
    amount_total = ndb.IntegerProperty()
    # public or private
    access = ndb.StringProperty()



    def put(self, **kwargs):
        ndb.Model.put(self, **kwargs)
        self.put_index()
        self.set_perm()

    def set_perm(self):
        about_item = str(self.key.id())

        perm = model.Permission(about_kind='Opportunity',
                         about_item=about_item,
                         type = 'user',
                         role = 'owner',
                         value = self.owner)
        perm.put()



    def put_index(self,data=None):
        """ index the element at each"""
        empty_string = lambda x: x if x else ""
        organization = str(self.organization.id())
        title_autocomplete = ','.join(tokenize_autocomplete(self.name))
        if data:
            search_key = ['infos','opportunities','tags','collaborators']
            for key in search_key:
                if key not in data.keys():
                    data[key] = ""
            my_document = search.Document(
            doc_id = str(data['id']),
            fields=[
                search.TextField(name=u'type', value=u'Opportunity'),
                search.TextField(name='organization', value = empty_string(organization) ),
                search.TextField(name='access', value = empty_string(self.access) ),
                search.TextField(name='owner', value = empty_string(self.owner) ),
                search.TextField(name='collaborators', value = data['collaborators']  ),
                search.TextField(name='title', value = empty_string(self.name) ),
                search.TextField(name='stagename', value = empty_string(self.stagename) ),
                search.TextField(name='description', value = empty_string(self.description)),
                search.TextField(name='account_name', value = empty_string(self.account_name)),
                search.NumberField(name='amount_total', value = int(self.amount_total)),
                search.DateField(name='created_at', value = self.created_at),
                search.TextField(name='infos', value= data['infos']),
                search.TextField(name='tags', value= data['tags']),
                search.TextField(name='opportunities', value= data['opportunities']),
                search.TextField(name='title_autocomplete', value = empty_string(title_autocomplete)),
            ])
        else:
            my_document = search.Document(
            doc_id = str(self.key.id()),
            fields=[
                search.TextField(name=u'type', value=u'Opportunity'),
                search.TextField(name='organization', value = empty_string(organization) ),
                search.TextField(name='access', value = empty_string(self.access) ),
                search.TextField(name='owner', value = empty_string(self.owner) ),

                search.TextField(name='title', value = empty_string(self.name) ),
                search.TextField(name='stagename', value = empty_string(self.stagename) ),
                search.TextField(name='description', value = empty_string(self.description)),
                search.TextField(name='account_name', value = empty_string(self.account_name)),
                search.NumberField(name='amount_total', value = int(self.amount_total)),
                #search.DateField(name='closed_date', value = self.closed_date),
                search.DateField(name='created_at', value = self.created_at),
                #search.DateField(name='reason_lost', value = self.reason_lost),
                search.TextField(name='title_autocomplete', value = empty_string(title_autocomplete)),
            ])
        my_index = search.Index(name="GlobalIndex")
        my_index.put(my_document)

    @classmethod
    def get_schema(cls,user_from_email,request):
        opportunity = Opportunity.get_by_id(int(request.id))
        if opportunity is None:
            raise endpoints.NotFoundException('Opportunity not found.')
        if not Node.check_permission( user_from_email, opportunity ):
            raise endpoints.UnauthorizedException('You don\'t have permissions.')
        parents_edge_list = Edge.list(
                                    start_node = opportunity.key,
                                    kind = 'parents'
                                    )
        account_schema = None
        for parent in parents_edge_list['items']:
            if parent.end_node.kind() == 'Account':
                account = parent.end_node.get()
                account_schema = AccountSchema(
                                        id = str( account.key.id() ),
                                        entityKey = account.key.urlsafe(),
                                        name = account.name
                                        )
        #list of tags related to this account
        tag_list = Tag.list_by_parent(opportunity.key)
        # list of infonodes
        infonodes = Node.list_info_nodes(
                                        parent_key = opportunity.key,
                                        request = request
                                        )
        #list of topics related to this account
        topics = None
        if request.topics:
            topics = Note.list_by_parent(
                                        parent_key = opportunity.key,
                                        request = request
                                        )
        tasks = None
        if request.tasks:
            tasks = Task.list_by_parent(
                                        parent_key = opportunity.key,
                                        request = request
                                        )
        events = None
        if request.events:
            events = Event.list_by_parent(
                                        parent_key = opportunity.key,
                                        request = request
                                        )
        documents = None
        if request.documents:
            documents = Document.list_by_parent(
                                        parent_key = opportunity.key,
                                        request = request
                                        )
        opportunity_stage_edges = Edge.list(
                                                start_node = opportunity.key,
                                                kind = 'stages',
                                                limit = 1
                                                )
        current_stage_schema = None
        if len(opportunity_stage_edges['items'])>0:
            current_stage = opportunity_stage_edges['items'][0].end_node.get()
            current_stage_schema = OpportunitystageSchema(
                                                        name=current_stage.name,
                                                        probability= current_stage.probability,
                                                        stage_changed_at=opportunity_stage_edges['items'][0].created_at.isoformat()
                                                        )
        closed_date = None
        if opportunity.closed_date:
            closed_date = opportunity.closed_date.strftime("%Y-%m-%dT%H:%M:00.000")
        opportunity_schema = OpportunitySchema(
                                  id = str( opportunity.key.id() ),
                                  entityKey = opportunity.key.urlsafe(),
                                  access = opportunity.access,
                                  account = account_schema,
                                  name = opportunity.name,
                                  opportunity_type = opportunity.opportunity_type,
                                  duration = opportunity.duration,
                                  duration_unit = opportunity.duration_unit,
                                  amount_per_unit = opportunity.amount_per_unit,
                                  amount_total = opportunity.amount_total,
                                  currency = opportunity.currency,
                                  closed_date = closed_date,
                                  competitor = opportunity.competitor,
                                  reason_lost = opportunity.reason_lost,
                                  description = opportunity.description,
                                  source = opportunity.source,
                                  current_stage = current_stage_schema,
                                  tags = tag_list,
                                  topics = topics,
                                  tasks = tasks,
                                  events = events,
                                  documents = documents,
                                  infonodes = infonodes,
                                  created_at = opportunity.created_at.strftime("%Y-%m-%dT%H:%M:00.000"),
                                  updated_at = opportunity.updated_at.strftime("%Y-%m-%dT%H:%M:00.000")
                                )
        return  opportunity_schema
    @classmethod
    def list(cls,user_from_email,request):
        curs = Cursor(urlsafe=request.pageToken)
        if request.limit:
            limit = int(request.limit)
        else:
            limit = 10
        items = list()
        you_can_loop = True
        count = 0
        while you_can_loop:
            if request.order:
                ascending = True
                if request.order.startswith('-'):
                    order_by = request.order[1:]
                    ascending = False
                else:
                    order_by = request.order
                attr = cls._properties.get(order_by)
                if attr is None:
                    raise AttributeError('Order attribute %s not defined.' % (attr_name,))
                if ascending:
                    opportunities, next_curs, more =  cls.query().filter(cls.organization==user_from_email.organization).order(+attr).fetch_page(limit, start_cursor=curs)
                else:
                    opportunities, next_curs, more = cls.query().filter(cls.organization==user_from_email.organization).order(-attr).fetch_page(limit, start_cursor=curs)
            else:
                opportunities, next_curs, more = cls.query().filter(cls.organization==user_from_email.organization).fetch_page(limit, start_cursor=curs)
            for opportunity in opportunities:
                if count<= limit:
                    is_filtered = True
                    if request.tags and is_filtered:
                        end_node_set = [ndb.Key(urlsafe=tag_key) for tag_key in request.tags]
                        if not Edge.find(start_node=opportunity.key,kind='tags',end_node_set=end_node_set,operation='AND'):
                            is_filtered = False
                    if request.owner and opportunity.owner!=request.owner and is_filtered:
                        is_filtered = False
                    if request.stagename and opportunity.stagename!=request.stagename and is_filtered:
                        is_filtered = False
                    if is_filtered and Node.check_permission( user_from_email, opportunity ):
                        count = count + 1
                        #list of tags related to this opportunity
                        tag_list = Tag.list_by_parent(parent_key = opportunity.key)
                        closed_date = None
                        if opportunity.closed_date:
                            closed_date = opportunity.closed_date.strftime("%Y-%m-%dT%H:%M:00.000")
                        opportunity_stage_edges = Edge.list(
                                                                start_node = opportunity.key,
                                                                kind = 'stages',
                                                                limit = 1
                                                                )
                        current_stage_schema = None
                        if len(opportunity_stage_edges['items'])>0:
                            current_stage = opportunity_stage_edges['items'][0].end_node.get()
                            current_stage_schema = OpportunitystageSchema(
                                                                        name=current_stage.name,
                                                                        probability= current_stage.probability,
                                                                        stage_changed_at=opportunity_stage_edges['items'][0].created_at.isoformat()
                                                                        )
                        opportunity_schema = OpportunitySchema(
                                  id = str( opportunity.key.id() ),
                                  entityKey = opportunity.key.urlsafe(),
                                  name = opportunity.name,
                                  opportunity_type = opportunity.opportunity_type,
                                  duration = opportunity.duration,
                                  duration_unit = opportunity.duration_unit,
                                  amount_per_unit = opportunity.amount_per_unit,
                                  amount_total = opportunity.amount_total,
                                  currency = opportunity.currency,
                                  closed_date=closed_date,
                                  current_stage = current_stage_schema,
                                  tags = tag_list,
                                  created_at = opportunity.created_at.strftime("%Y-%m-%dT%H:%M:00.000"),
                                  updated_at = opportunity.updated_at.strftime("%Y-%m-%dT%H:%M:00.000")
                                )
                        items.append(opportunity_schema)
            if (count == limit):
                you_can_loop = False
            if more and next_curs:
                curs = next_curs
            else:
                you_can_loop = False
        if next_curs and more:
            next_curs_url_safe = next_curs.urlsafe()
{'fraction_non_alphanumeric': 0.08474798281462852, 'fraction_numerical': 0.0068636697055433306, 'mean_word_length': 1.7332808248603753, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 183, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5881921', 'n_tokens_mistral': 9447, 'n_tokens_neox': 8972, 'n_words': 1659}