<commit_before>import user, toolkit, project, ontology, article
from toolkit import cached

def getCodedArticle(db, cjaid):
    data = db.doQuery("select codingjobid, setnr from codingjobs_articles where codingjob_articleid=%i"% cjaid)
    if not data: return None
    cjid, setnr = data[0]
    return Codingjob(db, cjid).getSet(setnr).getArticle(cjaid)
    
def getCodedArticleIdsFromArticleId(db, articleid):
    data = db.doQuery("select codingjob_articleid from codingjobs_articles where articleid=%i"% articleid)
    if not data: return None
    return (row[0] for row in data)
    

class Codingjob(object):

    def __init__(self, db, id):
        self.db = db
        self.id = id
        
    def _getFields(self):
        SQL = """select name, unitschemaid, articleschemaid, params, owner_userid, insertdate, projectid from codingjobs
              where codingjobid=%i"""%self.id
        if self._fields: return
        self._name, ua, aa, self._params, ownerid, insertdate, projectid = self.db.doQuery(SQL)[0]
        self._unitschema = AnnotationSchema(self.db, ua)
        self._artschema = AnnotationSchema(self.db, aa)
        self._owner = self.db.users.getUser(ownerid)
        self._project = project.Project(self.db, projectid)
        self._insertdate = insertdate
        self._fields = True
    _fields = False

    def getLink(self):
        return "codingjobDetails?codingjobid=%i" % self.id

    @property
    def name(self):
        self._getFields()
        return self._name

    @property
    def articleSchema(self):
        self._getFields()
        return self._artschema

    @property
    def unitSchema(self):
        self._getFields()
        return self._unitschema

    @property
    @cached
    def sets(self):
        SQL = "select setnr from codingjobs_sets where codingjobid=%i" % self.id
        return [CodingjobSet(self, setnr) for (setnr,) in  self.db.doQuery(SQL)]

    @property
    def owner(self):
        self._getFields()
        return self._owner
            
    @property
    def insertdate(self):
        self._getFields()
        return self._insertdate

    @property
    def project(self):
        self._getFields()
        return self._project
    
    def getSet(self, setnr):
        for set in self.sets:
            if set.setnr == setnr:
                return set
                
        
    def getNonCodedArticleids(self):
        sql = """
        select articleid from vw_codingjobs_articles_done where codingjobid = %d and has_arrow = 0 and has_artannot = 0 and (irrelevant IS NULL or irrelevant = 0)
        """ % self.id
        
        data = self.db.doQuery(sql)
        aids = [row[0] for row in data]
        # return aids
        result = []
        for aid in aids:
            if aids.count(aid) == 1:
                result.append(aid)
        return result
        

    def findCodedArticle(self, art=None, coder=None, cjaid=None):
        if art and coder:
            for set in self.sets:
                if set.coder == coder:
                    for ca in set.articles:
                        if ca.article == art:
                            return ca
        elif cjaid:
            for set in self.sets:
                for ca in set.articles:
                    if ca.cjaid == cjaid:
                        return ca
        else:
            raise Exception("Need either art and coder OR cjaid")

    def findCodedArticles(self, art=None, coder=None):
        raise Exception([art, coder])
        for set in self.sets:
            if coder and set.coder <> coder: continue
            for ca in set.articles:
                if art and ca.article <> art: continue
                yield ca

    def idname(self):
        return "%i - %s" % (self.id, self.name)

    def cacheValues(self, artannots = False, arrows = False, sentences = False, headlines=True):
        #nog mogelijke optimalisaties:
        # categorize 1x ophalen voor meer objecten
        # niet alle sentences ophalen (maar lastig om verwacht gedrag article.sentences niet te breken...)
        
        # cache article-level values if needed
        # do this first to allow caching (null) values for articles without annotations
        if artannots:
            aavalues = {}
            SQL = self.articleSchema.SQLSelect(extra=["codingjob_articleid"])
            SQL += " WHERE codingjob_articleid in (select codingjob_articleid from codingjobs_articles where codingjobid=%i)" % (self.id)
            for row in self.db.doQuery(SQL):
                cjaid = row[0]
                row = row[1:]
                aavalues[cjaid] = row

        #create articles
        ids = "select articleid from codingjobs_articles where codingjobid=%i" % (self.id,)
        articles = {}
        for a in article.articlesFromDB(self.db, ids, headline=headlines):
            articles[a.id] = a

        # create sets and codedarticles
        SQL = """select setnr, codingjob_articleid, articleid
        from codingjobs_articles where codingjobid=%i""" % self.id
        sets = []
        setsbynr = {}
        articlesperset = {}
        codedarticles = {}<commit_msg>[svn r808] add new set option<commit_after>import user, toolkit, project, ontology, article
from toolkit import cached

def getCodedArticle(db, cjaid):
    data = db.doQuery("select codingjobid, setnr from codingjobs_articles where codingjob_articleid=%i"% cjaid)
    if not data: return None
    cjid, setnr = data[0]
    return Codingjob(db, cjid).getSet(setnr).getArticle(cjaid)
    
def getCodedArticleIdsFromArticleId(db, articleid):
    data = db.doQuery("select codingjob_articleid from codingjobs_articles where articleid=%i"% articleid)
    if not data: return None
    return (row[0] for row in data)
    

class Codingjob(object):

    def __init__(self, db, id):
        self.db = db
        self.id = id
        
    def _getFields(self):
        SQL = """select name, unitschemaid, articleschemaid, params, owner_userid, insertdate, projectid from codingjobs
              where codingjobid=%i"""%self.id
        if self._fields: return
        self._name, ua, aa, self._params, ownerid, insertdate, projectid = self.db.doQuery(SQL)[0]
        self._unitschema = AnnotationSchema(self.db, ua)
        self._artschema = AnnotationSchema(self.db, aa)
        self._owner = self.db.users.getUser(ownerid)
        self._project = project.Project(self.db, projectid)
        self._insertdate = insertdate
        self._fields = True
    _fields = False

    def getLink(self):
        return "codingjobDetails?codingjobid=%i" % self.id

    @property
    def name(self):
        self._getFields()
        return self._name

    @property
    def articleSchema(self):
        self._getFields()
        return self._artschema

    @property
    def unitSchema(self):
        self._getFields()
        return self._unitschema

    @property
    @cached
    def sets(self):
        SQL = "select setnr from codingjobs_sets where codingjobid=%i" % self.id
        return [CodingjobSet(self, setnr) for (setnr,) in  self.db.doQuery(SQL)]

    @property
    def owner(self):
        self._getFields()
        return self._owner
            
    @property
    def insertdate(self):
        self._getFields()
        return self._insertdate

    @property
    def project(self):
        self._getFields()
        return self._project
    
    def getSet(self, setnr):
        for set in self.sets:
            if set.setnr == setnr:
                return set
                
        
    def getNonCodedArticleids(self):
        sql = """
        select articleid
        from vw_codingjobs_articles_done
        where codingjobid = %d and has_arrow = 0 and has_artannot = 0 and (irrelevant IS NULL or irrelevant = 0)
        """ % self.id
        
        data = self.db.doQuery(sql)
        aids = [row[0] for row in data]
        return aids
        # result = []
        # for aid in aids:
            # if aids.count(aid) == 1:
                # result.append(aid)
        # return result
        

    def findCodedArticle(self, art=None, coder=None, cjaid=None):
        if art and coder:
            for set in self.sets:
                if set.coder == coder:
                    for ca in set.articles:
                        if ca.article == art:
                            return ca
        elif cjaid:
            for set in self.sets:
                for ca in set.articles:
                    if ca.cjaid == cjaid:
                        return ca
        else:
            raise Exception("Need either art and coder OR cjaid")

    def findCodedArticles(self, art=None, coder=None):
        raise Exception([art, coder])
        for set in self.sets:
            if coder and set.coder <> coder: continue
            for ca in set.articles:
                if art and ca.article <> art: continue
                yield ca

    def idname(self):
        return "%i - %s" % (self.id, self.name)

        
    def addSet(self, userid, aids):
        setnr = len(self.sets) + 1
        self.db.insert('codingjobs_sets', {'codingjobid':self.id, 'setnr': setnr, 'coder_userid' : userid}, 
                    retrieveIdent = False)
        for aid in aids:
            self.db.insert('codingjobs_articles', {'codingjobid' : self.id, 'setnr' : setnr, 'articleid' : aid}, 
                retrieveIdent = False)
        self.db.commit()
        
        
        
    def cacheValues(self, artannots = False, arrows = False, sentences = False, headlines=True):
        #nog mogelijke optimalisaties:
        # categorize 1x ophalen voor meer objecten
        # niet alle sentences ophalen (maar lastig om verwacht gedrag article.sentences niet te breken...)
        
        # cache article-level values if needed
        # do this first to allow caching (null) values for articles without annotations
        if artannots:
            aavalues = {}
            SQL = self.articleSchema.SQLSelect(extra=["codingjob_articleid"])
            SQL += " WHERE codingjob_articleid in (select codingjob_articleid from codingjobs_articles where codingjobid=%i)" % (self.id)
            for row in self.db.doQuery(SQL):
                cjaid = row[0]
                row = row[1:]
                aavalues[cjaid] = row

        #create articles
        ids = "select articleid from codingjobs_articles where codingjobid=%i" % (self.id,)
        articles = {}
        for a in article.articlesFromDB(self.db, ids, headline=headlines):
            articles[a.id] = a

        # create sets and codedarticles
        SQL = """select setnr, codingjob_articleid, articleid
        from codingjobs_articles where codingjobid=%i""" % self.id
        sets = []
        setsbynr = {}
        articlesperset = {}
        codedarticles = {}
{'fraction_non_alphanumeric': 0.09188034188034189, 'fraction_numerical': 0.0024154589371980675, 'mean_word_length': 2.0069832402234637, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '169303', 'n_tokens_mistral': 3239, 'n_tokens_neox': 3075, 'n_words': 876}