<commit_before>from amcat.tools.cachable.cachable import Cachable, DBProperty, ForeignKey, DBProperties
from amcat.tools.cachable.latebind import LB
from amcat.tools.cachable import cacher
from amcat.tools import toolkit

import logging; log = logging.getLogger(__name__)

def cacheCodingJobs(codingjobs, values=False, sentences=True, articles=True, schema=True):
    # TODO make caching smarter so this can be dumber
    if schema:
        cacher.cache(codingjobs, unitSchema=[], articleSchema=[])
        schemas = set(toolkit.flatten((job.unitSchema, job.articleSchema) for job in codingjobs))
        cacher.cache(schemas, "location", "isarticleschema", fields=["fieldname", "fieldtype","codebook","keycolumn","labelcolumn", "table"])
    if articles:
        article = {}
        if sentences:
            if values:
                article['sentences'] = ["values"]
            else:
                article['sentences'] = []
        if values:
            article['values'] = []
        cacher.cache(codingjobs, sets=dict(articles=article))

def getCodedArticlesFromCodingjobs(codingjobs, cache=True, cacheSentences=False, cacheValues=True):
    if cache: cacheCodingJobs(codingjobs, sentences=cacheSentences, values=cacheValues)
    for cj in codingjobs:
        for s in cj.sets:
            for ca in s.articles:
                yield ca
                    
def getCodedSentencesFromCodingjobs(codingjobs, cache=True, cacheValues=True):
    if cache: cacheCodingJobs(codingjobs, values=cacheValues)
    for ca in getCodedArticlesFromCodingjobs(codingjobs):
        for cs in ca.sentences:
            yield cs
            
class CodingJob(Cachable):
    __table__ = 'codingjobs'
    __idcolumn__ = 'codingjobid'
    __labelprop__ = 'name'
    name, insertdate = DBProperties(2)

    unitSchema = DBProperty(LB("AnnotationSchema", package="amcat.model.coding"), getcolumn="unitschemaid")
    articleSchema = DBProperty(LB("AnnotationSchema", package="amcat.model.coding"), getcolumn="articleschemaid")
    project = DBProperty(LB("Project"))
    owner = DBProperty(LB("User"), getcolumn="owner_userid")

    sets = ForeignKey(LB("CodingJobSet", sub="coding"), includeOwnID=True)
        
    def getLink(self):
        return "codingjobDetails?codingjobid=%i" % self.id

    def getSet(self, setnr):
        for set in self.sets:
            if set.setnr == setnr:
                return set

    def getAllCodedArticles(self):
        for s in self.sets:
            for ca in s.articles:
                yield ca
    
    def getNonCodedArticleids(self): # articlids that do not have any codings in this job
        sql = """
        select v.articleid
        from vw_codingjobs_articles_done as v
        where v.codingjobid = %d and v.has_arrow = 0 and v.has_artannot = 0 and (v.irrelevant IS NULL or v.irrelevant = 0)
        and not exists (
            select * from vw_codingjobs_articles_done as vw where vw.codingjobid = %d and vw.has_arrow > 0 and vw.has_artannot > 0 and vw.articleid = v.articleid
        )
        """ % (self.id, self.id)
        
        data = self.db.doQuery(sql)<commit_msg>deleted getLink, corrected getcolumn<commit_after>from amcat.tools.cachable.cachable import Cachable, DBProperty, ForeignKey, DBProperties
from amcat.tools.cachable.latebind import LB
from amcat.tools.cachable import cacher
from amcat.tools import toolkit

import logging; log = logging.getLogger(__name__)

def cacheCodingJobs(codingjobs, values=False, sentences=True, articles=True, schema=True):
    # TODO make caching smarter so this can be dumber
    if schema:
        cacher.cache(codingjobs, unitSchema=[], articleSchema=[])
        schemas = set(toolkit.flatten((job.unitSchema, job.articleSchema) for job in codingjobs))
        cacher.cache(schemas, "location", "isarticleschema", fields=["fieldname", "fieldtype","codebook","keycolumn","labelcolumn", "table"])
    if articles:
        article = {}
        if sentences:
            if values:
                article['sentences'] = ["values"]
            else:
                article['sentences'] = []
        if values:
            article['values'] = []
        cacher.cache(codingjobs, sets=dict(articles=article))

def getCodedArticlesFromCodingjobs(codingjobs, cache=True, cacheSentences=False, cacheValues=True):
    if cache: cacheCodingJobs(codingjobs, sentences=cacheSentences, values=cacheValues)
    for cj in codingjobs:
        for s in cj.sets:
            for ca in s.articles:
                yield ca
                    
def getCodedSentencesFromCodingjobs(codingjobs, cache=True, cacheValues=True):
    if cache: cacheCodingJobs(codingjobs, values=cacheValues)
    for ca in getCodedArticlesFromCodingjobs(codingjobs):
        for cs in ca.sentences:
            yield cs
            
class CodingJob(Cachable):
    __table__ = 'codingjobs'
    __idcolumn__ = 'codingjobid'
    __labelprop__ = 'name'
    codingjobid, name, insertdate = DBProperties(3)

    unitSchema = DBProperty(LB("AnnotationSchema", package="amcat.model.coding"), getcolumn="unitschemaid")
    articleSchema = DBProperty(LB("AnnotationSchema", package="amcat.model.coding"), getcolumn="articleschemaid")
    project = DBProperty(LB("Project"))
    owner = DBProperty(LB("User"), getcolumn="insertuserid")

    sets = ForeignKey(LB("CodingJobSet", sub="coding"), includeOwnID=True)

    def getSet(self, setnr):
        for set in self.sets:
            if set.setnr == setnr:
                return set

    def getAllCodedArticles(self):
        for s in self.sets:
            for ca in s.articles:
                yield ca
    
    def getNonCodedArticleids(self): # articlids that do not have any codings in this job
        sql = """
        select v.articleid
        from vw_codingjobs_articles_done as v
        where v.codingjobid = %d and v.has_arrow = 0 and v.has_artannot = 0 and (v.irrelevant IS NULL or v.irrelevant = 0)
        and not exists (
            select * from vw_codingjobs_articles_done as vw where vw.codingjobid = %d and vw.has_arrow > 0 and vw.has_artannot > 0 and vw.articleid = v.articleid
        )
        """ % (self.id, self.id)
        
        data = self.db.doQuery(sql)
{'fraction_non_alphanumeric': 0.101535974130962, 'fraction_numerical': 0.0019401778496362166, 'mean_word_length': 2.973025048169557, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1328060', 'n_tokens_mistral': 1946, 'n_tokens_neox': 1809, 'n_words': 458}