<commit_before>from ..lm import DomainLMResource, GigawordLMResource
from ..data import Resource, get_resource_manager
from ..sc import SCChunkResource, IdfResource
from ..pipeline import (
    TfIdfExtractor, LMProbExtractor, BasicFeaturesExtractor,
    QueryFeaturesExtractor, GeoFeaturesExtractor
)
import os
from cuttsum.detector import ArticleDetector
import streamcorpus as sc
import re
import pandas as pd
from itertools import izip
import signal
import Queue
import cuttsum.srilm
import gzip
from datetime import datetime, timedelta

class ArticlesResource(Resource):
    def __init__(self):
        Resource.__init__(self)
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'articles')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

    def check_coverage(self, event, corpus, **kwargs):
        data_dir = os.path.join(self.dir_, event.fs_name())
        hours = event.list_event_hours()
        n_hours = len(hours)
        n_covered = 0
        for hour in hours:
            path = os.path.join(data_dir, '{}.sc.gz'.format(
                hour.strftime(u'%Y-%m-%d-%H')))
            if os.path.exists(path):
                n_covered += 1
        if n_hours == 0:
            return 0
        else:
            return n_covered / float(n_hours)


    def get_chunk_path(self, event, hour):
        data_dir = os.path.join(self.dir_, event.fs_name())
        return os.path.join(data_dir, u'{}.sc.gz'.format(
            hour.strftime(u'%Y-%m-%d-%H')))

    def get(self, event, corpus, **kwargs):
        data_dir = os.path.join(self.dir_, event.fs_name())
        if not os.path.exists(data_dir):
            os.makedirs(data_dir)
        chunks = SCChunkResource()
        

        overwrite = kwargs.get(u'overwrite', False)
        hours = event.list_event_hours()

        jobs = []
        for hour in hours: 
            path = self.get_chunk_path(event, hour)
            if os.path.exists(path) and overwrite is False:
                continue
            chunk_paths = chunks.get_chunks_for_hour(hour, corpus)
            jobs.append((path, chunk_paths))

        n_procs = kwargs.get(u'n_procs', 1)
        progress_bar = kwargs.get(u'progress_bar', False)
        self.do_work(_article_resource_worker, jobs, n_procs, progress_bar,
                     event=event, corpus=corpus)

    def dependencies(self):
        return tuple(['SCChunkResource'])

    def __unicode__(self):
        return u"cuttsum.pipeline.ArticlesResource"

def _article_resource_worker(job_queue, result_queue, **kwargs):
    signal.signal(signal.SIGINT, signal.SIG_IGN)
    event = kwargs.get(u'event')
    corpus = kwargs.get(u'corpus')
    while not job_queue.empty():
        try:
            opath, chunk_paths = job_queue.get(block=False)
            artcl_detect = ArticleDetector(event)
            patt = event.regex_pattern()
            with sc.Chunk(path=opath, mode='wb', message=corpus.sc_msg()) as ochunk:
                for path in chunk_paths:
                    for si in sc.Chunk(path=path, message=corpus.sc_msg()):
                        if si.body.clean_visible is None:
                            continue
                        
                        elif patt.search(si.body.clean_visible, re.I):
                            
                            #if corpus.annotator() not in si.body.sentences:
                            #    continue
                            sentences = corpus.get_sentences(si)
                            sent_idxs = artcl_detect.find_articles(
                                sentences)
                            if len(sent_idxs) > 0:
                                rel_sents = []
                                for sent_idx in sent_idxs:
                                    #for token in sentences[sent_idx].tokens:
                                    #    print token.token,
                                    #print
                                    rel_sents.append(sentences[sent_idx])
                                si.body.sentences[u'article-clf'] = rel_sents
                                ochunk.add(si)


            result_queue.put(None)
        except Queue.Empty:
            pass


class SentenceFeaturesResource(Resource):
    def __init__(self):
        Resource.__init__(self)
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'sentence-features')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

    def dependencies(self):
        return tuple([u'DomainLMResource', u'IdfResource',
                      u'ArticlesResource', u'GigawordLMResource',
                      u'SentenceStringsResource', u'GeoClustersResource'])

    def __unicode__(self):
        return u"cuttsum.pipeline.SentenceFeaturesResource"

    def get_tsv_path(self, event, hour):
        data_dir = os.path.join(self.dir_, event.fs_name())
        return os.path.join(data_dir, u'{}.tsv.gz'.format(
            hour.strftime(u'%Y-%m-%d-%H')))

    def check_coverage(self, event, corpus, **kwargs):
        
        n_hours = 0
        n_covered = 0

        strings = get_resource_manager(u'SentenceStringsResource')

        for hour in event.list_event_hours():
            if os.path.join(strings.get_tsv_path(event, hour)):
                n_hours += 1<commit_msg>Fixed resource coverage check bug<commit_after>from ..lm import DomainLMResource, GigawordLMResource
from ..data import Resource, get_resource_manager
from ..sc import SCChunkResource, IdfResource
from ..pipeline import (
    TfIdfExtractor, LMProbExtractor, BasicFeaturesExtractor,
    QueryFeaturesExtractor, GeoFeaturesExtractor
)
import os
from cuttsum.detector import ArticleDetector
import streamcorpus as sc
import re
import pandas as pd
from itertools import izip
import signal
import Queue
import cuttsum.srilm
import gzip
from datetime import datetime, timedelta

class ArticlesResource(Resource):
    def __init__(self):
        Resource.__init__(self)
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'articles')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

    def check_coverage(self, event, corpus, **kwargs):
        data_dir = os.path.join(self.dir_, event.fs_name())
        hours = event.list_event_hours()
        n_hours = len(hours)
        n_covered = 0
        for hour in hours:
            path = os.path.join(data_dir, '{}.sc.gz'.format(
                hour.strftime(u'%Y-%m-%d-%H')))
            if os.path.exists(path):
                n_covered += 1
        if n_hours == 0:
            return 0
        else:
            return n_covered / float(n_hours)


    def get_chunk_path(self, event, hour):
        data_dir = os.path.join(self.dir_, event.fs_name())
        return os.path.join(data_dir, u'{}.sc.gz'.format(
            hour.strftime(u'%Y-%m-%d-%H')))

    def get(self, event, corpus, **kwargs):
        data_dir = os.path.join(self.dir_, event.fs_name())
        if not os.path.exists(data_dir):
            os.makedirs(data_dir)
        chunks = SCChunkResource()
        

        overwrite = kwargs.get(u'overwrite', False)
        hours = event.list_event_hours()

        jobs = []
        for hour in hours: 
            path = self.get_chunk_path(event, hour)
            if os.path.exists(path) and overwrite is False:
                continue
            chunk_paths = chunks.get_chunks_for_hour(hour, corpus)
            jobs.append((path, chunk_paths))

        n_procs = kwargs.get(u'n_procs', 1)
        progress_bar = kwargs.get(u'progress_bar', False)
        self.do_work(_article_resource_worker, jobs, n_procs, progress_bar,
                     event=event, corpus=corpus)

    def dependencies(self):
        return tuple(['SCChunkResource'])

    def __unicode__(self):
        return u"cuttsum.pipeline.ArticlesResource"

def _article_resource_worker(job_queue, result_queue, **kwargs):
    signal.signal(signal.SIGINT, signal.SIG_IGN)
    event = kwargs.get(u'event')
    corpus = kwargs.get(u'corpus')
    while not job_queue.empty():
        try:
            opath, chunk_paths = job_queue.get(block=False)
            artcl_detect = ArticleDetector(event)
            patt = event.regex_pattern()
            with sc.Chunk(path=opath, mode='wb', message=corpus.sc_msg()) as ochunk:
                for path in chunk_paths:
                    for si in sc.Chunk(path=path, message=corpus.sc_msg()):
                        if si.body.clean_visible is None:
                            continue
                        
                        elif patt.search(si.body.clean_visible, re.I):
                            
                            #if corpus.annotator() not in si.body.sentences:
                            #    continue
                            sentences = corpus.get_sentences(si)
                            sent_idxs = artcl_detect.find_articles(
                                sentences)
                            if len(sent_idxs) > 0:
                                rel_sents = []
                                for sent_idx in sent_idxs:
                                    #for token in sentences[sent_idx].tokens:
                                    #    print token.token,
                                    #print
                                    rel_sents.append(sentences[sent_idx])
                                si.body.sentences[u'article-clf'] = rel_sents
                                ochunk.add(si)


            result_queue.put(None)
        except Queue.Empty:
            pass


class SentenceFeaturesResource(Resource):
    def __init__(self):
        Resource.__init__(self)
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'sentence-features')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

    def dependencies(self):
        return tuple([u'DomainLMResource', u'IdfResource',
                      u'ArticlesResource', u'GigawordLMResource',
                      u'SentenceStringsResource', u'GeoClustersResource'])

    def __unicode__(self):
        return u"cuttsum.pipeline.SentenceFeaturesResource"

    def get_tsv_path(self, event, hour):
        data_dir = os.path.join(self.dir_, event.fs_name())
        return os.path.join(data_dir, u'{}.tsv.gz'.format(
            hour.strftime(u'%Y-%m-%d-%H')))

    def check_coverage(self, event, corpus, **kwargs):
        
        n_hours = 0
        n_covered = 0

        strings = get_resource_manager(u'SentenceStringsResource')

        for hour in event.list_event_hours():
            if os.path.exists(strings.get_tsv_path(event, hour)):
                n_hours += 1
{'fraction_non_alphanumeric': 0.10190014903129657, 'fraction_numerical': 0.0016766020864381521, 'mean_word_length': 1.8262700710713347, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4930504', 'n_tokens_mistral': 3241, 'n_tokens_neox': 3098, 'n_words': 625}