<commit_before>import os
import sys
import re
import httplib2
import calendar

from django.db import models
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from speeches.models import Section

HTTPLIB2_HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/601.4.4 (KHTML, like Gecko) Version/9.0.3 Safari/601.4.4'
}

if 'ON_HEROKU' not in os.environ:
    # check that the cache is setup and the directory exists
    for setting_name in ('HANSARD_CACHE',
                         'COMMITTEE_CACHE',
                         'ANSWER_CACHE',
                         'QUESTION_CACHE',
                         'ANSWER_JSON_CACHE',
                         'QUESTION_JSON_CACHE'):
        try:
            directory = os.path.join(getattr(settings, setting_name))
            if not os.path.exists(directory):
                os.makedirs(directory)
        except AttributeError:
            raise ImproperlyConfigured(
                "Could not find {0} setting - please set it".format(setting_name))

# EXCEPTIONS


class SourceUrlCouldNotBeRetrieved(Exception):
    pass


class SourceCouldNotParseTimeString(Exception):
    pass


class SourceQuerySet(models.query.QuerySet):
    def requires_processing(self):
        return self.filter(last_processing_attempt=None)

    def requires_completion(self, retry_download=False):
        objects = self.filter(last_processing_success=None)
        if not retry_download:
            objects = objects.filter(is404=False)
        return objects


class Source(models.Model):
    """
    Sources of the hansard transcripts

    For example a Word transcript.
    """

    title = models.CharField(max_length=200)
    # bah, SHOULD be unique, but apparently isn't
    document_name = models.CharField(max_length=200)
    document_number = models.CharField(unique=True, max_length=200)
    date = models.DateField()
    url = models.URLField(max_length=1000)
    is404 = models.BooleanField(default=False)
    house = models.CharField(max_length=200)
    language = models.CharField(max_length=200)

    last_processing_attempt = models.DateTimeField(blank=True, null=True)
    last_processing_success = models.DateTimeField(blank=True, null=True)

    last_sayit_import = models.DateTimeField(blank=True, null=True)
    sayit_section = models.ForeignKey(Section, blank=True, null=True, on_delete=models.PROTECT,
                                      help_text='Associated Sayit section object, if imported')

    objects = SourceQuerySet.as_manager()

    class Meta:
        ordering = ['-date', 'document_name']

    def __unicode__(self):
        return self.document_name

    def delete(self):
        """After deleting from db, delete the cached file too"""
        cache_file_path = self.cache_file_path()
        super(Source, self).delete()

        if os.path.exists(cache_file_path):
            os.remove(cache_file_path)

    def file(self, debug=False):
        """
        Return as a file object the resource that the url is pointing to.

        Should check the local cache first, and fetch and store if it is not
        found there.

        Raises a SourceUrlCouldNotBeRetrieved exception if URL could not be
        retrieved.
        """
        cache_file_path = self.cache_file_path()

        found = os.path.isfile(cache_file_path)

        if debug:
            print >> sys.stderr, "%s (%s)" % (cache_file_path, found)

        # If the file exists open it, read it and return it
        if found:
            return cache_file_path

        # If not fetch the file, save to cache and then return fh
        h = httplib2.Http()
        url = self.url

        def request_url(url):
            if debug:
                print >> sys.stderr, 'Requesting %s' % url
            (response, content) = h.request(url, headers=HTTPLIB2_HEADERS)
            if response.status != 200:
                raise SourceUrlCouldNotBeRetrieved(
                    "status code: %s, url: %s" % (response.status, self.url))
            self.is404 = False
            self.save()
            return (response, content)

        try:
            (response, content) = request_url(url)
        except SourceUrlCouldNotBeRetrieved as e:
            try:
                if not url[-4:] == '.doc':
                    (response, content) = request_url(url + '.doc')
                    self.url = self.url + '.doc'
                    self.save()
                else:
                    raise e
            except:
                raise e

        if not content:
            raise SourceUrlCouldNotBeRetrieved("WTF?")
        with open(cache_file_path, "w") as new_cache_file:
            new_cache_file.write(content)

        return cache_file_path

    @property
    def section_parent_headings(self):
        return [
            "Hansard",
            str(self.date.year),
            calendar.month_name[self.date.month],
            "%02d" % self.date.day,
        ]

    def cache_file_path(self):
        """Absolute path to the cache file for this source"""

        id_str = "%05u" % self.id

        # do some simple partitioning
        # FIXME - put in something to prevent the test suite overwriting non-test files.
        aaa = id_str[-1]
        bbb = id_str[-2]
        cache_dir = os.path.join(settings.HANSARD_CACHE, aaa, bbb)

        # check that the dir exists
        if not os.path.exists(cache_dir):
            os.makedirs(cache_dir)

        d = self.date.strftime('%Y-%m-%d')

        # create the path to the file
        cache_file_path = os.path.join(
            cache_dir, '-'.join([d, id_str, self.document_name]))
        return cache_file_path

    def xml_file_path(self):
        xml_file_path = '%s.xml' % self.cache_file_path()
        if os.path.isfile(xml_file_path):
            return xml_file_path
        return None


class PMGCommitteeReport(models.Model):
    """
    Committe reports, scraped from PMG site
    """
    premium = models.BooleanField(default=None)
    processed = models.BooleanField(default=None)
    meeting_url = models.TextField()
    meeting_name = models.TextField(blank=True, null=True)
    committee_url = models.URLField(blank=True, null=True)
    committee_name = models.TextField(default='')
    meeting_date = models.DateField(blank=True, null=True)

    # For anything sourced from the PMG API, these two fields should
    # identify the meeting event:
    api_committee_id = models.IntegerField(null=True, blank=True)<commit_msg>Remove slashes in za_hansard source filenames<commit_after>import os
import sys
import re
import httplib2
import calendar

from django.db import models
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from speeches.models import Section

HTTPLIB2_HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/601.4.4 (KHTML, like Gecko) Version/9.0.3 Safari/601.4.4'
}

if 'ON_HEROKU' not in os.environ:
    # check that the cache is setup and the directory exists
    for setting_name in ('HANSARD_CACHE',
                         'COMMITTEE_CACHE',
                         'ANSWER_CACHE',
                         'QUESTION_CACHE',
                         'ANSWER_JSON_CACHE',
                         'QUESTION_JSON_CACHE'):
        try:
            directory = os.path.join(getattr(settings, setting_name))
            if not os.path.exists(directory):
                os.makedirs(directory)
        except AttributeError:
            raise ImproperlyConfigured(
                "Could not find {0} setting - please set it".format(setting_name))

# EXCEPTIONS


class SourceUrlCouldNotBeRetrieved(Exception):
    pass


class SourceCouldNotParseTimeString(Exception):
    pass


class SourceQuerySet(models.query.QuerySet):
    def requires_processing(self):
        return self.filter(last_processing_attempt=None)

    def requires_completion(self, retry_download=False):
        objects = self.filter(last_processing_success=None)
        if not retry_download:
            objects = objects.filter(is404=False)
        return objects


class Source(models.Model):
    """
    Sources of the hansard transcripts

    For example a Word transcript.
    """

    title = models.CharField(max_length=200)
    # bah, SHOULD be unique, but apparently isn't
    document_name = models.CharField(max_length=200)
    document_number = models.CharField(unique=True, max_length=200)
    date = models.DateField()
    url = models.URLField(max_length=1000)
    is404 = models.BooleanField(default=False)
    house = models.CharField(max_length=200)
    language = models.CharField(max_length=200)

    last_processing_attempt = models.DateTimeField(blank=True, null=True)
    last_processing_success = models.DateTimeField(blank=True, null=True)

    last_sayit_import = models.DateTimeField(blank=True, null=True)
    sayit_section = models.ForeignKey(Section, blank=True, null=True, on_delete=models.PROTECT,
                                      help_text='Associated Sayit section object, if imported')

    objects = SourceQuerySet.as_manager()

    class Meta:
        ordering = ['-date', 'document_name']

    def __unicode__(self):
        return self.document_name

    def delete(self):
        """After deleting from db, delete the cached file too"""
        cache_file_path = self.cache_file_path()
        super(Source, self).delete()

        if os.path.exists(cache_file_path):
            os.remove(cache_file_path)

    def file(self, debug=False):
        """
        Return as a file object the resource that the url is pointing to.

        Should check the local cache first, and fetch and store if it is not
        found there.

        Raises a SourceUrlCouldNotBeRetrieved exception if URL could not be
        retrieved.
        """
        cache_file_path = self.cache_file_path()

        found = os.path.isfile(cache_file_path)

        if debug:
            print >> sys.stderr, "%s (%s)" % (cache_file_path, found)

        # If the file exists open it, read it and return it
        if found:
            return cache_file_path

        # If not fetch the file, save to cache and then return fh
        h = httplib2.Http()
        url = self.url

        def request_url(url):
            if debug:
                print >> sys.stderr, 'Requesting %s' % url
            (response, content) = h.request(url, headers=HTTPLIB2_HEADERS)
            if response.status != 200:
                raise SourceUrlCouldNotBeRetrieved(
                    "status code: %s, url: %s" % (response.status, self.url))
            self.is404 = False
            self.save()
            return (response, content)

        try:
            (response, content) = request_url(url)
        except SourceUrlCouldNotBeRetrieved as e:
            try:
                if not url[-4:] == '.doc':
                    (response, content) = request_url(url + '.doc')
                    self.url = self.url + '.doc'
                    self.save()
                else:
                    raise e
            except:
                raise e

        if not content:
            raise SourceUrlCouldNotBeRetrieved("WTF?")
        with open(cache_file_path, "w") as new_cache_file:
            new_cache_file.write(content)

        return cache_file_path

    @property
    def section_parent_headings(self):
        return [
            "Hansard",
            str(self.date.year),
            calendar.month_name[self.date.month],
            "%02d" % self.date.day,
        ]

    def cache_file_path(self):
        """Absolute path to the cache file for this source"""

        id_str = "%05u" % self.id

        # do some simple partitioning
        # FIXME - put in something to prevent the test suite overwriting non-test files.
        aaa = id_str[-1]
        bbb = id_str[-2]
        cache_dir = os.path.join(settings.HANSARD_CACHE, aaa, bbb)

        # check that the dir exists
        if not os.path.exists(cache_dir):
            os.makedirs(cache_dir)

        d = self.date.strftime('%Y-%m-%d')

        # create the path to the file
        cache_file_path = os.path.join(
            cache_dir, '-'.join([d, id_str, self.document_name.replace('/', '-')]))
        return cache_file_path

    def xml_file_path(self):
        xml_file_path = '%s.xml' % self.cache_file_path()
        if os.path.isfile(xml_file_path):
            return xml_file_path
        return None


class PMGCommitteeReport(models.Model):
    """
    Committe reports, scraped from PMG site
    """
    premium = models.BooleanField(default=None)
    processed = models.BooleanField(default=None)
    meeting_url = models.TextField()
    meeting_name = models.TextField(blank=True, null=True)
    committee_url = models.URLField(blank=True, null=True)
    committee_name = models.TextField(default='')
    meeting_date = models.DateField(blank=True, null=True)

    # For anything sourced from the PMG API, these two fields should
    # identify the meeting event:
    api_committee_id = models.IntegerField(null=True, blank=True)
{'fraction_non_alphanumeric': 0.0918227501142074, 'fraction_numerical': 0.009593421653723162, 'mean_word_length': 2.3035714285714284, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6585379', 'n_tokens_mistral': 3951, 'n_tokens_neox': 3733, 'n_words': 958}