<commit_before># This file is part of OpenHatch.
# Copyright (C) 2011 <NAME>
# Copyright (C) 2010, 2011 OpenHatch, Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from collections import defaultdict
from django.core.management.base import BaseCommand
import twisted.web.client
import twisted.internet
import mysite.customs.profile_importers
import datetime
import logging
from types import NoneType
import django.db.models

import mysite.customs.models
import mysite.customs.bugimporters.base
import mysite.customs.bugimporters.bugzilla
import mysite.customs.bugimporters.google
import mysite.customs.bugimporters.roundup
import mysite.customs.bugimporters.trac
import mysite.profile.models
from mysite.search.models import Bug

tracker2importer = {
        # This first one catches any Bugs with bug.tracker = None and updates
        # that field. Then it passes the bugs back here for updating.
        NoneType:
            mysite.customs.bugimporters.base.AddTrackerForeignKeysToBugs,

        # This second one is specifically for Bugs belonging to trackers
        # that are hard-coded in for one reason or another.
        #mysite.customs.models.HardCodedTrackerModel:
        #    mysite.customs.bugimporters.base.HandleHardCodedBugTracker,

        # The rest of these link specific subclasses of TrackerModel to the
        # relevant BugImporter class. This includes the hard-coded special
        # cases, where the TrackerModel is just a shell linking to the location
        # of the hard-coded class and the BugImporter is a special one that
        # handles them specifically.

        # Bugzilla
        mysite.customs.models.BugzillaTrackerModel:
            mysite.customs.bugimporters.bugzilla.BugzillaBugImporter,
        # Google Code Issue Tracker
        mysite.customs.models.GoogleTrackerModel:
            mysite.customs.bugimporters.google.GoogleBugImporter,
        # Roundup
        mysite.customs.models.RoundupTrackerModel:
            mysite.customs.bugimporters.roundup.RoundupBugImporter,
        # Trac
        mysite.customs.models.TracTrackerModel:
            mysite.customs.bugimporters.trac.TracBugImporter,
        }

class Command(BaseCommand):
    help = "Call this when you want to run a Twisted reactor."

    def _get_importer_instance_for_tracker_model(self, tracker_model):
        '''This takes a TrackerModel as input, and returns an instantiated
        BugImporter subclass.'''
        # Some TrackerModel values have no corresponding BugImporter. Test
        # for that first so that we can bail early if needed.
        if tracker_model.__class__ not in tracker2importer:
            raise ValueError, "That tracker_model has no corresponding importer"

        # Okay, so success is possible.
        # We keep the collection of running importers indexed by the type of
        # TrackerModel. If we already such an importer running, we can
        # look it up and grab it from that collection.
        key = tracker_model
        if key in self.running_importers:
            return self.running_importers[key]

        # Okay, at this point we're going to have to create it. Note that
        # when we create it, we store it in the self.running_importers dict
        # so that later calls to this will find it.
        bug_importer_subclass = tracker2importer[tracker_model.__class__]
        bug_importer_instance = bug_importer_subclass(tracker_model, self)
        self.running_importers[key] = bug_importer_instance

        return self.running_importers[key]

    def update_trackers(self):
        print "For all tracker queries we know about, enqueue the stale ones."
        # Fetch a list of Queries that are stale.
        queries = mysite.customs.models.TrackerQueryModel.objects.filter(
                last_polled__lt=datetime.datetime.utcnow()-datetime.timedelta(days=1)
                ).select_subclasses()
        # Convert this list to a dictionary of TrackerModels.
        tm_list = [(query, query.tracker) for query in queries if hasattr(query, 'tracker')]
        tm_dict = defaultdict(list)
        [tm_dict[k].append(v) for v, k in tm_list]
        # For each TrackerModel, process its stale Queries.
        for tm, queries in tm_dict.items():
            try:
                importer = self._get_importer_instance_for_tracker_model(tm)<commit_msg>Provide support for using a bug_parser class specified in a TrackerModel.<commit_after># This file is part of OpenHatch.
# Copyright (C) 2011 <NAME>
# Copyright (C) 2010, 2011 OpenHatch, Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from collections import defaultdict
import importlib
from django.core.management.base import BaseCommand
import twisted.web.client
import twisted.internet
import mysite.customs.profile_importers
import datetime
import logging
from types import NoneType
import django.db.models

import mysite.customs.models
import mysite.customs.bugimporters.base
import mysite.customs.bugimporters.bugzilla
import mysite.customs.bugimporters.google
import mysite.customs.bugimporters.roundup
import mysite.customs.bugimporters.trac
import mysite.profile.models
from mysite.search.models import Bug

tracker2importer = {
        # This first one catches any Bugs with bug.tracker = None and updates
        # that field. Then it passes the bugs back here for updating.
        NoneType:
            mysite.customs.bugimporters.base.AddTrackerForeignKeysToBugs,

        # This second one is specifically for Bugs belonging to trackers
        # that are hard-coded in for one reason or another.
        #mysite.customs.models.HardCodedTrackerModel:
        #    mysite.customs.bugimporters.base.HandleHardCodedBugTracker,

        # The rest of these link specific subclasses of TrackerModel to the
        # relevant BugImporter class. This includes the hard-coded special
        # cases, where the TrackerModel is just a shell linking to the location
        # of the hard-coded class and the BugImporter is a special one that
        # handles them specifically.

        # Bugzilla
        mysite.customs.models.BugzillaTrackerModel:
            mysite.customs.bugimporters.bugzilla.BugzillaBugImporter,
        # Google Code Issue Tracker
        mysite.customs.models.GoogleTrackerModel:
            mysite.customs.bugimporters.google.GoogleBugImporter,
        # Roundup
        mysite.customs.models.RoundupTrackerModel:
            mysite.customs.bugimporters.roundup.RoundupBugImporter,
        # Trac
        mysite.customs.models.TracTrackerModel:
            mysite.customs.bugimporters.trac.TracBugImporter,
        }

class Command(BaseCommand):
    help = "Call this when you want to run a Twisted reactor."

    def _get_importer_instance_for_tracker_model(self, tracker_model):
        '''This takes a TrackerModel as input, and returns an instantiated
        BugImporter subclass.'''
        # Some TrackerModel values have no corresponding BugImporter. Test
        # for that first so that we can bail early if needed.
        if tracker_model.__class__ not in tracker2importer:
            raise ValueError, "That tracker_model has no corresponding importer"

        # Okay, so success is possible.
        # We keep the collection of running importers indexed by the type of
        # TrackerModel. If we already such an importer running, we can
        # look it up and grab it from that collection.
        key = (tracker_model, tracker_model.custom_parser)
        if key in self.running_importers:
            return self.running_importers[key]

        # Find the custom parser class, if specified.
        custom_parser_class = None
        if tracker_model.custom_parser:
            try:
                module_part, custom_parser_class_name = (
                    tracker_model.custom_parser.rsplit('.', 1))
                module = importlib.import_module(
                    'mysite.customs.bugimporters.' + module_part)
                custom_parser_class = getattr(module, custom_parser_class_name)
            except Exception:
                logging.error("Uh oh, failed trying to grab %s", custom_parser_class_name)
                logging.exception("We failed to import the requested custom importer.")

        # Okay, at this point we're going to have to create it. Note that
        # when we create it, we store it in the self.running_importers dict
        # so that later calls to this will find it.
        bug_importer_subclass = tracker2importer[tracker_model.__class__]
        bug_importer_instance = bug_importer_subclass(
            tracker_model, self, bug_parser=custom_parser_class)
        self.running_importers[key] = bug_importer_instance

        return self.running_importers[key]

    def update_trackers(self):
        print "For all tracker queries we know about, enqueue the stale ones."
        # Fetch a list of Queries that are stale.
        queries = mysite.customs.models.TrackerQueryModel.objects.filter(
                last_polled__lt=datetime.datetime.utcnow()-datetime.timedelta(days=1)
                ).select_subclasses()
        # Convert this list to a dictionary of TrackerModels.
        tm_list = [(query, query.tracker) for query in queries if hasattr(query, 'tracker')]
        tm_dict = defaultdict(list)
        [tm_dict[k].append(v) for v, k in tm_list]
        # For each TrackerModel, process its stale Queries.
        for tm, queries in tm_dict.items():
            try:
                importer = self._get_importer_instance_for_tracker_model(tm)
{'fraction_non_alphanumeric': 0.06064275733581742, 'fraction_numerical': 0.0032603632976245926, 'mean_word_length': 2.983673469387755, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3133253', 'n_tokens_mistral': 3024, 'n_tokens_neox': 2868, 'n_words': 1017}