<commit_before># This file is part of OpenHatch.
# Copyright (C) 2011 <NAME>
# Copyright (C) 2010, 2011 OpenHatch, Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from collections import defaultdict
from django.core.management.base import BaseCommand
import twisted.web.client
import twisted.internet
import mysite.customs.profile_importers
import datetime
import logging
from types import NoneType

import mysite.customs.models
import mysite.customs.bugimporters.base
import mysite.customs.bugimporters.bugzilla
import mysite.customs.bugimporters.google
import mysite.customs.bugimporters.roundup
import mysite.customs.bugimporters.trac
import mysite.profile.models
from mysite.search.models import Bug

tracker2importer = {
        # This first one catches any Bugs with bug.tracker = None and updates
        # that field. Then it passes the bugs back here for updating.
        NoneType:
            mysite.customs.bugimporters.base.AddTrackerForeignKeysToBugs,

        # The rest of these link specific subclasses of TrackerModel to the
        # relevant BugImporter class. This includes the hard-coded special
        # cases, where the TrackerModel is just a shell linking to the location
        # of the hard-coded class and the BugImporter is a special one that
        # handles them specifically.

        # Bugzilla
        mysite.customs.models.BugzillaTrackerModel:
            mysite.customs.bugimporters.bugzilla.BugzillaBugImporter,
        # Google Code Issue Tracker
        mysite.customs.models.GoogleTrackerModel:
            mysite.customs.bugimporters.google.GoogleBugImporter,
        # Roundup1
        mysite.customs.models.RoundupTrackerModel:
            mysite.customs.bugimporters.roundup.RoundupBugImporter,
        # Trac
        mysite.customs.models.TracTrackerModel:
            mysite.customs.bugimporters.trac.TracBugImporter,
        }

class Command(BaseCommand):
    help = "Call this when you want to run a Twisted reactor."

    def update_trackers(self):
        print "For all tracker queries we know about, enqueue the stale ones."
        # Fetch a list of Queries that are stale.
        queries = mysite.customs.models.TrackerQueryModel.objects.filter(
                last_polled__lt=datetime.datetime.utcnow()-datetime.timedelta(days=1)
                ).select_subclasses()
        # Convert this list to a dictionary of TrackerModules.
        tm_list = [(query, query.tracker) for query in queries if hasattr(query, 'tracker')]
        tm_dict = defaultdict(list)
        [tm_dict[k].append(v) for v, k in tm_list]
        # For each TrackerModel, process its stale Queries.
        for tm, queries in tm_dict.items():
            try:
                icls = tracker2importer[tm.__class__]
            except KeyError:
                # This TrackerModule doesn't have a BugImporter yet.
                continue
            else:
                # See if there is already an importer running for this tracker.
                im = self.running_importers.get(tm, None)
                if not im:
                    # There isn't, so create one.
                    im = icls(tm, self)
                    # And store it.
                    self.running_importers[tm] = im
                # Give the importer queries to process.
                im.process_queries(queries)

    def update_bugs(self, bug_list = None):
        print "For all Bugs we know of, enqueue the stale ones."
        # Check if we have been specifically passed Bugs to update.
        if bug_list:
            bugs = bug_list
        else:
            # Fetch a list of all Bugs that are stale.
            bugs = Bug.all_bugs.filter(
                    last_polled__lt=datetime.datetime.utcnow()-datetime.timedelta(days=1))
        # Convert this list to a dictionary of TrackerModules.
        tm_list = [(bug, bug.tracker) for bug in bugs]
        tm_dict = defaultdict(list)
        [tm_dict[k].append(v) for v, k in tm_list]
        # For each TrackerModel, process its stale Bugs.
        for tm, bugs in tm_dict.items():
            try:
                icls = tracker2importer[tm.__class__]
            except KeyError:
                # This TrackerModule doesn't have a BugImporter yet.
                continue
            else:
                # See if there is already an importer running for this tracker.
                im = self.running_importers.get(tm, None)<commit_msg>Fixed comments in customs_twist and added a pre-emptive comment for hardcoded trackers<commit_after># This file is part of OpenHatch.
# Copyright (C) 2011 <NAME>
# Copyright (C) 2010, 2011 OpenHatch, Inc.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from collections import defaultdict
from django.core.management.base import BaseCommand
import twisted.web.client
import twisted.internet
import mysite.customs.profile_importers
import datetime
import logging
from types import NoneType

import mysite.customs.models
import mysite.customs.bugimporters.base
import mysite.customs.bugimporters.bugzilla
import mysite.customs.bugimporters.google
import mysite.customs.bugimporters.roundup
import mysite.customs.bugimporters.trac
import mysite.profile.models
from mysite.search.models import Bug

tracker2importer = {
        # This first one catches any Bugs with bug.tracker = None and updates
        # that field. Then it passes the bugs back here for updating.
        NoneType:
            mysite.customs.bugimporters.base.AddTrackerForeignKeysToBugs,

        # This second one is specifically for Bugs belonging to trackers
        # that are hard-coded in for one reason or another.
        #mysite.customs.models.HardCodedTrackerModel:
        #    mysite.customs.bugimporters.base.HandleHardCodedBugTracker,

        # The rest of these link specific subclasses of TrackerModel to the
        # relevant BugImporter class. This includes the hard-coded special
        # cases, where the TrackerModel is just a shell linking to the location
        # of the hard-coded class and the BugImporter is a special one that
        # handles them specifically.

        # Bugzilla
        mysite.customs.models.BugzillaTrackerModel:
            mysite.customs.bugimporters.bugzilla.BugzillaBugImporter,
        # Google Code Issue Tracker
        mysite.customs.models.GoogleTrackerModel:
            mysite.customs.bugimporters.google.GoogleBugImporter,
        # Roundup
        mysite.customs.models.RoundupTrackerModel:
            mysite.customs.bugimporters.roundup.RoundupBugImporter,
        # Trac
        mysite.customs.models.TracTrackerModel:
            mysite.customs.bugimporters.trac.TracBugImporter,
        }

class Command(BaseCommand):
    help = "Call this when you want to run a Twisted reactor."

    def update_trackers(self):
        print "For all tracker queries we know about, enqueue the stale ones."
        # Fetch a list of Queries that are stale.
        queries = mysite.customs.models.TrackerQueryModel.objects.filter(
                last_polled__lt=datetime.datetime.utcnow()-datetime.timedelta(days=1)
                ).select_subclasses()
        # Convert this list to a dictionary of TrackerModels.
        tm_list = [(query, query.tracker) for query in queries if hasattr(query, 'tracker')]
        tm_dict = defaultdict(list)
        [tm_dict[k].append(v) for v, k in tm_list]
        # For each TrackerModel, process its stale Queries.
        for tm, queries in tm_dict.items():
            try:
                icls = tracker2importer[tm.__class__]
            except KeyError:
                # This TrackerModel doesn't have a BugImporter yet.
                continue
            else:
                # See if there is already an importer running for this tracker.
                im = self.running_importers.get(tm, None)
                if not im:
                    # There isn't, so create one.
                    im = icls(tm, self)
                    # And store it.
                    self.running_importers[tm] = im
                # Give the importer queries to process.
                im.process_queries(queries)

    def update_bugs(self, bug_list = None):
        print "For all Bugs we know of, enqueue the stale ones."
        # Check if we have been specifically passed Bugs to update.
        if bug_list:
            bugs = bug_list
        else:
            # Fetch a list of all Bugs that are stale.
            bugs = Bug.all_bugs.filter(
                    last_polled__lt=datetime.datetime.utcnow()-datetime.timedelta(days=1))
        # Convert this list to a dictionary of TrackerModels.
        tm_list = [(bug, bug.tracker) for bug in bugs]
        tm_dict = defaultdict(list)
        [tm_dict[k].append(v) for v, k in tm_list]
        # For each TrackerModel, process its stale Bugs.
        for tm, bugs in tm_dict.items():
            try:
                icls = tracker2importer[tm.__class__]
            except KeyError:
                # This TrackerModel doesn't have a BugImporter yet.
                continue
            else:
                # See if there is already an importer running for this tracker.
                im = self.running_importers.get(tm, None)
{'fraction_non_alphanumeric': 0.06517350764202634, 'fraction_numerical': 0.003556666346246275, 'mean_word_length': 2.4133858267716537, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6434128', 'n_tokens_mistral': 2952, 'n_tokens_neox': 2789, 'n_words': 987}