<commit_before>from django.core.management.base import BaseCommand, CommandError
from django.db import transaction

import sys
import json

from twit.models import User, Tweet, Retweet, Mention, UserMention

class Command(BaseCommand):
    """Scrape tweets from twitter and load into database."""
    help = __doc__

    def add_arguments(self, parser):
        import argparse
        parser.add_argument('--issues_dict', type=argparse.FileType('r'), help="Input file containing a issues and keywords corresponding to them.")

    def handle(self, *args, **options):
        for line in options['input']:
            for tweet in Tweet.objects.all():
                print(tweet) # do magic<commit_msg>Count tweets by the issues they reference<commit_after>from django.core.management.base import BaseCommand, CommandError
from django.db import transaction

from collections import Counter
import json
import re
import sys

from twit.models import User, Tweet, Retweet, Mention, UserMention

class Command(BaseCommand):
    """Scrape tweets from twitter and load into database."""
    help = __doc__

    def add_arguments(self, parser):
        import argparse
        # parser.add_argument('--issues_dict', type=argparse.FileType('r'), help="Input file containing a issues and keywords corresponding to them.")

    def handle(self, *args, **options):
        issues = {
            "Free Trade": "free trade|trade",
            "Energy & Oil": "energy|oil",
            "Jobs": "jobs",
            "Environment": "environment",
            "Corporations": "corporations",
            "Tax Reform": "tax|taxes|tax reform",
            "Tax Reform": "tax reform|tax|taxes",
            "Government Reform": "government reform",
            "Infrastructure & Technology": "infrastructure|technology",
            "Education": "education",
            "Health Care": "health care|healthcare|obamacare",
            "Civil Rights": "civil rights|equal rights",
            "Families & Children": "families|children",
            "Religion": "religion",
            "Criminal Justice": "criminal justice",
            "Gun Control": "gun control|gun|guns",
            "Welfare & Poverty": "welfare|poverty",
            "Crime": "crime",
            "Drugs": "drugs",
            "Immigration": "immigration|border patrol|border",
            "Foreign policy": "foreign policy",
            "War & Peace": "war|peace",
            "Homeland Security": "homeland security|dhs",
        }
        counts = Counter()

        print("Building counts by issue")
        for i, tweet in enumerate(Tweet.objects.all()):
            text = tweet.text.lower()
            for issue, regex in issues.items():
                if re.search(regex, text) is not None:
                    counts[issue] += 1
        print("Issues referenced most often in descending order")
        for issue, count in counts.most_common():
            print("%4d %s" % (count, issue))
{'fraction_non_alphanumeric': 0.1155149051490515, 'fraction_numerical': 0.0006775067750677507, 'mean_word_length': 2.6547029702970297, 'pattern_counts': {'":': 23, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6805573', 'n_tokens_mistral': 811, 'n_tokens_neox': 752, 'n_words': 246}