<commit_before>#!/usr/bin/env python

import collections
import logging
import re
import sys

from django.core.management.base import BaseCommand

from app.models import SourceLine


logging.basicConfig(
    format="%(asctime)-15s %(levelname)-8s %(message)s",
    stream=sys.stderr,
    level=logging.DEBUG)

# disable db query logs
logging.getLogger('django.db.backends').propagate = False
logger = logging.getLogger(__name__)


class Command(BaseCommand):
    help = 'beer'

    def add_arguments(self, parser):
        pass # parser.add_argument('projects', nargs='+')

    def handle(self, *args, **options):
        camelcase_pat = re.compile('[A-Z][a-z]*')
        namepaths = SourceLine.objects.filter(
            project='flask').values_list(
            'name', 'path')
        path_words = collections.defaultdict(
            collections.Counter)

        for num,(name, path) in enumerate(namepaths):
            names = [name.lower()]
            if '_' in name:
                names = name.lower().split('_')
            elif camelcase_pat.match(name):
                names = camelcase_pat.findall(name.lower())
            path_words[path].update(filter(None, names))
            if num > 500:
                break

        for path, words in sorted(path_words.iteritems()):
            relpath = re.sub('^.+?/', '', path)
            common = [(word, count)
                      for word, count in words.most_common(3)
                      if count > 1]
            if common:
                print '{:30} {}'.format(relpath, common if common else '')
<commit_msg>scan words for multiple projects<commit_after>#!/usr/bin/env python

import collections
import logging
import re
import sys

from django.core.management.base import BaseCommand

from app.models import SourceLine


logging.basicConfig(
    format="%(asctime)-15s %(levelname)-8s %(message)s",
    stream=sys.stderr,
    level=logging.DEBUG)

# disable db query logs
logging.getLogger('django.db.backends').propagate = False
logger = logging.getLogger(__name__)


class Command(BaseCommand):
    help = 'beer'

    def add_arguments(self, parser):
        parser.add_argument('projects', nargs='+')

    def handle(self, *args, **options):
        camelcase_pat = re.compile('[A-Z][a-z]*')
        for project in options['projects']:
            print '*', project.upper()
            namepaths = SourceLine.objects.filter(
                project=project).values_list(
                    'name', 'path')
            path_words = collections.defaultdict(
                collections.Counter)

            for num,(name, path) in enumerate(namepaths):
                names = [name.lower()]
                if '_' in name:
                    names = name.lower().split('_')
                elif camelcase_pat.match(name):
                    names = camelcase_pat.findall(name.lower())
                path_words[path].update(filter(None, names))

            for path, words in sorted(path_words.iteritems()):
                relpath = re.sub('^.+?/', '', path)
                common = [(word, count)
                          for word, count in words.most_common(3)
                          if count > 1]
                if common:
                    print '{:30} {}'.format(relpath, common if common else '')

{'fraction_non_alphanumeric': 0.12632535595274158, 'fraction_numerical': 0.0051499545592244775, 'mean_word_length': 2.1387832699619773, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3301909', 'n_tokens_mistral': 964, 'n_tokens_neox': 910, 'n_words': 217}