<commit_before>import collections
import os
import sys

from django.core.management.base import BaseCommand
from django.db.models import Avg, Max, Sum

from app.models import SourceFile, Symbol


def show_file_index(projects):
    FORMAT = '{name:20} {path:50} {num_lines:>5}'
    print FORMAT.format(name='NAME', path='PATH', num_lines="LINES")
    for project in projects:
        # pylint: disable=no-member
        files = SourceFile.objects.filter(
            project=project).order_by('path')
        for file_ in files:
            print FORMAT.format(**vars(file_))

def show_dir_index(projects):
    """
    for each directory, output total number of source lines
    """
    FORMAT = '{dir_path:50} {num_lines:>5}'

    for project in projects:
        data = SourceFile.objects.filter(
            project=project).values_list('path', 'num_lines')
        count = collections.Counter()
        for path,num_lines in data:
            count[os.path.dirname(path)] = num_lines

        for dir_path,num_lines in sorted(count.iteritems()):
            print FORMAT.format(**locals())

# XX V1
def show_symbol_index(projects):
    FORMAT = '{name:30} {path}:{line_number}'

    def fun_symbol(sym):
        return sym.name[0] != '_'

    for project in projects:
        # pylint: disable=no-member
        symbols = SourceLine.objects.filter(
            project=project).order_by('name')
        for symbol in filter(fun_symbol, symbols):
            print FORMAT.format(**symbol.__dict__)


def show_summary(projects):
    HEADER = '{:30} {:>9} {:>6} {:>7} {:>10} {:>9}'.format(
        'project', 'files', 'avglen', 'maxlen', 'total', 'symbols')
    FORMAT = (
        '{project:30}'
        ' {num_files:9,}'
        ' {avg_length:6,}'
        ' {max_length:7,}'
        ' {total_length:10,}'
        ' {num_symbols:9,}')

    print HEADER
    for project in projects:
        proj_qs = SourceFile.objects.filter(project=project)
        num_files = proj_qs.count()
        avg_length = int(proj_qs.aggregate(Avg('num_lines')).values()[0])
        max_length = proj_qs.aggregate(Max('num_lines')).values()[0]
        total_length = proj_qs.aggregate(Sum('num_lines')).values()[0]
        proj_symbols = Symbol.objects.filter(source_file__project=project)<commit_msg>add "--style=symbol_index" for outputting symbox index<commit_after>import collections
import os
import sys

from django.core.management.base import BaseCommand
from django.db.models import Avg, Max, Sum

from app.models import SourceFile, Symbol


def show_file_index(projects):
    FORMAT = '{name:20} {path:50} {num_lines:>5}'
    print FORMAT.format(name='NAME', path='PATH', num_lines="LINES")
    for project in projects:
        # pylint: disable=no-member
        files = SourceFile.objects.filter(
            project=project).order_by('path')
        for file_ in files:
            print FORMAT.format(**vars(file_))

def show_dir_index(projects):
    """
    for each directory, output total number of source lines
    """
    FORMAT = '{dir_path:50} {num_lines:>5}'

    for project in projects:
        data = SourceFile.objects.filter(
            project=project).values_list('path', 'num_lines')
        count = collections.Counter()
        for path,num_lines in data:
            count[os.path.dirname(path)] = num_lines

        for dir_path,num_lines in sorted(count.iteritems()):
            print FORMAT.format(**locals())


def show_symbol_index(projects):
    FORMAT = '{0:30} {1}:{2}'

    def fun_symbol(sym):
        return sym.name[0] != '_'

    for project in projects:
        # pylint: disable=no-member
        proj_symbols = Symbol.objects.filter(
            source_file__project=project)
        fun_symbols = proj_symbols.exclude(label__startswith='_').order_by('label')
        for symbol in fun_symbols:
            print FORMAT.format(symbol.label, symbol.source_file.path, symbol.line_number)


def show_summary(projects):
    HEADER = '{:30} {:>9} {:>6} {:>7} {:>10} {:>9}'.format(
        'project', 'files', 'avglen', 'maxlen', 'total', 'symbols')
    FORMAT = (
        '{project:30}'
        ' {num_files:9,}'
        ' {avg_length:6,}'
        ' {max_length:7,}'
        ' {total_length:10,}'
        ' {num_symbols:9,}')

    print HEADER
    for project in projects:
        proj_qs = SourceFile.objects.filter(project=project)
        num_files = proj_qs.count()
        avg_length = int(proj_qs.aggregate(Avg('num_lines')).values()[0])
        max_length = proj_qs.aggregate(Max('num_lines')).values()[0]
        total_length = proj_qs.aggregate(Sum('num_lines')).values()[0]
        proj_symbols = Symbol.objects.filter(source_file__project=project)
{'fraction_non_alphanumeric': 0.14078188421277504, 'fraction_numerical': 0.013672292245246742, 'mean_word_length': 3.064236111111111, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1071667', 'n_tokens_mistral': 1543, 'n_tokens_neox': 1472, 'n_words': 303}