<commit_before>#!/usr/bin/env python
import pdb

#==============================================================================
# Common Functionality
#==============================================================================
from ssa.common import *
import ssa.java

#==============================================================================
# Cmdline
#==============================================================================
import argparse

parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, 
    description="""
TODO
""")

parser.add_argument('-v', '--verbose',
        action="store_true",
        help="verbose output" )

parser.add_argument('-p', '--pattern',
        action='append',
        help="file patterns of the files to count." )

parser.add_argument('-i', '--ignore-pattern',
        action='append',
        help="file patterns to skip." )

parser.add_argument('-c', '--count',
        action="store_true",
        help="count the lines of code in each file." )

parser.add_argument('--interactive',
        action="store_true",
        help="prompts for input before exiting." )

args = None

#==============================================================================
# Main
#==============================================================================
import sys
import os

class LineCounter(FileSet):
    def __init__(self):
        super(LineCounter, self).__init__()

    def linecounts(self):
        counted = set()
        for path in self.files:
            with open(path.abspath, 'r') as f:
                n = 0
                for line in f:
                    line = line.strip()
                    # ignore blank lines.
                    if len(line) <= 0:
                        continue
                    # ignore comments.  TODO: ignore block comments?
                    if line.startswith('//'):
                        continue
                    # count the line.
                    n += 1
                assert(path not in counted)
                counted.add(path)
                yield path, n
    
    def count_lines(self):
        lines = 0
        files = 0
        for filename, linecount in self.linecounts():
            print '%10d' % linecount, filename
            files += 1
            lines += linecount
        print '%d lines in %d files' % (lines, files)


def get_files():
    paths = listify(args.args)
    # if no paths were specified, then switch to interactive mode and
    # prompt the user for paths.
    if len(paths) <= 0:
        args.interactive = True
        while True:
            line = raw_input('Enter source code path or file pattern.  To finish, enter a blank line: ').strip()
            if len(line) <= 0:
                break
            paths.append(line)
    # if no paths are specified, search the cwd.
    if len(paths) <= 0:
        print 'No paths specified, searching current dir: %s' % os.getcwd()
        paths.append('.')
    files = list(find_files(paths, patterns=args.pattern, ignore_patterns=args.ignore_pattern))
    return files

def run():<commit_msg>default to scanning Java files.<commit_after>#!/usr/bin/env python
import pdb

#==============================================================================
# Common Functionality
#==============================================================================
from ssa.common import *
import ssa.java

#==============================================================================
# Cmdline
#==============================================================================
import argparse

parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, 
    description="""
TODO
""")

parser.add_argument('-v', '--verbose',
        action="store_true",
        help="verbose output" )

parser.add_argument('-p', '--pattern',
        action='append',
        help="file patterns of the files to count." )

parser.add_argument('-i', '--ignore-pattern',
        action='append',
        help="file patterns to skip." )

parser.add_argument('-c', '--count',
        action="store_true",
        help="count the lines of code in each file." )

parser.add_argument('--interactive',
        action="store_true",
        help="prompts for input before exiting." )

args = None

#==============================================================================
# Main
#==============================================================================
import sys
import os

class LineCounter(FileSet):
    def __init__(self):
        super(LineCounter, self).__init__()

    def linecounts(self):
        counted = set()
        for path in self.files:
            with open(path.abspath, 'r') as f:
                n = 0
                for line in f:
                    line = line.strip()
                    # ignore blank lines.
                    if len(line) <= 0:
                        continue
                    # ignore comments.  TODO: ignore block comments?
                    if line.startswith('//'):
                        continue
                    # count the line.
                    n += 1
                assert(path not in counted)
                counted.add(path)
                yield path, n
    
    def count_lines(self):
        lines = 0
        files = 0
        for filename, linecount in self.linecounts():
            print '%10d' % linecount, filename
            files += 1
            lines += linecount
        print '%d lines in %d files' % (lines, files)


def get_files():
    paths = listify(args.args)
    # if no paths were specified, then switch to interactive mode and
    # prompt the user for paths.
    if len(paths) <= 0:
        args.interactive = True
        while True:
            line = raw_input('Enter source code path or file pattern.  To finish, enter a blank line: ').strip()
            if len(line) <= 0:
                break
            paths.append(line)
    # if no paths are specified, search the cwd.
    if len(paths) <= 0:
        print 'No paths specified, searching current dir: %s' % os.getcwd()
        paths.append('.')
    files = list(find_files(paths, patterns=args.pattern, ignore_patterns=args.ignore_pattern, verbose=True))
    return files

def run():
    if not args.count:
        # if we're not counting lines, then we're scanning Java code.
        args.pattern = listify(args.pattern)
        args.pattern += ['*.java']
    print 'Searching for files...'
{'fraction_non_alphanumeric': 0.2403400309119011, 'fraction_numerical': 0.0034003091190108192, 'mean_word_length': 2.3563278008298756, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7438180', 'n_tokens_mistral': 1667, 'n_tokens_neox': 1559, 'n_words': 473}