<commit_before>#!/usr/bin/env python3
from __future__ import print_function
from __future__ import absolute_import

import os
import sys
import subprocess
import re
from collections import defaultdict
import argparse
from six.moves import filter

from typing import Union, List, Dict

def get_ftype(fpath, use_shebang):
    # type: (str, bool) -> str
    ext = os.path.splitext(fpath)[1]
    if ext:
        return ext[1:]
    elif use_shebang:
        # opening a file may throw an OSError
        with open(fpath) as f:
            first_line = f.readline()
            if re.search(r'^#!.*\bpython', first_line):
                return 'py'
            elif re.search(r'^#!.*sh', first_line):
                return 'sh'
            elif re.search(r'^#!.*\bperl', first_line):
                return 'pl'
            elif re.search(r'^#!.*\bnode', first_line):
                return 'js'
            elif re.search(r'^#!.*\bruby', first_line):
                return 'rb'
            elif re.search(r'^#!', first_line):
                print('Error: Unknown shebang in file "%s":\n%s' % (fpath, first_line), file=sys.stderr)
                return ''
            else:
                return ''
    else:
        return ''

def list_files(targets=[], ftypes=[], use_shebang=True, modified_only=False,
               exclude=[], group_by_ftype=False, extless_only=False):
    # type: (List[str], List[str], bool, bool, List[str], bool, bool) -> Union[Dict[str, List[str]], List[str]]
    """<commit_msg>mypy: Use Python 3 type syntax in tools/lister.py.<commit_after>#!/usr/bin/env python3
from __future__ import print_function
from __future__ import absolute_import

import os
import sys
import subprocess
import re
from collections import defaultdict
import argparse
from six.moves import filter

from typing import Union, List, Dict

def get_ftype(fpath, use_shebang):
    # type: (str, bool) -> str
    ext = os.path.splitext(fpath)[1]
    if ext:
        return ext[1:]
    elif use_shebang:
        # opening a file may throw an OSError
        with open(fpath) as f:
            first_line = f.readline()
            if re.search(r'^#!.*\bpython', first_line):
                return 'py'
            elif re.search(r'^#!.*sh', first_line):
                return 'sh'
            elif re.search(r'^#!.*\bperl', first_line):
                return 'pl'
            elif re.search(r'^#!.*\bnode', first_line):
                return 'js'
            elif re.search(r'^#!.*\bruby', first_line):
                return 'rb'
            elif re.search(r'^#!', first_line):
                print('Error: Unknown shebang in file "%s":\n%s' % (fpath, first_line), file=sys.stderr)
                return ''
            else:
                return ''
    else:
        return ''

def list_files(targets: List[str]=[], ftypes: List[str]=[], use_shebang: bool=True,
               modified_only: bool=False, exclude: List[str]=[], group_by_ftype: bool=False,
               extless_only: bool=False) -> Union[Dict[str, List[str]], List[str]]:
    """
{'fraction_non_alphanumeric': 0.13470993117010815, 'fraction_numerical': 0.0022943297279580466, 'mean_word_length': 2.226215644820296, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7509017', 'n_tokens_mistral': 998, 'n_tokens_neox': 958, 'n_words': 252}