<commit_before>## $Id$
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
Check the kwalitee of the CDS Invenio Python code.

Q: What is kwalitee?
A: <http://qa.perl.org/phalanx/kwalitee.html>

Usage: python kwalitee.py <topsrcdir | file1.py file2.py ...>
Examples:
    $ python kwalitee.py ~/src/cds-invenio/
    $ python kwalitee.py ../../websearch/lib/*.py
"""

import os
import sre
import sys
import time

# pylint: disable-msg=C0301
# pylint: disable-msg=C0103

__revision__ = "$Id$"

def get_list_of_python_code_files(modulesdir, modulename):
    """Return list of Python source code files for MODULENAME in MODULESDIR,
       excluding test files.
    """
    out = []
    # firstly, find out *.py files:
    (dummy, pipe, dummy)= os.popen3("find %s/%s/ -name '*.py'" % (modulesdir, modulename))
    out.extend([filename.strip() for filename in pipe.readlines()])
    pipe.close()
    # secondly, find out *.py.wml files:
    (dummy, pipe, dummy) = os.popen3("find %s/%s/ -name '*.py.wml'" % (modulesdir, modulename))
    out.extend([filename.strip() for filename in pipe.readlines()])
    pipe.close()
    # thirdly, find out bin/*.in files:
    (dummy, pipe, dummy) = os.popen3("find %s/%s/bin/ -name '*.in'" % (modulesdir, modulename))
    out.extend([filename.strip() for filename in pipe.readlines()])
    pipe.close()
    # last, remove Makefile, test files, z_ files:
    # pylint: disable-msg=W0141
    out = filter(lambda x: not x.endswith("Makefile.in"), out)
    out = filter(lambda x: not x.endswith("_tests.py"), out)
    out = filter(lambda x: x.find("/z_") == -1, out)
    # return list:
    return out

def wash_list_of_python_files_for_pylinting(filenames):
    """Remove away some Python files that are not suitable for
       pylinting, e.g. known wrong test files or empty init files.
    """
    # pylint: disable-msg=W0141
    # take only .py files for pylinting:
    filenames = filter(lambda x: x.endswith(".py"),
                                 filenames)
    # remove empty __init__.py files (FIXME: we may check for file size here
    # in case we shall have non-empty __init__.py files one day)    
    filenames = filter(lambda x: not x.endswith("__init__.py"),
                                 filenames)
    # take out unloadable bibformat test files:
    filenames = filter(lambda x: not x.endswith("bfe_test_4.py"),
                                 filenames)
    # take out test unloadable file:
    filenames = filter(lambda x: not x.endswith("test3.py"),
                                 filenames)
    return filenames

def get_list_of_python_unit_test_files(modulesdir, modulename):
    """Return list of Python unit test files for MODULENAME in MODULESDIR."""
    out = []
    (dummy, pipe, dummy) = os.popen3("find %s/%s/ -name '*_tests.py'" % (modulesdir, modulename))
    out.extend([filename.strip() for filename in pipe.readlines()])
    pipe.close()
    # pylint: disable-msg=W0141
    out = filter(lambda x: not x.endswith("_regression_tests.py"), out)
    return out

def get_list_of_python_regression_test_files(modulesdir, modulename):
    """Return list of Python unit test files for MODULENAME in MODULESDIR."""
    out = []
    (dummy, pipe, dummy) = os.popen3("find %s/%s/ -name '*_regression_tests.py'" % (modulesdir, modulename))
    out.extend([filename.strip() for filename in pipe.readlines()])
    pipe.close()
    return out

def get_nb_lines_in_file(filename):
    """Return number of lines in FILENAME."""
    return len(open(filename).readlines())

def get_nb_test_cases_in_file(filename):
    """Return number of test cases in FILENAME."""
    (dummy, pipe, dummy) = os.popen3("grep ' def test' %s" % filename)
    return len(pipe.readlines())

def get_pylint_score(filename):
    """Run pylint and return the code score for FILENAME.  If score
       cannot be detected, print an error and return -999999999.
    """    
    (dummy, pipe, dummy) = os.popen3("pylint %s" % filename)
    pylint_output = pipe.read()
    pylint_score = -999999999
    pylint_score_matched = sre.search(r'Your code has been rated at ([0-9\.\-]+)\/10', pylint_output)
    if pylint_score_matched:
        pylint_score = pylint_score_matched.group(1)
    else:
        print "ERROR: cannot detect pylint score for %s" % filename
    return float(pylint_score)

def get_nb_pychecker_warnings(filename):
    """Run pychecker for FILENAME and return the number of warnings.
       Do not return warnings from imported files, only warnings found
       inside FILENAME.
    """
    nb_warnings_found = 0
    filename_to_watch_for = sre.sub(r'^\.\/', '', filename) # pychecker strips leading ./
    (dummy, pipe, dummy) = os.popen3("pychecker %s" % filename)
    pychecker_output_lines = pipe.readlines()
    for line in pychecker_output_lines:
        if line.find(filename_to_watch_for + ":") > -1:
            nb_warnings_found += 1            
    return nb_warnings_found    

def calculate_module_kwalitee(modulesdir, modulename):<commit_msg>Fixed detection of pychecker-stripped leading ../../ path.<commit_after>## $Id$
##
## This file is part of CDS Invenio.
## Copyright (C) 2002, 2003, 2004, 2005, 2006 CERN.
##
## CDS Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
Check the kwalitee of the CDS Invenio Python code.

Q: What is kwalitee?
A: <http://qa.perl.org/phalanx/kwalitee.html>

Usage: python kwalitee.py <topsrcdir | file1.py file2.py ...>
Examples:
    $ python kwalitee.py ~/src/cds-invenio/
    $ python kwalitee.py ../../websearch/lib/*.py
"""

import os
import sre
import sys
import time

# pylint: disable-msg=C0301
# pylint: disable-msg=C0103

__revision__ = "$Id$"

verbose = 0

def get_list_of_python_code_files(modulesdir, modulename):
    """Return list of Python source code files for MODULENAME in MODULESDIR,
       excluding test files.
    """
    out = []
    # firstly, find out *.py files:
    (dummy, pipe, dummy)= os.popen3("find %s/%s/ -name '*.py'" % (modulesdir, modulename))
    out.extend([filename.strip() for filename in pipe.readlines()])
    pipe.close()
    # secondly, find out *.py.wml files:
    (dummy, pipe, dummy) = os.popen3("find %s/%s/ -name '*.py.wml'" % (modulesdir, modulename))
    out.extend([filename.strip() for filename in pipe.readlines()])
    pipe.close()
    # thirdly, find out bin/*.in files:
    (dummy, pipe, dummy) = os.popen3("find %s/%s/bin/ -name '*.in'" % (modulesdir, modulename))
    out.extend([filename.strip() for filename in pipe.readlines()])
    pipe.close()
    # last, remove Makefile, test files, z_ files:
    # pylint: disable-msg=W0141
    out = filter(lambda x: not x.endswith("Makefile.in"), out)
    out = filter(lambda x: not x.endswith("_tests.py"), out)
    out = filter(lambda x: x.find("/z_") == -1, out)
    # return list:
    return out

def wash_list_of_python_files_for_pylinting(filenames):
    """Remove away some Python files that are not suitable for
       pylinting, e.g. known wrong test files or empty init files.
    """
    # pylint: disable-msg=W0141
    # take only .py files for pylinting:
    filenames = filter(lambda x: x.endswith(".py"),
                                 filenames)
    # remove empty __init__.py files (FIXME: we may check for file size here
    # in case we shall have non-empty __init__.py files one day)    
    filenames = filter(lambda x: not x.endswith("__init__.py"),
                                 filenames)
    # take out unloadable bibformat test files:
    filenames = filter(lambda x: not x.endswith("bfe_test_4.py"),
                                 filenames)
    # take out test unloadable file:
    filenames = filter(lambda x: not x.endswith("test3.py"),
                                 filenames)
    return filenames

def get_list_of_python_unit_test_files(modulesdir, modulename):
    """Return list of Python unit test files for MODULENAME in MODULESDIR."""
    out = []
    (dummy, pipe, dummy) = os.popen3("find %s/%s/ -name '*_tests.py'" % (modulesdir, modulename))
    out.extend([filename.strip() for filename in pipe.readlines()])
    pipe.close()
    # pylint: disable-msg=W0141
    out = filter(lambda x: not x.endswith("_regression_tests.py"), out)
    return out

def get_list_of_python_regression_test_files(modulesdir, modulename):
    """Return list of Python unit test files for MODULENAME in MODULESDIR."""
    out = []
    (dummy, pipe, dummy) = os.popen3("find %s/%s/ -name '*_regression_tests.py'" % (modulesdir, modulename))
    out.extend([filename.strip() for filename in pipe.readlines()])
    pipe.close()
    return out

def get_nb_lines_in_file(filename):
    """Return number of lines in FILENAME."""
    return len(open(filename).readlines())

def get_nb_test_cases_in_file(filename):
    """Return number of test cases in FILENAME."""
    (dummy, pipe, dummy) = os.popen3("grep ' def test' %s" % filename)
    return len(pipe.readlines())

def get_pylint_score(filename):
    """Run pylint and return the code score for FILENAME.  If score
       cannot be detected, print an error and return -999999999.
    """
    (dummy, pipe, dummy) = os.popen3("pylint %s" % filename)
    pylint_output = pipe.read()
    pylint_score = -999999999
    pylint_score_matched = sre.search(r'Your code has been rated at ([0-9\.\-]+)\/10', pylint_output)
    if pylint_score_matched:
        pylint_score = pylint_score_matched.group(1)
    else:
        print "ERROR: cannot detect pylint score for %s" % filename
    if verbose >= 9:
        print "get_pylint_score(%s) = %s" % (filename, pylint_score)
    return float(pylint_score)

def get_nb_pychecker_warnings(filename):
    """Run pychecker for FILENAME and return the number of warnings.
       Do not return warnings from imported files, only warnings found
       inside FILENAME.
    """
    nb_warnings_found = 0
    filename_to_watch_for = sre.sub(r'^[\.\/]+', '', filename) # pychecker strips leading ../.. stuff
    (dummy, pipe, dummy) = os.popen3("pychecker %s" % filename)
    pychecker_output_lines = pipe.readlines()
    for line in pychecker_output_lines:
        if line.find(filename_to_watch_for + ":") > -1:            
            nb_warnings_found += 1            
    if verbose >= 9:
        print "get_nb_pychecker_warnings(%s) = %s" % (filename, nb_warnings_found)
    return nb_warnings_found    

def calculate_module_kwalitee(modulesdir, modulename):
{'fraction_non_alphanumeric': 0.12117866847826086, 'fraction_numerical': 0.016219429347826088, 'mean_word_length': 3.2105827672506257, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3845911', 'n_tokens_mistral': 4000, 'n_tokens_neox': 3781, 'n_words': 1189}