<commit_before>#!/usr/bin/python
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Simple Code Hygiene tool meant to be run as a presubmit test or standalone

Usage:
./code_hygiene.py <file1> <file2> ...

For the styleguides see:
http://code.google.com/p/soc/wiki/PythonStyleGuide
http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml
"""

import os
import re
import signal
import subprocess
import sys
import time

# ======================================================================
VERBOSE = 0
LIMIT = 5
TIMEOUT = 10

# These programs are likely not available/working on windows.

# http://tidy.sourceforge.net/
HTML_CHECKER = ['tidy', '-errors']

# From depot_tools:
# To see a list of all filters run: 'depot_tools/cpplint.py --filter='
CPP_CHECKER = ['cpplint.py',
               '--filter='
               '-build/header_guard'
               ',-build/class'
               ',-build/include_order'
               ',-readability/casting'
               ',-readability/check'
               ',-readability/multiline_comment'
               ',-runtime/sizeof'
               ',-runtime/threadsafe_fn'
               ',-whitespace/newline'
               ]

# From depot_tools (currently not used -- too many false positives).
# To see a list of all filters run: 'depot_tools/cpplint.py --filter='
C_CHECKER = ['cpplint.py',
             '--filter='
             '-build/header_guard'
             ',-build/class'
             ',-build/include_order'
             ',-readability/casting'
             ',-readability/check'
             ',-readability/multiline_comment'
             ',-runtime/sizeof'
             ',-runtime/threadsafe_fn'
             ',-whitespace/newline'
             ]

# http://pychecker.sourceforge.net/
PYTHON_CHECKER = ['pychecker']

RE_IGNORE = re.compile(r'@IGNORE_LINES_FOR_CODE_HYGIENE\[([0-9]+)\]')

# ======================================================================
def Debug(s):
  if VERBOSE:
    print s


def RunCommand(cmd):
  """Run a shell command given an argv style vector."""
  Debug(str(cmd))

  start = time.time()
  p = subprocess.Popen(cmd,
                       bufsize=1000*1000,
                       stderr=subprocess.PIPE,
                       stdout=subprocess.PIPE)
  while p.poll() is None:
    time.sleep(1)
    now = time.time()
    if now - start > TIMEOUT:
      Debug('Error: timeout')
      os.kill(p.pid, signal.SIGKILL)
      os.waitpid(-1, os.WNOHANG)
      return -666, "", ""
  stdout = p.stdout.read()
  stderr = p.stderr.read()
  retcode = p.wait()

  return retcode, stdout, stderr


# ======================================================================
class ExternalChecker(object):
  """Base Class"""
  def __init__(self, commandline, use_stderr=False):
    self._commandline = commandline
    self._use_stderr = use_stderr

  def FindProblems(self, props, unused_data):
    filename = props['name']
    try:
      retcode, stdout, stderr = RunCommand(self._commandline + [filename])
    except Exception, err:
      print 'Error processing %s:' % filename
      print '  Cannot execute command %s. failed reason: %s ' % (
          str(self._commandline), str(err))
      return []
    if retcode == 0:
      return []
    if retcode == 666:
      print 'command %s timed out' % str(self._commandline)
      return []
    if self._use_stderr:
      return [stderr]
    else:
      return [stdout]

  def FileFilter(self, unused_props):
    raise NotImplementedError()

<commit_msg>Fix presubmit checks to not take 1 second per file: remove sleep() call<commit_after>#!/usr/bin/python
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Simple Code Hygiene tool meant to be run as a presubmit test or standalone

Usage:
./code_hygiene.py <file1> <file2> ...

For the styleguides see:
http://code.google.com/p/soc/wiki/PythonStyleGuide
http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml
"""

import os
import re
import subprocess
import sys

# ======================================================================
VERBOSE = 0
LIMIT = 5

# These programs are likely not available/working on windows.

# http://tidy.sourceforge.net/
HTML_CHECKER = ['tidy', '-errors']

# From depot_tools:
# To see a list of all filters run: 'depot_tools/cpplint.py --filter='
CPP_CHECKER = ['cpplint.py',
               '--filter='
               '-build/header_guard'
               ',-build/class'
               ',-build/include_order'
               ',-readability/casting'
               ',-readability/check'
               ',-readability/multiline_comment'
               ',-runtime/sizeof'
               ',-runtime/threadsafe_fn'
               ',-whitespace/newline'
               ]

# From depot_tools (currently not used -- too many false positives).
# To see a list of all filters run: 'depot_tools/cpplint.py --filter='
C_CHECKER = ['cpplint.py',
             '--filter='
             '-build/header_guard'
             ',-build/class'
             ',-build/include_order'
             ',-readability/casting'
             ',-readability/check'
             ',-readability/multiline_comment'
             ',-runtime/sizeof'
             ',-runtime/threadsafe_fn'
             ',-whitespace/newline'
             ]

# http://pychecker.sourceforge.net/
PYTHON_CHECKER = ['pychecker']

RE_IGNORE = re.compile(r'@IGNORE_LINES_FOR_CODE_HYGIENE\[([0-9]+)\]')

# ======================================================================
def Debug(s):
  if VERBOSE:
    print s


def RunCommand(cmd):
  """Run a shell command given an argv style vector."""
  Debug(str(cmd))

  p = subprocess.Popen(cmd,
                       bufsize=1000*1000,
                       stderr=subprocess.PIPE,
                       stdout=subprocess.PIPE)
  stdout, stderr = p.communicate()
  retcode = p.wait()

  return retcode, stdout, stderr


# ======================================================================
class ExternalChecker(object):
  """Base Class"""
  def __init__(self, commandline, use_stderr=False):
    self._commandline = commandline
    self._use_stderr = use_stderr

  def FindProblems(self, props, unused_data):
    filename = props['name']
    try:
      retcode, stdout, stderr = RunCommand(self._commandline + [filename])
    except Exception, err:
      print 'Error processing %s:' % filename
      print '  Cannot execute command %s. failed reason: %s ' % (
          str(self._commandline), str(err))
      return []
    if retcode == 0:
      return []
    if self._use_stderr:
      return [stderr]
    else:
      return [stdout]

  def FileFilter(self, unused_props):
    raise NotImplementedError()


{'fraction_non_alphanumeric': 0.1774123126182506, 'fraction_numerical': 0.007131421918206956, 'mean_word_length': 2.556935817805383, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2811111', 'n_tokens_mistral': 2127, 'n_tokens_neox': 1994, 'n_words': 510}