<commit_before>#!/usr/bin/env python
# Copyright 2014 The LUCI Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.

"""Compiles all *.proto files it finds into *_pb2.py."""

import logging
import optparse
import os
import re
import shutil
import subprocess
import sys
import tempfile


# Directory with this file.
THIS_DIR = os.path.dirname(os.path.abspath(__file__))


# Minimally required protoc version.
MIN_SUPPORTED_PROTOC_VERSION = (3, 0, 0)
# Maximally supported protoc version.
MAX_SUPPORTED_PROTOC_VERSION = (3, 0, 0)


# Printed if protoc is missing or too old.
PROTOC_INSTALL_HELP = (
    "Could not find working protoc (%s <= ver <= %s) in PATH." %
    (
      '.'.join(map(str, MIN_SUPPORTED_PROTOC_VERSION)),
      '.'.join(map(str, MAX_SUPPORTED_PROTOC_VERSION)),
    ))


# Paths that should not be searched for *.proto.
BLACKLISTED_PATHS = [
  re.compile(r'.*(/|\\)third_party(/|\\)?'),
]


def is_blacklisted(path, blacklist):
  """True if |path| matches any regexp in |blacklist|."""
  return any(b.match(path) for b in blacklist)


def find_proto_files(path, blacklist):
  """Recursively searches for *.proto files, yields absolute paths to them."""<commit_msg>Bump supported protoc version to 3.1.0.<commit_after>#!/usr/bin/env python
# Copyright 2014 The LUCI Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.

"""Compiles all *.proto files it finds into *_pb2.py."""

import logging
import optparse
import os
import re
import shutil
import subprocess
import sys
import tempfile


# Directory with this file.
THIS_DIR = os.path.dirname(os.path.abspath(__file__))


# Minimally required protoc version.
MIN_SUPPORTED_PROTOC_VERSION = (3, 0, 0)
# Maximally supported protoc version.
MAX_SUPPORTED_PROTOC_VERSION = (3, 1, 0)


# Printed if protoc is missing or too old.
PROTOC_INSTALL_HELP = (
    "Could not find working protoc (%s <= ver <= %s) in PATH." %
    (
      '.'.join(map(str, MIN_SUPPORTED_PROTOC_VERSION)),
      '.'.join(map(str, MAX_SUPPORTED_PROTOC_VERSION)),
    ))


# Paths that should not be searched for *.proto.
BLACKLISTED_PATHS = [
  re.compile(r'.*(/|\\)third_party(/|\\)?'),
]


def is_blacklisted(path, blacklist):
  """True if |path| matches any regexp in |blacklist|."""
  return any(b.match(path) for b in blacklist)


def find_proto_files(path, blacklist):
  """Recursively searches for *.proto files, yields absolute paths to them."""
{'fraction_non_alphanumeric': 0.11986035686578743, 'fraction_numerical': 0.011249030256012413, 'mean_word_length': 4.21010101010101, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5889613', 'n_tokens_mistral': 891, 'n_tokens_neox': 835, 'n_words': 284}