<commit_before>#!/usr/bin/env python

# +skip_license_check

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Verifies that all source files contain the necessary copyright boilerplate
# snippet.

from __future__ import print_function

import argparse
import datetime
import glob
import os
import re
import sys


def get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "filenames", help="list of files to check, all files if unspecified", nargs='*')

    rootdir = os.path.dirname(__file__) + "/../"
    rootdir = os.path.abspath(rootdir)
    parser.add_argument("--rootdir", default=rootdir,
                        help="root directory to examine")

    default_boilerplate_dir = os.path.join(rootdir, "hack/boilerplate")
    parser.add_argument("--boilerplate-dir", default=default_boilerplate_dir)
    return parser.parse_args()


def get_refs():
    refs = {}

    for path in glob.glob(os.path.join(ARGS.boilerplate_dir, "boilerplate.*.txt")):
        extension = os.path.basename(path).split(".")[1]

        ref_file = open(path, 'r')
        ref = ref_file.read().splitlines()
        ref_file.close()
        refs[extension] = ref

    return refs


def file_passes(filename, refs, regexs):  # pylint: disable=too-many-locals
    try:
        with open(filename, 'r') as fp:
            data = fp.read()
    except IOError:
        return False

    basename = os.path.basename(filename)
    extension = file_extension(filename)
    if extension != "":
        ref = refs[extension]
    else:
        ref = refs[basename]

    # remove build tags from the top of Go files
    if extension == "go":
        con = regexs["go_build_constraints"]
        (data, found) = con.subn("", data, 1)

    # remove shebang from the top of shell files
    if extension == "sh" or extension == "py":
        she = regexs["shebang"]
        (data, found) = she.subn("", data, 1)

    data = data.splitlines()

    # if our test file is smaller than the reference it surely fails!
    if len(ref) > len(data):
        return False

    # trim our file to the same number of lines as the reference file
    data = data[:len(ref)]

    year = regexs["year"]
    for datum in data:
        if year.search(datum):
            return False

    # Replace all occurrences of the regex "2017|2016|2015|2014" with "YEAR"
    when = regexs["date"]
    for idx, datum in enumerate(data):
        (data[idx], found) = when.subn('YEAR', datum)
        if found != 0:
            break

    # if we don't match the reference at this point, fail
    if ref != data:
        return False

    return True


def file_extension(filename):
    return os.path.splitext(filename)[1].split(".")[-1].lower()


SKIPPED_DIRS = [
    'Godeps', 'third_party', '_gopath', '_output',
    '.git', 'vendor', '__init__.py', 'node_modules',
]

# even when generated by bazel we will complain about some generated files
# not having the headers. since they're just generated, ignore them
IGNORE_HEADERS = [
    '// Code generated by go-bindata.',
    '// +skip_license_check',
    '# +skip_license_check',
]


def has_ignored_header(pathname):
    with open(pathname, 'r') as myfile:
        data = myfile.read()
        for header in IGNORE_HEADERS:
            if header in data:
                return True
    return False


def normalize_files(files):
    newfiles = []
    for pathname in files:
        if any(x in pathname for x in SKIPPED_DIRS):
            continue
        newfiles.append(pathname)
    for idx, pathname in enumerate(newfiles):
        if not os.path.isabs(pathname):
            newfiles[idx] = os.path.join(ARGS.rootdir, pathname)
    return newfiles


def get_files(extensions):
    files = []<commit_msg>Explicitly set encoding in verify_boilerplate.py<commit_after>#!/usr/bin/env python

# +skip_license_check

# Copyright 2015 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Verifies that all source files contain the necessary copyright boilerplate
# snippet.

from __future__ import print_function

import argparse
import datetime
import glob
import os
import re
import sys


def get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "filenames", help="list of files to check, all files if unspecified", nargs='*')

    rootdir = os.path.dirname(__file__) + "/../"
    rootdir = os.path.abspath(rootdir)
    parser.add_argument("--rootdir", default=rootdir,
                        help="root directory to examine")

    default_boilerplate_dir = os.path.join(rootdir, "hack/boilerplate")
    parser.add_argument("--boilerplate-dir", default=default_boilerplate_dir)
    return parser.parse_args()


def get_refs():
    refs = {}

    for path in glob.glob(os.path.join(ARGS.boilerplate_dir, "boilerplate.*.txt")):
        extension = os.path.basename(path).split(".")[1]

        ref_file = open(path, 'r', encoding="utf-8")
        ref = ref_file.read().splitlines()
        ref_file.close()
        refs[extension] = ref

    return refs


def file_passes(filename, refs, regexs):  # pylint: disable=too-many-locals
    try:
        with open(filename, 'r', encoding="utf-8") as fp:
            data = fp.read()
    except IOError:
        return False

    basename = os.path.basename(filename)
    extension = file_extension(filename)
    if extension != "":
        ref = refs[extension]
    else:
        ref = refs[basename]

    # remove build tags from the top of Go files
    if extension == "go":
        con = regexs["go_build_constraints"]
        (data, found) = con.subn("", data, 1)

    # remove shebang from the top of shell files
    if extension == "sh" or extension == "py":
        she = regexs["shebang"]
        (data, found) = she.subn("", data, 1)

    data = data.splitlines()

    # if our test file is smaller than the reference it surely fails!
    if len(ref) > len(data):
        return False

    # trim our file to the same number of lines as the reference file
    data = data[:len(ref)]

    year = regexs["year"]
    for datum in data:
        if year.search(datum):
            return False

    # Replace all occurrences of the regex "2017|2016|2015|2014" with "YEAR"
    when = regexs["date"]
    for idx, datum in enumerate(data):
        (data[idx], found) = when.subn('YEAR', datum)
        if found != 0:
            break

    # if we don't match the reference at this point, fail
    if ref != data:
        return False

    return True


def file_extension(filename):
    return os.path.splitext(filename)[1].split(".")[-1].lower()


SKIPPED_DIRS = [
    'Godeps', 'third_party', '_gopath', '_output',
    '.git', 'vendor', '__init__.py', 'node_modules',
]

# even when generated by bazel we will complain about some generated files
# not having the headers. since they're just generated, ignore them
IGNORE_HEADERS = [
    '// Code generated by go-bindata.',
    '// +skip_license_check',
    '# +skip_license_check',
]


def has_ignored_header(pathname):
    with open(pathname, 'r', encoding="utf-8") as myfile:
        data = myfile.read()
        for header in IGNORE_HEADERS:
            if header in data:
                return True
    return False


def normalize_files(files):
    newfiles = []
    for pathname in files:
        if any(x in pathname for x in SKIPPED_DIRS):
            continue
        newfiles.append(pathname)
    for idx, pathname in enumerate(newfiles):
        if not os.path.isabs(pathname):
            newfiles[idx] = os.path.join(ARGS.rootdir, pathname)
    return newfiles


def get_files(extensions):
    files = []
{'fraction_non_alphanumeric': 0.10916676324023641, 'fraction_numerical': 0.0073009618727546645, 'mean_word_length': 2.8803956834532376, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6480863', 'n_tokens_mistral': 2716, 'n_tokens_neox': 2566, 'n_words': 839}