<commit_before>#!/usr/bin/env python3
# Copyright (c) 2016 The Bitcoin Core developers
# Copyright (c) 2017 The Bitcoin developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

import re
import fnmatch
import sys
import subprocess
import datetime
import os

##########################################################################
# file filtering
##########################################################################

EXCLUDE = [
    # libsecp256k1:
    'src/secp256k1/include/secp256k1.h',
    'src/secp256k1/include/secp256k1_ecdh.h',
    'src/secp256k1/include/secp256k1_recovery.h',
    'src/secp256k1/include/secp256k1_schnorr.h',
    'src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.c',
    'src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.h',
    'src/secp256k1/src/java/org_bitcoin_Secp256k1Context.c',
    'src/secp256k1/src/java/org_bitcoin_Secp256k1Context.h',
    # univalue:
    'src/univalue/test/object.cpp',
    'src/univalue/lib/univalue_escapes.h',
    # auto generated:
    'src/qt/bitcoinstrings.cpp',
    'src/chainparamsseeds.h',
    # other external copyrights:
    'src/tinyformat.h',
    'src/leveldb/util/env_win.cc',
    'src/crypto/ctaes/bench.c',
    'test/functional/test_framework/bignum.py',
    # python init:
    '*__init__.py',
]
EXCLUDE_COMPILED = re.compile(
    '|'.join([fnmatch.translate(m) for m in EXCLUDE]))

INCLUDE = ['*.h', '*.cpp', '*.cc', '*.c', '*.py']
INCLUDE_COMPILED = re.compile(
    '|'.join([fnmatch.translate(m) for m in INCLUDE]))


def applies_to_file(filename):
    return ((EXCLUDE_COMPILED.match(filename) is None)
            and (INCLUDE_COMPILED.match(filename) is not None))

##########################################################################
# obtain list of files in repo according to INCLUDE and EXCLUDE
##########################################################################


GIT_LS_CMD = 'git ls-files'


def call_git_ls():
    out = subprocess.check_output(GIT_LS_CMD.split(' '))
    return [f for f in out.decode("utf-8").split('\n') if f != '']


def get_filenames_to_examine():
    filenames = call_git_ls()
    return sorted([filename for filename in filenames if
                   applies_to_file(filename)])

##########################################################################
# define and compile regexes for the patterns we are looking for
##########################################################################


COPYRIGHT_WITH_C = r'Copyright \(c\)'
COPYRIGHT_WITHOUT_C = 'Copyright'
ANY_COPYRIGHT_STYLE = '({}|{})'.format(COPYRIGHT_WITH_C, COPYRIGHT_WITHOUT_C)

YEAR = "20[0-9][0-9]"
YEAR_RANGE = '({})(-{})?'.format(YEAR, YEAR)
YEAR_LIST = '({})(, {})+'.format(YEAR, YEAR)
ANY_YEAR_STYLE = '({}|{})'.format(YEAR_RANGE, YEAR_LIST)
ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE = (
    "{} {}".format(ANY_COPYRIGHT_STYLE, ANY_YEAR_STYLE))

ANY_COPYRIGHT_COMPILED = re.compile(ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE)


def compile_copyright_regex(copyright_style, year_style, name):
    return re.compile('{} {} {}'.format(copyright_style, year_style, name))


EXPECTED_HOLDER_NAMES = [
    r"<NAME>\n",
    r"The Bitcoin Core developers\n",
    r"The Bitcoin Core developers \n",
    r"Bitcoin Core Developers\n",
    r"the Bitcoin Core developers\n",
    r"The Bitcoin developers\n",
    r"The LevelDB Authors\. All rights reserved\.\n",
    r"BitPay Inc\.\n",
    r"BitPay, Inc\.\n",
    r"University of Illinois at Urbana-Champaign\.\n",
    r"MarcoFalke\n",
    r"<NAME>\n",
    r"<NAME> +\*\n",
    r"<NAME>, <NAME> +\*\n",
    r"<NAME>, <NAME> +\*\n",
    r"<NAME> +\*\n",
    r"<NAME>\n",
    r"<NAME>\n",
    r"<NAME>, <NAME> +\*\n",
    r"<NAME>, <NAME> +\*\n",
    r"<NAME>\n",
    r"<NAME>\n",
    r"ArtForz -- public domain half-a-node\n",
    r"<NAME>\n",
    r"<NAME>\n",
]

DOMINANT_STYLE_COMPILED = {}
YEAR_LIST_STYLE_COMPILED = {}
WITHOUT_C_STYLE_COMPILED = {}

for holder_name in EXPECTED_HOLDER_NAMES:
    DOMINANT_STYLE_COMPILED[holder_name] = (
        compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_RANGE, holder_name))
    YEAR_LIST_STYLE_COMPILED[holder_name] = (
        compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_LIST, holder_name))
    WITHOUT_C_STYLE_COMPILED[holder_name] = (
        compile_copyright_regex(COPYRIGHT_WITHOUT_C, ANY_YEAR_STYLE,
                                holder_name))

##########################################################################
# search file contents for copyright message of particular category
##########################################################################<commit_msg>[backport#14785] Scripts: Fix detection of copyright holders<commit_after>#!/usr/bin/env python3
# Copyright (c) 2016 The Bitcoin Core developers
# Copyright (c) 2017 The Bitcoin developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

import re
import fnmatch
import sys
import subprocess
import datetime
import os

##########################################################################
# file filtering
##########################################################################

EXCLUDE = [
    # auto generated:
    'src/qt/bitcoinstrings.cpp',
    'src/chainparamsseeds.h',
    # other external copyrights:
    'src/tinyformat.h',
    'test/functional/test_framework/bignum.py',
    # python init:
    '*__init__.py',
]
EXCLUDE_COMPILED = re.compile(
    '|'.join([fnmatch.translate(m) for m in EXCLUDE]))

EXCLUDE_DIRS = [
    # git subtrees
    "src/crypto/ctaes/",
    "src/leveldb/",
    "src/secp256k1/",
    "src/univalue/",
]

INCLUDE = ['*.h', '*.cpp', '*.cc', '*.c', '*.py']
INCLUDE_COMPILED = re.compile(
    '|'.join([fnmatch.translate(m) for m in INCLUDE]))


def applies_to_file(filename):
    for excluded_dir in EXCLUDE_DIRS:
        if filename.startswith(excluded_dir):
            return False
    return ((EXCLUDE_COMPILED.match(filename) is None)
            and (INCLUDE_COMPILED.match(filename) is not None))

##########################################################################
# obtain list of files in repo according to INCLUDE and EXCLUDE
##########################################################################


GIT_LS_CMD = 'git ls-files'


def call_git_ls():
    out = subprocess.check_output(GIT_LS_CMD.split(' '))
    return [f for f in out.decode("utf-8").split('\n') if f != '']


def get_filenames_to_examine():
    filenames = call_git_ls()
    return sorted([filename for filename in filenames if
                   applies_to_file(filename)])

##########################################################################
# define and compile regexes for the patterns we are looking for
##########################################################################


COPYRIGHT_WITH_C = r'Copyright \(c\)'
COPYRIGHT_WITHOUT_C = 'Copyright'
ANY_COPYRIGHT_STYLE = '({}|{})'.format(COPYRIGHT_WITH_C, COPYRIGHT_WITHOUT_C)

YEAR = "20[0-9][0-9]"
YEAR_RANGE = '({})(-{})?'.format(YEAR, YEAR)
YEAR_LIST = '({})(, {})+'.format(YEAR, YEAR)
ANY_YEAR_STYLE = '({}|{})'.format(YEAR_RANGE, YEAR_LIST)
ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE = (
    "{} {}".format(ANY_COPYRIGHT_STYLE, ANY_YEAR_STYLE))

ANY_COPYRIGHT_COMPILED = re.compile(ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE)


def compile_copyright_regex(copyright_style, year_style, name):
    return re.compile('{} {},? {}'.format(copyright_style, year_style, name))


EXPECTED_HOLDER_NAMES = [
    r"<NAME>\n",
    r"The Bitcoin Core developers\n",
    r"The Bitcoin Core developers \n",
    r"Bitcoin Core Developers\n",
    r"the Bitcoin Core developers\n",
    r"The Bitcoin developers\n",
    r"The LevelDB Authors\. All rights reserved\.\n",
    r"BitPay Inc\.\n",
    r"BitPay, Inc\.\n",
    r"University of Illinois at Urbana-Champaign\.\n",
    r"MarcoFalke\n",
    r"<NAME>\n",
    r"<NAME> +\*\n",
    r"<NAME>, <NAME> +\*\n",
    r"<NAME>, <NAME> +\*\n",
    r"<NAME> +\*\n",
    r"<NAME>\n",
    r"<NAME>\n",
    r"<NAME>, <NAME> +\*\n",
    r"<NAME>, <NAME> +\*\n",
    r"<NAME>\n",
    r"<NAME>\n",
    r"ArtForz -- public domain half-a-node\n",
    r"<NAME>\n",
    r"Intel Corporation\n",
    r"The Zcash developers\n",
    r"<NAME>\n",
]

DOMINANT_STYLE_COMPILED = {}
YEAR_LIST_STYLE_COMPILED = {}
WITHOUT_C_STYLE_COMPILED = {}

for holder_name in EXPECTED_HOLDER_NAMES:
    DOMINANT_STYLE_COMPILED[holder_name] = (
        compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_RANGE, holder_name))
    YEAR_LIST_STYLE_COMPILED[holder_name] = (
        compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_LIST, holder_name))
    WITHOUT_C_STYLE_COMPILED[holder_name] = (
        compile_copyright_regex(COPYRIGHT_WITHOUT_C, ANY_YEAR_STYLE,
                                holder_name))

##########################################################################
# search file contents for copyright message of particular category
##########################################################################
{'fraction_non_alphanumeric': 0.2611874169251218, 'fraction_numerical': 0.012073548958794861, 'mean_word_length': 4.47876213592233, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5307617', 'n_tokens_mistral': 3190, 'n_tokens_neox': 3017, 'n_words': 560}