<commit_before>#!/usr/bin/env python

# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import fnmatch
import os
import sys
import json
import yaml


def get_godeps_dependencies(rootdir, components):
    all_dependencies = {}
    for component in components:
        with open(os.path.join(rootdir, component, "go.mod")) as f:
            print(component + " dependencies")
            all_dependencies[component] = []
            lines = list(set(f))
            lines.sort()
            for line in lines:
                for dep in components:
                    if dep == component:
                        continue
                    if ("k8s.io/" + dep + " v0.0.0") not in line:
                        continue
                    print("\t"+dep)
                    if dep not in all_dependencies[component]:
                        all_dependencies[component].append(dep)
    return all_dependencies


def get_rules_dependencies(rules_file):
    with open(rules_file) as f:
        data = yaml.load(f)
    return data


def main():
    rootdir = os.path.dirname(__file__) + "/../"
    rootdir = os.path.abspath(rootdir)

    components = []
    for component in os.listdir(rootdir + '/staging/src/k8s.io/'):
        components.append(component)
    components.sort()

    godep_dependencies = get_godeps_dependencies(rootdir + '/staging/src/k8s.io/', components)
    rules_dependencies = get_rules_dependencies(rootdir + "/staging/publishing/rules.yaml")

    processed_repos = []
    for rule in rules_dependencies["rules"]:
        branch = rule["branches"][0]
        if branch["name"] != "master":
            raise Exception("cannot find master branch for destination %s" % rule["destination"])
        if branch["source"]["branch"] != "master":
            raise Exception("cannot find master source branch for destination %s" % rule["destination"])

        print("processing : %s" % rule["destination"])
        if rule["destination"] not in godep_dependencies:
            raise Exception("missing Godeps.json for %s" % rule["destination"])
        processed_repos.append(rule["destination"])
        for dep in set(godep_dependencies[rule["destination"]]):
            found = False
            if "dependencies" in branch:
                for dep2 in branch["dependencies"]:
                    if dep2["branch"] != "master":
                        raise Exception("Looking for master branch and found : %s for destination", dep2,
                                        rule["destination"])
                    if dep2["repository"] == dep:
                        found = True
            else:
                raise Exception(
                    "destination %s does not have any dependencies (looking for %s)" % (rule["destination"], dep))
            if not found:
                raise Exception("destination %s does not have dependency %s" % (rule["destination"], dep))
            else:
                print("  found dependency %s" % dep)
    items = set(godep_dependencies.keys()) - set(processed_repos)
    if len(items) > 0:
        raise Exception("missing rules for %s" % ','.join(str(s) for s in items))
    print("Done.")


if __name__ == "__main__":
    sys.exit(main())
<commit_msg>Show all missing deps when publishing-bot has no listed dependencies<commit_after>#!/usr/bin/env python

# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import fnmatch
import os
import sys
import json
import yaml


def get_gomod_dependencies(rootdir, components):
    all_dependencies = {}
    for component in components:
        with open(os.path.join(rootdir, component, "go.mod")) as f:
            print(component + " dependencies")
            all_dependencies[component] = []
            lines = list(set(f))
            lines.sort()
            for line in lines:
                for dep in components:
                    if dep == component:
                        continue
                    if ("k8s.io/" + dep + " v0.0.0") not in line:
                        continue
                    print("\t"+dep)
                    if dep not in all_dependencies[component]:
                        all_dependencies[component].append(dep)
    return all_dependencies


def get_rules_dependencies(rules_file):
    with open(rules_file) as f:
        data = yaml.load(f)
    return data


def main():
    rootdir = os.path.dirname(__file__) + "/../"
    rootdir = os.path.abspath(rootdir)

    components = []
    for component in os.listdir(rootdir + '/staging/src/k8s.io/'):
        components.append(component)
    components.sort()

    rules_file = "/staging/publishing/rules.yaml"
    gomod_dependencies = get_gomod_dependencies(rootdir + '/staging/src/k8s.io/', components)
    rules_dependencies = get_rules_dependencies(rootdir + rules_file)

    processed_repos = []
    for rule in rules_dependencies["rules"]:
        branch = rule["branches"][0]
        if branch["name"] != "master":
            raise Exception("cannot find master branch for destination %s" % rule["destination"])
        if branch["source"]["branch"] != "master":
            raise Exception("cannot find master source branch for destination %s" % rule["destination"])

        print("processing : %s" % rule["destination"])
        if rule["destination"] not in gomod_dependencies:
            raise Exception("missing go.mod for %s" % rule["destination"])
        processed_repos.append(rule["destination"])
        for dep in set(gomod_dependencies[rule["destination"]]):
            found = False
            if "dependencies" in branch:
                for dep2 in branch["dependencies"]:
                    if dep2["branch"] != "master":
                        raise Exception("Looking for master branch and found : %s for destination", dep2,
                                        rule["destination"])
                    if dep2["repository"] == dep:
                        found = True
            else:
                raise Exception(
                    "Please add %s as dependencies under destination %s in %s" % (gomod_dependencies[rule["destination"]], rule["destination"], rules_file))
            if not found:
                raise Exception("Please add %s as a dependency under destination %s in %s" % (dep, rule["destination"], rules_file))
            else:
                print("  found dependency %s" % dep)
    items = set(gomod_dependencies.keys()) - set(processed_repos)
    if len(items) > 0:
        raise Exception("missing rules for %s" % ','.join(str(s) for s in items))
    print("Done.")


if __name__ == "__main__":
    sys.exit(main())

{'fraction_non_alphanumeric': 0.10663568293312678, 'fraction_numerical': 0.005163955589981926, 'mean_word_length': 2.33634797588286, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7194627', 'n_tokens_mistral': 2075, 'n_tokens_neox': 1952, 'n_words': 647}