<commit_before>#!/usr/bin/env python3
#
# Copyright 2018 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import base64
import json
import os
import re
import subprocess
import sys
import threading
import yaml
import urllib.request

import bazelci
from bazelci import BuildkiteException

BUILD_STATUS_API_URL = "https://api.buildkite.com/v2/organizations/bazel/pipelines/bazelisk-plus-incompatible-flags/builds/"

ENCRYPTED_BUILDKITE_API_TOKEN = """
<KEY>
""".strip()

INCOMPATIBLE_FLAGS = bazelci.fetch_incompatible_flags()

class LogFetcher(threading.Thread):
   def __init__(self, job):
      threading.Thread.__init__(self)
      self.job = job
      self.log = None

   def run(self):
      self.log = get_build_log(self.job)


def buildkite_token():
    return (
        subprocess.check_output(
            [
                bazelci.gcloud_command(),
                "kms",
                "decrypt",
                "--project",
                "bazel-untrusted",
                "--location",
                "global",
                "--keyring",
                "buildkite",
                "--key",
                "buildkite-untrusted-api-token",
                "--ciphertext-file",
                "-",
                "--plaintext-file",
                "-",
            ],
            input=base64.b64decode(ENCRYPTED_BUILDKITE_API_TOKEN),
            env=os.environ,
        )
        .decode("utf-8")
        .strip()
    )


def get_build_status_api_url(build_number):
    return BUILD_STATUS_API_URL + "%s?access_token=%s" % (build_number, buildkite_token())


def get_build_log_api_url(raw_log_url):
    return raw_log_url + "?access_token=" + buildkite_token()


def get_build_info(build_number):
    output = urllib.request.urlopen(get_build_status_api_url(build_number)).read().decode("utf-8")
    build_info = json.loads(output)
    return build_info


def get_build_log(job):
    log_url = job["raw_log_url"]
    output = urllib.request.urlopen(get_build_log_api_url(log_url)).read().decode("utf-8")
    return output


def process_build_log(failed_jobs_per_flag, log, job):
    # Remove this after Bazelisk returns non-zero when build fails without incompatible flags
    # See https://github.com/philwo/bazelisk/issues/34#issuecomment-467831202
    if "Failure: Command failed, even without incompatible flags." in log:
        return

    if "+++ Result" in log:
        index_success = log.rfind("Command was successful with the following flags:")
        index_failure = log.rfind("Migration is needed for the following flags:")
        if index_success== -1 or index_failure == -1:
            raise BuildkiteException("Cannot recognize log of " + job["web_url"])
        lines = log[index_failure:].split("\n")
        for line in lines:
            line = line.strip()
            if line.startswith("--incompatible_") and line in failed_jobs_per_flag:
                failed_jobs_per_flag[line].append(job)
        return

    if "Success: No migration needed." not in log:
        raise BuildkiteException("Cannot recognize log of " + job["web_url"])


def get_html_link_text(content, link):
    return f"<a href=\"{link}\" target=\"_blank\">{content}</a>"<commit_msg>Remove success message check, some jobs doesn't need to have this (eg. Setup XXX)<commit_after>#!/usr/bin/env python3
#
# Copyright 2018 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import base64
import json
import os
import re
import subprocess
import sys
import threading
import yaml
import urllib.request

import bazelci
from bazelci import BuildkiteException

BUILD_STATUS_API_URL = "https://api.buildkite.com/v2/organizations/bazel/pipelines/bazelisk-plus-incompatible-flags/builds/"

ENCRYPTED_BUILDKITE_API_TOKEN = """
<KEY>
""".strip()

INCOMPATIBLE_FLAGS = bazelci.fetch_incompatible_flags()

class LogFetcher(threading.Thread):
   def __init__(self, job):
      threading.Thread.__init__(self)
      self.job = job
      self.log = None

   def run(self):
      self.log = get_build_log(self.job)


def buildkite_token():
    return (
        subprocess.check_output(
            [
                bazelci.gcloud_command(),
                "kms",
                "decrypt",
                "--project",
                "bazel-untrusted",
                "--location",
                "global",
                "--keyring",
                "buildkite",
                "--key",
                "buildkite-untrusted-api-token",
                "--ciphertext-file",
                "-",
                "--plaintext-file",
                "-",
            ],
            input=base64.b64decode(ENCRYPTED_BUILDKITE_API_TOKEN),
            env=os.environ,
        )
        .decode("utf-8")
        .strip()
    )


def get_build_status_api_url(build_number):
    return BUILD_STATUS_API_URL + "%s?access_token=%s" % (build_number, buildkite_token())


def get_build_log_api_url(raw_log_url):
    return raw_log_url + "?access_token=" + buildkite_token()


def get_build_info(build_number):
    output = urllib.request.urlopen(get_build_status_api_url(build_number)).read().decode("utf-8")
    build_info = json.loads(output)
    return build_info


def get_build_log(job):
    log_url = job["raw_log_url"]
    output = urllib.request.urlopen(get_build_log_api_url(log_url)).read().decode("utf-8")
    return output


def process_build_log(failed_jobs_per_flag, log, job):
    # Remove this after Bazelisk returns non-zero when build fails without incompatible flags
    # See https://github.com/philwo/bazelisk/issues/34#issuecomment-467831202
    if "Failure: Command failed, even without incompatible flags." in log:
        return

    if "+++ Result" in log:
        index_success = log.rfind("Command was successful with the following flags:")
        index_failure = log.rfind("Migration is needed for the following flags:")
        if index_success== -1 or index_failure == -1:
            raise BuildkiteException("Cannot recognize log of " + job["web_url"])
        lines = log[index_failure:].split("\n")
        for line in lines:
            line = line.strip()
            if line.startswith("--incompatible_") and line in failed_jobs_per_flag:
                failed_jobs_per_flag[line].append(job)
        return


def get_html_link_text(content, link):
    return f"<a href=\"{link}\" target=\"_blank\">{content}</a>"
{'fraction_non_alphanumeric': 0.10665414542527502, 'fraction_numerical': 0.008585994097129058, 'mean_word_length': 2.786185881157948, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1457004', 'n_tokens_mistral': 2327, 'n_tokens_neox': 2217, 'n_words': 564}