<commit_before>#!/usr/bin/env python3

import sys
import os
import os.path
import pickle
import git
import requests
import shlex
import subprocess
import time


ASSET_FILES = [
    ".github/workflows/assets.yml",
    "public",
    "ui",
    "package.json",
    "yarn.lock",
]

ASSET_BUILDS_URL = "https://api.github.com/repos/ornicar/lila/actions/workflows/assets.yml/runs"

PROFILES = {
    "khiaw": {
        "ssh": "<EMAIL>",
        "files": ASSET_FILES,
        "workflow_url": ASSET_BUILDS_URL,
        "artifact_name": "lila-assets",
    }
}


def hash_files(tree, files):
    return tuple(tree[path].hexsha for path in files)


def find_commits(commit, files, wanted_hash):
    try:
        if hash_files(commit.tree, files) != wanted_hash:
            return
    except KeyError:
        return

    yield commit.hexsha

    for parent in commit.parents:
        yield from find_commits(parent, files, wanted_hash)


def workflow_runs(session, repo):
    with open(os.path.join(repo.common_dir, "workflow_runs.pickle"), "ab+") as f:
        try:
            f.seek(0)
            data = pickle.load(f)
        except EOFError:
            print("Created workflow run database.")
            data = {}

        try:
            new = 0
            synced = False
            url = ASSET_BUILDS_URL

            while not synced:
                print("Fetching workflow runs ...")
                res = session.get(url)
                if res.status_code != 200:
                    print(f"Unexpected response: {res.status_code} {res.text}")
                    break

                for run in res.json()["workflow_runs"]:
                    if run["id"] in data and data[run["id"]]["status"] == "completed":
                        synced = True
                    else:
                        new += 1
                    data[run["id"]] = run

                if "next" not in res.links:
                    break
                url = res.links["next"]["url"]
        finally:
            f.seek(0)
            f.truncate()
            pickle.dump(data, f)
            print(f"Added/updated {new} workflow run(s).")

        return data


def find_workflow_run(runs, wanted_commits):
    found = None

    print("Matching workflow runs:")
    for run in runs.values():
        if run["head_commit"]["id"] not in wanted_commits:
            continue

        if run["status"] != "completed":
            print(f"- {run['html_url']} pending.")
        elif run["conclusion"] != "success":
            print(f"- {run['html_url']} failed.")
        else:
            print(f"- {run['html_url']} succeeded.")
            if found is None:
                found = run

    if found is None:
        raise RuntimeError("Did not find successful matching workflow run.")

    print(f"Selected {found['html_url']}.")
    return found


def artifact_url(session, run, name):
    for artifact in session.get(run["artifacts_url"]).json()["artifacts"]:
        if artifact["name"] == name:
            if artifact["expired"]:
                print("Artifact expired.")
            return artifact["archive_download_url"]

    raise RuntimeError(f"Did not find artifact {name}.")


def main():
    try:
        github_api_token = os.environ["GITHUB_API_TOKEN"]
    except KeyError:
        print("Need environment variable GITHUB_API_TOKEN. See https://github.com/settings/tokens/new. Scope public_repo.")
        return 128

    session = requests.Session()
    session.headers["Authorization"] = f"token {github_api_token}"

    repo = git.Repo(search_parent_directories=True)
    runs = workflow_runs(session, repo)

    return deploy(PROFILES["khiaw"], session, repo, runs)


def deploy(profile, session, repo, runs):
    try:
        wanted_hash = hash_files(repo.head.commit.tree, profile["files"])
    except KeyError:
        print("Commit is missing asset file.")
        return 1

    wanted_commits = set(find_commits(repo.head.commit, profile["files"], wanted_hash))
    print(f"Found {len(wanted_commits)} matching commits.")

    run = find_workflow_run(runs, wanted_commits)
    url = artifact_url(session, run, profile["artifact_name"])

    print(f"Deploying {url} to {profile['ssh']} in 10s ...")
    time.sleep(5)
    header = f"Authorization: {session.headers['Authorization']}"
    artifact_target = f"/home/lichess-artifacts/{profile['artifact_name']}-{run['id']:d}.zip"
    command = ";".join([
        f"mkdir -p /home/lichess-artifacts",
        f"mkdir -p /home/lichess-deploy",
        f"wget --header={shlex.quote(header)} -O {shlex.quote(artifact_target)} --no-clobber {shlex.quote(url)}",
        f"unzip -q -o {shlex.quote(artifact_target)} -d /home/lichess-artifacts/{profile['artifact_name']}-{run['id']:d}",
        f"cat /home/lichess-artifacts/{profile['artifact_name']}-{run['id']:d}/commit.txt",
        f"ln -f -s /home/lichess-artifacts/{profile['artifact_name']}-{run['id']:d}/public /home/lichess-deploy/public",
        "/bin/bash",
    ])
    return subprocess.call(["ssh", "-t", "root@khiaw.lichess.ovh", "tmux", "new-session", "-s", "lila-deploy", f"/bin/sh -c {shlex.quote(command)}"], stdout=sys.stdout, stdin=sys.stdin)


if __name__ == "__main__":
    sys.exit(main())
<commit_msg>deploy profiles<commit_after>#!/usr/bin/env python3

import sys
import os
import os.path
import pickle
import git
import requests
import shlex
import subprocess
import time


ASSETS_FILES = [
    ".github/workflows/assets.yml",
    "public",
    "ui",
    "package.json",
    "yarn.lock",
]

SERVER_FILES = [
    ".github/workflows/server.yml",
    "app",
    "conf",
    "modules",
    "project",
    "build.sbt",
]

ASSETS_BUILD_URL = "https://api.github.com/repos/ornicar/lila/actions/workflows/assets.yml/runs"

SERVER_BUILD_URL = "https://api.github.com/repos/ornicar/lila/actions/workflows/server.yml/runs"

PROFILES = {
    "khiaw-assets": {
        "ssh": "<EMAIL>",
        "files": ASSETS_FILES,
        "workflow_url": ASSETS_BUILD_URL,
        "artifact_name": "lila-assets",
    },
    "khiaw-server": {
        "ssh": "<EMAIL>",
        "files": SERVER_FILES,
        "workflow_url": SERVER_BUILD_URL,
        "artifact_name": "lila-server",
    },
}


def hash_files(tree, files):
    return tuple(tree[path].hexsha for path in files)


def find_commits(commit, files, wanted_hash):
    try:
        if hash_files(commit.tree, files) != wanted_hash:
            return
    except KeyError:
        return

    yield commit.hexsha

    for parent in commit.parents:
        yield from find_commits(parent, files, wanted_hash)


def workflow_runs(profile, session, repo):
    with open(os.path.join(repo.common_dir, "workflow_runs.pickle"), "ab+") as f:
        try:
            f.seek(0)
            data = pickle.load(f)
        except EOFError:
            print("Created workflow run database.")
            data = {}

        try:
            new = 0
            synced = False
            url = profile["workflow_url"]

            while not synced:
                print("Fetching workflow runs ...")
                res = session.get(url)
                if res.status_code != 200:
                    print(f"Unexpected response: {res.status_code} {res.text}")
                    break

                for run in res.json()["workflow_runs"]:
                    if run["id"] in data and data[run["id"]]["status"] == "completed":
                        synced = True
                    else:
                        new += 1
                    data[run["id"]] = run

                if "next" not in res.links:
                    break
                url = res.links["next"]["url"]
        finally:
            f.seek(0)
            f.truncate()
            pickle.dump(data, f)
            print(f"Added/updated {new} workflow run(s).")

        return data


def find_workflow_run(runs, wanted_commits):
    found = None

    print("Matching workflow runs:")
    for run in runs.values():
        if run["head_commit"]["id"] not in wanted_commits:
            continue

        if run["status"] != "completed":
            print(f"- {run['html_url']} pending.")
        elif run["conclusion"] != "success":
            print(f"- {run['html_url']} failed.")
        else:
            print(f"- {run['html_url']} succeeded.")
            if found is None:
                found = run

    if found is None:
        raise RuntimeError("Did not find successful matching workflow run.")

    print(f"Selected {found['html_url']}.")
    return found


def artifact_url(session, run, name):
    for artifact in session.get(run["artifacts_url"]).json()["artifacts"]:
        if artifact["name"] == name:
            if artifact["expired"]:
                print("Artifact expired.")
            return artifact["archive_download_url"]

    raise RuntimeError(f"Did not find artifact {name}.")


def main(profile):
    try:
        github_api_token = os.environ["GITHUB_API_TOKEN"]
    except KeyError:
        print("Need environment variable GITHUB_API_TOKEN. See https://github.com/settings/tokens/new. Scope public_repo.")
        return 128

    session = requests.Session()
    session.headers["Authorization"] = f"token {github_api_token}"

    repo = git.Repo(search_parent_directories=True)
    runs = workflow_runs(profile, session, repo)

    return deploy(profile, session, repo, runs)


def deploy(profile, session, repo, runs):
    try:
        wanted_hash = hash_files(repo.head.commit.tree, profile["files"])
    except KeyError:
        print("Commit is missing asset file.")
        return 1

    wanted_commits = set(find_commits(repo.head.commit, profile["files"], wanted_hash))
    print(f"Found {len(wanted_commits)} matching commits.")

    run = find_workflow_run(runs, wanted_commits)
    url = artifact_url(session, run, profile["artifact_name"])

    print(f"Deploying {url} to {profile['ssh']} in 10s ...")
    time.sleep(5)
    header = f"Authorization: {session.headers['Authorization']}"
    artifact_target = f"/home/lichess-artifacts/{profile['artifact_name']}-{run['id']:d}.zip"
    command = ";".join([
        f"mkdir -p /home/lichess-artifacts",
        f"mkdir -p /home/lichess-deploy",
        f"wget --header={shlex.quote(header)} -O {shlex.quote(artifact_target)} --no-clobber {shlex.quote(url)}",
        f"unzip -q -o {shlex.quote(artifact_target)} -d /home/lichess-artifacts/{profile['artifact_name']}-{run['id']:d}",
        f"cat /home/lichess-artifacts/{profile['artifact_name']}-{run['id']:d}/commit.txt",
        f"ln -f -s /home/lichess-artifacts/{profile['artifact_name']}-{run['id']:d}/public /home/lichess-deploy/public",
        "/bin/bash",
    ])
    return subprocess.call(["ssh", "-t", "<EMAIL>", "tmux", "new-session", "-s", "lila-deploy", f"/bin/sh -c {shlex.quote(command)}"], stdout=sys.stdout, stdin=sys.stdin)


if __name__ == "__main__":
    sys.exit(main(PROFILES[sys.argv[1]]))

{'fraction_non_alphanumeric': 0.14587155963302753, 'fraction_numerical': 0.0028440366972477065, 'mean_word_length': 2.4097591492023773, 'pattern_counts': {'":': 23, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5539384', 'n_tokens_mistral': 3592, 'n_tokens_neox': 3400, 'n_words': 725}