<commit_before># coding: utf8

# Copyright 2018 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

"""
Project-independent library for Taskcluster decision tasks
"""

import datetime
import hashlib
import json
import os
import re
import sys
import taskcluster


class DecisionTask:
    DOCKER_IMAGE_ARTIFACT_FILENAME = "image.tar.lz4"

    # https://github.com/servo/taskcluster-bootstrap-docker-images#image-builder
    DOCKER_IMAGE_BUILDER_IMAGE = "servobrowser/taskcluster-bootstrap:image-builder@sha256:" \
        "0a7d012ce444d62ffb9e7f06f0c52fedc24b68c2060711b313263367f7272d9d"

    def __init__(self, project_name, *, route_prefix,
                 worker_type="github-worker", docker_image_cache_expiry="1 year"):
        self.project_name = project_name
        self.route_prefix = route_prefix
        self.worker_type = worker_type
        self.docker_image_cache_expiry = docker_image_cache_expiry

        # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/features#feature-taskclusterproxy
        self.queue_service = taskcluster.Queue(options={"baseUrl": "http://taskcluster/queue/v1/"})
        self.index_service = taskcluster.Index(options={"baseUrl": "http://taskcluster/index/v1/"})

        self.now = datetime.datetime.utcnow()
        self.found_or_created_routes = {}

    def from_now_json(self, offset):
        return taskcluster.stringDate(taskcluster.fromNow(offset, dateObj=self.now))

    def find_or_create_task(self, *, route_bucket, route_key, route_expiry, artifacts, **kwargs):
        route = "%s.%s.%s" % (self.route_prefix, route_bucket, route_key)

        task_id = self.found_or_created_routes.get(route)
        if task_id is not None:
            return task_id

        try:
            result = self.index_service.findTask(route)
            task_id = result["taskId"]
        except taskcluster.TaskclusterRestFailure as e:
            if e.status_code == 404:
                task_id = self.create_task(
                    routes=[
                        "index." + route,
                    ],
                    extra={
                        "index": {
                            "expires": self.from_now_json(self.docker_image_cache_expiry),
                        },
                    },
                    artifacts=[
                        (artifact, route_expiry)
                        for artifact in artifacts
                    ],
                    **kwargs
                )
            else:
                raise

        self.found_or_created_routes[route] = task_id
        return task_id

    def find_or_build_docker_image(self, dockerfile):
        dockerfile_contents = expand_dockerfile(dockerfile)
        digest = hashlib.sha256(dockerfile_contents).hexdigest()

        return self.find_or_create_task(
            route_bucket="docker-image",
            route_key=digest,
            route_expiry=self.docker_image_cache_expiry,

            task_name="Docker image build task for image: " + image_name(dockerfile),
            script="""
                echo "$DOCKERFILE" | docker build -t taskcluster-built -
                docker save taskcluster-built | lz4 > /%s
            """ % self.DOCKER_IMAGE_ARTIFACT_FILENAME,
            env={
                "DOCKERFILE": dockerfile_contents,
            },
            artifacts=[
                "/" + self.DOCKER_IMAGE_ARTIFACT_FILENAME,
            ],
            max_run_time_minutes=20,
            docker_image=self.DOCKER_IMAGE_BUILDER_IMAGE,
            features={
                "dind": True,  # docker-in-docker
            },<commit_msg>Shorten those task names<commit_after># coding: utf8

# Copyright 2018 The Servo Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

"""
Project-independent library for Taskcluster decision tasks
"""

import datetime
import hashlib
import json
import os
import re
import sys
import taskcluster


class DecisionTask:
    DOCKER_IMAGE_ARTIFACT_FILENAME = "image.tar.lz4"

    # https://github.com/servo/taskcluster-bootstrap-docker-images#image-builder
    DOCKER_IMAGE_BUILDER_IMAGE = "servobrowser/taskcluster-bootstrap:image-builder@sha256:" \
        "0a7d012ce444d62ffb9e7f06f0c52fedc24b68c2060711b313263367f7272d9d"

    def __init__(self, project_name, *, route_prefix,
                 worker_type="github-worker", docker_image_cache_expiry="1 year"):
        self.project_name = project_name
        self.route_prefix = route_prefix
        self.worker_type = worker_type
        self.docker_image_cache_expiry = docker_image_cache_expiry

        # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/features#feature-taskclusterproxy
        self.queue_service = taskcluster.Queue(options={"baseUrl": "http://taskcluster/queue/v1/"})
        self.index_service = taskcluster.Index(options={"baseUrl": "http://taskcluster/index/v1/"})

        self.now = datetime.datetime.utcnow()
        self.found_or_created_routes = {}

    def from_now_json(self, offset):
        return taskcluster.stringDate(taskcluster.fromNow(offset, dateObj=self.now))

    def find_or_create_task(self, *, route_bucket, route_key, route_expiry, artifacts, **kwargs):
        route = "%s.%s.%s" % (self.route_prefix, route_bucket, route_key)

        task_id = self.found_or_created_routes.get(route)
        if task_id is not None:
            return task_id

        try:
            result = self.index_service.findTask(route)
            task_id = result["taskId"]
        except taskcluster.TaskclusterRestFailure as e:
            if e.status_code == 404:
                task_id = self.create_task(
                    routes=[
                        "index." + route,
                    ],
                    extra={
                        "index": {
                            "expires": self.from_now_json(self.docker_image_cache_expiry),
                        },
                    },
                    artifacts=[
                        (artifact, route_expiry)
                        for artifact in artifacts
                    ],
                    **kwargs
                )
            else:
                raise

        self.found_or_created_routes[route] = task_id
        return task_id

    def find_or_build_docker_image(self, dockerfile):
        dockerfile_contents = expand_dockerfile(dockerfile)
        digest = hashlib.sha256(dockerfile_contents).hexdigest()

        return self.find_or_create_task(
            route_bucket="docker-image",
            route_key=digest,
            route_expiry=self.docker_image_cache_expiry,

            task_name="Docker image: " + image_name(dockerfile),
            script="""
                echo "$DOCKERFILE" | docker build -t taskcluster-built -
                docker save taskcluster-built | lz4 > /%s
            """ % self.DOCKER_IMAGE_ARTIFACT_FILENAME,
            env={
                "DOCKERFILE": dockerfile_contents,
            },
            artifacts=[
                "/" + self.DOCKER_IMAGE_ARTIFACT_FILENAME,
            ],
            max_run_time_minutes=20,
            docker_image=self.DOCKER_IMAGE_BUILDER_IMAGE,
            features={
                "dind": True,  # docker-in-docker
            },
{'fraction_non_alphanumeric': 0.09212518929833417, 'fraction_numerical': 0.01691065118626956, 'mean_word_length': 2.175080128205128, 'pattern_counts': {'":': 12, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3774076', 'n_tokens_mistral': 2439, 'n_tokens_neox': 2230, 'n_words': 423}