<commit_before># Copyright 2014 The Swarming Authors. All rights reserved.
# Use of this source code is governed by the Apache v2.0 license that can be
# found in the LICENSE file.

"""Task's common code and definition."""

import datetime

from google.appengine.api import datastore_errors

from components import utils


# The production server must handle up to 1000 task requests per second. The
# number of root entities must be a few orders of magnitude higher. The goal is
# to almost completely get rid of transactions conflicts. This means that the
# probability of two transactions happening on the same shard must be very low.
# This relates to number of transactions per second * seconds per transaction /
# number of shard.
#
# Intentionally starve the canary server by using only 256 root entities. This
# will force transaction conflicts. On the production server, use 16**5 (~1
# million) root entities to reduce the number of transaction conflict.
# TODO(maruel): Testing for load test, 5 on canary.
SHARDING_LEVEL = 2 if utils.is_canary() else 5


# Used to encode time.
UNIX_EPOCH = datetime.datetime(1970, 1, 1)


# Maximum acceptable priority value, which is effectively the lowest priority.
MAXIMUM_PRIORITY = 255


# Maximum number of shards for a single request.
MAXIMUM_SHARDS = 255


# Amount of time after which a bot is considered dead. In short, if a bot has
# not ping in the last 5 minutes while running a task, it is considered dead.
BOT_PING_TOLERANCE = datetime.timedelta(seconds=5*60)

<commit_msg>Increases sharding on the canary master from 256 to 4096.<commit_after># coding: utf-8
# Copyright 2014 The Swarming Authors. All rights reserved.
# Use of this source code is governed by the Apache v2.0 license that can be
# found in the LICENSE file.

"""Task's common code and definition."""

import datetime

from google.appengine.api import datastore_errors

from components import utils


# The production server must handle up to 1000 task requests per second. The
# number of root entities must be a few orders of magnitude higher. The goal is
# to almost completely get rid of transactions conflicts. This means that the
# probability of two transactions happening on the same shard must be very low.
# This relates to number of transactions per second * seconds per transaction /
# number of shard.
#
# Intentionally starve the canary server by using only 16³=4096 root entities.
# This will cause mild transaction conflicts during load tests. On the
# production server, use 16**5 (~1 million) root entities to reduce the number
# of transaction conflict.
SHARDING_LEVEL = 3 if utils.is_canary() else 5


# Used to encode time.
UNIX_EPOCH = datetime.datetime(1970, 1, 1)


# Maximum acceptable priority value, which is effectively the lowest priority.
MAXIMUM_PRIORITY = 255


# Maximum number of shards for a single request.
MAXIMUM_SHARDS = 255


# Amount of time after which a bot is considered dead. In short, if a bot has
# not ping in the last 5 minutes while running a task, it is considered dead.
BOT_PING_TOLERANCE = datetime.timedelta(seconds=5*60)


{'fraction_non_alphanumeric': 0.053479381443298966, 'fraction_numerical': 0.026417525773195876, 'mean_word_length': 4.335051546391752, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1996893', 'n_tokens_mistral': 906, 'n_tokens_neox': 818, 'n_words': 449}