<commit_before># Copyright 2014 The LUCI Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.

"""Task execution result models.

This module doesn't do the scheduling itself. It only describes the entities to
store tasks results.

- TaskResultSummary represents the overall result for the TaskRequest taking in
  account retries.
- TaskRunResult represents the result for one 'try'. There can
  be multiple tries for one job, for example if a bot dies.
- The stdout of the task is saved under TaskOutput, chunked in TaskOutputChunk
  entities to fit the entity size limit.

Graph of schema:

    +--------Root-------+
    |TaskRequest        |                                        task_request.py
    |  +--------------+ |
    |  |TaskProperties| |
    |  |  +--------+  | |
    |  |  |FilesRef|  | |
    |  |  +--------+  | |
    |  +--------------+ |
    |id=<based on epoch>|
    +-------------------+
        |
        v
    +-----------------+
    |TaskResultSummary|
    |  +--------+     |
    |  |FilesRef|     |
    |  +--------+     |
    |id=1             |
    +-----------------+
        |
        +----------------+
        |                |
        v                v
    +-------------+  +-------------+
    |TaskRunResult|  |TaskRunResult|
    |  +--------+ |  |  +--------+ |
    |  |FilesRef| |  |  |FilesRef| |
    |  +--------+ |  |  +--------+ |
    |id=1 <try #> |  |id=2         |
    +-------------+  +-------------+
        |
        +--------------------+
        |                    |
        v                    v
    +-----------------+  +----------------+
    |TaskOutput       |  |PerformanceStats|
    |id=1 (not stored)|  |id=1            |
    +-----------------+  +----------------+
        |
        +------------ ... ----+
        |                     |
        v                     v
    +---------------+     +---------------+
    |TaskOutputChunk| ... |TaskOutputChunk|
    |id=1           | ... |id=N           |
    +---------------+     +---------------+
"""

import collections
import datetime
import logging
import random
import re

from google.appengine.api import datastore_errors
from google.appengine.datastore import datastore_query
from google.appengine.ext import ndb

from components import datastore_utils
from components import utils
from server import large
from server import task_pack
from server import task_request

import cipd

# Amount of time after which a bot is considered dead. In short, if a bot has
# not ping in the last 2 minutes while running a task, it is considered dead.
#
# task_runner.MAX_PACKET_INTERVAL is 30 seconds.
BOT_PING_TOLERANCE = datetime.timedelta(seconds=2*60)


class State(object):
  """Represents the current task state.

  For documentation, see the comments in the swarming_rpcs.TaskState enum, which
  is using the same values:
  https://cs.chromium.org/chromium/infra/luci/appengine/swarming/swarming_rpcs.py?q=TaskState\(

  It's in fact an enum.
  """
  RUNNING = 0x10      # 16
  PENDING = 0x20      # 32
  EXPIRED = 0x30      # 48
  TIMED_OUT = 0x40    # 64
  BOT_DIED = 0x50     # 80
  CANCELED = 0x60     # 96
  COMPLETED = 0x70    # 112
  KILLED = 0x80       # 128
  NO_RESOURCE = 0x100 # 256

  STATES = (
      RUNNING, PENDING, EXPIRED, TIMED_OUT, BOT_DIED, CANCELED, COMPLETED,
      KILLED, NO_RESOURCE)
  # State will mutate again. Anything else means not queued, not running.<commit_msg>swarming: increase BOT_PING_TOLERANCE to 6 minutes<commit_after># Copyright 2014 The LUCI Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.

"""Task execution result models.

This module doesn't do the scheduling itself. It only describes the entities to
store tasks results.

- TaskResultSummary represents the overall result for the TaskRequest taking in
  account retries.
- TaskRunResult represents the result for one 'try'. There can
  be multiple tries for one job, for example if a bot dies.
- The stdout of the task is saved under TaskOutput, chunked in TaskOutputChunk
  entities to fit the entity size limit.

Graph of schema:

    +--------Root-------+
    |TaskRequest        |                                        task_request.py
    |  +--------------+ |
    |  |TaskProperties| |
    |  |  +--------+  | |
    |  |  |FilesRef|  | |
    |  |  +--------+  | |
    |  +--------------+ |
    |id=<based on epoch>|
    +-------------------+
        |
        v
    +-----------------+
    |TaskResultSummary|
    |  +--------+     |
    |  |FilesRef|     |
    |  +--------+     |
    |id=1             |
    +-----------------+
        |
        +----------------+
        |                |
        v                v
    +-------------+  +-------------+
    |TaskRunResult|  |TaskRunResult|
    |  +--------+ |  |  +--------+ |
    |  |FilesRef| |  |  |FilesRef| |
    |  +--------+ |  |  +--------+ |
    |id=1 <try #> |  |id=2         |
    +-------------+  +-------------+
        |
        +--------------------+
        |                    |
        v                    v
    +-----------------+  +----------------+
    |TaskOutput       |  |PerformanceStats|
    |id=1 (not stored)|  |id=1            |
    +-----------------+  +----------------+
        |
        +------------ ... ----+
        |                     |
        v                     v
    +---------------+     +---------------+
    |TaskOutputChunk| ... |TaskOutputChunk|
    |id=1           | ... |id=N           |
    +---------------+     +---------------+
"""

import collections
import datetime
import logging
import random
import re

from google.appengine.api import datastore_errors
from google.appengine.datastore import datastore_query
from google.appengine.ext import ndb

from components import datastore_utils
from components import utils
from server import large
from server import task_pack
from server import task_request

import cipd

# Amount of time after which a bot is considered dead. In short, if a bot has
# not sent an update in the last N minutes while running a task, it is
# considered dead.
#
# task_runner.MAX_PACKET_INTERVAL is 30 seconds.
#
# https://crbug.com/863175 "Temporarily" increase from 2 minutes to 6 minutes,
# as some task workload is so overwhelming on the host that the bot process
# doesn't get CPU time for several minutes. This should be decreased once the
# bot processes run at higher priority.
BOT_PING_TOLERANCE = datetime.timedelta(seconds=6*60)


class State(object):
  """Represents the current task state.

  For documentation, see the comments in the swarming_rpcs.TaskState enum, which
  is using the same values:
  https://cs.chromium.org/chromium/infra/luci/appengine/swarming/swarming_rpcs.py?q=TaskState\(

  It's in fact an enum.
  """
  RUNNING = 0x10      # 16
  PENDING = 0x20      # 32
  EXPIRED = 0x30      # 48
  TIMED_OUT = 0x40    # 64
  BOT_DIED = 0x50     # 80
  CANCELED = 0x60     # 96
  COMPLETED = 0x70    # 112
  KILLED = 0x80       # 128
  NO_RESOURCE = 0x100 # 256

  STATES = (
      RUNNING, PENDING, EXPIRED, TIMED_OUT, BOT_DIED, CANCELED, COMPLETED,
      KILLED, NO_RESOURCE)
  # State will mutate again. Anything else means not queued, not running.
{'fraction_non_alphanumeric': 0.1884730947828462, 'fraction_numerical': 0.019393608303742146, 'mean_word_length': 1.4508032128514057, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3211415', 'n_tokens_mistral': 2303, 'n_tokens_neox': 2112, 'n_words': 642}