<commit_before># Copyright 2017-2021 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import yaml

import guild

from guild import guildfile
from guild import model as modellib
from guild import plugin as pluginlib

dask_scheduler_description = """
Start a Dask scheduler.

Dask schedulers execute staged runs in parallel according to their \
number of workers. Specify the number of workers using the workers \
flags.

If the Bokeh Python package is installed, the schduler runs a \
dashboard application by default on port 8787. Specify a different \
port or binding address using the dashboard-address flag. To disable \
the dashboard, specify no for dashboard-address.

A Dask scheduler polls for staged runs and starts them in the order \
they were staged. Dask schedulers are queues that can process runs in \
parallel.

By default, a Dask scheduler runs staged runs even if there are other \
runs in progress. To force a scheduler to wait until other runs finish \
before starting a staged run, set wait-for-running to true when \
starting the scheduler.

Use run-once to start staged runs and stop without waiting for \
additional staged runs.
"""

dask_scheduler_flags_data = yaml.safe_load(
    """
workers:
  description: Number of workers in the Dask cluster
  type: int
loglevel:
  description: Log level used for Dask scheduler
  default: warn
  choices: [debug, info, warn, error]
dashboard-address:
  description: >
    Address to listen to for dashboard connections

    You can specify a port or an address like '0.0.0.0:8787'. Use no
    to disable the dashboard. Use 0 for a randomly assigned port.
  default: 8787
poll-interval:
  description: Minimum number of seconds between polls
  default: 10
  type: int
run-once:
  description: Run all staged runs and stop
  default: no
  arg-switch: yes
  type: boolean
wait-for-running:
  description: Wait for other runs to stop before starting staged runs<commit_msg>Dask scheduler CLI help<commit_after># Copyright 2017-2021 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import yaml

import guild

from guild import guildfile
from guild import model as modellib
from guild import plugin as pluginlib

dask_scheduler_description = """
Start a Dask scheduler.

Dask schedulers execute staged runs in parallel according to their \
number of workers. Specify the number of workers using the workers \
flags.

If the Bokeh Python package is installed, the schduler runs a \
dashboard application by default on port 8787. Specify a different \
port or binding address using the dashboard-address flag. To disable \
the dashboard, specify no for dashboard-address.

A Dask scheduler polls for staged runs and starts them in the order \
they were staged. Dask schedulers are queues that can process runs in \
parallel.

By default, a Dask scheduler runs staged runs even if there are other \
runs in progress. To force a scheduler to wait until other runs finish \
before starting a staged run, set wait-for-running to true when \
starting the scheduler.

Use run-once to start staged runs and stop without waiting for \
additional staged runs.
"""

dask_scheduler_flags_data = yaml.safe_load(
    """
workers:
  description: >
    Number of workers in the Dask cluster

    By default, the cluster uses one worker per CPU core available on
    the system.
  type: int
loglevel:
  description: Log level used for Dask scheduler
  default: warn
  choices: [debug, info, warn, error]
dashboard-address:
  description: >
    Address to listen to for dashboard connections

    You can specify a port or an address like '0.0.0.0:8787'. Use no
    to disable the dashboard. Use 0 for a randomly assigned port.
  default: 8787
poll-interval:
  description: Minimum number of seconds between polls
  default: 10
  type: int
run-once:
  description: Run all staged runs and stop
  default: no
  arg-switch: yes
  type: boolean
wait-for-running:
  description: Wait for other runs to stop before starting staged runs
{'fraction_non_alphanumeric': 0.05084745762711865, 'fraction_numerical': 0.01207870640950711, 'mean_word_length': 4.028403525954946, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3998533', 'n_tokens_mistral': 1421, 'n_tokens_neox': 1338, 'n_words': 724}