<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""run_bulk_pipelines.py - Start multiple pipeline instances from a template

Usage: run_bulk_pipelines.py template_uuid project_uuid [param [param ...]]

This script starts multiple pipeline instances from a single template, with
different parameters you specify.  It also creates a new subproject to hold
each pipeline instance and its associated data, under the specified project.

Each pipeline instance is started with parameters specified on the command
line, using the same syntax as arv-run-pipeline-instance.  It also reads
parameters from standard input by default, or one or more input files named
with `--input`.  Each line of the input creates a new pipeline instance, with
the parameters on that line.  The input follows shell quoting rules, so if you
need to include a literal newline, quote, or backslash in an input parameter,
you can write that in the input the same way you would write it when you run
arv-run-pipeline-instance in your shell.

If an input line sets a `PROJECT_NAME` parameter (`PROJECT_NAME="..."`), that
will be used as the name for the subproject created for the instance.  You
can change the name of this parameter with the `--project-name-param` option.
This parameter will not be passed to the pipeline instance.
"""
# Copyright 2016 Curoverse
# Written by <NAME> <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import argparse
import collections
import functools
import io
import locale
import logging
import shlex
import string
import subprocess
import sys
import time

import arvados

from arvados.api import OrderedJsonModel

logger = logging.getLogger('run_bulk_pipelines')

class ParameterParser(shlex.shlex, object):
    # shlex module uses iso-8859-1 encoding, at least in Python 2.7.
    SHLEX_CODING = 'iso-8859-1'

    def __init__(self, instream=None, infile=None):
        self.token = None
        super(ParameterParser, self).__init__(instream, infile, posix=True)
        self.wordchars = self.wordchars.decode(self.SHLEX_CODING)
        addchars = set(unicode(string.punctuation))
        addchars.difference_update(
            (self.quotes + self.escape + self.escapedquotes).decode(self.SHLEX_CODING))
        self.wordchars += ''.join(addchars)

    def read_token(self):
        self.token_lineno = self.lineno
        return super(ParameterParser, self).read_token()

    @property
    def lineno(self):
        return self._lineno

    @lineno.setter
    def lineno(self, value):
        self._lineno = value
        if not self.token:
            self.token_lineno = self._lineno

    def iterlines(self):
        """Iterate logical lines in the source.

        This method yields a list of tokens up to each unescaped newline in
        the source.
        """
        line = []
        on_lineno = self.lineno
        for token in self:
            for _ in xrange(on_lineno, self.token_lineno):
                yield line
                line = []
            line.append(token)
            on_lineno = self.token_lineno + token.count('\n')
        for _ in xrange(on_lineno, self.lineno):
            yield line
            line = []
        if line:
            yield line

    @classmethod
    def iter_parameters(cls, token_seq):
        assign_name = None
        for token in token_seq:
            if assign_name is not None:
                yield (assign_name, token)
                assign_name = None
            else:
                try:
                    name, value = token.split('=', 1)
                except ValueError:
                    assign_name = token
                else:
                    yield (name, value)


class PipelineRun(object):
    def __init__(self, project_uuid, template_uuid, parameters,
                 project_name=None, dry_run=False):
        self.parent_project_uuid = project_uuid
        self.template_uuid = template_uuid
        self.parameters = ['{}={}'.format(key, parameters[key]) for key in parameters]
        if project_name is None:
            keys = list(self.parameters)
            if not keys:
                param_desc = "no parameters specified"
            else:
                keys.sort(key=lambda s: s.rsplit(':', 1)[-1])
                param_desc = " ".join(parameters[key] for key in keys[:3])
            project_name = "Run with " + param_desc
        self.project_name = project_name
        self.dry_run = dry_run
        self.error = None
        self.returncode = None

    def _create_subproject(self, arv, num_retries):
        logger.info("Creating project under %s named %r",
                    self.parent_project_uuid, self.project_name)
        if self.dry_run:
            return 'dry run'
        create_call = arv.groups().create(body={
            'group_class': 'project',
            'owner_uuid': self.parent_project_uuid,
            'name': self.project_name,
        }, ensure_unique_name=True)
        try:
            subproject = create_call.execute(num_retries=num_retries)
        except Exception as error:
            logger.warning("Failed to create project: %s", error,
                           exc_info=logger.isEnabledFor(logging.DEBUG))
            self._save_error(error)
            return None
        else:
            return subproject['uuid']

    def _submit_pipeline(self, project_uuid):<commit_msg>run_bulk_pipelines: Fix default project name.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""run_bulk_pipelines.py - Start multiple pipeline instances from a template

Usage: run_bulk_pipelines.py template_uuid project_uuid [param [param ...]]

This script starts multiple pipeline instances from a single template, with
different parameters you specify.  It also creates a new subproject to hold
each pipeline instance and its associated data, under the specified project.

Each pipeline instance is started with parameters specified on the command
line, using the same syntax as arv-run-pipeline-instance.  It also reads
parameters from standard input by default, or one or more input files named
with `--input`.  Each line of the input creates a new pipeline instance, with
the parameters on that line.  The input follows shell quoting rules, so if you
need to include a literal newline, quote, or backslash in an input parameter,
you can write that in the input the same way you would write it when you run
arv-run-pipeline-instance in your shell.

If an input line sets a `PROJECT_NAME` parameter (`PROJECT_NAME="..."`), that
will be used as the name for the subproject created for the instance.  You
can change the name of this parameter with the `--project-name-param` option.
This parameter will not be passed to the pipeline instance.
"""
# Copyright 2016 Curoverse
# Written by <NAME> <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import argparse
import collections
import functools
import io
import locale
import logging
import shlex
import string
import subprocess
import sys
import time

import arvados

from arvados.api import OrderedJsonModel

logger = logging.getLogger('run_bulk_pipelines')

class ParameterParser(shlex.shlex, object):
    # shlex module uses iso-8859-1 encoding, at least in Python 2.7.
    SHLEX_CODING = 'iso-8859-1'

    def __init__(self, instream=None, infile=None):
        self.token = None
        super(ParameterParser, self).__init__(instream, infile, posix=True)
        self.wordchars = self.wordchars.decode(self.SHLEX_CODING)
        addchars = set(unicode(string.punctuation))
        addchars.difference_update(
            (self.quotes + self.escape + self.escapedquotes).decode(self.SHLEX_CODING))
        self.wordchars += ''.join(addchars)

    def read_token(self):
        self.token_lineno = self.lineno
        return super(ParameterParser, self).read_token()

    @property
    def lineno(self):
        return self._lineno

    @lineno.setter
    def lineno(self, value):
        self._lineno = value
        if not self.token:
            self.token_lineno = self._lineno

    def iterlines(self):
        """Iterate logical lines in the source.

        This method yields a list of tokens up to each unescaped newline in
        the source.
        """
        line = []
        on_lineno = self.lineno
        for token in self:
            for _ in xrange(on_lineno, self.token_lineno):
                yield line
                line = []
            line.append(token)
            on_lineno = self.token_lineno + token.count('\n')
        for _ in xrange(on_lineno, self.lineno):
            yield line
            line = []
        if line:
            yield line

    @classmethod
    def iter_parameters(cls, token_seq):
        assign_name = None
        for token in token_seq:
            if assign_name is not None:
                yield (assign_name, token)
                assign_name = None
            else:
                try:
                    name, value = token.split('=', 1)
                except ValueError:
                    assign_name = token
                else:
                    yield (name, value)


class PipelineRun(object):
    def __init__(self, project_uuid, template_uuid, parameters,
                 project_name=None, dry_run=False):
        self.parent_project_uuid = project_uuid
        self.template_uuid = template_uuid
        self.parameters = ['{}={}'.format(key, parameters[key]) for key in parameters]
        if project_name is None:
            keys = list(self.parameters)
            if not keys:
                param_desc = "no parameters specified"
            else:
                keys.sort(key=lambda s: s.rsplit(':', 1)[-1])
                param_desc = " ".join("{}={}".format(key, parameters[key])
                                      for key in keys[:3])
            project_name = "Run with " + param_desc
        self.project_name = project_name
        self.dry_run = dry_run
        self.error = None
        self.returncode = None

    def _create_subproject(self, arv, num_retries):
        logger.info("Creating project under %s named %r",
                    self.parent_project_uuid, self.project_name)
        if self.dry_run:
            return 'dry run'
        create_call = arv.groups().create(body={
            'group_class': 'project',
            'owner_uuid': self.parent_project_uuid,
            'name': self.project_name,
        }, ensure_unique_name=True)
        try:
            subproject = create_call.execute(num_retries=num_retries)
        except Exception as error:
            logger.warning("Failed to create project: %s", error,
                           exc_info=logger.isEnabledFor(logging.DEBUG))
            self._save_error(error)
            return None
        else:
            return subproject['uuid']

    def _submit_pipeline(self, project_uuid):
{'fraction_non_alphanumeric': 0.0784718211073521, 'fraction_numerical': 0.004125752949913359, 'mean_word_length': 2.5232558139534884, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '197425', 'n_tokens_mistral': 3355, 'n_tokens_neox': 3234, 'n_words': 1148}