<commit_before># Copyright 2017 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import logging
import os
import re
import shlex
import subprocess
import sys

import guild.plugin
import guild.run

from guild import deps
from guild import opref
from guild import util
from guild import var

log = logging.getLogger("core")

OP_RUNFILES = [
    "org_click",
    "org_psutil"
]

class InvalidCmd(ValueError):
    pass

class Operation(object):

    def __init__(self, model, opdef, rundir=None):
        self.model = model
        self.opdef = opdef
        self.cmd_args = _init_cmd_args(opdef)
        self.cmd_env = _init_cmd_env(opdef)
        self._rundir = rundir
        self._running = False
        self._started = None
        self._stopped = None
        self._run = None
        self._proc = None
        self._exit_status = None

    @property
    def rundir(self):
        return self._rundir or (self._run.path if self._run else None)

    def run(self):
        assert not self._running
        self._running = True
        self._started = guild.run.timestamp()
        self._init_run()
        self._init_attrs()
        self._resolve_deps()
        self._start_proc()
        self._wait_for_proc()
        self._finalize_attrs()
        return self._exit_status

    def _init_run(self):
        if self._rundir:
            run_id = os.path.basename(self._rundir)
            path = self._rundir
        else:
            run_id = guild.run.mkid()
            path = os.path.join(var.runs_dir(), run_id)
        self._run = guild.run.Run(run_id, path)
        log.debug("initializing run in %s", path)
        self._run.init_skel()

    def _init_attrs(self):
        assert self._run is not None
        self._run.write_attr("opref", self._opref_attr())
        self._run.write_attr("flags", self.opdef.flag_values())
        self._run.write_attr("cmd", self.cmd_args)
        self._run.write_attr("env", self.cmd_env)
        self._run.write_attr("started", self._started)

    def _opref_attr(self):
        ref = opref.OpRef.from_op(self.opdef.name, self.model.reference)
        return str(ref)

    def _resolve_deps(self):
        assert self._run is not None
        ctx = deps.ResolutionContext(
            target_dir=self._run.path,
            opdef=self.opdef)
        deps.resolve(self.opdef.dependencies, ctx)

    def _start_proc(self):
        assert self._proc is None
        assert self._run is not None
        args = self._proc_args()
        env = self._proc_env()
        cwd = self._run.path
        log.debug("starting operation run %s", self._run.id)
        log.debug("operation command: %s", args)
        log.debug("operation env: %s", env)
        log.debug("operation cwd: %s", cwd)
        self._proc = subprocess.Popen(args, env=env, cwd=cwd)
        _write_proc_lock(self._proc, self._run)

    def _proc_args(self):
        assert self._run
        return self.cmd_args + ["--rundir", self._run.path]

    def _proc_env(self):
        assert self._run is not None
        env = {}
        env.update(self.cmd_env)
        env["RUNDIR"] = self._run.path
        return env
<commit_msg>Don't include a --rundir opt in op commands<commit_after># Copyright 2017 TensorHub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division

import logging
import os
import re
import shlex
import subprocess
import sys

import guild.plugin
import guild.run

from guild import deps
from guild import opref
from guild import util
from guild import var

log = logging.getLogger("core")

OP_RUNFILES = [
    "org_click",
    "org_psutil"
]

class InvalidCmd(ValueError):
    pass

class Operation(object):

    def __init__(self, model, opdef, rundir=None):
        self.model = model
        self.opdef = opdef
        self.cmd_args = _init_cmd_args(opdef)
        self.cmd_env = _init_cmd_env(opdef)
        self._rundir = rundir
        self._running = False
        self._started = None
        self._stopped = None
        self._run = None
        self._proc = None
        self._exit_status = None

    @property
    def rundir(self):
        return self._rundir or (self._run.path if self._run else None)

    def run(self):
        assert not self._running
        self._running = True
        self._started = guild.run.timestamp()
        self._init_run()
        self._init_attrs()
        self._resolve_deps()
        self._start_proc()
        self._wait_for_proc()
        self._finalize_attrs()
        return self._exit_status

    def _init_run(self):
        if self._rundir:
            run_id = os.path.basename(self._rundir)
            path = self._rundir
        else:
            run_id = guild.run.mkid()
            path = os.path.join(var.runs_dir(), run_id)
        self._run = guild.run.Run(run_id, path)
        log.debug("initializing run in %s", path)
        self._run.init_skel()

    def _init_attrs(self):
        assert self._run is not None
        self._run.write_attr("opref", self._opref_attr())
        self._run.write_attr("flags", self.opdef.flag_values())
        self._run.write_attr("cmd", self.cmd_args)
        self._run.write_attr("env", self.cmd_env)
        self._run.write_attr("started", self._started)

    def _opref_attr(self):
        ref = opref.OpRef.from_op(self.opdef.name, self.model.reference)
        return str(ref)

    def _resolve_deps(self):
        assert self._run is not None
        ctx = deps.ResolutionContext(
            target_dir=self._run.path,
            opdef=self.opdef)
        deps.resolve(self.opdef.dependencies, ctx)

    def _start_proc(self):
        assert self._proc is None
        assert self._run is not None
        args = self._proc_args()
        env = self._proc_env()
        cwd = self._run.path
        log.debug("starting operation run %s", self._run.id)
        log.debug("operation command: %s", args)
        log.debug("operation env: %s", env)
        log.debug("operation cwd: %s", cwd)
        self._proc = subprocess.Popen(args, env=env, cwd=cwd)
        _write_proc_lock(self._proc, self._run)

    def _proc_args(self):
        return self.cmd_args

    def _proc_env(self):
        assert self._run is not None
        env = {}
        env.update(self.cmd_env)
        env["RUNDIR"] = self._run.path
        return env

{'fraction_non_alphanumeric': 0.09716599190283401, 'fraction_numerical': 0.0021592442645074223, 'mean_word_length': 2.495754716981132, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '166601', 'n_tokens_mistral': 2306, 'n_tokens_neox': 2254, 'n_words': 621}