<commit_before>#!/usr/bin/python
#
# Main entry point for ads. See README for overall description; run with -h for
# usage.
#
# @author adamcath


import sys


def debug(msg):
    print(msg)
    sys.stdout.flush()


def info(msg):
    print("--- " + msg)
    sys.stdout.flush()


def error(msg):
    sys.stderr.write("!!! " + msg + "\n")
    sys.stderr.flush()


try:
    import yaml
except ImportError:
    error(
        "ads requires the python package 'pyyaml'.\n"
        "Please install it with 'pip install pyyaml' or 'easy_install pyyaml'\n"
        "(disregard the message about 'forcing --no-libyaml')")
    sys.exit(1)

import os
import tempfile
import subprocess
import argparse
import glob
from collections import OrderedDict


##############################################
# Treelisting
##############################################

class Treelisting:

    def __init__(self, sections=None):
        self.sections = sections or []

    def with_section(self, heading, listing_dict, empty_msg=None):
        self.sections.append((heading, listing_dict, empty_msg))
        return self

    def pretty_print(self):
        all_keys = [
            k
            for (heading, listing_dict, _) in self.sections
            for k in listing_dict.keys()]
        if len(all_keys) == 0:
            return
        column_width = max(map(len, all_keys)) + 1
        for (heading, listing_dict, empty_msg) in self.sections:
            empty = len(listing_dict) == 0
            print("\n" + heading + (empty and "\n " + empty_msg or ""))
            if not empty:
                for (k, v) in listing_dict.items():
                    print(("%" + str(column_width) + "s: %s") % (k, v))


##############################################
# subprocess stuff
##############################################

def _shell_get_output(cmd_str, working_dir):
    process = subprocess.Popen(
        cmd_str,
        shell=True,
        stdout=subprocess.PIPE,
        stderr=file("/dev/null", "w"),
        close_fds=True,
        cwd=working_dir)

    return process.communicate()[0]


def _shell(cmd_str, working_dir, quiet=False):
    if not quiet:
        debug("cd " + working_dir)
    # TODO Can we avoid this nonsense using shell=true?
    cmd_file = tempfile.NamedTemporaryFile()
    cmd_file.write(cmd_str)
    cmd_file.flush()
    if not quiet:
        debug(cmd_str)
    try:
        status = subprocess.Popen(
            ["/bin/bash", cmd_file.name],
            close_fds=True,
            cwd=working_dir,
            stdout=quiet and file("/dev/null", "w") or None,
            stderr=quiet and file("/dev/null", "w") or None).wait()
    except KeyboardInterrupt:
        # Suppress python from printing a stack trace
        status = 47
        pass
    cmd_file.close()
    return status == 0


def _shell_quiet(cmd_str, working_dir):
    return _shell(cmd_str, working_dir, True)


##############################################
# YML stuff
##############################################

class ParseProjectException(Exception):

    def __init__(self, msg):
        super(ParseProjectException, self).__init__(msg)


def _expect(expected_type, actual, origin_file):
    if not isinstance(actual, expected_type):
        raise ParseProjectException(
            "%s: Expected %s, got %s: %s" %
            (origin_file, str(expected_type), type(actual), str(actual)))


def _load_spec_file(path):
    result = yaml.safe_load(file(path, "r").read()) or {}
    _expect(dict, result, path)
    return result


##############################################
# Service
##############################################

class Service:

    @classmethod
    def load(cls, svc_yml, name):
        spec = _load_spec_file(svc_yml)
        return Service(name,
                       os.path.dirname(svc_yml),
                       spec.get("description"),
                       spec.get("start"),
                       spec.get("stop"),
                       spec.get("status"),
                       spec.get("logs"))

    @classmethod
    def as_printable_dict(cls, services):
        return dict([
            (s.name, s.get_description_or_default()) for s in services])

    def __init__(self, name, home, description=None,
                 start=None, stop=None, status=None, logs=None):

        self.name = name
        self.home = home
        self.description = description

        self.start = start
        self.stop = stop
        self.status = status

        self.logs = logs or []

    def resolve_logs_relative_to_cwd(self):
        result = []
        for logfile in self.logs:
            abs_log_glob = os.path.join(self.home, logfile)
            result = result + [
                os.path.relpath(
                    abs_log_file,
                    os.path.abspath(os.curdir))
                for abs_log_file
                in glob.iglob(abs_log_glob)]
        return result

    def get_description_or_default(self):
        return self.description or "(No description)"


##############################################
# ServiceSet
##############################################

class BadSelectorException(Exception):
    def __init__(self, msg):
        super(BadSelectorException, self).__init__(msg)


def _resolve(selector, project, service_sets_by_name, selector_stack):
    assert selector

    if selector in selector_stack:<commit_msg>Improve msg when no logs found<commit_after>#!/usr/bin/python
#
# Main entry point for ads. See README for overall description; run with -h for
# usage.
#
# @author adamcath


import sys


def debug(msg):
    print(msg)
    sys.stdout.flush()


def info(msg):
    print("--- " + msg)
    sys.stdout.flush()


def error(msg):
    sys.stderr.write("!!! " + msg + "\n")
    sys.stderr.flush()


try:
    import yaml
except ImportError:
    error(
        "ads requires the python package 'pyyaml'.\n"
        "Please install it with 'pip install pyyaml' or 'easy_install pyyaml'\n"
        "(disregard the message about 'forcing --no-libyaml')")
    sys.exit(1)

import os
import tempfile
import subprocess
import argparse
import glob
from collections import OrderedDict


##############################################
# Treelisting
##############################################

class Treelisting:

    def __init__(self, sections=None):
        self.sections = sections or []

    def with_section(self, heading, listing_dict, empty_msg=None):
        self.sections.append((heading, listing_dict, empty_msg))
        return self

    def pretty_print(self):
        all_keys = [
            k
            for (heading, listing_dict, _) in self.sections
            for k in listing_dict.keys()]
        if len(all_keys) == 0:
            return
        column_width = max(map(len, all_keys)) + 1
        for (heading, listing_dict, empty_msg) in self.sections:
            empty = len(listing_dict) == 0
            print("\n" + heading + (empty and "\n " + empty_msg or ""))
            if not empty:
                for (k, v) in listing_dict.items():
                    print(("%" + str(column_width) + "s: %s") % (k, v))


##############################################
# subprocess stuff
##############################################

def _shell_get_output(cmd_str, working_dir):
    process = subprocess.Popen(
        cmd_str,
        shell=True,
        stdout=subprocess.PIPE,
        stderr=file("/dev/null", "w"),
        close_fds=True,
        cwd=working_dir)

    return process.communicate()[0]


def _shell(cmd_str, working_dir, quiet=False):
    if not quiet:
        debug("cd " + working_dir)
    # TODO Can we avoid this nonsense using shell=true?
    cmd_file = tempfile.NamedTemporaryFile()
    cmd_file.write(cmd_str)
    cmd_file.flush()
    if not quiet:
        debug(cmd_str)
    try:
        status = subprocess.Popen(
            ["/bin/bash", cmd_file.name],
            close_fds=True,
            cwd=working_dir,
            stdout=quiet and file("/dev/null", "w") or None,
            stderr=quiet and file("/dev/null", "w") or None).wait()
    except KeyboardInterrupt:
        # Suppress python from printing a stack trace
        status = 47
        pass
    cmd_file.close()
    return status == 0


def _shell_quiet(cmd_str, working_dir):
    return _shell(cmd_str, working_dir, True)


##############################################
# YML stuff
##############################################

class ParseProjectException(Exception):

    def __init__(self, msg):
        super(ParseProjectException, self).__init__(msg)


def _expect(expected_type, actual, origin_file):
    if not isinstance(actual, expected_type):
        raise ParseProjectException(
            "%s: Expected %s, got %s: %s" %
            (origin_file, str(expected_type), type(actual), str(actual)))


def _load_spec_file(path):
    result = yaml.safe_load(file(path, "r").read()) or {}
    _expect(dict, result, path)
    return result


##############################################
# Service
##############################################

class Service:

    @classmethod
    def load(cls, svc_yml, name):
        spec = _load_spec_file(svc_yml)
        return Service(name,
                       os.path.dirname(svc_yml),
                       spec.get("description"),
                       spec.get("start"),
                       spec.get("stop"),
                       spec.get("status"),
                       spec.get("logs"))

    @classmethod
    def as_printable_dict(cls, services):
        return dict([
            (s.name, s.get_description_or_default()) for s in services])

    def __init__(self, name, home, description=None,
                 start=None, stop=None, status=None, logs=None):

        self.name = name
        self.home = home
        self.description = description

        self.start = start
        self.stop = stop
        self.status = status

        self.logs = logs or []

    def resolve_logs_relative_to_cwd(self):
        result = []
        for logfile in self.logs:
            abs_log_glob = os.path.join(self.home, logfile)
            result = result + [
                os.path.relpath(
                    abs_log_file,
                    os.path.abspath(os.curdir))
                for abs_log_file
                in glob.iglob(abs_log_glob)]
        return result

    def get_description_or_default(self):
        return self.description or "(No description)"

    def __repr__(self):
        return self.name


##############################################
# ServiceSet
##############################################

class BadSelectorException(Exception):
    def __init__(self, msg):
        super(BadSelectorException, self).__init__(msg)


def _resolve(selector, project, service_sets_by_name, selector_stack):
    assert selector

    if selector in selector_stack:
{'fraction_non_alphanumeric': 0.1848879047882646, 'fraction_numerical': 0.0014761509364332502, 'mean_word_length': 2.5171966255678133, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '775058', 'n_tokens_mistral': 3192, 'n_tokens_neox': 3017, 'n_words': 752}