<commit_before>#!/usr/bin/env python3

# This is a python script that emulates a terminal session and runs
# commands from a supplied markdown file..

import difflib
import optparse
import os
import pexpect
import random
import time
import shlex
import sys
import json
import colorama
colorama.init(strip=None)

class Environment(object):
    def __init__(self, directory, copy_env=True):
        """Initialize the environment"""
        if copy_env:
            self.env = os.environ.copy()
        else:
            self.env = {}
        self.read_simdem_environment(directory)

    def read_simdem_environment(self, directory):
        """
        Populates each shell environment with a set of environment vars
        loaded via env.json and/or env.local.json files. Variables are loaded
        from the project root down, with the most local files taking
        precednece, and env.local.json > env.json.
        """
        env = {}

        if not directory.endswith('/'):
            directory = directory + "/"

        filename = directory + "../env.json"
        if os.path.isfile(directory + "../env.json"):
            with open(filename) as env_file:
                app_env = json.load(env_file)
                env.update(app_env)

        filename = directory + "env.json"
        if os.path.isfile(filename):
            with open(filename) as env_file:
                script_env = json.load(env_file)
                env.update(script_env)

        filename = directory + "../env.local.json"
        if os.path.isfile(filename):
            with open(filename) as env_file:
                local_env = json.load(env_file)
                env.update(local_env)

        filename = directory + "env.local.json"
        if os.path.isfile(filename):
            with open(filename) as env_file:
                local_env = json.load(env_file)
                env.update(local_env)

        self.env.update(env)

    def set(self, var, value):
        """Adds a new variable to the environment"""
        self.env[var] = value

    def get(self):
        """Returns a dictionary of the current environment"""
        return self.env

class Demo(object):
    def __init__(self, script_dir = "demo_scripts", filename = "script.md", is_simulation = True, is_automated = False, is_testing = False):
        """Initialize variables"""
        self.filename = filename
        self.script_dir = script_dir
        self.env = Environment(script_dir)
        self.is_simulation = is_simulation
        self.is_automated = is_automated
        self.is_testing = is_testing
        self.current_command = ""

    def run(self):
        """
        Reads a script.md file in the indicated directoy and runs the
        commands contained within. If simulation == True then human
        entry will be simulated (looks like typing and waits for
        keyboard input before proceeding to the next command). This is
        useful if you want to run a fully automated demo.

        The script.md file will be parsed as follows:

        ``` marks the start or end of a code block

        Each line in a code block will be treated as a separate command.
        All other lines will be ignored
        """
        if not self.script_dir.endswith('/'):
            self.script_dir = self.script_dir + "/"

        file = self.script_dir + self.filename

        lines = list(open(file))
        in_code_block = False
        in_results_section = False
        expected_results = ""
        actual_results = ""
        passed_tests = 0
        failed_tests = 0
        is_first_line = True
        executed_code_in_this_section = False

        for line in lines:
            if line.startswith("Results:"):
                # Entering results section
                in_results_section = True
                pos = line.lower().find("expected similarity: ")
                if pos >= 0:
                    pos = pos + len("expected similarity: ")
                    similarity = line[pos:]
                    expected_similarity = float(similarity)
                else:
                    expected_similarity = 0.66
            elif line.startswith("```") and not in_code_block:
                # Entering a code block, if in_results_section = True then it's a results block
                in_code_block = True
            elif line.startswith("```") and in_code_block and in_results_section:
                # Finishing results section
                if in_results_section and self.is_testing:
                    if test_results(expected_results, actual_results, expected_similarity):
                        passed_tests += 1
                    else:
                        failed_tests += 1
                expected_results = ""
                actual_results = ""
                in_results_section = False
                in_code_block = False
            elif line.startswith("```") and in_code_block:
                # Finishing code block
                in_code_block = False
                in_results_section = False
            elif in_results_section and in_code_block:
                expected_results += line
            elif in_code_block and not in_results_section:
                # Executable line
                print("$ ", end="", flush=True)
                self.current_command = line
                check_for_interactive_command(self)
                actual_results = simulate_command(self)
                executed_code_in_this_section = True
            elif line.startswith("#") and not in_code_block and not in_results_section and not self.is_automated:
                # Heading in descriptive text, indicating a new section
                if is_first_line:
                    run_command(self, "clear")
                elif executed_code_in_this_section:
                    executed_code_in_this_section = False
                    print("$ ", end="", flush=True)
                    check_for_interactive_command(self)
                    self.current_command = "clear"
                    simulate_command(self)
                    if not self.is_simulation:
                        print("$ ", end="", flush=True)
                        # Since this is a heading we are not really simulating a command, it appears as a comment
                        simulate_command(self)
            elif not self.is_simulation and not in_results_section:
                # Descriptive text
                print(colorama.Fore.CYAN, end="")
                print(line, end="", flush=True)
                print(colorama.Style.RESET_ALL, end="")

            is_first_line = False

        if self.is_testing:
            print("\n\n=============================\n\n")
            print("Test Run Complete.")
            if failed_tests > 0:
                print(colorama.Fore.RED + colorama.Style.BRIGHT)
                print("Failed Tests: " + str(failed_tests))
                print(colorama.Style.RESET_ALL)
            else:
                print(colorama.Fore.GREEN + colorama.Style.BRIGHT)
                print("No failed tests")
                print(colorama.Style.RESET_ALL)
                print("Passed Tests: " + str(passed_tests))
            if failed_tests > 0:
                print("\n\n")
                print("View failure reports in context in the above output.")
                print("\n\n=============================\n\n")

def input_interactive_variable(name):
    """
    Set a new environment with name
    from user input
    """
    print(colorama.Fore.MAGENTA + colorama.Style.BRIGHT, end="")
    print("\n\nEnter a value for ", end="")
    print(colorama.Fore.YELLOW + colorama.Style.BRIGHT, end="")
    print("$" + name, end="")
    print(colorama.Fore.MAGENTA + colorama.Style.BRIGHT, end="")
    print(": ", end="")
    print(colorama.Fore.WHITE + colorama.Style.BRIGHT, end="")
    value = input()
    return value

def type_command(command, script_dir, simulation, var_set=[]):
    """
    Displays the command on the screen
    If simulation == True then it will look like someone is typing the command
    """
    print(colorama.Fore.WHITE + colorama.Style.BRIGHT, end="")
    interactive_var = False
    for idx, char in enumerate(command):
        if char == "$" and var_set and idx in [command.find(i)-1 for i in var_set if command.find(i) > 0]:
            interactive_var = True
            print(colorama.Fore.YELLOW + colorama.Style.BRIGHT, end="")
        if char == " " and interactive_var:
            interactive_var = False
            print(colorama.Fore.WHITE + colorama.Style.BRIGHT, end="")
        if char != "\n":
            print(char, end="", flush=True)
        if simulation:
            delay = random.uniform(0.01, 0.04)
            time.sleep(delay)
    print(colorama.Style.RESET_ALL, end="")

def simulate_command(demo):
    """
    Types the command on the screen, executes it and outputs the
    results if simulation == True then system will make the "typing"
    look real and will wait for keyboard entry before proceeding to
    the next command
    """
    var_set = [v.split("$")[1] for v in demo.current_command.rstrip().split(" ") if v.startswith("$") and v.split("$")[1] not in demo.env.get()]
    type_command(demo.current_command, demo.script_dir, demo.is_simulation, var_set)

    for var_name in var_set:
        var_value = input_interactive_variable(var_name)
        demo.env.set(var_name, var_value)
<commit_msg>Add var_set to Demo class, add var_set init, remove var_set param from type_command<commit_after>#!/usr/bin/env python3

# This is a python script that emulates a terminal session and runs
# commands from a supplied markdown file..

import difflib
import optparse
import os
import pexpect
import random
import time
import shlex
import sys
import json
import colorama
colorama.init(strip=None)

class Environment(object):
    def __init__(self, directory, copy_env=True):
        """Initialize the environment"""
        if copy_env:
            self.env = os.environ.copy()
        else:
            self.env = {}
        self.read_simdem_environment(directory)

    def read_simdem_environment(self, directory):
        """
        Populates each shell environment with a set of environment vars
        loaded via env.json and/or env.local.json files. Variables are loaded
        from the project root down, with the most local files taking
        precednece, and env.local.json > env.json.
        """
        env = {}

        if not directory.endswith('/'):
            directory = directory + "/"

        filename = directory + "../env.json"
        if os.path.isfile(directory + "../env.json"):
            with open(filename) as env_file:
                app_env = json.load(env_file)
                env.update(app_env)

        filename = directory + "env.json"
        if os.path.isfile(filename):
            with open(filename) as env_file:
                script_env = json.load(env_file)
                env.update(script_env)

        filename = directory + "../env.local.json"
        if os.path.isfile(filename):
            with open(filename) as env_file:
                local_env = json.load(env_file)
                env.update(local_env)

        filename = directory + "env.local.json"
        if os.path.isfile(filename):
            with open(filename) as env_file:
                local_env = json.load(env_file)
                env.update(local_env)

        self.env.update(env)

    def set(self, var, value):
        """Adds a new variable to the environment"""
        self.env[var] = value

    def get(self):
        """Returns a dictionary of the current environment"""
        return self.env

class Demo(object):
    def __init__(self, script_dir = "demo_scripts", filename = "script.md", is_simulation = True, is_automated = False, is_testing = False):
        """Initialize variables"""
        self.filename = filename
        self.script_dir = script_dir
        self.env = Environment(script_dir)
        self.is_simulation = is_simulation
        self.is_automated = is_automated
        self.is_testing = is_testing
        self.current_command = ""
        self.var_set = []

    def run(self):
        """
        Reads a script.md file in the indicated directoy and runs the
        commands contained within. If simulation == True then human
        entry will be simulated (looks like typing and waits for
        keyboard input before proceeding to the next command). This is
        useful if you want to run a fully automated demo.

        The script.md file will be parsed as follows:

        ``` marks the start or end of a code block

        Each line in a code block will be treated as a separate command.
        All other lines will be ignored
        """
        if not self.script_dir.endswith('/'):
            self.script_dir = self.script_dir + "/"

        file = self.script_dir + self.filename

        lines = list(open(file))
        in_code_block = False
        in_results_section = False
        expected_results = ""
        actual_results = ""
        passed_tests = 0
        failed_tests = 0
        is_first_line = True
        executed_code_in_this_section = False

        for line in lines:
            if line.startswith("Results:"):
                # Entering results section
                in_results_section = True
                pos = line.lower().find("expected similarity: ")
                if pos >= 0:
                    pos = pos + len("expected similarity: ")
                    similarity = line[pos:]
                    expected_similarity = float(similarity)
                else:
                    expected_similarity = 0.66
            elif line.startswith("```") and not in_code_block:
                # Entering a code block, if in_results_section = True then it's a results block
                in_code_block = True
            elif line.startswith("```") and in_code_block and in_results_section:
                # Finishing results section
                if in_results_section and self.is_testing:
                    if test_results(expected_results, actual_results, expected_similarity):
                        passed_tests += 1
                    else:
                        failed_tests += 1
                expected_results = ""
                actual_results = ""
                in_results_section = False
                in_code_block = False
            elif line.startswith("```") and in_code_block:
                # Finishing code block
                in_code_block = False
                in_results_section = False
            elif in_results_section and in_code_block:
                expected_results += line
            elif in_code_block and not in_results_section:
                # Executable line
                print("$ ", end="", flush=True)
                self.current_command = line
                check_for_interactive_command(self)
                actual_results = simulate_command(self)
                executed_code_in_this_section = True
            elif line.startswith("#") and not in_code_block and not in_results_section and not self.is_automated:
                # Heading in descriptive text, indicating a new section
                if is_first_line:
                    run_command(self, "clear")
                elif executed_code_in_this_section:
                    executed_code_in_this_section = False
                    print("$ ", end="", flush=True)
                    check_for_interactive_command(self)
                    self.current_command = "clear"
                    simulate_command(self)
                    if not self.is_simulation:
                        print("$ ", end="", flush=True)
                        # Since this is a heading we are not really simulating a command, it appears as a comment
                        simulate_command(self)
            elif not self.is_simulation and not in_results_section:
                # Descriptive text
                print(colorama.Fore.CYAN, end="")
                print(line, end="", flush=True)
                print(colorama.Style.RESET_ALL, end="")

            is_first_line = False

        if self.is_testing:
            print("\n\n=============================\n\n")
            print("Test Run Complete.")
            if failed_tests > 0:
                print(colorama.Fore.RED + colorama.Style.BRIGHT)
                print("Failed Tests: " + str(failed_tests))
                print(colorama.Style.RESET_ALL)
            else:
                print(colorama.Fore.GREEN + colorama.Style.BRIGHT)
                print("No failed tests")
                print(colorama.Style.RESET_ALL)
                print("Passed Tests: " + str(passed_tests))
            if failed_tests > 0:
                print("\n\n")
                print("View failure reports in context in the above output.")
                print("\n\n=============================\n\n")

def input_interactive_variable(name):
    """
    Set a new environment with name
    from user input
    """
    print(colorama.Fore.MAGENTA + colorama.Style.BRIGHT, end="")
    print("\n\nEnter a value for ", end="")
    print(colorama.Fore.YELLOW + colorama.Style.BRIGHT, end="")
    print("$" + name, end="")
    print(colorama.Fore.MAGENTA + colorama.Style.BRIGHT, end="")
    print(": ", end="")
    print(colorama.Fore.WHITE + colorama.Style.BRIGHT, end="")
    value = input()
    return value

def type_command(command, script_dir, simulation):
    """
    Displays the command on the screen
    If simulation == True then it will look like someone is typing the command
    """
    print(colorama.Fore.WHITE + colorama.Style.BRIGHT, end="")
    interactive_var = False
    for idx, char in enumerate(command):
        if char == "$" and demo.var_set and idx in [command.find(i)-1 for i in demo.var_set if command.find(i) > 0]:
            interactive_var = True
            print(colorama.Fore.YELLOW + colorama.Style.BRIGHT, end="")
        if char == " " and interactive_var:
            interactive_var = False
            print(colorama.Fore.WHITE + colorama.Style.BRIGHT, end="")
        if char != "\n":
            print(char, end="", flush=True)
        if simulation:
            delay = random.uniform(0.01, 0.04)
            time.sleep(delay)
    print(colorama.Style.RESET_ALL, end="")

def simulate_command(demo):
    """
    Types the command on the screen, executes it and outputs the
    results if simulation == True then system will make the "typing"
    look real and will wait for keyboard entry before proceeding to
    the next command
    """
    demo.var_set = [v.split("$")[1] for v in demo.current_command.rstrip().split(" ") if v.startswith("$") and v.split("$")[1] not in demo.env.get()]
    type_command(demo.current_command, demo.script_dir, demo.is_simulation)

    for var_name in demo.var_set:
        var_value = input_interactive_variable(var_name)
        demo.env.set(var_name, var_value)

{'fraction_non_alphanumeric': 0.09689656989303967, 'fraction_numerical': 0.0022129722324674642, 'mean_word_length': 1.9522476279359153, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '698338', 'n_tokens_mistral': 5253, 'n_tokens_neox': 5024, 'n_words': 1437}