<commit_before>from argparse import ArgumentParser
from html import escape
from operator import attrgetter
from os import path
from siebenapp.cli import IO, ConsoleIO
from siebenapp.domain import EdgeType, Graph, RenderRow
from siebenapp.goaltree import Goals, GoalsData, EdgesData, OptionsData
from siebenapp.layers import get_root, persistent_layers
from siebenapp.open_view import ToggleOpenView
from siebenapp.progress_view import ToggleProgress
from siebenapp.switchable_view import ToggleSwitchableView
from siebenapp.system import load, save, split_long
from typing import List, Optional, Dict, Set, Any


def print_dot(args, io: IO):
    tree = load(args.db)
    if args.n:
        tree.accept(ToggleOpenView())
    if args.p:
        tree.accept(ToggleProgress())
    if args.t:
        tree.accept(ToggleSwitchableView())
    io.write(dot_export(tree))


def print_md(args, io: IO):
    tree = load(args.db)
    if args.n:
        tree.accept(ToggleOpenView())
    if args.p:
        tree.accept(ToggleProgress())
    if args.t:
        tree.accept(ToggleSwitchableView())
    io.write(markdown_export(tree))


def migrate(args, io: IO):
    goals = load(args.db)
    save(goals, args.db)


def extract(args, io: IO):
    tree = load(args.source_db).goaltree.goaltree
    assert not path.exists(args.target_db), f"File {args.target_db} already exists!"
    result = extract_subtree(tree, args.goal_id)
    save(result, args.target_db)


def _flag(parser, key, description) -> None:
    parser.add_argument(key, required=False, action="store_true", help=description)


def main(argv: Optional[List[str]] = None, io: Optional[IO] = None):
    parser = ArgumentParser()
    subparsers = parser.add_subparsers(title="commands")

    parser_dot = subparsers.add_parser("dot")
    _flag(parser_dot, "-n", "Show closed goals (same as n key in the app)")
    _flag(parser_dot, "-p", "Show goal progress (same as p key in the app)")
    _flag(parser_dot, "-t", "Show only switchable goals (same as t key in app)")
    parser_dot.add_argument("db", help="An existing file with goaltree.")
    parser_dot.set_defaults(func=print_dot)

    parser_dot = subparsers.add_parser("md")
    _flag(parser_dot, "-n", "Show closed goals (same as n key in the app)")
    _flag(parser_dot, "-p", "Show goal progress (same as p key in the app)")
    _flag(parser_dot, "-t", "Show only switchable goals (same as t key in app)")
    parser_dot.add_argument("db", help="An existing file with goaltree.")
    parser_dot.set_defaults(func=print_md)

    parser_migrate = subparsers.add_parser("migrate")
    parser_migrate.add_argument("db", help="An existing file with goaltree.")
    parser_migrate.set_defaults(func=migrate)

    parser_extract = subparsers.add_parser("extract")
    parser_extract.add_argument(
        "source_db",
        help="An existing file with goaltree to extract. "
        "This file will not be modified during the operation. You may/should do it manually.",
    )
    parser_extract.add_argument(
        "target_db",
        help="A new file to be created. It will contain a result of extraction. "
        "Only subgoals and edges are extracted, no zoom and/or autolink data is inherited.",
    )
    parser_extract.add_argument(
        "goal_id",
        type=int,
        help="An id of the goal that should be set as a root goal of a new goaltree. "
        "All of its subgoals will be copied too (but not blockers). "
        "The simplest way to find a real id of a goal is to run `sieben-manage dot` on the source file.",
    )
    parser_extract.set_defaults(func=extract)

    args = parser.parse_args(argv)
    io = io or ConsoleIO("> ")
    if "func" in dir(args):
        args.func(args, io)
    else:
        parser.print_help()


def _format_name(row: RenderRow) -> str:
    goal_name = escape(row.name)
    label = (
        f'"{row.goal_id}: {goal_name}"'
        if isinstance(row.goal_id, int) and row.goal_id >= 0
        else f'"{goal_name}"'
    )
    return split_long(label)


def dot_export(goals):
    render_result = goals.q()
    node_lines = []
    edge_lines = []
    for row in sorted(render_result.rows, key=attrgetter("goal_id")):
        attributes = {
            "label": _format_name(row),
            "color": "red" if row.is_open else "green",
        }
        if row.is_switchable and row.is_open:
            attributes["style"] = "bold"
        attributes_str = ", ".join(
            f"{k}={attributes[k]}"
            for k in ["label", "color", "style", "fillcolor"]
            if k in attributes and attributes[k]
        )
        node_lines.append(f"{row.goal_id} [{attributes_str}];")
        for edge in row.edges:
            color = "black" if render_result.by_id(edge[0]).is_open else "gray"
            line_attrs = f"color={color}"
            if edge[1] == EdgeType.BLOCKER:
                line_attrs += ", style=dashed"
            edge_lines.append(f"{edge[0]} -> {row.goal_id} [{line_attrs}];")
    body = "\n".join(node_lines + edge_lines)
    return f"digraph g {{\nnode [shape=box];\n{body}\n}}"


def markdown_export(goals):
    render_result = goals.q()
    output: List[str] = []
    for row in render_result.rows:
        output.append(_format_md_row(row))
    return "\n".join(output)


def _format_md_row(row: RenderRow) -> str:
    open_status = " " if row.is_open else "x"
    blockers: List[str] = [f"**{e[0]}**" for e in row.edges if e[1] == EdgeType.BLOCKER]
    blocked_status = "" if not blockers else f" (blocked by {', '.join(blockers)})"
    return f"* [{open_status}] **{row.goal_id}** {row.name}{blocked_status}"


def extract_subtree(source_goals: Graph, goal_id: int) -> Graph:
    root_goaltree: Goals = get_root(source_goals)
    render_result = root_goaltree.q()
    assert (
        render_result.by_id(goal_id) is not None
    ), f"Cannot find goal with id {goal_id}"
    target_goals: Set[int] = set()
    goals_to_add: Set[int] = {goal_id}
    goals_data: GoalsData = []
    edges_data: EdgesData = []
    options_data: OptionsData = []
    while goals_to_add:
        goal = goals_to_add.pop()
        row = render_result.by_id(goal)<commit_msg>Tree walk during md print (no shift yet) #311<commit_after>from argparse import ArgumentParser
from html import escape
from operator import attrgetter
from os import path
from siebenapp.cli import IO, ConsoleIO
from siebenapp.domain import EdgeType, Graph, RenderRow
from siebenapp.goaltree import Goals, GoalsData, EdgesData, OptionsData
from siebenapp.layers import get_root, persistent_layers
from siebenapp.open_view import ToggleOpenView
from siebenapp.progress_view import ToggleProgress
from siebenapp.switchable_view import ToggleSwitchableView
from siebenapp.system import load, save, split_long
from typing import List, Optional, Dict, Set, Any


def print_dot(args, io: IO):
    tree = load(args.db)
    if args.n:
        tree.accept(ToggleOpenView())
    if args.p:
        tree.accept(ToggleProgress())
    if args.t:
        tree.accept(ToggleSwitchableView())
    io.write(dot_export(tree))


def print_md(args, io: IO):
    tree = load(args.db)
    if args.n:
        tree.accept(ToggleOpenView())
    if args.p:
        tree.accept(ToggleProgress())
    if args.t:
        tree.accept(ToggleSwitchableView())
    io.write(markdown_export(tree))


def migrate(args, io: IO):
    goals = load(args.db)
    save(goals, args.db)


def extract(args, io: IO):
    tree = load(args.source_db).goaltree.goaltree
    assert not path.exists(args.target_db), f"File {args.target_db} already exists!"
    result = extract_subtree(tree, args.goal_id)
    save(result, args.target_db)


def _flag(parser, key, description) -> None:
    parser.add_argument(key, required=False, action="store_true", help=description)


def main(argv: Optional[List[str]] = None, io: Optional[IO] = None):
    parser = ArgumentParser()
    subparsers = parser.add_subparsers(title="commands")

    parser_dot = subparsers.add_parser("dot")
    _flag(parser_dot, "-n", "Show closed goals (same as n key in the app)")
    _flag(parser_dot, "-p", "Show goal progress (same as p key in the app)")
    _flag(parser_dot, "-t", "Show only switchable goals (same as t key in app)")
    parser_dot.add_argument("db", help="An existing file with goaltree.")
    parser_dot.set_defaults(func=print_dot)

    parser_dot = subparsers.add_parser("md")
    _flag(parser_dot, "-n", "Show closed goals (same as n key in the app)")
    _flag(parser_dot, "-p", "Show goal progress (same as p key in the app)")
    _flag(parser_dot, "-t", "Show only switchable goals (same as t key in app)")
    parser_dot.add_argument("db", help="An existing file with goaltree.")
    parser_dot.set_defaults(func=print_md)

    parser_migrate = subparsers.add_parser("migrate")
    parser_migrate.add_argument("db", help="An existing file with goaltree.")
    parser_migrate.set_defaults(func=migrate)

    parser_extract = subparsers.add_parser("extract")
    parser_extract.add_argument(
        "source_db",
        help="An existing file with goaltree to extract. "
        "This file will not be modified during the operation. You may/should do it manually.",
    )
    parser_extract.add_argument(
        "target_db",
        help="A new file to be created. It will contain a result of extraction. "
        "Only subgoals and edges are extracted, no zoom and/or autolink data is inherited.",
    )
    parser_extract.add_argument(
        "goal_id",
        type=int,
        help="An id of the goal that should be set as a root goal of a new goaltree. "
        "All of its subgoals will be copied too (but not blockers). "
        "The simplest way to find a real id of a goal is to run `sieben-manage dot` on the source file.",
    )
    parser_extract.set_defaults(func=extract)

    args = parser.parse_args(argv)
    io = io or ConsoleIO("> ")
    if "func" in dir(args):
        args.func(args, io)
    else:
        parser.print_help()


def _format_name(row: RenderRow) -> str:
    goal_name = escape(row.name)
    label = (
        f'"{row.goal_id}: {goal_name}"'
        if isinstance(row.goal_id, int) and row.goal_id >= 0
        else f'"{goal_name}"'
    )
    return split_long(label)


def dot_export(goals):
    render_result = goals.q()
    node_lines = []
    edge_lines = []
    for row in sorted(render_result.rows, key=attrgetter("goal_id")):
        attributes = {
            "label": _format_name(row),
            "color": "red" if row.is_open else "green",
        }
        if row.is_switchable and row.is_open:
            attributes["style"] = "bold"
        attributes_str = ", ".join(
            f"{k}={attributes[k]}"
            for k in ["label", "color", "style", "fillcolor"]
            if k in attributes and attributes[k]
        )
        node_lines.append(f"{row.goal_id} [{attributes_str}];")
        for edge in row.edges:
            color = "black" if render_result.by_id(edge[0]).is_open else "gray"
            line_attrs = f"color={color}"
            if edge[1] == EdgeType.BLOCKER:
                line_attrs += ", style=dashed"
            edge_lines.append(f"{edge[0]} -> {row.goal_id} [{line_attrs}];")
    body = "\n".join(node_lines + edge_lines)
    return f"digraph g {{\nnode [shape=box];\n{body}\n}}"


def markdown_export(goals):
    render_result = goals.q()
    all_ids = {r.goal_id for r in render_result.rows}
    non_roots = {
        e[0] for r in render_result.rows for e in r.edges if e[1] == EdgeType.PARENT
    }
    roots = all_ids.difference(non_roots)
    output: List[str] = []
    for root_id in sorted(list(roots)):
        output.extend(_md_tree(render_result, root_id, 0))
    return "\n".join(output)


def _md_tree(render_result, root_id, shift):
    row = render_result.by_id(root_id)
    result = [_format_md_row(row)]
    for edge in row.edges:
        if edge[1] == EdgeType.PARENT:
            result.extend(_md_tree(render_result, edge[0], shift + 1))
    return result


def _format_md_row(row: RenderRow) -> str:
    open_status = " " if row.is_open else "x"
    blockers: List[str] = [f"**{e[0]}**" for e in row.edges if e[1] == EdgeType.BLOCKER]
    blocked_status = "" if not blockers else f" (blocked by {', '.join(blockers)})"
    return f"* [{open_status}] **{row.goal_id}** {row.name}{blocked_status}"


def extract_subtree(source_goals: Graph, goal_id: int) -> Graph:
    root_goaltree: Goals = get_root(source_goals)
    render_result = root_goaltree.q()
    assert (
        render_result.by_id(goal_id) is not None
    ), f"Cannot find goal with id {goal_id}"
    target_goals: Set[int] = set()
    goals_to_add: Set[int] = {goal_id}
    goals_data: GoalsData = []
    edges_data: EdgesData = []
    options_data: OptionsData = []
    while goals_to_add:
        goal = goals_to_add.pop()
        row = render_result.by_id(goal)
{'fraction_non_alphanumeric': 0.12233296994237658, 'fraction_numerical': 0.0016352593054041427, 'mean_word_length': 3.2218934911242605, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2315147', 'n_tokens_mistral': 4301, 'n_tokens_neox': 4216, 'n_words': 1127}