<commit_before>from typing import Tuple, List, Set, Optional

from pathlib import Path

from factory import *


def dir_files(path: Path, suffix=None) -> Set[Path]:
    def suffix_ok(s):
        if suffix is None:
            return True
        elif isinstance(suffix, str):
            return s == suffix
        else:
            return s in suffix

    return {p for p in path.iterdir() if p.is_file() and suffix_ok(p.suffix)}


def subdirs(path: Path) -> Set[Path]:
    return {p for p in path.iterdir() if p.is_dir()}


def cmd_cp(src: Path, dst: Path) -> Cmd:
    return Cmd(cmd=["cp", src, dst], inputs=[src], output=dst)


def cmd_dd_create(output: Path, size_sectors: int) -> Cmd:
    assert "/dev" not in str(output), "Safety check"
    SECTOR_SIZE_BYTES = 512

    size_bytes = size_sectors * SECTOR_SIZE_BYTES

    # Choose better block size if possible
    if size_bytes % 0x10000 == 0:
        block_size = 0x10000
        count = size_bytes // 0x10000
    else:
        block_size = SECTOR_SIZE_BYTES
        count = size_sectors

    return Cmd(
        cmd=[
            "dd",
            "if=/dev/zero",
            "of=" + str(output),
            "bs=" + str(block_size),
            "conv=notrunc",
            "count=" + str(count),
        ]
    )


def cmd_dd_insert(
    output: Path, input: Path, offset: int, size_sectors: Optional[int] = None
) -> Cmd:
    assert "/dev" not in str(output), "Safety check"
    return Cmd(
        cmd=[
            "dd",
            "if=" + str(input),
            "of=" + str(output),
            "bs=512",
            "conv=notrunc",
            "seek=" + str(offset),
            None if size_sectors is None else "count=" + str(size_sectors),
        ]
    )


def cmd_nasm(input: Path, output: Path, format: str) -> Cmd:
    return Cmd(
        inputs=[input], output=output, cmd=["nasm", input, "-f", format, "-o", output]
    )


def cmd_cargo_build(pdir: Path, target=None) -> Cmd:
    out_path = pdir / "target" / "release"
    if target:
        out_path /= target
    return Cmd(
        inputs={pdir / "src/", pdir / "Cargo.toml"},
        output=out_path,
        cmd=["cargo", "build", "--release", "--color=always"],
        cwd=pdir,
    )


def cmd_cargo_xbuild(pdir: Path, target_json: Path) -> Cmd:
    out_path = pdir / "target" / target_json.stem / "release"
    import toml

    with open(pdir / "Cargo.toml") as f:
        t = toml.load(f)
        out_path /= "lib" + t["package"]["name"] + ".a"

    return Cmd(
        inputs={target_json, pdir / "src/", pdir / "Cargo.toml"},
        output=out_path,
        cmd=["cargo", "xbuild", "--target", target_json, "--release", "--color=always"],
        cwd=pdir,
    )


def cmd_ld(linker_script: Path, output: Path, inputs: List[Path]) -> Cmd:
    return Cmd(
        inputs={linker_script}.union(inputs),
        output=output,
        cmd=[
            "ld",
            "-z",
            "max-page-size=0x1000",
            "--gc-sections",
            "-T",<commit_msg>Recompile when depended local crates are modified<commit_after>from typing import Tuple, List, Set, Optional

from dataclasses import dataclass
from pathlib import Path
from subprocess import run
import re
import toml
import json

from factory import *


def dir_files(path: Path, suffix=None) -> Set[Path]:
    def suffix_ok(s):
        if suffix is None:
            return True
        elif isinstance(suffix, str):
            return s == suffix
        else:
            return s in suffix

    return {p for p in path.iterdir() if p.is_file() and suffix_ok(p.suffix)}


def subdirs(path: Path) -> Set[Path]:
    return {p for p in path.iterdir() if p.is_dir()}


def cmd_cp(src: Path, dst: Path) -> Cmd:
    return Cmd(cmd=["cp", src, dst], inputs=[src], output=dst)


def cmd_dd_create(output: Path, size_sectors: int) -> Cmd:
    assert "/dev" not in str(output), "Safety check"
    SECTOR_SIZE_BYTES = 512

    size_bytes = size_sectors * SECTOR_SIZE_BYTES

    # Choose better block size if possible
    if size_bytes % 0x10000 == 0:
        block_size = 0x10000
        count = size_bytes // 0x10000
    else:
        block_size = SECTOR_SIZE_BYTES
        count = size_sectors

    return Cmd(
        cmd=[
            "dd",
            "if=/dev/zero",
            "of=" + str(output),
            "bs=" + str(block_size),
            "conv=notrunc",
            "count=" + str(count),
        ]
    )


def cmd_dd_insert(
    output: Path, input: Path, offset: int, size_sectors: Optional[int] = None
) -> Cmd:
    assert "/dev" not in str(output), "Safety check"
    return Cmd(
        cmd=[
            "dd",
            "if=" + str(input),
            "of=" + str(output),
            "bs=512",
            "conv=notrunc",
            "seek=" + str(offset),
            None if size_sectors is None else "count=" + str(size_sectors),
        ]
    )


def cmd_nasm(input: Path, output: Path, format: str) -> Cmd:
    return Cmd(
        inputs=[input], output=output, cmd=["nasm", input, "-f", format, "-o", output]
    )


@dataclass
class CargoPackage:
    name: str
    sources: Set[Path]

    def load(pdir: Path) -> "CargoPackage":
        """If cargo build depends on local packages, return their inputs as well."""
        p = run(
            ["cargo", "metadata", "--format-version", "1"],
            capture_output=True,
            cwd=pdir,
            check=True,
        )

        data = json.loads(p.stdout.decode())
        sources = {pdir / "src/", pdir / "Cargo.toml"}
        for p in data["packages"]:
            m = re.match(r"(.+?)\s(.+?)\s\((.+?)\+(.+)\)", p["id"])
            assert m
            name, version, source_type, location = m.groups()
            if source_type == "path":
                schema, path = location.split("://")
                assert schema == "file"
                p = Path(path)
                sources.update({p / "src/", p / "Cargo.toml"})

        return CargoPackage(
            name=data["resolve"]["root"].split(" ", 1)[0], sources=sources
        )


def cmd_cargo_build(pdir: Path, target=None) -> Cmd:
    package = CargoPackage.load(pdir)
    out_path = pdir / "target" / "release"
    if target:
        out_path /= target
    return Cmd(
        inputs=package.sources,
        output=out_path,
        cmd=["cargo", "build", "--release", "--color=always"],
        cwd=pdir,
    )


def cmd_cargo_xbuild(pdir: Path, target_json: Path) -> Cmd:
    package = CargoPackage.load(pdir)
    out_path = (
        pdir / "target" / target_json.stem / "release" / ("lib" + package.name + ".a")
    )
    return Cmd(
        inputs={target_json}.union(package.sources),
        output=out_path,
        cmd=["cargo", "xbuild", "--target", target_json, "--release", "--color=always"],
        cwd=pdir,
    )


def cmd_ld(linker_script: Path, output: Path, inputs: List[Path]) -> Cmd:
    return Cmd(
        inputs={linker_script}.union(inputs),
        output=output,
        cmd=[
            "ld",
            "-z",
            "max-page-size=0x1000",
            "--gc-sections",
            "-T",
{'fraction_non_alphanumeric': 0.14672686230248308, 'fraction_numerical': 0.008888261851015801, 'mean_word_length': 2.0608808290155443, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4785722', 'n_tokens_mistral': 2413, 'n_tokens_neox': 2315, 'n_words': 601}