<commit_before>#!/usr/bin/env python3

# Copyright (c) 2019 Nordic Semiconductor ASA
# Copyright (c) 2019 Linaro Limited
# SPDX-License-Identifier: BSD-3-Clause

# This script uses edtlib to generate a header file and a .conf file (both
# containing the same values) from a devicetree (.dts) file. Information from
# binding files in YAML format is used as well.
#
# Bindings are files that describe devicetree nodes. Devicetree nodes are
# usually mapped to bindings via their 'compatible = "..."' property.
#
# See the docstring/comments at the top of edtlib.py for more information.
#
# Note: Do not access private (_-prefixed) identifiers from edtlib here (and
# also note that edtlib is not meant to expose the dtlib API directly).
# Instead, think of what API you need, and add it as a public documented API in
# edtlib. This will keep this script simple.

import argparse
import os
import pathlib
import sys

import edtlib


def main():
    global conf_file
    global header_file

    args = parse_args()

    try:
        edt = edtlib.EDT(args.dts, args.bindings_dirs)
    except edtlib.EDTError as e:
        sys.exit("devicetree error: " + str(e))

    conf_file = open(args.conf_out, "w", encoding="utf-8")
    header_file = open(args.header_out, "w", encoding="utf-8")

    out_comment("Generated by gen_defines.py", blank_before=False)
    out_comment("DTS input file: " + args.dts, blank_before=False)
    out_comment("Directories with bindings: " +
                ", ".join(map(relativize, args.bindings_dirs)),
                blank_before=False)

    active_compats = set()

    for node in edt.nodes:
        if node.enabled and node.matching_compat:
            # Skip 'fixed-partitions' devices since they are handled by
            # write_flash() and would generate extra spurious #defines
            if node.matching_compat == "fixed-partitions":
                continue

            out_comment("Devicetree node: " + node.path)
            out_comment("Binding (compatible = {}): {}".format(
                            node.matching_compat,
                            relativize(node.binding_path)),
                        blank_before=False)
            out_comment("Binding description: " + node.description,
                        blank_before=False)

            write_regs(node)
            write_irqs(node)
            write_props(node)
            write_clocks(node)
            write_spi_dev(node)
            write_bus(node)
            write_existence_flags(node)

            active_compats.update(node.compats)

    out_comment("Active compatibles (mentioned in DTS + binding found)")
    for compat in sorted(active_compats):
        #define DT_COMPAT_<COMPAT> 1
        out("COMPAT_{}".format(str2ident(compat)), 1)

    # These are derived from /chosen
    write_addr_size(edt, "zephyr,sram", "SRAM")
    write_addr_size(edt, "zephyr,ccm", "CCM")
    write_addr_size(edt, "zephyr,dtcm", "DTCM")

    write_flash(edt.chosen_node("zephyr,flash"))
    write_code_partition(edt.chosen_node("zephyr,code-partition"))

    flash_index = 0
    for node in edt.nodes:
        if node.name.startswith("partition@"):
            write_flash_partition(node, flash_index)
            flash_index += 1

    out_comment("Number of flash partitions")
    if flash_index != 0:
        out("FLASH_AREA_NUM", flash_index)

    print("Devicetree configuration written to " + args.conf_out)


def parse_args():
    # Returns parsed command-line arguments

    parser = argparse.ArgumentParser()
    parser.add_argument("--dts", required=True, help="DTS file")
    parser.add_argument("--bindings-dirs", nargs='+', required=True,
                        help="directory with bindings in YAML format, "
                        "we allow multiple")
    parser.add_argument("--header-out", required=True,
                        help="path to write header to")
    parser.add_argument("--conf-out", required=True,
                        help="path to write configuration file to")

    return parser.parse_args()


def relativize(path):
    # If 'path' is within $ZEPHYR_BASE, returns it relative to $ZEPHYR_BASE,
    # with a "$ZEPHYR_BASE/..." hint at the start of the string. Otherwise,
    # returns 'path' unchanged.

    zbase = os.getenv("ZEPHYR_BASE")
    if zbase is None:
        return path

    try:
        return str("$ZEPHYR_BASE" / pathlib.Path(path).relative_to(zbase))
    except ValueError:
        # Not within ZEPHYR_BASE
        return path


def write_regs(node):
    # Writes address/size output for the registers in the node's 'reg' property

    def reg_addr_name_alias(reg):
        return str2ident(reg.name) + "_BASE_ADDRESS" if reg.name else None

    def reg_size_name_alias(reg):
        return str2ident(reg.name) + "_SIZE" if reg.name else None

    for reg in node.regs:
        out_dev(node, reg_addr_ident(reg), hex(reg.addr),
                name_alias=reg_addr_name_alias(reg))

        if reg.size:
            out_dev(node, reg_size_ident(reg), reg.size,
                    name_alias=reg_size_name_alias(reg))


def write_props(node):
    # Writes any properties defined in the "properties" section of the binding
    # for the node

    for prop in node.props.values():
        # Skip #size-cell and other property starting with #. Also skip mapping
        # properties like 'gpio-map'.
        if prop.name[0] == "#" or prop.name.endswith("-map"):
            continue

        # See write_clocks()
        if prop.name == "clocks":
            continue

        # edtlib provides these as well (Property.val becomes an edtlib.Node
        # and a list of edtlib.Nodes, respectively). Nothing is generated for
        # them currently though.
        if prop.type in {"phandle", "phandles"}:
            continue

        # Skip properties that we handle elsewhere
        if prop.name in {
            "reg", "compatible", "status", "interrupts",
            "interrupt-controller", "gpio-controller"
        }:
            continue

        if prop.description is not None:
            out_comment(prop.description, blank_before=False)

        ident = str2ident(prop.name)

        if prop.type == "boolean":
            out_dev(node, ident, 1 if prop.val else 0)
        elif prop.type == "string":
            out_dev_s(node, ident, prop.val)
        elif prop.type == "int":
            out_dev(node, ident, prop.val)
        elif prop.type == "array":
            for i, val in enumerate(prop.val):
                out_dev(node, "{}_{}".format(ident, i), val)
        elif prop.type == "string-array":
            for i, val in enumerate(prop.val):
                out_dev_s(node, "{}_{}".format(ident, i), val)
        elif prop.type == "uint8-array":
            out_dev(node, ident,
                    "{ " + ", ".join("0x{:02x}".format(b) for b in prop.val) + " }")
        elif prop.type == "phandle-array":
            write_phandle_val_list(prop)

        # Generate DT_..._ENUM if there's an 'enum:' key in the binding
        if prop.enum_index is not None:
            out_dev(node, ident + "_ENUM", prop.enum_index)


def write_bus(node):
    # Generate bus-related #defines

    if not node.bus:
        return

    if node.parent.label is None:
        err("missing 'label' property on {!r}".format(node.parent))

    # #define DT_<DEV-IDENT>_BUS_NAME <BUS-LABEL>
    out_dev_s(node, "BUS_NAME", str2ident(node.parent.label))
<commit_msg>scripts: dts: generate array values as initializer lists<commit_after>#!/usr/bin/env python3

# Copyright (c) 2019 Nordic Semiconductor ASA
# Copyright (c) 2019 Linaro Limited
# SPDX-License-Identifier: BSD-3-Clause

# This script uses edtlib to generate a header file and a .conf file (both
# containing the same values) from a devicetree (.dts) file. Information from
# binding files in YAML format is used as well.
#
# Bindings are files that describe devicetree nodes. Devicetree nodes are
# usually mapped to bindings via their 'compatible = "..."' property.
#
# See the docstring/comments at the top of edtlib.py for more information.
#
# Note: Do not access private (_-prefixed) identifiers from edtlib here (and
# also note that edtlib is not meant to expose the dtlib API directly).
# Instead, think of what API you need, and add it as a public documented API in
# edtlib. This will keep this script simple.

import argparse
import os
import pathlib
import sys

import edtlib


def main():
    global conf_file
    global header_file

    args = parse_args()

    try:
        edt = edtlib.EDT(args.dts, args.bindings_dirs)
    except edtlib.EDTError as e:
        sys.exit("devicetree error: " + str(e))

    conf_file = open(args.conf_out, "w", encoding="utf-8")
    header_file = open(args.header_out, "w", encoding="utf-8")

    out_comment("Generated by gen_defines.py", blank_before=False)
    out_comment("DTS input file: " + args.dts, blank_before=False)
    out_comment("Directories with bindings: " +
                ", ".join(map(relativize, args.bindings_dirs)),
                blank_before=False)

    active_compats = set()

    for node in edt.nodes:
        if node.enabled and node.matching_compat:
            # Skip 'fixed-partitions' devices since they are handled by
            # write_flash() and would generate extra spurious #defines
            if node.matching_compat == "fixed-partitions":
                continue

            out_comment("Devicetree node: " + node.path)
            out_comment("Binding (compatible = {}): {}".format(
                            node.matching_compat,
                            relativize(node.binding_path)),
                        blank_before=False)
            out_comment("Binding description: " + node.description,
                        blank_before=False)

            write_regs(node)
            write_irqs(node)
            write_props(node)
            write_clocks(node)
            write_spi_dev(node)
            write_bus(node)
            write_existence_flags(node)

            active_compats.update(node.compats)

    out_comment("Active compatibles (mentioned in DTS + binding found)")
    for compat in sorted(active_compats):
        #define DT_COMPAT_<COMPAT> 1
        out("COMPAT_{}".format(str2ident(compat)), 1)

    # These are derived from /chosen
    write_addr_size(edt, "zephyr,sram", "SRAM")
    write_addr_size(edt, "zephyr,ccm", "CCM")
    write_addr_size(edt, "zephyr,dtcm", "DTCM")

    write_flash(edt.chosen_node("zephyr,flash"))
    write_code_partition(edt.chosen_node("zephyr,code-partition"))

    flash_index = 0
    for node in edt.nodes:
        if node.name.startswith("partition@"):
            write_flash_partition(node, flash_index)
            flash_index += 1

    out_comment("Number of flash partitions")
    if flash_index != 0:
        out("FLASH_AREA_NUM", flash_index)

    print("Devicetree configuration written to " + args.conf_out)


def parse_args():
    # Returns parsed command-line arguments

    parser = argparse.ArgumentParser()
    parser.add_argument("--dts", required=True, help="DTS file")
    parser.add_argument("--bindings-dirs", nargs='+', required=True,
                        help="directory with bindings in YAML format, "
                        "we allow multiple")
    parser.add_argument("--header-out", required=True,
                        help="path to write header to")
    parser.add_argument("--conf-out", required=True,
                        help="path to write configuration file to")

    return parser.parse_args()


def relativize(path):
    # If 'path' is within $ZEPHYR_BASE, returns it relative to $ZEPHYR_BASE,
    # with a "$ZEPHYR_BASE/..." hint at the start of the string. Otherwise,
    # returns 'path' unchanged.

    zbase = os.getenv("ZEPHYR_BASE")
    if zbase is None:
        return path

    try:
        return str("$ZEPHYR_BASE" / pathlib.Path(path).relative_to(zbase))
    except ValueError:
        # Not within ZEPHYR_BASE
        return path


def write_regs(node):
    # Writes address/size output for the registers in the node's 'reg' property

    def reg_addr_name_alias(reg):
        return str2ident(reg.name) + "_BASE_ADDRESS" if reg.name else None

    def reg_size_name_alias(reg):
        return str2ident(reg.name) + "_SIZE" if reg.name else None

    for reg in node.regs:
        out_dev(node, reg_addr_ident(reg), hex(reg.addr),
                name_alias=reg_addr_name_alias(reg))

        if reg.size:
            out_dev(node, reg_size_ident(reg), reg.size,
                    name_alias=reg_size_name_alias(reg))


def write_props(node):
    # Writes any properties defined in the "properties" section of the binding
    # for the node

    for prop in node.props.values():
        # Skip #size-cell and other property starting with #. Also skip mapping
        # properties like 'gpio-map'.
        if prop.name[0] == "#" or prop.name.endswith("-map"):
            continue

        # See write_clocks()
        if prop.name == "clocks":
            continue

        # edtlib provides these as well (Property.val becomes an edtlib.Node
        # and a list of edtlib.Nodes, respectively). Nothing is generated for
        # them currently though.
        if prop.type in {"phandle", "phandles"}:
            continue

        # Skip properties that we handle elsewhere
        if prop.name in {
            "reg", "compatible", "status", "interrupts",
            "interrupt-controller", "gpio-controller"
        }:
            continue

        if prop.description is not None:
            out_comment(prop.description, blank_before=False)

        ident = str2ident(prop.name)

        if prop.type == "boolean":
            out_dev(node, ident, 1 if prop.val else 0)
        elif prop.type == "string":
            out_dev_s(node, ident, prop.val)
        elif prop.type == "int":
            out_dev(node, ident, prop.val)
        elif prop.type == "array":
            for i, val in enumerate(prop.val):
                out_dev(node, "{}_{}".format(ident, i), val)
            out_dev(node, ident,
                    "{" + ", ".join(map(str, prop.val)) + "}")
        elif prop.type == "string-array":
            for i, val in enumerate(prop.val):
                out_dev_s(node, "{}_{}".format(ident, i), val)
        elif prop.type == "uint8-array":
            out_dev(node, ident,
                    "{ " + ", ".join("0x{:02x}".format(b) for b in prop.val) + " }")
        elif prop.type == "phandle-array":
            write_phandle_val_list(prop)

        # Generate DT_..._ENUM if there's an 'enum:' key in the binding
        if prop.enum_index is not None:
            out_dev(node, ident + "_ENUM", prop.enum_index)


def write_bus(node):
    # Generate bus-related #defines

    if not node.bus:
        return

    if node.parent.label is None:
        err("missing 'label' property on {!r}".format(node.parent))

    # #define DT_<DEV-IDENT>_BUS_NAME <BUS-LABEL>
    out_dev_s(node, "BUS_NAME", str2ident(node.parent.label))

{'fraction_non_alphanumeric': 0.10802160432086418, 'fraction_numerical': 0.003867440154697606, 'mean_word_length': 2.4391194680119237, 'pattern_counts': {'":': 18, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2854012', 'n_tokens_mistral': 4631, 'n_tokens_neox': 4458, 'n_words': 1243}