<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2013-2016, <NAME>
# Copyright (c)      2016, <NAME>
# All rights reserved.

import math
import logging

from ..device_tree import DeviceTree
from ..input.xml import XMLReader

from .avr_identifier import AVRIdentifier
from . import avr_io
from . import avr_mcu

LOGGER = logging.getLogger("dfg.avr.reader")

class AVRDeviceTree:
    """ AVRDeviceTree
    This AVR specific part description file reader knows the structure and
    translates the data into a platform independent format.
    """

    @staticmethod
    def _devices_from_file(filename):
        device_file = XMLReader(filename)
        devices = device_file.query("//variants/variant/@ordercode")
        return list(set(d for d in devices if d != "standard"))

    @staticmethod
    def _properties_from_file(filename, devname):
        p = {}

        device_file = XMLReader(filename)
        device = device_file.query("//device")[0]
        variant = device_file.query(f'//variants/variant[@ordercode="{devname}"]')[0]
        did = AVRIdentifier.from_string(devname.lower())
        p["id"] = did

        LOGGER.info("Parsing '%s'", did.string)

        mcu = avr_mcu.getMcuForDevice(did)
        mcu = "unsupported" if mcu == None else mcu
        p["mcu"] = mcu
        p["core"] = device.get("architecture").lower()
        # We don"t care about Microchip"s new abominations
        if p["core"] == "avr8x":
            return None

        p["define"] = "__AVR_" + device.get("name") + "__"

        p["max_fcpu"] = variant.get("speedmax")
        p["package"] = variant.get("package")
        p["pinout"] = variant.get("pinout")
        p["pinout_pins"] = {
            p.get("position"): p.get("pad")
            for p in device_file.query(f'//pinouts/pinout[@name="{p["pinout"]}"]/pin')
        }

        # find the values for flash, ram and (optional) eeprom
        for memory_segment in device_file.query("//memory-segment"):
            name = memory_segment.get("name")
            size = int(memory_segment.get("size"), 16)
            if name in ["FLASH", "APP_SECTION", "PROGMEM"]:
                p["flash"] = size
            elif name in ["IRAM", "SRAM", "INTERNAL_SRAM"]:
                p["ram"] = size
            elif name == "EEPROM":
                p["eeprom"] = size

        raw_modules = device_file.query("//peripherals/module/instance")
        modules = []
        signals = []
        gpios = []
        ports = []
        for m in raw_modules:
            tmp = {"module": m.getparent().get("name").lower(),
                   "instance": m.get("name").lower()}
            # module SPI has two instances USART0_SPI, USART1_SPI, wtf?
            if tmp["module"] == "spi" and tmp["instance"].startswith("usart"): continue;
            if tmp["module"] == "port":
                ports.append(tmp)
            else:
                modules.append(tmp)
        p["modules"] = sorted(list(set([(m["module"], m["instance"]) for m in modules])))

        raw_signals = device_file.query("//peripherals/module/instance/signals/signal")
        modules_only = not len(raw_signals)
        p["modules_only"] = modules_only

        if modules_only:
            # Parse GPIOs manually from mask
            for port in ports:
                port = port["instance"][-1:]
                port_mask = device_file.query('//modules/module[@name="PORT"]/register-group[@name="PORT{0}"]/register[@name="PORT{0}"]/@mask'.format(port.upper()))
                if len(port_mask) == 0:
                    # The port mask is split up
                    port_mask = device_file.query('//modules/module[@name="PORT"]/register-group[@name="PORT{0}"]/register[@name="PORT{0}"]/bitfield/@mask'.format(port.upper()))
                    port_mask = sum([int(mask, 16) for mask in port_mask])
                else:
                    port_mask = int(port_mask[0], 16)
                for pos in range(8):
                    if (1 << pos) & port_mask:
                        gpios.append((port, str(pos)))
            gpios = sorted(gpios)
        else:
            for s in raw_signals:
                tmp = {"module": s.getparent().getparent().getparent().get("name").lower(),
                       "instance": s.getparent().getparent().get("name").lower()}
                tmp.update({k:v.lower() for k,v in s.items()})
                if tmp["group"] in ["p", "pin"] or tmp["group"].startswith("port"):
                    gpios.append(tmp)
                else:
                    signals.append(tmp)
            gpios = sorted([(g["pad"][1], g["pad"][2]) for g in gpios])

        # Signal information is missing
        p["signals"] = signals
        # Filter gpios by pinout
        p["gpios"] = [pin for pin in gpios if f"P{pin[0].upper()}{pin[1]}" in p["pinout_pins"].values()]
        # print(signals)
        # print(p["gpios"])
        # print(p["pinout"], p["pinout_pins"])

        LOGGER.debug("Found GPIOs: [%s]", ", ".join([p.upper() + i for p,i in p["gpios"]]))
        LOGGER.debug("Available Modules are:\n" + AVRDeviceTree._modulesToString(p["modules"]))

        interrupts = []
        for i in device_file.query("//interrupts/interrupt"):
            interrupts.append({"position": i.get("index"), "name": i.get("name")})
        p["interrupts"] = interrupts

        return p
<commit_msg>[dfg] Remove AVRs without GPIOs<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2013-2016, <NAME>
# Copyright (c)      2016, <NAME>
# All rights reserved.

import math
import logging

from ..device_tree import DeviceTree
from ..input.xml import XMLReader

from .avr_identifier import AVRIdentifier
from . import avr_io
from . import avr_mcu

LOGGER = logging.getLogger("dfg.avr.reader")

class AVRDeviceTree:
    """ AVRDeviceTree
    This AVR specific part description file reader knows the structure and
    translates the data into a platform independent format.
    """

    @staticmethod
    def _devices_from_file(filename):
        device_file = XMLReader(filename)
        devices = device_file.query("//variants/variant/@ordercode")
        return list(set(d for d in devices if d != "standard"))

    @staticmethod
    def _properties_from_file(filename, devname):
        p = {}

        device_file = XMLReader(filename)
        device = device_file.query("//device")[0]
        variant = device_file.query(f'//variants/variant[@ordercode="{devname}"]')[0]
        did = AVRIdentifier.from_string(devname.lower())
        p["id"] = did

        LOGGER.info("Parsing '%s'", did.string)

        mcu = avr_mcu.getMcuForDevice(did)
        mcu = "unsupported" if mcu == None else mcu
        p["mcu"] = mcu
        p["core"] = device.get("architecture").lower()
        # We don"t care about Microchip's new abominations
        if p["core"] == "avr8x":
            return None

        p["define"] = "__AVR_" + device.get("name") + "__"

        p["max_fcpu"] = variant.get("speedmax")
        p["package"] = variant.get("package")
        p["pinout"] = variant.get("pinout")
        p["pinout_pins"] = {
            p.get("position"): p.get("pad")
            for p in device_file.query(f'//pinouts/pinout[@name="{p["pinout"]}"]/pin')
        }

        # find the values for flash, ram and (optional) eeprom
        for memory_segment in device_file.query("//memory-segment"):
            name = memory_segment.get("name")
            size = int(memory_segment.get("size"), 16)
            if name in ["FLASH", "APP_SECTION", "PROGMEM"]:
                p["flash"] = size
            elif name in ["IRAM", "SRAM", "INTERNAL_SRAM"]:
                p["ram"] = size
            elif name == "EEPROM":
                p["eeprom"] = size

        raw_modules = device_file.query("//peripherals/module/instance")
        modules = []
        signals = []
        gpios = []
        ports = []
        for m in raw_modules:
            tmp = {"module": m.getparent().get("name").lower(),
                   "instance": m.get("name").lower()}
            # module SPI has two instances USART0_SPI, USART1_SPI, wtf?
            if tmp["module"] == "spi" and tmp["instance"].startswith("usart"): continue;
            if tmp["module"] == "port":
                ports.append(tmp)
            else:
                modules.append(tmp)
        p["modules"] = sorted(list(set([(m["module"], m["instance"]) for m in modules])))

        raw_signals = device_file.query("//peripherals/module/instance/signals/signal")
        modules_only = not len(raw_signals)
        p["modules_only"] = modules_only

        if modules_only:
            # Parse GPIOs manually from mask
            for port in ports:
                port = port["instance"][-1:]
                port_mask = device_file.query('//modules/module[@name="PORT"]/register-group[@name="PORT{0}"]/register[@name="PORT{0}"]/@mask'.format(port.upper()))
                if len(port_mask) == 0:
                    # The port mask is split up
                    port_mask = device_file.query('//modules/module[@name="PORT"]/register-group[@name="PORT{0}"]/register[@name="PORT{0}"]/bitfield/@mask'.format(port.upper()))
                    port_mask = sum([int(mask, 16) for mask in port_mask])
                else:
                    port_mask = int(port_mask[0], 16)
                for pos in range(8):
                    if (1 << pos) & port_mask:
                        gpios.append((port, str(pos)))
            gpios = sorted(gpios)
        else:
            for s in raw_signals:
                tmp = {"module": s.getparent().getparent().getparent().get("name").lower(),
                       "instance": s.getparent().getparent().get("name").lower()}
                tmp.update({k:v.lower() for k,v in s.items()})
                if tmp["group"] in ["p", "pin"] or tmp["group"].startswith("port"):
                    gpios.append(tmp)
                else:
                    signals.append(tmp)
            gpios = sorted([(g["pad"][1], g["pad"][2]) for g in gpios])

        # Signal information is missing
        p["signals"] = signals
        # Filter gpios by pinout
        p["gpios"] = [pin for pin in gpios if f"P{pin[0].upper()}{pin[1]}" in p["pinout_pins"].values()]
        # print(signals)
        # print(p["gpios"])
        # print(p["pinout"], p["pinout_pins"])

        # if no GPIOs are found, don't bother with this device
        if not p["gpios"]:
            LOGGER.error("No GPIOs found for '{}'!".format(did.string))
            return None

        LOGGER.debug("Found GPIOs: [%s]", ", ".join([p.upper() + i for p,i in p["gpios"]]))
        LOGGER.debug("Available Modules are:\n" + AVRDeviceTree._modulesToString(p["modules"]))

        interrupts = []
        for i in device_file.query("//interrupts/interrupt"):
            interrupts.append({"position": i.get("index"), "name": i.get("name")})
        p["interrupts"] = interrupts

        return p

{'fraction_non_alphanumeric': 0.15739807604214384, 'fraction_numerical': 0.006779661016949152, 'mean_word_length': 2.189947399181765, 'pattern_counts': {'":': 18, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5990096', 'n_tokens_mistral': 3403, 'n_tokens_neox': 3298, 'n_words': 732}