<commit_before>#!/usr/bin/env python3

# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
#
# SPDX-License-Identifier: MIT

import json
import os
import subprocess
import sys
import sh
import base64
from datetime import date
from sh.contrib import git

sys.path.append("../docs")
import shared_bindings_matrix

sys.path.append("adabot")
import adabot.github_requests as github

SUPPORTED_PORTS = [
    "atmel-samd",
    "cxd56",
    "esp32s2",
    "litex",
    "mimxrt10xx",
    "nrf",
    "raspberrypi",
    "stm",
]

BIN = ("bin",)
UF2 = ("uf2",)
BIN_UF2 = ("bin", "uf2")
HEX = ("hex",)
HEX_UF2 = ("hex", "uf2")
SPK = ("spk",)
DFU = ("dfu",)
BIN_DFU = ("bin", "dfu")

# Default extensions
extension_by_port = {
    "nrf": UF2,
    "atmel-samd": UF2,
    "stm": BIN,
    "cxd56": SPK,
    "mimxrt10xx": HEX_UF2,
    "litex": DFU,
    "esp32s2": BIN_UF2,
    "raspberrypi": UF2,
}

# Per board overrides
extension_by_board = {
    # samd
    "arduino_mkr1300": BIN_UF2,
    "arduino_mkrzero": BIN_UF2,
    "arduino_nano_33_iot": BIN_UF2,
    "arduino_zero": BIN_UF2,
    "feather_m0_adalogger": BIN_UF2,
    "feather_m0_basic": BIN_UF2,
    "feather_m0_rfm69": BIN_UF2,
    "feather_m0_rfm9x": BIN_UF2,
    "uchip": BIN_UF2,
    # nRF52840 dev kits that may not have UF2 bootloaders,
    "makerdiary_nrf52840_mdk": HEX,
    "makerdiary_nrf52840_mdk_usb_dongle": HEX_UF2,
    "pca10056": BIN_UF2,
    "pca10059": BIN_UF2,
    "electronut_labs_blip": HEX,
    # stm32
    "meowbit_v121": UF2,
}

aliases_by_board = {
    "circuitplayground_express": [
        "circuitplayground_express_4h",
        "circuitplayground_express_digikey_pycon2019",
    ],
    "pybadge": ["edgebadge"],
    "pyportal": ["pyportal_pynt"],
    "gemma_m0": ["gemma_m0_pycon2018"],
    "pewpew10": ["pewpew13"],
}

language_allow_list = set([
    "ID",
    "de_DE",
    "en_US",
    "en_x_pirate",
    "es",
    "fil",
    "fr",
    "it_IT",
    "ja",
    "nl",
    "pl",
    "pt_BR",
    "sv",
    "zh_Latn_pinyin",
])


def get_languages(list_all = False):
    languages = set()
    for f in os.scandir("../locale"):
        if f.name.endswith(".po"):
            languages.add(f.name[:-3])
    if not list_all:
        languages = languages & language_allow_list
    return sorted(list(languages))


def get_board_mapping():
    boards = {}
    for port in SUPPORTED_PORTS:
        board_path = os.path.join("../ports", port, "boards")
        for board_path in os.scandir(board_path):
            if board_path.is_dir():
                board_files = os.listdir(board_path.path)
                board_id = board_path.name
                extensions = extension_by_port[port]
                extensions = extension_by_board.get(board_path.name, extensions)
                aliases = aliases_by_board.get(board_path.name, [])
                boards[board_id] = {
                    "port": port,
                    "extensions": extensions,
                    "download_count": 0,
                    "aliases": aliases,
                }
                for alias in aliases:
                    boards[alias] = {
                        "port": port,
                        "extensions": extensions,
                        "download_count": 0,
                        "alias": True,
                        "aliases": [],<commit_msg>sort languages in a way that is not case sensitive<commit_after>#!/usr/bin/env python3

# SPDX-FileCopyrightText: 2014 MicroPython & CircuitPython contributors (https://github.com/adafruit/circuitpython/graphs/contributors)
#
# SPDX-License-Identifier: MIT

import json
import os
import subprocess
import sys
import sh
import base64
from datetime import date
from sh.contrib import git

sys.path.append("../docs")
import shared_bindings_matrix

sys.path.append("adabot")
import adabot.github_requests as github

SUPPORTED_PORTS = [
    "atmel-samd",
    "cxd56",
    "esp32s2",
    "litex",
    "mimxrt10xx",
    "nrf",
    "raspberrypi",
    "stm",
]

BIN = ("bin",)
UF2 = ("uf2",)
BIN_UF2 = ("bin", "uf2")
HEX = ("hex",)
HEX_UF2 = ("hex", "uf2")
SPK = ("spk",)
DFU = ("dfu",)
BIN_DFU = ("bin", "dfu")

# Default extensions
extension_by_port = {
    "nrf": UF2,
    "atmel-samd": UF2,
    "stm": BIN,
    "cxd56": SPK,
    "mimxrt10xx": HEX_UF2,
    "litex": DFU,
    "esp32s2": BIN_UF2,
    "raspberrypi": UF2,
}

# Per board overrides
extension_by_board = {
    # samd
    "arduino_mkr1300": BIN_UF2,
    "arduino_mkrzero": BIN_UF2,
    "arduino_nano_33_iot": BIN_UF2,
    "arduino_zero": BIN_UF2,
    "feather_m0_adalogger": BIN_UF2,
    "feather_m0_basic": BIN_UF2,
    "feather_m0_rfm69": BIN_UF2,
    "feather_m0_rfm9x": BIN_UF2,
    "uchip": BIN_UF2,
    # nRF52840 dev kits that may not have UF2 bootloaders,
    "makerdiary_nrf52840_mdk": HEX,
    "makerdiary_nrf52840_mdk_usb_dongle": HEX_UF2,
    "pca10056": BIN_UF2,
    "pca10059": BIN_UF2,
    "electronut_labs_blip": HEX,
    # stm32
    "meowbit_v121": UF2,
}

aliases_by_board = {
    "circuitplayground_express": [
        "circuitplayground_express_4h",
        "circuitplayground_express_digikey_pycon2019",
    ],
    "pybadge": ["edgebadge"],
    "pyportal": ["pyportal_pynt"],
    "gemma_m0": ["gemma_m0_pycon2018"],
    "pewpew10": ["pewpew13"],
}

language_allow_list = set([
    "ID",
    "de_DE",
    "en_US",
    "en_x_pirate",
    "es",
    "fil",
    "fr",
    "it_IT",
    "ja",
    "nl",
    "pl",
    "pt_BR",
    "sv",
    "zh_Latn_pinyin",
])


def get_languages(list_all = False):
    languages = set()
    for f in os.scandir("../locale"):
        if f.name.endswith(".po"):
            languages.add(f.name[:-3])
    if not list_all:
        languages = languages & language_allow_list
    return sorted(list(languages), key = lambda s: s.casefold())


def get_board_mapping():
    boards = {}
    for port in SUPPORTED_PORTS:
        board_path = os.path.join("../ports", port, "boards")
        for board_path in os.scandir(board_path):
            if board_path.is_dir():
                board_files = os.listdir(board_path.path)
                board_id = board_path.name
                extensions = extension_by_port[port]
                extensions = extension_by_board.get(board_path.name, extensions)
                aliases = aliases_by_board.get(board_path.name, [])
                boards[board_id] = {
                    "port": port,
                    "extensions": extensions,
                    "download_count": 0,
                    "aliases": aliases,
                }
                for alias in aliases:
                    boards[alias] = {
                        "port": port,
                        "extensions": extensions,
                        "download_count": 0,
                        "alias": True,
                        "aliases": [],
{'fraction_non_alphanumeric': 0.13622471256003493, 'fraction_numerical': 0.03114539368359773, 'mean_word_length': 2.401980198019802, 'pattern_counts': {'":': 74, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1043675', 'n_tokens_mistral': 2758, 'n_tokens_neox': 2599, 'n_words': 471}