<commit_before>function assert(bool, message) {
  if (!bool) throw "Assertion failed! " + (message || "")
}
function isArray(o) {
  return o && o.constructor === Array
}

// List of classes we're allowed to override.

var overrideCategories = [
  "motion",
  "looks",
  "sound",
  "pen",
  "variables",
  "list",
  "events",
  "control",
  "sensing",
  "operators",
  "custom",
  "custom-arg",
  "extension",
  "grey",
  "obsolete",
  "music",
  "video",
  "tts",
  "translate",
  "wedo",
  "ev3",
  "microbit",
  "makeymakey",
]
var overrideShapes = ["hat", "cap", "stack", "boolean", "reporter", "ring"]

// languages that should be displayed right to left
var rtlLanguages = ["ar", "ckb", "fa", "he"]

// List of commands taken from Scratch
var scratchCommands = require("./commands.js")

var inputNumberPat = /\%([0-9]+)/
var inputPat = /(%[a-zA-Z0-9](?:\.[a-zA-Z0-9]+)?)/
var inputPatGlobal = new RegExp(inputPat.source, "g")
var iconPat = /(@[a-zA-Z]+)/
var splitPat = new RegExp(
  [inputPat.source, "|", iconPat.source, "| +"].join(""),
  "g"
)

var hexColorPat = /^#(?:[0-9a-fA-F]{3}){1,2}?$/

function parseInputNumber(part) {
  var m = inputNumberPat.exec(part)
  return m ? +m[1] : 0
}

// used for procDefs
function parseSpec(spec) {
  var parts = spec.split(splitPat).filter(x => !!x)
  var inputs = parts.filter(function(p) {
    return inputPat.test(p)
  })
  return {
    spec: spec,
    parts: parts,
    inputs: inputs,
    hash: hashSpec(spec),
  }
}

function hashSpec(spec) {
  return minifyHash(spec.replace(inputPatGlobal, " _ "))
}

function minifyHash(hash) {
  return hash
    .replace(/_/g, " _ ")
    .replace(/ +/g, " ")
    .replace(/[,%?:]/g, "")
    .replace(/ß/g, "ss")
    .replace(/ä/g, "a")
    .replace(/ö/g, "o")
    .replace(/ü/g, "u")
    .replace(". . .", "...")
    .replace(/^…$/, "...")
    .trim()
    .toLowerCase()
}

var blocksById = {}
var blocksBySpec = {}
var allBlocks = scratchCommands.map(function(def) {
  var spec = def.spec
  if (!def.id) {
    if (!def.selector) throw new Error("Missing ID: " + def.spec)
    def.id = "sb2:" + def.selector
  }
  if (!def.spec) throw new Error("Missing spec: " + def.id)

  var info = {
    id: def.id, // Used for Scratch 3 translations
    spec: def.spec, // Used for Scratch 2 translations
    parts: def.spec.split(splitPat).filter(x => !!x),
    selector: def.selector || "sb3:" + def.id, // Used for JSON marshalling
    inputs: def.inputs,
    shape: def.shape,
    category: def.category,
    hasLoopArrow: !!def.hasLoopArrow,
  }
  if (blocksById[info.id]) {
    throw new Error("Duplicate ID: " + info.id)
  }
  blocksById[info.id] = info
  return (blocksBySpec[spec] = info)
})

var unicodeIcons = {
  "@greenflag": "⚑",
  "@turnright": "↻",
  "@turnleft": "↺",
  "@addinput": "▸",
  "@delinput": "◂",
}

var allLanguages = {}
function loadLanguage(code, language) {
  var blocksByHash = (language.blocksByHash = {})

  Object.keys(language.commands).forEach(function(spec) {
    var nativeSpec = language.commands[spec]
    var block = blocksBySpec[spec]

    var nativeHash = hashSpec(nativeSpec)
    blocksByHash[nativeHash] = block

    // fallback image replacement, for languages without aliases
    var m = iconPat.exec(spec)
    if (m) {
      var image = m[0]
      var hash = nativeHash.replace(iconPat, image => unicodeIcons[image])
      blocksByHash[hash] = block
    }
  })

  language.nativeAliases = {}
  Object.keys(language.aliases).forEach(function(alias) {
    var spec = language.aliases[alias]
    var block = blocksBySpec[spec]
    if (block === undefined) {
      throw new Error("Invalid alias '" + spec + "'")
    }
    var aliasHash = hashSpec(alias)
    blocksByHash[aliasHash] = block

    language.nativeAliases[spec] = alias
  })

  // Some English blocks were renamed between Scratch 2 and Scratch 3. Wire them
  // into language.blocksByHash<commit_msg>Partially revert changes in 4d99a69e4f0bcb66040a56996dd1c49ddb21204c<commit_after>function assert(bool, message) {
  if (!bool) throw "Assertion failed! " + (message || "")
}
function isArray(o) {
  return o && o.constructor === Array
}

// List of classes we're allowed to override.

var overrideCategories = [
  "motion",
  "looks",
  "sound",
  "pen",
  "variables",
  "list",
  "events",
  "control",
  "sensing",
  "operators",
  "custom",
  "custom-arg",
  "extension",
  "grey",
  "obsolete",
  "music",
  "video",
  "tts",
  "translate",
  "wedo",
  "ev3",
  "microbit",
  "makeymakey",
]
var overrideShapes = ["hat", "cap", "stack", "boolean", "reporter", "ring"]

// languages that should be displayed right to left
var rtlLanguages = ["ar", "ckb", "fa", "he"]

// List of commands taken from Scratch
var scratchCommands = require("./commands.js")

var inputNumberPat = /\%([0-9]+)/
var inputPat = /(%[a-zA-Z0-9](?:\.[a-zA-Z0-9]+)?)/
var inputPatGlobal = new RegExp(inputPat.source, "g")
var iconPat = /(@[a-zA-Z]+)/
var splitPat = new RegExp(
  [inputPat.source, "|", iconPat.source, "| +"].join(""),
  "g"
)

var hexColorPat = /^#(?:[0-9a-fA-F]{3}){1,2}?$/

function parseInputNumber(part) {
  var m = inputNumberPat.exec(part)
  return m ? +m[1] : 0
}

// used for procDefs
function parseSpec(spec) {
  var parts = spec.split(splitPat).filter(x => !!x)
  var inputs = parts.filter(function(p) {
    return inputPat.test(p)
  })
  return {
    spec: spec,
    parts: parts,
    inputs: inputs,
    hash: hashSpec(spec),
  }
}

function hashSpec(spec) {
  return minifyHash(spec.replace(inputPatGlobal, " _ "))
}

function minifyHash(hash) {
  return hash
    .replace(/_/g, " _ ")
    .replace(/ +/g, " ")
    .replace(/[,%?:]/g, "")
    .replace(/ß/g, "ss")
    .replace(/ä/g, "a")
    .replace(/ö/g, "o")
    .replace(/ü/g, "u")
    .replace(". . .", "...")
    .replace(/^…$/, "...")
    .trim()
    .toLowerCase()
}

var blocksById = {}
var blocksBySpec = {}
var allBlocks = scratchCommands.map(function(def) {
  var spec = def.spec
  if (!def.id) {
    if (!def.selector) throw new Error("Missing ID: " + def.spec)
    def.id = "sb2:" + def.selector
  }
  if (!def.spec) throw new Error("Missing spec: " + def.id)

  var info = {
    id: def.id, // Used for Scratch 3 translations
    spec: def.spec, // Used for Scratch 2 translations
    parts: def.spec.split(splitPat).filter(x => !!x),
    selector: def.selector || "sb3:" + def.id, // Used for JSON marshalling
    inputs: def.inputs,
    shape: def.shape,
    category: def.category,
    hasLoopArrow: !!def.hasLoopArrow,
  }
  if (blocksById[info.id]) {
    throw new Error("Duplicate ID: " + info.id)
  }
  blocksById[info.id] = info
  return (blocksBySpec[spec] = info)
})

var unicodeIcons = {
  "@greenFlag": "⚑",
  "@turnRight": "↻",
  "@turnLeft": "↺",
  "@addInput": "▸",
  "@delInput": "◂",
}

var allLanguages = {}
function loadLanguage(code, language) {
  var blocksByHash = (language.blocksByHash = {})

  Object.keys(language.commands).forEach(function(spec) {
    var nativeSpec = language.commands[spec]
    var block = blocksBySpec[spec]

    var nativeHash = hashSpec(nativeSpec)
    blocksByHash[nativeHash] = block

    // fallback image replacement, for languages without aliases
    var m = iconPat.exec(spec)
    if (m) {
      var image = m[0]
      var hash = nativeHash.replace(image, unicodeIcons[image])
      blocksByHash[hash] = block
    }
  })

  language.nativeAliases = {}
  Object.keys(language.aliases).forEach(function(alias) {
    var spec = language.aliases[alias]
    var block = blocksBySpec[spec]
    if (block === undefined) {
      throw new Error("Invalid alias '" + spec + "'")
    }
    var aliasHash = hashSpec(alias)
    blocksByHash[aliasHash] = block

    language.nativeAliases[spec] = alias
  })

  // Some English blocks were renamed between Scratch 2 and Scratch 3. Wire them
  // into language.blocksByHash
{'fraction_non_alphanumeric': 0.18047034764826175, 'fraction_numerical': 0.008563394683026585, 'mean_word_length': 3.241192411924119, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '717623', 'n_tokens_mistral': 2883, 'n_tokens_neox': 2830, 'n_words': 656}