<commit_before>#! /usr/bin/env python

import wikipedia
import json
from subprocess import call

freeciv_wiki_doc = {};

def fix_tech(tech_name):
  if (tech_name.startswith("?")):
    tech_name = tech_name.partition(':')[2];

  if (tech_name == "Advanced Flight"): tech_name = "Aeronautics";
  if (tech_name == "Rocketry"): tech_name = "Rocket";
  if (tech_name == "Stealth"): tech_name = "Stealth technology";
  if (tech_name == "Tactics"): tech_name = "Military tactics";
  if (tech_name == "The Republic"): tech_name = "Repubic";
  if (tech_name == "The Corporation"): tech_name = "Corporation";
  if (tech_name == "The Wheel"): tech_name = "Wheel";
  if (tech_name == "Archers"): tech_name = "Archery";
  if (tech_name == "Partisan"): tech_name = "Partisan military";
  if (tech_name == "Horsemen"): tech_name = "Equestrianism";
  if (tech_name == "Fighter"): tech_name = "Fighter aircraft";
  if (tech_name == "Carrier"): tech_name = "Aircraft carrier";
  if (tech_name == "Legion"): tech_name = "Roman legion";
  if (tech_name == "Magellan's Expedition"): tech_name = "Ferd<NAME>an";
  if (tech_name == "Nuclear"): tech_name = "Nuclear weapon";
  if (tech_name == "Caravan"): tech_name = "Camel train";
  if (tech_name == "Aqueduct"): tech_name = "Aqueduct water supply";
  if (tech_name == "Barracks II"): tech_name = "Barracks";
  if (tech_name == "Barracks III"): tech_name = "Barracks";
  if (tech_name == "Mfg. Plant"): tech_name = "Factory";
  if (tech_name == "Sewer System"): tech_name = "Sanitary_sewer";
  if (tech_name == "Space Structural"): tech_name = "Spacecraft";
  if (tech_name == "A.Smith's Trading Co."): tech_name = "<NAME>";
  if (tech_name == "Colossus"): tech_name = "Colossus of Rhodes";
  if (tech_name == "Michelangelo's Chapel"): tech_name = "<NAME>";
  if (tech_name == "Shakespeare's Theater"): tech_name = "<NAME>";
  if (tech_name == "Coinage"): tech_name = "Coining (mint)";
  if (tech_name == "SDI Defense"): tech_name = "Strategic Defense Initiative";
  if (tech_name == "Coastal Defense"): tech_name = "Coastal defence and fortification";
  if (tech_name == "Copernicus' Observatory"): tech_name = "Space observatory";
  if (tech_name == "Mech. Inf."): tech_name = "Mechanized infantry";
  if (tech_name == "Sun Tzu's War Academy"): tech_name = "Sun Tzu";
  if (tech_name == "Mobile Warfare"): tech_name = "Mobile_Warfare";
  if (tech_name == "Leonardo's Workshop"): tech_name = "<NAME>";
  if (tech_name == "SETI Program"): tech_name = "Search for extraterrestrial intelligence";
  if (tech_name == "Mass Transit"): tech_name = "Public transport";
  if (tech_name == "Spy"): tech_name = "Espionage";
  if (tech_name == "Space Component"): tech_name = "Spacecraft propulsion";
  if (tech_name == "Space Module"): tech_name = "Life support system";
  if (tech_name == "Space Structural"): tech_name = "Spacecraft design";
  return tech_name;<commit_msg>update-wikipedia-docs: clarify <NAME>.<commit_after>#! /usr/bin/env python

import wikipedia
import json
from subprocess import call

freeciv_wiki_doc = {};

def fix_tech(tech_name):
  if (tech_name.startswith("?")):
    tech_name = tech_name.partition(':')[2];

  if (tech_name == "Advanced Flight"): tech_name = "Aeronautics";
  if (tech_name == "Rocketry"): tech_name = "Rocket";
  if (tech_name == "Stealth"): tech_name = "Stealth technology";
  if (tech_name == "Tactics"): tech_name = "Military tactics";
  if (tech_name == "The Republic"): tech_name = "Repubic";
  if (tech_name == "The Corporation"): tech_name = "Corporation";
  if (tech_name == "The Wheel"): tech_name = "Wheel";
  if (tech_name == "Archers"): tech_name = "Archery";
  if (tech_name == "Partisan"): tech_name = "Partisan military";
  if (tech_name == "Horsemen"): tech_name = "Equestrianism";
  if (tech_name == "Fighter"): tech_name = "Fighter aircraft";
  if (tech_name == "Carrier"): tech_name = "Aircraft carrier";
  if (tech_name == "Legion"): tech_name = "Roman legion";
  if (tech_name == "Magellan's Expedition"): tech_name = "Ferd<NAME>";
  if (tech_name == "Nuclear"): tech_name = "Nuclear weapon";
  if (tech_name == "Caravan"): tech_name = "Camel train";
  if (tech_name == "Aqueduct"): tech_name = "Aqueduct water supply";
  if (tech_name == "Barracks II"): tech_name = "Barracks";
  if (tech_name == "Barracks III"): tech_name = "Barracks";
  if (tech_name == "Mfg. Plant"): tech_name = "Factory";
  if (tech_name == "Sewer System"): tech_name = "Sanitary_sewer";
  if (tech_name == "Space Structural"): tech_name = "Spacecraft";
  if (tech_name == "A.Smith's Trading Co."): tech_name = "<NAME>";
  if (tech_name == "Colossus"): tech_name = "Colossus of Rhodes";
  if (tech_name == "Michelangelo's Chapel"): tech_name = "Sistine Chapel";
  if (tech_name == "Shakespeare's Theater"): tech_name = "<NAME>";
  if (tech_name == "Coinage"): tech_name = "Coining (mint)";
  if (tech_name == "SDI Defense"): tech_name = "Strategic Defense Initiative";
  if (tech_name == "Coastal Defense"): tech_name = "Coastal defence and fortification";
  if (tech_name == "Copernicus' Observatory"): tech_name = "Space observatory";
  if (tech_name == "Mech. Inf."): tech_name = "Mechanized infantry";
  if (tech_name == "Sun Tzu's War Academy"): tech_name = "Sun Tzu";
  if (tech_name == "Mobile Warfare"): tech_name = "Mobile_Warfare";
  if (tech_name == "Leonardo's Workshop"): tech_name = "<NAME>";
  if (tech_name == "SETI Program"): tech_name = "Search for extraterrestrial intelligence";
  if (tech_name == "<NAME>hedral"): tech_name = "<NAME>";
  if (tech_name == "Mass Transit"): tech_name = "Public transport";
  if (tech_name == "Spy"): tech_name = "Espionage";
  if (tech_name == "Space Component"): tech_name = "Spacecraft propulsion";
  if (tech_name == "Space Module"): tech_name = "Life support system";
  if (tech_name == "Space Structural"): tech_name = "Spacecraft design";
  return tech_name;
{'fraction_non_alphanumeric': 0.17304747320061256, 'fraction_numerical': 0.0003403096818104475, 'mean_word_length': 5.266524520255864, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2865055', 'n_tokens_mistral': 2067, 'n_tokens_neox': 2083, 'n_words': 561}