<commit_before>from urllib2 import urlopen
import csv
import lxml.builder as lb
from lxml import etree

# Returns CSV of all wikipedias, ordered by number of 'good' articles
URL = "https://wikistats.wmflabs.org/api.php?action=dump&table=wikipedias&format=csv&s=good"

data = csv.reader(urlopen(URL))

# Column 2 is the language code
lang_keys = [row[2] for row in data]

del lang_keys[0] # Get rid of the headers

# Generate the XML
x = lb.E

keys = [x.item(k) for k in lang_keys]

resources = x.resources(
    getattr(x, 'string-array')(*keys, name="preference_language_keys"),
)

open("languages_list.xml", "w").write(
    etree.tostring(resources, pretty_print=True, encoding="utf-8", xml_declaration=True)
)
<commit_msg>Modify language generation script to make JSON for iOS<commit_after>from urllib2 import urlopen
import csv
import json
import lxml.builder as lb
from lxml import etree

# Returns CSV of all wikipedias, ordered by number of 'good' articles
URL = "https://wikistats.wmflabs.org/api.php?action=dump&table=wikipedias&format=csv&s=good"

data = csv.reader(urlopen(URL))

lang_keys = []
lang_local_names = []
lang_eng_names = []
for row in data:
    lang_keys.append(row[2])
    lang_local_names.append(row[10])
    lang_eng_names.append(row[1])

# Generate the XML, for Android
x = lb.E

keys = [x.item(k) for k in lang_keys]

# Skip the headers!
del keys[0]
resources = x.resources(
    getattr(x, 'string-array')(*keys, name="preference_language_keys"),
)

open("languages_list.xml", "w").write(
    etree.tostring(resources, pretty_print=True, encoding="utf-8", xml_declaration=True)
)

# Generate the JSON, for iOS
langs_json = []

# Start from 1, to skip the headers
for i in xrange(1, len(lang_keys)):
    langs_json.append({
        "code": lang_keys[i],
        "name": lang_local_names[i],
        "canonical_name": lang_eng_names[i]
    })

open("languages_list.json", "w").write(json.dumps(langs_json, indent=4))
{'fraction_non_alphanumeric': 0.13079299691040164, 'fraction_numerical': 0.007723995880535531, 'mean_word_length': 4.960122699386503, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3727209', 'n_tokens_mistral': 708, 'n_tokens_neox': 697, 'n_words': 176}