<commit_before>#!/usr/bin/env python3
#
# This script is used for downloading the dataset used by the examples.
# Dataset used: UCI / Pima Indians Diabetes (in libsvm format)

import os
import urllib.request


DATASET_URL = 'http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/diabetes'
TARGET_PATH = os.path.dirname(os.path.realpath(__file__)) + '/dataset.txt'


def main():
    urllib.request.urlretrieve(DATASET_URL, TARGET_PATH)


if __name__ == '__main__': main()<commit_msg>Change dataset used in example (letter)<commit_after>#!/usr/bin/env python3
#
# This script is used for downloading the dataset used by the examples.
# Dataset used: Statlog / Letter (in libsvm format)

import os
import urllib.request
import random


DATASET_URL = 'http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass/letter.scale'
DATASET_SIZE = 1000
TARGET_PATH = os.path.dirname(os.path.realpath(__file__)) + '/dataset.txt'


def main():
    rows = list(urllib.request.urlopen(DATASET_URL))
    selected = random.sample(rows, DATASET_SIZE)
    with open(TARGET_PATH, 'wb') as f:
        for row in selected:
            f.write(row)


if __name__ == '__main__': main()
{'fraction_non_alphanumeric': 0.12340425531914893, 'fraction_numerical': 0.005106382978723404, 'mean_word_length': 4.321266968325792, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4404255', 'n_tokens_mistral': 424, 'n_tokens_neox': 407, 'n_words': 99}