<commit_before>from __future__ import division
import numpy as np


def xval(Xlist, K=10):
    """
    Input:
        - Xlist: list of ids [tip: maybe use np.random.permutation(len(Xlist)) before]
        - K: number of folds
    Generator yields:
        - training and test sets with ids
    """
    for k in xrange(K):
        train = [x for i, x in enumerate(Xlist) if i % K != k]
        test = [x for i, x in enumerate(Xlist) if i % K == k]
        yield train, test

def balanced_xval(Xdict, K=10, random_seed=None):
    """
    Input:
        - Xdict: dictionary with category : list of doc_ids
        - K: number of folds
    Generator yields:
        - training and test sets that are balanced with respect to
          the number of docs from each category (i.e. ratio stays the same)
    """
    if random_seed is not None:
        np.random.seed(random_seed)
    Xlist = []
    for cat in Xdict:
        Xlist.extend(list(np.random.permutation(Xdict[cat])))
    for k in xrange(K):
        train = [x for i, x in enumerate(Xlist) if i % K != k]
        test = [x for i, x in enumerate(Xlist) if i % K == k]
        yield train, test
<commit_msg>to make the code run identically on all systems, order the keys of the dict before randomizing<commit_after>from __future__ import division
import numpy as np


def xval(Xlist, K=10):
    """
    Input:
        - Xlist: list of ids [tip: maybe use np.random.permutation(len(Xlist)) before]
        - K: number of folds
    Generator yields:
        - training and test sets with ids
    """
    for k in xrange(K):
        train = [x for i, x in enumerate(Xlist) if i % K != k]
        test = [x for i, x in enumerate(Xlist) if i % K == k]
        yield train, test

def balanced_xval(Xdict, K=10, random_seed=None):
    """
    Input:
        - Xdict: dictionary with category : list of doc_ids
        - K: number of folds
    Generator yields:
        - training and test sets that are balanced with respect to
          the number of docs from each category (i.e. ratio stays the same)
    """
    if random_seed is not None:
        np.random.seed(random_seed)
    Xlist = []
    for cat in Xdict:
        Xlist.extend(list(np.random.permutation(sorted(Xdict[cat]))))
    for k in xrange(K):
        train = [x for i, x in enumerate(Xlist) if i % K != k]
        test = [x for i, x in enumerate(Xlist) if i % K == k]
        yield train, test

{'fraction_non_alphanumeric': 0.10012463647694225, 'fraction_numerical': 0.0033236393851267137, 'mean_word_length': 2.2672998643147895, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6218315', 'n_tokens_mistral': 783, 'n_tokens_neox': 757, 'n_words': 302}