<commit_before>#!/usr/bin/python

env_vars = {}


def read_file(path):
    try:
        with open(path, 'r') as f:
            return f.read().strip()
    except IOError:
        return None


def get_memory_limit():
    limit = read_file('/sys/fs/cgroup/memory/memory.limit_in_bytes')
    if limit:
        env_vars['MEMORY_LIMIT_IN_BYTES'] = limit


def get_number_of_cores():
    core_count = 0

    line = read_file('/sys/fs/cgroup/cpuset/cpuset.cpus')
    if line is None:
        return

    for group in line.split(','):
        core_ids = list(map(int, group.split('-')))
        if len(core_ids) == 2:
            core_count += core_ids[1] - core_ids[0] + 1
        else:
            core_count += 1

    env_vars['NUMBER_OF_CORES'] = str(core_count)

get_memory_limit()
get_number_of_cores()

print("MAX_MEMORY_LIMIT_IN_BYTES=9223372036854775807")<commit_msg>Print warnings to standard error<commit_after>#!/usr/bin/python

from __future__ import print_function
import sys

env_vars = {}


def read_file(path):
    try:
        with open(path, 'r') as f:
            return f.read().strip()
    except IOError:
        return None


def get_memory_limit():
    limit = read_file('/sys/fs/cgroup/memory/memory.limit_in_bytes')
    if limit is None:
        print("Warning: Can't detect memory limit from cgroups",
              file=sys.stderr)
        return
    env_vars['MEMORY_LIMIT_IN_BYTES'] = limit


def get_number_of_cores():
    core_count = 0

    line = read_file('/sys/fs/cgroup/cpuset/cpuset.cpus')
    if line is None:
        print("Warning: Can't detect number of CPU cores from cgroups",
              file=sys.stderr)
        return

    for group in line.split(','):
        core_ids = list(map(int, group.split('-')))
        if len(core_ids) == 2:
            core_count += core_ids[1] - core_ids[0] + 1
        else:
            core_count += 1

    env_vars['NUMBER_OF_CORES'] = str(core_count)

get_memory_limit()
get_number_of_cores()

print("MAX_MEMORY_LIMIT_IN_BYTES=9223372036854775807")
{'fraction_non_alphanumeric': 0.12246913580246914, 'fraction_numerical': 0.024691358024691357, 'mean_word_length': 2.731123388581952, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7350069', 'n_tokens_mistral': 746, 'n_tokens_neox': 702, 'n_words': 144}