<commit_before>from __future__ import division, print_function, absolute_import

import datetime
import os
import platform
import re
import socket
import sys
import time
try:
    import resource
except ImportError:
    resource = None

try:
    # Optional dependency
    import psutil
except ImportError:
    psutil = None

import perf
from perf._cli import format_metadata
from perf._cpu_utils import (format_cpu_list,
                             parse_cpu_list, get_isolated_cpus,
                             get_logical_cpu_count, format_cpu_infos,
                             set_cpu_affinity)
from perf._formatter import format_timedelta, format_datetime
from perf._utils import (MS_WINDOWS,
                         open_text, read_first_line, sysfs_path, proc_path)
if MS_WINDOWS:
    from perf._win_memory import check_tracking_memory, get_peak_pagefile_usage


def normalize_text(text):
    text = str(text)
    text = re.sub(r'\s+', ' ', text)
    return text.strip()


def collect_python_metadata(metadata):
    # Implementation
    impl = perf.python_implementation()
    metadata['python_implementation'] = impl

    # Version
    version = platform.python_version()

    match = re.search(r'\[(PyPy [^ ]+)', sys.version)
    if match:
        version = '%s (Python %s)' % (match.group(1), version)

    bits = platform.architecture()[0]
    if bits:
        if bits == '64bit':
            bits = '64-bit'
        elif bits == '32bit':
            bits = '32-bit'
        version = '%s (%s)' % (version, bits)

    # '74667320778e' in 'Python 2.7.12+ (2.7:74667320778e,'
    match = re.search(r'^[^(]+\([^:]+:([a-f0-9]{6,}\+?),', sys.version)
    if match:
        revision = match.group(1)
    else:
        # 'bbd45126bc691f669c4ebdfbd74456cd274c6b92'
        # in 'Python 2.7.10 (bbd45126bc691f669c4ebdfbd74456cd274c6b92,'
        match = re.search(r'^[^(]+\(([a-f0-9]{6,}\+?),', sys.version)
        if match:
            revision = match.group(1)
        else:
            revision = None
    if revision:
        version = '%s revision %s' % (version, revision)
    metadata['python_version'] = version

    if sys.executable:
        metadata['python_executable'] = sys.executable

    # Before PEP 393 (Python 3.3)
    if sys.version_info < (3, 3):
        if sys.maxunicode == 0xffff:
            unicode_impl = 'UTF-16'
        else:
            unicode_impl = 'UCS-4'
        metadata['python_unicode'] = unicode_impl

    # timer
    if (hasattr(time, 'perf_counter')
       and perf.perf_counter == time.perf_counter):

        info = time.get_clock_info('perf_counter')
        metadata['timer'] = ('%s, resolution: %s'
                             % (info.implementation,
                                format_timedelta(info.resolution)))
    elif perf.perf_counter == time.clock:
        metadata['timer'] = 'time.clock()'
    elif perf.perf_counter == time.time:
        metadata['timer'] = 'time.time()'

    # PYTHONHASHSEED
    if os.environ.get('PYTHONHASHSEED'):
        hash_seed = os.environ['PYTHONHASHSEED']
        try:
            if hash_seed != "random":
                hash_seed = int(hash_seed)
        except ValueError:
            pass
        else:
            metadata['python_hash_seed'] = hash_seed

    # CFLAGS
    try:
        import sysconfig
    except ImportError:
        pass
    else:
        cflags = sysconfig.get_config_var('CFLAGS')
        if cflags:
            cflags = normalize_text(cflags)
            metadata['python_cflags'] = cflags

    # GC disabled?
    try:
        import gc
    except ImportError:
        pass
    else:
        if not gc.isenabled():
            metadata['python_gc'] = 'disabled'


def read_proc(path):
    path = proc_path(path)
    try:
        fp = open_text(path)
        try:
            for line in fp:
                yield line.rstrip()
        finally:
            # don't use context manager to support StringIO on Python 2
            # for unit tests
            fp.close()
    except (OSError, IOError):
        return


def collect_linux_metadata(metadata):
    # ASLR
    for line in read_proc('sys/kernel/randomize_va_space'):
        if line == '0':
            metadata['aslr'] = 'No randomization'
        elif line == '1':
            metadata['aslr'] = 'Conservative randomization'
        elif line == '2':
            metadata['aslr'] = 'Full randomization'
        break


def get_cpu_affinity():
    if hasattr(os, 'sched_getaffinity'):
        return os.sched_getaffinity(0)

    if psutil is not None:
        proc = psutil.Process()
        # cpu_affinity() is only available on Linux, Windows and FreeBSD
        if hasattr(proc, 'cpu_affinity'):
            return proc.cpu_affinity()

    return None


def collect_system_metadata(metadata):
    metadata['platform'] = platform.platform(True, False)
    if sys.platform.startswith('linux'):
        collect_linux_metadata(metadata)

    # on linux, load average over 1 minute
    for line in read_proc("loadavg"):
        fields = line.split()
        loadavg = fields[0]
        metadata['load_avg_1min'] = float(loadavg)

        if len(fields) >= 4 and '/' in fields[3]:
            runnable_threads = fields[3].split('/', 1)[0]
            runnable_threads = int(runnable_threads)
            metadata['runnable_threads'] = runnable_threads

    if not 'load_avg_1min' in metadata and hasattr(os, 'getloadavg'):
        metadata['load_avg_1min'] = os.getloadavg()[0]

    # Hostname
    hostname = socket.gethostname()
    if hostname:
        metadata['hostname'] = hostname

    # Boot time
    boot_time = None
    for line in read_proc("stat"):
        if not line.startswith("btime "):
            continue<commit_msg>Fix PEP8<commit_after>from __future__ import division, print_function, absolute_import

import datetime
import os
import platform
import re
import socket
import sys
import time
try:
    import resource
except ImportError:
    resource = None

try:
    # Optional dependency
    import psutil
except ImportError:
    psutil = None

import perf
from perf._cli import format_metadata
from perf._cpu_utils import (format_cpu_list,
                             parse_cpu_list, get_isolated_cpus,
                             get_logical_cpu_count, format_cpu_infos,
                             set_cpu_affinity)
from perf._formatter import format_timedelta, format_datetime
from perf._utils import (MS_WINDOWS,
                         open_text, read_first_line, sysfs_path, proc_path)
if MS_WINDOWS:
    from perf._win_memory import check_tracking_memory, get_peak_pagefile_usage


def normalize_text(text):
    text = str(text)
    text = re.sub(r'\s+', ' ', text)
    return text.strip()


def collect_python_metadata(metadata):
    # Implementation
    impl = perf.python_implementation()
    metadata['python_implementation'] = impl

    # Version
    version = platform.python_version()

    match = re.search(r'\[(PyPy [^ ]+)', sys.version)
    if match:
        version = '%s (Python %s)' % (match.group(1), version)

    bits = platform.architecture()[0]
    if bits:
        if bits == '64bit':
            bits = '64-bit'
        elif bits == '32bit':
            bits = '32-bit'
        version = '%s (%s)' % (version, bits)

    # '74667320778e' in 'Python 2.7.12+ (2.7:74667320778e,'
    match = re.search(r'^[^(]+\([^:]+:([a-f0-9]{6,}\+?),', sys.version)
    if match:
        revision = match.group(1)
    else:
        # '<KEY>'
        # in 'Python 2.7.10 (bbd45126bc691f669c4ebdfbd74456cd274c6b92,'
        match = re.search(r'^[^(]+\(([a-f0-9]{6,}\+?),', sys.version)
        if match:
            revision = match.group(1)
        else:
            revision = None
    if revision:
        version = '%s revision %s' % (version, revision)
    metadata['python_version'] = version

    if sys.executable:
        metadata['python_executable'] = sys.executable

    # Before PEP 393 (Python 3.3)
    if sys.version_info < (3, 3):
        if sys.maxunicode == 0xffff:
            unicode_impl = 'UTF-16'
        else:
            unicode_impl = 'UCS-4'
        metadata['python_unicode'] = unicode_impl

    # timer
    if (hasattr(time, 'perf_counter')
       and perf.perf_counter == time.perf_counter):

        info = time.get_clock_info('perf_counter')
        metadata['timer'] = ('%s, resolution: %s'
                             % (info.implementation,
                                format_timedelta(info.resolution)))
    elif perf.perf_counter == time.clock:
        metadata['timer'] = 'time.clock()'
    elif perf.perf_counter == time.time:
        metadata['timer'] = 'time.time()'

    # PYTHONHASHSEED
    if os.environ.get('PYTHONHASHSEED'):
        hash_seed = os.environ['PYTHONHASHSEED']
        try:
            if hash_seed != "random":
                hash_seed = int(hash_seed)
        except ValueError:
            pass
        else:
            metadata['python_hash_seed'] = hash_seed

    # CFLAGS
    try:
        import sysconfig
    except ImportError:
        pass
    else:
        cflags = sysconfig.get_config_var('CFLAGS')
        if cflags:
            cflags = normalize_text(cflags)
            metadata['python_cflags'] = cflags

    # GC disabled?
    try:
        import gc
    except ImportError:
        pass
    else:
        if not gc.isenabled():
            metadata['python_gc'] = 'disabled'


def read_proc(path):
    path = proc_path(path)
    try:
        fp = open_text(path)
        try:
            for line in fp:
                yield line.rstrip()
        finally:
            # don't use context manager to support StringIO on Python 2
            # for unit tests
            fp.close()
    except (OSError, IOError):
        return


def collect_linux_metadata(metadata):
    # ASLR
    for line in read_proc('sys/kernel/randomize_va_space'):
        if line == '0':
            metadata['aslr'] = 'No randomization'
        elif line == '1':
            metadata['aslr'] = 'Conservative randomization'
        elif line == '2':
            metadata['aslr'] = 'Full randomization'
        break


def get_cpu_affinity():
    if hasattr(os, 'sched_getaffinity'):
        return os.sched_getaffinity(0)

    if psutil is not None:
        proc = psutil.Process()
        # cpu_affinity() is only available on Linux, Windows and FreeBSD
        if hasattr(proc, 'cpu_affinity'):
            return proc.cpu_affinity()

    return None


def collect_system_metadata(metadata):
    metadata['platform'] = platform.platform(True, False)
    if sys.platform.startswith('linux'):
        collect_linux_metadata(metadata)

    # on linux, load average over 1 minute
    for line in read_proc("loadavg"):
        fields = line.split()
        loadavg = fields[0]
        metadata['load_avg_1min'] = float(loadavg)

        if len(fields) >= 4 and '/' in fields[3]:
            runnable_threads = fields[3].split('/', 1)[0]
            runnable_threads = int(runnable_threads)
            metadata['runnable_threads'] = runnable_threads

    if 'load_avg_1min' not in metadata and hasattr(os, 'getloadavg'):
        metadata['load_avg_1min'] = os.getloadavg()[0]

    # Hostname
    hostname = socket.gethostname()
    if hostname:
        metadata['hostname'] = hostname

    # Boot time
    boot_time = None
    for line in read_proc("stat"):
        if not line.startswith("btime "):
            continue
{'fraction_non_alphanumeric': 0.11275026343519494, 'fraction_numerical': 0.01966982788900597, 'mean_word_length': 2.254, 'pattern_counts': {'":': 2, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3796453', 'n_tokens_mistral': 3667, 'n_tokens_neox': 3448, 'n_words': 818}