<commit_before>try:
    import configparser
except ImportError:
    import ConfigParser as configparser
import gzip
import json
import os
from collections import defaultdict
import sys
import textwrap
import random

import coverage
from testmon.process_code import checksum_coverage
from testmon.process_code import Module


def _get_python_lib_paths():
    res = [sys.prefix]
    for attr in ['exec_prefix', 'real_prefix', 'base_prefix']:
        if getattr(sys, attr, sys.prefix) not in res:
            res.append(getattr(sys, attr))
    return [os.path.join(d, "*") for d in res]


def is_dependent(node, changed_py_files):
    if node:
        for changed_file_name in set(node) & set(changed_py_files):
            new_checksums = set(changed_py_files[changed_file_name])
            if set(node[changed_file_name]) - new_checksums:
                return True
        return False
    else:
        # not enough data, means test should run
        return True


def affected_nodeids(nodes, changes):
    affected = []
    for filename in changes:
        for nodeid, node in nodes.items():
            if filename in node:
                new_checksums = set(changes[filename])
                if set(node[filename]) - new_checksums:
                    affected.append(nodeid)
    return affected


class Testmon(object):

    def __init__(self, project_dirs, testmon_labels=set()):
        self.project_dirs = project_dirs
        self.testmon_labels = testmon_labels
        self.setup_coverage(not('singleprocess' in testmon_labels))

    def setup_coverage(self, subprocess):

        includes = [os.path.join(path, '*') for path in self.project_dirs]
        if subprocess:
            if not os.path.exists('.tmontmp'):
                os.makedirs('.tmontmp')

            self.sub_cov_file = os.path.abspath('.tmontmp/.testmoncoverage' + str(random.randint(0, 1000000)))
            with open(self.sub_cov_file + "_rc", "w") as subprocess_rc:
                rc_content = textwrap.dedent("""\
                    [run]
                    data_file = {}
                    include = {}
                    omit = {}
                    parallel=True
                    """).format(self.sub_cov_file,
                                "\n ".join(includes),
                                "\n ".join(_get_python_lib_paths())
                                )
                subprocess_rc.write(rc_content)
            os.environ['COVERAGE_PROCESS_START'] = self.sub_cov_file + "_rc"

        self.cov = coverage.coverage(include=includes,
                                     omit=_get_python_lib_paths(),
                                     data_file=getattr(self, 'sub_cov_file', None),
                                     config_file=False, )
        self.cov.use_cache(False)

    def track_dependencies(self, callable_to_track, nodeid):
        self.cov.erase()
        self.cov.start()
        try:
            result = callable_to_track()
        except:
            raise
        finally:
            self.cov.stop()
            self.cov.save()
            if hasattr(self, 'sub_cov_file'):
                self.cov.combine()

        return result, self.cov.data

    def close(self):
        if hasattr(self, 'sub_cov_file'):
            os.remove(self.sub_cov_file + "_rc")


def eval_variant(run_variant, **kwargs):
    if not run_variant:
        return ''

    eval_locals = {'os': os, 'sys': sys}
    eval_locals.update(kwargs)

    try:
        return eval(run_variant, {}, eval_locals)
    except Exception as e:
        return repr(e)


def get_variant_inifile(inifile):
    config = configparser.ConfigParser()
    config.read(str(inifile),)
    if config.has_section('pytest') and config.has_option('pytest', 'run_variant_expression'):
        run_variant_expression = config.get('pytest', 'run_variant_expression')
    else:
        run_variant_expression = None

    return eval_variant(run_variant_expression)


class TestmonData(object):

    def __init__(self, directory, variant=None):

        self.variant = variant if variant else 'default'
        self.init_connection(directory)
        self.mtimes = {}
        self.node_data = {}
        self.modules_cache = {}
        self.lastfailed = []

    def __eq__(self, other):
        return (self.mtimes, \
                self.node_data, \
                self.lastfailed == other.mtimes, \
                other.node_data, \
                other.lastfailed)

    def is_unchanged(self):
        return (not self.affected and not self.newfile and not self.lastfailed and len(self.node_data)>0)

    def init_connection(self, directory):
        self.datafile = os.path.join(directory, '.testmondata')
        self.connection = None
        import sqlite3

        if os.path.exists(self.datafile):
            self.newfile = False
        else:
            self.newfile = True
        self.connection = sqlite3.connect(self.datafile)
        if getattr(self, 'newfile', False):
            self.init_tables()

    def _fetch_attribute(self, attribute, default=None):
        cursor = self.connection.execute("SELECT data FROM alldata WHERE dataid=?",
                                         [self.variant + ':' + attribute])
        result = cursor.fetchone()
        if result:
            return json.loads(result[0])
        else:
            return default

    def _write_attribute(self, attribute, data):
        dataid = self.variant + ':' + attribute
        cursor = self.connection.execute("UPDATE alldata SET data=? WHERE dataid=?",
                                         [json.dumps(data), dataid])
        if not cursor.rowcount:
            cursor.execute("INSERT INTO alldata VALUES (?, ?)",
                           [dataid, json.dumps(data)])

    def init_tables(self):
        self.connection.execute('CREATE TABLE alldata (dataid text primary key, data text)')

    def read_data(self):
        self.mtimes, \
        self.node_data, \
        self.lastfailed = self._fetch_attribute('mtimes', default={}), \
                          self._fetch_attribute('node_data', default={}), \
                          self._fetch_attribute('lastfailed', default=[])

    def write_data(self):<commit_msg>Adding zlib compression.<commit_after>import zlib

try:
    import configparser
except ImportError:
    import ConfigParser as configparser
import json
import os
from collections import defaultdict
import sys
import textwrap
import random

import coverage
from testmon.process_code import checksum_coverage
from testmon.process_code import Module

if sys.version_info > (3,):
    buffer = memoryview

def _get_python_lib_paths():
    res = [sys.prefix]
    for attr in ['exec_prefix', 'real_prefix', 'base_prefix']:
        if getattr(sys, attr, sys.prefix) not in res:
            res.append(getattr(sys, attr))
    return [os.path.join(d, "*") for d in res]


def is_dependent(node, changed_py_files):
    if node:
        for changed_file_name in set(node) & set(changed_py_files):
            new_checksums = set(changed_py_files[changed_file_name])
            if set(node[changed_file_name]) - new_checksums:
                return True
        return False
    else:
        # not enough data, means test should run
        return True


def affected_nodeids(nodes, changes):
    affected = []
    for filename in changes:
        for nodeid, node in nodes.items():
            if filename in node:
                new_checksums = set(changes[filename])
                if set(node[filename]) - new_checksums:
                    affected.append(nodeid)
    return affected


class Testmon(object):

    def __init__(self, project_dirs, testmon_labels=set()):
        self.project_dirs = project_dirs
        self.testmon_labels = testmon_labels
        self.setup_coverage(not('singleprocess' in testmon_labels))

    def setup_coverage(self, subprocess):

        includes = [os.path.join(path, '*') for path in self.project_dirs]
        if subprocess:
            if not os.path.exists('.tmontmp'):
                os.makedirs('.tmontmp')

            self.sub_cov_file = os.path.abspath('.tmontmp/.testmoncoverage' + str(random.randint(0, 1000000)))
            with open(self.sub_cov_file + "_rc", "w") as subprocess_rc:
                rc_content = textwrap.dedent("""\
                    [run]
                    data_file = {}
                    include = {}
                    omit = {}
                    parallel=True
                    """).format(self.sub_cov_file,
                                "\n ".join(includes),
                                "\n ".join(_get_python_lib_paths())
                                )
                subprocess_rc.write(rc_content)
            os.environ['COVERAGE_PROCESS_START'] = self.sub_cov_file + "_rc"

        self.cov = coverage.coverage(include=includes,
                                     omit=_get_python_lib_paths(),
                                     data_file=getattr(self, 'sub_cov_file', None),
                                     config_file=False, )
        self.cov.use_cache(False)

    def track_dependencies(self, callable_to_track, nodeid):
        self.cov.erase()
        self.cov.start()
        try:
            result = callable_to_track()
        except:
            raise
        finally:
            self.cov.stop()
            self.cov.save()
            if hasattr(self, 'sub_cov_file'):
                self.cov.combine()

        return result, self.cov.data

    def close(self):
        if hasattr(self, 'sub_cov_file'):
            os.remove(self.sub_cov_file + "_rc")


def eval_variant(run_variant, **kwargs):
    if not run_variant:
        return ''

    eval_locals = {'os': os, 'sys': sys}
    eval_locals.update(kwargs)

    try:
        return eval(run_variant, {}, eval_locals)
    except Exception as e:
        return repr(e)


def get_variant_inifile(inifile):
    config = configparser.ConfigParser()
    config.read(str(inifile),)
    if config.has_section('pytest') and config.has_option('pytest', 'run_variant_expression'):
        run_variant_expression = config.get('pytest', 'run_variant_expression')
    else:
        run_variant_expression = None

    return eval_variant(run_variant_expression)


class TestmonData(object):

    def __init__(self, directory, variant=None):

        self.variant = variant if variant else 'default'
        self.init_connection(directory)
        self.mtimes = {}
        self.node_data = {}
        self.modules_cache = {}
        self.lastfailed = []

    def __eq__(self, other):
        return (self.mtimes, \
                self.node_data, \
                self.lastfailed == other.mtimes, \
                other.node_data, \
                other.lastfailed)

    def is_unchanged(self):
        return (not self.affected and not self.newfile and not self.lastfailed and len(self.node_data)>0)

    def init_connection(self, directory):
        self.datafile = os.path.join(directory, '.testmondata')
        self.connection = None
        import sqlite3

        if os.path.exists(self.datafile):
            self.newfile = False
        else:
            self.newfile = True
        self.connection = sqlite3.connect(self.datafile)
        if getattr(self, 'newfile', False):
            self.init_tables()

    def _fetch_attribute(self, attribute, default=None):
        cursor = self.connection.execute("SELECT data FROM alldata WHERE dataid=?",
                                         [self.variant + ':' + attribute])
        result = cursor.fetchone()
        if result:
            return json.loads(zlib.decompress(result[0]).decode('utf-8)'))
        else:
            return default

    def _write_attribute(self, attribute, data):
        dataid = self.variant + ':' + attribute
        json_data = json.dumps(data).encode('utf-8')
        compressed_data_buffer = buffer(zlib.compress(json_data))
        cursor = self.connection.execute("UPDATE alldata SET data=? WHERE dataid=?",
                                         [compressed_data_buffer, dataid])
        if not cursor.rowcount:

            cursor.execute("INSERT INTO alldata VALUES (?, ?)",
                           [dataid, compressed_data_buffer])

    def init_tables(self):
        self.connection.execute('CREATE TABLE alldata (dataid text primary key, data blob)')

    def read_data(self):
        self.mtimes, \
        self.node_data, \
        self.lastfailed = self._fetch_attribute('mtimes', default={}), \
                          self._fetch_attribute('node_data', default={}), \
                          self._fetch_attribute('lastfailed', default=[])

    def write_data(self):
{'fraction_non_alphanumeric': 0.10325228923271235, 'fraction_numerical': 0.0021313545942532367, 'mean_word_length': 2.0185847033595423, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3480139', 'n_tokens_mistral': 3672, 'n_tokens_neox': 3531, 'n_words': 770}