<commit_before>import asyncio
import collections
import gc
import logging
import psycopg2
import pytest
import re
import socket
import sys
import time
import uuid
import warnings

from docker import Client as DockerClient

import aiopg
from aiopg import sa


@pytest.fixture(scope='session')
def unused_port():
    def f():
        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
            s.bind(('127.0.0.1', 0))
            return s.getsockname()[1]
    return f


@pytest.yield_fixture
def loop(request):
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(None)

    yield loop

    if not loop._closed:
        loop.call_soon(loop.stop)
        loop.run_forever()
        loop.close()
    gc.collect()
    asyncio.set_event_loop(None)


@pytest.mark.tryfirst
def pytest_pycollect_makeitem(collector, name, obj):
    if collector.funcnamefilter(name):
        if not callable(obj):
            return
        item = pytest.Function(name, parent=collector)
        if 'run_loop' in item.keywords:
            return list(collector._genfunctions(name, obj))


@pytest.mark.tryfirst
def pytest_pyfunc_call(pyfuncitem):
    """
    Run asyncio marked test functions in an event loop instead of a normal
    function call.
    """
    if 'run_loop' in pyfuncitem.keywords:
        funcargs = pyfuncitem.funcargs
        loop = funcargs['loop']
        testargs = {arg: funcargs[arg]
                    for arg in pyfuncitem._fixtureinfo.argnames}
        loop.run_until_complete(pyfuncitem.obj(**testargs))
        return True


def pytest_runtest_setup(item):
    if 'run_loop' in item.keywords and 'loop' not in item.fixturenames:
        # inject an event loop fixture for all async tests
        item.fixturenames.append('loop')


def pytest_ignore_collect(path, config):
    if 'pep492' in str(path):
        if sys.version_info < (3, 5, 0):
            return True


@pytest.fixture(scope='session')
def session_id():
    '''Unique session identifier, random string.'''
    return str(uuid.uuid4())


@pytest.fixture(scope='session')
def docker():
    return DockerClient(version='auto')


def pytest_addoption(parser):
    parser.addoption("--pg_tag", action="append", default=[],
                     help=("Postgres server versions. "
                           "May be used several times. "
                           "Available values: 9.3, 9.4, 9.5, all"))


def pytest_generate_tests(metafunc):
    if 'pg_tag' in metafunc.fixturenames:
        tags = set(metafunc.config.option.pg_tag)
        if not tags:
            tags = ['9.5']
        elif 'all' in tags:
            tags = ['9.3', '9.4', '9.5']
        else:
            tags = list(tags)
        metafunc.parametrize("pg_tag", tags, scope='session')


@pytest.yield_fixture(scope='session')
def pg_server(unused_port, docker, session_id, pg_tag):
    docker.pull('postgres:{}'.format(pg_tag))
    port = unused_port()
    container = docker.create_container(
        image='postgres:{}'.format(pg_tag),
        name='aiopg-test-server-{}-{}'.format(pg_tag, session_id),
        ports=[5432],
        detach=True,
        host_config=docker.create_host_config(port_bindings={5432: port})
    )
    docker.start(container=container['Id'])
    pg_params = dict(database='postgres',
                     user='postgres',
                     password='<PASSWORD>',
                     host='127.0.0.1',
                     port=port)
    delay = 0.001
    for i in range(100):
        try:
            conn = psycopg2.connect(**pg_params)
            cur = conn.cursor()
            cur.execute("CREATE EXTENSION hstore;")
            cur.close()
            conn.close()<commit_msg>Add cmdline option to skip postgres image pulling<commit_after>import asyncio
import collections
import gc
import logging
import psycopg2
import pytest
import re
import socket
import sys
import time
import uuid
import warnings

from docker import Client as DockerClient

import aiopg
from aiopg import sa


@pytest.fixture(scope='session')
def unused_port():
    def f():
        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
            s.bind(('127.0.0.1', 0))
            return s.getsockname()[1]
    return f


@pytest.yield_fixture
def loop(request):
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(None)

    yield loop

    if not loop._closed:
        loop.call_soon(loop.stop)
        loop.run_forever()
        loop.close()
    gc.collect()
    asyncio.set_event_loop(None)


@pytest.mark.tryfirst
def pytest_pycollect_makeitem(collector, name, obj):
    if collector.funcnamefilter(name):
        if not callable(obj):
            return
        item = pytest.Function(name, parent=collector)
        if 'run_loop' in item.keywords:
            return list(collector._genfunctions(name, obj))


@pytest.mark.tryfirst
def pytest_pyfunc_call(pyfuncitem):
    """
    Run asyncio marked test functions in an event loop instead of a normal
    function call.
    """
    if 'run_loop' in pyfuncitem.keywords:
        funcargs = pyfuncitem.funcargs
        loop = funcargs['loop']
        testargs = {arg: funcargs[arg]
                    for arg in pyfuncitem._fixtureinfo.argnames}
        loop.run_until_complete(pyfuncitem.obj(**testargs))
        return True


def pytest_runtest_setup(item):
    if 'run_loop' in item.keywords and 'loop' not in item.fixturenames:
        # inject an event loop fixture for all async tests
        item.fixturenames.append('loop')


def pytest_ignore_collect(path, config):
    if 'pep492' in str(path):
        if sys.version_info < (3, 5, 0):
            return True


@pytest.fixture(scope='session')
def session_id():
    '''Unique session identifier, random string.'''
    return str(uuid.uuid4())


@pytest.fixture(scope='session')
def docker():
    return DockerClient(version='auto')


def pytest_addoption(parser):
    parser.addoption("--pg_tag", action="append", default=[],
                     help=("Postgres server versions. "
                           "May be used several times. "
                           "Available values: 9.3, 9.4, 9.5, all"))
    parser.addoption("--no-pull", action="store_true", default=False,
                     help="Don't perform docker images pulling")


def pytest_generate_tests(metafunc):
    if 'pg_tag' in metafunc.fixturenames:
        tags = set(metafunc.config.option.pg_tag)
        if not tags:
            tags = ['9.5']
        elif 'all' in tags:
            tags = ['9.3', '9.4', '9.5']
        else:
            tags = list(tags)
        metafunc.parametrize("pg_tag", tags, scope='session')


@pytest.yield_fixture(scope='session')
def pg_server(unused_port, docker, session_id, pg_tag, request):
    if not request.config.option.no_pull:
        docker.pull('postgres:{}'.format(pg_tag))
    port = unused_port()
    container = docker.create_container(
        image='postgres:{}'.format(pg_tag),
        name='aiopg-test-server-{}-{}'.format(pg_tag, session_id),
        ports=[5432],
        detach=True,
        host_config=docker.create_host_config(port_bindings={5432: port})
    )
    # inspection = docker.inspect_container(container['Id'])
    # import pprint
    # pprint.pprint(inspection)
    # host = inspection['NetworkSettings']['IPAddress']
    docker.start(container=container['Id'])
    pg_params = dict(database='postgres',
                     user='postgres',
                     password='<PASSWORD>',
                     host='127.0.0.1',
                     port=port)
    delay = 0.001
    for i in range(100):
        try:
            conn = psycopg2.connect(**pg_params)
            cur = conn.cursor()
            cur.execute("CREATE EXTENSION hstore;")
            cur.close()
            conn.close()
{'fraction_non_alphanumeric': 0.11731339435348717, 'fraction_numerical': 0.013407245068969963, 'mean_word_length': 2.5440840566468705, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6465826', 'n_tokens_mistral': 2495, 'n_tokens_neox': 2414, 'n_words': 549}