<commit_before>#!/usr/bin/python
# coding=utf-8

import socket
import subprocess
import time
import os
import errno
from singleton import Singleton
import tempfile
import shutil
import stat
import json

import logging
logger = logging.getLogger(__name__)


HOSTNAME = 'localhost'


class PortPool(Singleton):

    __ports = set()
    __closed = set()
    __id = None

    def __init__(self, min_port=1025, max_port=2000, port_sequence=None):
        """
        Args:
            min_port - min port number  (ignoring if 'port_sequence' is not None)
            max_port - max port number  (ignoring if 'port_sequence' is not None)
            port_sequence - iterate sequence which contains numbers of ports
        """
        if not self.__id:  # singleton checker
            self.__id = id(self)
            self.__init_range(min_port, max_port, port_sequence)

    def __init_range(self, min_port=1025, max_port=2000, port_sequence=None):
        if port_sequence:
            self.__ports = set(port_sequence)
        else:
            self.__ports = set(xrange(min_port, max_port + 1))
        self.__closed = set()
        self.refresh()

    def __check_port(self, port):
        """check port status
        return True if port is free, False else
        """
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.bind((HOSTNAME, port))
            return True
        except socket.error:
            return False
        finally:
            s.close()

    def release_port(self, port):
        """release port"""
        if port in self.__closed:
            self.__closed.remove(port)
        self.__ports.add(port)

    def port(self, check=False):
        """return next opened port
        Args:
          check - check is port realy free
        """
        if not self.__ports:  # refresh ports if sequence is empty
            self.refresh()

        try:
            port = self.__ports.pop()
            if check:
                while not self.__check_port(port):
                    self.release_port(port)
                    port = self.__ports.pop()
        except (IndexError, KeyError):
            raise IndexError("Could not find a free port,\nclosed ports: {closed}".format(closed=self.__closed))
        self.__closed.add(port)
        return port

    def refresh(self, only_closed=False):
        """refresh ports status
        Args:
          only_closed - check status only for closed ports
        """
        if only_closed:
            opened = filter(self.__check_port, self.__closed)
            self.__closed = self.__closed.difference(opened)
            self.__ports = self.__ports.union(opened)
        else:
            ports = self.__closed.union(self.__ports)
            self.__ports = set(filter(self.__check_port, ports))
            self.__closed = ports.difference(self.__ports)

    def change_range(self, min_port=1025, max_port=2000, port_sequence=None):
        """change Pool port range"""
        self.__init_range(min_port, max_port, port_sequence)


def wait_for(port_num, timeout):
    """waits while process starts.
    Args:
        port_num    - port number
        timeout     - specify how long, in seconds, a command can take before times out.
    return True if process started, return False if not
    """
    logger.debug("wait for {port_num}".format(**locals()))
    t_start = time.time()
    sleeps = 1
    while time.time() - t_start < timeout:
        try:
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            try:
                s.connect((HOSTNAME, port_num))
                return True
            except (IOError, socket.error):
                time.sleep(sleeps)
        finally:
            s.close()
    return False


def mprocess(name, config_path, port=None, timeout=180):
    """start 'name' process with params from config_path.
    Args:
        name - process name or path
        config_path - path to file where should be stored configuration
        port - process's port
        timeout - specify how long, in seconds, a command can take before times out.
                  if timeout <=0 - doesn't wait for complete start process
    return tuple (pid, host) if process started, return (None, None) if not
    """

    logger.debug("mprocess({name}, {config_path}, {port}, {timeout})".format(**locals()))
    if not (config_path and isinstance(config_path, str) and os.path.exists(config_path)):
        raise OSError("can't find config file {config_path}".format(**locals()))

    cfg = read_config(config_path)
    cmd = [name, "--config", config_path]

    if cfg.get('port', None) is None or port:
        port = port or PortPool().port(check=True)
        cmd.extend(['--port', str(port)])
    host = "{HOSTNAME}:{port}".format(HOSTNAME=HOSTNAME, port=port)
    try:
        logger.debug("execute process: {cmd}".format(**locals()))
        proc = subprocess.Popen(cmd,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)

        if proc.poll() == 0:
            logger.debug("process is not alive")
            raise OSError
    except (OSError, TypeError) as err:
        logger.debug("exception while executing process: {err}".format(**locals()))
        raise OSError
    if timeout > 0 and wait_for(port, timeout):
        logger.debug("process '{name}' has started: pid={proc.pid}, host={host}".format(**locals()))
        return (proc.pid, host)
    elif timeout > 0:
        logger.debug("hasn't connected to pid={proc.pid} with host={host} during timeout {timeout} ".format(**locals()))
        logger.debug("terminate process with pid={proc.pid}".format(**locals()))
        kill_mprocess(proc.pid)
        proc_alive(proc.pid) and time.sleep(3)  # wait while process stoped
        message = "could not connect to process during {timeout} seconds".format(timeout=timeout)
        raise OSError(errno.ETIMEDOUT, message)
    return (proc.pid, host)


def kill_mprocess(pid, timeout=10):
    """kill process
    Args:
        pid - process pid
    """
    if pid and proc_alive(pid):
        os.kill(pid, 15)<commit_msg>kill mongod/mongos process with '-2' signal instead of '-15'; change timeout from 10 to 20 seconds<commit_after>#!/usr/bin/python
# coding=utf-8

import socket
import subprocess
import time
import os
import errno
from singleton import Singleton
import tempfile
import shutil
import stat
import json

import logging
logger = logging.getLogger(__name__)


HOSTNAME = 'localhost'


class PortPool(Singleton):

    __ports = set()
    __closed = set()
    __id = None

    def __init__(self, min_port=1025, max_port=2000, port_sequence=None):
        """
        Args:
            min_port - min port number  (ignoring if 'port_sequence' is not None)
            max_port - max port number  (ignoring if 'port_sequence' is not None)
            port_sequence - iterate sequence which contains numbers of ports
        """
        if not self.__id:  # singleton checker
            self.__id = id(self)
            self.__init_range(min_port, max_port, port_sequence)

    def __init_range(self, min_port=1025, max_port=2000, port_sequence=None):
        if port_sequence:
            self.__ports = set(port_sequence)
        else:
            self.__ports = set(xrange(min_port, max_port + 1))
        self.__closed = set()
        self.refresh()

    def __check_port(self, port):
        """check port status
        return True if port is free, False else
        """
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.bind((HOSTNAME, port))
            return True
        except socket.error:
            return False
        finally:
            s.close()

    def release_port(self, port):
        """release port"""
        if port in self.__closed:
            self.__closed.remove(port)
        self.__ports.add(port)

    def port(self, check=False):
        """return next opened port
        Args:
          check - check is port realy free
        """
        if not self.__ports:  # refresh ports if sequence is empty
            self.refresh()

        try:
            port = self.__ports.pop()
            if check:
                while not self.__check_port(port):
                    self.release_port(port)
                    port = self.__ports.pop()
        except (IndexError, KeyError):
            raise IndexError("Could not find a free port,\nclosed ports: {closed}".format(closed=self.__closed))
        self.__closed.add(port)
        return port

    def refresh(self, only_closed=False):
        """refresh ports status
        Args:
          only_closed - check status only for closed ports
        """
        if only_closed:
            opened = filter(self.__check_port, self.__closed)
            self.__closed = self.__closed.difference(opened)
            self.__ports = self.__ports.union(opened)
        else:
            ports = self.__closed.union(self.__ports)
            self.__ports = set(filter(self.__check_port, ports))
            self.__closed = ports.difference(self.__ports)

    def change_range(self, min_port=1025, max_port=2000, port_sequence=None):
        """change Pool port range"""
        self.__init_range(min_port, max_port, port_sequence)


def wait_for(port_num, timeout):
    """waits while process starts.
    Args:
        port_num    - port number
        timeout     - specify how long, in seconds, a command can take before times out.
    return True if process started, return False if not
    """
    logger.debug("wait for {port_num}".format(**locals()))
    t_start = time.time()
    sleeps = 1
    while time.time() - t_start < timeout:
        try:
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            try:
                s.connect((HOSTNAME, port_num))
                return True
            except (IOError, socket.error):
                time.sleep(sleeps)
        finally:
            s.close()
    return False


def mprocess(name, config_path, port=None, timeout=180):
    """start 'name' process with params from config_path.
    Args:
        name - process name or path
        config_path - path to file where should be stored configuration
        port - process's port
        timeout - specify how long, in seconds, a command can take before times out.
                  if timeout <=0 - doesn't wait for complete start process
    return tuple (pid, host) if process started, return (None, None) if not
    """

    logger.debug("mprocess({name}, {config_path}, {port}, {timeout})".format(**locals()))
    if not (config_path and isinstance(config_path, str) and os.path.exists(config_path)):
        raise OSError("can't find config file {config_path}".format(**locals()))

    cfg = read_config(config_path)
    cmd = [name, "--config", config_path]

    if cfg.get('port', None) is None or port:
        port = port or PortPool().port(check=True)
        cmd.extend(['--port', str(port)])
    host = "{HOSTNAME}:{port}".format(HOSTNAME=HOSTNAME, port=port)
    try:
        logger.debug("execute process: {cmd}".format(**locals()))
        proc = subprocess.Popen(cmd,
                                stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)

        if proc.poll() == 0:
            logger.debug("process is not alive")
            raise OSError
    except (OSError, TypeError) as err:
        logger.debug("exception while executing process: {err}".format(**locals()))
        raise OSError
    if timeout > 0 and wait_for(port, timeout):
        logger.debug("process '{name}' has started: pid={proc.pid}, host={host}".format(**locals()))
        return (proc.pid, host)
    elif timeout > 0:
        logger.debug("hasn't connected to pid={proc.pid} with host={host} during timeout {timeout} ".format(**locals()))
        logger.debug("terminate process with pid={proc.pid}".format(**locals()))
        kill_mprocess(proc.pid)
        proc_alive(proc.pid) and time.sleep(3)  # wait while process stoped
        message = "could not connect to process during {timeout} seconds".format(timeout=timeout)
        raise OSError(errno.ETIMEDOUT, message)
    return (proc.pid, host)


def kill_mprocess(pid, timeout=20):
    """kill process
    Args:
        pid - process pid
    """
    if pid and proc_alive(pid):
        os.kill(pid, 2)
{'fraction_non_alphanumeric': 0.10809938689899967, 'fraction_numerical': 0.006776379477250726, 'mean_word_length': 2.2874569079819675, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '176550', 'n_tokens_mistral': 3610, 'n_tokens_neox': 3440, 'n_words': 1004}