<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
from time import sleep
import paramiko
import socket
import re
from slugify import slugify as slugify_function
from django.contrib.auth.models import User
from django.http import HttpResponse
import json
import logging
import subprocess
import signal
import os
import traceback
import sys
from billiard import current_process
from django.utils.module_loading import import_by_path
from dns.resolver import Resolver
from dns.exception import DNSException


LOG = logging.getLogger(__name__)

# See http://docs.python.org/2/library/subprocess.html#popen-constructor if you
# have questions about this variable
DEFAULT_OUTPUT_BUFFER_SIZE = 16384
PROCESS_TIMEOUT = 4 * 60 * 60  # 4 horas


class AlarmException(Exception):
    pass


def alarm_handler(signum, frame):
    raise AlarmException


def slugify(string):
    return slugify_function(string, separator="_")


def make_db_random_password():
    return User.objects.make_random_password()


def as_json(f):
    def wrapper(request, *args, **kw):
        output = f(request, *args, **kw)
        if isinstance(output, HttpResponse):
            return output
        elif isinstance(output, basestring):
            return HttpResponse(output, content_type="text/plain")
        output = json.dumps(output, indent=4)
        return HttpResponse(output, content_type="application/json")
    return wrapper


def call_script(script_name, working_dir=None, split_lines=True, args=[],
                envs={}, shell=False, python_bin=None):

    args_copy = []
    for arg in args:
        if type(arg) == 'str' and arg.startswith("PASSWORD"):
            args_copy.append("xxx")
        else:
            args_copy.append(arg)

    if not working_dir:
        raise RuntimeError("Working dir is null")

    logging_cmdline = "%s %s" % (
        " ".join(["%s=%s" % (k, "xxx" if k.endswith("_PASSWORD") else v)
                  for (k, v) in envs.items()]),
        " ".join([script_name] + args_copy),
    )
    return_code = None
    output = []
    try:
        LOG.debug(
            'Running on path %s command: %s', working_dir, logging_cmdline)

        envs_with_path = {'PATH': os.getenv("PATH")}

        if envs:
            envs_with_path.update(envs)

        # For future, if scripts have lot of output can be better
        # create a temporary file for stdout. Scripts with lot
        # of output and subprocess.PIPE
        # can lock because this method not consume stdout without script finish
        # execute.

        LOG.info("Args: {}".format(args))

        if python_bin:
            exec_script = [python_bin, working_dir + script_name] + args
        else:
            exec_script = [working_dir + script_name] + args

        process = subprocess.Popen(
            exec_script,
            bufsize=DEFAULT_OUTPUT_BUFFER_SIZE,
            stdin=None,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,  # stderr and stdout are the same
            close_fds=True,
            cwd=working_dir,
            env=envs_with_path,
            universal_newlines=True,
            shell=shell)

        if not shell:
            signal.signal(signal.SIGALRM, alarm_handler)
            signal.alarm(PROCESS_TIMEOUT)
        try:
            process.wait()
            signal.alarm(0)  # Disable the alarm
        except AlarmException:
            LOG.error("Timeout %s exceeded for process id %s" %
                      (PROCESS_TIMEOUT, process.pid))
            process.kill()

        output = process.stdout.read()
        return_code = process.returncode

        LOG.debug("output: {} \n return_code: {}".format(output, return_code))

        if split_lines:
            return return_code, [s.strip() for s in output.splitlines()]
        else:
            return return_code, output
    except:
        # if any error happen, log cmdline to error
        LOG.error("Error running cmdline (exit code %s): %s",
                  return_code, logging_cmdline, exc_info=True)
        if not return_code:
            return_code = 1

        return return_code, output


def check_dns(dns_to_check, dns_server, retries=90, wait=10, ip_to_check=None):
    LOG.info("Cheking dns for {}...".format(dns_to_check))
    resolver = Resolver()
    resolver.nameservers = [dns_server]
    LOG.info("CHECK DNS: dns to check {}".format(dns_to_check))
    for attempt in range(0, retries):
        LOG.info("Cheking dns for {}... attempt number {}...".format(
            dns_to_check,
            str(attempt + 1)
        ))

        try:
            answer = resolver.query(dns_to_check)
        except DNSException:
            continue
        else:
            ips = map(str, answer)
            LOG.info("CHECK DNS: ips {}".format(ips))
        LOG.info("CHECK DNS: ip to check {}".format(ip_to_check))
        if (ip_to_check and ip_to_check in ips) or (not ip_to_check and ips):
            return True<commit_msg>Fix check_dns method.<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
from time import sleep
import paramiko
import socket
import re
from slugify import slugify as slugify_function
from django.contrib.auth.models import User
from django.http import HttpResponse
import json
import logging
import subprocess
import signal
import os
import traceback
import sys
from billiard import current_process
from django.utils.module_loading import import_by_path
from dns.resolver import Resolver
from dns.exception import DNSException


LOG = logging.getLogger(__name__)

# See http://docs.python.org/2/library/subprocess.html#popen-constructor if you
# have questions about this variable
DEFAULT_OUTPUT_BUFFER_SIZE = 16384
PROCESS_TIMEOUT = 4 * 60 * 60  # 4 horas


class AlarmException(Exception):
    pass


def alarm_handler(signum, frame):
    raise AlarmException


def slugify(string):
    return slugify_function(string, separator="_")


def make_db_random_password():
    return User.objects.make_random_password()


def as_json(f):
    def wrapper(request, *args, **kw):
        output = f(request, *args, **kw)
        if isinstance(output, HttpResponse):
            return output
        elif isinstance(output, basestring):
            return HttpResponse(output, content_type="text/plain")
        output = json.dumps(output, indent=4)
        return HttpResponse(output, content_type="application/json")
    return wrapper


def call_script(script_name, working_dir=None, split_lines=True, args=[],
                envs={}, shell=False, python_bin=None):

    args_copy = []
    for arg in args:
        if type(arg) == 'str' and arg.startswith("PASSWORD"):
            args_copy.append("xxx")
        else:
            args_copy.append(arg)

    if not working_dir:
        raise RuntimeError("Working dir is null")

    logging_cmdline = "%s %s" % (
        " ".join(["%s=%s" % (k, "xxx" if k.endswith("_PASSWORD") else v)
                  for (k, v) in envs.items()]),
        " ".join([script_name] + args_copy),
    )
    return_code = None
    output = []
    try:
        LOG.debug(
            'Running on path %s command: %s', working_dir, logging_cmdline)

        envs_with_path = {'PATH': os.getenv("PATH")}

        if envs:
            envs_with_path.update(envs)

        # For future, if scripts have lot of output can be better
        # create a temporary file for stdout. Scripts with lot
        # of output and subprocess.PIPE
        # can lock because this method not consume stdout without script finish
        # execute.

        LOG.info("Args: {}".format(args))

        if python_bin:
            exec_script = [python_bin, working_dir + script_name] + args
        else:
            exec_script = [working_dir + script_name] + args

        process = subprocess.Popen(
            exec_script,
            bufsize=DEFAULT_OUTPUT_BUFFER_SIZE,
            stdin=None,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,  # stderr and stdout are the same
            close_fds=True,
            cwd=working_dir,
            env=envs_with_path,
            universal_newlines=True,
            shell=shell)

        if not shell:
            signal.signal(signal.SIGALRM, alarm_handler)
            signal.alarm(PROCESS_TIMEOUT)
        try:
            process.wait()
            signal.alarm(0)  # Disable the alarm
        except AlarmException:
            LOG.error("Timeout %s exceeded for process id %s" %
                      (PROCESS_TIMEOUT, process.pid))
            process.kill()

        output = process.stdout.read()
        return_code = process.returncode

        LOG.debug("output: {} \n return_code: {}".format(output, return_code))

        if split_lines:
            return return_code, [s.strip() for s in output.splitlines()]
        else:
            return return_code, output
    except:
        # if any error happen, log cmdline to error
        LOG.error("Error running cmdline (exit code %s): %s",
                  return_code, logging_cmdline, exc_info=True)
        if not return_code:
            return_code = 1

        return return_code, output


def check_dns(dns_to_check, dns_server, retries=90, wait=10, ip_to_check=None):
    LOG.info("Cheking dns for {}...".format(dns_to_check))
    resolver = Resolver()
    resolver.nameservers = [dns_server]
    LOG.info("CHECK DNS: dns to check {}".format(dns_to_check))
    for attempt in range(0, retries):
        LOG.info("Cheking dns for {}... attempt number {}...".format(
            dns_to_check,
            str(attempt + 1)
        ))

        try:
            answer = resolver.query(dns_to_check)
        except DNSException:
            pass
        else:
            ips = map(str, answer)
            LOG.info("CHECK DNS: ips {}".format(ips))
            LOG.info("CHECK DNS: ip to check {}".format(ip_to_check))
            if (ip_to_check and ip_to_check in ips) or (not ip_to_check and ips):
                return True
{'fraction_non_alphanumeric': 0.09778708133971292, 'fraction_numerical': 0.0043859649122807015, 'mean_word_length': 2.4125850340136052, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2321103', 'n_tokens_mistral': 3037, 'n_tokens_neox': 2919, 'n_words': 801}