<commit_before>import re
import pwd
import zipfile
from io import BytesIO
from tempfile import mkdtemp
from datetime import timedelta
from os.path import join as pjoin

from concurrent.futures import ThreadPoolExecutor

from docker.errors import APIError

from dockerspawner import DockerSpawner
from textwrap import dedent
from traitlets import (
    Integer,
    Unicode,
)
from tornado import gen
from tornado.ioloop import IOLoop

from escapism import escape

import git

from .image_handler import ImageHandler

import ssl

import json

ssl._create_default_https_context = ssl._create_unverified_context

class CustomDockerSpawner(DockerSpawner):
    def __init__(self, **kwargs):
        self._user_log = []
        self._is_failed = False
        self._is_building = False
        self._image_handler = ImageHandler()
        self._cur_waiter = None
        super(CustomDockerSpawner, self).__init__(**kwargs)


    # We override the executor here to increase the number of threads
    @property
    def executor(self):
        """single global executor"""
        cls = self.__class__
        if cls._executor is None:
            cls._executor = ThreadPoolExecutor(20)
        return cls._executor


    def _docker(self, method, *args, **kwargs):
        """wrapper for calling docker methods

        to be passed to ThreadPoolExecutor
        """
        # methods that return a generator object return instantly
        # before the work they were meant to do is complete
        generator_methods = ('build',)
        m = getattr(self.client, method)

        if method in generator_methods:
            def lister(mm):
                ret = []
                for l in mm:
                    for lj in l.decode().split('\r\n'):
                        if len(lj) > 0:
                            ret.append(lj)
                            j = json.loads(lj)
                            # include only high-level docker's log
                            if 'stream' in j and not j['stream'].startswith(' --->'):
                            # self._add_to_log(l['stream'], 2)
                                self._cur_waiter.add_to_log(j['stream'], 2)
                return ret
            return lister(m(*args, **kwargs))
        else:
            return m(*args, **kwargs)

    _git_executor = None
    @property
    def git_executor(self):
        """single global git executor"""
        cls = self.__class__
        if cls._git_executor is None:
            cls._git_executor = ThreadPoolExecutor(1)<commit_msg>catch json errors<commit_after>import re
import pwd
import zipfile
from io import BytesIO
from tempfile import mkdtemp
from datetime import timedelta
from os.path import join as pjoin

from concurrent.futures import ThreadPoolExecutor

from docker.errors import APIError

from dockerspawner import DockerSpawner
from textwrap import dedent
from traitlets import (
    Integer,
    Unicode,
)
from tornado import gen
from tornado.ioloop import IOLoop

from escapism import escape

import git

from .image_handler import ImageHandler

import ssl

import json

ssl._create_default_https_context = ssl._create_unverified_context

class CustomDockerSpawner(DockerSpawner):
    def __init__(self, **kwargs):
        self._user_log = []
        self._is_failed = False
        self._is_building = False
        self._image_handler = ImageHandler()
        self._cur_waiter = None
        super(CustomDockerSpawner, self).__init__(**kwargs)


    # We override the executor here to increase the number of threads
    @property
    def executor(self):
        """single global executor"""
        cls = self.__class__
        if cls._executor is None:
            cls._executor = ThreadPoolExecutor(20)
        return cls._executor


    def _docker(self, method, *args, **kwargs):
        """wrapper for calling docker methods

        to be passed to ThreadPoolExecutor
        """
        # methods that return a generator object return instantly
        # before the work they were meant to do is complete
        generator_methods = ('build',)
        m = getattr(self.client, method)

        if method in generator_methods:
            def lister(mm):
                ret = []
                for l in mm:
                    for lj in l.decode().split('\r\n'):
                        if len(lj) > 0:
                            ret.append(lj)
                            try:
                                j = json.loads(lj)
                            except json.JSONDecodeError as e:
                                self.log.warn("Error decoding string to json: %s" % lj)
                            else:
                                if 'stream' in j and not j['stream'].startswith(' --->'):
                                # self._add_to_log(l['stream'], 2)
                                    self._cur_waiter.add_to_log(j['stream'], 2)
                return ret
            return lister(m(*args, **kwargs))
        else:
            return m(*args, **kwargs)

    _git_executor = None
    @property
    def git_executor(self):
        """single global git executor"""
        cls = self.__class__
        if cls._git_executor is None:
            cls._git_executor = ThreadPoolExecutor(1)
{'fraction_non_alphanumeric': 0.07642338555598013, 'fraction_numerical': 0.0022927015666794038, 'mean_word_length': 1.9426644182124788, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2314370', 'n_tokens_mistral': 1436, 'n_tokens_neox': 1393, 'n_words': 435}