<commit_before># Copyright (C) 2011, Quentin "mefyl" Hocquet
#
# This software is provided "as is" without warranty of any kind,
# either expressed or implied, including but not limited to the
# implied warranties of fitness for a particular purpose.
#
# See the LICENSE file for more information.

import greenlet
import sys
import threading
import time
import traceback
import types
import drake.debug
import drake.threadpool

class Frozen:
  pass

class CoroutineDone(Exception):
  pass

class CoroutineFrozen(Exception):
  pass

class Terminate(Exception):
  pass

class OrderedSet:

  def __init__(self, model = None):
    if model is None:
      self.__set = set()
      self.__list = []
    else:
      self.__set = set(model.__set)
      self.__list = list(model.__list)

  def add(self, v):
    self.__set.add(v)
    self.__list.append(v)

  def remove(self, v):
    self.__set.remove(v)

  def __iter__(self):
    remove = []
    i = 0
    seen = set()
    while i < len(self.__list):
      elt = self.__list[i]
      if elt in self.__set and elt not in seen:
        seen.add(elt)
        yield elt
        i += 1
      else:
        del self.__list[i]

  def __bool__(self):
    return bool(self.__set)

  def update(self, rhs):
    self.__set.update(rhs.__set)
    self.__list.extend(rhs.__list)

  def __repr__(self):
    return 'OrderedSet(%r)' % self.__list

class Scheduler:

  __instance = None
  __pool = None

  @classmethod
  def scheduler(self):
    return Scheduler.__instance

  def __init__(self):
    self.reset()
    Scheduler.__instance = self
    self.__lock = threading.Condition()
    self.__scheduled = []

  def reset(self):
    self.__coroutines = OrderedSet()
    self.__coroutines_frozen = set()
    self.__running = False
    self.__exception = None
    self.__traceback = None

  def running(self):
    return self.__running
<commit_msg>sched: Add OrderedDict.<commit_after># Copyright (C) 2011, Quentin "mefyl" Hocquet
#
# This software is provided "as is" without warranty of any kind,
# either expressed or implied, including but not limited to the
# implied warranties of fitness for a particular purpose.
#
# See the LICENSE file for more information.

import greenlet
import sys
import threading
import time
import traceback
import types
import drake.debug
import drake.threadpool

class Frozen:
  pass

class CoroutineDone(Exception):
  pass

class CoroutineFrozen(Exception):
  pass

class Terminate(Exception):
  pass

class OrderedSet:

  def __init__(self, model = None):
    if model is None:
      self.__set = set()
      self.__list = []
    else:
      self.__set = set(model.__set)
      self.__list = list(model.__list)

  def add(self, v):
    self.__set.add(v)
    self.__list.append(v)

  def remove(self, v):
    self.__set.remove(v)

  def __iter__(self):
    remove = []
    i = 0
    seen = set()
    while i < len(self.__list):
      elt = self.__list[i]
      if elt in self.__set and elt not in seen:
        seen.add(elt)
        yield elt
        i += 1
      else:
        del self.__list[i]

  def __bool__(self):
    return bool(self.__set)

  def update(self, rhs):
    self.__set.update(rhs.__set)
    self.__list.extend(rhs.__list)

  def __repr__(self):
    return 'OrderedSet(%r)' % self.__list


class OrderedDict:

  def __init__(self, model = None):
    if model is None:
      self.__dict = dict()
      self.__list = []
    else:
      self.__dict = dict(model.__dict)
      self.__list = list(model.__list)

  def __setitem__(self, k, v):
    self.__dict.__setitem__(k, v)
    self.__list.append((k, v))

  def __getitem__(self, k):
    return self.__dict.__getitem__(k)

  def __iter__(self):
    for k, v in self.items():
      yield k

  def __contains__(self, k):
    return self.__dict.__contains__(k)

  def items(self):
    remove = []
    i = len(self.__list) - 1
    seen = set()
    while i >= 0:
      key, value = self.__list[i]
      if key in self.__dict and key not in seen:
        seen.add(key)
        yield (key, value)
      else:
        del self.__list[i]
      i -= 1

  def __bool__(self):
    return bool(self.__set)

  def update(self, rhs):
    self.__set.update(rhs.__set)
    self.__list.extend(rhs.__list)

  def __repr__(self):
    return 'OrderedDict(%r)' % self.__list


class Scheduler:

  __instance = None
  __pool = None

  @classmethod
  def scheduler(self):
    return Scheduler.__instance

  def __init__(self):
    self.reset()
    Scheduler.__instance = self
    self.__lock = threading.Condition()
    self.__scheduled = []

  def reset(self):
    self.__coroutines = OrderedSet()
    self.__coroutines_frozen = set()
    self.__running = False
    self.__exception = None
    self.__traceback = None

  def running(self):
    return self.__running

{'fraction_non_alphanumeric': 0.10578131545873481, 'fraction_numerical': 0.003142019271051529, 'mean_word_length': 2.807814992025518, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '465613', 'n_tokens_mistral': 1584, 'n_tokens_neox': 1539, 'n_words': 445}