<commit_before>import os
import time
import logging
import Queue
import fnmatch
from collections import OrderedDict

from substance.monads import *
from substance.logs import *
from substance.shell import Shell
from substance.exceptions import (SubstanceError)
from substance.utils import pathComponents

from subwatch.watcher import (LocalWatchAgent, RemoteWatchAgent)
from subwatch.events import WatchEventHandler
from subwatch.events import (WatchEvent, EVENT_TYPE_MODIFIED)

from tornado import ioloop

logging.getLogger("watchdog").setLevel(logging.CRITICAL)

class SubstanceSyncher(object):

  UP = ">>"
  DOWN = "<<"
  PARTIAL_DIR = '.~subsync~'

  def __init__(self, engine, keyfile):
    self.keyfile = keyfile
    self.engine = engine
    self.localAgent = LocalWatchAgent(self.processLocalEvent)
    self.remoteAgent = RemoteWatchAgent(self.processRemoteEvent)
    self.unison = {}
    self.synching = {}
    self.toSync = {self.UP:{}, self.DOWN:{}}
    self.syncPeriod = 0.3
    self.schedule = {self.UP: False, self.DOWN: False}
    self.excludes = []
    
  def getFolders(self):
    return self.engine.getEngineFolders()

  def getFolderFromPath(self, path, direction=None):
    direction = self.UP if not direction else direction

    for folder in self.getFolders():
      if direction is self.UP and folder.hostPath == path:
        return OK(folder)
      elif direction is self.DOWN and folder.guestPath == path:
        return OK(folder)
    return Fail(NameError("%s is not a folder." % path))
 
  def start(self):
    op = self.initialSync()
    if op.isFail():
      return op
    
    folders = self.getFolders()
    for folder in folders:
      self.localAgent.startWatching(folder.hostPath)

    self.remoteAgent.connect("ws://%s:%s" % (self.engine.getPublicIP(), 1500))
    self.remoteAgent.startWatching(folder.guestPath)

    try:
      self.remoteAgent.start()
      ioloop.IOLoop.current().start()
      return OK(None)
    except KeyboardInterrupt:
      return self.stop()
    except Exception as err:
      logging.error(err)
      return Fail(err)
    finally:
      return self.stop()

    return OK(None)

  def stop(self):<commit_msg>Fail if no rsync folders found<commit_after>import os
import time
import logging
import Queue
import fnmatch
from collections import OrderedDict

from substance.monads import *
from substance.logs import *
from substance.shell import Shell
from substance.exceptions import (SubstanceError)
from substance.utils import pathComponents

from subwatch.watcher import (LocalWatchAgent, RemoteWatchAgent)
from subwatch.events import WatchEventHandler
from subwatch.events import (WatchEvent, EVENT_TYPE_MODIFIED)

from tornado import ioloop

logging.getLogger("watchdog").setLevel(logging.CRITICAL)

class SubstanceSyncher(object):

  UP = ">>"
  DOWN = "<<"
  PARTIAL_DIR = '.~subsync~'

  def __init__(self, engine, keyfile):
    self.keyfile = keyfile
    self.engine = engine
    self.localAgent = LocalWatchAgent(self.processLocalEvent)
    self.remoteAgent = RemoteWatchAgent(self.processRemoteEvent)
    self.unison = {}
    self.synching = {}
    self.toSync = {self.UP:{}, self.DOWN:{}}
    self.syncPeriod = 0.3
    self.schedule = {self.UP: False, self.DOWN: False}
    self.excludes = []
    
  def getFolders(self):
    return reduce(lambda acc, x: acc + [x] if x.mode == 'rsync' else acc, self.engine.getEngineFolders(), [])

  def getFolderFromPath(self, path, direction=None):
    direction = self.UP if not direction else direction

    for folder in self.getFolders():
      if direction is self.UP and folder.hostPath == path:
        return OK(folder)
      elif direction is self.DOWN and folder.guestPath == path:
        return OK(folder)
    return Fail(NameError("%s is not a folder." % path))
 
  def start(self):
    op = self.initialSync()
    if op.isFail():
      return op
    
    folders = self.getFolders()

    if len(folders) == 0:
      return Fail(SubstanceError("No rsync folders definedfor synchronization in engine configuration."))

    for folder in folders:
      self.localAgent.startWatching(folder.hostPath)
      self.remoteAgent.startWatching(folder.guestPath)

    self.remoteAgent.connect("ws://%s:%s" % (self.engine.getPublicIP(), 1500))

    try:
      self.remoteAgent.start()
      ioloop.IOLoop.current().start()
      return OK(None)
    except KeyboardInterrupt:
      return self.stop()
    except Exception as err:
      logging.error(err)
      return Fail(err)
    finally:
      return self.stop()

    return OK(None)

  def stop(self):
{'fraction_non_alphanumeric': 0.1051018173855923, 'fraction_numerical': 0.0028465075541931244, 'mean_word_length': 3.549800796812749, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '879241', 'n_tokens_mistral': 1419, 'n_tokens_neox': 1377, 'n_words': 362}