<commit_before>"""Run a simple web server which responds to requests to monitor a directory
tree and identify files over a certain size threshold. The directory is
scanned to pick up the current situation and then monitored for new / removed
files. By default, the page auto-refreshes every 60 seconds and shows the
top 50 files ordered by size.
"""
from __future__ import with_statement
import os, sys
import cgi
import datetime
import threading
import time
import Queue
import urllib
import urlparse
from wsgiref.simple_server import make_server
from wsgiref.util import shift_path_info

#~ import error_handler
from winsys import fs, misc

def get_files (path, size_threshold_mb, results, stop_event):
  """Intended to run inside a thread: scan the contents of
  a tree recursively, pushing every file which is at least
  as big as the size threshold onto a results queue. Stop
  if the stop_event is set.
  """
  size_threshold = size_threshold_mb * 1024 * 1024
  for f in fs.flat (path, ignore_access_errors=True):
    if stop_event.isSet (): break
    try:
      if f.size > size_threshold:
        results.put (f)
    except fs.exceptions.x_winsys:
      continue

def watch_files (path, size_threshold_mb, results, stop_event):
  """Intended to run inside a thread: monitor a directory tree
  for file changes. Convert the changed files to fs.File objects
  and push then onto a results queue. Stop if the stop_event is set.
  """
  size_threshold = size_threshold_mb * 1024 * 1024
  watcher = fs.watch (path, True)
  while True:
    if stop_event.isSet (): break
    action, old_file, new_file = watcher.next ()
    try:
      if old_file:
        if (not old_file) or (old_file and old_file.size > size_threshold):
          results.put (old_file)
      if new_file and new_file <> old_file:
        if new_file and new_file.size > size_threshold:
          results.put (new_file)
    except fs.x_no_such_file:
      pass
        
class Path (object):
  """Keep track of the files and changes under a particular
  path tree. No attempt is made to optimise the cases where
  one tree is contained within another.
  
  When the Path is started, it kicks of two threads: one to
  do a complete scan; the other to monitor changes. Both
  write back to the same results queue which is the basis
  for the set of files which will be sorted and presented
  on the webpage.
  
  For manageability, the files are pulled off the queue a
  chunk at a time (by default 1000).
  """
  
  def __init__ (self, path, size_threshold_mb, n_files_at_a_time):
    print "About to start Path for %s, %s" % (path, size_threshold_mb)
    self._path = path
    self._size_threshold_mb = size_threshold_mb
    self._n_files_at_a_time = n_files_at_a_time
    self._changes = Queue.Queue ()<commit_msg>Small robustness tweaks to directory monitor<commit_after>"""Run a simple web server which responds to requests to monitor a directory
tree and identify files over a certain size threshold. The directory is
scanned to pick up the current situation and then monitored for new / removed
files. By default, the page auto-refreshes every 60 seconds and shows the
top 50 files ordered by size.
"""
from __future__ import with_statement
import os, sys
import cgi
import datetime
import threading
import time
import traceback
import Queue
import urllib
import urlparse
from wsgiref.simple_server import make_server
from wsgiref.util import shift_path_info

#~ import error_handler
from winsys import core, fs, misc

def get_files (path, size_threshold_mb, results, stop_event):
  """Intended to run inside a thread: scan the contents of
  a tree recursively, pushing every file which is at least
  as big as the size threshold onto a results queue. Stop
  if the stop_event is set.
  """
  size_threshold = size_threshold_mb * 1024 * 1024
  for f in fs.flat (path, ignore_access_errors=True):
    if stop_event.isSet (): break
    try:
      if f.size > size_threshold:
        results.put (f)
    except fs.exceptions.x_winsys:
      continue

def watch_files (path, size_threshold_mb, results, stop_event):
  """Intended to run inside a thread: monitor a directory tree
  for file changes. Convert the changed files to fs.File objects
  and push then onto a results queue. Stop if the stop_event is set.
  """
  size_threshold = size_threshold_mb * 1024 * 1024
  watcher = fs.watch (path, True)
  while True:
    if stop_event.isSet (): break
    try:
      action, old_file, new_file = watcher.next ()
      if old_file:
        if (not old_file) or (old_file and old_file.size > size_threshold):
          results.put (old_file)
      if new_file and new_file <> old_file:
        if new_file and new_file.size > size_threshold:
          results.put (new_file)
    except fs.exceptions.x_winsys:
      pass
        
class Path (object):
  """Keep track of the files and changes under a particular
  path tree. No attempt is made to optimise the cases where
  one tree is contained within another.
  
  When the Path is started, it kicks of two threads: one to
  do a complete scan; the other to monitor changes. Both
  write back to the same results queue which is the basis
  for the set of files which will be sorted and presented
  on the webpage.
  
  For manageability, the files are pulled off the queue a
  chunk at a time (by default 1000).
  """
  
  def __init__ (self, path, size_threshold_mb, n_files_at_a_time):
    self._path = path
    self._size_threshold_mb = size_threshold_mb
    self._n_files_at_a_time = n_files_at_a_time
    self._changes = Queue.Queue ()
{'fraction_non_alphanumeric': 0.058093525179856116, 'fraction_numerical': 0.008633093525179856, 'mean_word_length': 3.5248169243287224, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4133202', 'n_tokens_mistral': 1686, 'n_tokens_neox': 1622, 'n_words': 714}