<commit_before>import os
import logging
import time
import threading
import Queue
import random
import taco.constants
import taco.globals
import uuid
from collections import defaultdict

if os.name=='nt':
  import ctypes
  def Get_Free_Space(path):
    free_bytes = ctypes.c_ulonglong(0)
    total = ctypes.c_ulonglong(0)
    ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(path), None, ctypes.pointer(total), ctypes.pointer(free_bytes))
    return (free_bytes.value,total.value)

elif os.name=='posix':
  def Get_Free_Space(path):
    try:
      data  = os.statvfs(path)
      free  = data.f_bavail * data.f_frsize
      total = data.f_blocks * data.f_frsize
      return (free,total)
    except:
      return (0,0)

def Is_Path_Under_A_Share(path):
  return_value = False
  if os.path.isdir(os.path.normpath(path)):
    with taco.globals.settings_lock:
      for [sharename,sharepath] in taco.globals.settings["Shares"]:
        dirpath = os.path.abspath(os.path.normcase(unicode(sharepath)))
        dirpath2 = os.path.abspath(os.path.normcase(unicode(path)))
        if os.path.commonprefix([dirpath,dirpath2]) == dirpath:
          return_value = True
          break
  #logging.debug(path + " -- " + str(return_value))
  return return_value

def Convert_Share_To_Path(share):
  return_val = ""
  with taco.globals.settings_lock:
    for [sharename,sharepath] in taco.globals.settings["Shares"]:
      if sharename==share:
        return_val = sharepath
        break
  #logging.debug(share + " -- " + str(return_val))
  return return_val

class TacoFilesystemManager(threading.Thread):
  def __init__(self):
    threading.Thread.__init__(self)

    self.stop = threading.Event()
    self.sleep = threading.Event()

    self.status_lock = threading.Lock()
    self.status = ""
    self.status_time = -1

    self.workers = []
    self.last_purge = time.time()

    self.listings_lock = threading.Lock()
    self.listings = {}

    self.listing_work_queue            = Queue.Queue()
    self.listing_results_queue         = Queue.Queue()
    self.chunk_requests_incoming_queue = Queue.Queue() 
    self.chunk_requests_outgoing_queue = Queue.Queue() 
    self.chunk_requests_ack_queue      = Queue.Queue() 

    self.results_to_return = []

    self.download_q_check_time = time.time()
    self.client_downloading = {}
    self.client_downloading_status = defaultdict(dict)
    self.client_downloading_pending_chunks = {}
    self.client_downloading_requested_chunks = {}
    self.client_downloading_filename = {} 
    self.files_w = {}
    self.files_r = {}
    self.files_r_last_access = {}
    self.files_w_last_access = {}
 
  def add_listing(self,thetime,sharedir,dirs,files):
    with self.listings_lock:
      self.listings[sharedir] = [thetime,dirs,files]

  def set_status(self,text,level=0):
    if   level==1: logging.info(text)
    elif level==0: logging.debug(text)
    elif level==2: logging.warning(text)
    elif level==3: logging.error(text)
    with self.status_lock:
      self.status = text
      self.status_time = time.time()

  def get_status(self):
    with self.status_lock:
      return (self.status,self.status_time)

  def run(self):
    self.set_status("Starting Up Filesystem Manager")
    for i in range(taco.constants.FILESYSTEM_WORKER_COUNT):
      self.workers.append(TacoFilesystemWorker(i))
    for i in self.workers:
      i.start()
    while not self.stop.is_set():
      self.set_status("FILESYS")
      self.sleep.wait(0.2)
      self.sleep.clear()
      if self.stop.is_set(): break
      
      #CHECK downloadq state
      if time.time() >= self.download_q_check_time:
        #self.set_status("Checking if the download q is in a good state")
        with taco.globals.settings_lock: local_copy_download_directory = os.path.normpath(taco.globals.settings["Download Location"])
        self.download_q_check_time = time.time() + taco.constants.DOWNLOAD_Q_CHECK_TIME

        #check for download q items
        with taco.globals.download_q_lock:
          for peer_uuid in taco.globals.download_q.keys():
            if len(taco.globals.download_q[peer_uuid]) == 0:
              self.client_downloading[peer_uuid] = 0
            else:
              (sharedir,filename,filesize,filemod) = taco.globals.download_q[peer_uuid][0]
              if not peer_uuid in self.client_downloading: self.client_downloading[peer_uuid] = 0
              if self.client_downloading[peer_uuid] != (sharedir,filename,filesize,filemod):
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.client_downloading[peer_uuid] = (sharedir,filename,filesize,filemod)
                self.client_downloading_pending_chunks[peer_uuid] = []
                self.client_downloading_requested_chunks[peer_uuid] = []
                if os.path.isdir(local_copy_download_directory):
                  filename_incomplete = os.path.normpath(local_copy_download_directory + u"/" + filename + taco.constants.FILESYSTEM_WORKINPROGRESS_SUFFIX)
                  filename_complete   = os.path.normpath(local_copy_download_directory + u"/" + filename)
                  if os.path.isfile(filename_complete):
                    pass
                  else:
                    try:
                      current_size = os.path.getsize(filename_incomplete)
                    except:
                      current_size = 0
                    if current_size != filesize:
                      self.set_status("Building in memory 'torrent'")<commit_msg>little insane on the last commit debug output<commit_after>import os
import logging
import time
import threading
import Queue
import random
import taco.constants
import taco.globals
import uuid
from collections import defaultdict

if os.name=='nt':
  import ctypes
  def Get_Free_Space(path):
    free_bytes = ctypes.c_ulonglong(0)
    total = ctypes.c_ulonglong(0)
    ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(path), None, ctypes.pointer(total), ctypes.pointer(free_bytes))
    return (free_bytes.value,total.value)

elif os.name=='posix':
  def Get_Free_Space(path):
    try:
      data  = os.statvfs(path)
      free  = data.f_bavail * data.f_frsize
      total = data.f_blocks * data.f_frsize
      return (free,total)
    except:
      return (0,0)

def Is_Path_Under_A_Share(path):
  return_value = False
  if os.path.isdir(os.path.normpath(path)):
    with taco.globals.settings_lock:
      for [sharename,sharepath] in taco.globals.settings["Shares"]:
        dirpath = os.path.abspath(os.path.normcase(unicode(sharepath)))
        dirpath2 = os.path.abspath(os.path.normcase(unicode(path)))
        if os.path.commonprefix([dirpath,dirpath2]) == dirpath:
          return_value = True
          break
  #logging.debug(path + " -- " + str(return_value))
  return return_value

def Convert_Share_To_Path(share):
  return_val = ""
  with taco.globals.settings_lock:
    for [sharename,sharepath] in taco.globals.settings["Shares"]:
      if sharename==share:
        return_val = sharepath
        break
  #logging.debug(share + " -- " + str(return_val))
  return return_val

class TacoFilesystemManager(threading.Thread):
  def __init__(self):
    threading.Thread.__init__(self)

    self.stop = threading.Event()
    self.sleep = threading.Event()

    self.status_lock = threading.Lock()
    self.status = ""
    self.status_time = -1

    self.workers = []
    self.last_purge = time.time()

    self.listings_lock = threading.Lock()
    self.listings = {}

    self.listing_work_queue            = Queue.Queue()
    self.listing_results_queue         = Queue.Queue()
    self.chunk_requests_incoming_queue = Queue.Queue() 
    self.chunk_requests_outgoing_queue = Queue.Queue() 
    self.chunk_requests_ack_queue      = Queue.Queue() 

    self.results_to_return = []

    self.download_q_check_time = time.time()
    self.client_downloading = {}
    self.client_downloading_status = defaultdict(dict)
    self.client_downloading_pending_chunks = {}
    self.client_downloading_requested_chunks = {}
    self.client_downloading_filename = {} 
    self.files_w = {}
    self.files_r = {}
    self.files_r_last_access = {}
    self.files_w_last_access = {}
 
  def add_listing(self,thetime,sharedir,dirs,files):
    with self.listings_lock:
      self.listings[sharedir] = [thetime,dirs,files]

  def set_status(self,text,level=0):
    if   level==1: logging.info(text)
    elif level==0: logging.debug(text)
    elif level==2: logging.warning(text)
    elif level==3: logging.error(text)
    with self.status_lock:
      self.status = text
      self.status_time = time.time()

  def get_status(self):
    with self.status_lock:
      return (self.status,self.status_time)

  def run(self):
    self.set_status("Starting Up Filesystem Manager")
    for i in range(taco.constants.FILESYSTEM_WORKER_COUNT):
      self.workers.append(TacoFilesystemWorker(i))
    for i in self.workers:
      i.start()
    while not self.stop.is_set():
      #self.set_status("FILESYS")
      self.sleep.wait(0.2)
      self.sleep.clear()
      if self.stop.is_set(): break
      
      #CHECK downloadq state
      if time.time() >= self.download_q_check_time:
        #self.set_status("Checking if the download q is in a good state")
        with taco.globals.settings_lock: local_copy_download_directory = os.path.normpath(taco.globals.settings["Download Location"])
        self.download_q_check_time = time.time() + taco.constants.DOWNLOAD_Q_CHECK_TIME

        #check for download q items
        with taco.globals.download_q_lock:
          for peer_uuid in taco.globals.download_q.keys():
            if len(taco.globals.download_q[peer_uuid]) == 0:
              self.client_downloading[peer_uuid] = 0
            else:
              (sharedir,filename,filesize,filemod) = taco.globals.download_q[peer_uuid][0]
              if not peer_uuid in self.client_downloading: self.client_downloading[peer_uuid] = 0
              if self.client_downloading[peer_uuid] != (sharedir,filename,filesize,filemod):
                self.set_status("File we should be downloading has changed:" + str((peer_uuid,sharedir,filename,filesize,filemod)))
                self.client_downloading[peer_uuid] = (sharedir,filename,filesize,filemod)
                self.client_downloading_pending_chunks[peer_uuid] = []
                self.client_downloading_requested_chunks[peer_uuid] = []
                if os.path.isdir(local_copy_download_directory):
                  filename_incomplete = os.path.normpath(local_copy_download_directory + u"/" + filename + taco.constants.FILESYSTEM_WORKINPROGRESS_SUFFIX)
                  filename_complete   = os.path.normpath(local_copy_download_directory + u"/" + filename)
                  if os.path.isfile(filename_complete):
                    pass
                  else:
                    try:
                      current_size = os.path.getsize(filename_incomplete)
                    except:
                      current_size = 0
                    if current_size != filesize:
                      self.set_status("Building in memory 'torrent'")
{'fraction_non_alphanumeric': 0.10906597753283055, 'fraction_numerical': 0.0022150730446706397, 'mean_word_length': 3.333180987202925, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 115, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '465870', 'n_tokens_mistral': 5752, 'n_tokens_neox': 5560, 'n_words': 1064}