<commit_before>""" 
Author: <NAME>

Start Date: August 4, 2008

Description:
A software updater for the node manager.   The focus is to make it secure, 
robust, and simple (in that order).

Usage:  ./softwareupdater.py


Updated 1/23/2009 use servicelogger to log errors - Xuanhua (Sean)s Ren


"""



import sys
import os

import daemon

import repyhelper

# I need to make a cachedir for repyhelper...
if not os.path.exists('softwareupdater.repyhelpercache'):
  os.mkdir('softwareupdater.repyhelpercache')

# prepend this to my python path
sys.path = ['softwareupdater.repyhelpercache'] + sys.path
repyhelpercachedir = repyhelper.set_importcachedir('softwareupdater.repyhelpercache')


# this is being done so that the resources accounting doesn't interfere with logging
from repyportability import *

import urllib      # to retrieve updates
import random
import shutil
import socket   # we'll make it so we don't hang...
import tempfile
import traceback    # For exception logging if the servicelogger fails.
import runonce
import harshexit  # Used for portablekill
import portable_popen


# Import servicelogger to do logging
import servicelogger

# This gives us do_sleep
import misc

repyhelper.translate_and_import("signeddata.repy")
repyhelper.translate_and_import("sha.repy")

# Armon: The port that should be used to update our time using NTP
TIME_PORT = 51234
TIME_PORT_2 = 42345

softwareurl = "http://seattle.cs.washington.edu/couvb/updatesite/0.1/"

# embedded this because it seems easier to update it along with this file
# Every computer running Seattle will have this same public key, and will trust
# files signed by this key.
softwareupdatepublickey = {'e':82832270266597330072676409661763231354244983360850404742185516224735762244569727906889368190381098316859532462559839005559035695542121011189767114678746829532642015227757061325811995458461556243183965254348908097559976740460038862499279411461302045605434778587281242796895759723616079286531587479712074947611, 'n':319621204384190529645831372818389656614287850643207619926347176392517761801427609535545760457027184668587674034177692977122041230031985031724339016854308623931563908276376263003735701277100364224187045110833742749159504168429702766032353498688487937836208653017735915837622736764430341063733201947629404712911592942893299407289815035924224344585640141382996031910529762480483482480840200108190644743566141062967857181966489101744929170144756204101501136046697030104623523067263295405505628760205318871212056879946829241448986763757070565574197490565540710448548232847380638562809965308287901471553677400477022039092783245720343246522144179191881098268618863594564939975401607436281396130900640289859459360314214324155479461961863933551434423773320970748327521097336640702078449006530782991443968680573263568609595969967079764427272827202433035192418494908184888678872217792993640959292902948045622147093326912328933981365394795535990933982037636876825043938697362285277475661382202880481400699819441979130858152032120174957606455858082332914545153781708896942610940094268714863253465554125515897189179557899347310399568254877069082016414203023408461051519104976942275899720740657969311479534442473551582563833145735116565451064388421}

# Whether the nodemanager should be told not to daemonize when it is restarted.
# This is only to assist our automated tests.
run_nodemanager_in_foreground = False

# Whether the softwareupdater should run in the foreground or not. Default
# to yes.
run_softwareupdater_in_foreground = True


# This code is in its own function called later rather than directly in the
# global scope right here because otherwise we need to ensure that the
# safe_log* methods are defined above this code or else they would cause a
# NameError because they aren't defined yet.
def safe_servicelogger_init():
  """
  This initializes the servicelogger in a way that will not throw an exception
  even if servicelogger.init() does.
  """
  # initialize the servicelogger to log on the softwareupdater file
  try:
    servicelogger.init("softwareupdater")
  except:
    # We assume that if servicelogger.init() fails, then safe_log will
    # fall back on using 'print' because servicelogger.log() calls
    # will throw a ValueError.
    safe_log("Servicelogger.init() failed. Will try to use stdout. The exception from servicelogger.init() was:")
    safe_log_last_exception()



def safe_log(message):
  """
  Log a message in a way that cannot throw an exception. First try to log using
  the servicelogger, then just try to print the message.
  """
  try:
    servicelogger.log(message)
  except:
    try:
      print message
    except:
      # As the standard output streams aren't closed, it would seem that this
      # should never happen. If it does, though, what can we do to log the
      # message, other than directly write to a file?
      pass



def safe_log_last_exception():
  """
  Log the last exception in a way that cannot throw an exception. First try to
  log using the servicelogger, then just try to print the message.
  """
  try:
    # Get the last exception in case the servicelogger fails.
    exceptionstr = traceback.format_exc()
  except:
    pass
  
  try:
    servicelogger.log_last_exception()
  except:
    try:
      print exceptionstr
    except:
      # As the standard output streams aren't closed, it would seem that this
      # should never happen. If it does, though, what can we do to log the
      # message, other than directly write to a file?
      pass




def get_file_hash(filename):
  fileobj = file(filename, 'rb')
  filedata = fileobj.read()
  fileobj.close()

  return sha_hexhash(filedata)



# We'll use this to get a file.   If it doesn't download in a reasonable time, 
# we'll fail. (BUG: doesn't do this yet.   I use timeouts, but they don't
# always work)
def safe_download(serverpath, filename, destdir, filesize):
  # TODO: filesize isn't being used.
  # TODO: raise an RsyncError from here if the download fails instead of
  #       returning True/False.
  try:
    urllib.urlretrieve(serverpath+filename,destdir+filename)
    return True
  except Exception,e:
    safe_log_last_exception()
    safe_log('[safe_download] Failed to download ' + serverpath + filename)
    return False
 
#  # how much we have left to download
#  remainingsize = filesize
#
#  # get a file-like object for the URL...
#  safefo = urllib.urlopen(filename)
#
#  # always close after this...
#  try:
#    # download up to "filesize" worth of data...   
#    # BUG: We also should check to see if this is too slow...
#    mydata
#  
#  
#  finally:
#    try:
#      safefo.close()
#    except:
#      pass




################### Begin Rsync ################### 
# I'd love to be able to put this in a separate module or repyify it, but 
# I'd need urllib...

class RsyncError(Exception):
  pass




def do_rsync(serverpath, destdir, tempdir):
  """
  <Purpose>
    This method is the one that attempts to download the metainfo file from
    the given serverpath, then uses that to attempt to do an update.  This
    method makes sure that the downloaded metainfo file is valid and signed
    correctly before changing any files.  Once the metainfo file is determined
    to be valid, it will then compare file hashes between the ones in the new
    metainfo file and the hashes of the files currently on disk.  If there is
    a difference, the new file is downloaded and added to the updated list.  
    Once all the new files have been downloaded, if they all did so 
    successfully they are then copied over the old ones, replacing them and
    completing the update of the files.  Then a list of the files updated is
    returned.

  <Arguments>
    serverpath - The url for the update site that we will try to contact.  
                 This should be the url of the directory that contains all of
                 the files that are being pushed as an update.
    destdir - This is the directory where the new files will end up if 
              everything goes well.
    tempdir - This is the directory where the new files will be initially
              downloaded to before their hashes are checked.  This is not
              cleaned up after finishing.

  <Exceptions>
    Will throw various socket errors if there is trouble getting a file from
    the webserver.
    Will throw an RsyncError if the downloaded metainfo is malformed, or if
    the hash of a downloaded file does not match the one listed in the 
    metainfo file.

  <Side Effects>
    Files will be downloaded to tempdir, and they might be copied over to
    destdir if everything is successful.

  <Returns>
    A list of files that have been updated.  The list is empty if nothing is
    to be updated.
  """

  # get the metainfo (like a directory listing)
  safe_download(serverpath, "metainfo", tempdir, 1024*32)

  # read the file data into a string
  newmetafileobject = file(tempdir+"metainfo")
  newmetafiledata = newmetafileobject.read()
  newmetafileobject.close()

  # Incorrectly signed, we don't update...
  if not signeddata_issignedcorrectly(newmetafiledata, softwareupdatepublickey):
    safe_log("[do_rsync] New metainfo not signed correctly. Not updating.")
    return []

  try:
    # read in the old file
    oldmetafileobject = file(destdir+"metainfo")
    oldmetafiledata = oldmetafileobject.read()
    oldmetafileobject.close()
  except:
    # The old file has problems.   We'll use the new one since it's signed
    pass

  else:
    try:
      # Armon: Update our time via NTP, before we check the meta info
      time_updatetime(TIME_PORT)
    except:
      time_updatetime(TIME_PORT_2)
    
    # they're both good.   Let's compare them...
    shoulduse, reasons = signeddata_shouldtrust(oldmetafiledata,newmetafiledata,softwareupdatepublickey)

    if shoulduse == True:
      # great!   All is well...
      pass
    elif shoulduse == None:
      # hmm, a warning...   
      if len(reasons) == 1 and reasons[0] == 'Cannot check expiration':
        # we should probably allow this.  The node may be offline
        # JCS: if it's offline, how is it downloading the metainfo or even
        # getting past the time_updatetime() calls above?
        safe_log("[do_rsync] Warning: " + str(reasons))
      elif 'Timestamps match' in reasons:
        # Already seen this one...
        safe_log("[do_rsync] The metainfo indicates no update is needed: " + str(reasons))
        return []

    elif shoulduse == False:
      if 'Public keys do not match' in reasons:
        # If the only complaint is that the oldmetafiledata and newmetafiledata
        # are signed by different keys, this is actually OK at this point.  We
        # know that the newmetafiledata was correctly signed with the key held
        # within this softwareupdater, so this should actually only happen when
        # the oldmetafiledata has an out of date signature.  However, we do 
        # still need to make sure there weren't any other fatal errors that 
        # we should distrust. - Brent
        reasons.remove('Public keys do not match')
        for comment in reasons:
          if comment in signeddata_fatal_comments:
            # If there is a different fatal comment still there, still log it
            # and don't perform the update.
            safe_log("[do_rsync] Serious problem with signed metainfo: " + str(reasons))
            return []
            
          if comment in signeddata_warning_comments:
            # If there is a different warning comment still there, log the
            # warning.  We will take care of specific behavior shortly.
            safe_log("[do_rsync] " + str(comment))
            
        if 'Timestamps match' in reasons:
          # Act as we do above when timestamps match
          # Already seen this one...
          safe_log("[do_rsync] The metainfo indicates no update is needed: " + str(reasons))
          return []
      else:
        # Let's assume this is a bad thing and exit
        safe_log("[do_rsync] Something is wrong with the metainfo: " + str(reasons))
        return []

  # now it's time to update
  updatedfiles = [ "metainfo" ]

  for line in file(tempdir+"metainfo"):

    # skip comments
    if line[0] == '#':
      continue
 
    # skip signature parts
    if line[0] == '!':
      continue
 
    # skip blank lines
    if line.strip() == '':
      continue

    linelist = line.split()
    if len(linelist)!= 3:
      raise RsyncError, "Malformed metainfo line: '"+line+"'"

    filename, filehash, filesize = linelist
    
    shoulddownloadfile = False
    
    # if the file is missing or the hash is different, we want to download...
    if not os.path.exists(destdir+filename):
      shoulddownloadfile = True
      safe_log("[do_rsync] Downloading file " + filename + " because it doesn't already exist at " + destdir+filename)
    elif get_file_hash(destdir+filename) != filehash:
      shoulddownloadfile = True
      safe_log("[do_rsync] Downloading file " + filename + " because the hash changed.")
      
    if shoulddownloadfile:
      # get the file
      safe_download(serverpath, filename, tempdir, filesize)

      # The hash doesn't match what we expected it to be according to the signed metainfo.
      if get_file_hash(tempdir+filename) != filehash:
        safe_log("[do_rsync] Hash mismatch on file '"+filename+"':" + filehash +
            " vs " + get_file_hash(tempdir+filename))
        raise RsyncError, "Hash of file '"+filename+"' does not match information in metainfo file"

      # put this file in the list of files we need to update
      updatedfiles.append(filename)      


  # copy the files to the local dir...
  safe_log("[do_rsync] Updating files: " + str(updatedfiles))
  for filename in updatedfiles:
    shutil.copy(tempdir+filename, destdir+filename)
    
  # done!   We updated the files
  return updatedfiles
  
################### End Rsync ################### 





# MUTEX  (how I prevent multiple copies)
# a new copy writes an "OK" file. if it's written the previous can exit.   
# a previous copy writes a "stop" file. if it's written the new copy must exit
# each new program has its own stop and OK files (listed by mutex number)
# 
# first program (fresh_software_updater)
#              get softwareupdater.new mutex
#              clean all mutex files
#              once in main, take softwareupdater.old, release softwareupdater.new
#              exit if we ever lose softwareupdater.old
#
# old program (restart_software_updater)
#              find an unused mutex 
#              starts new with arg that is the mutex
#              wait for some time
#              if "OK" file exists, release softwareupdater.old, remove it and exit
#              else write "stop" file
#              continue normal operation
#
# new program: (software_updater_start)
#              take softwareupdater.new mutex
#              initializes
#              if "stop" file exists, then exit
#              write "OK" file
#              while "OK" file exists
#                 if "stop" file exists, then exit
#              take softwareupdater.old, release softwareupdater.new
#              start normal operation
#


def init():
  """
  <Purpose>
    This method is here to do a runthrough of trying to update.  The idea is
    that if there is going to be a fatal error, we want to die immediately
    rather than later.  This way, when a node is updating to a flawed version,
    the old one won't die until we know the new one is working.  Also goes
    through the magic explained in the comment block above.

  <Arguments>
    None

  <Exceptions>
    See fresh_software_updater and software_updater_start.

  <Side Effects>
    If we can't get the lock, we will exit.
    We will hold the softwareupdater.new lock while trying to start, but if
    all goes well, we will release that lock and aquire the 
    softwareupdater.old lock.

  <Returns>
    None
  """
  gotlock = runonce.getprocesslock("softwareupdater.new")
  if gotlock == True:
    # I got the lock.   All is well...
    pass
  else:
    # didn't get the lock, and we like to be real quiet, so lets 
    # exit quietly
    sys.exit(55)

  # Close stdin because we don't read from stdin at all. We leave stdout and stderr
  # open because we haven't done anything to make sure that output to those (such as
  # uncaught python exceptions) go somewhere else useful.
  sys.stdin.close()<commit_msg>#767 was fixed indirectly by r3218, but add a comment to remind future developers to keep this issue in mind.<commit_after>""" 
Author: <NAME>

Start Date: August 4, 2008

Description:
A software updater for the node manager.   The focus is to make it secure, 
robust, and simple (in that order).

Usage:  ./softwareupdater.py


Updated 1/23/2009 use servicelogger to log errors - Xuanhua (Sean)s Ren


"""



import sys
import os

import daemon

import repyhelper

# I need to make a cachedir for repyhelper...
if not os.path.exists('softwareupdater.repyhelpercache'):
  os.mkdir('softwareupdater.repyhelpercache')

# prepend this to my python path
sys.path = ['softwareupdater.repyhelpercache'] + sys.path
repyhelpercachedir = repyhelper.set_importcachedir('softwareupdater.repyhelpercache')


# this is being done so that the resources accounting doesn't interfere with logging
from repyportability import *

import urllib      # to retrieve updates
import random
import shutil
import socket   # we'll make it so we don't hang...
import tempfile
import traceback    # For exception logging if the servicelogger fails.
import runonce
import harshexit  # Used for portablekill
import portable_popen


# Import servicelogger to do logging
import servicelogger

# This gives us do_sleep
import misc

repyhelper.translate_and_import("signeddata.repy")
repyhelper.translate_and_import("sha.repy")

# Armon: The port that should be used to update our time using NTP
TIME_PORT = 51234
TIME_PORT_2 = 42345

softwareurl = "http://seattle.cs.washington.edu/couvb/updatesite/0.1/"

# embedded this because it seems easier to update it along with this file
# Every computer running Seattle will have this same public key, and will trust
# files signed by this key.
softwareupdatepublickey = {'e':82832270266597330072676409661763231354244983360850404742185516224735762244569727906889368190381098316859532462559839005559035695542121011189767114678746829532642015227757061325811995458461556243183965254348908097559976740460038862499279411461302045605434778587281242796895759723616079286531587479712074947611, 'n':319621204384190529645831372818389656614287850643207619926347176392517761801427609535545760457027184668587674034177692977122041230031985031724339016854308623931563908276376263003735701277100364224187045110833742749159504168429702766032353498688487937836208653017735915837622736764430341063733201947629404712911592942893299407289815035924224344585640141382996031910529762480483482480840200108190644743566141062967857181966489101744929170144756204101501136046697030104623523067263295405505628760205318871212056879946829241448986763757070565574197490565540710448548232847380638562809965308287901471553677400477022039092783245720343246522144179191881098268618863594564939975401607436281396130900640289859459360314214324155479461961863933551434423773320970748327521097336640702078449006530782991443968680573263568609595969967079764427272827202433035192418494908184888678872217792993640959292902948045622147093326912328933981365394795535990933982037636876825043938697362285277475661382202880481400699819441979130858152032120174957606455858082332914545153781708896942610940094268714863253465554125515897189179557899347310399568254877069082016414203023408461051519104976942275899720740657969311479534442473551582563833145735116565451064388421}

# Whether the nodemanager should be told not to daemonize when it is restarted.
# This is only to assist our automated tests.
run_nodemanager_in_foreground = False

# Whether the softwareupdater should run in the foreground or not. Default
# to yes.
run_softwareupdater_in_foreground = True


# This code is in its own function called later rather than directly in the
# global scope right here because otherwise we need to ensure that the
# safe_log* methods are defined above this code or else they would cause a
# NameError because they aren't defined yet.
def safe_servicelogger_init():
  """
  This initializes the servicelogger in a way that will not throw an exception
  even if servicelogger.init() does.
  """
  # initialize the servicelogger to log on the softwareupdater file
  try:
    servicelogger.init("softwareupdater")
  except:
    # We assume that if servicelogger.init() fails, then safe_log will
    # fall back on using 'print' because servicelogger.log() calls
    # will throw a ValueError.
    safe_log("Servicelogger.init() failed. Will try to use stdout. The exception from servicelogger.init() was:")
    safe_log_last_exception()



def safe_log(message):
  """
  Log a message in a way that cannot throw an exception. First try to log using
  the servicelogger, then just try to print the message.
  """
  try:
    servicelogger.log(message)
  except:
    try:
      print message
    except:
      # As the standard output streams aren't closed, it would seem that this
      # should never happen. If it does, though, what can we do to log the
      # message, other than directly write to a file?
      pass



def safe_log_last_exception():
  """
  Log the last exception in a way that cannot throw an exception. First try to
  log using the servicelogger, then just try to print the message.
  """
  try:
    # Get the last exception in case the servicelogger fails.
    exceptionstr = traceback.format_exc()
  except:
    pass
  
  try:
    servicelogger.log_last_exception()
  except:
    try:
      print exceptionstr
    except:
      # As the standard output streams aren't closed, it would seem that this
      # should never happen. If it does, though, what can we do to log the
      # message, other than directly write to a file?
      pass




def get_file_hash(filename):
  fileobj = file(filename, 'rb')
  filedata = fileobj.read()
  fileobj.close()

  return sha_hexhash(filedata)



# We'll use this to get a file.   If it doesn't download in a reasonable time, 
# we'll fail. (BUG: doesn't do this yet.   I use timeouts, but they don't
# always work)
def safe_download(serverpath, filename, destdir, filesize):
  # TODO: filesize isn't being used.
  # TODO: raise an RsyncError from here if the download fails instead of
  #       returning True/False.
  try:
    urllib.urlretrieve(serverpath+filename,destdir+filename)
    return True
  except Exception,e:
    safe_log_last_exception()
    safe_log('[safe_download] Failed to download ' + serverpath + filename)
    return False
 
#  # how much we have left to download
#  remainingsize = filesize
#
#  # get a file-like object for the URL...
#  safefo = urllib.urlopen(filename)
#
#  # always close after this...
#  try:
#    # download up to "filesize" worth of data...   
#    # BUG: We also should check to see if this is too slow...
#    mydata
#  
#  
#  finally:
#    try:
#      safefo.close()
#    except:
#      pass




################### Begin Rsync ################### 
# I'd love to be able to put this in a separate module or repyify it, but 
# I'd need urllib...

class RsyncError(Exception):
  pass




def do_rsync(serverpath, destdir, tempdir):
  """
  <Purpose>
    This method is the one that attempts to download the metainfo file from
    the given serverpath, then uses that to attempt to do an update.  This
    method makes sure that the downloaded metainfo file is valid and signed
    correctly before changing any files.  Once the metainfo file is determined
    to be valid, it will then compare file hashes between the ones in the new
    metainfo file and the hashes of the files currently on disk.  If there is
    a difference, the new file is downloaded and added to the updated list.  
    Once all the new files have been downloaded, if they all did so 
    successfully they are then copied over the old ones, replacing them and
    completing the update of the files.  Then a list of the files updated is
    returned.

  <Arguments>
    serverpath - The url for the update site that we will try to contact.  
                 This should be the url of the directory that contains all of
                 the files that are being pushed as an update.
    destdir - This is the directory where the new files will end up if 
              everything goes well.
    tempdir - This is the directory where the new files will be initially
              downloaded to before their hashes are checked.  This is not
              cleaned up after finishing.

  <Exceptions>
    Will throw various socket errors if there is trouble getting a file from
    the webserver.
    Will throw an RsyncError if the downloaded metainfo is malformed, or if
    the hash of a downloaded file does not match the one listed in the 
    metainfo file.

  <Side Effects>
    Files will be downloaded to tempdir, and they might be copied over to
    destdir if everything is successful.

  <Returns>
    A list of files that have been updated.  The list is empty if nothing is
    to be updated.
  """

  # get the metainfo (like a directory listing)
  safe_download(serverpath, "metainfo", tempdir, 1024*32)

  # read the file data into a string
  newmetafileobject = file(tempdir+"metainfo")
  newmetafiledata = newmetafileobject.read()
  newmetafileobject.close()

  # Incorrectly signed, we don't update...
  if not signeddata_issignedcorrectly(newmetafiledata, softwareupdatepublickey):
    safe_log("[do_rsync] New metainfo not signed correctly. Not updating.")
    return []

  try:
    # read in the old file
    oldmetafileobject = file(destdir+"metainfo")
    oldmetafiledata = oldmetafileobject.read()
    oldmetafileobject.close()
  except:
    # The old file has problems.   We'll use the new one since it's signed
    pass

  else:
    try:
      # Armon: Update our time via NTP, before we check the meta info
      time_updatetime(TIME_PORT)
    except:
      time_updatetime(TIME_PORT_2)
    
    # they're both good.   Let's compare them...
    shoulduse, reasons = signeddata_shouldtrust(oldmetafiledata,newmetafiledata,softwareupdatepublickey)

    if shoulduse == True:
      # great!   All is well...
      pass
    elif shoulduse == None:
      # hmm, a warning...   
      if len(reasons) == 1 and reasons[0] == 'Cannot check expiration':
        # we should probably allow this.  The node may be offline
        # JCS: if it's offline, how is it downloading the metainfo or even
        # getting past the time_updatetime() calls above?
        safe_log("[do_rsync] Warning: " + str(reasons))
      elif 'Timestamps match' in reasons:
        # Already seen this one...
        safe_log("[do_rsync] The metainfo indicates no update is needed: " + str(reasons))
        return []

    elif shoulduse == False:
      if 'Public keys do not match' in reasons:
        # If the only complaint is that the oldmetafiledata and newmetafiledata
        # are signed by different keys, this is actually OK at this point.  We
        # know that the newmetafiledata was correctly signed with the key held
        # within this softwareupdater, so this should actually only happen when
        # the oldmetafiledata has an out of date signature.  However, we do 
        # still need to make sure there weren't any other fatal errors that 
        # we should distrust. - Brent
        reasons.remove('Public keys do not match')
        for comment in reasons:
          if comment in signeddata_fatal_comments:
            # If there is a different fatal comment still there, still log it
            # and don't perform the update.
            safe_log("[do_rsync] Serious problem with signed metainfo: " + str(reasons))
            return []
            
          if comment in signeddata_warning_comments:
            # If there is a different warning comment still there, log the
            # warning.  We will take care of specific behavior shortly.
            safe_log("[do_rsync] " + str(comment))
            
        if 'Timestamps match' in reasons:
          # Act as we do above when timestamps match
          # Already seen this one...
          safe_log("[do_rsync] The metainfo indicates no update is needed: " + str(reasons))
          return []
      else:
        # Let's assume this is a bad thing and exit
        safe_log("[do_rsync] Something is wrong with the metainfo: " + str(reasons))
        return []

  # now it's time to update
  updatedfiles = [ "metainfo" ]

  for line in file(tempdir+"metainfo"):

    # skip comments
    if line[0] == '#':
      continue
 
    # skip signature parts
    if line[0] == '!':
      continue
 
    # skip blank lines
    if line.strip() == '':
      continue

    linelist = line.split()
    if len(linelist)!= 3:
      raise RsyncError, "Malformed metainfo line: '"+line+"'"

    filename, filehash, filesize = linelist
    
    shoulddownloadfile = False
    
    # if the file is missing or the hash is different, we want to download...
    if not os.path.exists(destdir+filename):
      shoulddownloadfile = True
      safe_log("[do_rsync] Downloading file " + filename + " because it doesn't already exist at " + destdir+filename)
    elif get_file_hash(destdir+filename) != filehash:
      shoulddownloadfile = True
      safe_log("[do_rsync] Downloading file " + filename + " because the hash changed.")
      
    if shoulddownloadfile:
      # get the file
      safe_download(serverpath, filename, tempdir, filesize)

      # The hash doesn't match what we expected it to be according to the signed metainfo.
      if get_file_hash(tempdir+filename) != filehash:
        safe_log("[do_rsync] Hash mismatch on file '"+filename+"':" + filehash +
            " vs " + get_file_hash(tempdir+filename))
        raise RsyncError, "Hash of file '"+filename+"' does not match information in metainfo file"

      # put this file in the list of files we need to update
      updatedfiles.append(filename)      


  # copy the files to the local dir...
  safe_log("[do_rsync] Updating files: " + str(updatedfiles))
  for filename in updatedfiles:
    shutil.copy(tempdir+filename, destdir+filename)
    
  # done!   We updated the files
  return updatedfiles
  
################### End Rsync ################### 





# MUTEX  (how I prevent multiple copies)
# a new copy writes an "OK" file. if it's written the previous can exit.   
# a previous copy writes a "stop" file. if it's written the new copy must exit
# each new program has its own stop and OK files (listed by mutex number)
# 
# first program (fresh_software_updater)
#              get softwareupdater.new mutex
#              clean all mutex files
#              once in main, take softwareupdater.old, release softwareupdater.new
#              exit if we ever lose softwareupdater.old
#
# old program (restart_software_updater)
#              find an unused mutex 
#              starts new with arg that is the mutex
#              wait for some time
#              if "OK" file exists, release softwareupdater.old, remove it and exit
#              else write "stop" file
#              continue normal operation
#
# new program: (software_updater_start)
#              take softwareupdater.new mutex
#              initializes
#              if "stop" file exists, then exit
#              write "OK" file
#              while "OK" file exists
#                 if "stop" file exists, then exit
#              take softwareupdater.old, release softwareupdater.new
#              start normal operation
#


def init():
  """
  <Purpose>
    This method is here to do a runthrough of trying to update.  The idea is
    that if there is going to be a fatal error, we want to die immediately
    rather than later.  This way, when a node is updating to a flawed version,
    the old one won't die until we know the new one is working.  Also goes
    through the magic explained in the comment block above.

  <Arguments>
    None

  <Exceptions>
    See fresh_software_updater and software_updater_start.

  <Side Effects>
    If we can't get the lock, we will exit.
    We will hold the softwareupdater.new lock while trying to start, but if
    all goes well, we will release that lock and aquire the 
    softwareupdater.old lock.

  <Returns>
    None
  """
  # Note: be careful about making this init() method take too long. If it takes
  # longer to complete than the amount of time that restart_software_updater()
  # waits, then the new software updater will never be left running. Keep in
  # mind very slow systems and adjust the wait time in restart_software_updater()
  # if needed.
  
  gotlock = runonce.getprocesslock("softwareupdater.new")
  if gotlock == True:
    # I got the lock.   All is well...
    pass
  else:
    # didn't get the lock, and we like to be real quiet, so lets 
    # exit quietly
    sys.exit(55)

  # Close stdin because we don't read from stdin at all. We leave stdout and stderr
  # open because we haven't done anything to make sure that output to those (such as
  # uncaught python exceptions) go somewhere else useful.
  sys.stdin.close()
{'fraction_non_alphanumeric': 0.06576895393474089, 'fraction_numerical': 0.09497960652591171, 'mean_word_length': 3.2859897172236505, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 50, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4719988', 'n_tokens_mistral': 11852, 'n_tokens_neox': 9789, 'n_words': 3674}