parameters: (run, options)
docstring: Generate a folder containing handy debugging files including output thumbnails
docstring_summary: Generate a folder containing handy debugging files including output thumbnails
docstring_tokens: Generate a folder containing handy debugging files including output thumbnails
function: def generateFlightSummary(run, options):
      '''Generate a folder containing handy debugging files including output thumbnails'''
      
      # Copy logs to the output folder
      print 'Copying log files...'
      badImageFolder  = os.path.join(options.outputFolder, 'badImages')
      runFolder       = run.getFolder()
      procFolder      = run.getProcessFolder()
      navCameraFolder = run.getNavCameraFolder()
      os.system('mkdir -p ' + options.outputFolder)
      os.system('mkdir -p ' + badImageFolder)
      
      packedErrorLog = os.path.join(runFolder, 'packedErrors.log')
      if os.path.exists(packedErrorLog):
          try:
              shutil.copy(packedErrorLog, options.outputFolder)
          except Exception as e:
              # In case it complains about copying a file onto itself
              print("Warning: " + str(e))
              
      if not options.skipKml:
          # Copy the input camera kml file
          camerasInKmlPath = os.path.join(procFolder, 'cameras_in.kml')
          try:
              shutil.copy(camerasInKmlPath, options.outputFolder)
          except Exception as e:
              # In case it complains about copying a file onto itself
              print("Warning: " + str(e))
  
          # Copy the input camera kml file
          navCamerasKmlPath = os.path.join(navCameraFolder, 'nav_cameras.kml')
          try:
              shutil.copy(navCamerasKmlPath, options.outputFolder)
          except Exception as e:
              # In case it complains about copying a file onto itself
              print("Warning: " + str(e))
          
          # Create a merged version of all the bundle adjusted camera files
          # - The tool currently includes cameras more than once if they appear
          #   in multiple bundles.
          print 'Merging output camera kml files...'
          cmd = "find "+procFolder+" -name cameras_out.kml"
          p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, shell=False,
                               universal_newlines=True)
          textOutput, err = p.communicate()
          camKmlFiles = textOutput.replace('\n', ' ')
  
          # Write the list of files to process to disk. Otherwise, if we just
          # pass the full list on the command line, it may be too big
          # and the call will fail.
          kmlList = os.path.join(options.outputFolder, 'kml_list.txt')
          print("Writing: " + kmlList)
          with open(kmlList, 'w') as f:
              for filename in sorted(camKmlFiles.split()):
                  filename = filename.strip()
                  if filename == "":
                      continue
                  f.write(filename + "\n")
                  
          outputKml = os.path.join(options.outputFolder, 'cameras_out.kml')
          scriptPath = icebridge_common.fullPath('merge_orbitviz.py')
          cmd = 'python ' + scriptPath + ' ' + outputKml + ' -list ' + kmlList
          print(cmd)
          os.system(cmd)
          
          # Generate lidar kml files
          print("Generating lidar kml files")
          LIDAR_POINT_SKIP = 1527
          lidarFiles = run.getLidarList(prependFolder=True)
          lidarOutputFolder = os.path.join(options.outputFolder, 'lidar')
          os.system('mkdir -p ' + lidarOutputFolder)
          for f in lidarFiles:
              inputPath = os.path.splitext(f)[0] + '.csv'
              outputPath = os.path.join(lidarOutputFolder, os.path.basename(f)+'.kml')
              args = [inputPath, outputPath, '--skip', str(LIDAR_POINT_SKIP), '--color', 'red']
              if not os.path.exists(outputPath): # Don't recreate these files
                  print("Generating: " + outputPath) # This can be very slow, so print what is going on
                  try:
                      lvis2kml.main(args)
                  except Exception as e:
                      # Do not let this make our life miserable
                      print("Problem: " + str(e))
         
      # Collect per-batch information
      batchInfoPath   = os.path.join(options.outputFolder, 'batchInfoSummary.csv')
      failedBatchPath = os.path.join(options.outputFolder, 'failedBatchList.csv')
      print("Writing statistics to: " + batchInfoPath)
      print("Writing failures to: " + failedBatchPath)
      batchInfoLog = open(batchInfoPath, 'w')
      failureLog   = open(failedBatchPath, 'w')
  
      # Write the header for the batch log file
      batchInfoLog.write('# startFrame, stopFrame, centerLon, centerLat, meanAlt, ' +
                         'meanLidarDiff, meanInterDiff, meanFireDiff, meanFireLidarDiff, ' +
                         'percentValid, meanBlendDiff, meanBlendDiffInFireballFootprint, ' + \
                         'corrSearchWid, corrMem(GB), corrElapsedTime(minutes)\n')
      failureLog.write('# startFrame, stopFrame, errorCode, errorText\n')
  
      demList = run.getOutputFileList(icebridge_common.blendFileName())
      for (dem, frames) in demList:
  
          demFolder = os.path.dirname(dem)
  
          # Handle frame range option
          if (frames[0] < options.startFrame):
              continue
          if (frames[1] > options.stopFrame):
              break
  
          # Progress indication
          if frames[0] % 100 == 0:
              print("Frame: " + str(frames[0]))
              batchInfoLog.flush() # for instant gratification
              failureLog.flush()
  
          # Read in blend results which are not part of the consolidated stats file
          blendDiffPath = os.path.join(demFolder, 'out-blend-DEM-diff.csv')
          try:
              blendDiffResults = icebridge_common.readGeodiffOutput(blendDiffPath)
          except:
              blendDiffResults = {'Mean':-999}
  
          # Read in blend results which are not part of the consolidated stats file
          # for the blend done in the fireball footprint
          fireballBlendDiffPath = os.path.join(demFolder, 'out-blend-fb-footprint-diff.csv')
  
          try:
              fireballBlendDiffResults = icebridge_common.readGeodiffOutput(fireballBlendDiffPath)
          except:
              fireballBlendDiffResults = {'Mean':-999}
  
          runStatsFile = os.path.join(demFolder, icebridge_common.getRunStatsFile())
          statsLine = icebridge_common.readStats(runStatsFile)
          
          # All of the other results should be in a consolidated stats file
          consolidatedStatsPath = os.path.join(demFolder, 'out-consolidated_stats.txt')
  
          if not os.path.exists(consolidatedStatsPath):
              # Stats file not present, recreate it.
  
              print 'Recreating missing stats file: ' + consolidatedStatsPath
  
              # Get paths to the files of interest
              # This logic must be sync-ed up with cleanBatch().
              lidarDiffPath     = os.path.join(demFolder, 'out-diff.csv')
              interDiffPath     = os.path.join(demFolder, 'out_inter_diff_summary.csv')
              fireDiffPath      = os.path.join(demFolder, 'out_fireball_diff_summary.csv')
              fireLidarDiffPath = os.path.join(demFolder, 'out_fireLidar_diff_summary.csv')
              fractionValidPath = os.path.join(demFolder, 'valid_pixel_fraction.txt')
              process_icebridge_batch.consolidateStats(lidarDiffPath, interDiffPath, fireDiffPath,
                                                       fireLidarDiffPath, dem,
                                                       consolidatedStatsPath,
                                                       fractionValidPath, None, options.skipGeo)
          # Now the consolidated file should always be present
  
          with open(consolidatedStatsPath, 'r') as f:
              statsText = f.read()
  
          # Write info to summary file
          try:
              batchInfoLog.write('%d, %d, %s, %f, %f, %s\n' % 
                                 (frames[0], frames[1], statsText,
                                  blendDiffResults['Mean'], fireballBlendDiffResults['Mean'],
                                  statsLine))
          except:
              # Bugfix for corrupted data
              print("Problem parsing frame: " + str(frames[0]))
                  
          # Keep a list of batches that did not generate an output DEM
          parts = statsText.split(',')
          if len(parts) <= 2:
              print("Cannot parse " + consolidatedStatsPath + ", skipping.")
          else:
              if (float(parts[0]) == 0) and (float(parts[1]) == 0) and (float(parts[2]) == -999):
  
                  if os.path.exists(dem):
                      # Handle the case where the statistics are bad for some reason
                      errorCode = 0
                      errorText = 'Success but statistics are bad'
                  else: # A real failure, figure out the cause
                      batchFolder = os.path.dirname(dem)
                      (errorCode, errorText) = getFailureCause(batchFolder)
                      #print str((errorCode, errorText))
                  #if errorCode < 0: # Debug code for unknown errors
                      #print str((errorCode, errorText))
                      #print statsText
                      #print batchFolder
                      #raise Exception('DEBUG')
                  failureLog.write('%d, %d, %d, %s\n' %  (frames[0], frames[1], errorCode, errorText))
  
          # Make a link to the DEM thumbnail file in our summary folder
          hillshadePath = os.path.join(demFolder, 'out-blend-DEM_HILLSHADE_browse.tif')
          if os.path.exists(hillshadePath):
              thumbName = ('dem_%05d_%05d_browse.tif' % (frames[0], frames[1]))
              thumbPath = os.path.join(options.outputFolder, thumbName)
              icebridge_common.makeSymLink(hillshadePath, thumbPath, verbose=False)
          else:
              # If the DEM thumbnail does not exist, look for the input frame thumbnail.
              inPath    = os.path.join(demFolder, 'first_image_browse.tif')
              thumbName = ('input_%05d_browse.tif' % (frames[0]))
              thumbPath = os.path.join(badImageFolder, thumbName)
              if os.path.exists(inPath):
                  icebridge_common.makeSymLink(inPath, thumbPath, verbose=False)                
  
          # Make a link to the ortho thumbnail file in our summary folder
          orthoPath = os.path.join(demFolder,  icebridge_common.orthoPreviewFileName())
          if os.path.exists(orthoPath):
              thumbName = ('ortho_%05d_%05d_browse.jpg' % (frames[0], frames[1]))
              thumbPath = os.path.join(options.outputFolder, thumbName)
              icebridge_common.makeSymLink(orthoPath, thumbPath, verbose=False)
  
      # End loop through expected DEMs and writing log files
      batchInfoLog.close()
      failureLog.close()
      
      print 'Finished generating flight summary in folder: ' + options.outputFolder
      print("Wrote: " + batchInfoPath)
      print("Wrote: " + failedBatchPath)
function_tokens: def generateFlightSummary ( run , options ) : # Copy logs to the output folder print 'Copying log files...' badImageFolder = os . path . join ( options . outputFolder , 'badImages' ) runFolder = run . getFolder ( ) procFolder = run . getProcessFolder ( ) navCameraFolder = run . getNavCameraFolder ( ) os . system ( 'mkdir -p ' + options . outputFolder ) os . system ( 'mkdir -p ' + badImageFolder ) packedErrorLog = os . path . join ( runFolder , 'packedErrors.log' ) if os . path . exists ( packedErrorLog ) : try : shutil . copy ( packedErrorLog , options . outputFolder ) except Exception as e : # In case it complains about copying a file onto itself print ( "Warning: " + str ( e ) ) if not options . skipKml : # Copy the input camera kml file camerasInKmlPath = os . path . join ( procFolder , 'cameras_in.kml' ) try : shutil . copy ( camerasInKmlPath , options . outputFolder ) except Exception as e : # In case it complains about copying a file onto itself print ( "Warning: " + str ( e ) ) # Copy the input camera kml file navCamerasKmlPath = os . path . join ( navCameraFolder , 'nav_cameras.kml' ) try : shutil . copy ( navCamerasKmlPath , options . outputFolder ) except Exception as e : # In case it complains about copying a file onto itself print ( "Warning: " + str ( e ) ) # Create a merged version of all the bundle adjusted camera files # - The tool currently includes cameras more than once if they appear #   in multiple bundles. print 'Merging output camera kml files...' cmd = "find " + procFolder + " -name cameras_out.kml" p = subprocess . Popen ( cmd . split ( ) , stdout = subprocess . PIPE , shell = False , universal_newlines = True ) textOutput , err = p . communicate ( ) camKmlFiles = textOutput . replace ( '\n' , ' ' ) # Write the list of files to process to disk. Otherwise, if we just # pass the full list on the command line, it may be too big # and the call will fail. kmlList = os . path . join ( options . outputFolder , 'kml_list.txt' ) print ( "Writing: " + kmlList ) with open ( kmlList , 'w' ) as f : for filename in sorted ( camKmlFiles . split ( ) ) : filename = filename . strip ( ) if filename == "" : continue f . write ( filename + "\n" ) outputKml = os . path . join ( options . outputFolder , 'cameras_out.kml' ) scriptPath = icebridge_common . fullPath ( 'merge_orbitviz.py' ) cmd = 'python ' + scriptPath + ' ' + outputKml + ' -list ' + kmlList print ( cmd ) os . system ( cmd ) # Generate lidar kml files print ( "Generating lidar kml files" ) LIDAR_POINT_SKIP = 1527 lidarFiles = run . getLidarList ( prependFolder = True ) lidarOutputFolder = os . path . join ( options . outputFolder , 'lidar' ) os . system ( 'mkdir -p ' + lidarOutputFolder ) for f in lidarFiles : inputPath = os . path . splitext ( f ) [ 0 ] + '.csv' outputPath = os . path . join ( lidarOutputFolder , os . path . basename ( f ) + '.kml' ) args = [ inputPath , outputPath , '--skip' , str ( LIDAR_POINT_SKIP ) , '--color' , 'red' ] if not os . path . exists ( outputPath ) : # Don't recreate these files print ( "Generating: " + outputPath ) # This can be very slow, so print what is going on try : lvis2kml . main ( args ) except Exception as e : # Do not let this make our life miserable print ( "Problem: " + str ( e ) ) # Collect per-batch information batchInfoPath = os . path . join ( options . outputFolder , 'batchInfoSummary.csv' ) failedBatchPath = os . path . join ( options . outputFolder , 'failedBatchList.csv' ) print ( "Writing statistics to: " + batchInfoPath ) print ( "Writing failures to: " + failedBatchPath ) batchInfoLog = open ( batchInfoPath , 'w' ) failureLog = open ( failedBatchPath , 'w' ) # Write the header for the batch log file batchInfoLog . write ( '# startFrame, stopFrame, centerLon, centerLat, meanAlt, ' + 'meanLidarDiff, meanInterDiff, meanFireDiff, meanFireLidarDiff, ' + 'percentValid, meanBlendDiff, meanBlendDiffInFireballFootprint, ' + 'corrSearchWid, corrMem(GB), corrElapsedTime(minutes)\n' ) failureLog . write ( '# startFrame, stopFrame, errorCode, errorText\n' ) demList = run . getOutputFileList ( icebridge_common . blendFileName ( ) ) for ( dem , frames ) in demList : demFolder = os . path . dirname ( dem ) # Handle frame range option if ( frames [ 0 ] < options . startFrame ) : continue if ( frames [ 1 ] > options . stopFrame ) : break # Progress indication if frames [ 0 ] % 100 == 0 : print ( "Frame: " + str ( frames [ 0 ] ) ) batchInfoLog . flush ( ) # for instant gratification failureLog . flush ( ) # Read in blend results which are not part of the consolidated stats file blendDiffPath = os . path . join ( demFolder , 'out-blend-DEM-diff.csv' ) try : blendDiffResults = icebridge_common . readGeodiffOutput ( blendDiffPath ) except : blendDiffResults = { 'Mean' : - 999 } # Read in blend results which are not part of the consolidated stats file # for the blend done in the fireball footprint fireballBlendDiffPath = os . path . join ( demFolder , 'out-blend-fb-footprint-diff.csv' ) try : fireballBlendDiffResults = icebridge_common . readGeodiffOutput ( fireballBlendDiffPath ) except : fireballBlendDiffResults = { 'Mean' : - 999 } runStatsFile = os . path . join ( demFolder , icebridge_common . getRunStatsFile ( ) ) statsLine = icebridge_common . readStats ( runStatsFile ) # All of the other results should be in a consolidated stats file consolidatedStatsPath = os . path . join ( demFolder , 'out-consolidated_stats.txt' ) if not os . path . exists ( consolidatedStatsPath ) : # Stats file not present, recreate it. print 'Recreating missing stats file: ' + consolidatedStatsPath # Get paths to the files of interest # This logic must be sync-ed up with cleanBatch(). lidarDiffPath = os . path . join ( demFolder , 'out-diff.csv' ) interDiffPath = os . path . join ( demFolder , 'out_inter_diff_summary.csv' ) fireDiffPath = os . path . join ( demFolder , 'out_fireball_diff_summary.csv' ) fireLidarDiffPath = os . path . join ( demFolder , 'out_fireLidar_diff_summary.csv' ) fractionValidPath = os . path . join ( demFolder , 'valid_pixel_fraction.txt' ) process_icebridge_batch . consolidateStats ( lidarDiffPath , interDiffPath , fireDiffPath , fireLidarDiffPath , dem , consolidatedStatsPath , fractionValidPath , None , options . skipGeo ) # Now the consolidated file should always be present with open ( consolidatedStatsPath , 'r' ) as f : statsText = f . read ( ) # Write info to summary file try : batchInfoLog . write ( '%d, %d, %s, %f, %f, %s\n' % ( frames [ 0 ] , frames [ 1 ] , statsText , blendDiffResults [ 'Mean' ] , fireballBlendDiffResults [ 'Mean' ] , statsLine ) ) except : # Bugfix for corrupted data print ( "Problem parsing frame: " + str ( frames [ 0 ] ) ) # Keep a list of batches that did not generate an output DEM parts = statsText . split ( ',' ) if len ( parts ) <= 2 : print ( "Cannot parse " + consolidatedStatsPath + ", skipping." ) else : if ( float ( parts [ 0 ] ) == 0 ) and ( float ( parts [ 1 ] ) == 0 ) and ( float ( parts [ 2 ] ) == - 999 ) : if os . path . exists ( dem ) : # Handle the case where the statistics are bad for some reason errorCode = 0 errorText = 'Success but statistics are bad' else : # A real failure, figure out the cause batchFolder = os . path . dirname ( dem ) ( errorCode , errorText ) = getFailureCause ( batchFolder ) #print str((errorCode, errorText)) #if errorCode < 0: # Debug code for unknown errors #print str((errorCode, errorText)) #print statsText #print batchFolder #raise Exception('DEBUG') failureLog . write ( '%d, %d, %d, %s\n' % ( frames [ 0 ] , frames [ 1 ] , errorCode , errorText ) ) # Make a link to the DEM thumbnail file in our summary folder hillshadePath = os . path . join ( demFolder , 'out-blend-DEM_HILLSHADE_browse.tif' ) if os . path . exists ( hillshadePath ) : thumbName = ( 'dem_%05d_%05d_browse.tif' % ( frames [ 0 ] , frames [ 1 ] ) ) thumbPath = os . path . join ( options . outputFolder , thumbName ) icebridge_common . makeSymLink ( hillshadePath , thumbPath , verbose = False ) else : # If the DEM thumbnail does not exist, look for the input frame thumbnail. inPath = os . path . join ( demFolder , 'first_image_browse.tif' ) thumbName = ( 'input_%05d_browse.tif' % ( frames [ 0 ] ) ) thumbPath = os . path . join ( badImageFolder , thumbName ) if os . path . exists ( inPath ) : icebridge_common . makeSymLink ( inPath , thumbPath , verbose = False ) # Make a link to the ortho thumbnail file in our summary folder orthoPath = os . path . join ( demFolder , icebridge_common . orthoPreviewFileName ( ) ) if os . path . exists ( orthoPath ) : thumbName = ( 'ortho_%05d_%05d_browse.jpg' % ( frames [ 0 ] , frames [ 1 ] ) ) thumbPath = os . path . join ( options . outputFolder , thumbName ) icebridge_common . makeSymLink ( orthoPath , thumbPath , verbose = False ) # End loop through expected DEMs and writing log files batchInfoLog . close ( ) failureLog . close ( ) print 'Finished generating flight summary in folder: ' + options . outputFolder print ( "Wrote: " + batchInfoPath ) print ( "Wrote: " + failedBatchPath )