private void downloadFiles ( ) throws SnowflakeSQLException { try { threadExecutor = SnowflakeUtil . createDefaultExecutorService ( "sf-file-download-worker-" , 1 ) ; for ( String srcFile : sourceFiles ) { FileMetadata fileMetadata = fileMetadataMap . get ( srcFile ) ; // Check if the result status is already set so that we don't need to // upload it if ( fileMetadata . resultStatus != ResultStatus . UNKNOWN ) { logger . debug ( "Skipping {}, status: {}, details: {}" , srcFile , fileMetadata . resultStatus , fileMetadata . errorDetails ) ; continue ; } RemoteStoreFileEncryptionMaterial encMat = srcFileToEncMat . get ( srcFile ) ; threadExecutor . submit ( getDownloadFileCallable ( stageInfo , srcFile , localLocation , fileMetadataMap , ( stageInfo . getStageType ( ) == StageInfo . StageType . LOCAL_FS ) ? null : storageFactory . createClient ( stageInfo , parallel , encMat ) , connection , command , parallel , encMat ) ) ; logger . debug ( "submitted download job for: {}" , srcFile ) ; } threadExecutor . shutdown ( ) ; try { // wait for all threads to complete without timeout threadExecutor . awaitTermination ( Long . MAX_VALUE , TimeUnit . DAYS ) ; } catch ( InterruptedException ex ) { throw new SnowflakeSQLException ( SqlState . QUERY_CANCELED , ErrorCode . INTERRUPTED . getMessageCode ( ) ) ; } logger . debug ( "Done with downloading" ) ; } finally { if ( threadExecutor != null ) { threadExecutor . shutdownNow ( ) ; threadExecutor = null ; } } }
Helper to download files from remote