public void downloadAndExtract ( DataSetType set ) throws IOException { String localFilename = new File ( remoteDataUrl ( set ) ) . getName ( ) ; File tmpFile = new File ( System . getProperty ( "java.io.tmpdir" ) , localFilename ) ; File localCacheDir = getLocalCacheDir ( ) ; // check empty cache if ( localCacheDir . exists ( ) ) { File [ ] list = localCacheDir . listFiles ( ) ; if ( list == null || list . length == 0 ) localCacheDir . delete ( ) ; } File localDestinationDir = new File ( localCacheDir , dataSetName ( set ) ) ; if ( ! localDestinationDir . exists ( ) ) { localCacheDir . mkdirs ( ) ; tmpFile . delete ( ) ; log . info ( "Downloading dataset to " + tmpFile . getAbsolutePath ( ) ) ; FileUtils . copyURLToFile ( new URL ( remoteDataUrl ( set ) ) , tmpFile ) ; } else { //Directory exists and is non-empty - assume OK log . info ( "Using cached dataset at " + localCacheDir . getAbsolutePath ( ) ) ; return ; } if ( expectedChecksum ( set ) != 0L ) { log . info ( "Verifying download..." ) ; Checksum adler = new Adler32 ( ) ; FileUtils . checksum ( tmpFile , adler ) ; long localChecksum = adler . getValue ( ) ; log . info ( "Checksum local is " + localChecksum + ", expecting " + expectedChecksum ( set ) ) ; if ( expectedChecksum ( set ) != localChecksum ) { log . error ( "Checksums do not match. Cleaning up files and failing..." ) ; tmpFile . delete ( ) ; throw new IllegalStateException ( "Dataset file failed checksum: " + tmpFile + " - expected checksum " + expectedChecksum ( set ) + " vs. actual checksum " + localChecksum + ". If this error persists, please open an issue at https://github.com/deeplearning4j/deeplearning4j." ) ; } } try { ArchiveUtils . unzipFileTo ( tmpFile . getAbsolutePath ( ) , localCacheDir . getAbsolutePath ( ) ) ; } catch ( Throwable t ) { //Catch any errors during extraction, and delete the directory to avoid leaving the dir in an invalid state if ( localCacheDir . exists ( ) ) FileUtils . deleteDirectory ( localCacheDir ) ; throw t ; } }
Downloads and extracts the local dataset .