public static byte [ ] generateWARCInfo ( Map < String , String > fields ) { StringBuffer buffer = new StringBuffer ( ) ; buffer . append ( WARC_VERSION ) ; buffer . append ( CRLF ) ; buffer . append ( "WARC-Type: warcinfo" ) . append ( CRLF ) ; String mainID = UUID . randomUUID ( ) . toString ( ) ; // retrieve the date and filename from the map String date = fields . get ( "WARC-Date" ) ; buffer . append ( "WARC-Date: " ) . append ( date ) . append ( CRLF ) ; String filename = fields . get ( "WARC-Filename" ) ; buffer . append ( "WARC-Filename: " ) . append ( filename ) . append ( CRLF ) ; buffer . append ( "WARC-Record-ID" ) . append ( ": " ) . append ( "<urn:uuid:" ) . append ( mainID ) . append ( ">" ) . append ( CRLF ) ; buffer . append ( "Content-Type" ) . append ( ": " ) . append ( "application/warc-fields" ) . append ( CRLF ) ; StringBuilder fieldsBuffer = new StringBuilder ( ) ; // add WARC fields // http://bibnum.bnf.fr/warc/WARC_ISO_28500_version1_latestdraft.pdf Iterator < Entry < String , String > > iter = fields . entrySet ( ) . iterator ( ) ; while ( iter . hasNext ( ) ) { Entry < String , String > entry = iter . next ( ) ; String key = entry . getKey ( ) ; if ( key . startsWith ( "WARC-" ) ) continue ; fieldsBuffer . append ( key ) . append ( ": " ) . append ( entry . getValue ( ) ) . append ( CRLF ) ; } buffer . append ( "Content-Length" ) . append ( ": " ) . append ( fieldsBuffer . toString ( ) . getBytes ( StandardCharsets . UTF_8 ) . length ) . append ( CRLF ) ; buffer . append ( CRLF ) ; buffer . append ( fieldsBuffer . toString ( ) ) ; buffer . append ( CRLF ) ; buffer . append ( CRLF ) ; return buffer . toString ( ) . getBytes ( StandardCharsets . UTF_8 ) ; }
Generates a WARC info entry which can be stored at the beginning of each WARC file .