@ Override public void concat ( Path trg , Path [ ] psrcs ) throws IOException { logger . atFine ( ) . log ( "GHFS.concat: %s, %s" , trg , lazy ( ( ) -> Arrays . toString ( psrcs ) ) ) ; checkArgument ( psrcs . length > 0 , "psrcs must have at least one source" ) ; URI trgPath = getGcsPath ( trg ) ; List < URI > srcPaths = Arrays . stream ( psrcs ) . map ( this :: getGcsPath ) . collect ( toImmutableList ( ) ) ; checkArgument ( ! srcPaths . contains ( trgPath ) , "target must not be contained in sources" ) ; List < List < URI > > partitions = Lists . partition ( srcPaths , GoogleCloudStorage . MAX_COMPOSE_OBJECTS - 1 ) ; logger . atFine ( ) . log ( "GHFS.concat: %s, %d partitions" , trg , partitions . size ( ) ) ; for ( List < URI > partition : partitions ) { // We need to include the target in the list of sources to compose since // the GCS FS compose operation will overwrite the target, whereas the Hadoop // concat operation appends to the target. List < URI > sources = Lists . newArrayList ( trgPath ) ; sources . addAll ( partition ) ; logger . atFine ( ) . log ( "GHFS.concat compose: %s, %s" , trgPath , sources ) ; getGcsFs ( ) . compose ( sources , trgPath , CreateFileOptions . DEFAULT_CONTENT_TYPE ) ; } logger . atFine ( ) . log ( "GHFS.concat:=> " ) ; }
Concat existing files into one file .