public QueryBasedHivePublishEntity generatePublishQueries ( ) throws DataConversionException { QueryBasedHivePublishEntity publishEntity = new QueryBasedHivePublishEntity ( ) ; List < String > publishQueries = publishEntity . getPublishQueries ( ) ; Map < String , String > publishDirectories = publishEntity . getPublishDirectories ( ) ; List < String > cleanupQueries = publishEntity . getCleanupQueries ( ) ; List < String > cleanupDirectories = publishEntity . getCleanupDirectories ( ) ; String createFinalTableDDL = HiveConverterUtils . generateCreateDuplicateTableDDL ( outputDatabaseName , stagingTableName , outputTableName , outputDataLocation , Optional . of ( outputDatabaseName ) ) ; publishQueries . add ( createFinalTableDDL ) ; log . debug ( "Create final table DDL:\n" + createFinalTableDDL ) ; if ( ! this . supportTargetPartitioning || partitionsDDLInfo . size ( ) == 0 ) { log . debug ( "Snapshot directory to move: " + stagingDataLocation + " to: " + outputDataLocation ) ; publishDirectories . put ( stagingDataLocation , outputDataLocation ) ; String dropStagingTableDDL = HiveAvroORCQueryGenerator . generateDropTableDDL ( outputDatabaseName , stagingTableName ) ; log . debug ( "Drop staging table DDL: " + dropStagingTableDDL ) ; cleanupQueries . add ( dropStagingTableDDL ) ; log . debug ( "Staging table directory to delete: " + stagingDataLocation ) ; cleanupDirectories . add ( stagingDataLocation ) ; } else { String finalDataPartitionLocation = outputDataLocation + Path . SEPARATOR + stagingDataPartitionDirName ; Optional < Path > destPartitionLocation = HiveConverterUtils . getDestinationPartitionLocation ( destinationTableMeta , this . workUnitState , conversionEntity . getPartition ( ) . get ( ) . getName ( ) ) ; finalDataPartitionLocation = HiveConverterUtils . updatePartitionLocation ( finalDataPartitionLocation , this . workUnitState , destPartitionLocation ) ; log . debug ( "Partition directory to move: " + stagingDataPartitionLocation + " to: " + finalDataPartitionLocation ) ; publishDirectories . put ( stagingDataPartitionLocation , finalDataPartitionLocation ) ; List < String > dropPartitionsDDL = HiveAvroORCQueryGenerator . generateDropPartitionsDDL ( outputDatabaseName , outputTableName , partitionsDMLInfo ) ; log . debug ( "Drop partitions if exist in final table: " + dropPartitionsDDL ) ; publishQueries . addAll ( dropPartitionsDDL ) ; List < String > createFinalPartitionDDL = HiveAvroORCQueryGenerator . generateCreatePartitionDDL ( outputDatabaseName , outputTableName , finalDataPartitionLocation , partitionsDMLInfo , Optional . < String > absent ( ) ) ; log . debug ( "Create final partition DDL: " + createFinalPartitionDDL ) ; publishQueries . addAll ( createFinalPartitionDDL ) ; String dropStagingTableDDL = HiveAvroORCQueryGenerator . generateDropTableDDL ( outputDatabaseName , stagingTableName ) ; log . debug ( "Drop staging table DDL: " + dropStagingTableDDL ) ; cleanupQueries . add ( dropStagingTableDDL ) ; log . debug ( "Staging table directory to delete: " + stagingDataLocation ) ; cleanupDirectories . add ( stagingDataLocation ) ; publishQueries . addAll ( HiveAvroORCQueryGenerator . generateDropPartitionsDDL ( outputDatabaseName , outputTableName , AbstractAvroToOrcConverter . getDropPartitionsDDLInfo ( conversionEntity ) ) ) ; } log . info ( "Publish partition entity: " + publishEntity ) ; return publishEntity ; }
Returns a QueryBasedHivePublishEntity which includes publish level queries and cleanup commands .