private String parseHadoopOutputCommitter ( Path fullPath , boolean addTaskIdCompositeName , String hostNameScheme ) throws IOException { String path = fullPath . toString ( ) ; String noPrefix = path ; if ( path . startsWith ( hostNameScheme ) ) { noPrefix = path . substring ( hostNameScheme . length ( ) ) ; } int npIdx = noPrefix . indexOf ( HADOOP_TEMPORARY ) ; String objectName ; if ( npIdx >= 0 ) { if ( npIdx == 0 || npIdx == 1 && noPrefix . startsWith ( "/" ) ) { //no object name present, for example //schema://tone1.lvm/_temporary/0/_temporary/attempt_201610038_0001_m_000007_15/part-0007 //schema://tone1.lvm_temporary/0/_temporary/attempt_201610038_0001_m_000007_15/part-0007 throw new IOException ( "Object name is missing" ) ; } else { // Will give: schema://tone1.lvm/aa/bb/cc/one3.txt/ objectName = noPrefix . substring ( 0 , npIdx - 1 ) ; if ( addTaskIdCompositeName ) { String objName = null ; String taskAttempt = Utils . extractTaskID ( path , HADOOP_ATTEMPT ) ; if ( taskAttempt != null ) { int fIndex = fullPath . toString ( ) . indexOf ( taskAttempt + "/" ) ; if ( fIndex > 0 ) { fIndex = fIndex + taskAttempt . length ( ) + 1 ; } if ( fIndex < fullPath . toString ( ) . length ( ) ) { objName = fullPath . toString ( ) . substring ( fIndex ) ; } } if ( objName == null ) { objName = fullPath . getName ( ) ; } if ( taskAttempt != null && ! objName . startsWith ( HADOOP_ATTEMPT ) ) { // We want to prepend the attempt before the extension String extension = extractExtension ( objName ) ; objName = objName . replace ( "." + extension , "" ) + "-" + taskAttempt ; if ( ! extension . equals ( "" ) ) { objName += "." + extension ; } } objectName = objectName + "/" + objName ; } } return objectName ; } return noPrefix ; }
Main method to parse Hadoop OutputCommitter V1 or V2 as used by Hadoop M - R or Apache Spark Method transforms object name from the temporary path .