protected String valueForURL ( Tuple tuple ) { String url = tuple . getStringByField ( "url" ) ; Metadata metadata = ( Metadata ) tuple . getValueByField ( "metadata" ) ; // functionality deactivated if ( StringUtils . isBlank ( canonicalMetadataParamName ) ) { return url ; } String canonicalValue = metadata . getFirstValue ( canonicalMetadataName ) ; // no value found? if ( StringUtils . isBlank ( canonicalValue ) ) { return url ; } try { URL sURL = new URL ( url ) ; URL canonical = URLUtil . resolveURL ( sURL , canonicalValue ) ; String sDomain = PaidLevelDomain . getPLD ( sURL . getHost ( ) ) ; String canonicalDomain = PaidLevelDomain . getPLD ( canonical . getHost ( ) ) ; // check that the domain is the same if ( sDomain . equalsIgnoreCase ( canonicalDomain ) ) { return canonical . toExternalForm ( ) ; } else { LOG . info ( "Canonical URL references a different domain, ignoring in {} " , url ) ; } } catch ( MalformedURLException e ) { LOG . error ( "Malformed canonical URL {} was found in {} " , canonicalValue , url ) ; } return url ; }
Returns the value to be used as the URL for indexing purposes if present the canonical value is used instead