private void verifyUrls ( Set < CrawlerURL > allUrls , Set < HTMLPageResponse > verifiedUrls , Set < HTMLPageResponse > nonWorkingUrls , Map < String , String > requestHeaders ) { Set < CrawlerURL > urlsThatNeedsVerification = new LinkedHashSet < CrawlerURL > ( allUrls ) ; urlsThatNeedsVerification . removeAll ( verifiedUrls ) ; final Set < Callable < HTMLPageResponse > > tasks = new HashSet < Callable < HTMLPageResponse > > ( urlsThatNeedsVerification . size ( ) ) ; for ( CrawlerURL testURL : urlsThatNeedsVerification ) { tasks . add ( new HTMLPageResponseCallable ( testURL , responseFetcher , true , requestHeaders , false ) ) ; } try { // wait for all urls to verify List < Future < HTMLPageResponse >> responses = service . invokeAll ( tasks ) ; for ( Future < HTMLPageResponse > future : responses ) { if ( ! future . isCancelled ( ) ) { HTMLPageResponse response = future . get ( ) ; if ( response . getResponseCode ( ) == HttpStatus . SC_OK && response . getResponseType ( ) . indexOf ( "html" ) > 0 ) { // remove, way of catching interrupted / execution e urlsThatNeedsVerification . remove ( response . getPageUrl ( ) ) ; verifiedUrls . add ( response ) ; } else if ( response . getResponseCode ( ) == HttpStatus . SC_OK ) { // it is not HTML urlsThatNeedsVerification . remove ( response . getPageUrl ( ) ) ; } else { nonWorkingUrls . add ( response ) ; } } } } catch ( InterruptedException e1 ) { // TODO add some logging e1 . printStackTrace ( ) ; } catch ( ExecutionException e ) { // TODO Auto-generated catch block e . printStackTrace ( ) ; } // TODO: We can have a delta here if the exception occur }
Verify that all urls in allUrls returns 200 . If not they will be removed from that set and instead added to the nonworking list .