private void run ( ) { onStart ( ) ; while ( ! isStopping && crawlFrontier . hasNextCandidate ( ) ) { CrawlCandidate currentCandidate = crawlFrontier . getNextCandidate ( ) ; String candidateUrl = currentCandidate . getRequestUrl ( ) . toString ( ) ; HttpClientContext context = HttpClientContext . create ( ) ; CloseableHttpResponse httpHeadResponse = null ; boolean isUnsuccessfulRequest = false ; try { // Send an HTTP HEAD request to determine its availability and content type httpHeadResponse = getHttpHeadResponse ( candidateUrl , context ) ; } catch ( IOException exception ) { callbackManager . call ( CrawlEvent . REQUEST_ERROR , new RequestErrorEvent ( currentCandidate , exception ) ) ; isUnsuccessfulRequest = true ; } if ( ! isUnsuccessfulRequest ) { String responseUrl = getFinalResponseUrl ( context , candidateUrl ) ; if ( responseUrl . equals ( candidateUrl ) ) { String responseMimeType = getResponseMimeType ( httpHeadResponse ) ; if ( responseMimeType . equals ( ContentType . TEXT_HTML . getMimeType ( ) ) ) { boolean isTimedOut = false ; TimeoutException timeoutException = null ; try { // Open URL in browser webDriver . get ( candidateUrl ) ; } catch ( TimeoutException exception ) { isTimedOut = true ; timeoutException = exception ; } // Ensure the HTTP client and Selenium have the same state syncHttpClientCookies ( ) ; if ( isTimedOut ) { callbackManager . call ( CrawlEvent . PAGE_LOAD_TIMEOUT , new PageLoadTimeoutEvent ( currentCandidate , timeoutException ) ) ; } else { String loadedPageUrl = webDriver . getCurrentUrl ( ) ; if ( ! loadedPageUrl . equals ( candidateUrl ) ) { // Create a new crawl request for the redirected URL (JS redirect) handleRequestRedirect ( currentCandidate , loadedPageUrl ) ; } else { callbackManager . call ( CrawlEvent . PAGE_LOAD , new PageLoadEvent ( currentCandidate , webDriver ) ) ; } } } else { // URLs that point to non-HTML content should not be opened in the browser callbackManager . call ( CrawlEvent . NON_HTML_CONTENT , new NonHtmlContentEvent ( currentCandidate , responseMimeType ) ) ; } } else { // Create a new crawl request for the redirected URL handleRequestRedirect ( currentCandidate , responseUrl ) ; } } HttpClientUtils . closeQuietly ( httpHeadResponse ) ; performDelay ( ) ; } onStop ( ) ; }
Defines the workflow of the crawler .