protected void prepDocument ( ) { /** * In some cases a body element can't be found (if the HTML is totally * hosed for example) so we create a new body node and append it to the * document. */ if ( mDocument . body ( ) == null ) { mDocument . appendElement ( "body" ) ; } /* Remove all scripts */ Elements elementsToRemove = mDocument . getElementsByTag ( "script" ) ; for ( Element script : elementsToRemove ) { script . remove ( ) ; } /* Remove all stylesheets */ elementsToRemove = getElementsByTag ( mDocument . head ( ) , "link" ) ; for ( Element styleSheet : elementsToRemove ) { if ( "stylesheet" . equalsIgnoreCase ( styleSheet . attr ( "rel" ) ) ) { styleSheet . remove ( ) ; } } /* Remove all style tags in head */ elementsToRemove = mDocument . getElementsByTag ( "style" ) ; for ( Element styleTag : elementsToRemove ) { styleTag . remove ( ) ; } /* Turn all double br's into p's */ /* * TODO: this is pretty costly as far as processing goes. Maybe optimize * later. */ mDocument . body ( ) . html ( mDocument . body ( ) . html ( ) . replaceAll ( Patterns . REGEX_REPLACE_BRS , "</p><p>" ) . replaceAll ( Patterns . REGEX_REPLACE_FONTS , "<$1span>" ) ) ; }
Prepare the HTML document for readability to scrape it . This includes things like stripping javascript CSS and handling terrible markup .