private void cleanConditionally ( Element e , String tag ) { Elements tagsList = getElementsByTag ( e , tag ) ; /** * Gather counts for other typical elements embedded within. Traverse * backwards so we can remove nodes at the same time without effecting * the traversal. * * TODO: Consider taking into account original contentScore here. */ for ( Element node : tagsList ) { int weight = getClassWeight ( node ) ; dbg ( "Cleaning Conditionally (" + node . className ( ) + ":" + node . id ( ) + ")" + getContentScore ( node ) ) ; if ( weight < 0 ) { node . remove ( ) ; } else if ( getCharCount ( node , "," ) < 10 ) { /** * If there are not very many commas, and the number of * non-paragraph elements is more than paragraphs or other * ominous signs, remove the element. */ int p = getElementsByTag ( node , "p" ) . size ( ) ; int img = getElementsByTag ( node , "img" ) . size ( ) ; int li = getElementsByTag ( node , "li" ) . size ( ) - 100 ; int input = getElementsByTag ( node , "input" ) . size ( ) ; int embedCount = 0 ; Elements embeds = getElementsByTag ( node , "embed" ) ; for ( Element embed : embeds ) { if ( ! Patterns . get ( Patterns . RegEx . VIDEO ) . matcher ( embed . absUrl ( "src" ) ) . find ( ) ) { embedCount ++ ; } } float linkDensity = getLinkDensity ( node ) ; int contentLength = getInnerText ( node , true ) . length ( ) ; boolean toRemove = false ; if ( img > p ) { toRemove = true ; } else if ( li > p && ! "ul" . equalsIgnoreCase ( tag ) && ! "ol" . equalsIgnoreCase ( tag ) ) { toRemove = true ; } else if ( input > Math . floor ( p / 3 ) ) { toRemove = true ; } else if ( contentLength < 25 && ( img == 0 || img > 2 ) ) { toRemove = true ; } else if ( weight < 25 && linkDensity > 0.2f ) { toRemove = true ; } else if ( weight > 25 && linkDensity > 0.5f ) { toRemove = true ; } else if ( ( embedCount == 1 && contentLength < 75 ) || embedCount > 1 ) { toRemove = true ; } if ( toRemove ) { node . remove ( ) ; } } } }
Clean an element of all tags of type tag if they look fishy . Fishy is an algorithm based on content length classnames link density number of images & embeds etc .