public RobotsTxt readRobotsTxt ( InputStream inputStream ) throws IOException { BufferedReader reader = new BufferedReader ( new InputStreamReader ( inputStream , "UTF-8" ) ) ; Group currentGroup = null ; boolean startGroup = false ; RobotsTxtImpl robots = new RobotsTxtImpl ( matchingStrategy , winningStrategy ) ; for ( Entry entry = readEntry ( reader ) ; entry != null ; entry = readEntry ( reader ) ) { switch ( entry . getKey ( ) . toUpperCase ( ) ) { case "USER-AGENT" : if ( ! startGroup && currentGroup != null ) { robots . addGroup ( currentGroup ) ; currentGroup = null ; } if ( currentGroup == null ) { currentGroup = new Group ( ) ; } currentGroup . addUserAgent ( entry . getValue ( ) ) ; startGroup = true ; break ; case "DISALLOW" : if ( currentGroup != null ) { boolean access = entry . getValue ( ) . isEmpty ( ) ; currentGroup . addAccess ( new Access ( currentGroup , entry . getSource ( ) , entry . getValue ( ) , access ) ) ; startGroup = false ; } break ; case "ALLOW" : if ( currentGroup != null ) { boolean access = ! entry . getValue ( ) . isEmpty ( ) ; currentGroup . addAccess ( new Access ( currentGroup , entry . getSource ( ) , entry . getValue ( ) , access ) ) ; startGroup = false ; } break ; case "CRAWL-DELAY" : if ( currentGroup != null ) { try { int crawlDelay = Integer . parseInt ( entry . getValue ( ) ) ; currentGroup . setCrawlDelay ( crawlDelay ) ; startGroup = false ; } catch ( NumberFormatException ex ) { } } else { } break ; case "HOST" : robots . setHost ( entry . getValue ( ) ) ; startGroup = false ; break ; case "SITEMAP" : robots . getSitemaps ( ) . add ( entry . getValue ( ) ) ; startGroup = false ; break ; default : startGroup = false ; break ; } } if ( currentGroup != null ) { robots . addGroup ( currentGroup ) ; } return robots ; }
Reads robots txt .