def main_crawler(args, mailer=None):
    include_hosts = (args.include_hosts.split(',') if args.include_hosts else [])
    cookies_list = (args.cookies.split('|') if args.cookies else [''])
    jenkins_build_number = args.jenkins_build_number
    logs_folder = os.path.join(os.getcwd(), 'logs', '{}'.format(jenkins_build_number))
    web_crawler = WebCrawler(args.seeds, include_hosts, logs_folder, args.config_file)
    if (args.grey_user_agent and args.grey_traceid and args.grey_view_grey):
        web_crawler.set_grey_env(args.grey_user_agent, args.grey_traceid, args.grey_view_grey)
    canceled = False
    try:
        for cookies_str in cookies_list:
            cookies_str_list = cookies_str.split(',')
            cookies = {}
            for cookie_str in cookies_str_list:
                if (':' not in cookie_str):
                    continue
                (key, value) = cookie_str.split(':')
                cookies[key.strip()] = value.strip()
            web_crawler.start(cookies, args.crawl_mode, args.max_depth, args.concurrency)
        if (mailer and mailer.config_ready):
            subject = ('%s' % args.seeds)
            (mail_content_ordered_dict, flag_code) = web_crawler.get_mail_content_ordered_dict()
            mailer.send_mail(subject, mail_content_ordered_dict, flag_code)
    except KeyboardInterrupt:
        canceled = True
        color_logging('Canceling...', color='red')
    finally:
        save_results = (False if (args.save_results.upper() == 'NO') else True)
        web_crawler.print_result(canceled, save_results)
def main_crawler(args, mailer=None):
    include_hosts = (args.include_hosts.split(',') if args.include_hosts else [])
    cookies_list = (args.cookies.split('|') if args.cookies else [])
    jenkins_build_number = args.jenkins_build_number
    logs_folder = os.path.join(os.getcwd(), 'logs', '{}'.format(jenkins_build_number))
    web_crawler = WebCrawler(args.seeds, include_hosts, logs_folder, args.config_file)
    if (args.grey_user_agent and args.grey_traceid and args.grey_view_grey):
        web_crawler.set_grey_env(args.grey_user_agent, args.grey_traceid, args.grey_view_grey)
    canceled = False
    try:
        for cookies_str in cookies_list:
            cookies_str_list = cookies_str.split(',')
            cookies = {}
            for cookie_str in cookies_str_list:
                if (':' not in cookie_str):
                    continue
                (key, value) = cookie_str.split(':')
                cookies[key.strip()] = value.strip()
            web_crawler.start(cookies, args.crawl_mode, args.max_depth, args.concurrency)
        if (mailer and mailer.config_ready):
            subject = ('%s' % args.seeds)
            (mail_content_ordered_dict, flag_code) = web_crawler.get_mail_content_ordered_dict()
            mailer.send_mail(subject, mail_content_ordered_dict, flag_code)
    except KeyboardInterrupt:
        canceled = True
        color_logging('Canceling...', color='red')
    finally:
        save_results = (False if (args.save_results.upper() == 'NO') else True)
        web_crawler.print_result(canceled, save_results)