# Path: lib/core/common.py
# def getSafeExString(ex, encoding=None):
#     """
#     Safe way how to get the proper exception represtation as a string
#     (Note: errors to be avoided: 1) "%s" % Exception(u'\u0161') and 2) "%s" % str(Exception(u'\u0161'))
#     """
#     retVal = ex
# 
#     if getattr(ex, "message", None):
#         retVal = ex.message
#     elif getattr(ex, "msg", None):
#         retVal = ex.msg
# 
#     return getUnicode(retVal, encoding=encoding)
# 
# Path: lib/core/enums.py
# class PROXY_TYPE:  # keep same with SocksiPy(import socks)
#     PROXY_TYPE_SOCKS4 = SOCKS4 = 1
#     PROXY_TYPE_SOCKS5 = SOCKS5 = 2
#     PROXY_TYPE_HTTP = HTTP = 3
#     PROXY_TYPE_HTTP_NO_TUNNEL = 4
# 
# Path: lib/utils/config.py
# class ConfigFileParser:
#     @staticmethod
#     def _get_option(section, option):
#         try:
#             cf = ConfigParser.ConfigParser()
#             cf.read(paths.CONFIG_PATH)
#             return cf.get(section=section, option=option)
#         except ConfigParser.NoOptionError, e:
#             logger.warning('Missing essential options, please check your config-file')
#             logger.error(getSafeExString(e))
#             return ''
# 
#     def ZoomEyeEmail(self):
#         return self._get_option('zoomeye', 'email')
# 
#     def ZoomEyePassword(self):
#         return self._get_option('zoomeye', 'password')
# 
#     def UrlFilter(self):
#         return self._get_option('urlfilter', 'regex')
# 
# Path: lib/core/data.py



import re
import sys
import urllib
import urllib2
from urlparse import urlparse, urlunsplit, urljoin
from lib.core.common import getSafeExString
from lib.core.enums import PROXY_TYPE
from lib.utils.config import ConfigFileParser
from lib.core.data import logger, conf
from bs4 import BeautifulSoup as BS
    s = ''
    for each in _path_list:
        s += '/' + each
        _ans_list.append(urljoin(ori_str, s))
    return _ans_list

def BaiduSearch(query, limit=10, offset=0):
    urllist = {''}
    regex = str(ConfigFileParser().UrlFilter())
    try:
        while len(urllist)<limit:
            url = "http://www.baidu.com/s?{}".format(urllib.urlencode({'wd':query,'pn':str(offset)+'0','tn':'baidurt','ie':'utf-8','bsst':'1'}))
            request = urllib2.Request(url)
            response = urllib2.urlopen(request)
            html = response.read()
            soup = BS(html, "lxml")
            td = soup.find_all(class_='f')
            for t in td:
                if regex:
                    after_url = re.findall(regex, t.h3.a['href'])
                    if after_url:
                        urllist.add(after_url[0])
                else:
                    after_url = iterate_path(t.h3.a['href'])
                    for each_url in after_url:
                        urllist.add(each_url)
            offset = offset + 1
        return urllist
    except urllib2.URLError, e:
        logger.warning('It seems like URL is wrong')

# Path: lib/core/common.py
# def getSafeExString(ex, encoding=None):
#     """
#     Safe way how to get the proper exception represtation as a string
#     (Note: errors to be avoided: 1) "%s" % Exception(u'\u0161') and 2) "%s" % str(Exception(u'\u0161'))
#     """
#     retVal = ex
# 
#     if getattr(ex, "message", None):
#         retVal = ex.message
#     elif getattr(ex, "msg", None):
#         retVal = ex.msg
# 
#     return getUnicode(retVal, encoding=encoding)
# 
# Path: lib/core/enums.py
# class PROXY_TYPE:  # keep same with SocksiPy(import socks)
#     PROXY_TYPE_SOCKS4 = SOCKS4 = 1
#     PROXY_TYPE_SOCKS5 = SOCKS5 = 2
#     PROXY_TYPE_HTTP = HTTP = 3
#     PROXY_TYPE_HTTP_NO_TUNNEL = 4
# 
# Path: lib/utils/config.py
# class ConfigFileParser:
#     @staticmethod
#     def _get_option(section, option):
#         try:
#             cf = ConfigParser.ConfigParser()
#             cf.read(paths.CONFIG_PATH)
#             return cf.get(section=section, option=option)
#         except ConfigParser.NoOptionError, e:
#             logger.warning('Missing essential options, please check your config-file')
#             logger.error(getSafeExString(e))
#             return ''
# 
#     def ZoomEyeEmail(self):
#         return self._get_option('zoomeye', 'email')
# 
#     def ZoomEyePassword(self):
#         return self._get_option('zoomeye', 'password')
# 
#     def UrlFilter(self):
#         return self._get_option('urlfilter', 'regex')
# 
# Path: lib/core/data.py



# Path: lib/api/baidu/pack.py
import re
import sys
import urllib
import urllib2
from urlparse import urlparse, urlunsplit, urljoin
from lib.core.common import getSafeExString
from lib.core.enums import PROXY_TYPE
from lib.utils.config import ConfigFileParser
from lib.core.data import logger, conf
from bs4 import BeautifulSoup as BS

    s = ''
    for each in _path_list:
        s += '/' + each
        _ans_list.append(urljoin(ori_str, s))
    return _ans_list

def BaiduSearch(query, limit=10, offset=0):
    urllist = {''}
    regex = str(ConfigFileParser().UrlFilter())
    try:
        while len(urllist)<limit:
            url = "http://www.baidu.com/s?{}".format(urllib.urlencode({'wd':query,'pn':str(offset)+'0','tn':'baidurt','ie':'utf-8','bsst':'1'}))
            request = urllib2.Request(url)
            response = urllib2.urlopen(request)
            html = response.read()
            soup = BS(html, "lxml")
            td = soup.find_all(class_='f')
            for t in td:
                if regex:
                    after_url = re.findall(regex, t.h3.a['href'])
                    if after_url:
                        urllist.add(after_url[0])
                else:
                    after_url = iterate_path(t.h3.a['href'])
                    for each_url in after_url:
                        urllist.add(each_url)
            offset = offset + 1
        return urllist
    except urllib2.URLError, e:
        logger.warning('It seems like URL is wrong')

        sys.exit(logger.error(getSafeExString(e)))