# Path: falsy/netboy/request.py
# async def get_request(payload, share=None):
#     c = pycurl.Curl()
#     data_buf = BytesIO()
#     # header_buf = BytesIO()
#     headers = {'count': 0, 'content': [{}]}
#     try:
#         setup_curl_for_get(c, payload, data_buf, headers, share)  # header_buf)
# 
#         with aiohttp.Timeout(payload.get('aiohttp_timeout', 60)):
#             resp = await CurlLoop.handler_ready(c)
#             charset = None
#             if 'content-type' in headers:
#                 content_type = headers['content-type'].lower()
#                 match = re.search('charset=(\S+)', content_type)
#                 if match:
#                     charset = match.group(1)
#                     print('Decoding using %s' % charset)
#             body = data_buf.getvalue()
#             if len(body) == 0:
#                 data = ''
#                 charset = 'utf-8'
#             else:
#                 if charset is None:
#                     dammit = UnicodeDammit(body, ["utf-8", "gb2312", "gbk", "big5", "gb18030"], smart_quotes_to="html")
#                     data = dammit.unicode_markup
#                     charset = dammit.original_encoding
#                 else:
#                     data = body.decode(charset, 'ignore')
#             # headers.remove({})
#             headers['content'] = [h for h in headers['content'] if len(h) > 0]
#             soup_lxml = BeautifulSoup(data, 'lxml')
#             soup_html = BeautifulSoup(data, 'html.parser')
#             resp.update({
#                 'url': payload.get('url'),
#                 # 'soup': soup,
#                 'title': get_title(soup_lxml),
#                 'links': get_links(soup_lxml),
#                 'links2': get_links2(soup_lxml),
#                 'metas': get_metas(soup_lxml),
#                 'images': get_images(soup_lxml),
#                 'scripts': get_scripts(soup_lxml),
#                 'text': get_text(soup_html),
#                 'data': data,
#                 'headers': headers,
#                 'charset': charset,
#                 'spider': 'pycurl',
#                 'payload': payload,
#             })
#             post_func = payload.get('post_func')
#             if post_func:
#                 post_func = load(post_func)
#                 resp = post_func(payload, resp)
#             return resp
#     finally:
#         c.close()
# 
# async def post_request(payload, share=None):
#     c = pycurl.Curl()
#     data_buf = BytesIO()
#     # header_buf = BytesIO()
#     headers = {'count': 0, 'content': [{}]}
#     try:
#         setup_curl_for_post(c, payload, data_buf, headers, share)  # header_buf)
# 
#         with aiohttp.Timeout(payload.get('aiohttp_timeout', 60)):
#             resp = await CurlLoop.handler_ready(c)
#             # encoding = None
#             # if 'content-type' in headers:
#             #     content_type = headers['content-type'].lower()
#             #     match = re.search('charset=(\S+)', content_type)
#             #     if match:
#             #         encoding = match.group(1)
#             #         print('Decoding using %s' % encoding)
#             body = data_buf.getvalue()
#             encoding = 'utf-8'
#             data = body.decode(encoding, 'ignore') if len(body) > 0 else ''
# 
#             # if encoding is None:
#             #     dammit = UnicodeDammit(body, ["utf-8", "gb2312", "gbk", "big5", "gb18030"], smart_quotes_to="html")
#             #     data = dammit.unicode_markup
#             #     encoding = dammit.original_encoding
#             # else:
#             #     data = body.decode(encoding, 'ignore')
#             # headers.remove({})
#             headers['content'] = [h for h in headers['content'] if len(h) > 0]
# 
#             resp.update({
#                 # 'url': payload.get('url'),
#                 'data': data,
#                 'headers': headers,
#                 'encoding': encoding,
#             })
#             post_func = payload.get('post_func')
#             if type(post_func) == str:
#                 post_func = load(post_func)
#             if post_func:
#                 resp = post_func(payload, resp)
#             # post_func = payload.get('post_func')
#             # if post_func:
#             #     post_func = load(post_func)
#             #     resp = post_func(payload, resp)
#             return resp
#     finally:
#         c.close()


from falsy.netboy.request import get_request, post_request
import asyncio as aio



async def get_boy(payload):
    targets = []
    for p in payload:
        targets.append(get_request(p))
    res = await aio.gather(
        *targets, return_exceptions=True
    )
    return res

async def post_boy(payload):
    targets = []
    for p in payload:
        targets.append(post_request(p))
    res = await aio.gather(
        *targets, return_exceptions=True
    )
    return res

async def net_boy(payload, share=None):
    targets = []
    for p in payload:
        if p.get('postfields'):
            targets.append(post_request(p, share))
        else:

# Path: falsy/netboy/request.py
# async def get_request(payload, share=None):
#     c = pycurl.Curl()
#     data_buf = BytesIO()
#     # header_buf = BytesIO()
#     headers = {'count': 0, 'content': [{}]}
#     try:
#         setup_curl_for_get(c, payload, data_buf, headers, share)  # header_buf)
# 
#         with aiohttp.Timeout(payload.get('aiohttp_timeout', 60)):
#             resp = await CurlLoop.handler_ready(c)
#             charset = None
#             if 'content-type' in headers:
#                 content_type = headers['content-type'].lower()
#                 match = re.search('charset=(\S+)', content_type)
#                 if match:
#                     charset = match.group(1)
#                     print('Decoding using %s' % charset)
#             body = data_buf.getvalue()
#             if len(body) == 0:
#                 data = ''
#                 charset = 'utf-8'
#             else:
#                 if charset is None:
#                     dammit = UnicodeDammit(body, ["utf-8", "gb2312", "gbk", "big5", "gb18030"], smart_quotes_to="html")
#                     data = dammit.unicode_markup
#                     charset = dammit.original_encoding
#                 else:
#                     data = body.decode(charset, 'ignore')
#             # headers.remove({})
#             headers['content'] = [h for h in headers['content'] if len(h) > 0]
#             soup_lxml = BeautifulSoup(data, 'lxml')
#             soup_html = BeautifulSoup(data, 'html.parser')
#             resp.update({
#                 'url': payload.get('url'),
#                 # 'soup': soup,
#                 'title': get_title(soup_lxml),
#                 'links': get_links(soup_lxml),
#                 'links2': get_links2(soup_lxml),
#                 'metas': get_metas(soup_lxml),
#                 'images': get_images(soup_lxml),
#                 'scripts': get_scripts(soup_lxml),
#                 'text': get_text(soup_html),
#                 'data': data,
#                 'headers': headers,
#                 'charset': charset,
#                 'spider': 'pycurl',
#                 'payload': payload,
#             })
#             post_func = payload.get('post_func')
#             if post_func:
#                 post_func = load(post_func)
#                 resp = post_func(payload, resp)
#             return resp
#     finally:
#         c.close()
# 
# async def post_request(payload, share=None):
#     c = pycurl.Curl()
#     data_buf = BytesIO()
#     # header_buf = BytesIO()
#     headers = {'count': 0, 'content': [{}]}
#     try:
#         setup_curl_for_post(c, payload, data_buf, headers, share)  # header_buf)
# 
#         with aiohttp.Timeout(payload.get('aiohttp_timeout', 60)):
#             resp = await CurlLoop.handler_ready(c)
#             # encoding = None
#             # if 'content-type' in headers:
#             #     content_type = headers['content-type'].lower()
#             #     match = re.search('charset=(\S+)', content_type)
#             #     if match:
#             #         encoding = match.group(1)
#             #         print('Decoding using %s' % encoding)
#             body = data_buf.getvalue()
#             encoding = 'utf-8'
#             data = body.decode(encoding, 'ignore') if len(body) > 0 else ''
# 
#             # if encoding is None:
#             #     dammit = UnicodeDammit(body, ["utf-8", "gb2312", "gbk", "big5", "gb18030"], smart_quotes_to="html")
#             #     data = dammit.unicode_markup
#             #     encoding = dammit.original_encoding
#             # else:
#             #     data = body.decode(encoding, 'ignore')
#             # headers.remove({})
#             headers['content'] = [h for h in headers['content'] if len(h) > 0]
# 
#             resp.update({
#                 # 'url': payload.get('url'),
#                 'data': data,
#                 'headers': headers,
#                 'encoding': encoding,
#             })
#             post_func = payload.get('post_func')
#             if type(post_func) == str:
#                 post_func = load(post_func)
#             if post_func:
#                 resp = post_func(payload, resp)
#             # post_func = payload.get('post_func')
#             # if post_func:
#             #     post_func = load(post_func)
#             #     resp = post_func(payload, resp)
#             return resp
#     finally:
#         c.close()


# Path: falsy/netboy/fetch.py
from falsy.netboy.request import get_request, post_request
import asyncio as aio




async def get_boy(payload):
    targets = []
    for p in payload:
        targets.append(get_request(p))
    res = await aio.gather(
        *targets, return_exceptions=True
    )
    return res

async def post_boy(payload):
    targets = []
    for p in payload:
        targets.append(post_request(p))
    res = await aio.gather(
        *targets, return_exceptions=True
    )
    return res

async def net_boy(payload, share=None):
    targets = []
    for p in payload:
        if p.get('postfields'):
            targets.append(post_request(p, share))
        else:

            targets.append(get_request(p, share))