<commit_before># -*- coding:utf-8 -*-

"""
@version: 1.0
@author: marcovaldo
@license: Apache Licence
@contact: <EMAIL>
@site:
@software: PyCharm Community Edition
@file: __init__.py.py
@time: 16/11/26 下午15:45
"""
import sys
sys.path.append("..")
# from myfunc import *
from spiders.myfunc import *
from scrapy.http import HtmlResponse
from collections import defaultdict
import itertools
import time
import re
from bs4 import BeautifulSoup


def goodsOutline(url):
    '''
    获取三级目录详情
    :param url: 网站主页链接
    :return: [{'url':'','first_grade':'','second_grade': '', 'third_grade': ''},...{}]
    '''
    # 存储最后的三级列表页面
    outline_data = []
    # 解析页面
    body = getHtmlByRequests(url)
    html = HtmlResponse(url=url, body=str(body))
    soup = BeautifulSoup(body, 'lxml')      # xpath不好使就用beautisoup
    # 抓取一级目录名及链接，这里只要以下内容：
    # 泵阀门接头管件（不要化工泵）、紧固件密封件传动件（全要）、仪器仪表测量（只要气体检测仪，电力电工检测仪器）
    # 所以现在一级列表名称及链接不抓了，直接放到下面
    first_grade_name = ['紧固件密封件传动件', '泵阀门接头管件', '仪器仪表测量']
    first_grade_url = [url + '/category.php?id=207&amp;price_min=&amp;price_max=',
                       url + '/category.php?id=208&amp;price_min=&amp;price_max=',
                       url + '/category.php?id=191&amp;price_min=&amp;price_max=']
    # 这个for循环只处理了前两个一级目录，第三个里边只要两个二级目录
    for i in range(2):
        # 二级目录名及链接
        body = getHtmlByRequests(first_grade_url[i])
        html = HtmlResponse(url=first_grade_url[i], body=str(body))
        # soup = BeautifulSoup(body, 'lxml')
        # second_grade_name = soup.find_all()
        second_grade_name = []
        second_grade_url = []
        q = 1
        while(q):
            try:
                second_grade_name.append(html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/text()').extract()[0])
                second_grade_url.append(url + '/' + html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/@href').extract()[0])
                q += 1
            except:
                break
        # 在这个地方删掉不要的化工泵
        if second_grade_name[7] == u'\u5316\u5de5\u6cf5':
            del second_grade_name[7]
            del second_grade_url[7]
        for j in range(len(second_grade_name)):
            # 三级目录名及链接
            body =  getHtmlByRequests(second_grade_url[j])
            html = HtmlResponse(url=second_grade_url[j], body=str(body))
            third_grade_name = []
            third_grade_url = []
            q = 1
            while(q):
                try:
                    third_grade_name.append(html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/text()').extract()[0])
                    # print(html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/text()').extract()[0])
                    third_grade_url.append(html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/@href').extract()[0])
                    # print(html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/@href').extract()[0])
                    url_data = defaultdict()
                    url_data['url'] = url + '/' + third_grade_url[q-1]  # 把主页url和解析到的按三级目录url片段拼起来
                    url_data['third_grade'] = third_grade_name[q-1]
                    url_data['second_grade'] = second_grade_name[j]
                    url_data['first_grade'] = first_grade_name[i]
                    url_data['created'] = int(time.time())
                    url_data['updated'] = int(time.time())
                    outline_data.append(url_data)
                    q += 1
                except:
                    break
    # 下面抓第三个一级目录下的两个二级目录下的三级目录
    second_grade_name = ['气体检测仪', '电力电工检测仪器']
    second_grade_url = ['http://www.sssmro.com/category.php?id=702&price_min=&price_max=',
                        'http://www.sssmro.com/category.php?id=722&price_min=&price_max=']
    for j in range(len(second_grade_name)):
        # 三级目录名及链接
        body = getHtmlByRequests(second_grade_url[j])
        html = HtmlResponse(url=second_grade_url[j], body=str(body))
        third_grade_name = []
        third_grade_url = []
        q = 1
        while (q):
            try:
                third_grade_name.append(
                    html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/text()').extract()[0])
                # print(html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/text()').extract()[0])
                third_grade_url.append(
                    html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/@href').extract()[0])
                # print(html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/@href').extract()[0])
                url_data = defaultdict()
                url_data['url'] = url + '/' + third_grade_url[q - 1]  # 把主页url和解析到的按三级目录url片段拼起来
                url_data['third_grade'] = third_grade_name[q - 1]
                url_data['second_grade'] = second_grade_name[j]
                url_data['first_grade'] = first_grade_name[2]
                url_data['created'] = int(time.time())
                url_data['updated'] = int(time.time())
                outline_data.append(url_data)
                q += 1
            except:
                break
    # 至此，所有的三级页面首页目录链接都存在了outline_data里
    return outline_data

def goodsUrlList(home_url):
    '''
    根据三级目录商品首页获取所有详情页url
    :param home_url: http://www.sssmro.com//category.php?id=1137&price_min=&price_max=
    :return:url列表
    '''
    # 该网站不需要条件选择即可抓到所有产品
    # 保存所有goods的详情页url
    url_list = []
    # 解析首页，拿到该三级目录下有几页网页
    body = getHtmlByRequests(home_url)
    html = HtmlResponse(url=home_url, body=str(body))
    # 先拿到该三级目录下的产品有几页
    num = int(html.xpath('/html/body/div[9]/div[2]/div[4]/form/ul/li[1]/text()').extract()[0][2:])
    # 然后抓取每一页下的产品的url
    for i in range(1, num+1):
        iter_url = home_url + '&page=' + str(i) + '&sort=last_update&order=DESC'
        body = getHtml(iter_url)
        html = HtmlResponse(url=iter_url, body=str(body))
        alist = html.xpath('/html/body/div[9]/div[2]/form/div/div/div/p[1]/a/@href').extract()
        print(len(alist))
        for url in alist:
            url_list.append('http://www.sssmro.com//' + url + '|1')
    return url_list

def goodsDetail(detail_url):
    '''
    利用xpath解析关键字段
    :param detail_url: 详情页url
    :return: 因为每个详情页面可能会产生多条数据，所以返回值是一个以dict为元素的list，其中每一个dict是一条数据
    '''
    # 因为前面为了去重加了'|1'，现在要去除之
    detail_url = detail_url.split('|')[0]
    # 解析网页
    body = getHtml(detail_url)
    html = HtmlResponse(url=detail_url, body=body)
    # 价格
    price_list = html.selector.xpath('//*[@id="relative_goods"]/tr/td[7]/text()').re(r'(\d+\.\d+)')
    # 型号
    type_list = html.selector.xpath('//*[@id="relative_goods"]/tr/td[2]/text()').extract()[1:]
    # 名称
    name = html.selector.xpath('//*[@id="spec-list"]/ul/li/img/@alt').extract()[0]
    # 详情，包含两个标签，一个div，一个p，都是html语句，两个用换行符'\n'隔开
    table_name = html.selector.xpath('//*[@id="sub11"]/div[1]/ul/li/div[1]/text()').extract()    # div
    table_param = html.selector.xpath('//*[@id="sub11"]/div[1]/ul/li/div[2]/text()').extract()
    print(table_param)
    exit()
    detailInfo1 = handleTable(table_name,table_param)
    try:
        detailInfo2 = html.selector.xpath('//*[@id="sub11"]/div[3]/p/text()').extract()   # p标签里的内容
        detailInfo2 = handle(detailInfo2)
    except:
        detailInfo2 = ''
    detailInfo = detailInfo1 + '\n' + detailInfo2

    # 图片
    pics = []
    for pic in html.selector.xpath('//*[@id="spec-list"]/ul/li/img'):
        # 去除图片尺寸,方法图片('//*[@id="spec-n1"]/img')
        pics.append( 'www.sssmro.com/'+ pic.xpath('@src').extract()[0])
    pics = '|'.join(pics)
    goodslist = []
    for i in range(len(price_list)):
        goodslist.append(defaultdict())<commit_msg>remove exit<commit_after># -*- coding:utf-8 -*-

"""
@version: 1.0
@author: marcovaldo
@license: Apache Licence
@contact: <EMAIL>
@site:
@software: PyCharm Community Edition
@file: __init__.py.py
@time: 16/11/26 下午15:45
"""
import sys
sys.path.append("..")
# from myfunc import *
from spiders.myfunc import *
from scrapy.http import HtmlResponse
from collections import defaultdict
import itertools
import time
import re
from bs4 import BeautifulSoup


def goodsOutline(url):
    '''
    获取三级目录详情
    :param url: 网站主页链接
    :return: [{'url':'','first_grade':'','second_grade': '', 'third_grade': ''},...{}]
    '''
    # 存储最后的三级列表页面
    outline_data = []
    # 解析页面
    body = getHtmlByRequests(url)
    html = HtmlResponse(url=url, body=str(body))
    soup = BeautifulSoup(body, 'lxml')      # xpath不好使就用beautisoup
    # 抓取一级目录名及链接，这里只要以下内容：
    # 泵阀门接头管件（不要化工泵）、紧固件密封件传动件（全要）、仪器仪表测量（只要气体检测仪，电力电工检测仪器）
    # 所以现在一级列表名称及链接不抓了，直接放到下面
    first_grade_name = ['紧固件密封件传动件', '泵阀门接头管件', '仪器仪表测量']
    first_grade_url = [url + '/category.php?id=207&amp;price_min=&amp;price_max=',
                       url + '/category.php?id=208&amp;price_min=&amp;price_max=',
                       url + '/category.php?id=191&amp;price_min=&amp;price_max=']
    # 这个for循环只处理了前两个一级目录，第三个里边只要两个二级目录
    for i in range(2):
        # 二级目录名及链接
        body = getHtmlByRequests(first_grade_url[i])
        html = HtmlResponse(url=first_grade_url[i], body=str(body))
        # soup = BeautifulSoup(body, 'lxml')
        # second_grade_name = soup.find_all()
        second_grade_name = []
        second_grade_url = []
        q = 1
        while(q):
            try:
                second_grade_name.append(html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/text()').extract()[0])
                second_grade_url.append(url + '/' + html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/@href').extract()[0])
                q += 1
            except:
                break
        # 在这个地方删掉不要的化工泵
        if second_grade_name[7] == u'\u5316\u5de5\u6cf5':
            del second_grade_name[7]
            del second_grade_url[7]
        for j in range(len(second_grade_name)):
            # 三级目录名及链接
            body =  getHtmlByRequests(second_grade_url[j])
            html = HtmlResponse(url=second_grade_url[j], body=str(body))
            third_grade_name = []
            third_grade_url = []
            q = 1
            while(q):
                try:
                    third_grade_name.append(html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/text()').extract()[0])
                    # print(html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/text()').extract()[0])
                    third_grade_url.append(html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/@href').extract()[0])
                    # print(html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/@href').extract()[0])
                    url_data = defaultdict()
                    url_data['url'] = url + '/' + third_grade_url[q-1]  # 把主页url和解析到的按三级目录url片段拼起来
                    url_data['third_grade'] = third_grade_name[q-1]
                    url_data['second_grade'] = second_grade_name[j]
                    url_data['first_grade'] = first_grade_name[i]
                    url_data['created'] = int(time.time())
                    url_data['updated'] = int(time.time())
                    outline_data.append(url_data)
                    q += 1
                except:
                    break
    # 下面抓第三个一级目录下的两个二级目录下的三级目录
    second_grade_name = ['气体检测仪', '电力电工检测仪器']
    second_grade_url = ['http://www.sssmro.com/category.php?id=702&price_min=&price_max=',
                        'http://www.sssmro.com/category.php?id=722&price_min=&price_max=']
    for j in range(len(second_grade_name)):
        # 三级目录名及链接
        body = getHtmlByRequests(second_grade_url[j])
        html = HtmlResponse(url=second_grade_url[j], body=str(body))
        third_grade_name = []
        third_grade_url = []
        q = 1
        while (q):
            try:
                third_grade_name.append(
                    html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/text()').extract()[0])
                # print(html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/text()').extract()[0])
                third_grade_url.append(
                    html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/@href').extract()[0])
                # print(html.xpath('/html/body/div[9]/div[2]/div[1]/dl/dd/ul/li[' + str(q) + ']/a/@href').extract()[0])
                url_data = defaultdict()
                url_data['url'] = url + '/' + third_grade_url[q - 1]  # 把主页url和解析到的按三级目录url片段拼起来
                url_data['third_grade'] = third_grade_name[q - 1]
                url_data['second_grade'] = second_grade_name[j]
                url_data['first_grade'] = first_grade_name[2]
                url_data['created'] = int(time.time())
                url_data['updated'] = int(time.time())
                outline_data.append(url_data)
                q += 1
            except:
                break
    # 至此，所有的三级页面首页目录链接都存在了outline_data里
    return outline_data

def goodsUrlList(home_url):
    '''
    根据三级目录商品首页获取所有详情页url
    :param home_url: http://www.sssmro.com//category.php?id=1137&price_min=&price_max=
    :return:url列表
    '''
    # 该网站不需要条件选择即可抓到所有产品
    # 保存所有goods的详情页url
    url_list = []
    # 解析首页，拿到该三级目录下有几页网页
    body = getHtmlByRequests(home_url)
    html = HtmlResponse(url=home_url, body=str(body))
    # 先拿到该三级目录下的产品有几页
    num = int(html.xpath('/html/body/div[9]/div[2]/div[4]/form/ul/li[1]/text()').extract()[0][2:])
    # 然后抓取每一页下的产品的url
    for i in range(1, num+1):
        iter_url = home_url + '&page=' + str(i) + '&sort=last_update&order=DESC'
        body = getHtml(iter_url)
        html = HtmlResponse(url=iter_url, body=str(body))
        alist = html.xpath('/html/body/div[9]/div[2]/form/div/div/div/p[1]/a/@href').extract()
        print(len(alist))
        for url in alist:
            url_list.append('http://www.sssmro.com//' + url + '|1')
    return url_list

def goodsDetail(detail_url):
    '''
    利用xpath解析关键字段
    :param detail_url: 详情页url
    :return: 因为每个详情页面可能会产生多条数据，所以返回值是一个以dict为元素的list，其中每一个dict是一条数据
    '''
    # 因为前面为了去重加了'|1'，现在要去除之
    detail_url = detail_url.split('|')[0]
    # 解析网页
    body = getHtml(detail_url)
    html = HtmlResponse(url=detail_url, body=body)
    # 价格
    price_list = html.selector.xpath('//*[@id="relative_goods"]/tr/td[7]/text()').re(r'(\d+\.\d+)')
    # 型号
    type_list = html.selector.xpath('//*[@id="relative_goods"]/tr/td[2]/text()').extract()[1:]
    # 名称
    name = html.selector.xpath('//*[@id="spec-list"]/ul/li/img/@alt').extract()[0]
    # 详情，包含两个标签，一个div，一个p，都是html语句，两个用换行符'\n'隔开
    table_name = html.selector.xpath('//*[@id="sub11"]/div[1]/ul/li/div[1]/text()').extract()    # div
    table_param = html.selector.xpath('//*[@id="sub11"]/div[1]/ul/li/div[2]/text()').extract()

    detailInfo1 = handleTable(table_name,table_param)
    try:
        detailInfo2 = html.selector.xpath('//*[@id="sub11"]/div[3]/p/text()').extract()   # p标签里的内容
        detailInfo2 = handle(detailInfo2)
    except:
        detailInfo2 = ''
    detailInfo = detailInfo1 + '\n' + detailInfo2

    # 图片
    pics = []
    for pic in html.selector.xpath('//*[@id="spec-list"]/ul/li/img'):
        # 去除图片尺寸,方法图片('//*[@id="spec-n1"]/img')
        pics.append( 'www.sssmro.com/'+ pic.xpath('@src').extract()[0])
    pics = '|'.join(pics)
    goodslist = []
    for i in range(len(price_list)):
        goodslist.append(defaultdict())
{'fraction_non_alphanumeric': 0.17855995889267134, 'fraction_numerical': 0.01721369387886184, 'mean_word_length': 2.4546261371200355, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 10, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3481816', 'n_tokens_mistral': 6437, 'n_tokens_neox': 6613, 'n_words': 698}