<commit_before>#D:\Python\Python35\python.exe
# -*- coding:utf-8 -*-
import requests
from bs4 import BeautifulSoup
import re,traceback,os,csv
import

def getHTMLText(url):
    try:
        r=requests.get(url)
        r.raise_for_status()
        r.encoding=r.apparent_encoding
        return r.text
    except:
        print('获取页面异常:'+url)

def getStockList(lst,stockURL):
    html=getHTMLText(stockURL)
    sh_list=re.findall(r'<li><a target="_blank" href="http://quote.eastmoney.com/(sh[\d]{6}).html">(.*?)\([\d]*\)</a></li>',html)
    sz_list=re.findall(r'<li><a target="_blank" href="http://quote.eastmoney.com/(sz[\d]{6}).html">(.*?)\([\d]*\)</a></li>',html)
    lst.extend(sh_list)
    lst.extend(sz_list)

def getStockInfo(lst,stockURL,fpath):
    for stock in lst:
        stockURL=stockURL + stock[0] + '.html'
        html=getHTMLText(stockURL)

        # if not '雾霾太大' in html:
        #     pass
        # else:
        #     pass
        try:
            if html=='':
                continue
            infoDict={}
            soup=BeautifulSoup(html,'html.parser')
            stockInfo=soup.find('div',attrs={'class':'stock-bets'})

            name=stockInfo.find_all(attrs={'class':'stock-bets'})[0]
            infoDict.update({'股票名称':name.text.split()[0]})

            keylist=stockInfo.find_all('dt')
            valuelist=stockInfo.find_all('dd')
            for i in range(len(keylist)):
                key=keylist[i].text
                val=valuelist[i].text
                infoDict[key]=val

            #以字典形式写入CSV


        except:
            traceback.print_exc()<commit_msg>20170324_0019<commit_after>#D:\Python\Python35\python.exe
# -*- coding:utf-8 -*-
import requests
from bs4 import BeautifulSoup
import re,traceback,os,csv
import

def getHTMLText(url):
    try:
        r=requests.get(url)
        r.raise_for_status()
        r.encoding=r.apparent_encoding
        return r.text
    except:
        print('获取页面异常:'+url)

def getStockList(lst,stockURL):
    html=getHTMLText(stockURL)
    sh_list=re.findall(r'<li><a target="_blank" href="http://quote.eastmoney.com/(sh[\d]{6}).html">(.*?)\([\d]*\)</a></li>',html)
    sz_list=re.findall(r'<li><a target="_blank" href="http://quote.eastmoney.com/(sz[\d]{6}).html">(.*?)\([\d]*\)</a></li>',html)
    lst.extend(sh_list)
    lst.extend(sz_list)

def getStockInfo(lst,stockURL,fpath):
    for stock in lst:
        stockURL=stockURL + stock[0] + '.html'
        html=getHTMLText(stockURL)

        # if not '雾霾太大' in html:
        #     pass
        # else:
        #     pass
        try:
            if html=='':
                continue
            infoDict={}
            soup=BeautifulSoup(html,'html.parser')
            stockInfo=soup.find('div',attrs={'class':'stock-bets'})

            name=stockInfo.find_all(attrs={'class':'bets-name'})[0]
            infoDict.update({'股票名称':name.text.split()[0]})

            keylist=stockInfo.find_all('dt')
            valuelist=stockInfo.find_all('dd')
            for i in range(len(keylist)):
                key=keylist[i].text
                val=valuelist[i].text
                infoDict[key]=val

            #以字典形式写入CSV


        except:
            traceback.print_exc()
{'fraction_non_alphanumeric': 0.17096673919801056, 'fraction_numerical': 0.009325458501709667, 'mean_word_length': 2.3072970195272355, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3671301', 'n_tokens_mistral': 1171, 'n_tokens_neox': 1146, 'n_words': 141}