<commit_before># -*- coding: utf-8 -*-
#网络爬虫0.6版
# Python3.4.1

from tkinter import *   #Tk模块
from tkinter.scrolledtext import *
import urllib.request   #网络模块
import re               #正则表达式模块
import codecs           #文件编码转换
import sys
import tkinter.font
import tkinter.messagebox

###############################################################
#窗口模块开始

def close_window(window):
    """关闭窗口时给出提示"""
    if tkinter.messagebox.askyesno("关闭窗口", "真的要退出吗？(是/否)", icon="question"):
        window.destroy() 


class App(Frame):
    """构建窗口程序
    """
    def __init__(self, master=None):
        Frame.__init__(self, master)
        self.pack()
        self.createWidgets()

    def createWidgets(self):
        #创建一个标签，设置文字及背景属性
        self.label = Label(self, text='网络爬虫（股票抓取）', font='Helvetica -24 bold', fg='snow', bg='tomato')
        #将标签放置在窗口中的位置
        self.label.pack(fill='none', expand=1, ipadx=500, ipady=20, padx=1, pady=1, side='top')

        #创建网址前部文本输入框
        self.label1 = Label(text='输入网页抓取网址前部分：', font='Helvetica -15 bold')
        self.label1.pack(fill='none', expand=1, ipadx=400,side='top')
        self.text1=Entry()
        self.text1.pack(fill='none', expand=1, ipadx=100, ipady=5, padx=5, pady=5, side='top')
        self.contents = StringVar()
        self.contents.set("")
        self.text1["textvariable"] = self.contents
        self.text1.config(font="Arial 10 bold")

        #创建数字文本输入框
        self.label2 = Label(text='输入起始序号：', font='Helvetica -15 bold')
        self.label2.pack(fill='none', expand=1, ipadx=400,side='top')
        self.text2=Entry()
        self.text2.pack(fill='none', expand=1, ipadx=100, ipady=5, padx=5, pady=5, side='top')
        self.contents2 = StringVar()
        self.contents2.set("")
        self.text2["textvariable"] = self.contents2
        self.text2.config(font="Arial 10 bold")

        self.label3 = Label(text='输入结束序号：', font='Helvetica -15 bold')
        self.label3.pack(fill='none', expand=1, ipadx=400,side='top')
        self.text3=Entry()
        self.text3.pack(fill='none', expand=1, ipadx=100, ipady=5, padx=5, pady=5, side='top')
        self.contents3 = StringVar()
        self.contents3.set("")
        self.text3["textvariable"] = self.contents3
        self.text3.config(font="Arial 10 bold")

        #创建执行结果输出文本框
        self.textout= ScrolledText(wrap = WORD,height=10)
        self.textout.pack(fill='none', expand=1, padx=5, pady=5, side='top')
             
        #创建一个按钮，点击时运行网络爬虫
        self.hi_there = Button()
        self.hi_there['text'] = "点击股票抓取"
        self.hi_there['command'] = self.do_stock
        self.hi_there.pack(ipadx=50, ipady=10, padx=20, pady=10, side='left')

        #创建退出按钮
        self.QUIT = Button(text='QUIT', fg='red', command=lambda : close_window(root))
        self.QUIT.pack(ipadx=60, ipady=10, padx=20, pady=10, side='right')

    def do_stock(self):
        
        #运行网络爬虫
        spider_go(str(self.contents.get()), str(self.contents2.get()), str(self.contents3.get()))

        #可选，删除之前已获取股票数据中的重复行
        remove_r('new.txt','old.txt','save.txt')

        #可选，功能测试，在窗口中显示爬虫抓取后并去重的数据
        try:
            with open('f:/temp/stock/save.txt', 'r') as f:
                textoutstr = str(f.read())

            self.textout.insert(INSERT, textoutstr)
        except FileNotFoundError:
            print("FileNotFoundError!", end='')

#窗口模块结束

##############################################################
##网络爬虫模块开始
    
def get_stock_info(url):
    """抓取网页，利用正则表达式匹配股票信息。
    网页请求头部为Chrome信息，网页编码为gb2312。
    若网页中股票代码存在，返回股票信息。
    """
    user_agent = 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.6 Safari/537.36'
    req = urllib.request.Request(url)
    req.add_header('User-Agent', user_agent)

    response = urllib.request.urlopen(req)  #抓取网页      
    text = response.read().decode('gb2312', 'ignore')
    
    stock_id = re.findall(r'\d+\.S[ZH]', str(text))                                             #正则匹配股票代码
    price_inid = re.findall(r'<span id="ctl04_lbSpj">(\d+\.\d+)</span>', str(text))             #正则匹配初始价
    price_target = re.findall(r'<span id="ctl04_txTgtPrice">(\d+\.\d+)</span>', str(text))      #正则匹配目标价
    grade = re.findall(r'<span id="ctl04_bgpj">([\u4e00-\u9fa5]+)</span>', str(text))           #正则匹配评级
    trade = re.findall(r'<span id="ctl04_lbHylbmc">([\u4e00-\u9fa5]+)</span>', str(text))       #正则匹配行业
    trade = trade if len(trade) != 0 else '无'
    stock_date = re.findall(r'<span id="ctl04_lblzhxgrq" style="display:inline-block;width:90%;">(\d+\-\d+\-\d+)</span>', str(text))   #正则匹配日期
    
    if len(stock_id) != 0:
        name = stock_id[0]
        sname = '1' + name[0:6] if name[-1] == 'Z' else '0' + name[0:6]
        return '  ["{0}", "{1}", "{2}", "{3}",  "{4}", "{5}"],\n'.format(sname, trade[0], stock_date[0], price_inid[0], price_target[0], grade[0])
    else:
        return ''

def write_stock_file(mystr, file_name):
    """以写入方式打开文件，写入字符串mystr并自动关闭文件。
    """
    with open(file_name, 'a') as f:
        f.write(str(mystr))


def spider_go(url_head, start, end):<commit_msg>自动检测网页编码<commit_after># -*- coding: utf-8 -*-
#网络爬虫0.6版
# Python3.4.1

from tkinter import *   #Tk模块
from tkinter.scrolledtext import *
import urllib.request   #网络模块
import re               #正则表达式模块
import codecs           #文件编码转换
import chardet          #检测网页编码模块
import sys
import tkinter.font
import tkinter.messagebox

###############################################################
#窗口模块开始

def close_window(window):
    """关闭窗口时给出提示"""
    if tkinter.messagebox.askyesno("关闭窗口", "真的要退出吗？(是/否)", icon="question"):
        window.destroy() 


class App(Frame):
    """构建窗口程序
    """
    def __init__(self, master=None):
        Frame.__init__(self, master)
        self.pack()
        self.createWidgets()

    def createWidgets(self):
        #创建一个标签，设置文字及背景属性
        self.label = Label(self, text='网络爬虫（股票抓取）', font='Helvetica -24 bold', fg='snow', bg='tomato')
        #将标签放置在窗口中的位置
        self.label.pack(fill='none', expand=1, ipadx=500, ipady=20, padx=1, pady=1, side='top')

        #创建网址前部文本输入框
        self.label1 = Label(text='输入网页抓取网址前部分：', font='Helvetica -15 bold')
        self.label1.pack(fill='none', expand=1, ipadx=400,side='top')
        self.text1=Entry()
        self.text1.pack(fill='none', expand=1, ipadx=100, ipady=5, padx=5, pady=5, side='top')
        self.contents = StringVar()
        self.contents.set("")
        self.text1["textvariable"] = self.contents
        self.text1.config(font="Arial 10 bold")

        #创建数字文本输入框
        self.label2 = Label(text='输入起始序号：', font='Helvetica -15 bold')
        self.label2.pack(fill='none', expand=1, ipadx=400,side='top')
        self.text2=Entry()
        self.text2.pack(fill='none', expand=1, ipadx=100, ipady=5, padx=5, pady=5, side='top')
        self.contents2 = StringVar()
        self.contents2.set("")
        self.text2["textvariable"] = self.contents2
        self.text2.config(font="Arial 10 bold")

        self.label3 = Label(text='输入结束序号：', font='Helvetica -15 bold')
        self.label3.pack(fill='none', expand=1, ipadx=400,side='top')
        self.text3=Entry()
        self.text3.pack(fill='none', expand=1, ipadx=100, ipady=5, padx=5, pady=5, side='top')
        self.contents3 = StringVar()
        self.contents3.set("")
        self.text3["textvariable"] = self.contents3
        self.text3.config(font="Arial 10 bold")

        #创建执行结果输出文本框
        self.textout= ScrolledText(wrap = WORD,height=10)
        self.textout.pack(fill='none', expand=1, padx=5, pady=5, side='top')
             
        #创建一个按钮，点击时运行网络爬虫
        self.hi_there = Button()
        self.hi_there['text'] = "点击股票抓取"
        self.hi_there['command'] = self.do_stock
        self.hi_there.pack(ipadx=50, ipady=10, padx=20, pady=10, side='left')

        #创建退出按钮
        self.QUIT = Button(text='QUIT', fg='red', command=lambda : close_window(root))
        self.QUIT.pack(ipadx=60, ipady=10, padx=20, pady=10, side='right')

    def do_stock(self):
        
        #运行网络爬虫
        spider_go(str(self.contents.get()), str(self.contents2.get()), str(self.contents3.get()))

        #可选，删除之前已获取股票数据中的重复行
        remove_r('new.txt','old.txt','save.txt')

        #可选，功能测试，在窗口中显示爬虫抓取后并去重的数据
        try:
            with open('f:/temp/stock/save.txt', 'r') as f:
                textoutstr = str(f.read())

            self.textout.insert(INSERT, textoutstr)
        except FileNotFoundError:
            print("FileNotFoundError!", end='')

#窗口模块结束

##############################################################
##网络爬虫模块开始
    
def get_stock_info(url):
    """抓取网页，利用正则表达式匹配股票信息。
    网页请求头部为Chrome信息，网页编码为gb2312。
    若网页中股票代码存在，返回股票信息。
    """
    user_agent = 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.6 Safari/537.36'
    req = urllib.request.Request(url)
    req.add_header('User-Agent', user_agent)

    data = urllib.request.urlopen(req).read()  #抓取网页
    html_code = chardet.detect(data)           #通过chardet模块检测编码
    text = data.decode(html_code['encoding'])
    
    stock_id = re.findall(r'\d+\.S[ZH]', str(text))                                             #正则匹配股票代码
    price_inid = re.findall(r'<span id="ctl04_lbSpj">(\d+\.\d+)</span>', str(text))             #正则匹配初始价
    price_target = re.findall(r'<span id="ctl04_txTgtPrice">(\d+\.\d+)</span>', str(text))      #正则匹配目标价
    grade = re.findall(r'<span id="ctl04_bgpj">([\u4e00-\u9fa5]+)</span>', str(text))           #正则匹配评级
    trade = re.findall(r'<span id="ctl04_lbHylbmc">([\u4e00-\u9fa5]+)</span>', str(text))       #正则匹配行业
    trade = trade if len(trade) != 0 else '无'
    stock_date = re.findall(r'<span id="ctl04_lblzhxgrq" style="display:inline-block;width:90%;">(\d+\-\d+\-\d+)</span>', str(text))   #正则匹配日期
    
    if len(stock_id) != 0:
        name = stock_id[0]
        sname = '1' + name[0:6] if name[-1] == 'Z' else '0' + name[0:6]
        return '  ["{0}", "{1}", "{2}", "{3}",  "{4}",  "{5}"],\n'.format(sname, trade[0], stock_date[0], price_inid[0], price_target[0], grade[0])
    else:
        return ''

def write_stock_file(mystr, file_name):
    """以写入方式打开文件，写入字符串mystr并自动关闭文件。
    """
    with open(file_name, 'a') as f:
        f.write(str(mystr))


def spider_go(url_head, start, end):
{'fraction_non_alphanumeric': 0.19573436742607853, 'fraction_numerical': 0.0347067377605429, 'mean_word_length': 2.9434250764525993, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '65214', 'n_tokens_mistral': 4389, 'n_tokens_neox': 4503, 'n_words': 606}