<jupyter_start><jupyter_text>政府网站公开数据爬取  爬取漳州历史天气数据<jupyter_code>import requests
header = {
    'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36',
}
res_list = []
for year in range(2020,2022):
    for month in range(1,13):
        url = 'https://lishi.tianqi.com/zhangzhou/{}{:0>2d}.html'.format(year,month)
        # print(url)
        res = requests.get(url=url,headers=header)
        res_list.append(res)

from lxml import etree

# 2020年每天的天气情况
day = 1
csv_str = ""
for month in range(1,13):
    dom = etree.HTML(res_list[month].text)
    values = dom.xpath('/html/body/div[7]/div[1]/div[4]/ul/li')
    for one in values:
        high_temp = int(one.xpath('./div[2]/text()')[0][0:-1])
        low_temp = int(one.xpath('./div[3]/text()')[0][0:-1])
        weather = one.xpath('./div[4]/text()')[0]

        # print(day,high_temp,low_temp,weather)
        line = "{},{},{},{}\n".format(day,high_temp,low_temp,weather)
        csv_str = csv_str + line
        day += 1

head_str = "day,high,low,weather\n"
csv_str = head_str + csv_str
# print(csv_str)
with open('weather.csv','w',encoding='utf-8') as f:
    f.write(csv_str)<jupyter_output><empty_output><jupyter_text>福建爬取福建省生态环保厅的数据<jupyter_code>import requests
import time

header = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36'}
url = 'http://sthjt.fujian.gov.cn/was5/web/search'
params = {
    'channelid':280067,
    'sortfield':'-s4',
    'classsql':'(dockind=10)',
    'r':'0.3624881561901028',
    'prepage':100,
    'page':1
}
res_list = []
for i in range(1,177):
    params['page'] = i
    res = requests.get(url=url,params=params,headers=header)
    res_list.append(res)
    time.sleep(0.1)

print('done')


import json
docs_list = []
for one in res_list:
    doc = json.loads(one.text.replace('\r','').replace('\n',''))
    docs_list = docs_list + doc['docs']
print('done 1')


csv_str = ""
for i in range(len(docs_list)):
    one = docs_list[i]
    try:
        line = "{},{},{},{},{},{},{},{},{},{},{},{},{}\n".format(one['s1'] ,one['s2'],one['s3'],
                                           one['s4'],one['s5'],one['s6'],
                                           one['s7'],one['f1'],one['f2'],
                                           one['f3'],one['f4'],one['f5'],one['f6'],)
    except Exception as e:
        # print(e)
        # print(i)
        continue
    csv_str = csv_str + line
    # if i  == 2:
    #     break

# print(csv_str)
print('done 2')


head_str = '水系,点位名称,断面名称,年,周,起始时间,结束时间,pH,DO,COD,TP,氨氮,总氮\n'
csv_str = head_str + csv_str
with open('water.csv','w',encoding='utf-8') as f:
    f.write(csv_str)

print('done')
import pandas as pd
import numpy as np

fujian_df = pd.read_csv('data/network/gov/fujian/fujian.csv')

print(fujian_df.describe())

fujian_df['年'] = fujian_df['年'].astype(int)

for year in range(2004,2022):
    year_df = fujian_df[(fujian_df['年'] >= year) & (fujian_df['年'] < year + 1) ]
    print("{}年记录总数:{}".format(year,len(year_df)))
year_df = fujian_df[(fujian_df['年'] >= 2017) & (fujian_df['年'] < 2018) ]
print("记录总数:{}".format(len(year_df)))
site_list = year_df['断面名称'].unique()
print("站点总数：{}".format(len(site_list)))
for site in site_list:
    one = year_df[year_df['断面名称'] ==  site]
    print("{},{}".format(site,len(one)))<jupyter_output>记录总数:3520
站点总数：58
姑田溪(龙岩-三明交界断面),55
尤溪(三明－南平交界断面),55
干流(对照),55
翠江河(宁化-清流交界断面),55
干流(周宁-福安交界断面),55
饮用水水源地,330
水库、湖泊,110
富屯溪(三明-南平交界断面),55
黄潭河(上杭-永定交界断面),55
西溪(龙岩-漳州交界断面),55
干流(上杭湖洋乡涧头村),55
干流(寿宁-福安交界断面),55
九龙溪(清流-永安交界断面),55
沙溪(三明-南平交界断面),55
富屯溪(邵武-顺昌交界断面),55
北溪(控制断面),55
西溪(安溪-南安交界断面),55
濉溪(建宁-泰宁交界断面),55
黄潭河(新罗-上杭交界断面),55
鱼塘溪(明溪-三元交界断面),55
建溪(武夷山-建阳交界断面),55
干流(宁德-福州交界断面),55
干流(仙游-城厢交界断面),55
北团溪(龙岩-三明交界断面),55
北溪 (龙岩－漳州交界断面),55
干流(宁德－福州交界断面),55
干流(南安－丰泽交界断面),55
北溪(新罗-漳平交界断面),55
北溪 (华安－芗城交界断面),55
北溪(厦门－漳州交界断面),55
桃溪(永春-南安交界断面),55
干流(罗源－连江交界断面),55
干流(屏南-蕉城交界断面),55
沙溪(永安-三元交界断面),55
沙溪(梅列-沙县交界断面),55
金溪(泰宁-将乐交界断面),55
富屯溪(光泽-邵武交界断面),55
富屯溪(顺昌-延平交界断面),55
建溪(建阳-建瓯交界断面),55
省界(浙－闽),55
建溪(政和－建瓯交界断面),55
均溪(大田－尤溪交界断面),55
干流(南平-宁德交界断面),55
建溪(浦城－建阳交界断面),55
建溪(松溪－政和交界断面),55
建溪(建瓯－延平交界断面),55
干流(闽侯-福州交界断面),55
大樟溪(泉州-福州交界断面),55
干流(闽江入海口),55
北溪(长泰-龙文交界断面),55
西溪(平和-南靖交界断面),55
西溪(南靖-芗城交界断面),55
干流(长汀-上杭交界断面),55
干流(上杭-永定交界断面),55
文川河(龙岩-三明交界断面),55
干流(连江-马尾交界断面),55
大樟溪(永泰-闽侯交界断面),55
干流(闽清-闽侯交界断面),55<jupyter_text>山东爬取山东省生态环保厅的数据http://sthj.shandong.gov.cn/<jupyter_code>import requests
import json

url = 'http://fb.sdem.org.cn:8801/wryfb/ajax/map.ashx'
param = {
    'Method':'SelectSubList',
    'stcode':'0',
    'type':'WasteWaterGis',
    'isall':'0'
}
header = {
    'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36',
    # 'Referer':'http://fb.sdem.org.cn:8801/wryfb/WebGis/WasteWaterGis/HistoryData_New.aspx?SubId=29941%20&SubName=%u79D1%u6CD3%u79D1%u6280%u96C6%u56E2%u6709%u9650%u516C%u53F8',
    # 'X-Requested-With':'XMLHttpRequest',
    # 'Origin':'http://fb.sdem.org.cn:8801',
    # 'Pragma':'no-cache',
    # 'Cookie':'ASP.NET_SessionId=y5zc211bfydoziryjxzimesx; ASP.NET_SessionId_NS_Sig=oenCV6md0Dtq6Bby',
    # 'Content-Type':'application/x-www-form-urlencoded; charset=UTF-8'
}
res = requests.post(url=url, data=param, headers=header)
site_list = json.loads(res.text)['items']
print("站点总数：{}".format(len(site_list)))

import time

url = 'http://fb.sdem.org.cn:8801/wryfb/ajax/WasteWaterGis/WasteWaterHandler.ashx'
param = {
    'Method':'GetHisChart_New',
    'strID':'2326',
    'strTime':'2021-07-01'
}
header = {
    'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36',
}
month_data_list = []
for year in range(2010,2021):
    print("爬取{}年".format(year))
    for month in range(1,13):
        str_time = '{}-{}-01'.format(year,month)
        param['strTime'] = str_time

        month_res = requests.post(url=url, data=param, headers=header)
        month_data = json.loads(month_res.text)
        month_data_list.append(month_data)

        time.sleep(0.05)
        # break
    # break

# print(res.text)
# doc = json.loads(res.text)
# print(doc)<jupyter_output>爬取2010年
爬取2011年
爬取2012年
爬取2013年
爬取2014年
爬取2015年
爬取2016年
爬取2017年
爬取2018年
爬取2019年
爬取2020年<jupyter_text>中国环境监测总站  水质自动检测周报http://www.cnemc.cn/sssj/szzdjczb/<jupyter_code>import requests
import time

page_list = []
for i in range(25):

    if i == 0:
        page = requests.get(url='http://www.cnemc.cn/sssj/szzdjczb/index.shtml')
    else:
        url = 'http://www.cnemc.cn/sssj/szzdjczb/index_{}.shtml'.format(i)
        page = requests.get(url=url)
    time.sleep(0.05)
    page_list.append(page)
print('done')
# print(page.text)
from lxml import etree
import requests
import time

url_base = 'http://www.cnemc.cn/sssj/szzdjczb'

for page in page_list:
    dom = etree.HTML(page.text)
    for i in range(1,21):
        xpath_str = '//*[@id="contentPageData"]/li[{}]/a/@href'.format(i)
        line = dom.xpath(xpath_str)[0]
        if line.split('.')[-1] == 'shtml':
            url = url_base + line[1:]
            page =  requests.get(url=url)

            # 获取下载页
            doc_page = etree.HTML(page.text)
            # 获取所有a标签的链接
            all_link = doc_page.xpath('//a/@href')
            found = False
            for link in all_link:
                last_str = link.split('.')[-1]
                if (last_str == 'doc' or last_str == 'pdf') and link[:2] == './':
                    found = True
                    # 获取下载链接
                    download_link = "/".join(url.split("/")[:-1]) + link[1:]
                    doc_file = requests.get(url=download_link)

                    # 下载保存
                    dir = 'data/network/gov/cnemc/'
                    filename = dir + link[2:]
                    with open(filename,'wb') as f:
                        f.write(doc_file.content)
                    # print(download_link)
            if not found:
                print('{}没有找到下载链接'.format(url))

            time.sleep(0.05)
            # break
        else:
            print('{}可以直接下载'.format(line))
    # break


import os #用于获取目标文件所在路径
path="E:\\project\\mvp\\Graph-WaveNet\\data\\network\\gov\\cnemc\\" # 文件夹绝对路径
files=[]
for file in os.listdir(path):
    if file.endswith(".doc"): #排除文件夹内的其它干扰文件，只获取".doc"后缀的word文件
        files.append(path+file)

from win32com import client as wc #导入模块
word = wc.Dispatch("Word.Application") # 打开word应用程序
for file in files:
    doc = word.Documents.Open(file) #打开word文件
    doc.SaveAs("{}x".format(file), 12)#另存为后缀为".docx"的文件，其中参数12指docx文件
    doc.Close() #关闭原来word文件
word.Quit()
print("完成！")
# import docx
from docx import Document

docFile = 'data/network/gov/cnemc/2018-51.docx'
document = Document(docFile) #读入文件
table = document.tables[0] #获取文件中的表格集

table.cell(2,0).text<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18749299562927266, 'fraction_numerical': 0.05771601479323098, 'mean_word_length': 3.3090294543698695, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 5, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '420284', 'license': 'MIT', 'n_tokens_mistral': 4576, 'n_tokens_neox': 4557, 'n_words': 527, 'path': 'datasetHandle/gov/spider.ipynb', 'repo_name': 'LeonardYoung/Graph-WaveNet', 'size': '8923'}