<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-26-261361188ae0>, line 3)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-26-261361188ae0>\"\u001b[0;36m, line \u001b[0;32m3\u001b[0m\n\u001b[0;31m    import '/Users/laiyao/Documents/成本/投放成本录入准确性核对/今日头条数据/ID-Name-Python.py'\u001b[0m\n\u001b[0m                                                                           ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "# def 各种必要环境\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import logging,os,sys,time\n",
    "from selenium import webdriver\n",
    "def start():\n",
    "\t##/初始化路径\n",
    "\tchromedriver=\"/Users/laiyao/Documents/GitHub/MyApp/PythonApplication1/爬虫练习/chromedriver/chromedriver-2\"\n",
    "\t##引入selenium，设置Chrome\n",
    "\tfrom selenium import webdriver\n",
    "\timport selenium\n",
    "\tos.environ[\"webdriver.chrome.driver\"] = chromedriver\n",
    "\toption = webdriver.ChromeOptions()#自定义设置\n",
    "\toption.add_argument('--user-agent=Mozilla/5.0 (Linux; U; Android 2.2.1; zh-cn; HTC_Wildfire_A3333 Build/FRG83D) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1') #修改浏览器的User-Agent来伪装你的浏览器\n",
    "\toption.add_argument('--process-per-site') #每个站点使用单独进程\n",
    "\toption.add_argument('--lang=zh-CN') #设置语言为简体中文\n",
    "\t##启动浏览器\n",
    "\tdriver = webdriver.Chrome(chromedriver,chrome_options=option)\n",
    "\tdriver.get('about:version')\n",
    "\treturn driver\n",
    "\n",
    "def openNewTab(driver,url='https://www.sogou.com'):\n",
    "    js='window.open(\"'+url+'\");'\n",
    "    driver.execute_script(js)\n",
    "\n",
    "def quit(driver):\n",
    "    driver.close()\n",
    "    driver.quit()\n",
    "    logging.info(['QUIT success'])\n",
    "    return\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 初始化数据储存：\n",
    "df=pd.DataFrame(columns =['日期','ID','账户名称','总支出','现金支出','赠款支出'] ,index=[0])\n",
    "df = df.append({'日期':'2000-01-01','ID':0,'账户名称':'涉及金额的账户需要除以100000','总支出':0,'现金支出':0,'赠款支出':0}, ignore_index=True)  #忽略索引,往dataframe中插入一行数据\n",
    "FailSubAccount={}\n",
    "def loginToutiao():\n",
    "    # 开浏览器窗口（三个窗口）\n",
    "    driver=start()\n",
    "    openNewTab(driver)\n",
    "    openNewTab(driver)\n",
    "    # 业务代码：登录（登录需要手工）\n",
    "    loginURL='https://ad.toutiao.com/pages/login/index.html'\n",
    "    driver.switch_to.window(driver.window_handles[0])\n",
    "    driver.get(loginURL)\n",
    "    return driver\n",
    "    \n",
    "def loginSubAccount(driver,subAccountID=0,name='默认名字'):\n",
    "    # 业务代码：登录子账户（自动）\n",
    "    subAccountURL='https://ad.toutiao.com/marco/account/account_cutover/?advertiser_id='\n",
    "    driver.switch_to.window(driver.window_handles[1])\n",
    "    driver.get(subAccountURL+str(subAccountID))\n",
    "    time.sleep(3)\n",
    "    SubAccountData=getSubAccountData(driver)\n",
    "    if SubAccountData==None:\n",
    "        global FailSubAccount\n",
    "        FailSubAccount[subAccountID]=name\n",
    "    for data in SubAccountData:\n",
    "        if not subAccountID ==data[\"advertiser_id\"]:\n",
    "            logging.error(jsonData)\n",
    "            logging.error('账户错误')\n",
    "        else:\n",
    "            global df\n",
    "            df = df.append({'日期':data[\"date\"],'ID':data[\"advertiser_id\"],'账户名称':name,'总支出':data[\"cost\"],'现金支出':data[\"cash_cost\"] ,'赠款支出':data[\"reward_cost\"]}, ignore_index=True)  #忽略索引,往dataframe中插入一行数据\n",
    "    pass\n",
    "\n",
    "def getSubAccountData(driver,start_date=\"2019-03-01\",end_date=\"2019-03-08\"):\n",
    "    dataURL='https://ad.toutiao.com/overture/cash/get_cash_flow/?page=1&start_date='+start_date+'&end_date='+end_date\n",
    "    driver.switch_to.window(driver.window_handles[2])\n",
    "    driver.get(dataURL)\n",
    "    # 去掉HTML里面乱七八糟的东西，拿到真实的JSON数据\n",
    "    htmlText=driver.page_source.replace('<html xmlns=\"http://www.w3.org/1999/xhtml\"><head></head><body><pre style=\"word-wrap: break-word; white-space: pre-wrap;\">','').replace('</pre></body></html>','')\n",
    "    if htmlText[0] =='{':\n",
    "        jsonData=json.loads(htmlText)\n",
    "    else:\n",
    "        logging.error(htmlText)\n",
    "        logging.error('浏览器获取数据失败')\n",
    "        return None\n",
    "    # 验证\"status\": \"success\"\n",
    "    if not jsonData[\"status\"] ==\"success\":\n",
    "        logging.error(jsonData)\n",
    "        logging.error('头条系统获取数据失败')\n",
    "    return jsonData['data'][\"items\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:20: DeprecationWarning: use options instead of chrome_options\n"
     ]
    }
   ],
   "source": [
    "# 业务执行（登录头条）\n",
    "driver=loginToutiao()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据源来自这个文件：\n",
    "# import '/Users/laiyao/Documents/成本/投放成本录入准确性核对/今日头条数据/ID-Name-Python.py'\n",
    "\n",
    "# 业务执行（拿数据）\n",
    "rundata=liangyuanSubAccount_ID_Name\n",
    "for k,v in rundata.items():\n",
    "    loginSubAccount(driver,k,v)\n",
    "    print(k,v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "#// 需要的数据在 jsonData['data'][\"items\"][0][\"cost\"]  //总支出\n",
    "#// 需要的数据在 jsonData['data'][\"items\"][0][\"cash_cost\"]  //现金支出\n",
    "#// 需要的数据在 jsonData['data'][\"items\"][0][\"reward_cost\"]  //赠款支出\n",
    "#// 需要的数据在 jsonData['data'][\"items\"][0][\"date\"]  //日期\n",
    "\n",
    "#// 验证字段： jsonData['data'][\"items\"][0][\"advertiser_id\"]  //子账户ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(path_or_buf='aa.csv')\n",
    "\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "quit(driver)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3<commit_msg>更新了一下爬虫代码，提高点效率<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def 各种必要环境\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import logging,os,sys,time\n",
    "logging.basicConfig(level=logging.WARNING, format='levelname:%(levelname)s filename: %(filename)s ，outputNumber: [%(lineno)d]  thread: %(threadName)s output msg:  %(message)s - %(asctime)s', datefmt='[%d/%b/%Y %H:%M:%S]')\n",
    "logging.info(\"开工干活\")\n",
    "\n",
    "from selenium import webdriver\n",
    "def start():\n",
    "\t##/初始化路径\n",
    "\tchromedriver=\"/Users/laiyao/Documents/GitHub/MyApp/PythonApplication1/爬虫练习/chromedriver/chromedriver-2\"\n",
    "\t##引入selenium，设置Chrome\n",
    "\tfrom selenium import webdriver\n",
    "\timport selenium\n",
    "\tos.environ[\"webdriver.chrome.driver\"] = chromedriver\n",
    "\toption = webdriver.ChromeOptions()#自定义设置\n",
    "\toption.add_argument('--user-data-dir=' + '/Users/laiyao/Library/Application Support/Google/Chrome/') #设置成用户自己的数据目录##注意退出当前的chrome,MAC 为~/Library/Application Support/Google/Chrome/Default\n",
    "\toption.add_argument('--user-agent=Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36') #修改浏览器的User-Agent来伪装你的浏览器\n",
    "\toption.add_argument('--process-per-site') #每个站点使用单独进程\n",
    "\toption.add_argument('--lang=zh-CN') #设置语言为简体中文\n",
    "\t##启动浏览器\n",
    "\tdriver = webdriver.Chrome(chromedriver,options=option)\n",
    "\tdriver.get('about:version')\n",
    "\treturn driver\n",
    "\n",
    "def openNewTab(driver,url='https://www.sogou.com'):\n",
    "    js='window.open(\"'+url+'\");'\n",
    "    driver.execute_script(js)\n",
    "\n",
    "def quit(driver):\n",
    "    driver.quit()\n",
    "    logging.info(['QUIT success'])\n",
    "    return\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 初始化数据储存：\n",
    "df=pd.DataFrame(columns =['日期','ID','账户名称','总支出','现金支出','赠款支出'] ,index=[0])\n",
    "df = df.append({'日期':'2000-01-01','ID':0,'账户名称':'涉及金额的账户需要除以100000','总支出':0,'现金支出':0,'赠款支出':0}, ignore_index=True)  #忽略索引,往dataframe中插入一行数据\n",
    "FailSubAccount={}\n",
    "def loginToutiao():\n",
    "    # 开浏览器窗口（三个窗口）\n",
    "    driver=start()\n",
    "    openNewTab(driver)\n",
    "    openNewTab(driver)\n",
    "    # 业务代码：登录（登录需要手工）\n",
    "    loginURL='https://ad.toutiao.com/pages/login/index.html?advertiser_id='\n",
    "    driver.switch_to.window(driver.window_handles[0])\n",
    "    driver.get(loginURL)\n",
    "    return driver\n",
    "    \n",
    "def loginSubAccount(driver,subAccountID=0,name='默认名字'):\n",
    "    # 业务代码：登录子账户（自动）\n",
    "    subAccountURL='https://ad.toutiao.com/marco/account/account_cutover/?advertiser_id='\n",
    "    # driver.switch_to.window(driver.window_handles[1])\n",
    "    driver.get(subAccountURL+str(subAccountID))\n",
    "    time.sleep(4)\n",
    "    subAccountURL='https://ad.toutiao.com/overture/index/?advertiser_id='\n",
    "    driver.get(subAccountURL+str(subAccountID))\n",
    "    time.sleep(0.6)\n",
    "    pass\n",
    "\n",
    "def saveData(SubAccountData=None,subAccountID=0,name='默认名字'):\n",
    "    if SubAccountData==None:\n",
    "        global FailSubAccount\n",
    "        FailSubAccount[subAccountID]=name\n",
    "        logging.error('当前失败账户：'+name + subAccountURL)\n",
    "        return None\n",
    "    for data in SubAccountData:\n",
    "        if not subAccountID ==data[\"advertiser_id\"]:\n",
    "            logging.error(jsonData)\n",
    "            logging.error('账户错误')\n",
    "            FailSubAccount[subAccountID]=name\n",
    "        else:\n",
    "            global df\n",
    "            df = df.append({'日期':data[\"date\"],'ID':data[\"advertiser_id\"],'账户名称':name,'总支出':data[\"cost\"],'现金支出':data[\"cash_cost\"] ,'赠款支出':data[\"reward_cost\"]}, ignore_index=True)  #忽略索引,往dataframe中插入一行数据\n",
    "    pass\n",
    "\n",
    "def getSubAccountData(driver,start_date=\"2019-03-01\",end_date=time.strftime(\"%Y-%m-%d\", time.localtime())):\n",
    "    dataURL='https://ad.toutiao.com/overture/cash/get_cash_flow/?page=1&start_date='+start_date+'&end_date='+end_date\n",
    "    # driver.switch_to.window(driver.window_handles[2])\n",
    "    driver.get(dataURL)\n",
    "    time.sleep(0.6)\n",
    "    # 去掉HTML里面乱七八糟的东西，拿到真实的JSON数据\n",
    "    htmlText=driver.page_source.replace('<html xmlns=\"http://www.w3.org/1999/xhtml\"><head></head><body><pre style=\"word-wrap: break-word; white-space: pre-wrap;\">','').replace('</pre></body></html>','')\n",
    "    if htmlText[0] =='{':\n",
    "        jsonData=json.loads(htmlText)\n",
    "    else:\n",
    "        logging.error(htmlText)\n",
    "        logging.error('浏览器获取数据失败')\n",
    "        return None\n",
    "    # 验证\"status\": \"success\"\n",
    "    if not jsonData[\"status\"] ==\"success\":\n",
    "        logging.error(jsonData)\n",
    "        logging.error('头条系统获取数据失败')\n",
    "        return None\n",
    "    return jsonData['data'][\"items\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 业务执行（登录头条），记得要下载汇总数据\n",
    "driver=loginToutiao()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据源来自这个文件：\n",
    "# import '/Users/laiyao/Documents/成本/投放成本录入准确性核对/今日头条数据/ID-Name-Python.py'\n",
    "# h核心数据不能乱传\n",
    "\n",
    "sys.path.append('/Users/laiyao/Documents/成本/头条-投放成本核对/今日头条数据/')\n",
    "import id_Name_Python\n",
    "\n",
    "# 下载汇总数据\n",
    "driver.switch_to.window(driver.window_handles[1])\n",
    "dataURL='https://ad.toutiao.com/marco/account/get_majordomo_binded_account_stat_info/?start_time=2019-01-01&end_time='+time.strftime(\"%Y-%m-%d\", time.localtime())+'&action=download&single=1'\n",
    "driver.get(dataURL)\n",
    "\n",
    "# 业务执行（拿数据） 上海鱼元yuyuanSubAccount_ID_Name，南京大众dazhongSubAccount_ID_Name，量元liangyuanSubAccount_ID_Name\n",
    "rundata=id_Name_Python.yuyuanSubAccount_ID_Name\n",
    "for k,v in rundata.items():\n",
    "    print(\"正在抓取：\",k,v, end=\"  \")\n",
    "    loginSubAccount(driver,k,v)\n",
    "    saveData(getSubAccountData(driver,start_date=\"2019-01-01\",end_date=\"2019-01-15\"),k,v)\n",
    "    saveData(getSubAccountData(driver,start_date=\"2019-01-16\",end_date=\"2019-02-01\"),k,v)\n",
    "    saveData(getSubAccountData(driver,start_date=\"2019-02-02\",end_date=\"2019-02-14\"),k,v)\n",
    "    saveData(getSubAccountData(driver,start_date=\"2019-02-15\",end_date=\"2019-03-01\"),k,v)\n",
    "    saveData(getSubAccountData(driver,start_date=\"2019-03-02\"),k,v)\n",
    "    print(\"抓取完成：\",k,v)\n",
    "\n",
    "\n",
    "print('抓取数据完成：' , v[0:4], FailSubAccount,\"抓取过程貌似无异常\" if not FailSubAccount else \"抓取过程中异常数据为：\" + str(FailSubAccount))\n",
    "\n",
    "# 运行一下这个save 数据\n",
    "df.to_csv(path_or_buf=v[0:4] + time.strftime(\"%Y-%m-%d\", time.localtime()) +'.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "#// 需要的数据在 jsonData['data'][\"items\"][0][\"cost\"]  //总支出\n",
    "#// 需要的数据在 jsonData['data'][\"items\"][0][\"cash_cost\"]  //现金支出\n",
    "#// 需要的数据在 jsonData['data'][\"items\"][0][\"reward_cost\"]  //赠款支出\n",
    "#// 需要的数据在 jsonData['data'][\"items\"][0][\"date\"]  //日期\n",
    "\n",
    "#// 验证字段： jsonData['data'][\"items\"][0][\"advertiser_id\"]  //子账户ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 运行这个退出浏览器\n",
    "quit(driver)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
{'fraction_non_alphanumeric': 0.2335791111411757, 'fraction_numerical': 0.023878779679361428, 'mean_word_length': 2.967793880837359, 'pattern_counts': {'":': 92, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 10, 'lorem ipsum': 0, 'www.': 4, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6645407', 'n_tokens_mistral': 6614, 'n_tokens_neox': 6597, 'n_words': 616}