<jupyter_start><jupyter_text>下载PixivCrawler抓取到的图片<jupyter_code>import os
import time
import random
import pandas as pd
import sqlite3 as lite
from tqdm import tqdm_notebook as tqdm
from sqlalchemy import create_engine
from pixivpy3 import *


class PixivDownloader(object):

    def __init__(self, illust_db='pixiv_illusts.db'):
        self.illust_db = illust_db

    def DBIllusts(self, sql="SELECT * FROM illusts WHERE illust_id > 0"):
        with lite.connect(self.illust_db) as conn:
            sql_df = pd.read_sql_query(sql, conn, index_col='illust_id')

        # 还原json字段
        sql_df['image_urls'] = sql_df.image_urls.apply(json.loads)
        sql_df['meta_pages'] = sql_df.meta_pages.apply(json.loads)
        sql_df['meta_single_page'] = sql_df.meta_single_page.apply(json.loads)
        sql_df['series'] = sql_df.series.apply(json.loads)
        sql_df['tags'] = sql_df.tags.apply(json.loads)
        sql_df['tools'] = sql_df.tools.apply(json.loads)
        sql_df['user'] = sql_df.user.apply(json.loads)
        return sql_df

    def randSleep(self, base=0.1, rand=0.5):
        "休眠随机的时间"
        time.sleep(base + rand*random.random())

    def getImageUrl(self, illust):
        return illust['meta_single_page'].get('original_image_url',
                                              illust['image_urls']['large'])

    def StartDownload(self, path='./illusts'):
        if not os.path.exists(path):
            os.mkdir(path)

        api = AppPixivAPI(timeout=3)
        df = self.DBIllusts()
        for _, illust in tqdm(df.iterrows(), total=df.shape[0]):
            image_url = self.getImageUrl(illust)
            for i in range(3):
                try:
                    if api.download(image_url, path=path):
                        self.randSleep()
                    break
                except Exception as e:
                    print(">> Download %s failed: %s" % (image_url, e))


dl = PixivDownloader()
df = dl.StartDownload()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11711711711711711, 'fraction_numerical': 0.005005005005005005, 'mean_word_length': 2.4947552447552446, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '280541', 'license': 'Unlicense', 'n_tokens_mistral': 697, 'n_tokens_neox': 678, 'n_words': 123, 'path': 'notebooks/pixivDownloader.ipynb', 'repo_name': 'lkfo415579/pixivpy', 'size': '1998'}