<commit_before>#!/usr/bin/env python3
# coding=utf-8
import os
import sys
import re
import copy
import zlib
import sched
import queue
import base64
import random
import traceback
import ipaddress
import threading

from fnmatch import fnmatch
from time import time, sleep, process_time
from html import escape as html_escape
from datetime import datetime, timedelta
from urllib.parse import urljoin, urlsplit, urlunsplit, quote_plus
import urllib.parse
import requests
from flask import Flask, request, make_response, Response, redirect
from . import CONSTS

try:
    # for python 3.5+ Type Hint
    from typing import Union, List, Any, Tuple
except:
    pass
try:  # 用于检测html的文本编码, cchardet是chardet的c语言实现, 非常快
    from cchardet import detect as c_chardet
except:
    cchardet_available = False
else:
    cchardet_available = True

if os.path.abspath(os.getcwd()) != CONSTS.ZMIRROR_ROOT:
    os.chdir(CONSTS.ZMIRROR_ROOT)

from .external_pkgs.ColorfulPyPrint import *  # TODO: Migrate logging tools to the stdlib logging

if "ZMIRROR_UNITTEST" in os.environ:
    # 这边根据环境变量得到的unittest_mode信息会被config中的覆盖掉
    # 只是因为此时还没有加载 config, 所以先根据env里的临时定一下
    unittest_mode = True
else:
    unittest_mode = False

try:  # lru_cache的c语言实现, 比Python内置lru_cache更快
    from fastcache import lru_cache  # lru_cache用于缓存函数的执行结果
except:
    from functools import lru_cache

    warnprint('package fastcache not found, '
              'fallback to stdlib lru_cache, '
              'no FUNCTION is effected, only maybe a bit slower. '
              'Considering install it using "pip3 install fastcache"'
              )
else:
    if not unittest_mode:
        infoprint('lru_cache loaded successfully from fastcache')

from .threadlocal import ZmirrorThreadLocal

if not unittest_mode:  # 在unittest时不输出这几行
    infoprint('zmirror version: {version} author: {author}'.format(version=CONSTS.__VERSION__, author=CONSTS.__AUTHOR__))
    infoprint('Github: {site_url}'.format(site_url=CONSTS.__GITHUB_URL__))

try:  # 加载默认设置
    from config_default import *
except:  # coverage: exclude
    errprint('the config_default.py is missing, this program may not works normally\n'
             'config_default.py 文件丢失, 这会导致配置文件不向后兼容, 请重新下载一份 config_default.py')
    raise  # v0.23.1+ 当 config_default.py 不存在时, 程序会终止运行

try:  # 加载用户自定义配置文件, 覆盖掉默认配置的同名项
    from config import *
except:  # coverage: exclude
    errprint(
        'the config_default.py is missing, fallback to default configs(if we can), '
        'please COPY the config_default.py to config.py, and change it\'s content, '
        'or use the configs in the more_configs folder\n'
        '自定义配置文件 config.py 丢失或存在错误, 将使用默认设置, 请将 config_default.py 复制一份为 config.py, '
        '并根据自己的需求修改里面的设置'
        '(或者使用 more_configs 中的配置文件)'
    )
    raise  # v0.23.1+ 当config文件存在错误或不存在时, 程序会终止运行
else:
    target_domain = target_domain.strip("./ \t").replace("https://", "").replace("http://", "")
    infoprint('config file found, mirroring: ', target_domain)

if unittest_mode:
    import importlib

    importlib.reload(importlib.import_module("zmirror.utils"))
    importlib.reload(importlib.import_module("zmirror.connection_pool"))

from .utils import *
from .lru_dict import LRUDict
from . import connection_pool

if local_cache_enable:
    try:
        from .cache_system import FileCache, get_expire_from_mime

        cache = FileCache()
    except:  # coverage: exclude
        traceback.print_exc()
        errprint('Can Not Create Local File Cache, local file cache is disabled automatically.')
        local_cache_enable = False
    else:
        if not unittest_mode:
            infoprint('Local file cache enabled')

# ########## Basic Init #############
# 开始从配置文件加载配置, 在读代码时可以先跳过这部分, 从 main_function() 开始看
ColorfulPyPrint_set_verbose_level(verbose_level)

if developer_enable_experimental_feature:  # coverage: exclude
    # 先处理实验性功能开关
    pass

my_host_name_no_port = my_host_name  # 不带有端口号的本机域名

if my_host_port is not None:
    my_host_name += ':' + str(my_host_port)  # 带有端口号的本机域名, 如果为标准端口则不带显式端口号
    my_host_name_urlencoded = quote_plus(my_host_name)  # url编码后的
else:
    my_host_name_urlencoded = my_host_name

if external_domains is None:
    external_domains = []
external_domains = list([d.strip("./ \t").replace("https://", "").replace("http://", "") for d in external_domains])

external_domains_set = set(external_domains or [])
allowed_domains_set = external_domains_set.copy()
allowed_domains_set.add(target_domain)
for _domain in external_domains:  # for support domain with port
    allowed_domains_set.add(urlsplit('http://' + _domain).hostname)

domain_alias_to_target_set = set()  # 那些被视为主域名的域名, 如 www.google.com和google.com可以都被视为主域名
domain_alias_to_target_set.add(target_domain)
domains_alias_to_target_domain = list(domains_alias_to_target_domain)
if domains_alias_to_target_domain:
    for _domain in domains_alias_to_target_domain:
        allowed_domains_set.add(_domain)
        domain_alias_to_target_set.add(_domain)
    domains_alias_to_target_domain.append(target_domain)
else:
    domains_alias_to_target_domain = [target_domain]
my_host_scheme_escaped = my_host_scheme.replace('/', r'\/')
myurl_prefix = my_host_scheme + my_host_name  # http(s)://www.my-mirror-site.com  末尾没有反斜线
myurl_prefix_escaped = myurl_prefix.replace('/', r'\/')
cdn_domains_number = len(CDN_domains)
allowed_remote_response_headers = {
    'content-type', 'date', 'expires', 'cache-control', 'last-modified', 'server', 'location',
    'accept-ranges',
    'access-control-allow-origin', 'access-control-allow-headers', 'access-control-allow-methods',
    'access-control-expose-headers', 'access-control-max-age', 'access-control-allow-credentials',
    'timing-allow-origin',
}
allowed_remote_response_headers.update(custom_allowed_remote_headers)
# ## Get Target Domain and MyHostName's Root Domain ##
target_domain_root = extract_root_domain(target_domain)[0]  # type: str
my_host_name_root = extract_root_domain(target_domain)[0]  # type: str

# ########## Handle dependencies #############

if not enable_stream_content_transfer:
    steamed_mime_keywords = ()

if not url_custom_redirect_enable:
    url_custom_redirect_list = {}
    url_custom_redirect_regex = ()
    shadow_url_redirect_regex = ()
    plain_replace_domain_alias = []

if isinstance(plain_replace_domain_alias, tuple):
    plain_replace_domain_alias = list(plain_replace_domain_alias)

if not enable_stream_content_transfer:
    enable_stream_transfer_async_preload = False

if not enable_automatic_domains_whitelist:
    domains_whitelist_auto_add_glob_list = tuple()

if not enable_individual_sites_isolation:
    isolated_domains = set()
else:
    for isolated_domain in isolated_domains:
        if isolated_domain not in external_domains_set:
            warnprint('An isolated domain:', isolated_domain,
                      'would not have effect because it did not appears in the `external_domains` list')

if enable_custom_access_cookie_generate_and_verify:
    human_ip_verification_whitelist_from_cookies = False

if not is_use_proxy:
    requests_proxies = None
if human_ip_verification_enabled:
    buff = []
    for network in human_ip_verification_default_whitelist_networks:
        buff.append(ipaddress.ip_network(network, strict=False))
    human_ip_verification_default_whitelist_networks = tuple(buff)
    for question in human_ip_verification_questions:
        human_ip_verification_answers_hash_str += question[1]
else:
    identity_verify_required = False
    human_ip_verification_whitelist_from_cookies = False
    must_verify_cookies = False
if not human_ip_verification_whitelist_from_cookies and not enable_custom_access_cookie_generate_and_verify:
    must_verify_cookies = False

# ########### Global Variables ###############
# 与flask的request变量功能类似, 存储了一些解析后的请求信息, 在程序中会经常被调用
parse = ZmirrorThreadLocal()

# task_scheduler
task_scheduler = sched.scheduler(time, sleep)

# 记录一个URL的一些信息, 以及是否应该使用CDN
url_to_use_cdn = LRUDict(40960)
# 结构例子见下
url_to_use_cdn["www.fake-domain.com/folder/foo/bar.png"] = [
    True,  # Should this url use CDN
    "image/png",  # MIME
    17031,  # size, if size too small, will not redirect to cdn
]

# 记录最近请求的100个域名, 用于 domain_guess
# 虽然是个 dict, 但是只有key有用, value是无用的, 暂时全部赋值为 True
recent_domains = LRUDict(100)
recent_domains[target_domain] = True

# domain_guess 中已知的记录
# 对于已知的记录, 会使用307重定向
domain_guess_cache = LRUDict(1000)
# 格式如下:
domain_guess_cache[("example.com", "/path/no/query/string")] = "target.domain.com"

# ########### PreCompile Regex ###############

# 冒号(colon :)可能的值为:
#    : %3A %253A  完整列表见 tests.TestRegex.REGEX_POSSIBLE_COLON
REGEX_COLON = r"""(?::|%(?:25)?3[Aa])"""
# 斜线(slash /)可能的值为(包括大小写):
# 完整列表见 tests.TestRegex.REGEX_POSSIBLE_COLON
#    / \/ \\/ \\\(N个反斜线)/ %2F %5C%2F %5C%5C(N个5C)%2F %255C%252F %255C%255C%252F \x2F
REGEX_SLASH = r"""(?:\\*(?:/|x2[Ff])|%(?:(?:25)?5[Cc]%)*(?:25)?2[Ff])"""
# 引号 可能值的完整列表见 tests.TestRegex.REGEX_POSSIBLE_QUOTE
# " ' \\(可能有N个反斜线)' \\(可能有N个反斜线)"
# %22 %27 %5C(可能N个5C)%22 %5C(可能N个5C)%27
# %2522 %2527 %255C%2522 %255C%2527
# &quot;
REGEX_QUOTE = r"""(?:\\*["']|%(?:(?:25)?5[Cc]%)*2(?:52)?[27]|&quot;)"""

# 代表本镜像域名的正则
if my_host_port is not None:
    REGEX_MY_HOST_NAME = r'(?:' + re.escape(my_host_name_no_port) + REGEX_COLON + re.escape(str(my_host_port)) \
                         + r'|' + re.escape(my_host_name_no_port) + r')'
else:
    REGEX_MY_HOST_NAME = re.escape(my_host_name)

# Advanced url rewriter, see function response_text_rewrite()
# #### 这个正则表达式是整个程序的最核心的部分, 它的作用是从 html/css/js 中提取出长得类似于url的东西 ####
# 如果需要阅读这个表达式, 请一定要在IDE(如PyCharm)的正则高亮下阅读
# 这个正则并不保证匹配到的东西一定是url, 在 regex_url_reassemble() 中会进行进一步验证是否是url
regex_adv_url_rewriter = re.compile(
    # 前缀, 必须有  'action='(表单) 'href='(链接) 'src=' 'url('(css) '@import'(css) '":'(js/json, "key":"value")
    # \s 表示空白字符,如空格tab
    r"""(?P<prefix>\b(?:(?:src|href|action)\s*=|url\s*\(|@import\s*|"\s*:)\s*)""" +  # prefix, eg: src=
    # 左边引号, 可选 (因为url()允许没有引号). 如果是url以外的, 必须有引号且左右相等(在重写函数中判断, 写在正则里可读性太差)
    r"""(?P<quote_left>["'])?""" +  # quote  "'
    # 域名和协议头, 可选. http:// https:// // http:\/\/ (json) https:\/\/ (json) \/\/ (json)
    r"""(?P<domain_and_scheme>(?P<scheme>(?:https?:)?\\?/\\?/)(?P<domain>(?:[-a-z0-9]+\.)+[a-z]+(?P<port>:\d{1,5})?))?""" +
    # url路径, 含参数 可选
    r"""(?P<path>[^\s;+$?#'"\{}]*?""" +  # full path(with query string)  /foo/bar.js?love=luciaZ
    # 查询字符串, 可选
    r"""(?P<query_string>\?[^\s?#'"]*?)?)""" +  # query string  ?love=luciaZ
    # 右引号(可以是右括弧), 必须
    r"""(?P<quote_right>["')])(?P<right_suffix>\W)""",  # right quote  "'
    flags=re.IGNORECASE
)

# Response Cookies Rewriter, see response_cookie_rewrite()
regex_cookie_rewriter = re.compile(r'\bdomain=(\.?([\w-]+\.)+\w+)\b', flags=re.IGNORECASE)
regex_cookie_path_rewriter = re.compile(r'(?P<prefix>[pP]ath)=(?P<path>[\w\._/-]+?;)')

# Request Domains Rewriter, see client_requests_text_rewrite()
# 该正则用于匹配类似于下面的东西
#   [[[http(s):]//]www.mydomain.com/]extdomains/(https-)target.com
# 兼容各种urlencode/escape
#
# 注意, 若想阅读下面的正则表达式, 请一定要在 Pycharm 的正则高亮下进行
# 否则不对可能的头晕/恶心负责
# 下面那个正则, 在组装以后的样子大概是这样的(已大幅简化):
# 假设b.test.com是本机域名
#   ((https?:/{2})?b\.test\.com/)?extdomains/(https-)?((?:[\w-]+\.)+\w+)\b
#
# 对应的 unittest 见 TestRegex.test__regex_request_rewriter_extdomains()
regex_request_rewriter_extdomains = re.compile(
    r"""(?P<domain_prefix>""" +
    (  # [[[http(s):]//]www.mydomain.com/]
        r"""(?P<scheme>""" +
        (  # [[http(s):]//]
            (  # [http(s):]
                r"""(?:https?(?P<colon>{REGEX_COLON}))?""".format(REGEX_COLON=REGEX_COLON)  # https?:
            ) +
            r"""(?P<scheme_slash>%s)(?P=scheme_slash)""" % REGEX_SLASH  # //
        ) +
        r""")?""" +
        REGEX_MY_HOST_NAME +  # www.mydomain.com[:port] 本部分的正则在上面单独组装
        r"""(?P<slash2>(?(scheme_slash)(?P=scheme_slash)|{REGEX_SLASH}))""".format(REGEX_SLASH=REGEX_SLASH)  # # /
    ) +
    r""")?""" +

    r"""extdomains(?(slash2)(?P=slash2)|{REGEX_SLASH})(?P<is_https>https-)?""".format(
        REGEX_SLASH=REGEX_SLASH) +  # extdomains/(https-)
    r"""(?P<real_domain>(?:[\w-]+\.)+\w+)\b""",  # target.com
    flags=re.IGNORECASE,
)
regex_request_rewriter_main_domain = re.compile(REGEX_MY_HOST_NAME)


# 以下正则为*实验性*的 response_text_basic_rewrite() 的替代品
# 用于函数 response_text_basic_mirrorlization()
# 理论上, 在大量域名的情况下, 会比现有的暴力字符串替换要快, 并且未来可以更强大的域名通配符
# v0.28.0加入, v0.28.3后默认启用
def _regex_generate__basic_mirrorlization():
    """产生 regex_basic_mirrorlization
    用一个函数包裹起来是因为在 try_match_and_add_domain_to_rewrite_white_list()
    中需要动态修改 external_domains, 修改以后可能需要随之生成新的正则, 包裹一下比较容易调用
    """
    from collections import Counter

    # 统计各个后缀出现的频率, 并且按照出现频率降序排列, 有助于提升正则效率
    c = Counter(re.escape(x.split(".")[-1]) for x in allowed_domains_set)
    regex_all_remote_tld = sorted(list(c.keys()), key=lambda x: c[x], reverse=True)

    regex_all_remote_tld = "(?:" + "|".join(regex_all_remote_tld) + ")"
    return re.compile(
        r"""(?:""" +
        (  # [[http(s):]//] or [\?["']] or %27 %22 or &quot;
            r"""(?P<scheme>""" +
            (  # [[http(s):]//]
                (  # [http(s):]
                    r"""(?:https?(?P<colon>{REGEX_COLON}))?""".format(REGEX_COLON=REGEX_COLON)  # https?:
                ) +
                r"""(?P<scheme_slash>%s)(?P=scheme_slash)""" % REGEX_SLASH  # //
            ) +
            r""")""" +
            r"""|""" +
            # [\?["']] or %27 %22 or &quot
            r"""(?P<quote>{REGEX_QUOTE})""".format(REGEX_QUOTE=REGEX_QUOTE)
        ) +
        r""")""" +
        # End prefix.
        # Begin domain
        r"""(?P<domain>([a-zA-Z0-9-]+\.){1,5}%s)\b""" % regex_all_remote_tld +
        # Optional suffix slash
        r"""(?P<suffix_slash>(?(scheme_slash)(?P=scheme_slash)|{SLASH}))?""".format(SLASH=REGEX_SLASH) +

        # right quote (if we have left quote)
        r"""(?(quote)(?P=quote))"""
    )


regex_basic_mirrorlization = _regex_generate__basic_mirrorlization()

# 用于移除掉cookie中类似于 zmirror_verify=75bf23086a541e1f; 的部分
regex_remove__zmirror_verify__header = re.compile(
    r"""zmirror_verify=[a-zA-Z0-9]+\b;? ?"""
)

# ########## Flask app ###########

app = Flask(  # type: Flask
    __name__ if not unittest_mode
    else 'unittest' + str(random.random()).replace('.', '')
)
<commit_msg>[/] 修复flask static 导致的 /static/*** 路径无法镜像<commit_after>#!/usr/bin/env python3
# coding=utf-8
import os
import sys
import re
import copy
import zlib
import sched
import queue
import base64
import random
import traceback
import ipaddress
import threading

from fnmatch import fnmatch
from time import time, sleep, process_time
from html import escape as html_escape
from datetime import datetime, timedelta
from urllib.parse import urljoin, urlsplit, urlunsplit, quote_plus
import urllib.parse
import requests
from flask import Flask, request, make_response, Response, redirect
from . import CONSTS

try:
    # for python 3.5+ Type Hint
    from typing import Union, List, Any, Tuple
except:
    pass
try:  # 用于检测html的文本编码, cchardet是chardet的c语言实现, 非常快
    from cchardet import detect as c_chardet
except:
    cchardet_available = False
else:
    cchardet_available = True

if os.path.abspath(os.getcwd()) != CONSTS.ZMIRROR_ROOT:
    os.chdir(CONSTS.ZMIRROR_ROOT)

from .external_pkgs.ColorfulPyPrint import *  # TODO: Migrate logging tools to the stdlib logging

if "ZMIRROR_UNITTEST" in os.environ:
    # 这边根据环境变量得到的unittest_mode信息会被config中的覆盖掉
    # 只是因为此时还没有加载 config, 所以先根据env里的临时定一下
    unittest_mode = True
else:
    unittest_mode = False

try:  # lru_cache的c语言实现, 比Python内置lru_cache更快
    from fastcache import lru_cache  # lru_cache用于缓存函数的执行结果
except:
    from functools import lru_cache

    warnprint('package fastcache not found, '
              'fallback to stdlib lru_cache, '
              'no FUNCTION is effected, only maybe a bit slower. '
              'Considering install it using "pip3 install fastcache"'
              )
else:
    if not unittest_mode:
        infoprint('lru_cache loaded successfully from fastcache')

from .threadlocal import ZmirrorThreadLocal

if not unittest_mode:  # 在unittest时不输出这几行
    infoprint('zmirror version: {version} author: {author}'.format(version=CONSTS.__VERSION__, author=CONSTS.__AUTHOR__))
    infoprint('Github: {site_url}'.format(site_url=CONSTS.__GITHUB_URL__))

try:  # 加载默认设置
    from config_default import *
except:  # coverage: exclude
    errprint('the config_default.py is missing, this program may not works normally\n'
             'config_default.py 文件丢失, 这会导致配置文件不向后兼容, 请重新下载一份 config_default.py')
    raise  # v0.23.1+ 当 config_default.py 不存在时, 程序会终止运行

try:  # 加载用户自定义配置文件, 覆盖掉默认配置的同名项
    from config import *
except:  # coverage: exclude
    errprint(
        'the config_default.py is missing, fallback to default configs(if we can), '
        'please COPY the config_default.py to config.py, and change it\'s content, '
        'or use the configs in the more_configs folder\n'
        '自定义配置文件 config.py 丢失或存在错误, 将使用默认设置, 请将 config_default.py 复制一份为 config.py, '
        '并根据自己的需求修改里面的设置'
        '(或者使用 more_configs 中的配置文件)'
    )
    raise  # v0.23.1+ 当config文件存在错误或不存在时, 程序会终止运行
else:
    target_domain = target_domain.strip("./ \t").replace("https://", "").replace("http://", "")
    infoprint('config file found, mirroring: ', target_domain)

if unittest_mode:
    import importlib

    importlib.reload(importlib.import_module("zmirror.utils"))
    importlib.reload(importlib.import_module("zmirror.connection_pool"))

from .utils import *
from .lru_dict import LRUDict
from . import connection_pool

if local_cache_enable:
    try:
        from .cache_system import FileCache, get_expire_from_mime

        cache = FileCache()
    except:  # coverage: exclude
        traceback.print_exc()
        errprint('Can Not Create Local File Cache, local file cache is disabled automatically.')
        local_cache_enable = False
    else:
        if not unittest_mode:
            infoprint('Local file cache enabled')

# ########## Basic Init #############
# 开始从配置文件加载配置, 在读代码时可以先跳过这部分, 从 main_function() 开始看
ColorfulPyPrint_set_verbose_level(verbose_level)

if developer_enable_experimental_feature:  # coverage: exclude
    # 先处理实验性功能开关
    pass

my_host_name_no_port = my_host_name  # 不带有端口号的本机域名

if my_host_port is not None:
    my_host_name += ':' + str(my_host_port)  # 带有端口号的本机域名, 如果为标准端口则不带显式端口号
    my_host_name_urlencoded = quote_plus(my_host_name)  # url编码后的
else:
    my_host_name_urlencoded = my_host_name

if external_domains is None:
    external_domains = []
external_domains = list([d.strip("./ \t").replace("https://", "").replace("http://", "") for d in external_domains])

external_domains_set = set(external_domains or [])
allowed_domains_set = external_domains_set.copy()
allowed_domains_set.add(target_domain)
for _domain in external_domains:  # for support domain with port
    allowed_domains_set.add(urlsplit('http://' + _domain).hostname)

domain_alias_to_target_set = set()  # 那些被视为主域名的域名, 如 www.google.com和google.com可以都被视为主域名
domain_alias_to_target_set.add(target_domain)
domains_alias_to_target_domain = list(domains_alias_to_target_domain)
if domains_alias_to_target_domain:
    for _domain in domains_alias_to_target_domain:
        allowed_domains_set.add(_domain)
        domain_alias_to_target_set.add(_domain)
    domains_alias_to_target_domain.append(target_domain)
else:
    domains_alias_to_target_domain = [target_domain]
my_host_scheme_escaped = my_host_scheme.replace('/', r'\/')
myurl_prefix = my_host_scheme + my_host_name  # http(s)://www.my-mirror-site.com  末尾没有反斜线
myurl_prefix_escaped = myurl_prefix.replace('/', r'\/')
cdn_domains_number = len(CDN_domains)
allowed_remote_response_headers = {
    'content-type', 'date', 'expires', 'cache-control', 'last-modified', 'server', 'location',
    'accept-ranges',
    'access-control-allow-origin', 'access-control-allow-headers', 'access-control-allow-methods',
    'access-control-expose-headers', 'access-control-max-age', 'access-control-allow-credentials',
    'timing-allow-origin',
}
allowed_remote_response_headers.update(custom_allowed_remote_headers)
# ## Get Target Domain and MyHostName's Root Domain ##
target_domain_root = extract_root_domain(target_domain)[0]  # type: str
my_host_name_root = extract_root_domain(target_domain)[0]  # type: str

# ########## Handle dependencies #############

if not enable_stream_content_transfer:
    steamed_mime_keywords = ()

if not url_custom_redirect_enable:
    url_custom_redirect_list = {}
    url_custom_redirect_regex = ()
    shadow_url_redirect_regex = ()
    plain_replace_domain_alias = []

if isinstance(plain_replace_domain_alias, tuple):
    plain_replace_domain_alias = list(plain_replace_domain_alias)

if not enable_stream_content_transfer:
    enable_stream_transfer_async_preload = False

if not enable_automatic_domains_whitelist:
    domains_whitelist_auto_add_glob_list = tuple()

if not enable_individual_sites_isolation:
    isolated_domains = set()
else:
    for isolated_domain in isolated_domains:
        if isolated_domain not in external_domains_set:
            warnprint('An isolated domain:', isolated_domain,
                      'would not have effect because it did not appears in the `external_domains` list')

if enable_custom_access_cookie_generate_and_verify:
    human_ip_verification_whitelist_from_cookies = False

if not is_use_proxy:
    requests_proxies = None
if human_ip_verification_enabled:
    buff = []
    for network in human_ip_verification_default_whitelist_networks:
        buff.append(ipaddress.ip_network(network, strict=False))
    human_ip_verification_default_whitelist_networks = tuple(buff)
    for question in human_ip_verification_questions:
        human_ip_verification_answers_hash_str += question[1]
else:
    identity_verify_required = False
    human_ip_verification_whitelist_from_cookies = False
    must_verify_cookies = False
if not human_ip_verification_whitelist_from_cookies and not enable_custom_access_cookie_generate_and_verify:
    must_verify_cookies = False

# ########### Global Variables ###############
# 与flask的request变量功能类似, 存储了一些解析后的请求信息, 在程序中会经常被调用
parse = ZmirrorThreadLocal()

# task_scheduler
task_scheduler = sched.scheduler(time, sleep)

# 记录一个URL的一些信息, 以及是否应该使用CDN
url_to_use_cdn = LRUDict(40960)
# 结构例子见下
url_to_use_cdn["www.fake-domain.com/folder/foo/bar.png"] = [
    True,  # Should this url use CDN
    "image/png",  # MIME
    17031,  # size, if size too small, will not redirect to cdn
]

# 记录最近请求的100个域名, 用于 domain_guess
# 虽然是个 dict, 但是只有key有用, value是无用的, 暂时全部赋值为 True
recent_domains = LRUDict(100)
recent_domains[target_domain] = True

# domain_guess 中已知的记录
# 对于已知的记录, 会使用307重定向
domain_guess_cache = LRUDict(1000)
# 格式如下:
domain_guess_cache[("example.com", "/path/no/query/string")] = "target.domain.com"

# ########### PreCompile Regex ###############

# 冒号(colon :)可能的值为:
#    : %3A %253A  完整列表见 tests.TestRegex.REGEX_POSSIBLE_COLON
REGEX_COLON = r"""(?::|%(?:25)?3[Aa])"""
# 斜线(slash /)可能的值为(包括大小写):
# 完整列表见 tests.TestRegex.REGEX_POSSIBLE_COLON
#    / \/ \\/ \\\(N个反斜线)/ %2F %5C%2F %5C%5C(N个5C)%2F %255C%252F %255C%255C%252F \x2F
REGEX_SLASH = r"""(?:\\*(?:/|x2[Ff])|%(?:(?:25)?5[Cc]%)*(?:25)?2[Ff])"""
# 引号 可能值的完整列表见 tests.TestRegex.REGEX_POSSIBLE_QUOTE
# " ' \\(可能有N个反斜线)' \\(可能有N个反斜线)"
# %22 %27 %5C(可能N个5C)%22 %5C(可能N个5C)%27
# %2522 %2527 %255C%2522 %255C%2527
# &quot;
REGEX_QUOTE = r"""(?:\\*["']|%(?:(?:25)?5[Cc]%)*2(?:52)?[27]|&quot;)"""

# 代表本镜像域名的正则
if my_host_port is not None:
    REGEX_MY_HOST_NAME = r'(?:' + re.escape(my_host_name_no_port) + REGEX_COLON + re.escape(str(my_host_port)) \
                         + r'|' + re.escape(my_host_name_no_port) + r')'
else:
    REGEX_MY_HOST_NAME = re.escape(my_host_name)

# Advanced url rewriter, see function response_text_rewrite()
# #### 这个正则表达式是整个程序的最核心的部分, 它的作用是从 html/css/js 中提取出长得类似于url的东西 ####
# 如果需要阅读这个表达式, 请一定要在IDE(如PyCharm)的正则高亮下阅读
# 这个正则并不保证匹配到的东西一定是url, 在 regex_url_reassemble() 中会进行进一步验证是否是url
regex_adv_url_rewriter = re.compile(
    # 前缀, 必须有  'action='(表单) 'href='(链接) 'src=' 'url('(css) '@import'(css) '":'(js/json, "key":"value")
    # \s 表示空白字符,如空格tab
    r"""(?P<prefix>\b(?:(?:src|href|action)\s*=|url\s*\(|@import\s*|"\s*:)\s*)""" +  # prefix, eg: src=
    # 左边引号, 可选 (因为url()允许没有引号). 如果是url以外的, 必须有引号且左右相等(在重写函数中判断, 写在正则里可读性太差)
    r"""(?P<quote_left>["'])?""" +  # quote  "'
    # 域名和协议头, 可选. http:// https:// // http:\/\/ (json) https:\/\/ (json) \/\/ (json)
    r"""(?P<domain_and_scheme>(?P<scheme>(?:https?:)?\\?/\\?/)(?P<domain>(?:[-a-z0-9]+\.)+[a-z]+(?P<port>:\d{1,5})?))?""" +
    # url路径, 含参数 可选
    r"""(?P<path>[^\s;+$?#'"\{}]*?""" +  # full path(with query string)  /foo/bar.js?love=luciaZ
    # 查询字符串, 可选
    r"""(?P<query_string>\?[^\s?#'"]*?)?)""" +  # query string  ?love=luciaZ
    # 右引号(可以是右括弧), 必须
    r"""(?P<quote_right>["')])(?P<right_suffix>\W)""",  # right quote  "'
    flags=re.IGNORECASE
)

# Response Cookies Rewriter, see response_cookie_rewrite()
regex_cookie_rewriter = re.compile(r'\bdomain=(\.?([\w-]+\.)+\w+)\b', flags=re.IGNORECASE)
regex_cookie_path_rewriter = re.compile(r'(?P<prefix>[pP]ath)=(?P<path>[\w\._/-]+?;)')

# Request Domains Rewriter, see client_requests_text_rewrite()
# 该正则用于匹配类似于下面的东西
#   [[[http(s):]//]www.mydomain.com/]extdomains/(https-)target.com
# 兼容各种urlencode/escape
#
# 注意, 若想阅读下面的正则表达式, 请一定要在 Pycharm 的正则高亮下进行
# 否则不对可能的头晕/恶心负责
# 下面那个正则, 在组装以后的样子大概是这样的(已大幅简化):
# 假设b.test.com是本机域名
#   ((https?:/{2})?b\.test\.com/)?extdomains/(https-)?((?:[\w-]+\.)+\w+)\b
#
# 对应的 unittest 见 TestRegex.test__regex_request_rewriter_extdomains()
regex_request_rewriter_extdomains = re.compile(
    r"""(?P<domain_prefix>""" +
    (  # [[[http(s):]//]www.mydomain.com/]
        r"""(?P<scheme>""" +
        (  # [[http(s):]//]
            (  # [http(s):]
                r"""(?:https?(?P<colon>{REGEX_COLON}))?""".format(REGEX_COLON=REGEX_COLON)  # https?:
            ) +
            r"""(?P<scheme_slash>%s)(?P=scheme_slash)""" % REGEX_SLASH  # //
        ) +
        r""")?""" +
        REGEX_MY_HOST_NAME +  # www.mydomain.com[:port] 本部分的正则在上面单独组装
        r"""(?P<slash2>(?(scheme_slash)(?P=scheme_slash)|{REGEX_SLASH}))""".format(REGEX_SLASH=REGEX_SLASH)  # # /
    ) +
    r""")?""" +

    r"""extdomains(?(slash2)(?P=slash2)|{REGEX_SLASH})(?P<is_https>https-)?""".format(
        REGEX_SLASH=REGEX_SLASH) +  # extdomains/(https-)
    r"""(?P<real_domain>(?:[\w-]+\.)+\w+)\b""",  # target.com
    flags=re.IGNORECASE,
)
regex_request_rewriter_main_domain = re.compile(REGEX_MY_HOST_NAME)


# 以下正则为*实验性*的 response_text_basic_rewrite() 的替代品
# 用于函数 response_text_basic_mirrorlization()
# 理论上, 在大量域名的情况下, 会比现有的暴力字符串替换要快, 并且未来可以更强大的域名通配符
# v0.28.0加入, v0.28.3后默认启用
def _regex_generate__basic_mirrorlization():
    """产生 regex_basic_mirrorlization
    用一个函数包裹起来是因为在 try_match_and_add_domain_to_rewrite_white_list()
    中需要动态修改 external_domains, 修改以后可能需要随之生成新的正则, 包裹一下比较容易调用
    """
    from collections import Counter

    # 统计各个后缀出现的频率, 并且按照出现频率降序排列, 有助于提升正则效率
    c = Counter(re.escape(x.split(".")[-1]) for x in allowed_domains_set)
    regex_all_remote_tld = sorted(list(c.keys()), key=lambda x: c[x], reverse=True)

    regex_all_remote_tld = "(?:" + "|".join(regex_all_remote_tld) + ")"
    return re.compile(
        r"""(?:""" +
        (  # [[http(s):]//] or [\?["']] or %27 %22 or &quot;
            r"""(?P<scheme>""" +
            (  # [[http(s):]//]
                (  # [http(s):]
                    r"""(?:https?(?P<colon>{REGEX_COLON}))?""".format(REGEX_COLON=REGEX_COLON)  # https?:
                ) +
                r"""(?P<scheme_slash>%s)(?P=scheme_slash)""" % REGEX_SLASH  # //
            ) +
            r""")""" +
            r"""|""" +
            # [\?["']] or %27 %22 or &quot
            r"""(?P<quote>{REGEX_QUOTE})""".format(REGEX_QUOTE=REGEX_QUOTE)
        ) +
        r""")""" +
        # End prefix.
        # Begin domain
        r"""(?P<domain>([a-zA-Z0-9-]+\.){1,5}%s)\b""" % regex_all_remote_tld +
        # Optional suffix slash
        r"""(?P<suffix_slash>(?(scheme_slash)(?P=scheme_slash)|{SLASH}))?""".format(SLASH=REGEX_SLASH) +

        # right quote (if we have left quote)
        r"""(?(quote)(?P=quote))"""
    )


regex_basic_mirrorlization = _regex_generate__basic_mirrorlization()

# 用于移除掉cookie中类似于 zmirror_verify=75bf23086a541e1f; 的部分
regex_remove__zmirror_verify__header = re.compile(
    r"""zmirror_verify=[a-zA-Z0-9]+\b;? ?"""
)

# ########## Flask app ###########

app = Flask(  # type: Flask
    __name__ if not unittest_mode
    else 'unittest' + str(random.random()).replace('.', ''),
    static_folder=None,
    template_folder=None,
)

{'fraction_non_alphanumeric': 0.15194247617717896, 'fraction_numerical': 0.011309295263359978, 'mean_word_length': 4.229056397152765, 'pattern_counts': {'":': 4, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 6, 'lorem ipsum': 0, 'www.': 12, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '189250', 'n_tokens_mistral': 12119, 'n_tokens_neox': 12287, 'n_words': 1837}