<commit_before>import collections
import functools
import json
import os
import re
import textwrap
from collections import OrderedDict

import simple_yaml as yaml
from flask import abort as flask_abort
from flask import Flask, jsonify, request
from jinja2 import Template
from markdown import markdown as origin_markdown
from validr import Invalid, SchemaError, SchemaParser
from validr.schema import MarkKey

RE_DIRECTIVE = re.compile(r'[\t ]*\$\w+:')
RE_SHARED = re.compile(r'[\t ]*\@\w+:')

_DOC_PATH = os.path.join(os.path.dirname(__file__), 'document.jinja2')
with open(_DOC_PATH) as f:
    _DOC_TMPL = Template(f.read())


def dumps(v):
    return json.dumps(v, ensure_ascii=False, indent=4)


def markdown(text):
    """用Markdown解析文档"""
    extensions = ['nl2br', 'tables', 'fenced_code']
    return origin_markdown(text, extensions=extensions)


def render(desc='', shared=None, resources=None, directives=None):
    """渲染文档"""
    desc = markdown(desc)
    if shared is None:
        shared = OrderedDict()
    else:
        shared = OrderedDict([(k, dumps(v)) for k, v in shared.items()])
    if resources is None:
        resources = OrderedDict()
    else:
        resources = OrderedDict(sorted(resources.items(), key=lambda x: x[0]))
    if directives is None:
        directives = OrderedDict()
    else:
        directives = OrderedDict(
            [(k, dumps(v)) for k, v in directives.items()])
    return _DOC_TMPL.render(
        desc=desc, shared=shared, directives=directives, resources=resources)


def accept_json():
    """判断客户端是否接受JSON响应"""
    mediatype = request.accept_mimetypes.best_match(
        ['text/html', 'application/json'], default='text/html')
    return mediatype == 'application/json' or 'json' in request.args


def abort(message):
    """
    Abort with 400 response

    Args:
        message (str): 错误提示
    """
    response = jsonify(message)
    response.status_code = 400
    flask_abort(400, response=response)


def get_request_data():
    """Get request data based on request mimetype"""
    if request.mimetype == 'application/json':
        try:
            data = request.get_json()
        except:
            abort("Invalid JSON content")
        if not isinstance(data, collections.Mapping):
            abort("JSON content must be object")
        return data
    else:
        return request.form


def _parse_doc(doc, mark):
    """
    Parse YAML syntax data from doc

    if doc is None, return ('', OrderedDict())
    if doc has no YAML data, return (doc, OrderedDict())
    else, parse YAML data, return (doc, data)

    Args:
        doc (str): doc to be parsed
        mark (Regex): which marks the start position of data
    Returns:
        tuple(desc, data): data is an OrderedDict contains information of doc
    """
    if doc is None:<commit_msg>fix:py3.10 更新collections下有abc<commit_after>import collections
import functools
import json
import os
import re
import textwrap
from collections import OrderedDict

import simple_yaml as yaml
from flask import abort as flask_abort
from flask import Flask, jsonify, request
from jinja2 import Template
from markdown import markdown as origin_markdown
from validr import Invalid, SchemaError, SchemaParser
from validr.schema import MarkKey

RE_DIRECTIVE = re.compile(r'[\t ]*\$\w+:')
RE_SHARED = re.compile(r'[\t ]*\@\w+:')

_DOC_PATH = os.path.join(os.path.dirname(__file__), 'document.jinja2')
with open(_DOC_PATH) as f:
    _DOC_TMPL = Template(f.read())


def dumps(v):
    return json.dumps(v, ensure_ascii=False, indent=4)


def markdown(text):
    """用Markdown解析文档"""
    extensions = ['nl2br', 'tables', 'fenced_code']
    return origin_markdown(text, extensions=extensions)


def render(desc='', shared=None, resources=None, directives=None):
    """渲染文档"""
    desc = markdown(desc)
    if shared is None:
        shared = OrderedDict()
    else:
        shared = OrderedDict([(k, dumps(v)) for k, v in shared.items()])
    if resources is None:
        resources = OrderedDict()
    else:
        resources = OrderedDict(sorted(resources.items(), key=lambda x: x[0]))
    if directives is None:
        directives = OrderedDict()
    else:
        directives = OrderedDict(
            [(k, dumps(v)) for k, v in directives.items()])
    return _DOC_TMPL.render(
        desc=desc, shared=shared, directives=directives, resources=resources)


def accept_json():
    """判断客户端是否接受JSON响应"""
    mediatype = request.accept_mimetypes.best_match(
        ['text/html', 'application/json'], default='text/html')
    return mediatype == 'application/json' or 'json' in request.args


def abort(message):
    """
    Abort with 400 response

    Args:
        message (str): 错误提示
    """
    response = jsonify(message)
    response.status_code = 400
    flask_abort(400, response=response)


def get_request_data():
    """Get request data based on request mimetype"""
    if request.mimetype == 'application/json':
        try:
            data = request.get_json()
        except:
            abort("Invalid JSON content")
        if not isinstance(data, collections.abc.Mapping):
            abort("JSON content must be object")
        return data
    else:
        return request.form


def _parse_doc(doc, mark):
    """
    Parse YAML syntax data from doc

    if doc is None, return ('', OrderedDict())
    if doc has no YAML data, return (doc, OrderedDict())
    else, parse YAML data, return (doc, data)

    Args:
        doc (str): doc to be parsed
        mark (Regex): which marks the start position of data
    Returns:
        tuple(desc, data): data is an OrderedDict contains information of doc
    """
    if doc is None:
{'fraction_non_alphanumeric': 0.11289752650176678, 'fraction_numerical': 0.005477031802120141, 'mean_word_length': 3.2183308494783907, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3047288', 'n_tokens_mistral': 1807, 'n_tokens_neox': 1759, 'n_words': 532}