<commit_before>
import os
import datetime

from clastic import redirect, render_basic
from clastic.errors import BadRequest
from mwoauth import Handshaker, RequestToken
from markdown import Markdown
from markdown.extensions.codehilite import CodeHiliteExtension
from chert import hypertext as html_utils

from mw import public
from rdb import User, PublicDAO
from labs import get_files, get_file_info

from utils import load_env_config, DoesNotExist, InvalidAction

config = load_env_config()
DEBUG = config.get('debug', False)

CUR_PATH = os.path.dirname(os.path.abspath(__file__))
DOCS_PATH = CUR_PATH + '/docs'

WIKI_OAUTH_URL = "https://meta.wikimedia.org/w/index.php"

MD_EXTENSIONS = ['markdown.extensions.def_list',
                 'markdown.extensions.footnotes',
                 'markdown.extensions.fenced_code',
                 'markdown.extensions.tables',
                 CodeHiliteExtension(pygments_style='emacs')]


def get_public_routes():
    ui = [('/', home),
          ('/login', login),
          ('/logout', logout),
          ('/complete_login', complete_login),
          ('/campaign/<campaign_id:int>', get_report, 'report.html'),
          ('/docs/<path*>', get_doc, render_basic)]
    api = [('/campaign', get_all_reports),
           ('/series/<series_id?int>', get_series),
           ('/series', get_series),
           ('/entry/<entry_name:str>', get_entry_info),
           ('/campaign', get_all_reports),
           ('/utils/category', get_file_info_by_category),
           ('/utils/file', get_files_info_by_name)]
    return api, ui


@public
def get_doc(ashes_renderer, path):
    if not path:
        path = ['index']
    render = ashes_renderer('docs/base.html')
    doc_rel_path = '/'.join(path)
    doc_path = DOCS_PATH + '/' + doc_rel_path + '.md'
    doc_path = os.path.normpath(doc_path)
    if not doc_path.startswith(DOCS_PATH):
        raise BadRequest('invalid doc path: %r' % doc_path)
    try:
        doc_md = open(doc_path, 'rb').read()
    except OSError:
        raise BadRequest('could not open doc: %r' % doc_rel_path)

    lines = doc_md.splitlines()
    if not lines:
        raise BadRequest('empty doc: %r' % doc_rel_path)
    title_line, body_md = lines[0], '\n'.join(lines[1:])
    _, sep, title = title_line.partition('#')
    if not sep or not title:
        raise BadRequest('bad doc title. expected "# Title", not: %r'
                         % title_line)

    md_converter = Markdown(extensions=MD_EXTENSIONS)
    body_html = md_converter.convert(body_md)

    html_tree = html_utils.html_text_to_tree(body_html)
    html_utils.retarget_links(html_tree)
    html_utils.add_toc(html_tree)
    body_html = html_utils.html_tree_to_text(html_tree)

    # TODO: cached version
    return render({'title': title, 'body': body_html})


@public
def get_file_info_by_category(request_dict):
    try:
        category_name = request_dict['name']
    except Exception:
        raise InvalidAction('missing name=category_name query parameter')
    files = get_files(category_name)
    return {'file_infos': files}


@public
def get_files_info_by_name(request_dict):
    try:
        file_names = request_dict['names']
    except Exception:
        raise InvalidAction('must provide a list of names')
    files = []
    for file_name in file_names:
        file_info = get_file_info(file_name)
        files.append(file_info)
    return {'file_infos': files}


@public
def home(cookie, request):
    headers = dict([(k, v) for k, v in
                    request.environ.items() if k.startswith('HTTP_')])
    return {'cookie': dict(cookie),
            'headers': headers}


@public
def login(request, consumer_token, cookie, root_path):
    handshaker = Handshaker(WIKI_OAUTH_URL, consumer_token)

    redirect_url, request_token = handshaker.initiate()

    cookie['request_token_key'] = request_token.key
    cookie['request_token_secret'] = request_token.secret

    cookie['return_to_url'] = request.args.get('next', root_path)
    return redirect(redirect_url)

<commit_msg>backend: let me know when remote file info is missing<commit_after>
import os
import datetime

from clastic import redirect, render_basic
from clastic.errors import BadRequest
from mwoauth import Handshaker, RequestToken
from markdown import Markdown
from markdown.extensions.codehilite import CodeHiliteExtension
from chert import hypertext as html_utils

from mw import public
from rdb import User, PublicDAO
from labs import get_files, get_file_info

from utils import load_env_config, DoesNotExist, InvalidAction

config = load_env_config()
DEBUG = config.get('debug', False)

CUR_PATH = os.path.dirname(os.path.abspath(__file__))
DOCS_PATH = CUR_PATH + '/docs'

WIKI_OAUTH_URL = "https://meta.wikimedia.org/w/index.php"

MD_EXTENSIONS = ['markdown.extensions.def_list',
                 'markdown.extensions.footnotes',
                 'markdown.extensions.fenced_code',
                 'markdown.extensions.tables',
                 CodeHiliteExtension(pygments_style='emacs')]


def get_public_routes():
    ui = [('/', home),
          ('/login', login),
          ('/logout', logout),
          ('/complete_login', complete_login),
          ('/campaign/<campaign_id:int>', get_report, 'report.html'),
          ('/docs/<path*>', get_doc, render_basic)]
    api = [('/campaign', get_all_reports),
           ('/series/<series_id?int>', get_series),
           ('/series', get_series),
           ('/entry/<entry_name:str>', get_entry_info),
           ('/campaign', get_all_reports),
           ('/utils/category', get_file_info_by_category),
           ('/utils/file', get_files_info_by_name)]
    return api, ui


@public
def get_doc(ashes_renderer, path):
    if not path:
        path = ['index']
    render = ashes_renderer('docs/base.html')
    doc_rel_path = '/'.join(path)
    doc_path = DOCS_PATH + '/' + doc_rel_path + '.md'
    doc_path = os.path.normpath(doc_path)
    if not doc_path.startswith(DOCS_PATH):
        raise BadRequest('invalid doc path: %r' % doc_path)
    try:
        doc_md = open(doc_path, 'rb').read()
    except OSError:
        raise BadRequest('could not open doc: %r' % doc_rel_path)

    lines = doc_md.splitlines()
    if not lines:
        raise BadRequest('empty doc: %r' % doc_rel_path)
    title_line, body_md = lines[0], '\n'.join(lines[1:])
    _, sep, title = title_line.partition('#')
    if not sep or not title:
        raise BadRequest('bad doc title. expected "# Title", not: %r'
                         % title_line)

    md_converter = Markdown(extensions=MD_EXTENSIONS)
    body_html = md_converter.convert(body_md)

    html_tree = html_utils.html_text_to_tree(body_html)
    html_utils.retarget_links(html_tree)
    html_utils.add_toc(html_tree)
    body_html = html_utils.html_tree_to_text(html_tree)

    # TODO: cached version
    return render({'title': title, 'body': body_html})


@public
def get_file_info_by_category(request_dict):
    try:
        category_name = request_dict['name']
    except Exception:
        raise InvalidAction('missing name=category_name query parameter')
    files = get_files(category_name)
    return {'file_infos': files}


@public
def get_files_info_by_name(request_dict):
    try:
        file_names = request_dict['names']
    except Exception:
        raise InvalidAction('must provide a list of names')
    files = []
    no_info = []
    for file_name in file_names:
        file_info = get_file_info(file_name)
        if not file_info:
            no_info.append(file_name)
        else:
            files.append(file_info)
    return {'file_infos': files,
            'no_info': no_info}


@public
def home(cookie, request):
    headers = dict([(k, v) for k, v in
                    request.environ.items() if k.startswith('HTTP_')])
    return {'cookie': dict(cookie),
            'headers': headers}


@public
def login(request, consumer_token, cookie, root_path):
    handshaker = Handshaker(WIKI_OAUTH_URL, consumer_token)

    redirect_url, request_token = handshaker.initiate()

    cookie['request_token_key'] = request_token.key
    cookie['request_token_secret'] = request_token.secret

    cookie['return_to_url'] = request.args.get('next', root_path)
    return redirect(redirect_url)


{'fraction_non_alphanumeric': 0.11765418635647644, 'fraction_numerical': 0.00048467224039743125, 'mean_word_length': 3.292251690067603, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '940019', 'n_tokens_mistral': 2675, 'n_tokens_neox': 2607, 'n_words': 577}