<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import json
import re
import logging
import requests
import subprocess
from datetime import date
from argparse import ArgumentParser

from toollib import metautil, DATE_RE, YEAR_RE, year_to_western

# Note the full-width brackets and slash.
MULTIPART_RE = re.compile(u'^(.*)(（.*／.*）.*|（その.*）)')

# Simplify a couple of docset types
DOCSETTYPE_MAPPING = {
    u'政党の届出事項の異動の届出': u'異動の届出',
    u'政治団体の届出事項の異動の届出': u'異動の届出',
}


def title_date(title):
    result = DATE_RE.search(title)

    if result is None:
        import pdb; pdb.set_trace()
        raise ValueError("Title didn't match expected format: %s" % title)

    (emp, year, month, day) = result.groups()[:4]
    year = int(year.strip())
    month = int(month.strip())
    day = int(day.strip())
    year = year_to_western(emp, year)

    return date(year, month, day)


def makefilter(field, val, op='=='):
    fparams = {'filter': [{'name': field, 'op': op, 'val': val}]}
    return fparams


def _get_all(s, api_root, otype):
    LIST_ALL_PARAM = {'results_per_page': '50000'}
    objs = s.get(api_root + otype, params=LIST_ALL_PARAM).json()

    # FIXME: This may break eventually..
    assert objs['total_pages'] <= 1
    return objs['objects']


_group_cache = {}
_docset_cache = {}
_grouptype_cache = {}
_doctype_cache = {}
_pubtype_cache = {}


def get_existing_types(s, api_root):
    grouptype = _get_all(s, api_root, 'group_type')
    doctype = _get_all(s, api_root, 'doc_type')
    pubtype = _get_all(s, api_root, 'pub_type')

    for r in (grouptype, doctype, pubtype):
        assert len(r) > 0

    # Build maps of existing types
    _grouptype_cache.update({x['name']: x['id'] for x in grouptype})
    _doctype_cache.update({x['name']: x['id'] for x in doctype})
    _pubtype_cache.update({x['name']: x['id'] for x in pubtype})


def fill_caches(s, api_root, groups=True, docsets=True):
    if groups:
        groups = _get_all(s, api_root, 'group')
        for g in groups:
            _group_cache[g['name']] = g
    if docsets:
        docsets = _get_all(s, api_root, 'doc_set')
        for d in docsets:
            assert d['pubtype_id'] in _pubtype_cache.values()
            assert d['doctype_id'] in _doctype_cache.values()
            _docset_cache[(d['published'], d['pubtype_id'], d['doctype_id'])] = d


def get_existing_docs(s, api_root):
    return _get_all(s, api_root, 'document')


def get_or_make_group(s, api_root, name, gtype, parent):
    if parent == name:
        # don't make circular links, it makes me sad.
        parent = None
    if gtype is not None and gtype not in _grouptype_cache:
        logging.error("Grouptype %s is not in DB!!" % (gtype,))

    # Slight hack: Maybe the parent is missing "本部" from the name?
    if parent and (parent not in _group_cache) and (parent + u'本部' in _group_cache):
        parent = parent + u'本部'

    if name in _group_cache:
        cached_group = _group_cache[name]
        if parent is not None and cached_group['parent_id'] is None:
            if parent in _group_cache:
                # update name of parent
                parentid = _group_cache[parent]['id']
                cached_group['parent_id'] = parentid
                result = s.patch(api_root + 'group/%d' % cached_group['id'],
                                 data=json.dumps({'parent_id': parentid})).json()
                assert result['parent_id']
                return cached_group
            else:
                logging.info("Update group %s parent %s unknown" % (name, parent))

        if gtype and _grouptype_cache.get(gtype) != int(cached_group['type_id']):
            logging.warn("Group %s type %s (%s) previously recorded as type %s!!" %
                         (name, gtype, _grouptype_cache.get(gtype), _group_cache[name]['type_id']))
        return cached_group

    assert(gtype is not None)

    parent_id = None
    if parent is not None:
        if parent in _group_cache:
            parent_id = _group_cache[parent]['id']
        else:
            logging.info("Update group %s parent %s unknown" % (name, parent))

    if gtype not in _grouptype_cache:
        logging.warn("Unknown group type: %s" % gtype)
        return

    obj = {'name': name, 'type_id': _grouptype_cache[gtype], 'parent_id': parent_id}
    result = s.post(api_root + 'group', data=json.dumps(obj)).json()
    if 'id' not in result:
        raise ValueError('Group add: got back %s' % result)
    _group_cache[name] = result
    return result


def get_or_make_docset(s, api_root, title, docset_type, docdir):
    docset_date = title_date(title)

    if docset_date is None:
        logging.warn(u"Couldn't guess pub date from title %s" % title)
        return

    docset_pubtype = None
    for pubtype in _pubtype_cache:
        if pubtype in title:
            docset_pubtype = _pubtype_cache[pubtype]
            break

    if docset_pubtype is None:
        logging.warn(u"Couldn't guess pubtype from title %s" % title)
        return

    if docset_type in DOCSETTYPE_MAPPING:
        docset_type = DOCSETTYPE_MAPPING[docset_type]

    if docset_type not in _doctype_cache:
        logging.warn(u"Doc type %s not in DB" % docset_type)
        # Search for similar docsets in the cache
        keys = filter(lambda x: x[0] == docset_date and x[1] == docset_pubtype, _docset_cache.keys())
        if len(keys) == 1:
            docset_type = keys[0][2]
            logging.warn(u" .. assuming type %s already in DB" % docset_type)
        else:
            return
    else:
        docset_type = _doctype_cache[docset_type]

    key = (docset_date, docset_pubtype, docset_type)
    if key in _docset_cache:
        return _docset_cache[key]

    obj = {'published': str(docset_date), 'pubtype_id': docset_pubtype, 'doctype_id': docset_type, 'path': docdir}
    result = s.post(api_root + 'doc_set', data=json.dumps(obj)).json()
    if 'id' not in result:
        raise ValueError('Docset add: got back %s' % result)
    _docset_cache[key] = result

    return result


def make_doc(s, api_root, docsetid, year, groupid, docfname, url, srcurl, fsize, pagecount, note):
    m = YEAR_RE.match(year)
    if not m:
        raise ValueError("Year %s in metadata for %s doesn't look like a year." % (year, docfname))
    year = year_to_western(*m.groups())
    obj = {'docset_id': docsetid, 'year': year, 'group_id': groupid, 'filename': docfname,
           'url': url, 'srcurl': srcurl, 'size': fsize, 'pages': pagecount, 'note': note}
    result = s.post(api_root + 'document', data=json.dumps(obj)).json()
    if 'id' not in result:
        raise ValueError('Document add: got back %s' % result)

    return result


def check_pdf(s, pdf_path, pdf_root, api_root, docs_by_url, nodefer, groupsonly):
    relative_path = pdf_path[len(pdf_root):]
    meta = metautil.get_meta(pdf_path)

    if meta is None:
        logging.warn("Skip %s which has no metadata!" % relative_path)
        return

    if 'title' not in meta or 'url' not in meta or 'srcurl' not in meta:
        logging.warn("Invalid metadata for %s!" % relative_path)
        return

    url = meta['url']
    if url in docs_by_url:
        # Verify the contents.. should be the same source
        record = docs_by_url[url]<commit_msg>Don't verify the api cert because I'm lazy and want to use localhost<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import json
import re
import logging
import requests
import subprocess
from datetime import date
from argparse import ArgumentParser

from toollib import metautil, DATE_RE, YEAR_RE, year_to_western

# Note the full-width brackets and slash.
MULTIPART_RE = re.compile(u'^(.*)(（.*／.*）.*|（その.*）)')

# Simplify a couple of docset types
DOCSETTYPE_MAPPING = {
    u'政党の届出事項の異動の届出': u'異動の届出',
    u'政治団体の届出事項の異動の届出': u'異動の届出',
}


def title_date(title):
    result = DATE_RE.search(title)

    if result is None:
        import pdb; pdb.set_trace()
        raise ValueError("Title didn't match expected format: %s" % title)

    (emp, year, month, day) = result.groups()[:4]
    year = int(year.strip())
    month = int(month.strip())
    day = int(day.strip())
    year = year_to_western(emp, year)

    return date(year, month, day)


def makefilter(field, val, op='=='):
    fparams = {'filter': [{'name': field, 'op': op, 'val': val}]}
    return fparams


def _get_all(s, api_root, otype):
    LIST_ALL_PARAM = {'results_per_page': '50000'}
    objs = s.get(api_root + otype, params=LIST_ALL_PARAM, verify=False).json()

    # FIXME: This may break eventually..
    assert objs['total_pages'] <= 1
    return objs['objects']


_group_cache = {}
_docset_cache = {}
_grouptype_cache = {}
_doctype_cache = {}
_pubtype_cache = {}


def get_existing_types(s, api_root):
    grouptype = _get_all(s, api_root, 'group_type')
    doctype = _get_all(s, api_root, 'doc_type')
    pubtype = _get_all(s, api_root, 'pub_type')

    for r in (grouptype, doctype, pubtype):
        assert len(r) > 0

    # Build maps of existing types
    _grouptype_cache.update({x['name']: x['id'] for x in grouptype})
    _doctype_cache.update({x['name']: x['id'] for x in doctype})
    _pubtype_cache.update({x['name']: x['id'] for x in pubtype})


def fill_caches(s, api_root, groups=True, docsets=True):
    if groups:
        groups = _get_all(s, api_root, 'group')
        for g in groups:
            _group_cache[g['name']] = g
    if docsets:
        docsets = _get_all(s, api_root, 'doc_set')
        for d in docsets:
            assert d['pubtype_id'] in _pubtype_cache.values()
            assert d['doctype_id'] in _doctype_cache.values()
            _docset_cache[(d['published'], d['pubtype_id'], d['doctype_id'])] = d


def get_existing_docs(s, api_root):
    return _get_all(s, api_root, 'document')


def get_or_make_group(s, api_root, name, gtype, parent):
    if parent == name:
        # don't make circular links, it makes me sad.
        parent = None
    if gtype is not None and gtype not in _grouptype_cache:
        logging.error("Grouptype %s is not in DB!!" % (gtype,))

    # Slight hack: Maybe the parent is missing "本部" from the name?
    if parent and (parent not in _group_cache) and (parent + u'本部' in _group_cache):
        parent = parent + u'本部'

    if name in _group_cache:
        cached_group = _group_cache[name]
        if parent is not None and cached_group['parent_id'] is None:
            if parent in _group_cache:
                # update name of parent
                parentid = _group_cache[parent]['id']
                cached_group['parent_id'] = parentid
                result = s.patch(api_root + 'group/%d' % cached_group['id'],
                                 data=json.dumps({'parent_id': parentid})).json()
                assert result['parent_id']
                return cached_group
            else:
                logging.info("Update group %s parent %s unknown" % (name, parent))

        if gtype and _grouptype_cache.get(gtype) != int(cached_group['type_id']):
            logging.warn("Group %s type %s (%s) previously recorded as type %s!!" %
                         (name, gtype, _grouptype_cache.get(gtype), _group_cache[name]['type_id']))
        return cached_group

    assert(gtype is not None)

    parent_id = None
    if parent is not None:
        if parent in _group_cache:
            parent_id = _group_cache[parent]['id']
        else:
            logging.info("Update group %s parent %s unknown" % (name, parent))

    if gtype not in _grouptype_cache:
        logging.warn("Unknown group type: %s" % gtype)
        return

    obj = {'name': name, 'type_id': _grouptype_cache[gtype], 'parent_id': parent_id}
    result = s.post(api_root + 'group', data=json.dumps(obj), verify=False).json()
    if 'id' not in result:
        raise ValueError('Group add: got back %s' % result)
    _group_cache[name] = result
    return result


def get_or_make_docset(s, api_root, title, docset_type, docdir):
    docset_date = title_date(title)

    if docset_date is None:
        logging.warn(u"Couldn't guess pub date from title %s" % title)
        return

    docset_pubtype = None
    for pubtype in _pubtype_cache:
        if pubtype in title:
            docset_pubtype = _pubtype_cache[pubtype]
            break

    if docset_pubtype is None:
        logging.warn(u"Couldn't guess pubtype from title %s" % title)
        return

    if docset_type in DOCSETTYPE_MAPPING:
        docset_type = DOCSETTYPE_MAPPING[docset_type]

    if docset_type not in _doctype_cache:
        logging.warn(u"Doc type %s not in DB" % docset_type)
        # Search for similar docsets in the cache
        keys = filter(lambda x: x[0] == docset_date and x[1] == docset_pubtype, _docset_cache.keys())
        if len(keys) == 1:
            docset_type = keys[0][2]
            logging.warn(u" .. assuming type %s already in DB" % docset_type)
        else:
            return
    else:
        docset_type = _doctype_cache[docset_type]

    key = (docset_date, docset_pubtype, docset_type)
    if key in _docset_cache:
        return _docset_cache[key]

    obj = {'published': str(docset_date), 'pubtype_id': docset_pubtype, 'doctype_id': docset_type, 'path': docdir}
    result = s.post(api_root + 'doc_set', data=json.dumps(obj), verify=False).json()
    if 'id' not in result:
        raise ValueError('Docset add: got back %s' % result)
    _docset_cache[key] = result

    return result


def make_doc(s, api_root, docsetid, year, groupid, docfname, url, srcurl, fsize, pagecount, note):
    m = YEAR_RE.match(year)
    if not m:
        raise ValueError("Year %s in metadata for %s doesn't look like a year." % (year, docfname))
    year = year_to_western(*m.groups())
    obj = {'docset_id': docsetid, 'year': year, 'group_id': groupid, 'filename': docfname,
           'url': url, 'srcurl': srcurl, 'size': fsize, 'pages': pagecount, 'note': note}
    result = s.post(api_root + 'document', data=json.dumps(obj), verify=False).json()
    if 'id' not in result:
        raise ValueError('Document add: got back %s' % result)

    return result


def check_pdf(s, pdf_path, pdf_root, api_root, docs_by_url, nodefer, groupsonly):
    relative_path = pdf_path[len(pdf_root):]
    meta = metautil.get_meta(pdf_path)

    if meta is None:
        logging.warn("Skip %s which has no metadata!" % relative_path)
        return

    if 'title' not in meta or 'url' not in meta or 'srcurl' not in meta:
        logging.warn("Invalid metadata for %s!" % relative_path)
        return

    url = meta['url']
    if url in docs_by_url:
        # Verify the contents.. should be the same source
        record = docs_by_url[url]
{'fraction_non_alphanumeric': 0.12119561516987812, 'fraction_numerical': 0.0019064478790767345, 'mean_word_length': 2.917844758602294, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4360082', 'n_tokens_mistral': 5167, 'n_tokens_neox': 4977, 'n_words': 1377}