<commit_before>import cyst
import poppler

from numm3 import image2np, np2image
import seatbelt.seatbelt as seatbelt # ...internals

import os
import numpy as np

from twisted.web.resource import Resource
from twisted.internet import reactor

class PdfPage(cyst.Insist):
    def __init__(self, H, pdf, pageno, cachedir):
        self.H = H
        self.pdf = pdf
        self.pageno = pageno
        cyst.Insist.__init__(self, os.path.join(cachedir, "x%d-%d.jpg" % (H, pageno)))

    def serialize_computation(self, outpath):
        fname = self.pdf.dump_to_height(h=self.H, start=self.pageno, end=self.pageno+1)[0]
        os.rename(fname, outpath)

class PdfMosaic(cyst.Insist):
    def __init__(self, pdf, cachedir, W=50, H=72, N_COLS=20):
        self.W = W
        self.H = H
        self.N_COLS = N_COLS
        self.pdf = pdf
        cyst.Insist.__init__(self, os.path.join(cachedir, "50x72.jpg"))

    def serialize_computation(self, outpath):
        # Blank output image
        out = 255 * np.ones((self.H * int(np.ceil((self.pdf.npages / float(self.N_COLS)))),
                             min(self.N_COLS, self.pdf.npages) * self.W,
                             3), dtype=np.uint8)

        for idx, fpath in enumerate(self.pdf.dump_to_height(h=self.H)):
            arr = image2np(fpath)
            if arr.shape[1] > self.W:
                # Crop horizontally
                dw = arr.shape[1] - self.W
                arr = arr[:,(dw/2):(dw/2)+self.W]

            if arr.shape[0] > self.H:
                # Crop y
                arr = arr[:self.H]

            row = idx / self.N_COLS
            col = idx % self.N_COLS

            out[row*self.H:(row+1)*self.H,
                col*self.W:col*self.W + arr.shape[1]] = arr

        np2image(out, outpath)

class PdfDerivatives(Resource):
    def __init__(self, pdfpath, cacheroot):
        self.pdf = poppler.Pdf(pdfpath)
        self.cacheroot = cacheroot
        if not os.path.exists(cacheroot):
            os.makedirs(cacheroot)

        Resource.__init__(self)

        # Create lazy resources for derivatives
        mos_page = PdfMosaic(self.pdf, self.cacheroot)
        self.putChild("50x72.jpg", mos_page)
        for sz_name, h in [("small", 128), ("med", 1024), ("large", 2048)]:
            for p_no in range(self.pdf.npages):
                pp = PdfPage(h, self.pdf, p_no, self.cacheroot)
                self.putChild("x%d-%d.jpg" % (h, p_no), pp)

class LibraryDatabase(seatbelt.Database):
    def __init__(self, *a, **kw):
        self._derivs = {}       # id -> PdfDerivatives
        self._deriv_resource = Resource()

        seatbelt.Database.__init__(self, *a, **kw)

        self.putChild("_pdf_derivs", self._deriv_resource)

    def _create_derivative_resource_async(self, docid, pdfname):
        self._derivs[docid] = PdfDerivatives(
            os.path.join(self.dbpath, docid, pdfname),
            os.path.join(CACHEROOT, self.dbname, docid))

        reactor.callFromThread(
            self._with_derivative_resource_sync, docid)

    def _with_derivative_resource_sync(self, docid):
        self._deriv_resource.putChild(docid, self._derivs[docid])

        # Add PDF metadata to document
        change = False
        doc = self._all_docs[docid]
        for k,v in self._derivs[docid].pdf.meta.items():
            if doc.get(k) != v:
                doc[k] = v
                change = True
        if change:
            # Push a notification out
            self._change(doc)

    def _serve_doc(self, docid):
        seatbelt.Database._serve_doc(self, docid)

        doc = self._all_docs[docid]
        if doc.get("type", "pdf"):
            pdfs = [X for X in doc.get("_attachments", {}).keys() if X.endswith(".pdf")]
            if len(pdfs) > 0:
                pdfname = pdfs[0]

                if docid not in self._derivs:
                    reactor.callInThread(
                        self._create_derivative_resource_async,
                        docid, pdfname)

seatbelt.PARTS_BIN["Database"] = LibraryDatabase

if __name__=='__main__':
    import sys

    CACHEROOT = "/tmp/looseleaf"

    PORT = 6989
    WEB_SRC = "www/loose"
    dbdir  = sys.argv[1]

    print "http://localhost:%d" % ( PORT )

    seatbelt.serve(dbdir, port=PORT, defaultdb="looseleaf", defaultddocs=WEB_SRC)
<commit_msg>also generate 128x128 mosaics<commit_after>import cyst
import poppler

from numm3 import image2np, np2image
import seatbelt.seatbelt as seatbelt # ...internals

import os
import numpy as np

from twisted.web.resource import Resource
from twisted.internet import reactor

class PdfPage(cyst.Insist):
    def __init__(self, H, pdf, pageno, cachedir):
        self.H = H
        self.pdf = pdf
        self.pageno = pageno
        cyst.Insist.__init__(self, os.path.join(cachedir, "x%d-%d.jpg" % (H, pageno)))

    def serialize_computation(self, outpath):
        fname = self.pdf.dump_to_height(h=self.H, start=self.pageno, end=self.pageno+1)[0]
        os.rename(fname, outpath)

class PdfMosaic(cyst.Insist):
    def __init__(self, pdf, cachedir, W=50, H=72, N_COLS=20):
        self.W = W
        self.H = H
        self.N_COLS = N_COLS
        self.pdf = pdf
        cyst.Insist.__init__(self, os.path.join(cachedir, "%dx%d.jpg" % (W,H)))

    def serialize_computation(self, outpath):
        # Blank output image
        out = 255 * np.ones((self.H * int(np.ceil((self.pdf.npages / float(self.N_COLS)))),
                             min(self.N_COLS, self.pdf.npages) * self.W,
                             3), dtype=np.uint8)

        for idx, fpath in enumerate(self.pdf.dump_to_height(h=self.H, codec="png")):
            arr = image2np(fpath)
            if arr.shape[1] > self.W:
                # Crop horizontally
                dw = arr.shape[1] - self.W
                arr = arr[:,(dw/2):(dw/2)+self.W]

            if arr.shape[0] > self.H:
                # Crop y
                arr = arr[:self.H]

            row = idx / self.N_COLS
            col = idx % self.N_COLS

            out[row*self.H:(row+1)*self.H,
                col*self.W:col*self.W + arr.shape[1]] = arr

        np2image(out, outpath)

class PdfDerivatives(Resource):
    def __init__(self, pdfpath, cacheroot):
        self.pdf = poppler.Pdf(pdfpath)
        self.cacheroot = cacheroot
        if not os.path.exists(cacheroot):
            os.makedirs(cacheroot)

        Resource.__init__(self)

        # Create lazy resources for derivatives
        mos_page = PdfMosaic(self.pdf, self.cacheroot)
        gpu_mos_page = PdfMosaic(self.pdf, self.cacheroot, W=128, H=128, N_COLS=32)
        self.putChild("50x72.jpg", mos_page)
        self.putChild("128x128.jpg", gpu_mos_page)
        for sz_name, h in [("small", 128), ("med", 1024), ("large", 2048)]:
            for p_no in range(self.pdf.npages):
                pp = PdfPage(h, self.pdf, p_no, self.cacheroot)
                self.putChild("x%d-%d.jpg" % (h, p_no), pp)

class LibraryDatabase(seatbelt.Database):
    def __init__(self, *a, **kw):
        self._derivs = {}       # id -> PdfDerivatives
        self._deriv_resource = Resource()

        seatbelt.Database.__init__(self, *a, **kw)

        self.putChild("_pdf_derivs", self._deriv_resource)

    def _create_derivative_resource_async(self, docid, pdfname):
        self._derivs[docid] = PdfDerivatives(
            os.path.join(self.dbpath, docid, pdfname),
            os.path.join(CACHEROOT, self.dbname, docid))

        reactor.callFromThread(
            self._with_derivative_resource_sync, docid)

    def _with_derivative_resource_sync(self, docid):
        self._deriv_resource.putChild(docid, self._derivs[docid])

        # Add PDF metadata to document
        change = False
        doc = self._all_docs[docid]
        for k,v in self._derivs[docid].pdf.meta.items():
            if doc.get(k) != v:
                doc[k] = v
                change = True
        if change:
            # Push a notification out
            self._change(doc)

    def _serve_doc(self, docid):
        seatbelt.Database._serve_doc(self, docid)

        doc = self._all_docs[docid]
        if doc.get("type", "pdf"):
            pdfs = [X for X in doc.get("_attachments", {}).keys() if X.endswith(".pdf")]
            if len(pdfs) > 0:
                pdfname = pdfs[0]

                if docid not in self._derivs:
                    reactor.callInThread(
                        self._create_derivative_resource_async,
                        docid, pdfname)

seatbelt.PARTS_BIN["Database"] = LibraryDatabase

if __name__=='__main__':
    import sys

    CACHEROOT = "/tmp/looseleaf"

    PORT = 6989
    WEB_SRC = "www/loose"
    dbdir  = sys.argv[1]

    print "http://localhost:%d" % ( PORT )

    seatbelt.serve(dbdir, port=PORT, defaultdb="looseleaf", defaultddocs=WEB_SRC)

{'fraction_non_alphanumeric': 0.12465970961887478, 'fraction_numerical': 0.01338475499092559, 'mean_word_length': 2.1045774647887323, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '706100', 'n_tokens_mistral': 3170, 'n_tokens_neox': 2933, 'n_words': 599}