<commit_before>#!/usr/bin/env python

import re
import sys
import tei
import json
import pyld

from six.moves.urllib.parse import urljoin
from rdflib.plugin import register, Parser, Serializer
from rdflib import ConjunctiveGraph, URIRef, RDF, RDFS, BNode, Literal

from .namespaces import DC, OA, OAX, ORE, SC, SGA, TEI, EXIF, CNT


class Manifest(object):

    def __init__(self, tei_filename, manifest_uri, page=None, skip_annos=False):
        """
        Create a Shared Canvas manifest using the path to a given TEI file 
        and the URI where the manifest will be published. 

        m = Manifest("/path/to/tei.xml", "http://example.com/manifest.jsonld")
        
        Optionally pass in a page number if you are debugging and want 
        to limit the manifest to a specific page.

        Optionally set the skip_annos parameter to true to skip text annotations.
        """

        g = self.g = ConjunctiveGraph()
        self.tei = tei.Document(tei_filename)
        self.uri = URIRef(manifest_uri)

        ta = self.text_annotations = BNode()
        g.add((self.uri, ORE.aggregates, ta))
        g.add((ta, RDF.type, ORE.Aggregation))
        g.add((ta, RDF.type, SC.AnnotationList))
        g.add((ta, RDF.type, SC.Layer))
        g.add((ta, RDFS.label, Literal("Transcription")))
        g.add((ta, SC.forMotivation, SC.painting))

        za = self.zone_annotations = BNode()
        g.add((self.uri, ORE.aggregates, za))
        g.add((za, RDF.type, ORE.Aggregation))
        g.add((za, RDF.type, SC.AnnotationList))
        g.add((za, RDF.type, SC.Layer))
        g.add((za, RDFS.label, Literal("Zones")))

        ha = self.html_annotations = BNode()
        g.add((self.uri, ORE.aggregates, ha))
        g.add((ha, RDF.type, SC.AnnotationList))
        g.add((ha, RDF.type, ORE.Aggregation))
        g.add((ha, RDF.type, SC.Layer))
        g.add((ha, SC.forMotivation, SGA.reading))
        g.add((ha, RDFS.label, Literal("Reading layer")))

        xa = self.xml_annotations = BNode()
        g.add((self.uri, ORE.aggregates, xa))
        g.add((xa, RDF.type, SC.AnnotationList))
        g.add((xa, RDF.type, ORE.Aggregation))
        g.add((xa, RDF.type, SC.Layer))
        g.add((xa, SC.forMotivation, SGA.source))
        g.add((xa, RDFS.label, Literal("TEI source")))

        self._build(page, skip_annos)

    def jsonld(self, indent=2):
        # somewhat inefficient since we are serializing the json
        # and then reading it back in, to compact it with jsonld
        # jsonld's compaction actually works properly with the context
        # unlike rdflib_jsonld's at the moment. Ideally we could 
        # also use rdflib_jsonld.serlializer.from_rdf to skip 
        # the serialization, but unofortunately it seems to introduce 
        # errors into the graph.
        j = self.g.serialize(format='json-ld')
        j = json.loads(j)
        j = pyld.jsonld.compact(j, self._context())
        return j

    def tei_url(self, surface):
        path = surface.relative_path
        path = path.replace('/data', '')
        return urljoin(self.uri, path)

    def html_url(self, surface):
        # a hack until we've got a better way of coordinating the deployment
        # of the xml, html and images
        html_url = 'http://54.166.84.180/tei/readingTEI/html'
        html_url += surface.relative_path.replace('/data/tei/ox', '')
        html_url = html_url.replace('.xml', '.html')
        return html_url

    def _build(self, page=None, skip_annos=None):
        self.g.add((self.uri, RDF.type, SC.Manifest))
        self.g.add((self.uri, RDFS.label, Literal(self.tei.label)))
        self.g.add((self.uri, DC.title, Literal(self.tei.title)))
        self.g.add((self.uri, SC.agentLabel, Literal(self.tei.agent)))
        self.g.add((self.uri, SC.attributionLabel, Literal(self.tei.attribution)))
        self.g.add((self.uri, SC.dateLabel, Literal(self.tei.date)))
        self.g.add((self.uri, SGA.stateLabel, Literal(self.tei.state)))
        self.g.add((self.uri, SC.service, URIRef(self.tei.service)))
        self._add_canvases(page, skip_annos)

    def _add_canvases(self, page=None, skip_annos=None):
        g = self.g

        # add the list of sequences
        sequences_uri = BNode()
        g.add((self.uri, SC.hasSequences, sequences_uri))
        g.add((sequences_uri, RDF.type, RDF.List))

        # add the sequence, which itself is list of canvases
        sequence_uri = BNode()
        g.add((sequences_uri, RDF.first, sequence_uri))
        g.add((sequences_uri, RDF.rest, RDF.nil))
        g.add((sequence_uri, RDF.type, SC.Sequence))
        g.add((sequence_uri, RDF.type, RDF.List))
        g.add((sequence_uri, RDF.rest, RDF.nil))
        g.add((sequence_uri, RDFS.label, Literal("Physical sequence")))

        # add the image list
        image_list_uri = BNode()
        g.add((self.uri, SC.hasImageAnnotations, image_list_uri))
        g.add((image_list_uri, RDF.type, RDF.List))

        # add the canvas list
        canvas_list_uri = BNode()
        g.add((self.uri, SC.hasCanvases, canvas_list_uri))
        g.add((canvas_list_uri, RDF.type, RDF.List))

        # now add each surface
        page_count=0

        for surface in self.tei.surfaces:

            page_count += 1
            if page is not None and page_count != page:
                continue

            # add the canvas
            canvas_uri = BNode()
            g.add((canvas_uri, RDF.type, SC.Canvas))
            g.add((canvas_uri, RDFS.label, Literal(surface.folio)))
            g.add((canvas_uri, SGA.folioLabel, Literal(surface.folio)))
            g.add((canvas_uri, SGA.shelfmarkLabel, Literal(surface.shelfmark)))
            g.add((canvas_uri, SGA.handLabel, Literal(surface.hands_label)))
            g.add((canvas_uri, EXIF.height, Literal(surface.height)))<commit_msg>first attempt at adding ranges<commit_after>#!/usr/bin/env python

import re
import sys
import tei
import json
import pyld

from six.moves.urllib.parse import urljoin
from rdflib.plugin import register, Parser, Serializer
from rdflib import ConjunctiveGraph, URIRef, RDF, RDFS, BNode, Literal

from .namespaces import DC, OA, OAX, ORE, SC, SGA, TEI, EXIF, CNT


class Manifest(object):

    def __init__(self, tei_filename, manifest_uri, page=None, skip_annos=False):
        """
        Create a Shared Canvas manifest using the path to a given TEI file 
        and the URI where the manifest will be published. 

        m = Manifest("/path/to/tei.xml", "http://example.com/manifest.jsonld")
        
        Optionally pass in a page number if you are debugging and want 
        to limit the manifest to a specific page.

        Optionally set the skip_annos parameter to true to skip text annotations.
        """

        g = self.g = ConjunctiveGraph()
        self.tei = tei.Document(tei_filename)
        self.uri = URIRef(manifest_uri)

        ta = self.text_annotations = BNode()
        g.add((self.uri, ORE.aggregates, ta))
        g.add((ta, RDF.type, ORE.Aggregation))
        g.add((ta, RDF.type, SC.AnnotationList))
        g.add((ta, RDF.type, SC.Layer))
        g.add((ta, RDFS.label, Literal("Transcription")))
        g.add((ta, SC.forMotivation, SC.painting))

        za = self.zone_annotations = BNode()
        g.add((self.uri, ORE.aggregates, za))
        g.add((za, RDF.type, ORE.Aggregation))
        g.add((za, RDF.type, SC.AnnotationList))
        g.add((za, RDF.type, SC.Layer))
        g.add((za, RDFS.label, Literal("Zones")))

        ha = self.html_annotations = BNode()
        g.add((self.uri, ORE.aggregates, ha))
        g.add((ha, RDF.type, SC.AnnotationList))
        g.add((ha, RDF.type, ORE.Aggregation))
        g.add((ha, RDF.type, SC.Layer))
        g.add((ha, SC.forMotivation, SGA.reading))
        g.add((ha, RDFS.label, Literal("Reading layer")))

        xa = self.xml_annotations = BNode()
        g.add((self.uri, ORE.aggregates, xa))
        g.add((xa, RDF.type, SC.AnnotationList))
        g.add((xa, RDF.type, ORE.Aggregation))
        g.add((xa, RDF.type, SC.Layer))
        g.add((xa, SC.forMotivation, SGA.source))
        g.add((xa, RDFS.label, Literal("TEI source")))

        self._build(page, skip_annos)

    def jsonld(self, indent=2):
        # somewhat inefficient since we are serializing the json
        # and then reading it back in, to compact it with jsonld
        # jsonld's compaction actually works properly with the context
        # unlike rdflib_jsonld's at the moment. Ideally we could 
        # also use rdflib_jsonld.serlializer.from_rdf to skip 
        # the serialization, but unofortunately it seems to introduce 
        # errors into the graph.
        j = self.g.serialize(format='json-ld')
        j = json.loads(j)
        j = pyld.jsonld.compact(j, self._context())
        return j

    def tei_url(self, surface):
        path = surface.relative_path
        path = path.replace('/data', '')
        return urljoin(self.uri, path)

    def html_url(self, surface):
        # a hack until we've got a better way of coordinating the deployment
        # of the xml, html and images
        html_url = 'http://192.168.3.11/tei/readingTEI/html'
        html_url += surface.relative_path.replace('/data/tei/ox', '')
        html_url = html_url.replace('.xml', '.html')
        return html_url

    def _build(self, page=None, skip_annos=None):
        self.g.add((self.uri, RDF.type, SC.Manifest))
        self.g.add((self.uri, RDFS.label, Literal(self.tei.label)))
        self.g.add((self.uri, DC.title, Literal(self.tei.title)))
        self.g.add((self.uri, SC.agentLabel, Literal(self.tei.agent)))
        self.g.add((self.uri, SC.attributionLabel, Literal(self.tei.attribution)))
        self.g.add((self.uri, SC.dateLabel, Literal(self.tei.date)))
        self.g.add((self.uri, SGA.stateLabel, Literal(self.tei.state)))
        self.g.add((self.uri, SC.service, URIRef(self.tei.service)))
        self._add_canvases(page, skip_annos)

    def _add_canvases(self, page=None, skip_annos=None):
        g = self.g

        # add the list of sequences
        sequences_uri = BNode()
        g.add((self.uri, SC.hasSequences, sequences_uri))
        g.add((sequences_uri, RDF.type, RDF.List))

        # add the sequence, which itself is list of canvases
        sequence_uri = BNode()
        g.add((sequences_uri, RDF.first, sequence_uri))
        g.add((sequences_uri, RDF.rest, RDF.nil))
        g.add((sequence_uri, RDF.type, SC.Sequence))
        g.add((sequence_uri, RDF.type, RDF.List))
        g.add((sequence_uri, RDF.rest, RDF.nil))
        g.add((sequence_uri, RDFS.label, Literal("Physical sequence")))

        # add the list of ranges
        ranges_uri = BNode()
        g.add((self.uri, SC.hasRanges, ranges_uri))
        g.add((ranges_uri, RDF.type, RDF.List))

        # add each range, which itself is list of canvases
        for ran in self.tei.ranges:
            range_uri = BNode()
            g.add((ranges_uri, RDF.first, range_uri))
            g.add((ranges_uri, RDF.rest, RDF.nil))
            g.add((range_uri, RDF.type, SC.Range))
            g.add((range_uri, RDF.type, RDF.List))
            g.add((range_uri, RDF.rest, RDF.nil))
            g.add((range_uri, RDFS.label, Literal(ran)))

        # add the image list
        image_list_uri = BNode()
        g.add((self.uri, SC.hasImageAnnotations, image_list_uri))
        g.add((image_list_uri, RDF.type, RDF.List))

        # add the canvas list
        canvas_list_uri = BNode()
        g.add((self.uri, SC.hasCanvases, canvas_list_uri))
        g.add((canvas_list_uri, RDF.type, RDF.List))

        # now add each surface
        page_count=0

        for surface in self.tei.surfaces:

            page_count += 1
            if page is not None and page_count != page:
                continue

            # add the canvas
            canvas_uri = BNode()
            g.add((canvas_uri, RDF.type, SC.Canvas))
            g.add((canvas_uri, RDFS.label, Literal(surface.folio)))
            g.add((canvas_uri, SGA.folioLabel, Literal(surface.folio)))
            g.add((canvas_uri, SGA.shelfmarkLabel, Literal(surface.shelfmark)))
            g.add((canvas_uri, SGA.handLabel, Literal(surface.hands_label)))
            g.add((canvas_uri, EXIF.height, Literal(surface.height)))
{'fraction_non_alphanumeric': 0.13114487689932558, 'fraction_numerical': 0.0020313642642398634, 'mean_word_length': 2.4797851286400903, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2230981', 'n_tokens_mistral': 4254, 'n_tokens_neox': 4080, 'n_words': 947}