<commit_before># -*- coding: iso-8859-1 -*-
# 
"""

"""

import hashlib
import datetime
import urllib, urllib2
from gettext import gettext as _

from dateutil.parser import parse as dateparse

from functools import partial
from itertools import islice, dropwhile

import amara
from amara import tree, bindery
from amara.bindery import html
from amara.lib.util import first_item
from amara.lib import inputsource
#from amara import inputsource as baseinputsource
from amara.lib.irihelpers import resolver as baseresolver
#from amara.namespaces import *
#from amara.xslt import transform
#from amara.writers.struct import *
#from amara.bindery.html import parse as htmlparse
from amara.lib import U
from amara.lib.iri import split_uri_ref, split_fragment, relativize, absolutize, IriError, join
#from amara.bindery.model import examplotron_model, generate_metadata, metadata_dict
from amara.bindery.util import dispatcher, node_handler, property_sequence_getter
from amara.thirdparty import json

from akara import httplib2
from akara.util import copy_auth
from akara.util.moin import wiki_uri, wiki_normalize, ORIG_BASE_HEADER, DOCBOOK_IMT, RDF_IMT, HTML_IMT, XML_IMT
from akara.services import simple_service

try:
    from akara import logger
except ImportError:
    logger = None

from zenlib import zservice, service_proxy
from zenlib.util import find_peer_service

MOINREST_SERVICE_ID = 'http://purl.org/xml3k/akara/services/demo/moinrest'


def cleanup_text_blocks(text):
    return '\n'.join([line.strip() for line in text.splitlines() ])


def linkify(link, wikibase):
    '''
    Try to construct Moin-style link markup from a given link
    '''
    rel = relativize(link, wikibase)
    if rel:
        return u'[[%s]]'%rel
    else:
        return u'[[%s]]'%link


def zenuri_to_moinrest(environ, uri=None):
    #self_end_point = environ['SCRIPT_NAME'].rstrip('/') #$ServerPath/zen
    #self_end_point = request_uri(environ, include_query=False).rstrip('/')
    #self_end_point = guess_self_uri(environ)
    #absolutize(environ['SCRIPT_NAME'].rstrip('/'), request_uri(environ, include_query=False))
    #logger.debug('moinrest_uri: ' + repr((self_end_point, MOINREST_SERVICE_ID)))
    moinresttop = find_peer_service(environ, MOINREST_SERVICE_ID)
    #logger.debug('zenuri_to_moinrest: ' + repr((moinresttop, environ['PATH_INFO'], environ['SCRIPT_NAME'])))
    if uri:
        if uri.startswith(moinresttop):
        #if moinresttop.split('/')[-1] == environ['SCRIPT_NAME'].strip('/'):
            #It is already a moin URL
            return uri or request_uri(environ)
        else:
            raise NotImplementedError('For now a Zen uri is required')
    else:
        moinrest_uri = join(moinresttop, environ['PATH_INFO'].lstrip('/'))
    logger.debug('moinrest_uri: ' + repr(moinrest_uri))
    return moinrest_uri


class moinrest_resolver(baseresolver):
    """
    Resolver that uses a specialized URL opener
    """
    def __init__(self, authorizations=None, lenient=True, opener=None):
        """
        """
        self.opener = opener or urllib2.build_opener()
        self.last_lookup_headers = None
        baseresolver.__init__(self, authorizations, lenient)

    def resolve(self, uri, base=None):
        if not isinstance(uri, urllib2.Request):
            if base is not None:
                uri = self.absolutize(uri, base)
            req = urllib2.Request(uri)
        else:
            req, uri = uri, uri.get_full_url()
        try:
            #stream = self.opener(uri)
            resp = self.opener.open(req)
            stream = resp
            self.last_lookup_headers = resp.info()
        except IOError, e:
            raise IriError(IriError.RESOURCE_ERROR,
                               uri=uri, loc=uri, msg=str(e))
        return stream


#FIXME: consolidate URIs, opener, etc. into an InputSource derivative
#class inputsource(baseinputsource):
#    def __new__(cls, arg, uri=None, encoding=None, resolver=None, sourcetype=0, opener=None):
#       isrc = baseinputsource.__new__(cls, arg, uri, encoding, resolver, sourcetype)
#        isrc.opener = opener
#        return isrc

#    def __init__(self, arg, uri=None, encoding=None, resolver=None, sourcetype=0, opener=None):
#        baseinputsource.__init__(cls, arg, uri, encoding, resolver, sourcetype)

RESOURCE_TYPE_TYPE = u'http://purl.org/xml3k/akara/cms/resource-type'


UNSPECIFIED = object()

class node(object):
    '''
    Akara Moin/CMS node, a Moin wiki page that follows a template to direct workflow
    activity, including metadata extraction
    '''
    AKARA_TYPE = u'http://purl.org/xml3k/akara/cms/resource-type'
    NODES = {}
    ENDPOINTS = None

    _instance_cache = {}
    H = httplib2.Http('.cache')

    @staticmethod
    def lookup(rest_uri, opener=None, resolver=None):
        '''
        rest_uri - URI of the moinrest-wrapped version of the page
        opener - for specializing the HTTP request (e.g. to use auth)
        '''
        if rest_uri in node._instance_cache:
            #FIXME: Check for cache invalidation first. Right now this cache will last as long as the akara process
            return node._instance_cache[rest_uri]
        if not resolver:
            resolver = moinrest_resolver(opener=opener)
        if logger: logger.debug('node.lookup rest_uri: ' + rest_uri)
        isrc, resp = parse_moin_xml(rest_uri, resolver=resolver)
        doc = bindery.parse(isrc)
        #doc = bindery.parse(isrc, standalone=True, model=MOIN_DOCBOOK_MODEL)
        original_base, wrapped_base, original_page = resp.info()[ORIG_BASE_HEADER].split()
        atype = resource_type.construct_id(doc, original_base, wrapped_base, rest_uri)
        if logger: logger.debug('node.lookup akara type: ' + atype)
        #Older Moin CMS resource types are implemented by registration to the global node.NODES
        #Newer Moin CMS resource types are implemented by discovery of a URL,
        #to which a POST request executes the desired action
        cls = node.NODES.get(atype, node)
        instance = cls(doc, rest_uri, original_base, wrapped_base, akara_type=atype, resolver=resolver)
        node._instance_cache[rest_uri] = instance
        return instance
        #return node.ENDPOINTS and (rest_uri, akara_type, node.ENDPOINTS[akara_type], doc, metadata, original_wiki_base)

    @staticmethod
    def create(resource_type, body, ctype, opener=None, resolver=None):
        '''
        resource_type - type of the new resource to be created
        body - input information or document required to construct the resource page, according to the rule sheet
        '''
        if not resolver:
            resolver = moinrest_resolver(opener=opener)
        resource_type = node.lookup(resource_type, resolver=resolver)
        handler = resource_type.run_rulesheet('POST', ctype)
        url, wikified = handler(body)

        resp, content = self.H.request(url, "PUT", body=wikified, headers={'Content-Type' : 'text/plain'})

        return
    
    def __init__(self, doc, rest_uri, original_base, wrapped_base, akara_type=None, resolver=None):
        '''
        rest_uri - the full URI to the Moin/REST wrapper for this page
        relative - the URI of this page relative to the Wiki base
        '''
        self.doc = doc
        self.rest_uri = rest_uri
        self.original_base = original_base
        self.wrapped_base = wrapped_base
        self.resolver = resolver
        self.rulesheet = None
        self.resource_type = None
        if node.ENDPOINTS and akara_type in node.ENDPOINTS:
            #Uses old-style Akara services registered to endpoints
            self.endpoint = node.ENDPOINTS[akara_type]
        else:
            #Uses rulesheets
            self.endpoint = None
            #FIXME: Inelegant not to use polymorphism for the RESOURCE_TYPE_TYPE test
            if akara_type and isinstance(akara_type, basestring) and akara_type != RESOURCE_TYPE_TYPE:
                try:
                    self.resource_type = node.lookup(akara_type, resolver=self.resolver)
                except (KeyboardInterrupt, SystemExit):
                    raise
                except Exception as e:
                    #If there is an error looking up the resource type, just leave off.  Some operations will then fail
                    logger.debug('Exception looking up resource type %s: %s'%(akara_type, repr(e)))
                    pass
        return

    def load(self):
        raise NotImplementedError

    #def render(self):
    #    raise NotImplementedError

    def up_to_date(self, force_update=False):
        '''
        Checks whether there needs to be an update of the output
        '''
        #By default just always update
        return False

    def akara_type(self):
        return resource_type.construct_id(self.doc, self.original_base, self.wrapped_base, self.rest_uri)

    def section(self, title):
        '''
        Helper to extract content from a specific section within the page
        '''
        #FIXME: rethink this "caching" business
        logger.debug("section_titled: " + repr(title))
        return first_item(self.doc.xml_select(u'//*[@title = "%s"]'%title))

    def definition_list(self, list_path, contextnode=None, patterns=None):
        '''
        Helper to construct a dictionary from an indicated definition list on the page
        '''
        #FIXME: rethink this "caching" business
        #Use defaultdict instead, for performance
        #patterns = patterns or {None: lambda x: U(x) if x else None}
        patterns = patterns or {None: lambda x: x}
        contextnode = contextnode or self.doc.s1
        top = contextnode.xml_select(list_path)
        if not top:
            return None
        #Go over the glossentries, and map from term to def, applying the matching
        #Unit transform function from the patterns dict
        result = dict((U(l), patterns.get(U(l), patterns[None])(first_item(l.xml_select(u'following-sibling::item'))))
                      for l in top[0].label)
        logger.debug("definition_list: " + repr(result))
        return result

    def definition_section(self, title, patterns=None):
        '''
        Helper to extract the first definition list from a named section
        '''
        return self.definition_list(u'.//gloss', contextnode=self.section(title), patterns=patterns)

    def get_proxy(self, method, accept=None):
        return self.resource_type.run_rulesheet(method, accept)

    def absolute_wrap(self, link):
        link = '/' + link.lstrip('/')
        #if logger: logger.debug('absolute_wrap: ' + repr((self.original_base, self.wrapped_base, link, self.rest_uri)))
        wrapped_link, orig_link = wiki_uri(self.original_base, self.wrapped_base, link, self.rest_uri)
        #if logger: logger.debug('absolute_wrap: ' + repr((link, wrapped_link, orig_link)))
        return wrapped_link


node.NODES[node.AKARA_TYPE] = node

def parse_moin_xml(uri, resolver=None):
    #Stupid Moin XML export uses bogus nbsps, so this function encapsulates the kludge<commit_msg>Updating two calls to run_rulesheet to use the new signature (includes "environ"). Uses an empty dictionary for now, as I'm not sure how to get the environ to run_rulesheet via get_proxy.<commit_after># -*- coding: iso-8859-1 -*-
# 
"""

"""

import hashlib
import datetime
import urllib, urllib2
from gettext import gettext as _

from dateutil.parser import parse as dateparse

from functools import partial
from itertools import islice, dropwhile

import amara
from amara import tree, bindery
from amara.bindery import html
from amara.lib.util import first_item
from amara.lib import inputsource
#from amara import inputsource as baseinputsource
from amara.lib.irihelpers import resolver as baseresolver
#from amara.namespaces import *
#from amara.xslt import transform
#from amara.writers.struct import *
#from amara.bindery.html import parse as htmlparse
from amara.lib import U
from amara.lib.iri import split_uri_ref, split_fragment, relativize, absolutize, IriError, join
#from amara.bindery.model import examplotron_model, generate_metadata, metadata_dict
from amara.bindery.util import dispatcher, node_handler, property_sequence_getter
from amara.thirdparty import json

from akara import httplib2
from akara.util import copy_auth
from akara.util.moin import wiki_uri, wiki_normalize, ORIG_BASE_HEADER, DOCBOOK_IMT, RDF_IMT, HTML_IMT, XML_IMT
from akara.services import simple_service

try:
    from akara import logger
except ImportError:
    logger = None

from zenlib import zservice, service_proxy
from zenlib.util import find_peer_service

MOINREST_SERVICE_ID = 'http://purl.org/xml3k/akara/services/demo/moinrest'


def cleanup_text_blocks(text):
    return '\n'.join([line.strip() for line in text.splitlines() ])


def linkify(link, wikibase):
    '''
    Try to construct Moin-style link markup from a given link
    '''
    rel = relativize(link, wikibase)
    if rel:
        return u'[[%s]]'%rel
    else:
        return u'[[%s]]'%link


def zenuri_to_moinrest(environ, uri=None):
    #self_end_point = environ['SCRIPT_NAME'].rstrip('/') #$ServerPath/zen
    #self_end_point = request_uri(environ, include_query=False).rstrip('/')
    #self_end_point = guess_self_uri(environ)
    #absolutize(environ['SCRIPT_NAME'].rstrip('/'), request_uri(environ, include_query=False))
    #logger.debug('moinrest_uri: ' + repr((self_end_point, MOINREST_SERVICE_ID)))
    moinresttop = find_peer_service(environ, MOINREST_SERVICE_ID)
    #logger.debug('zenuri_to_moinrest: ' + repr((moinresttop, environ['PATH_INFO'], environ['SCRIPT_NAME'])))
    if uri:
        if uri.startswith(moinresttop):
        #if moinresttop.split('/')[-1] == environ['SCRIPT_NAME'].strip('/'):
            #It is already a moin URL
            return uri or request_uri(environ)
        else:
            raise NotImplementedError('For now a Zen uri is required')
    else:
        moinrest_uri = join(moinresttop, environ['PATH_INFO'].lstrip('/'))
    logger.debug('moinrest_uri: ' + repr(moinrest_uri))
    return moinrest_uri


class moinrest_resolver(baseresolver):
    """
    Resolver that uses a specialized URL opener
    """
    def __init__(self, authorizations=None, lenient=True, opener=None):
        """
        """
        self.opener = opener or urllib2.build_opener()
        self.last_lookup_headers = None
        baseresolver.__init__(self, authorizations, lenient)

    def resolve(self, uri, base=None):
        if not isinstance(uri, urllib2.Request):
            if base is not None:
                uri = self.absolutize(uri, base)
            req = urllib2.Request(uri)
        else:
            req, uri = uri, uri.get_full_url()
        try:
            #stream = self.opener(uri)
            resp = self.opener.open(req)
            stream = resp
            self.last_lookup_headers = resp.info()
        except IOError, e:
            raise IriError(IriError.RESOURCE_ERROR,
                               uri=uri, loc=uri, msg=str(e))
        return stream


#FIXME: consolidate URIs, opener, etc. into an InputSource derivative
#class inputsource(baseinputsource):
#    def __new__(cls, arg, uri=None, encoding=None, resolver=None, sourcetype=0, opener=None):
#       isrc = baseinputsource.__new__(cls, arg, uri, encoding, resolver, sourcetype)
#        isrc.opener = opener
#        return isrc

#    def __init__(self, arg, uri=None, encoding=None, resolver=None, sourcetype=0, opener=None):
#        baseinputsource.__init__(cls, arg, uri, encoding, resolver, sourcetype)

RESOURCE_TYPE_TYPE = u'http://purl.org/xml3k/akara/cms/resource-type'


UNSPECIFIED = object()

class node(object):
    '''
    Akara Moin/CMS node, a Moin wiki page that follows a template to direct workflow
    activity, including metadata extraction
    '''
    AKARA_TYPE = u'http://purl.org/xml3k/akara/cms/resource-type'
    NODES = {}
    ENDPOINTS = None

    _instance_cache = {}
    H = httplib2.Http('.cache')

    @staticmethod
    def lookup(rest_uri, opener=None, resolver=None):
        '''
        rest_uri - URI of the moinrest-wrapped version of the page
        opener - for specializing the HTTP request (e.g. to use auth)
        '''
        if rest_uri in node._instance_cache:
            #FIXME: Check for cache invalidation first. Right now this cache will last as long as the akara process
            return node._instance_cache[rest_uri]
        if not resolver:
            resolver = moinrest_resolver(opener=opener)
        if logger: logger.debug('node.lookup rest_uri: ' + rest_uri)
        isrc, resp = parse_moin_xml(rest_uri, resolver=resolver)
        doc = bindery.parse(isrc)
        #doc = bindery.parse(isrc, standalone=True, model=MOIN_DOCBOOK_MODEL)
        original_base, wrapped_base, original_page = resp.info()[ORIG_BASE_HEADER].split()
        atype = resource_type.construct_id(doc, original_base, wrapped_base, rest_uri)
        if logger: logger.debug('node.lookup akara type: ' + atype)
        #Older Moin CMS resource types are implemented by registration to the global node.NODES
        #Newer Moin CMS resource types are implemented by discovery of a URL,
        #to which a POST request executes the desired action
        cls = node.NODES.get(atype, node)
        instance = cls(doc, rest_uri, original_base, wrapped_base, akara_type=atype, resolver=resolver)
        node._instance_cache[rest_uri] = instance
        return instance
        #return node.ENDPOINTS and (rest_uri, akara_type, node.ENDPOINTS[akara_type], doc, metadata, original_wiki_base)

    @staticmethod
    def create(resource_type, body, ctype, opener=None, resolver=None):
        '''
        resource_type - type of the new resource to be created
        body - input information or document required to construct the resource page, according to the rule sheet
        '''
        if not resolver:
            resolver = moinrest_resolver(opener=opener)
        resource_type = node.lookup(resource_type, resolver=resolver)
        handler = resource_type.run_rulesheet({},'POST', ctype)
        url, wikified = handler(body)

        resp, content = self.H.request(url, "PUT", body=wikified, headers={'Content-Type' : 'text/plain'})

        return
    
    def __init__(self, doc, rest_uri, original_base, wrapped_base, akara_type=None, resolver=None):
        '''
        rest_uri - the full URI to the Moin/REST wrapper for this page
        relative - the URI of this page relative to the Wiki base
        '''
        self.doc = doc
        self.rest_uri = rest_uri
        self.original_base = original_base
        self.wrapped_base = wrapped_base
        self.resolver = resolver
        self.rulesheet = None
        self.resource_type = None
        if node.ENDPOINTS and akara_type in node.ENDPOINTS:
            #Uses old-style Akara services registered to endpoints
            self.endpoint = node.ENDPOINTS[akara_type]
        else:
            #Uses rulesheets
            self.endpoint = None
            #FIXME: Inelegant not to use polymorphism for the RESOURCE_TYPE_TYPE test
            if akara_type and isinstance(akara_type, basestring) and akara_type != RESOURCE_TYPE_TYPE:
                try:
                    self.resource_type = node.lookup(akara_type, resolver=self.resolver)
                except (KeyboardInterrupt, SystemExit):
                    raise
                except Exception as e:
                    #If there is an error looking up the resource type, just leave off.  Some operations will then fail
                    logger.debug('Exception looking up resource type %s: %s'%(akara_type, repr(e)))
                    pass
        return

    def load(self):
        raise NotImplementedError

    #def render(self):
    #    raise NotImplementedError

    def up_to_date(self, force_update=False):
        '''
        Checks whether there needs to be an update of the output
        '''
        #By default just always update
        return False

    def akara_type(self):
        return resource_type.construct_id(self.doc, self.original_base, self.wrapped_base, self.rest_uri)

    def section(self, title):
        '''
        Helper to extract content from a specific section within the page
        '''
        #FIXME: rethink this "caching" business
        logger.debug("section_titled: " + repr(title))
        return first_item(self.doc.xml_select(u'//*[@title = "%s"]'%title))

    def definition_list(self, list_path, contextnode=None, patterns=None):
        '''
        Helper to construct a dictionary from an indicated definition list on the page
        '''
        #FIXME: rethink this "caching" business
        #Use defaultdict instead, for performance
        #patterns = patterns or {None: lambda x: U(x) if x else None}
        patterns = patterns or {None: lambda x: x}
        contextnode = contextnode or self.doc.s1
        top = contextnode.xml_select(list_path)
        if not top:
            return None
        #Go over the glossentries, and map from term to def, applying the matching
        #Unit transform function from the patterns dict
        result = dict((U(l), patterns.get(U(l), patterns[None])(first_item(l.xml_select(u'following-sibling::item'))))
                      for l in top[0].label)
        logger.debug("definition_list: " + repr(result))
        return result

    def definition_section(self, title, patterns=None):
        '''
        Helper to extract the first definition list from a named section
        '''
        return self.definition_list(u'.//gloss', contextnode=self.section(title), patterns=patterns)

    def get_proxy(self, method, accept=None):
        return self.resource_type.run_rulesheet({},method, accept)

    def absolute_wrap(self, link):
        link = '/' + link.lstrip('/')
        #if logger: logger.debug('absolute_wrap: ' + repr((self.original_base, self.wrapped_base, link, self.rest_uri)))
        wrapped_link, orig_link = wiki_uri(self.original_base, self.wrapped_base, link, self.rest_uri)
        #if logger: logger.debug('absolute_wrap: ' + repr((link, wrapped_link, orig_link)))
        return wrapped_link


node.NODES[node.AKARA_TYPE] = node

def parse_moin_xml(uri, resolver=None):
    #Stupid Moin XML export uses bogus nbsps, so this function encapsulates the kludge
{'fraction_non_alphanumeric': 0.09423989966854789, 'fraction_numerical': 0.0017020514198692108, 'mean_word_length': 2.9805669459796755, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3550857', 'n_tokens_mistral': 6812, 'n_tokens_neox': 6674, 'n_words': 1855}