<commit_before>from __future__ import with_statement
import os, os.path, re, fnmatch, shutil, shlex, string,  codecs
import model, utils, html, state, template, widgets, markup

_ConfFile = "index.py"

class Options:
    """
        An Options object, injected into the namespace as "options".

        On the commandline, options are specified either as "name=value" pairs,
        or just as names, in which case the value is True. 
    """
    def __init__(self, lst):
        self.__dict__["opts"] = {}
        for i in lst:
            parts = i.split("=", 1)
            if len(parts) == 1:
                self.__dict__["opts"][i] = True
            else:
                self.__dict__["opts"][parts[0]] = parts[1]

    def __getattr__(self, attr):
        if "opts" in self.__dict__:
            if attr in self.__dict__["opts"]:
                return self.__dict__["opts"][attr]
        return False

    def __setattr__(self, attr, value):
        self.__dict__["opts"][attr] = value

    def __str__(self):
        r = []
        for k, v in self.__dict__["opts"].items():
            r.append("%s=%s"%(k, v))
        return "\n".join(r)
            

class _Bunch:
    def __init__(self, **kwds):
        self.__dict__.update(kwds)

    def getDict(self):
        return self.__dict__


def readFrom(path):
    f = open(path)
    return f.read()


class _DocMixin:
    def _getNamespace(self):
        d = {}
        for i in reversed(list(self.attrsToRoot("namespace"))):
            d.update(i)
        d["this"] = self
        if self.namespace:
            d.update(self.namespace)
        self.namespace = d

    def _nameSrc(self, name, src):
        if src:
            return name, src
        else:
            return os.path.basename(name), name

    def _setDirectory(self, src):
        self.src = os.path.join(src, self.src)
    

class _DocHTMLPage(model.HTMLPage, _DocMixin):
    link = model.Link([])
    _verbatimComponents = "pageTitle"
    _pageTitle = None
    def __init__(self, name, title, namespace=None, src=None, pageTitle=None):
        model.HTMLPage.__init__(self)
        self.name, self.src = self._nameSrc(name, src)
        self.title = title
        if not namespace:
            self.namespace = {}
        else:
            self.namespace = namespace
        self._pageTitle = pageTitle

    def pageTitle(self, *args, **kwargs):
        t = self._pageTitle or model.HTMLPage.pageTitle(self)
        prefix = self.findAttr("titlePrefix")
        if prefix:
            return "%s%s"%(prefix, t)
        else:
            return t

    def _prime(self, app):
        model.HTMLPage._prime(self, app)
        self._getNamespace()

    def _getLayoutComponent(self, attr):
        if attr in self.namespace:
            r = self.namespace[attr]
        else:
            r = model.HTMLPage._getLayoutComponent(self, attr)
        base = None
        if self.src:
            base = os.path.dirname(self.src)
        with utils.InDir(base or "."):
            if utils.isStringLike(r) and (attr not in self._verbatimComponents):
                return template.Template(self.findAttr("markup"), unicode(r))
            else:
                return r


class Page(_DocHTMLPage):
    """
        Pages are always guaranteed to have a Directory object parent.
    """ 
    def __init__(self, name, title=None, namespace=None, src=None, pageTitle=None):
        htmlextensions=['html','htm']
        htmlext=os.path.extsep+'html'
        filename_ext=os.path.splitext(name)
        self.fileext=None
        if len(filename_ext[1])==0 or (len(filename_ext[1])>0 and (filename_ext[1][1:]).lower() not in htmlextensions): 
            self.fileext=filename_ext[1]
            name = "%s%s"%(filename_ext[0],htmlext)
        _DocHTMLPage.__init__(self, name, title, namespace, src, pageTitle)

    def _prime(self, app):
        _DocHTMLPage._prime(self, app)
        dt = self.findAttr("contentName")
        if not dt in self.namespace:
            if self.fileext is not None:
                filepath = os.path.splitext(os.path.join(self.src))[0]+self.fileext
            else:
                filepath = self.src
                
            try:
                s = codecs.open(os.path.join(filepath), "r", "utf-8").read()
            except UnicodeDecodeError:
                s = codecs.open(os.path.join(filepath), "r", "latin-1").read()            
                
            if self.fileext is not None and self.fileext.lower() in [".md", ".markdown", ".mdtext"]:
                    self.namespace[dt] = template.Template(markup.Markdown(), s)
            elif self.fileext is not None and (self.fileext).lower() in [".rst", ".rest", ".rstext"]:
                    self.namespace[dt] = template.Template(markup.RST, s)
            else:
                self.namespace[dt] = template.Template(self.findAttr("markup"), s)<commit_msg>BUGFIX: missing ()<commit_after>from __future__ import with_statement
import os, os.path, re, fnmatch, shutil, shlex, string,  codecs
import model, utils, html, state, template, widgets, markup

_ConfFile = "index.py"

class Options:
    """
        An Options object, injected into the namespace as "options".

        On the commandline, options are specified either as "name=value" pairs,
        or just as names, in which case the value is True. 
    """
    def __init__(self, lst):
        self.__dict__["opts"] = {}
        for i in lst:
            parts = i.split("=", 1)
            if len(parts) == 1:
                self.__dict__["opts"][i] = True
            else:
                self.__dict__["opts"][parts[0]] = parts[1]

    def __getattr__(self, attr):
        if "opts" in self.__dict__:
            if attr in self.__dict__["opts"]:
                return self.__dict__["opts"][attr]
        return False

    def __setattr__(self, attr, value):
        self.__dict__["opts"][attr] = value

    def __str__(self):
        r = []
        for k, v in self.__dict__["opts"].items():
            r.append("%s=%s"%(k, v))
        return "\n".join(r)
            

class _Bunch:
    def __init__(self, **kwds):
        self.__dict__.update(kwds)

    def getDict(self):
        return self.__dict__


def readFrom(path):
    f = open(path)
    return f.read()


class _DocMixin:
    def _getNamespace(self):
        d = {}
        for i in reversed(list(self.attrsToRoot("namespace"))):
            d.update(i)
        d["this"] = self
        if self.namespace:
            d.update(self.namespace)
        self.namespace = d

    def _nameSrc(self, name, src):
        if src:
            return name, src
        else:
            return os.path.basename(name), name

    def _setDirectory(self, src):
        self.src = os.path.join(src, self.src)
    

class _DocHTMLPage(model.HTMLPage, _DocMixin):
    link = model.Link([])
    _verbatimComponents = "pageTitle"
    _pageTitle = None
    def __init__(self, name, title, namespace=None, src=None, pageTitle=None):
        model.HTMLPage.__init__(self)
        self.name, self.src = self._nameSrc(name, src)
        self.title = title
        if not namespace:
            self.namespace = {}
        else:
            self.namespace = namespace
        self._pageTitle = pageTitle

    def pageTitle(self, *args, **kwargs):
        t = self._pageTitle or model.HTMLPage.pageTitle(self)
        prefix = self.findAttr("titlePrefix")
        if prefix:
            return "%s%s"%(prefix, t)
        else:
            return t

    def _prime(self, app):
        model.HTMLPage._prime(self, app)
        self._getNamespace()

    def _getLayoutComponent(self, attr):
        if attr in self.namespace:
            r = self.namespace[attr]
        else:
            r = model.HTMLPage._getLayoutComponent(self, attr)
        base = None
        if self.src:
            base = os.path.dirname(self.src)
        with utils.InDir(base or "."):
            if utils.isStringLike(r) and (attr not in self._verbatimComponents):
                return template.Template(self.findAttr("markup"), unicode(r))
            else:
                return r


class Page(_DocHTMLPage):
    """
        Pages are always guaranteed to have a Directory object parent.
    """ 
    def __init__(self, name, title=None, namespace=None, src=None, pageTitle=None):
        htmlextensions=['html','htm']
        htmlext=os.path.extsep+'html'
        filename_ext=os.path.splitext(name)
        self.fileext=None
        if len(filename_ext[1])==0 or (len(filename_ext[1])>0 and (filename_ext[1][1:]).lower() not in htmlextensions): 
            self.fileext=filename_ext[1]
            name = "%s%s"%(filename_ext[0],htmlext)
        _DocHTMLPage.__init__(self, name, title, namespace, src, pageTitle)

    def _prime(self, app):
        _DocHTMLPage._prime(self, app)
        dt = self.findAttr("contentName")
        if not dt in self.namespace:
            if self.fileext is not None:
                filepath = os.path.splitext(os.path.join(self.src))[0]+self.fileext
            else:
                filepath = self.src
                
            try:
                s = codecs.open(os.path.join(filepath), "r", "utf-8").read()
            except UnicodeDecodeError:
                s = codecs.open(os.path.join(filepath), "r", "latin-1").read()            
                
            if self.fileext is not None and self.fileext.lower() in [".md", ".markdown", ".mdtext"]:
                    self.namespace[dt] = template.Template(markup.Markdown(), s)
            elif self.fileext is not None and (self.fileext).lower() in [".rst", ".rest", ".rstext"]:
                    self.namespace[dt] = template.Template(markup.RST(), s)
            else:
                self.namespace[dt] = template.Template(self.findAttr("markup"), s)
{'fraction_non_alphanumeric': 0.12675191815856776, 'fraction_numerical': 0.003069053708439898, 'mean_word_length': 2.0350822725861533, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7106852', 'n_tokens_mistral': 3007, 'n_tokens_neox': 2934, 'n_words': 713}