<commit_before>#!/usr/bin/env python
'''
An example script for RSON that handles macros and include files.

This could use a bit better error reporting, etc. but is basically functional.

TODO:
    - Use RSON error reporting
    - Perhaps move under RSON, and start distributing RSON as a package rather
      than a single module.
    - Figure out way to specify an include path
    - Maybe make it where you don't need to do str()

Copyright (c) 2010, <NAME>.  All rights reserved.

See http://code.google.com/p/rson/source/browse/trunk/license.txt

Usage:

When the loads() inside this file is used, any string which is unquoted and which
doesn't follow "=" is checked for the following special cases:

  - If it starts with @, it references another rson file which is included
  - If it contains $, then macro processing is performed.

Macro processing uses the following special characters:

  - $xxxx   starts a macro, which ends on the first character which is not in A-Za-z0-9_[].
  - $(xxx)  contains a macro

Within a macro, the characters .[] are special.  They are all separator characters between
macro identifiers.

Macros can reference any data in the same file that:

  - has already been encountered; and
  - is not in the same nested RSON structure

If the the data is in the same nested RSON structure, it can be used, but
must be dereferenced later (e.g. via str()).  This doesn't work for macros
used in include file names, where the names must be in a different top-level
RSON structure than anything using the name via $
'''


import sys
import re
sys.path.insert(0, '..')
from rson import RsonSystem

class StringProxy(object):
    macro_special_chars = '[]().$'
    macro_split = re.compile(r'([A-Za-z0-9_]+|%s)' % '|'.join('\\'+x for x in macro_special_chars)).split

    def __init__(self, token):
        self.token = token

    @property
    def dereference(self):
        token = self.token
        top_obj = token[-1].top_object
        strings = [x for x in self.macro_split(token[2]) if x]
        strings = iter(strings)
        result = []

        def recurse(result, endch=None):
            for s in strings:
                while s == '$':
                    s = handle_dollar(result)
                if s == endch:
                    return s
                result.append(s)


        def handle_dollar(result):
            lookup = []
            gotparen = False
            for s in strings:
                if s == '$':
                    if not lookup:
                        result.append('$$')
                        return ''
                    break
                elif s.replace('_', '').isalnum():
                    lookup.append(s)
                    s = ''
                elif s == '[':
                    s = recurse(lookup, ']')
                    assert s == ']'
                    s = ''
                elif s == '.':
                    continue
                elif s == '(' and not lookup:
                    gotparen = True
                else:
                    break
            if gotparen:
                assert s == ')'
                s = ''
            nexts = s

            myobj = top_obj
            for s in lookup:
                try:
                    ints = int(s)
                except ValueError:
                    pass
                else:
                    try:
                        myobj = myobj[ints]
                    except:
                        pass
                    else:
                        continue
                myobj = myobj[s]
            result.append(myobj)
            return nexts

        result = []
        recurse(result)
        if len(result) == 1:
            return result[0]
        return ''.join(str(x) for x in result)

    def __repr__(self):
        return unicode(self.dereference, 'utf-8')

class RsonMacros(RsonSystem):
    @classmethod
    def parse_unquoted_str(cls, token, unicode=unicode):
        s = token[2]
        if token[1] != '=':
            if s.startswith('@'):
                return cls.handle_include(token)
            if '$' in s:
                return cls.handle_macro(token)
        return unicode(s, 'utf-8')

    @classmethod
    def handle_include(cls, token):
        token = list(token)
        token[2] = token[2][1:]
        assert token[2] and not token[2].startswith('@'), token[2]
        fname = cls.parse_unquoted_str(tuple(token))
        # Do whatever stuff you want here to fix the filename up
        # (look in the current directory, whatever)
        f = open(fname, 'rb')
        data = f.read()
        f.close()
        return loads(data)

    @staticmethod
    def handle_macro(token):
        value = StringProxy(token)
        try:
            return value.dereference
        except:
            return value

loads = RsonMacros.dispatcher_factory()

if __name__ == '__main__':
    test1 = '''
fnames: []
    foobar.txt
foobar: @$fnames[0]
messages:
    {}
        stream: sys.stderr
        message: Welcome
    {}
        stream : $messages[0].stream
        message: Bienvenue
    '''
    print loads(test1)<commit_msg>Always process any proxy objects before returning from loads. Use lazy processing on proxy objects so that order doesn't matter.<commit_after>#!/usr/bin/env python
'''
An example script for RSON that handles macros and include files.

This could use a bit better error reporting, etc. but is basically functional.

TODO:
    - Use RSON error reporting
    - Perhaps move under RSON, and start distributing RSON as a package rather
      than a single module.
    - Figure out way to specify an include path

Copyright (c) 2010, <NAME>.  All rights reserved.

See http://code.google.com/p/rson/source/browse/trunk/license.txt

Usage:

When the loads() inside this file is used, any string which is unquoted and which
doesn't follow "=" is checked for the following special cases:

  - If it starts with @, it references another rson file which is included
  - If it contains $, then macro processing is performed.

Macro processing uses the following special characters:

  - $xxxx   starts a macro, which ends on the first character which is not in A-Za-z0-9_[].
  - $(xxx)  contains a macro

Within a macro, the characters .[] are special.  They are all separator characters between
macro identifiers.
'''


import sys
import re
sys.path.insert(0, '..')
from rson import RsonSystem, Tokenizer
from rson.baseobjects import make_hashable

class MacroProxy(object):
    macro_special_chars = '[]().$'
    macro_split = re.compile(r'([A-Za-z0-9_]+|%s)' % '|'.join('\\'+x for x in macro_special_chars)).split
    actual = None

    def __init__(self, token):
        self.token = token

    def findobj(self, lookup):
        myobj = self.token[-1].top_object
        for s in lookup:
            try:
                index = int(s)
                myobj[index]
            except:
                index = str(s)
            subobj = myobj[index]
            if isinstance(subobj, type(self)):
                myobj[index] = subobj = subobj.dereference()
            myobj = subobj
        return myobj

    def handle_include(self, s):
        assert s and not s.startswith('@'), s
        fname = self.handle_macro(s)
        # Do whatever stuff you want here to fix the filename up
        # (look in the current directory, whatever)
        f = open(fname, 'rb')
        data = f.read()
        f.close()
        return loads(data)

    def dereference(self):
        if self.actual is not None:
            return self.actual
        s = self.token[2]
        if s.startswith('@'):
            result = self.handle_include(s[1:])
        else:
            result = self.handle_macro(s)
        self.actual = result
        return result

    def handle_macro(self, s):
        strings = [x for x in self.macro_split(s) if x]
        strings = iter(strings)
        result = []

        def recurse(result, endch=None):
            for s in strings:
                while s == '$':
                    s = handle_dollar(result)
                if s == endch:
                    return s
                result.append(unicode(s, 'utf-8'))

        def handle_dollar(result):
            lookup = []
            gotparen = False
            for s in strings:
                if s == '$':
                    if not lookup:
                        result.append('$$')
                        return ''
                    break
                elif s.replace('_', '').isalnum():
                    lookup.append(s)
                    s = ''
                elif s == '[':
                    s = recurse(lookup, ']')
                    assert s == ']'
                    s = ''
                elif s == '.':
                    continue
                elif s == '(' and not lookup:
                    gotparen = True
                else:
                    break
            if gotparen:
                assert s == ')'
                s = ''
            result.append(self.findobj(lookup))
            return s

        result = []
        recurse(result)
        if len(result) == 1:
            result = result[0]
        result = u''.join(unicode(x) for x in result)
        return result

class RsonMacros(RsonSystem):

    class Tokenizer(Tokenizer):
        def __init__(self):
            self.proxylist = []

    @staticmethod
    def post_parse(tokens, value):
        for obj, index in tokens.proxylist:
            subobj = obj[index]
            if isinstance(subobj, MacroProxy):
                obj[index] = subobj.dereference()
        return value

    class default_array(list):
        def __init__(self, startlist, token):
            if token is not None:
                self.proxylist = token[-1].proxylist
            list.__init__(self, startlist)
        def append(self, value):
            if isinstance(value, MacroProxy):
                self.proxylist.append((self, len(self)))
            list.append(self, value)

    class default_object(dict):
        ''' By default, RSON objects are dictionaries that
            allow attribute access to their existing contents.
        '''
        def __init__(self):
            self.proxylist = []

        def append(self, itemlist):
            mydict = self
            value = itemlist.pop()
            itemlist = [make_hashable(x) for x in itemlist]
            lastkey = itemlist.pop()

            if itemlist:
                itemlist.reverse()
                while itemlist:
                    key = itemlist.pop()
                    subdict = mydict.get(key)
                    if not isinstance(subdict, dict):
                        subdict = mydict[key] = type(self)()
                    mydict = subdict
            if isinstance(value, dict):
                oldvalue = mydict.get(lastkey)
                if isinstance(oldvalue, dict):
                    oldvalue.update(value)
                    return
            mydict[lastkey] = value
            if isinstance(value, MacroProxy):
                self.proxylist.append((mydict, lastkey))

        def get_result(self, token):
            if token is not None:
                token[-1].proxylist.extend(self.proxylist)
            self.proxylist[:] = []
            return self

    @classmethod
    def parse_unquoted_str(cls, token, unicode=unicode):
        s = token[2]
        if token[1] != '=' and (s.startswith('@') or '$' in s):
            return MacroProxy(token)
        return unicode(s, 'utf-8')

loads = RsonMacros.dispatcher_factory()

if __name__ == '__main__':
    test1 = '''
fnames: []
    ignore me
    $(fnames[3]).txt
    ignore me too
    $lastname
foobar: @$fnames[1]
messages:
    {}
        stream: sys.stderr
        message: Welcome
    {}
        stream : $messages[0].stream
        message: Bienvenue
lastname: foobar
    '''
    print loads(test1)
{'fraction_non_alphanumeric': 0.09763288686715525, 'fraction_numerical': 0.004380422879285654, 'mean_word_length': 1.6242263483642794, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5682334', 'n_tokens_mistral': 3364, 'n_tokens_neox': 3203, 'n_words': 1039}