<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2016 - 2018  Ternaris.
# SPDX-License-Identifier: AGPL-3.0-only

from __future__ import absolute_import, division, print_function

import os
from collections import Mapping, Sequence

from capnp.lib.capnp import _DynamicStructReader as StructReader
from capnp.lib.capnp import _DynamicListReader as ListReader
from marv_node.setid import SetID


class ListWrapper(object):
    def __init__(self, list_reader, streamdir, setdir):
        assert isinstance(list_reader, ListReader), type(list_reader)
        self._reader = list_reader
        self._streamdir = streamdir
        self._setdir = setdir

    def __getitem__(self, idx):
        # pycapnp can't handle slices
        if isinstance(idx, slice):
            x = list(self._reader)[idx]
        else:
            x = self._reader[idx]
        streamdir = self._streamdir
        setdir = self._setdir
        return (Wrapper(x, streamdir, setdir) if isinstance(x, StructReader) else
                ListWrapper(x, streamdir, setdir) if isinstance(x, ListReader) else x)

    def __iter__(self):
        streamdir = self._streamdir
        setdir = self._setdir
        return (Wrapper(x, streamdir, setdir) if isinstance(x, StructReader) else
                ListWrapper(x, streamdir, setdir) if isinstance(x, ListReader) else x
                for x in self._reader)

    def __len__(self):
        return len(self._reader)


class Wrapper(object):
    def __init__(self, struct_reader, streamdir, setdir):
        assert isinstance(struct_reader, StructReader), type(struct_reader)
        self._reader = struct_reader
        self._streamdir = streamdir  # HACK: overloaded
        self._setdir = setdir

    # @property
    # def path(self):  # HACK: overload
    #     return os.path.realpath(os.path.join(self._streamdir, self.name))

    @property
    def relpath(self):  # HACK: overload
        path = os.path.relpath(self.path, self._setdir)
        return path.lstrip('.')

    @classmethod
    def from_dict(cls, schema, data):
        setid = data.pop('id', None)
        if isinstance(setid, SetID):
            data['id0'], data['id1'] = setid.lohi
        dct = cls._unwrap(data)
        struct_reader = schema.new_message(**dct).as_reader()
        return cls(struct_reader, None, None)

    def load(self, node):
        return node.load(self._setdir)
<commit_msg>[marv] Fix loading nodes via dataset<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2016 - 2018  Ternaris.
# SPDX-License-Identifier: AGPL-3.0-only

from __future__ import absolute_import, division, print_function

import os
from collections import Mapping, Sequence

from capnp.lib.capnp import _DynamicStructReader as StructReader
from capnp.lib.capnp import _DynamicListReader as ListReader
from flask import current_app

from marv_node.setid import SetID


class ListWrapper(object):
    def __init__(self, list_reader, streamdir, setdir):
        assert isinstance(list_reader, ListReader), type(list_reader)
        self._reader = list_reader
        self._streamdir = streamdir
        self._setdir = setdir

    def __getitem__(self, idx):
        # pycapnp can't handle slices
        if isinstance(idx, slice):
            x = list(self._reader)[idx]
        else:
            x = self._reader[idx]
        streamdir = self._streamdir
        setdir = self._setdir
        return (Wrapper(x, streamdir, setdir) if isinstance(x, StructReader) else
                ListWrapper(x, streamdir, setdir) if isinstance(x, ListReader) else x)

    def __iter__(self):
        streamdir = self._streamdir
        setdir = self._setdir
        return (Wrapper(x, streamdir, setdir) if isinstance(x, StructReader) else
                ListWrapper(x, streamdir, setdir) if isinstance(x, ListReader) else x
                for x in self._reader)

    def __len__(self):
        return len(self._reader)


class Wrapper(object):
    def __init__(self, struct_reader, streamdir, setdir):
        assert isinstance(struct_reader, StructReader), type(struct_reader)
        self._reader = struct_reader
        self._streamdir = streamdir  # HACK: overloaded
        self._setdir = setdir

    # @property
    # def path(self):  # HACK: overload
    #     return os.path.realpath(os.path.join(self._streamdir, self.name))

    @property
    def relpath(self):  # HACK: overload
        path = os.path.relpath(self.path, self._setdir)
        return path.lstrip('.')

    @classmethod
    def from_dict(cls, schema, data):
        setid = data.pop('id', None)
        if isinstance(setid, SetID):
            data['id0'], data['id1'] = setid.lohi
        dct = cls._unwrap(data)
        struct_reader = schema.new_message(**dct).as_reader()
        return cls(struct_reader, None, None)

    def load(self, node):
        from marv_store import Store
        storedir = current_app.site.config.marv.storedir
        store = Store(storedir, {node.name: node})
        return store.load(self._setdir, node, default=None)

{'fraction_non_alphanumeric': 0.09904153354632587, 'fraction_numerical': 0.005191693290734824, 'mean_word_length': 2.8120243531202433, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6940240', 'n_tokens_mistral': 1535, 'n_tokens_neox': 1491, 'n_words': 389}