<commit_before>import random
import asyncio
import contextlib
import collections

import synapse.lib.coro as s_coro

from synapse.tests.utils import alist
import synapse.tests.utils as s_t_utils

class SnapTest(s_t_utils.SynTest):

    async def test_snap_eval_storm(self):

        async with self.getTestCore() as core:

            async with await core.snap() as snap:

                await snap.addNode('teststr', 'hehe')
                await snap.addNode('teststr', 'haha')

                self.len(2, await alist(snap.eval('teststr')))

                await snap.addNode('teststr', 'hoho')

                self.len(3, await alist(snap.storm('teststr')))

    async def test_snap_feed_genr(self):

        async def testGenrFunc(snap, items):
            yield await snap.addNode('teststr', 'foo')
            yield await snap.addNode('teststr', 'bar')

        async with self.getTestCore() as core:

            core.setFeedFunc('test.genr', testGenrFunc)

            await core.addFeedData('test.genr', [])
            self.len(2, await alist(core.eval('teststr')))

            async with await core.snap() as snap:
                nodes = await alist(snap.addFeedNodes('test.genr', []))
                self.len(2, nodes)

    async def test_same_node_different_object(self):
        '''
        Test the problem in which a live node might be evicted out of the snap's buidcache causing two node
        objects to be representing the same logical thing.

        Also tests that creating a node then querying it back returns the same object
        '''
        async with self.getTestCore() as core:
            async with await core.snap() as snap:
                # Reduce the buid cache so we don't have to make 100K nodes
                snap.buidcache = collections.deque(maxlen=10)

                node0 = await snap.addNode('testint', 0)

                node = await snap.getNodeByNdef(('testint', 0))

                # Test write then read coherency

                self.eq(node0.buid, node.buid)
                self.eq(id(node0), id(node))

                # Test read, then a bunch of reads, then read coherency

                await alist(snap.addNodes([(('testint', x), {}) for x in range(1, 20)]))
                nodes = await alist(snap.getNodesBy('testint'))

                self.eq(nodes[0].buid, node0.buid)
                self.eq(id(nodes[0]), id(node0))

    async def test_addNodes(self):
        async with self.getTestCore() as core:
            async with await core.snap() as snap:
                ndefs = ()
                self.len(0, await alist(snap.addNodes(ndefs)))

                ndefs = (
                    (('teststr', 'hehe'), {'props': {'.created': 5, 'tick': 3}, 'tags': {'cool': (1, 2)}}, ),
                )
                result = await alist(snap.addNodes(ndefs))
                self.len(1, result)

                node = result[0]
                self.eq(node.props.get('tick'), 3)
                self.ge(node.props.get('.created', 0), 5)
                self.eq(node.tags.get('cool'), (1, 2))

                nodes = await alist(snap.getNodesBy('teststr', 'hehe'))
                self.len(1, nodes)
                self.eq(nodes[0], node)<commit_msg>Add test to verify that nodes aren't cached forever<commit_after>import random
import asyncio
import contextlib
import collections

import synapse.lib.coro as s_coro

from synapse.tests.utils import alist
import synapse.tests.utils as s_t_utils

class SnapTest(s_t_utils.SynTest):

    async def test_snap_eval_storm(self):

        async with self.getTestCore() as core:

            async with await core.snap() as snap:

                await snap.addNode('teststr', 'hehe')
                await snap.addNode('teststr', 'haha')

                self.len(2, await alist(snap.eval('teststr')))

                await snap.addNode('teststr', 'hoho')

                self.len(3, await alist(snap.storm('teststr')))

    async def test_snap_feed_genr(self):

        async def testGenrFunc(snap, items):
            yield await snap.addNode('teststr', 'foo')
            yield await snap.addNode('teststr', 'bar')

        async with self.getTestCore() as core:

            core.setFeedFunc('test.genr', testGenrFunc)

            await core.addFeedData('test.genr', [])
            self.len(2, await alist(core.eval('teststr')))

            async with await core.snap() as snap:
                nodes = await alist(snap.addFeedNodes('test.genr', []))
                self.len(2, nodes)

    async def test_same_node_different_object(self):
        '''
        Test the problem in which a live node might be evicted out of the snap's buidcache causing two node
        objects to be representing the same logical thing.

        Also tests that creating a node then querying it back returns the same object
        '''
        async with self.getTestCore() as core:
            async with await core.snap() as snap:
                nodebuid = None
                nodeid = None
                snap.buidcache = collections.deque(maxlen=10)

                async def doit():
                    nonlocal nodeid, nodebuid
                    # Reduce the buid cache so we don't have to make 100K nodes

                    node0 = await snap.addNode('testint', 0)

                    node = await snap.getNodeByNdef(('testint', 0))

                    # Test write then read coherency

                    self.eq(node0.buid, node.buid)
                    self.eq(id(node0), id(node))
                    nodeid = id(node)
                    nodebuid = node.buid

                    # Test read, then a bunch of reads, then read coherency

                    await alist(snap.addNodes([(('testint', x), {}) for x in range(1, 20)]))
                    nodes = await alist(snap.getNodesBy('testint'))

                    self.eq(nodes[0].buid, node0.buid)
                    self.eq(id(nodes[0]), id(node0))

                await doit()  # run in separate function so that objects are gc'd

                await alist(snap.addNodes([(('testint', x), {}) for x in range(20, 30)]))

                # Test that coherency goes away (and we don't store all nodes forever)
                node = await snap.getNodeByNdef(('testint', 0))
                self.eq(nodebuid, node.buid)
                self.ne(nodeid, id(node))

    async def test_addNodes(self):
        async with self.getTestCore() as core:
            async with await core.snap() as snap:
                ndefs = ()
                self.len(0, await alist(snap.addNodes(ndefs)))

                ndefs = (
                    (('teststr', 'hehe'), {'props': {'.created': 5, 'tick': 3}, 'tags': {'cool': (1, 2)}}, ),
                )
                result = await alist(snap.addNodes(ndefs))
                self.len(1, result)

                node = result[0]
                self.eq(node.props.get('tick'), 3)
                self.ge(node.props.get('.created', 0), 5)
                self.eq(node.tags.get('cool'), (1, 2))

                nodes = await alist(snap.getNodesBy('teststr', 'hehe'))
                self.len(1, nodes)
                self.eq(nodes[0], node)
{'fraction_non_alphanumeric': 0.11564720635362966, 'fraction_numerical': 0.010450048766894246, 'mean_word_length': 1.8270972823946436, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '793898', 'n_tokens_mistral': 2194, 'n_tokens_neox': 2027, 'n_words': 566}