def test_plain_ngrams_processor(self):
    from collections import deque

    def ngram_processor(items, ngram_len):
        '\n            Given a sequence or iterable of arbitrary items, return an iterator of\n            item ngrams tuples of length ngram_len. Buffers at most ngram_len iterable\n            items.\n\n            For example::\n\n            >>> list(ngram_processor([1, 2, 3, 4, 5], ngram_len=3))\n            [(1, 2, 3), (2, 3, 4), (3, 4, 5)]\n            '
        ngram = deque()
        current_len = 0
        for item in items:
            if (current_len == ngram_len):
                (yield tuple(ngram))
                ngram.popleft()
                current_len -= 1
            ngram.append(item)
            current_len += 1
        (yield tuple(ngram))
    text = u'/*COMMENT\n            COMMENT COMMENT\n            - COMMENT\n            */\n            public static boolean activateSearchResultView() {\n            String defaultPerspectiveId= SearchUI.getDefaultPerspectiveId();\n            if (defaultPerspectiveId != null) {\n                IWorkbenchWindow window= SearchPlugin.getActiveWorkbenchWindow();\n                if (window != null && window.getShell() != null && !window.getShell().isDisposed()) {\n                    try {\n                        PlatformUI.getWorkbench().showPerspective(defaultPerspectiveId, window);\n                    } catch (WorkbenchException ex) {\n                        // show view in current perspective\n                    }\n                }\n            }'
    expected = [(u'comment', u'comment', u'comment', u'comment', u'public', u'static'), (u'comment', u'comment', u'comment', u'public', u'static', u'boolean'), (u'comment', u'comment', u'public', u'static', u'boolean', u'activatesearchresultview'), (u'comment', u'public', u'static', u'boolean', u'activatesearchresultview', u'string'), (u'public', u'static', u'boolean', u'activatesearchresultview', u'string', u'defaultperspectiveid'), (u'static', u'boolean', u'activatesearchresultview', u'string', u'defaultperspectiveid', u'searchui'), (u'boolean', u'activatesearchresultview', u'string', u'defaultperspectiveid', u'searchui', u'getdefaultperspectiveid'), (u'activatesearchresultview', u'string', u'defaultperspectiveid', u'searchui', u'getdefaultperspectiveid', u'if'), (u'string', u'defaultperspectiveid', u'searchui', u'getdefaultperspectiveid', u'if', u'defaultperspectiveid'), (u'defaultperspectiveid', u'searchui', u'getdefaultperspectiveid', u'if', u'defaultperspectiveid', u'null'), (u'searchui', u'getdefaultperspectiveid', u'if', u'defaultperspectiveid', u'null', u'iworkbenchwindow'), (u'getdefaultperspectiveid', u'if', u'defaultperspectiveid', u'null', u'iworkbenchwindow', u'window'), (u'if', u'defaultperspectiveid', u'null', u'iworkbenchwindow', u'window', u'searchplugin'), (u'defaultperspectiveid', u'null', u'iworkbenchwindow', u'window', u'searchplugin', u'getactiveworkbenchwindow'), (u'null', u'iworkbenchwindow', u'window', u'searchplugin', u'getactiveworkbenchwindow', u'if'), (u'iworkbenchwindow', u'window', u'searchplugin', u'getactiveworkbenchwindow', u'if', u'window'), (u'window', u'searchplugin', u'getactiveworkbenchwindow', u'if', u'window', u'null'), (u'searchplugin', u'getactiveworkbenchwindow', u'if', u'window', u'null', u'window'), (u'getactiveworkbenchwindow', u'if', u'window', u'null', u'window', u'getshell'), (u'if', u'window', u'null', u'window', u'getshell', u'null'), (u'window', u'null', u'window', u'getshell', u'null', u'window'), (u'null', u'window', u'getshell', u'null', u'window', u'getshell'), (u'window', u'getshell', u'null', u'window', u'getshell', u'isdisposed'), (u'getshell', u'null', u'window', u'getshell', u'isdisposed', u'try'), (u'null', u'window', u'getshell', u'isdisposed', u'try', u'platformui'), (u'window', u'getshell', u'isdisposed', u'try', u'platformui', u'getworkbench'), (u'getshell', u'isdisposed', u'try', u'platformui', u'getworkbench', u'showperspective'), (u'isdisposed', u'try', u'platformui', u'getworkbench', u'showperspective', u'defaultperspectiveid'), (u'try', u'platformui', u'getworkbench', u'showperspective', u'defaultperspectiveid', u'window'), (u'platformui', u'getworkbench', u'showperspective', u'defaultperspectiveid', u'window', u'catch'), (u'getworkbench', u'showperspective', u'defaultperspectiveid', u'window', u'catch', u'workbenchexception'), (u'showperspective', u'defaultperspectiveid', u'window', u'catch', u'workbenchexception', u'ex'), (u'defaultperspectiveid', u'window', u'catch', u'workbenchexception', u'ex', u'show'), (u'window', u'catch', u'workbenchexception', u'ex', u'show', u'view'), (u'catch', u'workbenchexception', u'ex', u'show', u'view', u'in'), (u'workbenchexception', u'ex', u'show', u'view', u'in', u'current'), (u'ex', u'show', u'view', u'in', u'current', u'perspective')]
    unigrams = (x.value for x in unigram_splitter(text.splitlines()))
    result = list(ngram_processor(unigrams, ngram_len=6))
    assert (expected == result)
def test_plain_ngrams_processor(self):
    from collections import deque

    def ngram_processor(items, ngram_len):
        '\n            Given a sequence or iterable of arbitrary items, return an iterator of\n            item ngrams tuples of length ngram_len. Buffers at most ngram_len iterable\n            items.\n\n            For example::\n\n            >>> list(ngram_processor([1, 2, 3, 4, 5], ngram_len=3))\n            [(1, 2, 3), (2, 3, 4), (3, 4, 5)]\n            '
        ngram = deque()
        current_len = 0
        for item in items:
            if (current_len == ngram_len):
                (yield tuple(ngram))
                ngram.popleft()
                current_len -= 1
            ngram.append(item)
            current_len += 1
        (yield tuple(ngram))
    text = u'/*COMMENT\n            COMMENT COMMENT\n            - COMMENT\n            */\n            public static boolean activateSearchResultView() {\n            String defaultPerspectiveId= SearchUI.getDefaultPerspectiveId();\n            if (defaultPerspectiveId != null) {\n                IWorkbenchWindow window= SearchPlugin.getActiveWorkbenchWindow();\n                if (window != null && window.getShell() != null && !window.getShell().isDisposed()) {\n                    try {\n                        PlatformUI.getWorkbench().showPerspective(defaultPerspectiveId, window);\n                    } catch (WorkbenchException ex) {\n                        // show view in current perspective\n                    }\n                }\n            }'
    expected = [(u'comment', u'comment', u'comment', u'comment', u'public', u'static'), (u'comment', u'comment', u'comment', u'public', u'static', u'boolean'), (u'comment', u'comment', u'public', u'static', u'boolean', u'activatesearchresultview'), (u'comment', u'public', u'static', u'boolean', u'activatesearchresultview', u'string'), (u'public', u'static', u'boolean', u'activatesearchresultview', u'string', u'defaultperspectiveid'), (u'static', u'boolean', u'activatesearchresultview', u'string', u'defaultperspectiveid', u'searchui'), (u'boolean', u'activatesearchresultview', u'string', u'defaultperspectiveid', u'searchui', u'getdefaultperspectiveid'), (u'activatesearchresultview', u'string', u'defaultperspectiveid', u'searchui', u'getdefaultperspectiveid', u'if'), (u'string', u'defaultperspectiveid', u'searchui', u'getdefaultperspectiveid', u'if', u'defaultperspectiveid'), (u'defaultperspectiveid', u'searchui', u'getdefaultperspectiveid', u'if', u'defaultperspectiveid', u'null'), (u'searchui', u'getdefaultperspectiveid', u'if', u'defaultperspectiveid', u'null', u'iworkbenchwindow'), (u'getdefaultperspectiveid', u'if', u'defaultperspectiveid', u'null', u'iworkbenchwindow', u'window'), (u'if', u'defaultperspectiveid', u'null', u'iworkbenchwindow', u'window', u'searchplugin'), (u'defaultperspectiveid', u'null', u'iworkbenchwindow', u'window', u'searchplugin', u'getactiveworkbenchwindow'), (u'null', u'iworkbenchwindow', u'window', u'searchplugin', u'getactiveworkbenchwindow', u'if'), (u'iworkbenchwindow', u'window', u'searchplugin', u'getactiveworkbenchwindow', u'if', u'window'), (u'window', u'searchplugin', u'getactiveworkbenchwindow', u'if', u'window', u'null'), (u'searchplugin', u'getactiveworkbenchwindow', u'if', u'window', u'null', u'window'), (u'getactiveworkbenchwindow', u'if', u'window', u'null', u'window', u'getshell'), (u'if', u'window', u'null', u'window', u'getshell', u'null'), (u'window', u'null', u'window', u'getshell', u'null', u'window'), (u'null', u'window', u'getshell', u'null', u'window', u'getshell'), (u'window', u'getshell', u'null', u'window', u'getshell', u'isdisposed'), (u'getshell', u'null', u'window', u'getshell', u'isdisposed', u'try'), (u'null', u'window', u'getshell', u'isdisposed', u'try', u'platformui'), (u'window', u'getshell', u'isdisposed', u'try', u'platformui', u'getworkbench'), (u'getshell', u'isdisposed', u'try', u'platformui', u'getworkbench', u'showperspective'), (u'isdisposed', u'try', u'platformui', u'getworkbench', u'showperspective', u'defaultperspectiveid'), (u'try', u'platformui', u'getworkbench', u'showperspective', u'defaultperspectiveid', u'window'), (u'platformui', u'getworkbench', u'showperspective', u'defaultperspectiveid', u'window', u'catch'), (u'getworkbench', u'showperspective', u'defaultperspectiveid', u'window', u'catch', u'workbenchexception'), (u'showperspective', u'defaultperspectiveid', u'window', u'catch', u'workbenchexception', u'ex'), (u'defaultperspectiveid', u'window', u'catch', u'workbenchexception', u'ex', u'show'), (u'window', u'catch', u'workbenchexception', u'ex', u'show', u'view'), (u'catch', u'workbenchexception', u'ex', u'show', u'view', u'in'), (u'workbenchexception', u'ex', u'show', u'view', u'in', u'current'), (u'ex', u'show', u'view', u'in', u'current', u'perspective')]
    unigrams = (x.value for x in unigram_splitter(text.splitlines()))
    result = list(ngram_processor(unigrams, ngram_len=6))
    assert (expected == result)