<commit_before># -*- coding: utf-8 -*-
# vim: sw=4:ts=4:expandtab

"""
riko demo
~~~~~~~~~

Word Count

	>>> from riko.lib.collections import SyncPipe
	>>> from riko import get_path
	>>>
	>>> url = get_path('users.jyu.fi.html')
	>>> fetch_conf = {
	...     'url': url, 'start': '<body>', 'end': '</body>', 'detag': True}
	>>> replace_conf = {'rule': {'find': '\\n', 'replace': ' '}}
	>>>
	>>> counts = (SyncPipe('fetchpage', conf=fetch_conf)
	...     .strreplace(conf=replace_conf, assign='content')
	...     .stringtokenizer(conf={'delimiter': ' '}, emit=True)
	...     .count()
	...     .output)
	>>>
	>>> next(counts)
	{u'count': 70}

Fetching feeds

	>>> from riko.modules.pipefetch import pipe as fetch
	>>>
	>>> feed = fetch(conf={'url': 'https://news.ycombinator.com/rss'})
	>>> item = next(feed)
	>>> sorted(item.keys())
	[u'author.name', u'author.uri', 'comments', 'content', u'dc:creator', \
'link', u'pubDate', 'summary', 'title', 'updated', 'updated_parsed', u'y:id', \
u'y:published', u'y:title']
	>>> item['title'], item['link']  # doctest: +ELLIPSIS
	(u'...', u'http...')
"""
<commit_msg>Fix demo spacing and switch to local urls<commit_after># -*- coding: utf-8 -*-
# vim: sw=4:ts=4:expandtab

"""
riko demo
~~~~~~~~~

Word Count

    >>> from riko.lib.collections import SyncPipe
    >>> from riko import get_path
    >>>
    >>> url = get_path('users.jyu.fi.html')
    >>> fetch_conf = {
    ...     'url': url, 'start': '<body>', 'end': '</body>', 'detag': True}
    >>> replace_conf = {'rule': {'find': '\\n', 'replace': ' '}}
    >>>
    >>> counts = (SyncPipe('fetchpage', conf=fetch_conf)
    ...     .strreplace(conf=replace_conf, assign='content')
    ...     .stringtokenizer(conf={'delimiter': ' '}, emit=True)
    ...     .count()
    ...     .output)
    >>>
    >>> next(counts)
    {u'count': 70}

Fetching feeds

    >>> from riko.modules.pipefetch import pipe as fetch
    >>>
    >>> url = get_path('feeds.gawker.com_jalopnik_full.xml')
    >>> feed = fetch(conf={'url': url})
    >>> item = next(feed)
    >>> sorted(item.keys())
    ['author', u'author.name', u'author.uri', 'content', u'dc:creator', 'id', \
'link', u'pubDate', 'summary', 'title', 'updated', 'updated_parsed', u'y:id', \
u'y:published', u'y:title']
    >>> item['title'], item['link']  # doctest: +ELLIPSIS
    (u'...', u'http...')
"""
from __future__ import (
    absolute_import, division, print_function, unicode_literals)

from twisted.internet.defer import inlineCallbacks
from riko import get_path
from riko.lib.collections import SyncPipe
from riko.twisted.collections import AsyncPipe

replace_conf = {'rule': {'find': '\n', 'replace': ' '}}
url1 = get_path('news.yahoo.com_rss_health.xml')
url2 = get_path('www.caltrain.com_Fares_farechart.html')
fetch_conf = {
    'url': url2, 'start': '<body>', 'end': '</body>', 'detag': True}

def pipe(test=False):
    s1 = SyncPipe('fetch', test=test, conf={'url': url1}).output
    s2 = (SyncPipe('fetchpage', test=test, conf=fetch_conf)
        .strreplace(conf=replace_conf, assign='content')
        .stringtokenizer(conf={'delimiter': ' '}, emit=True)
        .count()
        .output)

    print(next(s1)['title'], next(s2)['count'])


@inlineCallbacks
def asyncPipe(reactor, test=False):
    s1 = yield AsyncPipe('fetch', test=test, conf={'url': url1}).output
    s2 = yield (AsyncPipe('fetchpage', test=test, conf=fetch_conf)
        .strreplace(conf=replace_conf, assign='content')
        .stringtokenizer(conf={'delimiter': ' '}, emit=True)
        .count()
        .output)

    print(next(s1)['title'], next(s2)['count'])

{'fraction_non_alphanumeric': 0.22798216276477146, 'fraction_numerical': 0.00641025641025641, 'mean_word_length': 3.766268260292165, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 108, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7343479', 'n_tokens_mistral': 1304, 'n_tokens_neox': 1305, 'n_words': 251}