<commit_before>"""
Some API handling code.  Predominantly this is to centralize common
alterations we make to API calls, such as filtering by router ids.
"""

import collections
import collections.abc
import fnmatch
import getpass
import hashlib
import html
import html.parser
import itertools
import json
import os
import re
import shellish
import shutil
import syndicate
import syndicate.client
import syndicate.data
import textwrap
import tornado
from syndicate.adapters.sync import LoginAuth


class HTMLJSONDecoder(syndicate.data.NormalJSONDecoder):

    def parse_object(self, data):
        data = super().parse_object(data)
        for key, value in data.items():
            if isinstance(value, str):
                data[key] = html.unescape(value)
        return data


def text_pager(data):
    width, height = shutil.get_terminal_size()
    i = 0
    for section in data:
        lines = textwrap.wrap(section, width-4)
        if not lines:
            i += 1
            print()
        for x in lines:
            i += 1
            if i == height - 2:
                input("<Press enter to view next page>")
                i = 0
            print(x)


syndicate.data.serializers['htmljson'] = syndicate.data.Serializer(
    'application/json',
    syndicate.data.serializers['json'].encode,
    HTMLJSONDecoder().decode
)<commit_msg>Fix for TOS pager skipping pages.<commit_after>"""
Some API handling code.  Predominantly this is to centralize common
alterations we make to API calls, such as filtering by router ids.
"""

import collections
import collections.abc
import fnmatch
import getpass
import hashlib
import html
import html.parser
import itertools
import json
import os
import re
import shellish
import shutil
import syndicate
import syndicate.client
import syndicate.data
import textwrap
import tornado
from syndicate.adapters.sync import LoginAuth


class HTMLJSONDecoder(syndicate.data.NormalJSONDecoder):

    def parse_object(self, data):
        data = super().parse_object(data)
        for key, value in data.items():
            if isinstance(value, str):
                data[key] = html.unescape(value)
        return data


def text_pager(data):
    width, height = shutil.get_terminal_size()
    i = 0
    for section in data:
        lines = textwrap.wrap(section, width-4)
        if not lines:
            i += 1
            print()
        for x in lines:
            i += 1
            if i >= height - 2:
                input("<Press enter to view next page>")
                i = 0
            print(x)


syndicate.data.serializers['htmljson'] = syndicate.data.Serializer(
    'application/json',
    syndicate.data.serializers['json'].encode,
    HTMLJSONDecoder().decode
)
{'fraction_non_alphanumeric': 0.0818048422597212, 'fraction_numerical': 0.004402054292002935, 'mean_word_length': 2.6604026845637585, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4687363', 'n_tokens_mistral': 777, 'n_tokens_neox': 750, 'n_words': 257}