<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json
from abc import ABCMeta

from collections import OrderedDict
from functools import wraps

import sys
from os.path import dirname, abspath
# just until ransom becomes its own package
sys.path.append(dirname(dirname(abspath(__file__))))
import ransom

from params import SingleParam
from models import get_unique_func, get_priority_func
from utils import (PriorityQueue,
                   MaxInt,
                   chunked_iter,
                   make_type_wrapper,
                   OperationExample)

# TODO: handle automatic redirecting better
# TODO: support batching and optimization limits
# TODO: concurrency. get_current_task() -> get_current_tasks()
# TODO: wrap exceptions
# TODO: separate structure for saving completed subops (for debugging?)
# TODO: WebRequestOperation: accepts URL, action (default: GET)
# TODO: Model links (url attribute)
# TODO: support field param_type (for cases with ints and strs)
# TODO: use source descriptor instead of api_url? (for op.source)
# TODO: check that subop_chain types match up
# TODO: check that priority attribute exists on output_type where applicable

"""
- what if operations were iterable over their results and process()
  returned the operation itself? (more expensive to iterate and find
  non-dupe results, would set ops help?)
- client -> root_owner.  parent operation (client
  if no parent op) -> owner.
- pregenerate MediawikiCalls/URLs on QueryOperations

Operation modifiers:
- Prioritized
- Recursive
- Buffered

fun metadata:

- operations executed
- suboperations skipped (from dedupe/prioritization/laziness)
- web requests executed, kb downloaded

retry strategies:

- absolute number of failures
- streaks/runs of failures
- fail if first operation fails
- reduce batch size/query limit on timeouts

prioritization/batching/concurrency implementation thoughts:

- hands-off implementation via multiplexing?
- separate priority queues for params and suboperations?
- fancy new datastructure with dedupe + priority queueing built-in
- buffering: do 3/5/10 GetCategoryInfos before fetching member pages
- early subop production based on next parameter priority
  sinking below a certain threshold?
  (e.g., next param's subcats=5 -> fetch more category infos)
"""

DEFAULT_API_URL = 'http://en.wikipedia.org/w/api.php'
DEFAULT_BASE_URL = 'http://en.wikipedia.org/wiki/'

DEFAULT_HEADERS = {'User-Agent': ('Wapiti/0.0.0 <NAME>'
                                  ' <EMAIL>') }

ALL = MaxInt('ALL')
DEFAULT_MIN = 50


class WapitiException(Exception):
    pass


class NoMoreResults(Exception):
    pass


DEFAULT_WEB_CLIENT = ransom.Client({'headers': DEFAULT_HEADERS})


class MockClient(object):
    def __init__(self, is_bot=False):
        self.web_client = DEFAULT_WEB_CLIENT
        self.api_url = DEFAULT_API_URL
        self.is_bot = is_bot


DEFAULT_CLIENT = MockClient()


Tune = make_type_wrapper('Tune', [('priority', None), ('buffer', None)])
Recursive = make_type_wrapper('Recursive', [('is_recursive', True)])


def get_unwrapped_options(wr_type):
    try:
        return dict(wr_type._wrapped_dict), wr_type._wrapped<commit_msg>fixing mock client<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json
from abc import ABCMeta

from collections import OrderedDict
from functools import wraps

import sys
from os.path import dirname, abspath
# just until ransom becomes its own package
sys.path.append(dirname(dirname(abspath(__file__))))
import ransom

from params import SingleParam
from models import get_unique_func, get_priority_func
from utils import (PriorityQueue,
                   MaxInt,
                   chunked_iter,
                   make_type_wrapper,
                   OperationExample)

# TODO: handle automatic redirecting better
# TODO: support batching and optimization limits
# TODO: concurrency. get_current_task() -> get_current_tasks()
# TODO: wrap exceptions
# TODO: separate structure for saving completed subops (for debugging?)
# TODO: WebRequestOperation: accepts URL, action (default: GET)
# TODO: Model links (url attribute)
# TODO: support field param_type (for cases with ints and strs)
# TODO: use source descriptor instead of api_url? (for op.source)
# TODO: check that subop_chain types match up
# TODO: check that priority attribute exists on output_type where applicable

"""
- what if operations were iterable over their results and process()
  returned the operation itself? (more expensive to iterate and find
  non-dupe results, would set ops help?)
- client -> root_owner.  parent operation (client
  if no parent op) -> owner.
- pregenerate MediawikiCalls/URLs on QueryOperations

Operation modifiers:
- Prioritized
- Recursive
- Buffered

fun metadata:

- operations executed
- suboperations skipped (from dedupe/prioritization/laziness)
- web requests executed, kb downloaded

retry strategies:

- absolute number of failures
- streaks/runs of failures
- fail if first operation fails
- reduce batch size/query limit on timeouts

prioritization/batching/concurrency implementation thoughts:

- hands-off implementation via multiplexing?
- separate priority queues for params and suboperations?
- fancy new datastructure with dedupe + priority queueing built-in
- buffering: do 3/5/10 GetCategoryInfos before fetching member pages
- early subop production based on next parameter priority
  sinking below a certain threshold?
  (e.g., next param's subcats=5 -> fetch more category infos)
"""

DEFAULT_API_URL = 'http://en.wikipedia.org/w/api.php'
DEFAULT_BASE_URL = 'http://en.wikipedia.org/wiki/'

DEFAULT_HEADERS = {'User-Agent': ('Wapiti/0.0.0 <NAME>'
                                  ' <EMAIL>') }

ALL = MaxInt('ALL')
DEFAULT_MIN = 50


class WapitiException(Exception):
    pass


class NoMoreResults(Exception):
    pass


DEFAULT_WEB_CLIENT = ransom.Client({'headers': DEFAULT_HEADERS})


class MockClient(object):
    def __init__(self, is_bot=False, **kwargs):
        self.web_client = DEFAULT_WEB_CLIENT
        self.api_url = DEFAULT_API_URL
        self.is_bot = is_bot


DEFAULT_CLIENT = MockClient()


Tune = make_type_wrapper('Tune', [('priority', None), ('buffer', None)])
Recursive = make_type_wrapper('Recursive', [('is_recursive', True)])


def get_unwrapped_options(wr_type):
    try:
        return dict(wr_type._wrapped_dict), wr_type._wrapped
{'fraction_non_alphanumeric': 0.08412377546260302, 'fraction_numerical': 0.003420929870937646, 'mean_word_length': 4.072555205047319, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1430357', 'n_tokens_mistral': 1871, 'n_tokens_neox': 1812, 'n_words': 622}