<commit_before># -*- coding: utf-8 -*-
__author__ = 'isparks'

from rwslib import RWSConnection
from rwslib.rws_requests.odm_adapter import AuditRecordsRequest
from urlparse import urlparse, parse_qs
from parser import parse
import logging


class ODMAdapter(object):
    """A self-contained data fetcher and parser using a RWSConnection and an event class provided by the user"""
    def __init__(self, rws_connection, study, environment, eventer):
        self.rws_connection = rws_connection
        self.eventer = eventer
        self.study = study
        self.environment = environment
        self.start_id = 0


    def get_next_start_id(self):
        """If link for next result set has been passed, extract it and get the next set start id"""
        link = self.rws_connection.last_result.links.get("next",None)
        if link:
            link = link['url']
            p = urlparse(link)
            start_id = long(parse_qs(p.query)['startid'][0])
            return start_id

        return None

    def run(self, start_id=0, max_pages=-1, per_page=1000):
        page = 0
        self.start_id = start_id
        while max_pages == -1 or (page < max_pages):

            req = AuditRecordsRequest(self.study,self.environment, startid=self.start_id, per_page=per_page)
            try:
                #Get the ODM data
                odm = self.rws_connection.send_request(req)
                #Send it for parsing
                parse(odm, self.eventer)
                page += 1
            except Exception, e:
                logging.error(e.message)

            #Check if we were passed the next startid
            self.start_id = self.get_next_start_id()
            if not self.start_id:
                break<commit_msg>Added kwarg passthrough for ODMAdapter<commit_after># -*- coding: utf-8 -*-
__author__ = 'isparks'

from rwslib import RWSConnection
from rwslib.rws_requests.odm_adapter import AuditRecordsRequest
from urlparse import urlparse, parse_qs
from parser import parse
import logging


class ODMAdapter(object):
    """A self-contained data fetcher and parser using a RWSConnection and an event class provided by the user"""
    def __init__(self, rws_connection, study, environment, eventer):
        self.rws_connection = rws_connection
        self.eventer = eventer
        self.study = study
        self.environment = environment
        self.start_id = 0


    def get_next_start_id(self):
        """If link for next result set has been passed, extract it and get the next set start id"""
        link = self.rws_connection.last_result.links.get("next",None)
        if link:
            link = link['url']
            p = urlparse(link)
            start_id = long(parse_qs(p.query)['startid'][0])
            return start_id

        return None

    def run(self, start_id=0, max_pages=-1, per_page=1000, **kwargs):
        page = 0
        self.start_id = start_id
        while max_pages == -1 or (page < max_pages):

            req = AuditRecordsRequest(self.study,self.environment, startid=self.start_id, per_page=per_page)
            try:
                #Get the ODM data
                odm = self.rws_connection.send_request(req, **kwargs)
                #Send it for parsing
                parse(odm, self.eventer)
                page += 1
            except Exception, e:
                logging.error(e.message)

            #Check if we were passed the next startid
            self.start_id = self.get_next_start_id()
            if not self.start_id:
                break
{'fraction_non_alphanumeric': 0.08349788434414669, 'fraction_numerical': 0.006770098730606488, 'mean_word_length': 2.2090497737556563, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5265994', 'n_tokens_mistral': 1058, 'n_tokens_neox': 1013, 'n_words': 295}