<commit_before>from collections import namedtuple
from datetime import MAXYEAR, datetime
from time import sleep

from dateutil.relativedelta import relativedelta
from flask import current_app
import redis
from redis.exceptions import ConnectionError
from sqlalchemy.types import Enum as SQLA_Enum
from werkzeug.exceptions import BadRequest

from ..audit import auditable_event
from ..cache import cache, TWO_HOURS
from ..database import db
from ..date_tools import FHIR_datetime, RelativeDelta
from ..set_tools import left_center_right
from ..timeout_lock import TimeoutLock
from ..trace import trace
from .overall_status import OverallStatus
from .qbd import QBD
from .questionnaire_bank import (
    qbs_by_intervention,
    qbs_by_rp,
    trigger_date,
    visit_name,
)
from .questionnaire_response import QNR_results, QuestionnaireResponse
from .research_protocol import ResearchProtocol
from .role import ROLE
from .user import User
from .user_consent import consent_withdrawal_dates


class QBT(db.Model):
    """Effectively a view, to simplify QB status lookups over time

    A user has a number of QBT rows, at least one for each questionnaire
    bank that has been due for the user, or will be in the future.

    The following events would invalidate the respective rows (delete
    said rows to invalidate):
     - user's trigger date changed (consent or new procedure)
     - user submits a QuestionnaireResponse
     - the definition of a QB or an organization's research protocol

    The table is populated up to the next known event for a user.  If a
    future date isn't found, that user's data is due for update.

    """
    __tablename__ = 'qb_timeline'
    id = db.Column(db.Integer, primary_key=True)
    user_id = db.Column(db.ForeignKey(
        'users.id', ondelete='cascade'), nullable=False)
    at = db.Column(
        db.DateTime, nullable=False, index=True,
        doc="initial date time for state of row")
    qb_id = db.Column(db.ForeignKey(
        'questionnaire_banks.id', ondelete='cascade'), nullable=True)
    qb_recur_id = db.Column(db.ForeignKey(
        'recurs.id', ondelete='cascade'), nullable=True)
    qb_iteration = db.Column(db.Integer, nullable=True)
    status = db.Column(SQLA_Enum(OverallStatus), nullable=False, index=True)

    def qbd(self):
        """Generate and return a QBD instance from self data"""
        return QBD(
            relative_start=self.at, iteration=self.qb_iteration,
            recur_id=self.qb_recur_id, qb_id=self.qb_id)


class AtOrderedList(list):
    """Specialize ``list`` to maintain insertion order and ``at`` attribute

    When building up QBTs for a user, we need to maintain both order and
    sort by the ``QBT.at`` attribute.  As there are rows with identical ``at``
    values, it can't simply be sorted by a field, as insertion order matters
    in the case of a tie (as say the ``due`` status needs to precede a
    ``completed``, which does happen with paper entry).

    As the build order matters, continue to add QBTs to the end of the list,
    taking special care to insert those with earlier ``at`` values in
    the correct place.  Two or more identical ``at`` values should result
    in the latest addition following preexisting.

    """

    def append(self, value):
        """Maintain order by appending or inserting as needed

        If the given value.at is > current_end.at, append to end.
        Otherwise, walk backwards till the new can be inserted
        so the list remains ordered by the 'at' attribute, with
        new matching values following existing

        """
        if not self.__len__():
            return super(AtOrderedList, self).append(value)

        # Expecting to build in order; common case new value
        # lands at end.
        if self[-1].at <= value.at:
            return super(AtOrderedList, self).append(value)

        # Otherwise, walk backwards till new value < existing
        for i, e in reversed(list(enumerate(self))):
            if i > 0:
                # If not at start and previous is also greater
                # than new, continue to walk backwards
                if self[i-1].at > value.at:
                    continue
            if e.at > value.at:
                return self.insert(i, value)

        raise ValueError("still here?")


def ordered_rp_qbs(rp_id, trigger_date):
    """Generator to yield ordered qbs by research protocol alone"""
    baselines = qbs_by_rp(rp_id, 'baseline')
    if len(baselines) != 1:
        raise RuntimeError(
            "Expect exactly one QB for baseline by rp {}".format(rp_id))
    baseline = baselines[0]
    if baseline not in db.session:
        baseline = db.session.merge(baseline, load=False)
    start = baseline.calculated_start(trigger_date=trigger_date)
    yield start

    qbs_by_start = {}
    recurring = qbs_by_rp(rp_id, 'recurring')
    for qb in recurring:
        if qb not in db.session:
            qb = db.session.merge(qb, load=False)

        for qbd in qb.recurring_starts(trigger_date):
            qbs_by_start[qbd.relative_start] = qbd

    # continue to yield in order
    trace("found {} total recurring QBs".format(len(qbs_by_start)))
    for start_date in sorted(qbs_by_start.keys()):
        yield qbs_by_start[start_date]


def ordered_intervention_qbs(user, trigger_date):
    """Generator to yield ordered qbs by intervention"""
    baselines = qbs_by_intervention(user, 'baseline')
    if not baselines:
        return
    if len(baselines) > 1:
        raise RuntimeError(
            "{} has {} baselines by intervention (expected ONE)".format(
                user, len(baselines)))
    baseline = baselines[0]
    if baseline not in db.session:
        baseline = db.session.merge(baseline, load=False)
    start = baseline.calculated_start(trigger_date=trigger_date)
    yield start

    qbs_by_start = {}
    recurring = qbs_by_intervention(user, 'recurring')
    for qb in recurring:
        if qb not in db.session:
            qb = db.session.merge(qb, load=False)

        for qbd in qb.recurring_starts(trigger_date=trigger_date):
            qbs_by_start[qbd.relative_start] = qbd

    # continue to yield in order
    for start_date in sorted(qbs_by_start.keys()):
        yield qbs_by_start[start_date]


def indef_qbs_by_rp(rp_id, trigger_date):
    """Generator to yield ordered `indefinite` qbs by research protocol

    At the moment, only expecting to yield one, but following generator
    pattern to facilitate polymorphic code.

    """
    indefinites = qbs_by_rp(rp_id, 'indefinite')
    for indefinite in indefinites:
        if indefinite not in db.session:
            indefinite = db.session.merge(indefinite, load=False)
        start = indefinite.calculated_start(trigger_date=trigger_date)
        yield start


def indef_intervention_qbs(user, trigger_date):
    """Generator to yield indefinite qbs by intervention"""
    indefinites = qbs_by_intervention(user, 'indefinite')
    for indefinite in indefinites:
        if indefinite not in db.session:
            indefinite = db.session.merge(indefinite, load=False)
        start = indefinite.calculated_start(trigger_date=trigger_date)
        yield start


def calc_and_adjust_start(user, qbd, initial_trigger):
    """Calculate correct start for user on given QBD

    A QBD is initially generated with a generic trigger date for
    caching and sorting needs.  This function translates the
    given QBD.relative_start to the users situation.

    :param user: subject user
    :param qbd: QBD with respect to system trigger
    :param initial_trigger: datetime value used in initial QBD calculation

    :returns adjusted `relative_start` for user

    """
    users_trigger = trigger_date(user, qbd.questionnaire_bank)
    if not users_trigger:
        trace(
            "no valid trigger, default to initial value: {}".format(
                initial_trigger))
        users_trigger = initial_trigger
    if initial_trigger > users_trigger:
        trace(
            "user {} has unexpected trigger date before consent date".format(
                user.id))

    delta = users_trigger - initial_trigger
    if not qbd.relative_start:
        raise RuntimeError("can't adjust without relative_start")
    return qbd.relative_start + delta


def calc_and_adjust_expired(user, qbd, initial_trigger):
    """Calculate correct expired for user on given QBD

    A QBD is initially generated with a generic trigger date for
    caching and sorting needs.  This function translates the
    given QBD.relative_start to the users situation.

    :param user: subject user
    :param qbd: QBD with respect to system trigger
    :param initial_trigger: datetime value used in initial QBD calculation

    :returns adjusted `relative_start` for user

    """
    users_trigger = trigger_date(user, qbd.questionnaire_bank)
    if not users_trigger:
        trace(
            "no valid trigger, default to initial value: {}".format(
                initial_trigger))
        users_trigger = initial_trigger
    if initial_trigger > users_trigger:
        trace(
            "user {} has unexpected trigger date before consent date".format(
                user.id))

    expired = qbd.questionnaire_bank.expired
    delta = users_trigger - initial_trigger + RelativeDelta(expired)
    if not qbd.relative_start:
        raise RuntimeError("can't adjust without relative_start")
    return qbd.relative_start + delta


max_sort_time = datetime(year=MAXYEAR, month=12, day=31)


def second_null_safe_datetime(x):
    """datetime sort accessor treats None as far off in the future"""
    if not x[1]:
        return max_sort_time
    return x[1]


RPD = namedtuple("RPD", ['rp', 'retired', 'qbds'])


def cur_next_rp_gen(user, classification, trigger_date):
    """Generator to manage transitions through research protocols

    Returns a *pair* of research protocol data (RPD) namedtuples,
    one for current (active) RP data, one for the "next".

    :param user: applicable patient
    :param classification: None or 'indefinite' for special handling
    :param trigger_date: patient's initial trigger date

    :yields: cur_RPD, next_RPD

    """
    rps = ResearchProtocol.assigned_to(user)
    sorted_rps = sorted(
        list(rps), key=second_null_safe_datetime, reverse=True)

    def qbds_for_rp(rp, classification, trigger_date):
        if rp is None:
            return None
        if classification == 'indefinite':
            return indef_qbs_by_rp(rp.id, trigger_date=trigger_date)
        return ordered_rp_qbs(rp.id, trigger_date=trigger_date)

    while sorted_rps:
        # Start with oldest RP (current) till it's time to progress (next)
        current_rp, current_retired = sorted_rps.pop()
        if sorted_rps:
            next_rp, next_retired = sorted_rps[-1]
        else:
            next_rp = None
        trace("current RP '{}'({}), next RP '{}'({})".format(
            current_rp.name, current_rp.id, next_rp.name
            if next_rp else 'None', next_rp.id if next_rp else 'None'))

        curRPD = RPD(
            rp=current_rp,
            retired=current_retired,
            qbds=qbds_for_rp(current_rp, classification, trigger_date))
        if next_rp:
            nextRPD = RPD(
                rp=next_rp,
                retired=next_retired,
                qbds=qbds_for_rp(next_rp, classification, trigger_date)
            )
            if curRPD.retired == nextRPD.retired:
                raise ValueError(
                    "Invalid state: multiple RPs w/ same retire date")
        else:
            nextRPD = None
        yield curRPD, nextRPD


class RP_flyweight(object):
    """maintains state for RPs as it transitions

    Any number of Research Protocols may apply to a user.  This
    class manages transitioning through the applicable as time
    passes, maintaining a "current" and "next" as well as the ability
    to continue marching forward.

    """

    def __init__(self, user, trigger_date, classification):
        """Initialize flyweight state
        :param user: the patient
        :param trigger_date: the patient's initial trigger date
        :param classification: `indefinite` or None
        """
        self.user = user
        self.td = trigger_date
        self.classification = classification
        self.rp_walker = cur_next_rp_gen(
            user=self.user, trigger_date=self.td,
            classification=self.classification)
        self.cur_rpd, self.nxt_rpd = next(self.rp_walker, (None, None))
        self.skipped_nxt_start = None

    def adjust_start(self):
        """The QB start may need a minor adjustment, done once when ready"""
        self.cur_qbd.relative_start = self.cur_start

        # sanity check - make sure we don't adjust twice
        if hasattr(self.cur_qbd, 'already_adjusted'):
            raise RuntimeError(
                'already adjusted the qbd relative start')
        self.cur_qbd.already_adjusted = True

    def consider_transition(self):
        """Returns true only if state suggests it *may* be transtion time"""
        return self.nxt_rpd and self.cur_rpd.retired < self.cur_exp

    def next_qbd(self):
        """Advance to next qbd on applicable RPs"""
        self.cur_qbd, self.cur_start, self.cur_exp = None, None, None
        if self.cur_rpd:
            self.cur_qbd = next(self.cur_rpd.qbds, None)
        if self.cur_qbd:
            self.cur_start = calc_and_adjust_start(
                user=self.user, qbd=self.cur_qbd, initial_trigger=self.td)
            self.cur_exp = calc_and_adjust_expired(
                user=self.user, qbd=self.cur_qbd, initial_trigger=self.td)

        self.nxt_qbd, self.nxt_start = None, None
        if self.nxt_rpd:
            self.nxt_qbd = next(self.nxt_rpd.qbds, None)
        if self.nxt_qbd:
            self.nxt_start = calc_and_adjust_start(
                user=self.user, qbd=self.nxt_qbd, initial_trigger=self.td)
            if self.cur_qbd is None:
                trace("Finished cur RP with remaining QBs in next")
                self.cur_start = self.nxt_start
                self.transition()
            elif self.cur_start > self.nxt_start + relativedelta(months=1):
                # The plus one month covers RP v5 date adjustments.

                # Valid only when the RP being replaced doesn't have all the
                # visits defined in the next one (i.e. v3 doesn't have months
                # 27 or 33 and v5 does). Look ahead for a match
                self.skipped_nxt_start = self.nxt_start
                self.nxt_start = None
                self.nxt_qbd = next(self.nxt_rpd.qbds, None)
                if self.nxt_qbd:
                    self.nxt_start = calc_and_adjust_start(
                        user=self.user, qbd=self.nxt_qbd,
                        initial_trigger=self.td)
                if self.cur_start > self.nxt_start:
                    # Still no match means poorly defined RP QBs
                    raise ValueError(
                        "Invalid state {}:{} not in lock-step even on "
                        "look ahead; RPs need to maintain same "
                        "schedule {}, {}, {}".format(
                            self.cur_rpd.rp.name, self.nxt_rpd.rp.name,
                            self.cur_start, self.nxt_start,
                            self.skipped_nxt_start))
        if self.cur_qbd:
            trace("advanced to next QB: {} w/ start {}".format(
                self.cur_qbd.questionnaire_bank.name, self.cur_start))
        else:
            trace("out of QBs!")

    def transition(self):
        """Transition internal state to 'next' Research Protocol"""
        trace("transitioning to the next RP w/ cur_start {}".format(
            self.cur_start))
        if self.cur_start > self.nxt_start + relativedelta(months=1):
            # The plus one month covers RP v5 shift
            raise ValueError(<commit_msg>Found another exception needed for v5 date changes. (#3676)<commit_after>from collections import namedtuple
from datetime import MAXYEAR, datetime
from time import sleep

from dateutil.relativedelta import relativedelta
from flask import current_app
import redis
from redis.exceptions import ConnectionError
from sqlalchemy.types import Enum as SQLA_Enum
from werkzeug.exceptions import BadRequest

from ..audit import auditable_event
from ..cache import cache, TWO_HOURS
from ..database import db
from ..date_tools import FHIR_datetime, RelativeDelta
from ..set_tools import left_center_right
from ..timeout_lock import TimeoutLock
from ..trace import trace
from .overall_status import OverallStatus
from .qbd import QBD
from .questionnaire_bank import (
    qbs_by_intervention,
    qbs_by_rp,
    trigger_date,
    visit_name,
)
from .questionnaire_response import QNR_results, QuestionnaireResponse
from .research_protocol import ResearchProtocol
from .role import ROLE
from .user import User
from .user_consent import consent_withdrawal_dates


class QBT(db.Model):
    """Effectively a view, to simplify QB status lookups over time

    A user has a number of QBT rows, at least one for each questionnaire
    bank that has been due for the user, or will be in the future.

    The following events would invalidate the respective rows (delete
    said rows to invalidate):
     - user's trigger date changed (consent or new procedure)
     - user submits a QuestionnaireResponse
     - the definition of a QB or an organization's research protocol

    The table is populated up to the next known event for a user.  If a
    future date isn't found, that user's data is due for update.

    """
    __tablename__ = 'qb_timeline'
    id = db.Column(db.Integer, primary_key=True)
    user_id = db.Column(db.ForeignKey(
        'users.id', ondelete='cascade'), nullable=False)
    at = db.Column(
        db.DateTime, nullable=False, index=True,
        doc="initial date time for state of row")
    qb_id = db.Column(db.ForeignKey(
        'questionnaire_banks.id', ondelete='cascade'), nullable=True)
    qb_recur_id = db.Column(db.ForeignKey(
        'recurs.id', ondelete='cascade'), nullable=True)
    qb_iteration = db.Column(db.Integer, nullable=True)
    status = db.Column(SQLA_Enum(OverallStatus), nullable=False, index=True)

    def qbd(self):
        """Generate and return a QBD instance from self data"""
        return QBD(
            relative_start=self.at, iteration=self.qb_iteration,
            recur_id=self.qb_recur_id, qb_id=self.qb_id)


class AtOrderedList(list):
    """Specialize ``list`` to maintain insertion order and ``at`` attribute

    When building up QBTs for a user, we need to maintain both order and
    sort by the ``QBT.at`` attribute.  As there are rows with identical ``at``
    values, it can't simply be sorted by a field, as insertion order matters
    in the case of a tie (as say the ``due`` status needs to precede a
    ``completed``, which does happen with paper entry).

    As the build order matters, continue to add QBTs to the end of the list,
    taking special care to insert those with earlier ``at`` values in
    the correct place.  Two or more identical ``at`` values should result
    in the latest addition following preexisting.

    """

    def append(self, value):
        """Maintain order by appending or inserting as needed

        If the given value.at is > current_end.at, append to end.
        Otherwise, walk backwards till the new can be inserted
        so the list remains ordered by the 'at' attribute, with
        new matching values following existing

        """
        if not self.__len__():
            return super(AtOrderedList, self).append(value)

        # Expecting to build in order; common case new value
        # lands at end.
        if self[-1].at <= value.at:
            return super(AtOrderedList, self).append(value)

        # Otherwise, walk backwards till new value < existing
        for i, e in reversed(list(enumerate(self))):
            if i > 0:
                # If not at start and previous is also greater
                # than new, continue to walk backwards
                if self[i-1].at > value.at:
                    continue
            if e.at > value.at:
                return self.insert(i, value)

        raise ValueError("still here?")


def ordered_rp_qbs(rp_id, trigger_date):
    """Generator to yield ordered qbs by research protocol alone"""
    baselines = qbs_by_rp(rp_id, 'baseline')
    if len(baselines) != 1:
        raise RuntimeError(
            "Expect exactly one QB for baseline by rp {}".format(rp_id))
    baseline = baselines[0]
    if baseline not in db.session:
        baseline = db.session.merge(baseline, load=False)
    start = baseline.calculated_start(trigger_date=trigger_date)
    yield start

    qbs_by_start = {}
    recurring = qbs_by_rp(rp_id, 'recurring')
    for qb in recurring:
        if qb not in db.session:
            qb = db.session.merge(qb, load=False)

        for qbd in qb.recurring_starts(trigger_date):
            qbs_by_start[qbd.relative_start] = qbd

    # continue to yield in order
    trace("found {} total recurring QBs".format(len(qbs_by_start)))
    for start_date in sorted(qbs_by_start.keys()):
        yield qbs_by_start[start_date]


def ordered_intervention_qbs(user, trigger_date):
    """Generator to yield ordered qbs by intervention"""
    baselines = qbs_by_intervention(user, 'baseline')
    if not baselines:
        return
    if len(baselines) > 1:
        raise RuntimeError(
            "{} has {} baselines by intervention (expected ONE)".format(
                user, len(baselines)))
    baseline = baselines[0]
    if baseline not in db.session:
        baseline = db.session.merge(baseline, load=False)
    start = baseline.calculated_start(trigger_date=trigger_date)
    yield start

    qbs_by_start = {}
    recurring = qbs_by_intervention(user, 'recurring')
    for qb in recurring:
        if qb not in db.session:
            qb = db.session.merge(qb, load=False)

        for qbd in qb.recurring_starts(trigger_date=trigger_date):
            qbs_by_start[qbd.relative_start] = qbd

    # continue to yield in order
    for start_date in sorted(qbs_by_start.keys()):
        yield qbs_by_start[start_date]


def indef_qbs_by_rp(rp_id, trigger_date):
    """Generator to yield ordered `indefinite` qbs by research protocol

    At the moment, only expecting to yield one, but following generator
    pattern to facilitate polymorphic code.

    """
    indefinites = qbs_by_rp(rp_id, 'indefinite')
    for indefinite in indefinites:
        if indefinite not in db.session:
            indefinite = db.session.merge(indefinite, load=False)
        start = indefinite.calculated_start(trigger_date=trigger_date)
        yield start


def indef_intervention_qbs(user, trigger_date):
    """Generator to yield indefinite qbs by intervention"""
    indefinites = qbs_by_intervention(user, 'indefinite')
    for indefinite in indefinites:
        if indefinite not in db.session:
            indefinite = db.session.merge(indefinite, load=False)
        start = indefinite.calculated_start(trigger_date=trigger_date)
        yield start


def calc_and_adjust_start(user, qbd, initial_trigger):
    """Calculate correct start for user on given QBD

    A QBD is initially generated with a generic trigger date for
    caching and sorting needs.  This function translates the
    given QBD.relative_start to the users situation.

    :param user: subject user
    :param qbd: QBD with respect to system trigger
    :param initial_trigger: datetime value used in initial QBD calculation

    :returns adjusted `relative_start` for user

    """
    users_trigger = trigger_date(user, qbd.questionnaire_bank)
    if not users_trigger:
        trace(
            "no valid trigger, default to initial value: {}".format(
                initial_trigger))
        users_trigger = initial_trigger
    if initial_trigger > users_trigger:
        trace(
            "user {} has unexpected trigger date before consent date".format(
                user.id))

    delta = users_trigger - initial_trigger
    if not qbd.relative_start:
        raise RuntimeError("can't adjust without relative_start")
    return qbd.relative_start + delta


def calc_and_adjust_expired(user, qbd, initial_trigger):
    """Calculate correct expired for user on given QBD

    A QBD is initially generated with a generic trigger date for
    caching and sorting needs.  This function translates the
    given QBD.relative_start to the users situation.

    :param user: subject user
    :param qbd: QBD with respect to system trigger
    :param initial_trigger: datetime value used in initial QBD calculation

    :returns adjusted `relative_start` for user

    """
    users_trigger = trigger_date(user, qbd.questionnaire_bank)
    if not users_trigger:
        trace(
            "no valid trigger, default to initial value: {}".format(
                initial_trigger))
        users_trigger = initial_trigger
    if initial_trigger > users_trigger:
        trace(
            "user {} has unexpected trigger date before consent date".format(
                user.id))

    expired = qbd.questionnaire_bank.expired
    delta = users_trigger - initial_trigger + RelativeDelta(expired)
    if not qbd.relative_start:
        raise RuntimeError("can't adjust without relative_start")
    return qbd.relative_start + delta


max_sort_time = datetime(year=MAXYEAR, month=12, day=31)


def second_null_safe_datetime(x):
    """datetime sort accessor treats None as far off in the future"""
    if not x[1]:
        return max_sort_time
    return x[1]


RPD = namedtuple("RPD", ['rp', 'retired', 'qbds'])


def cur_next_rp_gen(user, classification, trigger_date):
    """Generator to manage transitions through research protocols

    Returns a *pair* of research protocol data (RPD) namedtuples,
    one for current (active) RP data, one for the "next".

    :param user: applicable patient
    :param classification: None or 'indefinite' for special handling
    :param trigger_date: patient's initial trigger date

    :yields: cur_RPD, next_RPD

    """
    rps = ResearchProtocol.assigned_to(user)
    sorted_rps = sorted(
        list(rps), key=second_null_safe_datetime, reverse=True)

    def qbds_for_rp(rp, classification, trigger_date):
        if rp is None:
            return None
        if classification == 'indefinite':
            return indef_qbs_by_rp(rp.id, trigger_date=trigger_date)
        return ordered_rp_qbs(rp.id, trigger_date=trigger_date)

    while sorted_rps:
        # Start with oldest RP (current) till it's time to progress (next)
        current_rp, current_retired = sorted_rps.pop()
        if sorted_rps:
            next_rp, next_retired = sorted_rps[-1]
        else:
            next_rp = None
        trace("current RP '{}'({}), next RP '{}'({})".format(
            current_rp.name, current_rp.id, next_rp.name
            if next_rp else 'None', next_rp.id if next_rp else 'None'))

        curRPD = RPD(
            rp=current_rp,
            retired=current_retired,
            qbds=qbds_for_rp(current_rp, classification, trigger_date))
        if next_rp:
            nextRPD = RPD(
                rp=next_rp,
                retired=next_retired,
                qbds=qbds_for_rp(next_rp, classification, trigger_date)
            )
            if curRPD.retired == nextRPD.retired:
                raise ValueError(
                    "Invalid state: multiple RPs w/ same retire date")
        else:
            nextRPD = None
        yield curRPD, nextRPD


class RP_flyweight(object):
    """maintains state for RPs as it transitions

    Any number of Research Protocols may apply to a user.  This
    class manages transitioning through the applicable as time
    passes, maintaining a "current" and "next" as well as the ability
    to continue marching forward.

    """

    def __init__(self, user, trigger_date, classification):
        """Initialize flyweight state
        :param user: the patient
        :param trigger_date: the patient's initial trigger date
        :param classification: `indefinite` or None
        """
        self.user = user
        self.td = trigger_date
        self.classification = classification
        self.rp_walker = cur_next_rp_gen(
            user=self.user, trigger_date=self.td,
            classification=self.classification)
        self.cur_rpd, self.nxt_rpd = next(self.rp_walker, (None, None))
        self.skipped_nxt_start = None

    def adjust_start(self):
        """The QB start may need a minor adjustment, done once when ready"""
        self.cur_qbd.relative_start = self.cur_start

        # sanity check - make sure we don't adjust twice
        if hasattr(self.cur_qbd, 'already_adjusted'):
            raise RuntimeError(
                'already adjusted the qbd relative start')
        self.cur_qbd.already_adjusted = True

    def consider_transition(self):
        """Returns true only if state suggests it *may* be transtion time"""
        return self.nxt_rpd and self.cur_rpd.retired < self.cur_exp

    def next_qbd(self):
        """Advance to next qbd on applicable RPs"""
        self.cur_qbd, self.cur_start, self.cur_exp = None, None, None
        if self.cur_rpd:
            self.cur_qbd = next(self.cur_rpd.qbds, None)
        if self.cur_qbd:
            self.cur_start = calc_and_adjust_start(
                user=self.user, qbd=self.cur_qbd, initial_trigger=self.td)
            self.cur_exp = calc_and_adjust_expired(
                user=self.user, qbd=self.cur_qbd, initial_trigger=self.td)

        self.nxt_qbd, self.nxt_start = None, None
        if self.nxt_rpd:
            self.nxt_qbd = next(self.nxt_rpd.qbds, None)
        if self.nxt_qbd:
            self.nxt_start = calc_and_adjust_start(
                user=self.user, qbd=self.nxt_qbd, initial_trigger=self.td)
            if self.cur_qbd is None:
                trace("Finished cur RP with remaining QBs in next")
                self.cur_start = self.nxt_start
                self.transition()
            elif self.cur_start > self.nxt_start + relativedelta(months=1):
                # The plus one month covers RP v5 date adjustments.

                # Valid only when the RP being replaced doesn't have all the
                # visits defined in the next one (i.e. v3 doesn't have months
                # 27 or 33 and v5 does). Look ahead for a match
                self.skipped_nxt_start = self.nxt_start
                self.nxt_start = None
                self.nxt_qbd = next(self.nxt_rpd.qbds, None)
                if self.nxt_qbd:
                    self.nxt_start = calc_and_adjust_start(
                        user=self.user, qbd=self.nxt_qbd,
                        initial_trigger=self.td)
                if self.cur_start > self.nxt_start + relativedelta(months=1):
                    # Still no match means poorly defined RP QBs
                    raise ValueError(
                        "Invalid state {}:{} not in lock-step even on "
                        "look ahead; RPs need to maintain same "
                        "schedule {}, {}, {}".format(
                            self.cur_rpd.rp.name, self.nxt_rpd.rp.name,
                            self.cur_start, self.nxt_start,
                            self.skipped_nxt_start))
        if self.cur_qbd:
            trace("advanced to next QB: {} w/ start {}".format(
                self.cur_qbd.questionnaire_bank.name, self.cur_start))
        else:
            trace("out of QBs!")

    def transition(self):
        """Transition internal state to 'next' Research Protocol"""
        trace("transitioning to the next RP w/ cur_start {}".format(
            self.cur_start))
        if self.cur_start > self.nxt_start + relativedelta(months=1):
            # The plus one month covers RP v5 shift
            raise ValueError(
{'fraction_non_alphanumeric': 0.07883713660518912, 'fraction_numerical': 0.0016880275085964364, 'mean_word_length': 2.563265760748496, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7031119', 'n_tokens_mistral': 9549, 'n_tokens_neox': 9066, 'n_words': 2936}