<commit_before>"""
Contains the core building blocks of the framework.
"""
import math
from copy import deepcopy

import pandas as pd
import numpy as np
import cython as cy


class Node(object):

    """
    The Node is the main building block in bt's tree structure design.
    Both StrategyBase and SecurityBase inherit Node. It contains the
    core functionality of a tree node.

    Args:
        * name (str): The Node name
        * parent (Node): The parent Node
        * children (dict, list): A collection of children. If dict,
            the format is {name: child}, if list then list of children.

    Attributes:
        * name (str): Node name
        * parent (Node): Node parent
        * root (Node): Root node of the tree (topmost node)
        * children (dict): Node's children
        * now (datetime): Used when backtesting to store current date
        * stale (bool): Flag used to determine if Node is stale and need
            updating
        * prices (TimeSeries): Prices of the Node. Prices for a security will
            be the security's price, for a strategy it will be an index that
            reflects the value of the strategy over time.
        * price (float): last price
        * value (float): last value
        * weight (float): weight in parent
        * full_name (str): Name including parents' names
        * members (list): Current Node + node's children

    """

    _price = cy.declare(cy.double)
    _value = cy.declare(cy.double)
    _weight = cy.declare(cy.double)
    _issec = cy.declare(cy.bint)
    _has_strat_children = cy.declare(cy.bint)

    def __init__(self, name, parent=None, children=None):

        self.name = name

        # strategy children helpers
        self._has_strat_children = False
        self._strat_children = []

        # if children is not None, we assume that we want to limit the
        # available children space to the provided list.
        if children is not None:
            if isinstance(children, list):
                # if all strings - just save as universe_filter
                if all(isinstance(x, str) for x in children):
                    self._universe_tickers = children
                    # empty dict - don't want to uselessly create
                    # tons of children when they might not be needed
                    children = {}
                else:
                    # this will be case if we pass in children
                    # (say a bunch of sub-strategies)
                    tmp = {}
                    ut = []
                    for c in children:
                        if type(c) == str:
                            tmp[c] = SecurityBase(c)
                            ut.append(c)
                        else:
                            tmp[c.name] = c

                            # if strategy, turn on flag and add name to list
                            # strategy children have special treatment
                            if isinstance(c, StrategyBase):
                                self._has_strat_children = True
                                self._strat_children.append(c.name)
                            # if not strategy, then we will want to add this to
                            # universe_tickers to filter on setup
                            else:
                                ut.append(c.name)

                    children = tmp
                    # we want to keep whole universe in this case
                    # so set to None
                    self._universe_tickers = ut

        if parent is None:
            self.parent = self
            self.root = self
        else:
            self.parent = parent
            self.root = parent.root
            parent._add_child(self)

        # default children
        if children is None:
            children = {}
            self._universe_tickers = None
        self.children = children

        self._childrenv = children.values()
        for c in self._childrenv:
            c.parent = self
            c.root = self.root

        # set default value for now
        self.now = 0
        # make sure root has stale flag
        # used to avoid unncessary update
        # sometimes we change values in the tree and we know that we will need
        # to update if another node tries to access a given value (say weight).
        # This avoid calling the update until it is actually needed.
        self.root.stale = False

        # helper vars
        self._price = 0
        self._value = 0
        self._weight = 0

        # is security flag - used to avoid updating 0 pos securities
        self._issec = False

    def __getitem__(self, key):
        return self.children[key]

    @property
    def prices(self):
        """
        A TimeSeries of the Node's price.
        """
        # can optimize depending on type -
        # securities don't need to check stale to
        # return latest prices, whereas strategies do...
        raise NotImplementedError()

    @property
    def price(self):
        """
        Current price of the Node
        """
        # can optimize depending on type -
        # securities don't need to check stale to
        # return latest prices, whereas strategies do...
        raise NotImplementedError()

    @property
    def value(self):
        """
        Current value of the Node
        """
        if self.root.stale:
            self.root.update(self.root.now, None)
        return self._value

    @property
    def weight(self):
        """
        Current weight of the Node (with respect to the parent).
        """
        if self.root.stale:
            self.root.update(self.root.now, None)
        return self._weight

    def setup(self, dates):
        """
        Setup method used to initialize a Node with a set of dates.
        """
        raise NotImplementedError()

    def _add_child(self, child):
        child.parent = self
        child.root = self.root
        if self.children is None:
            self.children = {child.name: child}
        else:
            self.children[child.name] = child

        self._childrenv = self.children.values()

    def update(self, date, data=None):
        """
        Update Node with latest date, and optionally some data.
        """
        raise NotImplementedError()

    def adjust(self, amount, update=True, isflow=True):
        """
        Adjust Node value by amount.
        """
        raise NotImplementedError()

    def allocate(self, amount, update=True):
        """
        Allocate capital to Node.
        """
        raise NotImplementedError()

    @property
    def members(self):
        """
        Node members. Members include current node as well as Node's
        children.
        """
        res = [self]
        for c in self.children.values():
            res.extend(c.members)
        return res

    @property
    def full_name(self):
        if self.parent == self:
            return self.name
        else:
            return '%s>%s' % (self.parent.full_name, self.name)


class StrategyBase(Node):

    """
    Strategy Node. Used to define strategy logic within a tree.
    A Strategy's role is to allocate capital to it's children
    based on a function.

    Args:
        * name (str): Strategy name
        * children (dict, list): A collection of children. If dict,
            the format is {name: child}, if list then list of children.
            Children can be any type of Node.
        * parent (Node): The parent Node

    Attributes:
        * name (str): Strategy name
        * parent (Strategy): Strategy parent
        * root (Strategy): Root node of the tree (topmost node)
        * children (dict): Strategy's children
        * now (datetime): Used when backtesting to store current date
        * stale (bool): Flag used to determine if Strategy is stale and need
            updating
        * prices (TimeSeries): Prices of the Strategy - basically an index that
            reflects the value of the strategy over time.
        * price (float): last price
        * value (float): last value
        * weight (float): weight in parent
        * full_name (str): Name including parents' names
        * members (list): Current Strategy + strategy's children
        * commission_fn (fn(quantity, price)): A function used to determine the
            commission (transaction fee) amount. Could be used to model slippage
            (implementation shortfall).
        * capital (float): Capital amount in Strategy - cash
        * universe (DataFrame): Data universe available at the current time.
            Universe contains the data passed in when creating a Backtest. Use
            this data to determine strategy logic.

    """

    _capital = cy.declare(cy.double)
    _net_flows = cy.declare(cy.double)
    _last_value = cy.declare(cy.double)
    _last_price = cy.declare(cy.double)
    _paper_trade = cy.declare(cy.bint)

    def __init__(self, name, children=None, parent=None):
        Node.__init__(self, name, children=children, parent=parent)
        self._capital = 0
        self._weight = 1
        self._value = 0
        self._price = 100

        # helper vars
        self._net_flows = 0
        self._last_value = 0
        self._last_price = 100

        # default commission function
        self.commission_fn = self._dflt_comm_fn

        self._paper_trade = False
        self._positions = None

    @property
    def price(self):
        """
        Current price.
        """
        if self.root.stale:
            self.root.update(self.now, None)
        return self._price

    @property
    def prices(self):
        """
        TimeSeries of prices.
        """
        if self.root.stale:
            self.root.update(self.now, None)
        return self._prices.ix[:self.now]

    @property
    def values(self):
        """
        TimeSeries of values.
        """
        if self.root.stale:
            self.root.update(self.now, None)
        return self._values.ix[:self.now]

    @property
    def capital(self):
        """
        Current capital - amount of unallocated capital left in strategy.
        """
        # no stale check needed
        return self._capital

    @property
    def universe(self):
        """
        Data universe available at the current time.
        Universe contains the data passed in when creating a Backtest.
        Use this data to determine strategy logic.
        """
        # avoid windowing every time
        # if calling and on same date return
        # cached value
        if self.now == self._last_chk:
            return self._funiverse
        else:
            self._last_chk = self.now
            self._funiverse = self._universe.ix[:self.now]
            return self._funiverse

    @property
    def positions(self):
        """
        TimeSeries of positions.
        """
        # if accessing and stale - update first
        if self.root.stale:
            self.root.update(self.root.now, None)

        if self._positions is not None:
            return self._positions
        else:
            vals = pd.DataFrame({x.name: x.positions for x in self.members
                                 if isinstance(x, SecurityBase)})
            self._positions = vals
            return vals

    def setup(self, universe):
        """
        Setup strategy with universe. This will speed up future calculations
        and updates.
        """
        # save full universe in case we need it
        self._original_data = universe

        # determine if needs paper trading
        # and setup if so
        if self is not self.parent:
            self._paper_trade = True
            self._paper_amount = 1000000

            paper = deepcopy(self)
            paper.parent = paper
            paper.root = paper
            paper._paper_trade = False
            paper.setup(self._original_data)
            paper.adjust(self._paper_amount)
            self._paper = paper

        # setup universe
        funiverse = universe

        if self._universe_tickers is not None:
            # if we have universe_tickers defined, limit universe to
            # those tickers
            valid_filter = list(set(universe.columns)
                                .intersection(self._universe_tickers))

            funiverse = universe[valid_filter]

            # if we have strat children, we will need to create their columns
            # in the new universe
            if self._has_strat_children:
                for c in self._strat_children:
                    funiverse[c] = np.nan

            # must create to avoid pandas warning
            funiverse = pd.DataFrame(funiverse)

        self._universe = funiverse
        # holds filtered universe
        self._funiverse = funiverse
        self._last_chk = None

        # setup internal data
        self.data = pd.DataFrame(index=funiverse.index,
                                 columns=['price', 'value'],
                                 data=0.0)

        self._prices = self.data['price']
        self._values = self.data['value']

        # setup children as well - use original universe here - don't want to
        # pollute with potential strategy children in funiverse
        if self.children is not None:
            [c.setup(universe) for c in self._childrenv]

    @cy.locals(newpt=cy.bint, val=cy.double, ret=cy.double)
    def update(self, date, data=None):
        """
        Update strategy. Updates prices, values, weight, etc.
        """
        # resolve stale state
        self.root.stale = False

        # update helpers on date change
        # also set newpt flag
        newpt = False
        if self.now == 0:
            newpt = True
        elif date != self.now:
            self._net_flows = 0
            self._last_price = self._price
            self._last_value = self._value
            newpt = True

        # update now
        self.now = date

        # update children if any and calculate value
        val = self._capital  # default if no children

        if self.children is not None:
            for c in self._childrenv:
                # avoid useless update call
                if c._issec and not c._needupdate:
                    continue
                c.update(date, data)
                val += c.value

        if self.root == self:
            if val < 0:
                raise ValueError('negative root node value!')

        # update data if this value is different or
        # if now has changed - avoid all this if not since it
        # won't change
        if newpt or self._value != val:
            self._value = val
            self._values[date] = val

            try:
                ret = self._value / (self._last_value
                                     + self._net_flows) - 1
            except ZeroDivisionError:
                if self._value == 0:
                    ret = 0
                else:
                    raise ZeroDivisionError(
                        'Could not update %s. Last value '
                        'was %s and net flows were %s. Current'
                        'value is %s. Therefore, '
                        'we are dividing by zero to obtain the return '
                        'for the period.' % (self.name,
                                             self._last_value,
                                             self._net_flows,
                                             self._value))

            self._price = self._last_price * (1 + ret)
            self._prices[date] = self._price

        # update children weights
        if self.children is not None:
            for c in self._childrenv:
                # avoid useless update call
                if c._issec and not c._needupdate:
                    continue
                try:
                    c._weight = c.value / val
                except ZeroDivisionError:
                    c._weight = 0.0

        # if we have strategy children, we will need to update them in universe
        if self._has_strat_children:
            for c in self._strat_children:
                self._universe.loc[date, c] = self.children[c].price

        # update paper trade if necessary
        if newpt and self._paper_trade:
            self._paper.update(date)
            self._paper.run()
            self._paper.update(date)
            # update price
            self._price = self._paper.price
            self._prices[date] = self._price

    @cy.locals(amount=cy.double, update=cy.bint, flow=cy.bint)
    def adjust(self, amount, update=True, flow=True):
        """<commit_msg>ENH: unallocated cash is saved in "data" structure of the Strategy<commit_after>"""
Contains the core building blocks of the framework.
"""
import math
from copy import deepcopy

import pandas as pd
import numpy as np
import cython as cy


class Node(object):

    """
    The Node is the main building block in bt's tree structure design.
    Both StrategyBase and SecurityBase inherit Node. It contains the
    core functionality of a tree node.

    Args:
        * name (str): The Node name
        * parent (Node): The parent Node
        * children (dict, list): A collection of children. If dict,
            the format is {name: child}, if list then list of children.

    Attributes:
        * name (str): Node name
        * parent (Node): Node parent
        * root (Node): Root node of the tree (topmost node)
        * children (dict): Node's children
        * now (datetime): Used when backtesting to store current date
        * stale (bool): Flag used to determine if Node is stale and need
            updating
        * prices (TimeSeries): Prices of the Node. Prices for a security will
            be the security's price, for a strategy it will be an index that
            reflects the value of the strategy over time.
        * price (float): last price
        * value (float): last value
        * weight (float): weight in parent
        * full_name (str): Name including parents' names
        * members (list): Current Node + node's children

    """

    _price = cy.declare(cy.double)
    _value = cy.declare(cy.double)
    _weight = cy.declare(cy.double)
    _issec = cy.declare(cy.bint)
    _has_strat_children = cy.declare(cy.bint)

    def __init__(self, name, parent=None, children=None):

        self.name = name

        # strategy children helpers
        self._has_strat_children = False
        self._strat_children = []

        # if children is not None, we assume that we want to limit the
        # available children space to the provided list.
        if children is not None:
            if isinstance(children, list):
                # if all strings - just save as universe_filter
                if all(isinstance(x, str) for x in children):
                    self._universe_tickers = children
                    # empty dict - don't want to uselessly create
                    # tons of children when they might not be needed
                    children = {}
                else:
                    # this will be case if we pass in children
                    # (say a bunch of sub-strategies)
                    tmp = {}
                    ut = []
                    for c in children:
                        if type(c) == str:
                            tmp[c] = SecurityBase(c)
                            ut.append(c)
                        else:
                            tmp[c.name] = c

                            # if strategy, turn on flag and add name to list
                            # strategy children have special treatment
                            if isinstance(c, StrategyBase):
                                self._has_strat_children = True
                                self._strat_children.append(c.name)
                            # if not strategy, then we will want to add this to
                            # universe_tickers to filter on setup
                            else:
                                ut.append(c.name)

                    children = tmp
                    # we want to keep whole universe in this case
                    # so set to None
                    self._universe_tickers = ut

        if parent is None:
            self.parent = self
            self.root = self
        else:
            self.parent = parent
            self.root = parent.root
            parent._add_child(self)

        # default children
        if children is None:
            children = {}
            self._universe_tickers = None
        self.children = children

        self._childrenv = children.values()
        for c in self._childrenv:
            c.parent = self
            c.root = self.root

        # set default value for now
        self.now = 0
        # make sure root has stale flag
        # used to avoid unncessary update
        # sometimes we change values in the tree and we know that we will need
        # to update if another node tries to access a given value (say weight).
        # This avoid calling the update until it is actually needed.
        self.root.stale = False

        # helper vars
        self._price = 0
        self._value = 0
        self._weight = 0

        # is security flag - used to avoid updating 0 pos securities
        self._issec = False

    def __getitem__(self, key):
        return self.children[key]

    @property
    def prices(self):
        """
        A TimeSeries of the Node's price.
        """
        # can optimize depending on type -
        # securities don't need to check stale to
        # return latest prices, whereas strategies do...
        raise NotImplementedError()

    @property
    def price(self):
        """
        Current price of the Node
        """
        # can optimize depending on type -
        # securities don't need to check stale to
        # return latest prices, whereas strategies do...
        raise NotImplementedError()

    @property
    def value(self):
        """
        Current value of the Node
        """
        if self.root.stale:
            self.root.update(self.root.now, None)
        return self._value

    @property
    def weight(self):
        """
        Current weight of the Node (with respect to the parent).
        """
        if self.root.stale:
            self.root.update(self.root.now, None)
        return self._weight

    def setup(self, dates):
        """
        Setup method used to initialize a Node with a set of dates.
        """
        raise NotImplementedError()

    def _add_child(self, child):
        child.parent = self
        child.root = self.root
        if self.children is None:
            self.children = {child.name: child}
        else:
            self.children[child.name] = child

        self._childrenv = self.children.values()

    def update(self, date, data=None):
        """
        Update Node with latest date, and optionally some data.
        """
        raise NotImplementedError()

    def adjust(self, amount, update=True, isflow=True):
        """
        Adjust Node value by amount.
        """
        raise NotImplementedError()

    def allocate(self, amount, update=True):
        """
        Allocate capital to Node.
        """
        raise NotImplementedError()

    @property
    def members(self):
        """
        Node members. Members include current node as well as Node's
        children.
        """
        res = [self]
        for c in self.children.values():
            res.extend(c.members)
        return res

    @property
    def full_name(self):
        if self.parent == self:
            return self.name
        else:
            return '%s>%s' % (self.parent.full_name, self.name)


class StrategyBase(Node):

    """
    Strategy Node. Used to define strategy logic within a tree.
    A Strategy's role is to allocate capital to it's children
    based on a function.

    Args:
        * name (str): Strategy name
        * children (dict, list): A collection of children. If dict,
            the format is {name: child}, if list then list of children.
            Children can be any type of Node.
        * parent (Node): The parent Node

    Attributes:
        * name (str): Strategy name
        * parent (Strategy): Strategy parent
        * root (Strategy): Root node of the tree (topmost node)
        * children (dict): Strategy's children
        * now (datetime): Used when backtesting to store current date
        * stale (bool): Flag used to determine if Strategy is stale and need
            updating
        * prices (TimeSeries): Prices of the Strategy - basically an index that
            reflects the value of the strategy over time.
        * price (float): last price
        * value (float): last value
        * weight (float): weight in parent
        * full_name (str): Name including parents' names
        * members (list): Current Strategy + strategy's children
        * commission_fn (fn(quantity, price)): A function used to determine the
            commission (transaction fee) amount. Could be used to model slippage
            (implementation shortfall).
        * capital (float): Capital amount in Strategy - cash
        * universe (DataFrame): Data universe available at the current time.
            Universe contains the data passed in when creating a Backtest. Use
            this data to determine strategy logic.

    """

    _capital = cy.declare(cy.double)
    _net_flows = cy.declare(cy.double)
    _last_value = cy.declare(cy.double)
    _last_price = cy.declare(cy.double)
    _paper_trade = cy.declare(cy.bint)

    def __init__(self, name, children=None, parent=None):
        Node.__init__(self, name, children=children, parent=parent)
        self._capital = 0
        self._weight = 1
        self._value = 0
        self._price = 100

        # helper vars
        self._net_flows = 0
        self._last_value = 0
        self._last_price = 100

        # default commission function
        self.commission_fn = self._dflt_comm_fn

        self._paper_trade = False
        self._positions = None

    @property
    def price(self):
        """
        Current price.
        """
        if self.root.stale:
            self.root.update(self.now, None)
        return self._price

    @property
    def prices(self):
        """
        TimeSeries of prices.
        """
        if self.root.stale:
            self.root.update(self.now, None)
        return self._prices.ix[:self.now]

    @property
    def values(self):
        """
        TimeSeries of values.
        """
        if self.root.stale:
            self.root.update(self.now, None)
        return self._values.ix[:self.now]

    @property
    def capital(self):
        """
        Current capital - amount of unallocated capital left in strategy.
        """
        # no stale check needed
        return self._capital

    @property
    def cash(self):
        """
        TimeSeries of unallocated capital.
        """
        # no stale check needed
        return self._cash

    @property
    def universe(self):
        """
        Data universe available at the current time.
        Universe contains the data passed in when creating a Backtest.
        Use this data to determine strategy logic.
        """
        # avoid windowing every time
        # if calling and on same date return
        # cached value
        if self.now == self._last_chk:
            return self._funiverse
        else:
            self._last_chk = self.now
            self._funiverse = self._universe.ix[:self.now]
            return self._funiverse

    @property
    def positions(self):
        """
        TimeSeries of positions.
        """
        # if accessing and stale - update first
        if self.root.stale:
            self.root.update(self.root.now, None)

        if self._positions is not None:
            return self._positions
        else:
            vals = pd.DataFrame({x.name: x.positions for x in self.members
                                 if isinstance(x, SecurityBase)})
            self._positions = vals
            return vals

    def setup(self, universe):
        """
        Setup strategy with universe. This will speed up future calculations
        and updates.
        """
        # save full universe in case we need it
        self._original_data = universe

        # determine if needs paper trading
        # and setup if so
        if self is not self.parent:
            self._paper_trade = True
            self._paper_amount = 1000000

            paper = deepcopy(self)
            paper.parent = paper
            paper.root = paper
            paper._paper_trade = False
            paper.setup(self._original_data)
            paper.adjust(self._paper_amount)
            self._paper = paper

        # setup universe
        funiverse = universe

        if self._universe_tickers is not None:
            # if we have universe_tickers defined, limit universe to
            # those tickers
            valid_filter = list(set(universe.columns)
                                .intersection(self._universe_tickers))

            funiverse = universe[valid_filter]

            # if we have strat children, we will need to create their columns
            # in the new universe
            if self._has_strat_children:
                for c in self._strat_children:
                    funiverse[c] = np.nan

            # must create to avoid pandas warning
            funiverse = pd.DataFrame(funiverse)

        self._universe = funiverse
        # holds filtered universe
        self._funiverse = funiverse
        self._last_chk = None

        # setup internal data
        self.data = pd.DataFrame(index=funiverse.index,
                                 columns=['price', 'value', 'cash', 'fees'],
                                 data=0.0)

        self._prices = self.data['price']
        self._values = self.data['value']
        self._cash = self.data['cash']
        self._fees = self.data['fees']

        # setup children as well - use original universe here - don't want to
        # pollute with potential strategy children in funiverse
        if self.children is not None:
            [c.setup(universe) for c in self._childrenv]

    @cy.locals(newpt=cy.bint, val=cy.double, ret=cy.double)
    def update(self, date, data=None):
        """
        Update strategy. Updates prices, values, weight, etc.
        """
        # resolve stale state
        self.root.stale = False

        # update helpers on date change
        # also set newpt flag
        newpt = False
        if self.now == 0:
            newpt = True
        elif date != self.now:
            self._net_flows = 0
            self._last_price = self._price
            self._last_value = self._value
            newpt = True

        # update now
        self.now = date

        # update children if any and calculate value
        val = self._capital  # default if no children

        if self.children is not None:
            for c in self._childrenv:
                # avoid useless update call
                if c._issec and not c._needupdate:
                    continue
                c.update(date, data)
                val += c.value

        if self.root == self:
            if val < 0:
                raise ValueError('negative root node value!')

        # update data if this value is different or
        # if now has changed - avoid all this if not since it
        # won't change
        if newpt or self._value != val:
            self._value = val
            self._values[date] = val

            try:
                ret = self._value / (self._last_value
                                     + self._net_flows) - 1
            except ZeroDivisionError:
                if self._value == 0:
                    ret = 0
                else:
                    raise ZeroDivisionError(
                        'Could not update %s. Last value '
                        'was %s and net flows were %s. Current'
                        'value is %s. Therefore, '
                        'we are dividing by zero to obtain the return '
                        'for the period.' % (self.name,
                                             self._last_value,
                                             self._net_flows,
                                             self._value))

            self._price = self._last_price * (1 + ret)
            self._prices[date] = self._price

        # update children weights
        if self.children is not None:
            for c in self._childrenv:
                # avoid useless update call
                if c._issec and not c._needupdate:
                    continue
                try:
                    c._weight = c.value / val
                except ZeroDivisionError:
                    c._weight = 0.0

        # if we have strategy children, we will need to update them in universe
        if self._has_strat_children:
            for c in self._strat_children:
                self._universe.loc[date, c] = self.children[c].price

        # Cash should track the unallocated capital at the end of the day, so
        # we should update it every time we call "update"
        self._cash[self.now] = self._capital

        # update paper trade if necessary
        if newpt and self._paper_trade:
            self._paper.update(date)
            self._paper.run()
            self._paper.update(date)
            # update price
            self._price = self._paper.price
            self._prices[date] = self._price

    @cy.locals(amount=cy.double, update=cy.bint, flow=cy.bint)
    def adjust(self, amount, update=True, flow=True):
        """
{'fraction_non_alphanumeric': 0.07505695097778173, 'fraction_numerical': 0.002011774799562143, 'mean_word_length': 1.53445302541801, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '699701', 'n_tokens_mistral': 8733, 'n_tokens_neox': 8284, 'n_words': 3025}