<commit_before>"""
cartoframe methods
~~~~~~~~~~~~~~~~~~

A pandas dataframe interface for working with CARTO maps and tables.

<NAME> and <NAME>, 2017

Project goals
    * Interact with a CARTO table fully within a Jupyter notebook/pandas
      workflow (read and sync dataframe changes)
    * Like geopandas, have a .carto_map() method which gives back the data
      as a map using carto's maps api and TurboCartoCSS on an optional
      attribute
    * Add CARTO services like the Data Observatory as methods to a dataframe

Features to add: see issues in the repository https://github.com/CartoDB/cartoframes/issues?q=is%3Aopen+is%3Aissue+label%3Aenhancement
"""

# TODO: hook into pandas.core?
import pandas as pd
import cartoframes.utils as utils
import cartoframes.maps as maps
import carto


# NOTE: this is compatible with v1.0.0 of carto-python client
def read_carto(username=None, api_key=None, onprem_url=None, tablename=None,
               query=None, include_geom=True, limit=None, cdb_client=None,
               index='cartodb_id', debug=False):
    """Create a DataFrame from a CARTO table, storing table information in
       pandas metadata that allows for further cartoframe options like syncing,
       map creation, and augmentation from the Data Observatory.

       :param username: CARTO username (default None)
       :type username: string
       :param api_key: CARTO API key
       :type api_key: string
       :param onprem_url: BASEURL for onprem
       :type onprem_url: string
       :param tablename: Table to create a cartoframe from
       :type tablename: string
       :param query: Query for generating a cartoframe
       :type query: string
       :param include_geom: Not implemented
       :type include_geom: boolean
       :param limit: (optional) The maximum number of rows to pull
       :type limit: integer
       :param cdb_client: (optional) CARTO Python SDK authentication client object (default None)
       :type cdb_client: object
       :param index: (optional) Column to use for the index (default `cartodb_id`)
       :type index: string

       :returns: A pandas DataFrame linked to a CARTO table
       :rtype: cartoframe
    """
    import cartoframes.maps as maps
    # TODO: if onprem, use the specified template/domain? instead
    # either cdb_client or user credentials have to be specified
    sql = utils.get_auth_client(username=username,
                                api_key=api_key,
                                baseurl=onprem_url,
                                cdb_client=cdb_client)
    is_org_user = utils.get_is_org_user(sql)

    # construct query
    if tablename is not None and query is None:
        tablequery = 'SELECT * FROM "{tablename}"'.format(tablename=tablename)
        # Add limit if requested
        if limit:
            # TODO: ensure that this does not cause sync problems
            if (limit >= 0) and isinstance(limit, int):
                tablequery += ' LIMIT {limit}'.format(limit=limit)
            else:
                raise ValueError("`limit` parameter must an integer >= 0")
        if debug: print(tablequery)
        _df = utils.df_from_table(tablequery, sql, index=index)
    elif query:
        # NOTE: this registers a carto table if `tablename` is specified
        _df = utils.df_from_query(query, sql, is_org_user, username,
                                  tablename=tablename, debug=debug)
    else:
        raise ValueError("Either `tablename` or `query` (or both) need to be "
                         "specified.")

    # TODO: find out of there's a max length to clip on
    # only make map and set metadata if it's becoming a cartoframe
    if tablename:
        named_map_name = maps.create_named_map(username, api_key,
                                               tablename=tablename)
        print("Named map name: {}".format(named_map_name))

        _df.set_metadata(tablename=tablename,
                         username=username,
                         api_key=api_key,
                         named_map_name=named_map_name,
                         include_geom=include_geom,
                         limit=limit,
                         is_org_user=is_org_user,
                         geomtype=utils.get_geom_type(sql,
                                                      tablename=tablename))

        # save the state for later use
        # NOTE: this doubles the size of the dataframe
        _df.set_last_state()

    # store carto sql client for later use
    _df.set_carto_sql_client(sql)

    return _df


def carto_registered(self):
    """Says whether dataframe is registered as a table on CARTO

    :returns: Whether a DataFrame is registered as a table on CARTO
    :rtype: boolean
    """
    try:
        return self.get_carto_tablename() is not None
    except IndexError:
        return False

def carto_insync(self):
    """Says whether current cartoframe is in sync with last saved state

    :returns: Whether the current state of the cartoframe has changed since last sychronized with CARTO
    :rtype: boolean
    """
    return self.equals(self.carto_last_state)

def set_last_state(self):
    """
    Store the state of the cartoframe

    :returns: None
    """
    self.carto_last_state = self.copy(deep=True)

def set_carto_sql_client(self, sql_client):
    """
    Store the SQL client for later use

    :returns: None
    """
    self.carto_sql_client = sql_client
    # self._metadata[-1] = json.dumps()

def get_carto_sql_client(self):
    """
    Retrieves the internally stored SQL Auth client

    :returns: the internally stored sql client
    :rtype: CARTO SQL Auth client object
    """
    return self.carto_sql_client

# TODO: write a decorator for the following `get` functions
def get_carto_api_key(self):
    """return the username of a cartoframe

    :returns: CARTO API key associated with cartoframe
    :rtype: string
    """
    try:
        return self.get_carto('carto_api_key')
    except KeyError:
        raise Exception("This cartoframe is not registered. "
                        "Use `DataFrame.carto_register()`.")

def get_carto_username(self):
    """return the username of a cartoframe

    :returns: CARTO username associated with cartoframe
    :rtype: string
    """
    try:
        return self.get_carto('carto_username')
    except KeyError:
        raise Exception("This cartoframe is not registered. "
                        "Use `DataFrame.carto_register()`.")

def get_carto_datapage(self):
    """Return the CARTO dataset page

    :returns: URL of data page of dataset on CARTO (behind password if private)
    :rtype: string
    """
    # TODO: generalize this for onprem
    # e.g., https://eschbacher.carto.com/dataset/research_team
    url_template = 'https://{username}.carto.com/dataset/{tablename}'

    return url_template.format(username=self.get_carto_username(),
                               tablename=self.get_carto_tablename())

def get_carto_tablename(self):
    """return the username of a cartoframe

    :returns: Table that cartoframe is associated with
    :rtype: string
    """
    try:
        return self.get_carto('carto_table')
    except KeyError:
        raise Exception("This cartoframe is not registered. "
                        "Use `DataFrame.carto_register()`.")


def get_carto_is_org_user(self):
    """Return whether carto `username` is in a multiuser account or not
    """
    return self.get_carto('carto_is_org_user')


def get_carto_geomtype(self):
    """return the geometry type of the cartoframe

    :returns: Geometry type in table (one of 'point', 'line', 'polygon', or 'None')
    :rtype: text
    """
    return self.get_carto('carto_geomtype')

def get_carto_namedmap(self):
    """Return the named map associated with a cartoframe

    :returns: Name of named map
    :rtype: string
    """
    return self.get_carto('carto_named_map')

def get_carto(self, key):
    """General get method for reading from cartoframe metadata

    :param key: key of item to fetch from metadata. One of `carto_named_map`, `carto_geomtype`, `carto_username`, `carto_table`.
    :type key: string
    :returns: Value stored in cartoframe metadata
    :rtype: any
    """
    import json
    try:
        return json.loads(self._metadata[-1])[key]
    except IndexError:
        raise Exception('DataFrame not linked to CARTO. Use '
                        '`DataFrame.sync_carto()` with a new tablename.')


def set_metadata(self, tablename=None, username=None, api_key=None,
                 include_geom=None, limit=None, is_org_user=None,
                 geomtype=None, named_map_name=None):
    """
    Set method for storing metadata in a dataframe

    :returns: None
    """
    import json
    # NOTE: pylint complains that we're accessing a 'protected member
    #       _metadata of a client class' (appending to _metadata only works
    #       with strings, not JSON, so we're serializing here)
    self._metadata.append(
        json.dumps({'carto_table': tablename,
                    'carto_username': username,
                    'carto_api_key': api_key,
                    'carto_named_map': named_map_name,
                    'carto_include_geom': include_geom,
                    'carto_is_org_user': is_org_user,
                    'carto_limit': limit,
                    'carto_geomtype': geomtype}))
    return None



# TODO: make less buggy about the diff between NaNs and nulls
def sync_carto(self, username=None, api_key=None, requested_tablename=None,
               n_batch=20, lnglat_cols=None, debug=False):
    """If an existing cartoframe, this method syncs with the CARTO table a
        cartoframe is associated with. If syncing a DataFrame which has not yet
        been linked with CARTO, it creates a new table if the tablename does
        not yet exist and updates the metadata in the DataFrame.

    :param username: CARTO username credentials. Needed only if the DataFrame
        is not yet linked to CARTO (e.g., it was created with
        ``pd.DataFrame.read_carto``)
    :type username: string
    :param api_key: CARTO API key. Needed only if linking a DataFrame with a
        table on CARTO.
    :type api_key: string
    :param requested_tablename: if set, creates a new table with name
        `new_tablename` on account connected through the auth_client. If this
        tablename exists on CARTO already, an exception will be thrown.
    :type requested_tablename: string
    :param n_batch: Number of queries to include in a batch update to the
        database (experimental).
    :type n_batch: integer
    :param lnglat_cols: Columns which have the latitude and longitude (in that
        order) for creating the geometry in the database. Once this cartoframe
        syncs, a new column called `the_geom` will be pulled down that is a
        text representation of the geometry.
    :type lnglat_cols: tuple
    """

    if (requested_tablename is not None and username is not None and
            api_key is not None):
        # create table on carto if it doesn't not already exist
        # TODO: make this the main way of intereacting with carto_create
        self.carto_create(username, api_key, requested_tablename, debug=debug,
                          is_org_user=self.get_carto_is_org_user(),
                          lnglat_cols=lnglat_cols)
        self.set_last_state()
        return None
    elif not self.carto_registered():
        raise Exception("Table not registered with CARTO. Set "
                        "`requested_tablename` flag to a new tablename and " "enter credentials.")

    if self.equals(self.carto_last_state):
        print("Cartoframe is already synced")
        return None

    # create new column if needed
    # TODO: extract to function
    if len(set(self.columns) - set(self.carto_last_state.columns)) > 0:
        newcols = set(self.columns) - set(self.carto_last_state.columns)
        for col in newcols:
            utils.add_col(self, col, n_batch=n_batch,
                                      debug=debug)

    # drop column if needed
    # TODO: extract to function
    if len(set(self.carto_last_state.columns) - set(self.columns)) > 0:
        discardedcols = set(self.carto_last_state.columns) - set(self.columns)
        for col in discardedcols:
            utils.drop_col(self, col, debug=debug)

    # sync updated values
    # TODO: what happens if rows are removed?
    common_cols = list(set(self.columns) & set(self.carto_last_state.columns))
    if not self[common_cols].equals(self.carto_last_state[common_cols]):
        # NOTE: this updates null-valued cells which have not changed since
        #       np.nan != np.nan is True
        df_diff = (self[common_cols] !=
                   self.carto_last_state[common_cols]).stack()
        df_diff = df_diff[df_diff]
        utils.upsert_table(self, df_diff, debug=debug)

    # update state of dataframe
    self.set_last_state()
    print("Sync completed successfully")

# TODO: add geometry creation (now there is no trigger to fill in `the_geom`
#       like there is for the import api)
def carto_create(self, username, api_key, tablename, lnglat_cols=None,
                 debug=False):
    """Create and populate a table on carto with a dataframe.
    This is a private method, but can be used to create a new table on
    CARTO. It is used in carto_sync if a DataFrame is not yet linked to a
    CARTO table."""

    # give dataframe authentication client
    self.set_carto_sql_client(
        utils.get_auth_client(username=username,
                              api_key=api_key))
    is_org_user = utils.get_is_org_user(self.carto_sql_client)

    final_tablename = self._carto_create_table(tablename, username,
                            is_org_user=is_org_user, debug=debug)
    if debug: print("final_tablename: {}".format(final_tablename))
    self.set_metadata(tablename=final_tablename,
                      username=username,
                      is_org_user=is_org_user,
                      api_key=api_key,
                      include_geom=None,
                      limit=None,
                      geomtype='point' if lnglat_cols else None)
    # TODO: would it be better to cartodbfy after the inserts?
    # TODO: how to ensure some consistency between old index and new one? can cartodb_id be zero-valued?
    self._carto_insert_values(debug=debug)

    # override index<commit_msg>adds map creation in carto_create<commit_after>"""
cartoframe methods
~~~~~~~~~~~~~~~~~~

A pandas dataframe interface for working with CARTO maps and tables.

<NAME> and <NAME>, 2017

Project goals
    * Interact with a CARTO table fully within a Jupyter notebook/pandas
      workflow (read and sync dataframe changes)
    * Like geopandas, have a .carto_map() method which gives back the data
      as a map using carto's maps api and TurboCartoCSS on an optional
      attribute
    * Add CARTO services like the Data Observatory as methods to a dataframe

Features to add: see issues in the repository https://github.com/CartoDB/cartoframes/issues?q=is%3Aopen+is%3Aissue+label%3Aenhancement
"""

# TODO: hook into pandas.core?
import pandas as pd
import cartoframes.utils as utils
import cartoframes.maps as maps
import carto


# NOTE: this is compatible with v1.0.0 of carto-python client
def read_carto(username=None, api_key=None, onprem_url=None, tablename=None,
               query=None, include_geom=True, limit=None, cdb_client=None,
               index='cartodb_id', debug=False):
    """Create a DataFrame from a CARTO table, storing table information in
       pandas metadata that allows for further cartoframe options like syncing,
       map creation, and augmentation from the Data Observatory.

       :param username: CARTO username (default None)
       :type username: string
       :param api_key: CARTO API key
       :type api_key: string
       :param onprem_url: BASEURL for onprem
       :type onprem_url: string
       :param tablename: Table to create a cartoframe from
       :type tablename: string
       :param query: Query for generating a cartoframe
       :type query: string
       :param include_geom: Not implemented
       :type include_geom: boolean
       :param limit: (optional) The maximum number of rows to pull
       :type limit: integer
       :param cdb_client: (optional) CARTO Python SDK authentication client object (default None)
       :type cdb_client: object
       :param index: (optional) Column to use for the index (default `cartodb_id`)
       :type index: string

       :returns: A pandas DataFrame linked to a CARTO table
       :rtype: cartoframe
    """
    import cartoframes.maps as maps
    # TODO: if onprem, use the specified template/domain? instead
    # either cdb_client or user credentials have to be specified
    sql = utils.get_auth_client(username=username,
                                api_key=api_key,
                                baseurl=onprem_url,
                                cdb_client=cdb_client)
    is_org_user = utils.get_is_org_user(sql)

    # construct query
    if tablename is not None and query is None:
        tablequery = 'SELECT * FROM "{tablename}"'.format(tablename=tablename)
        # Add limit if requested
        if limit:
            # TODO: ensure that this does not cause sync problems
            if (limit >= 0) and isinstance(limit, int):
                tablequery += ' LIMIT {limit}'.format(limit=limit)
            else:
                raise ValueError("`limit` parameter must an integer >= 0")
        if debug: print(tablequery)
        _df = utils.df_from_table(tablequery, sql, index=index)
    elif query:
        # NOTE: this registers a carto table if `tablename` is specified
        _df = utils.df_from_query(query, sql, is_org_user, username,
                                  tablename=tablename, debug=debug)
    else:
        raise ValueError("Either `tablename` or `query` (or both) need to be "
                         "specified.")

    # TODO: find out of there's a max length to clip on
    # only make map and set metadata if it's becoming a cartoframe
    if tablename:
        named_map_name = maps.create_named_map(username, api_key,
                                               tablename=tablename)
        print("Named map name: {}".format(named_map_name))

        _df.set_metadata(tablename=tablename,
                         username=username,
                         api_key=api_key,
                         named_map_name=named_map_name,
                         include_geom=include_geom,
                         limit=limit,
                         is_org_user=is_org_user,
                         geomtype=utils.get_geom_type(sql,
                                                      tablename=tablename))

        # save the state for later use
        # NOTE: this doubles the size of the dataframe
        _df.set_last_state()

    # store carto sql client for later use
    _df.set_carto_sql_client(sql)

    return _df


def carto_registered(self):
    """Says whether dataframe is registered as a table on CARTO

    :returns: Whether a DataFrame is registered as a table on CARTO
    :rtype: boolean
    """
    try:
        return self.get_carto_tablename() is not None
    except IndexError:
        return False

def carto_insync(self):
    """Says whether current cartoframe is in sync with last saved state

    :returns: Whether the current state of the cartoframe has changed since last sychronized with CARTO
    :rtype: boolean
    """
    return self.equals(self.carto_last_state)

def set_last_state(self):
    """
    Store the state of the cartoframe

    :returns: None
    """
    self.carto_last_state = self.copy(deep=True)

def set_carto_sql_client(self, sql_client):
    """
    Store the SQL client for later use

    :returns: None
    """
    self.carto_sql_client = sql_client
    # self._metadata[-1] = json.dumps()

def get_carto_sql_client(self):
    """
    Retrieves the internally stored SQL Auth client

    :returns: the internally stored sql client
    :rtype: CARTO SQL Auth client object
    """
    return self.carto_sql_client

# TODO: write a decorator for the following `get` functions
def get_carto_api_key(self):
    """return the username of a cartoframe

    :returns: CARTO API key associated with cartoframe
    :rtype: string
    """
    try:
        return self.get_carto('carto_api_key')
    except KeyError:
        raise Exception("This cartoframe is not registered. "
                        "Use `DataFrame.carto_register()`.")

def get_carto_username(self):
    """return the username of a cartoframe

    :returns: CARTO username associated with cartoframe
    :rtype: string
    """
    try:
        return self.get_carto('carto_username')
    except KeyError:
        raise Exception("This cartoframe is not registered. "
                        "Use `DataFrame.carto_register()`.")

def get_carto_datapage(self):
    """Return the CARTO dataset page

    :returns: URL of data page of dataset on CARTO (behind password if private)
    :rtype: string
    """
    # TODO: generalize this for onprem
    # e.g., https://eschbacher.carto.com/dataset/research_team
    url_template = 'https://{username}.carto.com/dataset/{tablename}'

    return url_template.format(username=self.get_carto_username(),
                               tablename=self.get_carto_tablename())

def get_carto_tablename(self):
    """return the username of a cartoframe

    :returns: Table that cartoframe is associated with
    :rtype: string
    """
    try:
        return self.get_carto('carto_table')
    except KeyError:
        raise Exception("This cartoframe is not registered. "
                        "Use `DataFrame.carto_register()`.")


def get_carto_is_org_user(self):
    """Return whether carto `username` is in a multiuser account or not
    """
    return self.get_carto('carto_is_org_user')


def get_carto_geomtype(self):
    """return the geometry type of the cartoframe

    :returns: Geometry type in table (one of 'point', 'line', 'polygon', or 'None')
    :rtype: text
    """
    return self.get_carto('carto_geomtype')

def get_carto_namedmap(self):
    """Return the named map associated with a cartoframe

    :returns: Name of named map
    :rtype: string
    """
    return self.get_carto('carto_named_map')

def get_carto(self, key):
    """General get method for reading from cartoframe metadata

    :param key: key of item to fetch from metadata. One of `carto_named_map`, `carto_geomtype`, `carto_username`, `carto_table`.
    :type key: string
    :returns: Value stored in cartoframe metadata
    :rtype: any
    """
    import json
    try:
        return json.loads(self._metadata[-1])[key]
    except IndexError:
        raise Exception('DataFrame not linked to CARTO. Use '
                        '`DataFrame.sync_carto()` with a new tablename.')


def set_metadata(self, tablename=None, username=None, api_key=None,
                 include_geom=None, limit=None, is_org_user=None,
                 geomtype=None, named_map_name=None):
    """
    Set method for storing metadata in a dataframe

    :returns: None
    """
    import json
    # NOTE: pylint complains that we're accessing a 'protected member
    #       _metadata of a client class' (appending to _metadata only works
    #       with strings, not JSON, so we're serializing here)
    self._metadata.append(
        json.dumps({'carto_table': tablename,
                    'carto_username': username,
                    'carto_api_key': api_key,
                    'carto_named_map': named_map_name,
                    'carto_include_geom': include_geom,
                    'carto_is_org_user': is_org_user,
                    'carto_limit': limit,
                    'carto_geomtype': geomtype}))
    return None



# TODO: make less buggy about the diff between NaNs and nulls
# NOTE: write
def sync_carto(self, username=None, api_key=None, requested_tablename=None,
               n_batch=20, lnglat_cols=None, debug=False):
    """If an existing cartoframe, this method syncs with the CARTO table a
        cartoframe is associated with. If syncing a DataFrame which has not yet
        been linked with CARTO, it creates a new table if the tablename does
        not yet exist and updates the metadata in the DataFrame.

    :param username: CARTO username credentials. Needed only if the DataFrame
        is not yet linked to CARTO (e.g., it was created with
        ``pd.DataFrame.read_carto``)
    :type username: string
    :param api_key: CARTO API key. Needed only if linking a DataFrame with a
        table on CARTO.
    :type api_key: string
    :param requested_tablename: if set, creates a new table with name
        `new_tablename` on account connected through the auth_client. If this
        tablename exists on CARTO already, an exception will be thrown.
    :type requested_tablename: string
    :param n_batch: Number of queries to include in a batch update to the
        database (experimental).
    :type n_batch: integer
    :param lnglat_cols: Columns which have the latitude and longitude (in that
        order) for creating the geometry in the database. Once this cartoframe
        syncs, a new column called `the_geom` will be pulled down that is a
        text representation of the geometry.
    :type lnglat_cols: tuple
    """

    if (requested_tablename is not None and username is not None and
            api_key is not None):
        # create table on carto if it doesn't not already exist
        # TODO: make this the main way of intereacting with carto_create
        self.carto_create(username, api_key, requested_tablename, debug=debug,
                          lnglat_cols=lnglat_cols)
        self.set_last_state()
        return None
    elif not self.carto_registered():
        raise Exception("Table not registered with CARTO. Set "
                        "`requested_tablename` flag to a new tablename and " "enter credentials.")

    if self.equals(self.carto_last_state):
        print("Cartoframe is already synced")
        return None

    # create new column if needed
    # TODO: extract to function
    if len(set(self.columns) - set(self.carto_last_state.columns)) > 0:
        newcols = set(self.columns) - set(self.carto_last_state.columns)
        for col in newcols:
            utils.add_col(self, col, n_batch=n_batch,
                                      debug=debug)

    # drop column if needed
    # TODO: extract to function
    if len(set(self.carto_last_state.columns) - set(self.columns)) > 0:
        discardedcols = set(self.carto_last_state.columns) - set(self.columns)
        for col in discardedcols:
            utils.drop_col(self, col, debug=debug)

    # sync updated values
    # TODO: what happens if rows are removed?
    common_cols = list(set(self.columns) & set(self.carto_last_state.columns))
    if not self[common_cols].equals(self.carto_last_state[common_cols]):
        # NOTE: this updates null-valued cells which have not changed since
        #       np.nan != np.nan is True
        df_diff = (self[common_cols] !=
                   self.carto_last_state[common_cols]).stack()
        df_diff = df_diff[df_diff]
        utils.upsert_table(self, df_diff, debug=debug)

    # update state of dataframe
    self.set_last_state()
    print("Sync completed successfully")

# TODO: add geometry creation (now there is no trigger to fill in `the_geom`
#       like there is for the import api)
def carto_create(self, username, api_key, tablename, lnglat_cols=None,
                 debug=False):
    """Create and populate a table on carto with a dataframe.
    This is a private method, but can be used to create a new table on
    CARTO. It is used in carto_sync if a DataFrame is not yet linked to a
    CARTO table."""

    # give dataframe authentication client
    self.set_carto_sql_client(
        utils.get_auth_client(username=username,
                              api_key=api_key))
    is_org_user = utils.get_is_org_user(self.carto_sql_client)

    final_tablename = self._carto_create_table(tablename, username,
                            is_org_user=is_org_user, debug=debug)
    named_map_name = maps.create_named_map(username, api_key,
                          tablename=final_tablename)
    if debug: print("final_tablename: {}".format(final_tablename))
    self.set_metadata(tablename=final_tablename,
                      username=username,
                      is_org_user=is_org_user,
                      api_key=api_key,
                      named_map_name=named_map_name,
                      include_geom=None,
                      limit=None,
                      geomtype='point' if lnglat_cols else None)
    # TODO: would it be better to cartodbfy after the inserts?
    # TODO: how to ensure some consistency between old index and new one? can cartodb_id be zero-valued?
    self._carto_insert_values(debug=debug)

    # override index
{'fraction_non_alphanumeric': 0.076621500790976, 'fraction_numerical': 0.0012380493844143339, 'mean_word_length': 2.515778019586507, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1634773', 'n_tokens_mistral': 8584, 'n_tokens_neox': 8322, 'n_words': 2725}