<jupyter_start><jupyter_text>Pandas> Utilities for pandas<jupyter_code>#hide
from nbdev.showdoc import *
# export
import pandas as pd
import pandas_flavor
from datetime import datetime 
import typing as T
import numpy as np
import time<jupyter_output><empty_output><jupyter_text>`DataFrame` methods  `ends`This function shows combines the `.head()` and `.tail()` method of the `pd.DataFrame`.Options:* `n`: number of rows to show for the head and tail. The resulting df will have 2\*n rows, unless the df has <2\*n rows.<jupyter_code># export
@pandas_flavor.register_dataframe_method
@pandas_flavor.register_series_method
def ends(self, n:int=3)-> pd.DataFrame:
    """
    Combines the .head and .tail methods to show both ends
    of a pd.DataFrame.
    
    Options:
    * n: number of rows to show for the head and tail. The resulting
         DataFrame will have 2*n rows, unless the df has <2*n rows.
    """
    return pd.concat([self.head(n), self.tail(n)], axis=0).drop_duplicates()
df = pd.DataFrame(np.random.rand(20, 5))
df.head()
df.tail()
df.ends()<jupyter_output><empty_output><jupyter_text>`group_by_summary`This method returns a df with the mean, median, sem, std, and count of each column.Options:* `by`: Colums to group by. These will be the index of the df returned.* `filter_cols`: if `None`, will use all the columns. Otherwise, pass column name or list of column names.<jupyter_code># export
@pandas_flavor.register_dataframe_method
def group_by_summary(self, by:T.Union[T.AnyStr, T.List], filter_cols:T.Union[None, T.List[str]]=None)-> pd.DataFrame:
    """
    
    """
    if not filter_cols:
        filter_cols = self.columns[~self.columns.isin([by] if isinstance(by, str) else by)]
    return self.groupby(by)[filter_cols].agg(['mean','median','sem','std','count'])
df['cat_col'] = df[0] > 0.5
df.group_by_summary('cat_col')
df.group_by_summary(by='cat_col', filter_cols=1)
df.group_by_summary(['cat_col'], filter_cols=1)<jupyter_output><empty_output><jupyter_text>datetime_from_timestampConverts a pd.Series of integer timestamps to datetime format.Options:* `s`: pd.Series of timestamps* `ts_type`: type of timestamp.   * `ms`: default, milliseconds  * `ns`: nanoseconds  * `s`: seconds<jupyter_code># export
def datetime_from_timestamp(s:pd.Series, ts_type:str='ms') -> pd.Series:
    lookup = {
        'ms':1e3,
        's':1,
        'ns': 1e6
    }
    return s.map(lambda x: datetime.fromtimestamp(x/lookup[ts_type]))
datetime_from_timestamp(pd.Series([1627417286117]))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13249211356466878, 'fraction_numerical': 0.012618296529968454, 'mean_word_length': 4.955399061032864, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '598038', 'license': 'Apache-2.0', 'n_tokens_mistral': 902, 'n_tokens_neox': 862, 'n_words': 233, 'path': '02_pandas.ipynb', 'repo_name': 'mgfrantz/dessiccate', 'size': '2536'}