CACHE = {}
STORE = 'store.h5'

def process_row(d, key, max_len=5000, _cache=CACHE):
    """
    Append row d to the store 'key'.
    When the number of items in the key's cache reaches max_len,
    append the list of rows to the HDF5 store and clear the list.
    """
lst = _cache.setdefault(key, [])
if len(lst) >= max_len:
    store_and_clear(lst, key)
lst.append(d)

def store_and_clear(lst, key):
    """
    Convert key's cache list to a DataFrame and append that to HDF5.
    """
df = pd.DataFrame(lst)
with pd.HDFStore(STORE) as store:
    store.append(key, df)
lst.clear()
Как обрабатывать входящие данные в реальном времени с помощью python pandas