df = pd.read_csv('temp.txt', header=None, delim_whitespace=True, names=['data']
    )
bins = df.data.str.endswith('[a-token]')
idx_bins = df[bins][:]
idx_bins.data = idx_bins.data.str.rstrip(to_strip='[a-token]')
idx_vals = df[~bins][:]
a = pd.DataFrame(idx_bins.index.values, columns=['a'])
b = pd.DataFrame(idx_vals.index.values, columns=['b'])
merge_df = pd.merge_asof(b, a, left_on='b', right_on='a')
new_df = pd.DataFrame({'A': idx_bins.data.loc[list(merge_df.a)].values, 'B':
    idx_vals.data.values})
Pandas: parsing values in structured non tabular text