path = '../lastfm_train/'

def data_iterator(path):
    for root, dirs, files in os.walk(path):
        for f in files:
            if f.endswith('.json'):
                fp = os.path.join(root, f)
with open(fp) as o:
    data = json.load(o)
yield {'similars': data['similars'], 'track_id': data['track_id']}
df = pd.DataFrame(data_iterator(path))
Чтение огромного количества файлов json в Python?