cli/isp-upload-cli.py

#!/usr/bin/env python3

import glob
import os

import random
import requests

from datasets import load_dataset, Dataset, DatasetDict

folder_path = 'isp-data-json/'  # Replace with your folder path

# Search for all .json files in the folder
json_files = glob.glob(os.path.join(folder_path, '*.jsonl'))

path = 'pminervini/inverse-scaling'

for json_path in json_files:
    base_name = os.path.basename(json_path)
    name = base_name.split("_")[0]

    ds = load_dataset("json", data_files={'data': json_path})
    ds.push_to_hub(path, config_name=name)