<jupyter_start><jupyter_text>json to validation script<jupyter_code>import json
req = json.loads(json.load(open("sync_ml_11_28.txt", "r")))["body"]
seg_list = []
for seg in req["segments"]:
    seg_list.append((seg["id"], seg["originalText"], seg["startTime"]))
seg_list = sorted(seg_list, key=lambda kv:kv[-1], reverse=False)

print (len(seg_list))
for seg in seg_list:
    print ("Segment: ", seg[0], seg[1])
    print ("\n\n")


for index, seg in enumerate(req["segments"]):
    if seg["id"] == '6574a7eb96dc4b9a845526b88f4f3227':
        req["segments"][index]["tag"] = 3
import json
json.dump(json.dumps({"body":req}), open("set_1.txt", "w") )
test = json.loads(json.load(open("set_1_raw.txt", "r")))
test
for index, seg in enumerate(test["body"]["segments"]):
    try:
        print (seg["tag"])
    except Exception as e:
        print (seg["id"], index)
test["body"]["segments"][47]["tag"] = 2
import json
json.dump(json.dumps(test), open("validation_set_4.txt", "w") )
import json
filename = "sync_eng_12_19.txt"
test = json.loads(json.load(open(filename, "r")))
req = test["detail"]
json.dump(json.dumps({"body":req}), open(filename, "w") )
req = test["detail"]
req<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2091286307053942, 'fraction_numerical': 0.03319502074688797, 'mean_word_length': 5.813559322033898, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '613689', 'license': 'MIT', 'n_tokens_mistral': 479, 'n_tokens_neox': 466, 'n_words': 89, 'path': 'community_detection/group_segments/topic_testing/csv_to_json_reformt_req.ipynb', 'repo_name': 'etherlabsio/hinton', 'size': '1205'}