<commit_before>import sys
from datetime import datetime
import json
import hashlib


def print_rsf(item, key_field):
    key = item[key_field]
    timestamp = datetime.now().isoformat(sep='T')
    item_str = json.dumps(item, separators=(',', ':'), sort_keys=True)
    item_hash = hashlib.sha256(item_str.encode("utf-8")).hexdigest()
    item_line = "add-item\t" + item_str
    entry_line = "append-entry\t{0}\t{1}\tsha-256:{2}".format(
            key, timestamp, item_hash)
    print(item_line)
    print(entry_line)


item_arr = json.load(sys.stdin)
if len(sys.argv) < 2:
    sys.exit("Usage: cat foo.json | python json2rsf.py [key field name]")
if not isinstance(item_arr, list):
    sys.exit("Error: input must be json array")
key = sys.argv[1]
[print_rsf(item, key) for item in item_arr]
<commit_msg>Fix time format<commit_after>import sys
from datetime import datetime
import json
import hashlib


def print_rsf(item, key_field):
    key = item[key_field]
    timestamp = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
    item_str = json.dumps(item, separators=(',', ':'), sort_keys=True)
    item_hash = hashlib.sha256(item_str.encode("utf-8")).hexdigest()
    item_line = "add-item\t" + item_str
    entry_line = "append-entry\t{0}\t{1}\tsha-256:{2}".format(
            key, timestamp, item_hash)
    print(item_line)
    print(entry_line)


item_arr = json.load(sys.stdin)
if len(sys.argv) < 2:
    sys.exit("Usage: cat foo.json | python json2rsf.py [key field name]")
if not isinstance(item_arr, list):
    sys.exit("Error: input must be json array")
key = sys.argv[1]
[print_rsf(item, key) for item in item_arr]

{'fraction_non_alphanumeric': 0.1530424093423479, 'fraction_numerical': 0.015980331899200985, 'mean_word_length': 4.1682539682539685, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2991078', 'n_tokens_mistral': 615, 'n_tokens_neox': 594, 'n_words': 130}