<commit_before>"""
squash_tweets.py

USAGE:
$ python squash_tweets.py [-h] [--parse] path

Squash multiple jsons into one.

positional arguments:
  path        path to directory containing tweet jsons. Needs trailing "/"

optional arguments:
  -h, --help  show this help message and exit
  --parse     use if jsons to squash need to be parsed first
"""

import sys
import os
import glob
import argparse

from datetime import datetime

from json_io import list_to_json, list_from_json


if __name__ == "__main__":
    # Setup CLA parser
    parser = argparse.ArgumentParser(description='Squash multiple jsons into one')
    parser.add_argument('path', help='path to directory containing tweet jsons. Needs trailing "/"')
    parser.add_argument('--delete', action="store_true", help='delete old JSONs after squashing')
    # Parse CLAs
    args = parser.parse_args()
    if not os.path.exists(args.path):
        raise Exception("{} is not a valid path.".format(args.path))

    # Populate list with paths to jsons
    json_paths_lst = glob.glob(args.path + "*.json")

    # create one list of all merged tweets
    merged_lst = [ tweet for json_path in json_paths_lst for tweet in list_from_json(json_path)]

    # Get earliest and latest date of jsons squashed for naming purposes of merged file.
    parse_date_from_filename = lambda fn: fn.split('/')[-1].split('.')[0].split('_')
    sorted_dates = sorted([datetime.strptime(date, "%Y-%m-%d") for fn in json_paths_lst for date in parse_date_from_filename(fn)])
    from_date = datetime.strftime(sorted_dates[0], "%Y-%m-%d")
    to_date = datetime.strftime(sorted_dates[-1], "%Y-%m-%d")
    filename = "{}_{}.json".format(from_date, to_date)

    # Save merged list to json
    list_to_json(merged_lst, args.path + filename)

    # Delete old files if flag is set
    if args.delete:
        for json_path in json_paths_lst:
            os.remove(json_path)<commit_msg>use json_io function, minor doctstring fix<commit_after>"""
squash_tweets.py

USAGE: squash_tweets.py [-h] [--delete] path

Squash multiple jsons into one

positional arguments:
  path        path to directory containing tweet jsons. Needs trailing "/"

optional arguments:
  -h, --help  show this help message and exit
  --delete    delete old JSONs after squashing
"""

import sys
import os
import glob
import argparse

from json_io import list_to_json, list_from_json, merge_json_filenames


if __name__ == "__main__":
    # Setup CLA parser
    parser = argparse.ArgumentParser(description='Squash multiple jsons into one')
    parser.add_argument('path', help='path to directory containing tweet jsons. Needs trailing "/"')
    parser.add_argument('--delete', action="store_true", help='delete old JSONs after squashing')
    # Parse CLAs
    args = parser.parse_args()
    if not os.path.exists(args.path):
        raise Exception("{} is not a valid path.".format(args.path))

    # Populate list with paths to jsons
    json_paths_lst = glob.glob(args.path + "*-*-*_*-*-*.json")

    # create one list of all merged tweets
    merged_lst = [ tweet for json_path in json_paths_lst for tweet in list_from_json(json_path)]

    # Get earliest and latest date of jsons squashed for naming purposes of merged file.
    filename = merge_json_filenames(json_paths_lst)

    # Save merged list to json
    list_to_json(merged_lst, args.path + filename)

    # Delete old files if flag is set
    if args.delete:
        for json_path in json_paths_lst:
            os.remove(json_path)
{'fraction_non_alphanumeric': 0.10071326676176891, 'fraction_numerical': 0.0011412268188302425, 'mean_word_length': 3.643708609271523, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5590735', 'n_tokens_mistral': 1105, 'n_tokens_neox': 1065, 'n_words': 353}