<commit_before>#######################################################################
# PackageEvaluator
# https://github.com/IainNZ/PackageEvaluator.jl
# (c) <NAME> 2015
# Licensed under the MIT License
#######################################################################
# Combines the JSON files in a folder into one
#######################################################################

import JSON

# ARGS[1]: Folder containing JSONs
# ARGS[2]: Output filename (excluding .json)
@assert length(ARGS) == 2

# Parse each JSON and combine
all_pkgs = Dict[]
for file in readdir(ARGS[1])
    !ismatch(r"json", file) && continue
    try
        push!(all_pkgs, JSON.parsefile(joinpath(ARGS[1],file)))
    catch e
        if isa(e, SystemError)
            # Propbably mmap failure due to empty file
            println("$file: $(e.prefix)")
        else
            println("Error for $file:")
            println(e)
        end
    end
end

# Then write them all to one big file
println("Writing $(length(all_pkgs)) packages to $(ARGS[2]).json")
cat_fp = open("$(ARGS[2]).json","w")
JSON.print(cat_fp, all_pkgs)
close(cat_fp)<commit_msg>use a do block instead of close<commit_after>#######################################################################
# PackageEvaluator
# https://github.com/IainNZ/PackageEvaluator.jl
# (c) <NAME> 2015
# Licensed under the MIT License
#######################################################################
# Combines the JSON files in a folder into one
#######################################################################

import JSON

# ARGS[1]: Folder containing JSONs
# ARGS[2]: Output filename (excluding .json)
@assert length(ARGS) == 2

# Parse each JSON and combine
all_pkgs = Dict[]
for file in readdir(ARGS[1])
    !ismatch(r"json", file) && continue
    try
        push!(all_pkgs, JSON.parsefile(joinpath(ARGS[1],file)))
    catch e
        if isa(e, SystemError)
            # Propbably mmap failure due to empty file
            println("$file: $(e.prefix)")
        else
            println("Error for $file:")
            println(e)
        end
    end
end

# Then write them all to one big file
println("Writing $(length(all_pkgs)) packages to $(ARGS[2]).json")
open("$(ARGS[2]).json","w") do cat_fp
    JSON.print(cat_fp, all_pkgs)
end
{'fraction_non_alphanumeric': 0.2892561983471074, 'fraction_numerical': 0.009569377990430622, 'mean_word_length': 4.145413870246085, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5652419', 'n_tokens_mistral': 681, 'n_tokens_neox': 639, 'n_words': 188}