<commit_before>import datetime
import json
from random import choice, randint
from pymongo import MongoClient
import sys

HOST = 'localhost'
PORT = 27017
DB_NAME = 'backdrop'
BUCKET = 'licensify'

# IMPORTANT:
#
# Dates are required to be stored as ISODate so you need to run these
# postprocessing scripts after loading data into db:
#
# /usr/bin/mongo your_database --eval "
#   db.your_collection.find({ _timestamp: { \$type: 2}}).forEach(
#     function(doc){
#       doc._timestamp = new ISODate(doc._timestamp);
#       db.your_collection.save(doc)
#     });"
#
# /usr/bin/mongo your_database --eval "
#   db.your_collection.find({ _week_start_at: { \$type: 2}}).forEach(
#     function(doc){
#       doc._week_start_at = new ISODate(doc._week_start_at);
#       db.your_collection.save(doc)
#     });"

USAGE = """
./generate_fake_licensing_data.py (save_to_db|print_json)
"""<commit_msg>Update fake licensing generator to use correct bucket<commit_after>import datetime
import json
from random import choice, randint
from pymongo import MongoClient
import sys

HOST = 'localhost'
PORT = 27017
DB_NAME = 'backdrop'
BUCKET = 'licensing'

# IMPORTANT:
#
# Dates are required to be stored as ISODate so you need to run these
# postprocessing scripts after loading data into db:
#
# /usr/bin/mongo your_database --eval "
#   db.your_collection.find({ _timestamp: { \$type: 2}}).forEach(
#     function(doc){
#       doc._timestamp = new ISODate(doc._timestamp);
#       db.your_collection.save(doc)
#     });"
#
# /usr/bin/mongo your_database --eval "
#   db.your_collection.find({ _week_start_at: { \$type: 2}}).forEach(
#     function(doc){
#       doc._week_start_at = new ISODate(doc._week_start_at);
#       db.your_collection.save(doc)
#     });"

USAGE = """
./generate_fake_licensing_data.py (save_to_db|print_json)
"""
{'fraction_non_alphanumeric': 0.13224043715846995, 'fraction_numerical': 0.007650273224043716, 'mean_word_length': 3.9755434782608696, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3806729', 'n_tokens_mistral': 638, 'n_tokens_neox': 620, 'n_words': 151}