<commit_before>#!/bin/python

import argparse
import datetime

import lglass.nic
import lglass.dn42

argparser = argparse.ArgumentParser(
        description="Sync local database with DN42 database")
argparser.add_argument("src")
argparser.add_argument("dst")

args = argparser.parse_args()

src = lglass.dn42.DN42Database(args.src)
dst = lglass.nic.FileDatabase(args.dst)

last_update = dst.manifest.last_modified_datetime

for obj in src.find():
    if obj.last_modified_datetime > last_update:
        if src.database_name not in obj:
            obj.append("source", src.database_name)
        print("Update {} {}".format(obj.object_class, obj.primary_key))
        for fix in lglass.dn42.fix_object(obj):
            dst.save(fix)

for obj in dst.find():
    if src.database_name not in obj.get("source"):
        continue
    original_primary_key = src.primary_key(obj)
    original_object = src.try_fetch(obj.object_class, original_primary_key)
    delete = original_object is None or (obj.object_class in {"route", "route6"} and
            obj["origin"] not in original_object.get("origin"))
    if delete:
        print(original_primary_key)
        print("Delete {} {}".format(obj.object_class, obj.primary_key))
        dst.delete(obj)

dst.manifest.remove("last-modified")
dst.manifest.last_modified = datetime.datetime.now()
dst.save_manifest()
<commit_msg>Added new options to sync_db contrib script<commit_after>#!/bin/python

import argparse
import datetime

import lglass.nic
import lglass.dn42

def sync(src, dst, dn42=False, delete=True):
    last_update = dst.manifest.last_modified_datetime

    for obj in src.find():
        if obj.last_modified_datetime > last_update:
            if src.database_name not in obj:
                obj.append("source", src.database_name)
            if dn42:
                for fix in lglass.dn42.fix_object(obj):
                    yield ('ADD', fix.object_class, fix.primary_key)
                    dst.save(fix)
            else:
                yield ('ADD', obj.object_class, obj.primary_key)
                dst.save(obj)

    if delete:
        for obj in dst.find():
            if src.database_name not in obj.get("source"):
                continue
            original_primary_key = src.primary_key(obj)
            original_object = src.try_fetch(obj.object_class, original_primary_key)
            d = original_object is None or (obj.object_class in {"route", "route6"} and
                    obj["origin"] not in original_object.get("origin") and dn42)
            if d:
                yield ('DEL', obj.object_class, obj.primary_key)
                dst.delete(obj)

    dst.manifest.remove("last-modified")
    dst.manifest.last_modified = datetime.datetime.now()
    dst.save_manifest()

if __name__ == "__main__":
    argparser = argparse.ArgumentParser(
            description="Sync local database with DN42 database")
    argparser.add_argument("--from-dn42", action="store_true")
    argparser.add_argument("--no-delete", action="store_true")
    argparser.add_argument("--mqtt-broker")
    argparser.add_argument("--mqtt-topic")
    argparser.add_argument("--quiet", action="store_true")
    argparser.add_argument("source")
    argparser.add_argument("destination")

    args = argparser.parse_args()

    if args.from_dn42:
        src = lglass.dn42.DN42Database(args.source)
    else:
        src = lglass.nic.FileDatabase(args.source)
    dst = lglass.nic.FileDatabase(args.destination)

    actions = []
    for action, object_class, object_key in sync(src, dst, dn42=args.from_dn42,
            delete=not args.no_delete):
        if not args.quiet:
            print("{} {}:   {}".format(action, object_class, object_key))
        actions.append((action, object_class, object_key))

    if args.mqtt_broker:
        import paho.mqtt.client as mqtt
        mqttc = mqtt.Client()
        mqttc.connect(args.mqtt_broker)
        mqttc.loop_start()
        for a in actions:
            mqttc.publish(args.mqtt_topic, "{} {}: {}".format(*a))
        mqttc.loop_stop(force=False)

{'fraction_non_alphanumeric': 0.11630194246373247, 'fraction_numerical': 0.008851733464470126, 'mean_word_length': 2.8269049858889934, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2835837', 'n_tokens_mistral': 1295, 'n_tokens_neox': 1242, 'n_words': 256}