<commit_before>import os
import re
import gzip
import magic
import shutil
import subprocess
import random
import pymongo

import virtool.utils
import virtool.files
import virtool.plots
import virtool.gen
import virtool.database
import virtool.job
import virtool.pathoscope


class Collection(virtool.database.Collection):
    """
    A connection to the pymongo samples collection. Provides methods for viewing and modifying the
    collection.

    """
    def __init__(self, dispatcher):
        super(Collection, self).__init__("samples", dispatcher)

        dispatcher.watcher.register("reads", self.watch)

        # Extend sync_projector. These fields will be passed to the client to populate sample tables.
        self.sync_projector.update({field: True for field in [
            "name",
            "added",
            "username",
            "analyses",
            "imported",
            "archived",
            "analyzed",
            "group",
            "group_read",
            "group_write",
            "all_read",
            "all_write"
        ]})

        db_sync = virtool.utils.get_db_client(self.settings, sync=True)

        db_sync.analyses.update({"algorithm": {"$exists": False}}, {
            "$set": {<commit_msg>Add code to change NuVs records to use HMM database for joins<commit_after>import os
import re
import gzip
import magic
import shutil
import subprocess
import random
import pymongo

import virtool.utils
import virtool.files
import virtool.plots
import virtool.gen
import virtool.database
import virtool.job
import virtool.pathoscope


class Collection(virtool.database.Collection):
    """
    A connection to the pymongo samples collection. Provides methods for viewing and modifying the
    collection.

    """
    def __init__(self, dispatcher):
        super(Collection, self).__init__("samples", dispatcher)

        dispatcher.watcher.register("reads", self.watch)

        # Extend sync_projector. These fields will be passed to the client to populate sample tables.
        self.sync_projector.update({field: True for field in [
            "name",
            "added",
            "username",
            "analyses",
            "imported",
            "archived",
            "analyzed",
            "group",
            "group_read",
            "group_write",
            "all_read",
            "all_write"
        ]})

        db_sync = virtool.utils.get_db_client(self.settings, sync=True)

        for analysis in db_sync.analyses.find({"algorithm": "nuvs"}):

            if "definition" in analysis["hmm"][0]:

                hits = analysis["hmm"]

                for hit in hits:
                    cluster = int(hit["hit"].split("_")[1])
                    hmm = db_sync.hmm.find_one({"cluster": cluster}, {"_id": True})

                    hit.pop("definition")
                    hit.pop("families")

                    hit["hit"] = hmm["_id"]

                db_sync.analyses.update({"_id": analysis["_id"]}, {
                    "$set": {
                        "hmm": hits
                    }
                })

        db_sync.analyses.update({"algorithm": {"$exists": False}}, {
            "$set": {
{'fraction_non_alphanumeric': 0.10686305233469132, 'fraction_numerical': 0.0006267627702914447, 'mean_word_length': 2.0604026845637584, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1312128', 'n_tokens_mistral': 902, 'n_tokens_neox': 841, 'n_words': 228}