# Path: virtool/api/json.py
# class CustomEncoder(json.JSONEncoder):
#     """
#     A custom :class:`JSONEncoder` that converts :class:`datetime` objects to
#     ISO-formatting date strings.
# 
#     """
# 
#     def default(self, obj):
#         if isinstance(obj, datetime.datetime):
#             return isoformat(obj)
# 
#         return json.JSONEncoder.default(self, obj)
# 
# Path: virtool/history/db.py
# async def patch_to_version(app, otu_id: str, version: Union[str, int]) -> tuple:
#     """
#     Take a joined otu back in time to the passed ``version``.
# 
#     Uses the diffs in the change documents associated with the otu.
# 
#     :param app: the application object
#     :param otu_id: the id of the otu to patch
#     :param version: the version to patch to
#     :return: the current joined otu, patched otu, and the ids of reverted changes
# 
#     """
#     db = app["db"]
# 
#     reverted_history_ids = list()
# 
#     current = await virtool.otus.db.join(db, otu_id) or dict()
# 
#     if "version" in current and current["version"] == version:
#         return current, deepcopy(current), reverted_history_ids
# 
#     patched = deepcopy(current)
# 
#     # Sort the changes by descending timestamp.
#     async for change in db.history.find({"otu.id": otu_id}, sort=[("otu.version", -1)]):
#         if change["otu"]["version"] == "removed" or change["otu"]["version"] > version:
#             reverted_history_ids.append(change["_id"])
# 
#             if change["diff"] == "file":
#                 change["diff"] = await virtool.history.utils.read_diff_file(
#                     app["config"].data_path, otu_id, change["otu"]["version"]
#                 )
# 
#             if change["method_name"] == "remove":
#                 patched = change["diff"]
# 
#             elif change["method_name"] == "create":
#                 patched = None
# 
#             else:
#                 diff = dictdiffer.swap(change["diff"])
#                 patched = dictdiffer.patch(diff, patched)
#         else:
#             break
# 
#     if current == {}:
#         current = None
# 
#     return current, patched, reverted_history_ids
# 
# Path: virtool/history/utils.py
# async def remove_diff_files(app, id_list: List[str]):
#     """
#     Remove multiple diff files given a list of change IDs (`id_list`).
# 
#     :param app: the application object
#     :param id_list: a list of change IDs to remove diff files for
# 
#     """
#     data_path = app["config"].data_path
# 
#     for change_id in id_list:
#         otu_id, otu_version = change_id.split(".")
# 
#         path = join_diff_path(data_path, otu_id, otu_version)
# 
#         try:
#             await app["run_in_thread"](os.remove, path)
#         except FileNotFoundError:
#             pass


import asyncio
import json
import os
import shutil
import aiohttp
import arrow
import virtool.errors
import virtool.otus.db
import virtool.tasks.pg
from datetime import timedelta
from pathlib import Path
from semver import VersionInfo
from virtool.api.json import CustomEncoder
from virtool.github import create_update_subdocument
from virtool.history.db import patch_to_version
from virtool.history.utils import remove_diff_files
from virtool.http.utils import download_file
from virtool.references.db import (
    download_and_parse_release,
    fetch_and_update_release,
    insert_change,
    insert_joined_otu,
    update_joined_otu,
)
from virtool.references.utils import check_import_data, load_reference_file
from virtool.tasks.task import Task
from virtool.utils import compress_json_with_gzip, get_temp_dir
            {"reference.id": {"$in": self.non_existent_references}}
        )

        await virtool.tasks.pg.update(
            self.pg, self.id, progress=tracker.step_completed, step="remove_indexes"
        )

    async def remove_unreferenced_otus(self):
        tracker = await self.get_tracker()

        for ref_id in self.non_existent_references:
            referenced_otu_ids = await self.db.analyses.distinct(
                "results.otu.id", {"reference.id": ref_id}
            )

            unreferenced_otu_ids = await self.db.otus.distinct(
                "_id",
                {"reference.id": ref_id, "_id": {"$not": {"$in": referenced_otu_ids}}},
            )

            diff_file_change_ids = await self.db.history.distinct(
                "_id", {"diff": "file", "otu.id": {"$in": unreferenced_otu_ids}}
            )

            await asyncio.gather(
                self.db.otus.delete_many({"_id": {"$in": unreferenced_otu_ids}}),
                self.db.history.delete_many({"otu.id": {"$in": unreferenced_otu_ids}}),
                self.db.sequences.delete_many(
                    {"otu_id": {"$in": unreferenced_otu_ids}}
                ),

# Path: virtool/api/json.py
# class CustomEncoder(json.JSONEncoder):
#     """
#     A custom :class:`JSONEncoder` that converts :class:`datetime` objects to
#     ISO-formatting date strings.
# 
#     """
# 
#     def default(self, obj):
#         if isinstance(obj, datetime.datetime):
#             return isoformat(obj)
# 
#         return json.JSONEncoder.default(self, obj)
# 
# Path: virtool/history/db.py
# async def patch_to_version(app, otu_id: str, version: Union[str, int]) -> tuple:
#     """
#     Take a joined otu back in time to the passed ``version``.
# 
#     Uses the diffs in the change documents associated with the otu.
# 
#     :param app: the application object
#     :param otu_id: the id of the otu to patch
#     :param version: the version to patch to
#     :return: the current joined otu, patched otu, and the ids of reverted changes
# 
#     """
#     db = app["db"]
# 
#     reverted_history_ids = list()
# 
#     current = await virtool.otus.db.join(db, otu_id) or dict()
# 
#     if "version" in current and current["version"] == version:
#         return current, deepcopy(current), reverted_history_ids
# 
#     patched = deepcopy(current)
# 
#     # Sort the changes by descending timestamp.
#     async for change in db.history.find({"otu.id": otu_id}, sort=[("otu.version", -1)]):
#         if change["otu"]["version"] == "removed" or change["otu"]["version"] > version:
#             reverted_history_ids.append(change["_id"])
# 
#             if change["diff"] == "file":
#                 change["diff"] = await virtool.history.utils.read_diff_file(
#                     app["config"].data_path, otu_id, change["otu"]["version"]
#                 )
# 
#             if change["method_name"] == "remove":
#                 patched = change["diff"]
# 
#             elif change["method_name"] == "create":
#                 patched = None
# 
#             else:
#                 diff = dictdiffer.swap(change["diff"])
#                 patched = dictdiffer.patch(diff, patched)
#         else:
#             break
# 
#     if current == {}:
#         current = None
# 
#     return current, patched, reverted_history_ids
# 
# Path: virtool/history/utils.py
# async def remove_diff_files(app, id_list: List[str]):
#     """
#     Remove multiple diff files given a list of change IDs (`id_list`).
# 
#     :param app: the application object
#     :param id_list: a list of change IDs to remove diff files for
# 
#     """
#     data_path = app["config"].data_path
# 
#     for change_id in id_list:
#         otu_id, otu_version = change_id.split(".")
# 
#         path = join_diff_path(data_path, otu_id, otu_version)
# 
#         try:
#             await app["run_in_thread"](os.remove, path)
#         except FileNotFoundError:
#             pass


# Path: virtool/references/tasks.py
import asyncio
import json
import os
import shutil
import aiohttp
import arrow
import virtool.errors
import virtool.otus.db
import virtool.tasks.pg
from datetime import timedelta
from pathlib import Path
from semver import VersionInfo
from virtool.api.json import CustomEncoder
from virtool.github import create_update_subdocument
from virtool.history.db import patch_to_version
from virtool.history.utils import remove_diff_files
from virtool.http.utils import download_file
from virtool.references.db import (
    download_and_parse_release,
    fetch_and_update_release,
    insert_change,
    insert_joined_otu,
    update_joined_otu,
)
from virtool.references.utils import check_import_data, load_reference_file
from virtool.tasks.task import Task
from virtool.utils import compress_json_with_gzip, get_temp_dir

            {"reference.id": {"$in": self.non_existent_references}}
        )

        await virtool.tasks.pg.update(
            self.pg, self.id, progress=tracker.step_completed, step="remove_indexes"
        )

    async def remove_unreferenced_otus(self):
        tracker = await self.get_tracker()

        for ref_id in self.non_existent_references:
            referenced_otu_ids = await self.db.analyses.distinct(
                "results.otu.id", {"reference.id": ref_id}
            )

            unreferenced_otu_ids = await self.db.otus.distinct(
                "_id",
                {"reference.id": ref_id, "_id": {"$not": {"$in": referenced_otu_ids}}},
            )

            diff_file_change_ids = await self.db.history.distinct(
                "_id", {"diff": "file", "otu.id": {"$in": unreferenced_otu_ids}}
            )

            await asyncio.gather(
                self.db.otus.delete_many({"_id": {"$in": unreferenced_otu_ids}}),
                self.db.history.delete_many({"otu.id": {"$in": unreferenced_otu_ids}}),
                self.db.sequences.delete_many(
                    {"otu_id": {"$in": unreferenced_otu_ids}}
                ),

                remove_diff_files(self.app, diff_file_change_ids),