<commit_before>#!/usr/bin/env python
#
# Copyright (C) Citrix Inc
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

# Example storage backend using SMAPIv2

# WARNING: this API is considered to be unstable and may be changed at-will

import os, sys, commands, xmlrpclib

import xcp
from storage import *

import vhd, tapdisk, mount, util

raw_suffix = ".raw"
vhd_suffix = ".vhd"
metadata_suffix = ".xml"

metadata_dir = "metadata"
data_dir = "data"

def highest_id(xs):
    highest = 0L
    for x in xs:
        try:
            y = long(x)
            if y > highest:
                highest = y
        except:
            pass
    return highest

def unlink_safe(path):
    try:
        os.unlink(path)
        log("os.unlink %s OK" % path)
    except Exception, e:
        log("os.unlink %s: %s" % (path, str(e)))


class Repo:
    """Encapsulates all relevant SR operations

The on-disk structure looks like this:

<root>/metadata/name.xml    -- VDI metadata for a disk with name "name"
<root>/metadata/name.1.xml  -- VDI metadata for another disk with name "name"

<root>/data/1.vhd           -- vhd format disk (parent or leaf)
<root>/data/2.vhd           -- vhd format disk (parent or leaf)
<root>/data/3.raw           -- raw format disk

There is an injective relationship between metadata/ files and data/
i.e. all metadata/ files reference exactly one data/ file but not all
data/ files are referenced directly by a metadata/ file.
"""
    def __init__(self, path):
        self.path = path
        # Used to cache which tapdisks are activated read/write (XXX persistence)
        self.writable = {}
        # Load the metadata first
        self.metadata = {}
        metadata_path = path + "/" + metadata_dir
        if not (os.path.exists(metadata_path)):
            os.mkdir(metadata_path)
        for name in os.listdir(metadata_path):
            if name.endswith(metadata_suffix):
                md = metadata_path + "/" + name
                f = open(md, "r")
                try:
                    try:
                        vdi_info = xmlrpclib.loads(f.read())[0][0]
                        self.metadata[name[0:-len(metadata_suffix)]] = vdi_info
                    except:
                        log("%s is corrupt: ignoring" % md)
                finally:
                    f.close()
        # Load data about the data second
        self.data = {}
        data_path = path + "/" + data_dir
        if not(os.path.exists(data_path)):
            os.mkdir(data_path)
        for name in os.listdir(data_path):
            if name.endswith(raw_suffix):
                self.data[name[0:-len(raw_suffix)]] = { "type": "raw" }
        all = vhd.list(data_path + "/*.vhd")
        for name in all.keys():
            log("data[%s] = %s" % (name, repr(all[name])))
            self.data[name] = all[name]
        self._data_highest_id = highest_id(self.data.keys()) + 1L
        # Query running tapdisks. We assume that all tapdisks in this domain
        # reference this SR.
        self.tapdisks = {}
        for tap in tapdisk.list():
            if not tap.file:
                log("%s has no file open: detaching and freeing" % str(tap))
                tap.detach()
                tap.free()
            else:
                ty, path = tap.file
                if path.startswith(data_path):
                    filename = os.path.basename(path)
                    if ty == "raw":
                        name = filename[0:-len(raw_suffix)]
                        log("%s open by %s" % (name, str(tap)))
                        self.tapdisks[name] = tap
                    elif ty == "vhd":
                        name = filename[0:-len(vhd_suffix)]
                        log("%s open by %s" % (name, str(tap)))
                        self.tapdisks[name] = tap
                    else:
                        log("Unknown tapdisk type: %s" % ty)

        # Integrity check: each metadata record should point to a data record
        for vdi_info in self.metadata.values():
            if vdi_info["data"] not in self.data:
                log("WARNING: vdi %s has data %s but this does not exist" % (vdi_info["vdi"], vdi_info["data"]))

    def get_deletable(self):
        """Return a set of the data blobs which are not reachable from
        any of the VDIs """
        reachable = set()
        for vdi_info in self.metadata.values():
            x = vdi_info["data"]
            while x not in reachable:
                reachable.add(x)
                info = self.data[x]
                if "parent" in info:
                    x = info["parent"]
        all = set(self.data.keys())
        return all.difference(reachable)

    def make_fresh_metadata_name(self, vdi_info):
        # To make this vaguely human-readable, sanitise the first 32 chars of the
        # VDI's name and then add a numerical suffix to make unique.
        name = vdi_info["name_label"]
        if len(name) > 32:
            # arbitrary cut-off
            name = name[0:32]
        valid = '_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
        name = ''.join(c for c in name if c in valid)
        if name == "":
            name = "_"
        taken = self.metadata.keys()
        if name not in taken:
            return name
        else:
            superstrings = filter(lambda x:x.startswith(name + "."), taken)
            suffixes = map(lambda x:x[len(name) + 1:], superstrings)
            suffix = highest_id(suffixes) + 1L
            return "%s.%Ld" % (name, suffix)

    def make_fresh_data_name(self):
        name = str(self._data_highest_id)
        self._data_highest_id = self._data_highest_id + 1
        return name

    def metadata_path_of_vdi(self, vdi):
        return self.path + "/" + metadata_dir + "/" + vdi + metadata_suffix

    def data_path_of_key(self, key):
        data = self.data[key]
        if data["type"] == "raw":
            return self.path + "/" + data_dir + "/" + key + raw_suffix
        elif data["type"] == "vhd":
            return self.path + "/" + data_dir + "/" + key + vhd_suffix
        else:
            raise (Vdi_does_not_exist(vdi))

    def data_path_of_vdi(self, vdi):
        vdi_info = self.metadata[vdi]
        key = vdi_info["data"]
        return self.data_path_of_key(key)

    def update_vdi_info(self, vdi, vdi_info):
        path = self.metadata_path_of_vdi(vdi)
        f = open(path, "w")
        try:
            try:
                f.write(xmlrpclib.dumps((vdi_info,), allow_none=True))
                self.metadata[vdi] = vdi_info
            except Exception, e:
                log("Exception writing metadata: %s" % (str(e)))
                # remove corrupt file
                os.unlink(path)
        finally:
            f.close()

    def create(self, vdi_info, params):
        vdi = self.make_fresh_metadata_name(vdi_info)
        vdi_info["vdi"] = vdi

        data = self.make_fresh_data_name()
        vdi_info["data"] = data

        stem = self.path + "/" + data_dir + "/" + data
        virtual_size = long(vdi_info["virtual_size"])
        if "type" in params and params["type"] == "raw":
            f = open(stem + raw_suffix, "wc")
            try:
                f.truncate(virtual_size)
                self.data[data] = { "type": "raw" }
            finally:
                f.close()
        else:
            # xmlrpclib will reject longs which are > 32 bits since it will try
            # to marshal them as XMLRPC integers.
            vdi_info["virtual_size"] = str(vhd.create(virtual_size, stem + vhd_suffix))
            self.data[data] = { "type": "vhd" }

        self.update_vdi_info(vdi, vdi_info)
        return vdi_info

    def gc(self):
        deletable = self.get_deletable()
        for x in deletable:
            log("Data %s is unreachable now" % x)
            path = self.data_path_of_key(x)
            unlink_safe(path)
            del self.data[x]

    def destroy(self, vdi):
        meta = self.metadata_path_of_vdi(vdi)
        unlink_safe(meta)
        del self.metadata[vdi]
        self.gc()

    def clone(self, vdi, vdi_info, params):
        parent = self.data_path_of_vdi(vdi)

        # Create two vhd leaves whose parent is [vdi]
        left = self.make_fresh_data_name()
        self.data[left] = vhd.make_leaf(self.path + "/" + data_dir + "/" + left + vhd_suffix, parent)

        right = self.make_fresh_data_name()
        self.data[right] = vhd.make_leaf(self.path + "/" + data_dir + "/" + right + vhd_suffix, parent)

        # Remap the original [vdi]'s location to point to the first leaf's path
        parent_info = self.metadata[vdi]
        parent_info["data"] = left
        self.update_vdi_info(vdi, parent_info)

        # The cloned vdi's location points to the second leaf's path
        clone = self.make_fresh_metadata_name(vdi_info)
        vdi_info["vdi"] = clone
        vdi_info["data"] = right
        vdi_info["virtual_size"] = parent_info["virtual_size"]
        vdi_info["content_id"] = parent_info["content_id"]
        if vdi_info["content_id"] == "":
            vdi_info["content_id"] = util.gen_uuid()
        vdi_info["read_only"] = parent_info["read_only"]
        self.update_vdi_info(clone, vdi_info)

        return vdi_info

    def snapshot(self, vdi, vdi_info, params):
        # Before modifying the vhd-tree, take a note of the
        # currently-active leaf so we can find its tapdisk later
        old_leaf = self.metadata[vdi]["data"]

        # The vhd-tree manipulation is the same as clone...
        vdi_info = self.clone(vdi, vdi_info, params)
        vdi_info["snapshot_of"] = vdi
        # XXX vdi_info["snapshot_time"]
        # ... but we also re-open any active tapdisks
        if old_leaf in self.tapdisks:
            tapdisk = self.tapdisks[old_leaf]
            if "mirror" in params:
                tapdisk.mirror = params["mirror"]
            else:
                tapdisk.mirror = None
            # Find the new vhd leaf
            new_leaf = self.metadata[vdi]["data"]
            tapdisk.file = (self.data[new_leaf]["type"], self.data_path_of_key(new_leaf))
            tapdisk.reopen()
            del self.tapdisks[old_leaf]
            self.tapdisks[new_leaf] = tapdisk
        return vdi_info

    def stat(self, vdi):
        return self.metadata[vdi]

    def set_persistent(self, vdi, persistent):
        meta = self.metadata[vdi]

        if meta["persistent"] and not persistent:
            # We need to make a fresh vhd leaf
            parent = self.data_path_of_vdi(vdi)

            leaf = self.make_fresh_data_name()
            self.data[leaf] = vhd.make_leaf(self.path + "/" + data_dir + "/" + leaf + vhd_suffix, parent)

            # Remap the original [vdi]'s location to point to the new leaf
            meta["data"] = leaf

        meta["persistent"] = persistent
        self.update_vdi_info(vdi, meta)

    def set_content_id(self, vdi, content_id):
        meta = self.metadata[vdi]
        meta["content_id"] = content_id
        self.update_vdi_info(vdi, meta)

    def get_by_name(self, name):
        for vdi in self.metadata.keys():
            md = self.metadata[vdi]
            if vdi == name:
                return md
        raise Vdi_does_not_exist(name)

    def similar_content(self, vdi):
        chains = {}
        if vdi not in self.metadata.keys():
            raise Vdi_does_not_exist(vdi)
        for v in self.metadata.keys():
            md = self.metadata[v]
            data = md["data"]
            chain = set([data])
            while "parent" in self.data[data]:
                data = self.data[data]["parent"]
                chain.add(data)
            chains[v] = chain
        distance = {}
        target_chain = chains[vdi]
        for v in self.metadata.keys():
            this_chain = chains[v]
            difference = target_chain.symmetric_difference(this_chain)
            # if two disks have nothing in common then we don't consider them
            # as similar at all
            if target_chain.intersection(this_chain) <> set([]):
                distance[v] = len(difference)
        import operator
        ordered = sorted(distance.iteritems(), key=operator.itemgetter(1))
        return map(lambda x:self.metadata[x[0]], ordered)

    def maybe_reset(self, vdi):
        meta = self.metadata[vdi]
        if not(meta["persistent"]):
            # we throw away updates to the leaf
            leaf = meta["data"]
            leaf_info = self.data[leaf]
            new_leaf = self.make_fresh_data_name()
            if "parent" in leaf_info:
                # Create a new leaf
                child = self.path + "/" + data_dir + "/" + new_leaf + vhd_suffix
                parent = self.path + "/" + data_dir + "/" + leaf_info["parent"] + vhd_suffix
                self.data[new_leaf] = vhd.make_leaf(child, parent)
                meta["data"] = new_leaf
                self.update_vdi_info(vdi, meta)
                self.gc()
            else:
                # Recreate a whole blank disk
                new_info = self.create(meta, leaf_info)
                meta["data"] = new_info["data"]
                self.update_vdi_info(vdi, meta)
                self.destroy(new_info["vdi"])

    def epoch_begin(self, vdi):
        self.maybe_reset(vdi)

    def epoch_end(self, vdi):
        self.maybe_reset(vdi)

    def open_dummy_vhd(self, vdi_info, tapdisk):
        # blkback will immediately close if it encounters a blktap
        # device which doesn't have a backing file. It probably needs
        # to know the number of sectors? We can satisfy this with a
        # temporary .vhd file
        dummy_data_name = self.make_fresh_data_name()
        dummy_path = self.path + "/" + data_dir + "/" + dummy_data_name + ".dummy" + vhd_suffix
        virtual_size = long(vdi_info["virtual_size"])
        vhd.create(virtual_size, dummy_path)
        tapdisk.open("vhd", dummy_path)
        tapdisk.pause()
        os.unlink(dummy_path)

    def attach(self, vdi, read_write):
        md = self.metadata[vdi]
        data = md["data"]
        if data in self.tapdisks:
            raise Backend_error("VDI_ALREADY_ATTACHED", [ vdi ])
        self.tapdisks[data] = tapdisk.Tapdisk()
        device = self.tapdisks[data].get_device()
        self.writable[vdi] = read_write

        self.open_dummy_vhd(md, self.tapdisks[data])

        return { "params": device,
                 "xenstore_data": {} }

    def activate(self, vdi):
        md = self.metadata[vdi]
        if self.writable[vdi]:
            md["content_id"] = ""
            self.update_vdi_info(vdi, md)
        data = md["data"]
        if data not in self.tapdisks:
            raise Backend_error("VDI_NOT_ATTACHED", [ vdi ])
        self.tapdisks[data].file = (self.data[data]["type"], self.data_path_of_key(data))
        self.tapdisks[data].unpause()

    def deactivate(self, vdi):
        md = self.metadata[vdi]
        if md["content_id"] == "":
            md["content_id"] = util.gen_uuid()
            self.update_vdi_info(vdi, md)
        data = md["data"]
        if data not in self.tapdisks:
            raise Backend_error("VDI_NOT_ATTACHED", [ vdi ])
        self.tapdisks[data].close()
<commit_msg>fs: avoid leaving the tapdisk paused because it blocks close<commit_after>#!/usr/bin/env python
#
# Copyright (C) Citrix Inc
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

# Example storage backend using SMAPIv2

# WARNING: this API is considered to be unstable and may be changed at-will

import os, sys, commands, xmlrpclib

import xcp
from storage import *

import vhd, tapdisk, mount, util

raw_suffix = ".raw"
vhd_suffix = ".vhd"
metadata_suffix = ".xml"

metadata_dir = "metadata"
data_dir = "data"

def highest_id(xs):
    highest = 0L
    for x in xs:
        try:
            y = long(x)
            if y > highest:
                highest = y
        except:
            pass
    return highest

def unlink_safe(path):
    try:
        os.unlink(path)
        log("os.unlink %s OK" % path)
    except Exception, e:
        log("os.unlink %s: %s" % (path, str(e)))


class Repo:
    """Encapsulates all relevant SR operations

The on-disk structure looks like this:

<root>/metadata/name.xml    -- VDI metadata for a disk with name "name"
<root>/metadata/name.1.xml  -- VDI metadata for another disk with name "name"

<root>/data/1.vhd           -- vhd format disk (parent or leaf)
<root>/data/2.vhd           -- vhd format disk (parent or leaf)
<root>/data/3.raw           -- raw format disk

There is an injective relationship between metadata/ files and data/
i.e. all metadata/ files reference exactly one data/ file but not all
data/ files are referenced directly by a metadata/ file.
"""
    def __init__(self, path):
        self.path = path
        # Used to cache which tapdisks are activated read/write (XXX persistence)
        self.writable = {}
        # Load the metadata first
        self.metadata = {}
        metadata_path = path + "/" + metadata_dir
        if not (os.path.exists(metadata_path)):
            os.mkdir(metadata_path)
        for name in os.listdir(metadata_path):
            if name.endswith(metadata_suffix):
                md = metadata_path + "/" + name
                f = open(md, "r")
                try:
                    try:
                        vdi_info = xmlrpclib.loads(f.read())[0][0]
                        self.metadata[name[0:-len(metadata_suffix)]] = vdi_info
                    except:
                        log("%s is corrupt: ignoring" % md)
                finally:
                    f.close()
        # Load data about the data second
        self.data = {}
        data_path = path + "/" + data_dir
        if not(os.path.exists(data_path)):
            os.mkdir(data_path)
        for name in os.listdir(data_path):
            if name.endswith(raw_suffix):
                self.data[name[0:-len(raw_suffix)]] = { "type": "raw" }
        all = vhd.list(data_path + "/*.vhd")
        for name in all.keys():
            log("data[%s] = %s" % (name, repr(all[name])))
            self.data[name] = all[name]
        self._data_highest_id = highest_id(self.data.keys()) + 1L
        # Query running tapdisks. We assume that all tapdisks in this domain
        # reference this SR.
        self.tapdisks = {}
        for tap in tapdisk.list():
            if not tap.file:
                log("%s has no file open: detaching and freeing" % str(tap))
                tap.detach()
                tap.free()
            else:
                ty, path = tap.file
                if path.startswith(data_path):
                    filename = os.path.basename(path)
                    if ty == "raw":
                        name = filename[0:-len(raw_suffix)]
                        log("%s open by %s" % (name, str(tap)))
                        self.tapdisks[name] = tap
                    elif ty == "vhd":
                        name = filename[0:-len(vhd_suffix)]
                        log("%s open by %s" % (name, str(tap)))
                        self.tapdisks[name] = tap
                    else:
                        log("Unknown tapdisk type: %s" % ty)

        # Integrity check: each metadata record should point to a data record
        for vdi_info in self.metadata.values():
            if vdi_info["data"] not in self.data:
                log("WARNING: vdi %s has data %s but this does not exist" % (vdi_info["vdi"], vdi_info["data"]))

    def get_deletable(self):
        """Return a set of the data blobs which are not reachable from
        any of the VDIs """
        reachable = set()
        for vdi_info in self.metadata.values():
            x = vdi_info["data"]
            while x not in reachable:
                reachable.add(x)
                info = self.data[x]
                if "parent" in info:
                    x = info["parent"]
        all = set(self.data.keys())
        return all.difference(reachable)

    def make_fresh_metadata_name(self, vdi_info):
        # To make this vaguely human-readable, sanitise the first 32 chars of the
        # VDI's name and then add a numerical suffix to make unique.
        name = vdi_info["name_label"]
        if len(name) > 32:
            # arbitrary cut-off
            name = name[0:32]
        valid = '_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
        name = ''.join(c for c in name if c in valid)
        if name == "":
            name = "_"
        taken = self.metadata.keys()
        if name not in taken:
            return name
        else:
            superstrings = filter(lambda x:x.startswith(name + "."), taken)
            suffixes = map(lambda x:x[len(name) + 1:], superstrings)
            suffix = highest_id(suffixes) + 1L
            return "%s.%Ld" % (name, suffix)

    def make_fresh_data_name(self):
        name = str(self._data_highest_id)
        self._data_highest_id = self._data_highest_id + 1
        return name

    def metadata_path_of_vdi(self, vdi):
        return self.path + "/" + metadata_dir + "/" + vdi + metadata_suffix

    def data_path_of_key(self, key):
        data = self.data[key]
        if data["type"] == "raw":
            return self.path + "/" + data_dir + "/" + key + raw_suffix
        elif data["type"] == "vhd":
            return self.path + "/" + data_dir + "/" + key + vhd_suffix
        else:
            raise (Vdi_does_not_exist(vdi))

    def data_path_of_vdi(self, vdi):
        vdi_info = self.metadata[vdi]
        key = vdi_info["data"]
        return self.data_path_of_key(key)

    def update_vdi_info(self, vdi, vdi_info):
        path = self.metadata_path_of_vdi(vdi)
        f = open(path, "w")
        try:
            try:
                f.write(xmlrpclib.dumps((vdi_info,), allow_none=True))
                self.metadata[vdi] = vdi_info
            except Exception, e:
                log("Exception writing metadata: %s" % (str(e)))
                # remove corrupt file
                os.unlink(path)
        finally:
            f.close()

    def create(self, vdi_info, params):
        vdi = self.make_fresh_metadata_name(vdi_info)
        vdi_info["vdi"] = vdi

        data = self.make_fresh_data_name()
        vdi_info["data"] = data

        stem = self.path + "/" + data_dir + "/" + data
        virtual_size = long(vdi_info["virtual_size"])
        if "type" in params and params["type"] == "raw":
            f = open(stem + raw_suffix, "wc")
            try:
                f.truncate(virtual_size)
                self.data[data] = { "type": "raw" }
            finally:
                f.close()
        else:
            # xmlrpclib will reject longs which are > 32 bits since it will try
            # to marshal them as XMLRPC integers.
            vdi_info["virtual_size"] = str(vhd.create(virtual_size, stem + vhd_suffix))
            self.data[data] = { "type": "vhd" }

        self.update_vdi_info(vdi, vdi_info)
        return vdi_info

    def gc(self):
        deletable = self.get_deletable()
        for x in deletable:
            log("Data %s is unreachable now" % x)
            path = self.data_path_of_key(x)
            unlink_safe(path)
            del self.data[x]

    def destroy(self, vdi):
        meta = self.metadata_path_of_vdi(vdi)
        unlink_safe(meta)
        del self.metadata[vdi]
        self.gc()

    def clone(self, vdi, vdi_info, params):
        parent = self.data_path_of_vdi(vdi)

        # Create two vhd leaves whose parent is [vdi]
        left = self.make_fresh_data_name()
        self.data[left] = vhd.make_leaf(self.path + "/" + data_dir + "/" + left + vhd_suffix, parent)

        right = self.make_fresh_data_name()
        self.data[right] = vhd.make_leaf(self.path + "/" + data_dir + "/" + right + vhd_suffix, parent)

        # Remap the original [vdi]'s location to point to the first leaf's path
        parent_info = self.metadata[vdi]
        parent_info["data"] = left
        self.update_vdi_info(vdi, parent_info)

        # The cloned vdi's location points to the second leaf's path
        clone = self.make_fresh_metadata_name(vdi_info)
        vdi_info["vdi"] = clone
        vdi_info["data"] = right
        vdi_info["virtual_size"] = parent_info["virtual_size"]
        vdi_info["content_id"] = parent_info["content_id"]
        if vdi_info["content_id"] == "":
            vdi_info["content_id"] = util.gen_uuid()
        vdi_info["read_only"] = parent_info["read_only"]
        self.update_vdi_info(clone, vdi_info)

        return vdi_info

    def snapshot(self, vdi, vdi_info, params):
        # Before modifying the vhd-tree, take a note of the
        # currently-active leaf so we can find its tapdisk later
        old_leaf = self.metadata[vdi]["data"]

        # The vhd-tree manipulation is the same as clone...
        vdi_info = self.clone(vdi, vdi_info, params)
        vdi_info["snapshot_of"] = vdi
        # XXX vdi_info["snapshot_time"]
        # ... but we also re-open any active tapdisks
        if old_leaf in self.tapdisks:
            tapdisk = self.tapdisks[old_leaf]
            if "mirror" in params:
                tapdisk.mirror = params["mirror"]
            else:
                tapdisk.mirror = None
            # Find the new vhd leaf
            new_leaf = self.metadata[vdi]["data"]
            tapdisk.file = (self.data[new_leaf]["type"], self.data_path_of_key(new_leaf))
            tapdisk.reopen()
            del self.tapdisks[old_leaf]
            self.tapdisks[new_leaf] = tapdisk
        return vdi_info

    def stat(self, vdi):
        return self.metadata[vdi]

    def set_persistent(self, vdi, persistent):
        meta = self.metadata[vdi]

        if meta["persistent"] and not persistent:
            # We need to make a fresh vhd leaf
            parent = self.data_path_of_vdi(vdi)

            leaf = self.make_fresh_data_name()
            self.data[leaf] = vhd.make_leaf(self.path + "/" + data_dir + "/" + leaf + vhd_suffix, parent)

            # Remap the original [vdi]'s location to point to the new leaf
            meta["data"] = leaf

        meta["persistent"] = persistent
        self.update_vdi_info(vdi, meta)

    def set_content_id(self, vdi, content_id):
        meta = self.metadata[vdi]
        meta["content_id"] = content_id
        self.update_vdi_info(vdi, meta)

    def get_by_name(self, name):
        for vdi in self.metadata.keys():
            md = self.metadata[vdi]
            if vdi == name:
                return md
        raise Vdi_does_not_exist(name)

    def similar_content(self, vdi):
        chains = {}
        if vdi not in self.metadata.keys():
            raise Vdi_does_not_exist(vdi)
        for v in self.metadata.keys():
            md = self.metadata[v]
            data = md["data"]
            chain = set([data])
            while "parent" in self.data[data]:
                data = self.data[data]["parent"]
                chain.add(data)
            chains[v] = chain
        distance = {}
        target_chain = chains[vdi]
        for v in self.metadata.keys():
            this_chain = chains[v]
            difference = target_chain.symmetric_difference(this_chain)
            # if two disks have nothing in common then we don't consider them
            # as similar at all
            if target_chain.intersection(this_chain) <> set([]):
                distance[v] = len(difference)
        import operator
        ordered = sorted(distance.iteritems(), key=operator.itemgetter(1))
        return map(lambda x:self.metadata[x[0]], ordered)

    def maybe_reset(self, vdi):
        meta = self.metadata[vdi]
        if not(meta["persistent"]):
            # we throw away updates to the leaf
            leaf = meta["data"]
            leaf_info = self.data[leaf]
            new_leaf = self.make_fresh_data_name()
            if "parent" in leaf_info:
                # Create a new leaf
                child = self.path + "/" + data_dir + "/" + new_leaf + vhd_suffix
                parent = self.path + "/" + data_dir + "/" + leaf_info["parent"] + vhd_suffix
                self.data[new_leaf] = vhd.make_leaf(child, parent)
                meta["data"] = new_leaf
                self.update_vdi_info(vdi, meta)
                self.gc()
            else:
                # Recreate a whole blank disk
                new_info = self.create(meta, leaf_info)
                meta["data"] = new_info["data"]
                self.update_vdi_info(vdi, meta)
                self.destroy(new_info["vdi"])

    def epoch_begin(self, vdi):
        self.maybe_reset(vdi)

    def epoch_end(self, vdi):
        self.maybe_reset(vdi)

    def open_dummy_vhd(self, vdi_info, tapdisk):
        # blkback will immediately close if it encounters a blktap
        # device which doesn't have a backing file. It probably needs
        # to know the number of sectors? We can satisfy this with a
        # temporary .vhd file
        dummy_data_name = self.make_fresh_data_name()
        dummy_path = self.path + "/" + data_dir + "/" + dummy_data_name + ".dummy" + vhd_suffix
        virtual_size = long(vdi_info["virtual_size"])
        vhd.create(virtual_size, dummy_path)
        tapdisk.open("vhd", dummy_path)
        os.unlink(dummy_path)

    def attach(self, vdi, read_write):
        md = self.metadata[vdi]
        data = md["data"]
        if data in self.tapdisks:
            raise Backend_error("VDI_ALREADY_ATTACHED", [ vdi ])
        self.tapdisks[data] = tapdisk.Tapdisk()
        device = self.tapdisks[data].get_device()
        self.writable[vdi] = read_write

        self.open_dummy_vhd(md, self.tapdisks[data])

        return { "params": device,
                 "xenstore_data": {} }

    def activate(self, vdi):
        md = self.metadata[vdi]
        if self.writable[vdi]:
            md["content_id"] = ""
            self.update_vdi_info(vdi, md)
        data = md["data"]
        if data not in self.tapdisks:
            raise Backend_error("VDI_NOT_ATTACHED", [ vdi ])
        tapdisk = self.tapdisks[data]
        tapdisk.close()
        tapdisk.open(self.data[data]["type"], self.data_path_of_key(data))

    def deactivate(self, vdi):
        md = self.metadata[vdi]
        if md["content_id"] == "":
            md["content_id"] = util.gen_uuid()
            self.update_vdi_info(vdi, md)
        data = md["data"]
        if data not in self.tapdisks:
            raise Backend_error("VDI_NOT_ATTACHED", [ vdi ])
        self.tapdisks[data].close()

{'fraction_non_alphanumeric': 0.10861646674621617, 'fraction_numerical': 0.0023236827231049423, 'mean_word_length': 1.9504354270891235, 'pattern_counts': {'":': 26, '<': 15, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2676060', 'n_tokens_mistral': 10105, 'n_tokens_neox': 9635, 'n_words': 2547}