<commit_before># Copyright (c) 2005-2007 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.

import sys

from conary import files, trove, versions
from conary.dbstore import migration, sqlerrors, sqllib
from conary.lib.tracelog import logMe
from conary.deps import deps
from conary.repository.netrepos import versionops, trovestore
from conary.server import schema

# SCHEMA Migration
class SchemaMigration(migration.SchemaMigration):
    def message(self, msg = None):
        if msg is None:
            msg = self.msg
        if msg == "":
            msg = "Finished migration to schema version %s" % (self.Version,)
        logMe(1, msg)
        self.msg = msg

# dummy migration class that maintains compatbility with schema version13
class MigrateTo_13(SchemaMigration):
    Version = (13,1)
    # fix a migration step that could have potentially failed in the
    # migrations before conary 1.0
    def migrate1(self):
        # flavorId = 0 is now ''
        cu = self.db.cursor()
        cu.execute("UPDATE Flavors SET flavor = '' WHERE flavorId = 0")
        return True
    def migrate(self):
        return self.Version

# schema versions lower than 13 are not supported for migration through this pathway
class MigrateTo_14(SchemaMigration):
    Version = 14
    def migrate(self):
        self.message('WARNING: do NOT interrupt this migration, you will leave your DB in a messy state')

        updateCursor = self.db.cursor()
        self.cu.execute("""
        CREATE TEMPORARY TABLE tmpSha1s(
        streamId INTEGER,
        sha1  BINARY(20))""",
                        start_transaction=False)
        self.cu.execute('CREATE INDEX tmpSha1sIdx ON tmpSha1s(streamId)')
        total = self.cu.execute('SELECT max(streamId) FROM FileStreams').fetchall()[0][0]
        pct = 0
        for idx, (streamId, fileId, stream) in \
                enumerate(updateCursor.execute("SELECT streamId, fileId, stream FROM "
                                "FileStreams ORDER BY StreamId")):
            if stream and files.frozenFileHasContents(stream):
                contents = files.frozenFileContentInfo(stream)
                sha1 = contents.sha1()
                self.cu.execute("INSERT INTO tmpSha1s (streamId, sha1) VALUES (?,?)",
                                (streamId, self.cu.binary(sha1)))
            newPct = (streamId * 100)/total
            if newPct >= pct:
                logMe(3, 'Calculating sha1 for fileStream %s/%s (%02d%%)...' % (streamId, total, pct))
                pct = newPct + 5

        logMe(2, 'Populating FileStream Table with sha1s...')

        # delay this as long as possible, any CTRL-C after this point
        # will make future migrations fail.
        self.cu.execute("ALTER TABLE FileStreams ADD COLUMN "
                        "sha1        %(BINARY20)s"
                        % self.db.keywords)

        self.cu.execute("""
        UPDATE FileStreams
        SET sha1 = (
            SELECT sha1 FROM tmpSha1s
            WHERE FileStreams.streamid = tmpSha1s.streamid )
        """)
        self.cu.execute('DROP TABLE tmpSha1s')

        # because of the foreign key referential mess, we need to
        # destroy the FKs relationships, recreate the Entitlement
        # tables, and restore the data
        logMe(2, 'Updating Entitlements...')
        self.cu.execute("CREATE TABLE Entitlements2 AS SELECT * FROM "
                        "Entitlements")
        self.cu.execute("CREATE TABLE EntitlementGroups2 AS SELECT * FROM "
                        "EntitlementGroups")
        self.cu.execute("CREATE TABLE EntitlementOwners2 AS SELECT * FROM "
                        "EntitlementOwners")
        self.cu.execute("DROP TABLE Entitlements")
        self.cu.execute("DROP TABLE EntitlementOwners")
        self.cu.execute("DROP TABLE EntitlementGroups")

        self.db.loadSchema()
        schema.createEntitlements(self.db)

        self.cu.execute("INSERT INTO EntitlementGroups (entGroup, entGroupId) "
                        "SELECT entGroup, entGroupId FROM EntitlementGroups2")
        self.cu.execute("INSERT INTO EntitlementAccessMap (entGroupId, "
                        "userGroupId) SELECT entGroupId, userGroupId FROM "
                        "EntitlementGroups2")
        self.cu.execute("INSERT INTO Entitlements SELECT * FROM Entitlements2")
        self.cu.execute("INSERT INTO EntitlementOwners SELECT * FROM EntitlementOwners2")
        self.cu.execute("DROP TABLE Entitlements2")
        self.cu.execute("DROP TABLE EntitlementGroups2")
        self.cu.execute("DROP TABLE EntitlementOwners2")

        logMe(2, "Updating the Permissions table...")
        self.cu.execute("ALTER TABLE Permissions ADD COLUMN "
                        "canRemove   INTEGER NOT NULL DEFAULT 0"
                        % self.db.keywords)

        logMe(2, "Updating Instances table...")
        self.db.renameColumn("Instances", "isRedirect", "troveType")
        self.db.loadSchema()
        return self.Version

class MigrateTo_15(SchemaMigration):
    Version = (15, 5)
    def updateLatest(self, cu):
        logMe(2, "Updating the Latest table...")
        cu.execute("DROP TABLE Latest")
        self.db.loadSchema()
        schema.createLatest(self.db)

        cu.execute("""
            insert into Latest (itemId, branchId, flavorId, versionId,
                                latestType)
                select
                    instances.itemid as itemid,
                    nodes.branchid as branchid,
                    instances.flavorid as flavorid,
                    nodes.versionid as versionid,
                    %d
                from
                    ( select
                        i.itemid as itemid,
                        n.branchid as branchid,
                        i.flavorid as flavorid,
                        max(n.finalTimestamp) as finaltimestamp
                      from
                        instances as i, nodes as n
                      where
                            i.itemid = n.itemid
                        and i.versionid = n.versionid
                      group by i.itemid, n.branchid, i.flavorid
                    ) as tmp
                    join nodes on
                      tmp.itemid = nodes.itemid and
                      tmp.branchid = nodes.branchid and
                      tmp.finaltimestamp = nodes.finaltimestamp
                    join instances on
                      nodes.itemid = instances.itemid and
                      nodes.versionid = instances.versionid and
                      instances.flavorid = tmp.flavorid
        """ % versionops.LATEST_TYPE_ANY)

        self.cu.execute("""
            insert into Latest (itemId, branchId, flavorId, versionId,
                                latestType)
                select
                    instances.itemid as itemid,
                    nodes.branchid as branchid,
                    instances.flavorid as flavorid,
                    nodes.versionid as versionid,
                    %d
                from
                    ( select
                        i.itemid as itemid,
                        n.branchid as branchid,
                        i.flavorid as flavorid,
                        max(n.finalTimestamp) as finaltimestamp
                      from
                        instances as i, nodes as n
                      where
                            i.itemid = n.itemid
                        and i.versionid = n.versionid
                        and i.troveType != %d
                      group by i.itemid, n.branchid, i.flavorid
                    ) as tmp
                    join nodes on
                      tmp.itemid = nodes.itemid and
                      tmp.branchid = nodes.branchid and
                      tmp.finaltimestamp = nodes.finaltimestamp
                    join instances on
                      nodes.itemid = instances.itemid and
                      nodes.versionid = instances.versionid and
                      instances.flavorid = tmp.flavorid
        """ % (versionops.LATEST_TYPE_PRESENT, trove.TROVE_TYPE_REMOVED))

        self.cu.execute("""
            insert into Latest (itemId, branchId, flavorId, versionId,
                                latestType)
                select
                    instances.itemid as itemid,
                    latest.branchid as branchid,
                    instances.flavorid as flavorid,
                    instances.versionid as versionid,
                    %d
                from Latest join Instances
                    using (itemId, versionId, flavorId)
                where
                    latest.latestType = %d AND
                    instances.troveType = %d
        """ % (versionops.LATEST_TYPE_NORMAL,
               versionops.LATEST_TYPE_PRESENT, trove.TROVE_TYPE_NORMAL))

    # update a trove signatures, if required
    def fixTroveSig(self, repos, instanceId):
        cu = self.db.cursor()
        cu.execute("""
        select Items.item as name, Versions.version, Flavors.flavor
        from Instances
        join Items using (itemId)
        join Versions on
            Instances.versionId = Versions.versionId
        join Flavors on
            Instances.flavorId = Flavors.flavorId
        where Instances.instanceId = ?""", instanceId)
        (name, version, flavor) = cu.fetchall()[0]
        # check the signature
        trv = repos.getTrove(name, versions.VersionFromString(version),
                             deps.ThawFlavor(flavor))
        if trv.verifyDigests():
            return
        logMe(3, "updating trove sigs: %s %s %s" % (name, version, flavor))
        trv.computeDigests()
        cu.execute("delete from TroveInfo where instanceId = ? "
                   "and infoType = ?", (instanceId, trove._TROVEINFO_TAG_SIGS))
        cu.execute("insert into TroveInfo (instanceId, infoType, data) "
                   "values (?, ?, ?)", (
            instanceId, trove._TROVEINFO_TAG_SIGS,
            cu.binary(trv.troveInfo.sigs.freeze())))

    def fixRedirects(self, repos):
        logMe(2, "removing dep provisions from redirects...")
        self.db.loadSchema()
        # avoid creating this index until we had a chance to check the path indexes
        self.db.tables["TroveFiles"].append("TroveFilesPathIdx")
        # remove dependency provisions from redirects -- the conary 1.0
        # branch would set redirects to provide their own names. this doesn't
        # clean up the dependency table; that would only matter on a trove
        # which was cooked as only a redirect in the repository; any other
        # instances would still need the depId anyway
        cu = self.db.cursor()
        cu.execute("delete from provides where instanceId in "
                   "(select instanceId from instances "
                   "where troveType=? and isPresent=1)",
                   trove.TROVE_TYPE_REDIRECT)
        # need to make sure TroveRedirects is defined...
        schema.createTroves(self.db)
        # loop over redirects...
        cu.execute("select instanceId from instances "
                   "where troveType=? and isPresent=1",
                   trove.TROVE_TYPE_REDIRECT)
        for (instanceId,) in cu:
            self.fixTroveSig(repos, instanceId)

    # fix the duplicate path problems, if any
    def fixDuplicatePaths(self, repos):
        logMe(2, "checking database for duplicate path entries...")
        cu = self.db.cursor()
        # we'll have to do a full table scan on TroveFiles. no way
        # around it...
        cu.execute("""
        create temporary table tmpDups(
            instanceId integer,
            path %(PATHTYPE)s,
            counter integer
        ) %(TABLEOPTS)s""" % self.db.keywords)
        logMe(2, "looking for troves with duplicate paths...")
        # sqlite has a real challenge dealing with large datasets
        if self.db.driver == 'sqlite':
            cu2 = self.db.cursor()
            cu.execute("select distinct instanceId from TroveFiles")
            # so we split this in very little tasks. lots of them
            for (instanceId,) in cu:
                cu2.execute("""
                insert into tmpDups (instanceId, path, counter)
                select instanceId, path, count(*)
                from TroveFiles
                where instanceId = ?
                group by instanceId, path
                having count(*) > 1""", instanceId)
        else: # other backends should be able to process this in one shot
            cu.execute("""
            insert into tmpDups (instanceId, path, counter)
            select instanceId, path, count(*)
            from TroveFiles
            group by instanceId, path
            having count(*) > 1""")
        counter = cu.execute("select count(*) from tmpDups").fetchall()[0][0]
        if counter > 0:
            # drop the old index, if any
            self.db.loadSchema()
            self.db.dropIndex("TroveFiles", "TroveFilesPathIdx")
        logMe(3, "detected %d duplicates" % (counter,))
        # loop over every duplicate and apply the appropiate fix
        cu.execute("select instanceId, path from tmpDups")
        for (instanceId, path) in cu.fetchall():
            cu.execute("""select distinct
            instanceId, streamId, versionId, pathId, path
            from trovefiles where instanceId = ? and path = ?
            order by streamId, versionId, pathId""", (instanceId, path))
            ret = cu.fetchall()
            # delete all the duplicates and put the first one back
            cu.execute("delete from trovefiles "
                       "where instanceId = ? and path = ?",
                       (instanceId, path))
            # in case they are different, we pick the oldest, chances are it is
            # more "original"
            cu.execute("insert into trovefiles "
                       "(instanceId, streamId, versionId, pathId, path) "
                       "values (?,?,?,?,?)", tuple(ret[0]))
            if len(ret) > 1:
                # need to recompute the sha1 - we might have changed the trove manifest
                # if the records were different
                self.fixTroveSig(repos, instanceId)
        # recreate the indexes and triggers - including new path
        # index for TroveFiles.  Also recreates the indexes table.
        logMe(2, 'Recreating indexes... (this could take a while)')
        cu.execute("drop table tmpDups")
        self.db.loadSchema()
        schema.createTroves(self.db)
        logMe(2, 'Indexes created.')

    # drop the unused views
    def dropViews(self):
        logMe(2, "dropping unused views")
        self.db.loadSchema()
        cu = self.db.cursor()
        for v in ["UsersView", "NodesView", "InstancesView", "LatestView"]:
            if v in self.db.views:
                cu.execute("drop view %s" % (v,))

    # add the CapId reference to Permissions and Latest
    def fixPermissions(self):
        # because we need to add a foreign key to the Permissions and Latest
        # tables, it is easier if we save the tables and recreate them
        logMe(2, "Updating the Permissions table...")
        cu = self.db.cursor()
        cu.execute("""create table tmpPerm as
        select permissionId, userGroupId, labelId, itemId, admin, canWrite, canRemove
        from Permissions""")
        cu.execute("drop table Permissions")
        self.db.loadSchema()
        schema.createUsers(self.db)
        cu.execute("""insert into Permissions
        (permissionId, userGroupId, labelId, itemId, admin, canWrite, canRemove)
        select permissionId, userGroupId, labelId, itemId, admin, canWrite, canRemove
        from tmpPerm
        """)
        cu.execute("drop table tmpPerm")

    # migrate to 15.0
    def migrate(self):
        schema.setupTempTables(self.db)
        cu = self.db.cursor()
        # needed for signature recalculation
        repos = trovestore.TroveStore(self.db)
        self.dropViews()
        self.fixRedirects(repos)
        self.fixDuplicatePaths(repos)
        self.fixPermissions()
        self.updateLatest(cu)
        return True
    # migrate to 15.1
    def migrate1(self):
        # drop the pinned index on Instances and recreate it as not-pinned
        self.db.dropTrigger("Instances", "UPDATE")
        ret = schema.createTrigger(self.db, "Instances")
        self.db.loadSchema()
        return ret
    # this migration has been moved to a major schema migration
    def migrate2(self):
        return True
    # migrate to 15.3
    def migrate3(self):
        # we create a whole bunch of indexes for foreign keys to smooth out the differences
        # between MySQL and PostgreSQL
        self.db.loadSchema()
        # indexes on Nodes have to be cleaned up
        self.db.dropIndex("Nodes", "NodesItemBranchVersionIdx")
        self.db.createIndex("Nodes", "NodesItemVersionBranchIdx",
                            "itemId, versionId, branchId", unique = True)
        self.db.dropIndex("Nodes", "NodesItemVersionIdx")
        self.db.dropIndex("Latest", "LatestIdx")
        self.db.loadSchema()
        schema.createSchema(self.db)
        return True
    # conary 1.1.22 went out with a busted definition of LabelMap - we need to fix it
    def migrate4(self):
        return createLabelMap(self.db)
    # 15.5
    def migrate5(self):
        # drop the index in case a user created it by hand (CNY-1704)
        self.db.dropIndex('LabelMap', 'LabelMapItemIdBranchIdIdx')
        return self.db.createIndex('LabelMap', 'LabelMapItemIdBranchIdIdx', 'itemId, branchId')

# looks like this LabelMap has to be recreated multiple times by
# different stages of migraton :-(
def createLabelMap(db):
    cu = db.cursor()
    cu.execute("create table OldLabelMap as select * from LabelMap")
    cu.execute("drop table LabelMap")
    db.loadSchema()
    schema.createLabelMap(db)
    cu.execute("insert into LabelMap (itemId, labelId, branchId) "
               "select itemId, labelId, branchId from OldLabelMap ")
    cu.execute("drop table OldLabelMap")
    db.loadSchema()
    return True
    
class MigrateTo_16(SchemaMigration):
    Version = (16.0)
    # create a primary key for labelmap
    def migrate(self):
        return createLabelMap(self.db)

def _getMigration(major):
    try:
        ret = sys.modules[__name__].__dict__['MigrateTo_' + str(major)]
    except KeyError:
        return None
    return ret

# return the last major.minor version for a given major<commit_msg>clean up wrapping<commit_after># Copyright (c) 2005-2007 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.

import sys

from conary import files, trove, versions
from conary.dbstore import migration, sqlerrors, sqllib
from conary.lib.tracelog import logMe
from conary.deps import deps
from conary.repository.netrepos import versionops, trovestore
from conary.server import schema

# SCHEMA Migration
class SchemaMigration(migration.SchemaMigration):
    def message(self, msg = None):
        if msg is None:
            msg = self.msg
        if msg == "":
            msg = "Finished migration to schema version %s" % (self.Version,)
        logMe(1, msg)
        self.msg = msg

# dummy migration class that maintains compatbility with schema version13
class MigrateTo_13(SchemaMigration):
    Version = (13,1)
    # fix a migration step that could have potentially failed in the
    # migrations before conary 1.0
    def migrate1(self):
        # flavorId = 0 is now ''
        cu = self.db.cursor()
        cu.execute("UPDATE Flavors SET flavor = '' WHERE flavorId = 0")
        return True
    def migrate(self):
        return self.Version

# schema versions lower than 13 are not supported for migration through this pathway
class MigrateTo_14(SchemaMigration):
    Version = 14
    def migrate(self):
        self.message('WARNING: do NOT interrupt this migration, you will leave your DB in a messy state')

        updateCursor = self.db.cursor()
        self.cu.execute("""
        CREATE TEMPORARY TABLE tmpSha1s(
        streamId INTEGER,
        sha1  BINARY(20))""",
                        start_transaction=False)
        self.cu.execute('CREATE INDEX tmpSha1sIdx ON tmpSha1s(streamId)')
        total = self.cu.execute('SELECT max(streamId) FROM FileStreams').fetchall()[0][0]
        pct = 0
        for idx, (streamId, fileId, stream) in \
                enumerate(updateCursor.execute("SELECT streamId, fileId, stream FROM "
                                "FileStreams ORDER BY StreamId")):
            if stream and files.frozenFileHasContents(stream):
                contents = files.frozenFileContentInfo(stream)
                sha1 = contents.sha1()
                self.cu.execute("INSERT INTO tmpSha1s (streamId, sha1) VALUES (?,?)",
                                (streamId, self.cu.binary(sha1)))
            newPct = (streamId * 100)/total
            if newPct >= pct:
                logMe(3, 'Calculating sha1 for fileStream %s/%s (%02d%%)...' % (streamId, total, pct))
                pct = newPct + 5

        logMe(2, 'Populating FileStream Table with sha1s...')

        # delay this as long as possible, any CTRL-C after this point
        # will make future migrations fail.
        self.cu.execute("ALTER TABLE FileStreams ADD COLUMN "
                        "sha1        %(BINARY20)s"
                        % self.db.keywords)

        self.cu.execute("""
        UPDATE FileStreams
        SET sha1 = (
            SELECT sha1 FROM tmpSha1s
            WHERE FileStreams.streamid = tmpSha1s.streamid )
        """)
        self.cu.execute('DROP TABLE tmpSha1s')

        # because of the foreign key referential mess, we need to
        # destroy the FKs relationships, recreate the Entitlement
        # tables, and restore the data
        logMe(2, 'Updating Entitlements...')
        self.cu.execute("CREATE TABLE Entitlements2 AS SELECT * FROM "
                        "Entitlements")
        self.cu.execute("CREATE TABLE EntitlementGroups2 AS SELECT * FROM "
                        "EntitlementGroups")
        self.cu.execute("CREATE TABLE EntitlementOwners2 AS SELECT * FROM "
                        "EntitlementOwners")
        self.cu.execute("DROP TABLE Entitlements")
        self.cu.execute("DROP TABLE EntitlementOwners")
        self.cu.execute("DROP TABLE EntitlementGroups")

        self.db.loadSchema()
        schema.createEntitlements(self.db)

        self.cu.execute("INSERT INTO EntitlementGroups (entGroup, entGroupId) "
                        "SELECT entGroup, entGroupId FROM EntitlementGroups2")
        self.cu.execute("INSERT INTO EntitlementAccessMap (entGroupId, "
                        "userGroupId) SELECT entGroupId, userGroupId FROM "
                        "EntitlementGroups2")
        self.cu.execute("INSERT INTO Entitlements SELECT * FROM Entitlements2")
        self.cu.execute("INSERT INTO EntitlementOwners SELECT * FROM EntitlementOwners2")
        self.cu.execute("DROP TABLE Entitlements2")
        self.cu.execute("DROP TABLE EntitlementGroups2")
        self.cu.execute("DROP TABLE EntitlementOwners2")

        logMe(2, "Updating the Permissions table...")
        self.cu.execute("ALTER TABLE Permissions ADD COLUMN "
                        "canRemove   INTEGER NOT NULL DEFAULT 0"
                        % self.db.keywords)

        logMe(2, "Updating Instances table...")
        self.db.renameColumn("Instances", "isRedirect", "troveType")
        self.db.loadSchema()
        return self.Version

class MigrateTo_15(SchemaMigration):
    Version = (15, 5)
    def updateLatest(self, cu):
        logMe(2, "Updating the Latest table...")
        cu.execute("DROP TABLE Latest")
        self.db.loadSchema()
        schema.createLatest(self.db)

        cu.execute("""
            insert into Latest (itemId, branchId, flavorId, versionId,
                                latestType)
                select
                    instances.itemid as itemid,
                    nodes.branchid as branchid,
                    instances.flavorid as flavorid,
                    nodes.versionid as versionid,
                    %d
                from
                    ( select
                        i.itemid as itemid,
                        n.branchid as branchid,
                        i.flavorid as flavorid,
                        max(n.finalTimestamp) as finaltimestamp
                      from
                        instances as i, nodes as n
                      where
                            i.itemid = n.itemid
                        and i.versionid = n.versionid
                      group by i.itemid, n.branchid, i.flavorid
                    ) as tmp
                    join nodes on
                      tmp.itemid = nodes.itemid and
                      tmp.branchid = nodes.branchid and
                      tmp.finaltimestamp = nodes.finaltimestamp
                    join instances on
                      nodes.itemid = instances.itemid and
                      nodes.versionid = instances.versionid and
                      instances.flavorid = tmp.flavorid
        """ % versionops.LATEST_TYPE_ANY)

        self.cu.execute("""
            insert into Latest (itemId, branchId, flavorId, versionId,
                                latestType)
                select
                    instances.itemid as itemid,
                    nodes.branchid as branchid,
                    instances.flavorid as flavorid,
                    nodes.versionid as versionid,
                    %d
                from
                    ( select
                        i.itemid as itemid,
                        n.branchid as branchid,
                        i.flavorid as flavorid,
                        max(n.finalTimestamp) as finaltimestamp
                      from
                        instances as i, nodes as n
                      where
                            i.itemid = n.itemid
                        and i.versionid = n.versionid
                        and i.troveType != %d
                      group by i.itemid, n.branchid, i.flavorid
                    ) as tmp
                    join nodes on
                      tmp.itemid = nodes.itemid and
                      tmp.branchid = nodes.branchid and
                      tmp.finaltimestamp = nodes.finaltimestamp
                    join instances on
                      nodes.itemid = instances.itemid and
                      nodes.versionid = instances.versionid and
                      instances.flavorid = tmp.flavorid
        """ % (versionops.LATEST_TYPE_PRESENT, trove.TROVE_TYPE_REMOVED))

        self.cu.execute("""
            insert into Latest (itemId, branchId, flavorId, versionId,
                                latestType)
                select
                    instances.itemid as itemid,
                    latest.branchid as branchid,
                    instances.flavorid as flavorid,
                    instances.versionid as versionid,
                    %d
                from Latest join Instances
                    using (itemId, versionId, flavorId)
                where
                    latest.latestType = %d AND
                    instances.troveType = %d
        """ % (versionops.LATEST_TYPE_NORMAL,
               versionops.LATEST_TYPE_PRESENT, trove.TROVE_TYPE_NORMAL))

    # update a trove signatures, if required
    def fixTroveSig(self, repos, instanceId):
        cu = self.db.cursor()
        cu.execute("""
        select Items.item as name, Versions.version, Flavors.flavor
        from Instances
        join Items using (itemId)
        join Versions on
            Instances.versionId = Versions.versionId
        join Flavors on
            Instances.flavorId = Flavors.flavorId
        where Instances.instanceId = ?""", instanceId)
        (name, version, flavor) = cu.fetchall()[0]
        # check the signature
        trv = repos.getTrove(name, versions.VersionFromString(version),
                             deps.ThawFlavor(flavor))
        if trv.verifyDigests():
            return
        logMe(3, "updating trove sigs: %s %s %s" % (name, version, flavor))
        trv.computeDigests()
        cu.execute("delete from TroveInfo where instanceId = ? "
                   "and infoType = ?", (instanceId, trove._TROVEINFO_TAG_SIGS))
        cu.execute("insert into TroveInfo (instanceId, infoType, data) "
                   "values (?, ?, ?)", (
            instanceId, trove._TROVEINFO_TAG_SIGS,
            cu.binary(trv.troveInfo.sigs.freeze())))

    def fixRedirects(self, repos):
        logMe(2, "removing dep provisions from redirects...")
        self.db.loadSchema()
        # avoid creating this index until we had a chance to check the path indexes
        self.db.tables["TroveFiles"].append("TroveFilesPathIdx")
        # remove dependency provisions from redirects -- the conary 1.0
        # branch would set redirects to provide their own names. this doesn't
        # clean up the dependency table; that would only matter on a trove
        # which was cooked as only a redirect in the repository; any other
        # instances would still need the depId anyway
        cu = self.db.cursor()
        cu.execute("delete from provides where instanceId in "
                   "(select instanceId from instances "
                   "where troveType=? and isPresent=1)",
                   trove.TROVE_TYPE_REDIRECT)
        # need to make sure TroveRedirects is defined...
        schema.createTroves(self.db)
        # loop over redirects...
        cu.execute("select instanceId from instances "
                   "where troveType=? and isPresent=1",
                   trove.TROVE_TYPE_REDIRECT)
        for (instanceId,) in cu:
            self.fixTroveSig(repos, instanceId)

    # fix the duplicate path problems, if any
    def fixDuplicatePaths(self, repos):
        logMe(2, "checking database for duplicate path entries...")
        cu = self.db.cursor()
        # we'll have to do a full table scan on TroveFiles. no way
        # around it...
        cu.execute("""
        create temporary table tmpDups(
            instanceId integer,
            path %(PATHTYPE)s,
            counter integer
        ) %(TABLEOPTS)s""" % self.db.keywords)
        logMe(2, "looking for troves with duplicate paths...")
        # sqlite has a real challenge dealing with large datasets
        if self.db.driver == 'sqlite':
            cu2 = self.db.cursor()
            cu.execute("select distinct instanceId from TroveFiles")
            # so we split this in very little tasks. lots of them
            for (instanceId,) in cu:
                cu2.execute("""
                insert into tmpDups (instanceId, path, counter)
                select instanceId, path, count(*)
                from TroveFiles
                where instanceId = ?
                group by instanceId, path
                having count(*) > 1""", instanceId)
        else: # other backends should be able to process this in one shot
            cu.execute("""
            insert into tmpDups (instanceId, path, counter)
            select instanceId, path, count(*)
            from TroveFiles
            group by instanceId, path
            having count(*) > 1""")
        counter = cu.execute("select count(*) from tmpDups").fetchall()[0][0]
        if counter > 0:
            # drop the old index, if any
            self.db.loadSchema()
            self.db.dropIndex("TroveFiles", "TroveFilesPathIdx")
        logMe(3, "detected %d duplicates" % (counter,))
        # loop over every duplicate and apply the appropiate fix
        cu.execute("select instanceId, path from tmpDups")
        for (instanceId, path) in cu.fetchall():
            cu.execute("""select distinct
            instanceId, streamId, versionId, pathId, path
            from trovefiles where instanceId = ? and path = ?
            order by streamId, versionId, pathId""", (instanceId, path))
            ret = cu.fetchall()
            # delete all the duplicates and put the first one back
            cu.execute("delete from trovefiles "
                       "where instanceId = ? and path = ?",
                       (instanceId, path))
            # in case they are different, we pick the oldest, chances are it is
            # more "original"
            cu.execute("insert into trovefiles "
                       "(instanceId, streamId, versionId, pathId, path) "
                       "values (?,?,?,?,?)", tuple(ret[0]))
            if len(ret) > 1:
                # need to recompute the sha1 - we might have changed the trove manifest
                # if the records were different
                self.fixTroveSig(repos, instanceId)
        # recreate the indexes and triggers - including new path
        # index for TroveFiles.  Also recreates the indexes table.
        logMe(2, 'Recreating indexes... (this could take a while)')
        cu.execute("drop table tmpDups")
        self.db.loadSchema()
        schema.createTroves(self.db)
        logMe(2, 'Indexes created.')

    # drop the unused views
    def dropViews(self):
        logMe(2, "dropping unused views")
        self.db.loadSchema()
        cu = self.db.cursor()
        for v in ["UsersView", "NodesView", "InstancesView", "LatestView"]:
            if v in self.db.views:
                cu.execute("drop view %s" % (v,))

    # add the CapId reference to Permissions and Latest
    def fixPermissions(self):
        # because we need to add a foreign key to the Permissions and Latest
        # tables, it is easier if we save the tables and recreate them
        logMe(2, "Updating the Permissions table...")
        cu = self.db.cursor()
        cu.execute("""create table tmpPerm as
        select permissionId, userGroupId, labelId, itemId, admin, canWrite, canRemove
        from Permissions""")
        cu.execute("drop table Permissions")
        self.db.loadSchema()
        schema.createUsers(self.db)
        cu.execute("""insert into Permissions
        (permissionId, userGroupId, labelId, itemId, admin, canWrite, canRemove)
        select permissionId, userGroupId, labelId, itemId, admin, canWrite, canRemove
        from tmpPerm
        """)
        cu.execute("drop table tmpPerm")

    # migrate to 15.0
    def migrate(self):
        schema.setupTempTables(self.db)
        cu = self.db.cursor()
        # needed for signature recalculation
        repos = trovestore.TroveStore(self.db)
        self.dropViews()
        self.fixRedirects(repos)
        self.fixDuplicatePaths(repos)
        self.fixPermissions()
        self.updateLatest(cu)
        return True
    # migrate to 15.1
    def migrate1(self):
        # drop the pinned index on Instances and recreate it as not-pinned
        self.db.dropTrigger("Instances", "UPDATE")
        ret = schema.createTrigger(self.db, "Instances")
        self.db.loadSchema()
        return ret
    # this migration has been moved to a major schema migration
    def migrate2(self):
        return True
    # migrate to 15.3
    def migrate3(self):
        # we create a whole bunch of indexes for foreign keys to smooth out the differences
        # between MySQL and PostgreSQL
        self.db.loadSchema()
        # indexes on Nodes have to be cleaned up
        self.db.dropIndex("Nodes", "NodesItemBranchVersionIdx")
        self.db.createIndex("Nodes", "NodesItemVersionBranchIdx",
                            "itemId, versionId, branchId", unique = True)
        self.db.dropIndex("Nodes", "NodesItemVersionIdx")
        self.db.dropIndex("Latest", "LatestIdx")
        self.db.loadSchema()
        schema.createSchema(self.db)
        return True
    # conary 1.1.22 went out with a busted definition of LabelMap - we need to fix it
    def migrate4(self):
        return createLabelMap(self.db)
    # 15.5
    def migrate5(self):
        # drop the index in case a user created it by hand (CNY-1704)
        self.db.dropIndex('LabelMap', 'LabelMapItemIdBranchIdIdx')
        return self.db.createIndex('LabelMap', 'LabelMapItemIdBranchIdIdx',
                                   'itemId, branchId')

# looks like this LabelMap has to be recreated multiple times by
# different stages of migraton :-(
def createLabelMap(db):
    cu = db.cursor()
    cu.execute("create table OldLabelMap as select * from LabelMap")
    cu.execute("drop table LabelMap")
    db.loadSchema()
    schema.createLabelMap(db)
    cu.execute("insert into LabelMap (itemId, labelId, branchId) "
               "select itemId, labelId, branchId from OldLabelMap ")
    cu.execute("drop table OldLabelMap")
    db.loadSchema()
    return True
    
class MigrateTo_16(SchemaMigration):
    Version = (16.0)
    # create a primary key for labelmap
    def migrate(self):
        return createLabelMap(self.db)

def _getMigration(major):
    try:
        ret = sys.modules[__name__].__dict__['MigrateTo_' + str(major)]
    except KeyError:
        return None
    return ret

# return the last major.minor version for a given major
{'fraction_non_alphanumeric': 0.08456470958767487, 'fraction_numerical': 0.007191412298889793, 'mean_word_length': 1.8442818751866228, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1039570', 'n_tokens_mistral': 10508, 'n_tokens_neox': 9977, 'n_words': 3089}