<commit_before>#
# Copyright (c) 2005-2009 rPath, Inc.
#
# All Rights Reserved
#
from mint import config
from mint import helperfuncs
from mint.db import pkgindex
from mint.db import projects
from mint.lib import database
from mint.lib import scriptlibrary

from conary import conaryclient
from conary import conarycfg
from conary import dbstore
from conary import versions
from conary.repository import repository

import os
import time

hiddenLabels = [
    versions.Label('conary.rpath.com@rpl:rpl1'),
    versions.Label('con<EMAIL>@ravenous:bugblatterbeast')
]

class PackageIndexer(scriptlibrary.SingletonScript):
    db = None
    cfgPath = config.RBUILDER_CONFIG

    def __init__(self, aLockPath = scriptlibrary.DEFAULT_LOCKPATH,
            aMintServer=None):
        self.cfg = config.MintConfig()
        self.cfg.read(self.cfgPath)
        if self.logFileName:
            self.logPath = os.path.join(self.cfg.logPath, self.logFileName)

        scriptlibrary.SingletonScript.__init__(self, aLockPath)

    def cleanup(self):
        if self.db:
            self.db.close()


class UpdatePackageIndex(PackageIndexer):
    logFileName = 'package-index.log'

    def action(self):
        self.log.info("Updating package index")

        self.db = dbstore.connect(self.cfg.dbPath, driver = self.cfg.dbDriver)
        self.db.loadSchema()
        cu = self.db.cursor()

        try:
            cu.execute('SELECT COUNT(*) FROM PackageIndexMark')
            if not cu.fetchone()[0]:
                cu.execute('INSERT INTO PackageIndexMark VALUES(0)')
            cu.execute("SELECT COALESCE(MAX(timestamp), 0) FROM Commits")
            newMark = cu.fetchone()[0]

            # Clear out Package index if the timestamp in PackageIndexMark == 0
            cu.execute("""DELETE FROM PackageIndex WHERE
                              (SELECT mark FROM PackageIndexMark) = 0""")

            cu.execute("""SELECT Projects.projectId, troveName, Commits.version,
                                 timestamp
                              FROM Commits
                              LEFT JOIN Projects
                                  ON Commits.projectId=Projects.projectId
                              WHERE (troveName NOT LIKE '%:%' OR troveName LIKE '%:source')
                              AND hidden=0 AND disabled=0
                              AND timestamp >=
                                  (SELECT mark FROM PackageIndexMark)""")

            res = cu.fetchall()
            numPkgs = len(res) - 1
            if numPkgs > 0:
                self.log.info("Indexing %d packages" % numPkgs)
            else:
                self.log.info("Package index is up to date")
            packageDict = {}
            labelMap = {}
            for projectId, troveName, verStr, timeStamp in res:
                troveEntry = packageDict.setdefault(troveName, {})
                version = versions.VersionFromString(verStr)
                label = str(version.branch().label())
                versionList = troveEntry.setdefault(label, [])

                version = version.copy()
                version.resetTimeStamps(timeStamp)
                versionList.append(version)
                labelMap[label] = projectId

            packageIndex = []
            for troveName in packageDict:
                for label in packageDict[troveName]:
                    packageIndex.append((labelMap[label], troveName,
                                         max(packageDict[troveName][label])))

            inserts = []
            updates = []
            for projectId, troveName, version in packageIndex:
                cu.execute("""SELECT pkgId, version FROM PackageIndex
                                  WHERE projectId=? AND name=?""",
                           projectId, troveName)

                res = [x[0] for x in cu.fetchall() if \
                       versions.VersionFromString(x[1]).branch().label() == \
                       version.branch().label()]

                label = version.branch().label()
                serverName = label.getHost()
                branchName = label.getNamespace() + ":" + label.getLabel()

                isSource = int(troveName.endswith(':source'))
                if not res:
                    inserts.append((projectId, troveName, str(version), serverName,
                        branchName, isSource))
                else:
                    pkgId = res[0]
                    updates.append((projectId, troveName, str(version), serverName,
                        branchName, isSource, pkgId))

            if inserts or updates:
                self.db.transaction()
                if inserts:
                    cu.executemany("""INSERT INTO PackageIndex
                                      (projectId, name, version,
                                       serverName, branchName, isSource)
                                      VALUES (?, ?, ?, ?, ?, ?)""", inserts)
                if updates:
                    cu.executemany("""UPDATE PackageIndex SET
                                          projectId=?, name=?, version=?,
                                          serverName=?, branchName=?, isSource=?
                                          WHERE pkgId=?""", updates)
                self.db.commit()

            cu.execute("UPDATE PackageIndexMark SET mark=?", int(newMark))
        except Exception, e:
            self.log.error("Error occurred: %s" % str(e))
            self.db.rollback()
            exitcode = 1
            raise
        else:
            self.log.info("Completed successfully: %d" % len(inserts))
            exitcode = 0
            self.db.commit()
        return exitcode


class UpdatePackageIndexExternal(PackageIndexer):
    logFileName = 'package-index-external.log'

    def updateMark(self):
        # This code exists to overcome the situation where there are no
        # internal projects on the rBuilder. internal package index code will
        # delete the package index if there is no mark or a mark of zero.
        # this code sets the mark to "1" to ensure no race conditions exist
        # sorrounding the setting of the mark.
        cu = self.db.transaction()
        cu.execute("SELECT COUNT(*) FROM Projects WHERE external = 0")
        if not cu.fetchone()[0]:
            cu.execute("DELETE FROM PackageIndexMark")
            cu.execute("INSERT INTO PackageIndexMark ( mark ) VALUES ( 1 )")
            self.db.commit()
        else:
            self.db.rollback()
<commit_msg>Fix package index mark so that if the external index doesn't continually get blown away by the internal indexer if no commits have been made to an internal project<commit_after>#
# Copyright (c) 2005-2009 rPath, Inc.
#
# All Rights Reserved
#
from mint import config
from mint import helperfuncs
from mint.db import pkgindex
from mint.db import projects
from mint.lib import database
from mint.lib import scriptlibrary

from conary import conaryclient
from conary import conarycfg
from conary import dbstore
from conary import versions
from conary.repository import repository

import os
import time

hiddenLabels = [
    versions.Label('<EMAIL>:rpl1'),
    versions.Label('<EMAIL>:bugblatterbeast')
]

class PackageIndexer(scriptlibrary.SingletonScript):
    db = None
    cfgPath = config.RBUILDER_CONFIG

    def __init__(self, aLockPath = scriptlibrary.DEFAULT_LOCKPATH,
            aMintServer=None):
        self.cfg = config.MintConfig()
        self.cfg.read(self.cfgPath)
        if self.logFileName:
            self.logPath = os.path.join(self.cfg.logPath, self.logFileName)

        scriptlibrary.SingletonScript.__init__(self, aLockPath)

    def cleanup(self):
        if self.db:
            self.db.close()


class UpdatePackageIndex(PackageIndexer):
    logFileName = 'package-index.log'

    def action(self):
        self.log.info("Updating package index")

        self.db = dbstore.connect(self.cfg.dbPath, driver = self.cfg.dbDriver)
        self.db.loadSchema()
        cu = self.db.cursor()

        try:
            cu.execute('SELECT COUNT(*) FROM PackageIndexMark')
            if not cu.fetchone()[0]:
                cu.execute('INSERT INTO PackageIndexMark VALUES(0)')
                oldMark = 0
            else:
                cu.execute("SELECT MAX(mark) FROM PackageIndexMark")
                oldMark = cu.fetchone()[0]
            cu.execute("SELECT COALESCE(MAX(timestamp), 0) FROM Commits")
            newMark = cu.fetchone()[0]
            # If the oldMark was 1, and there have been no comits (newMark is
            # 0), preserve the oldMark of 1, so that the external package
            # index does not get blown away.
            if newMark == 0 and oldMark == 1:
                newMark = 1

            # Clear out Package index if the timestamp in PackageIndexMark == 0
            cu.execute("""DELETE FROM PackageIndex WHERE
                              (SELECT mark FROM PackageIndexMark) = 0""")

            cu.execute("""SELECT Projects.projectId, troveName, Commits.version,
                                 timestamp
                              FROM Commits
                              LEFT JOIN Projects
                                  ON Commits.projectId=Projects.projectId
                              WHERE (troveName NOT LIKE '%:%' OR troveName LIKE '%:source')
                              AND hidden=0 AND disabled=0
                              AND timestamp >=
                                  (SELECT mark FROM PackageIndexMark)""")

            res = cu.fetchall()
            numPkgs = len(res) - 1
            if numPkgs > 0:
                self.log.info("Indexing %d packages" % numPkgs)
            else:
                self.log.info("Package index is up to date")
            packageDict = {}
            labelMap = {}
            for projectId, troveName, verStr, timeStamp in res:
                troveEntry = packageDict.setdefault(troveName, {})
                version = versions.VersionFromString(verStr)
                label = str(version.branch().label())
                versionList = troveEntry.setdefault(label, [])

                version = version.copy()
                version.resetTimeStamps(timeStamp)
                versionList.append(version)
                labelMap[label] = projectId

            packageIndex = []
            for troveName in packageDict:
                for label in packageDict[troveName]:
                    packageIndex.append((labelMap[label], troveName,
                                         max(packageDict[troveName][label])))

            inserts = []
            updates = []
            for projectId, troveName, version in packageIndex:
                cu.execute("""SELECT pkgId, version FROM PackageIndex
                                  WHERE projectId=? AND name=?""",
                           projectId, troveName)

                res = [x[0] for x in cu.fetchall() if \
                       versions.VersionFromString(x[1]).branch().label() == \
                       version.branch().label()]

                label = version.branch().label()
                serverName = label.getHost()
                branchName = label.getNamespace() + ":" + label.getLabel()

                isSource = int(troveName.endswith(':source'))
                if not res:
                    inserts.append((projectId, troveName, str(version), serverName,
                        branchName, isSource))
                else:
                    pkgId = res[0]
                    updates.append((projectId, troveName, str(version), serverName,
                        branchName, isSource, pkgId))

            if inserts or updates:
                self.db.transaction()
                if inserts:
                    cu.executemany("""INSERT INTO PackageIndex
                                      (projectId, name, version,
                                       serverName, branchName, isSource)
                                      VALUES (?, ?, ?, ?, ?, ?)""", inserts)
                if updates:
                    cu.executemany("""UPDATE PackageIndex SET
                                          projectId=?, name=?, version=?,
                                          serverName=?, branchName=?, isSource=?
                                          WHERE pkgId=?""", updates)
                self.db.commit()

            cu.execute("UPDATE PackageIndexMark SET mark=?", int(newMark))
        except Exception, e:
            self.log.error("Error occurred: %s" % str(e))
            self.db.rollback()
            exitcode = 1
            raise
        else:
            self.log.info("Completed successfully: %d" % len(inserts))
            exitcode = 0
            self.db.commit()
        return exitcode


class UpdatePackageIndexExternal(PackageIndexer):
    logFileName = 'package-index-external.log'

    def updateMark(self):
        # This code exists to overcome the situation where there are no
        # internal projects on the rBuilder, or there are internal projects but
        # they haven't had any commits. internal package index code will delete
        # the package index if there is no mark or a mark of zero.  this code
        # sets the mark to "1" to ensure no race conditions exist
        # sorrounding the setting of the mark.
        cu = self.db.transaction()
        cu.execute("SELECT COUNT(*) FROM Projects WHERE external = 0")
        internalProjects = cu.fetchone()[0]
        cu.execute("SELECT COUNT(*) FROM Commits")
        commits = cu.fetchone()[0]
        if not internalProjects or not commits:
            cu.execute("DELETE FROM PackageIndexMark")
            cu.execute("INSERT INTO PackageIndexMark ( mark ) VALUES ( 1 )")
            self.db.commit()
        else:
            self.db.rollback()

{'fraction_non_alphanumeric': 0.09123744852994294, 'fraction_numerical': 0.004695513978183919, 'mean_word_length': 1.622466376207615, 'pattern_counts': {'":': 2, '<': 6, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4302281', 'n_tokens_mistral': 3585, 'n_tokens_neox': 3368, 'n_words': 943}