<commit_before>#!/usr/bin/python
#
# Copyright (c) 2009 rPath, Inc.
#
# All Rights Reserved
#

from mint import buildtypes
import testsetup

import base64
import os
import re
import subprocess
import tempfile
import time
from conary.dbstore.sqllib import CaselessDict
from conary import conaryclient

from restlib.http import handler
from restlib.http import request
from mint import helperfuncs
from mint import users
from mint.mint_error import UserAlreadyExists
from mint.rest.api import site
from mint.rest.middleware import auth
from mint.rest.middleware import formatter
from mint.rest.modellib import converter
from rpath_proddef import api1 as proddef

from mint_test import mint_rephelp

URLQUOTE_RE = re.compile('%([A-Z0-9]{2})')

class BaseRestTest(mint_rephelp.MintDatabaseHelper):
    buildDefs = [
        ('Citrix XenServer 32-bit', 'xen', 'x86', 'xenOvaImage'),
        ('Citrix XenServer 64-bit', 'xen', 'x86_64', 'xenOvaImage'),
        ('VMware ESX 32-bit', 'vmware', 'x86', 'vmwareEsxImage'),
        ('VMware ESX 64-bit', 'vmware', 'x86_64', 'vmwareEsxImage'),
    ]
    buildTemplates = buildDefs
    architectures = [ ]
    containerTemplates = [ ]
    flavorSets = [ ]
    productVersion = '1.0'
    productName = 'Project 1'
    productShortName = 'testproject'
    productVersionDescription = 'Version description'
    productDomainName = mint_rephelp.MINT_PROJECT_DOMAIN
    productHostname = "%s.%s" % (productShortName, productDomainName)

    def setUp(self):
        mint_rephelp.MintDatabaseHelper.setUp(self)
        self.mockProddef()
        
    def tearDown(self):
        mint_rephelp.MintDatabaseHelper.tearDown(self)
        self.unMockProddef()

    ControllerFactory = None

    def setupProduct(self):
        self.setUpProductDefinition()

        pd = self._setupProduct()
        # used by other tests before setupProduct was fixturized.
        self.productDefinition = pd

    @mint_rephelp.restFixturize('apitest.setupProduct')
    def _setupProduct(self):
        version = self.productVersion
        projectName = self.productName
        shortName = self.productShortName
        domainName = self.productDomainName
        description = self.productVersionDescription
        db = self.openRestDatabase()
        self.createUser('adminuser', admin=True)
        self.setDbUser(db, 'adminuser')
        self.createProduct(shortName,
                           name=projectName,
                           domainname=domainName, db=db)
        self.createProductVersion(db, shortName,
                                  version,
                                  description=description,
                                  namespace=self.mintCfg.namespace)
        pd = helperfuncs.sanitizeProductDefinition(
            projectName, '', shortName, domainName,
            shortName, version, '', self.mintCfg.namespace)
        stageRefs = [ x.name for x in pd.getStages() ]
        for _name, displayName, _flavor in self.architectures:
            pd.addArchitecture(_name, displayName, _flavor)
        for _name, displayName, _flavor in self.flavorSets:
            pd.addFlavorSet(_name, displayName, _flavor)
        for _name, opts in self.containerTemplates:
            pd.addContainerTemplate(pd.imageType(_name, opts))
        for buildName, flavorSetRef, archRef, containerTemplateRef in \
                    self.buildTemplates:
            pd.addBuildTemplate(name = buildName,
                displayName = buildName,
                flavorSetRef = flavorSetRef,
                architectureRef = archRef,
                containerTemplateRef = containerTemplateRef)
        for buildName, flavorSetRef, archRef, containerTemplateRef in \
                    self.buildDefs:
            pd.addBuildDefinition(name = buildName,
                flavorSetRef = flavorSetRef,
                architectureRef = archRef,
                containerTemplateRef = containerTemplateRef,
                stages = stageRefs)
        client = db.productMgr.reposMgr.getConaryClientForProduct(shortName)
        pd.saveToRepository(client, 'Product Definition commit\n')
        return pd

    @mint_rephelp.restFixturize('apitest.setupReleases')
    def setupReleases(self):
        self.setupProduct()
        # Add a group
        label = self.productDefinition.getDefaultLabel()
        db = self.openRestDatabase()
        self.setDbUser(db, 'adminuser')
        client = db.productMgr.reposMgr.getConaryClientForProduct(self.productShortName)
        repos = client.getRepos()
        self.addComponent("foo:bin=%s" % label, repos=repos)
        self.addCollection("foo=%s" % label, ['foo:bin'], repos=repos)
        groupTrv = self.addCollection("group-foo=%s" % label, ['foo'],
                                      repos=repos)

        # Let's add some images
        images = [
            ('Image 1', buildtypes.INSTALLABLE_ISO),
            ('Image 2', buildtypes.TARBALL),
        ]
        groupName = groupTrv.getName()
        groupVer = groupTrv.getVersion().freeze()
        groupFlv = str(groupTrv.getFlavor())
        imageIds = []
        for imageName, imageType in images:
            imageId = self.createImage(db, self.productShortName,
                imageType, name = imageName,
                description = "Description for %s" % imageName,
                troveName = groupName,
                troveVersion = groupVer,
                troveFlavor = groupFlv)
            self.setImageFiles(db, self.productShortName, imageId)
            imageIds.append(imageId)

        # make sure the times are different fo
        imageId3 = self.createImage(db, self.productShortName,
                                   buildtypes.TARBALL,
                                   name = 'Image 3')
        self.setImageFiles(db, self.productShortName, imageId3)

        releaseId = db.createRelease(self.productShortName, 'Release Name', '',
                'v1', imageIds)
        releaseId2 = db.createRelease(self.productShortName, 'Release Name', '',
                'v1', [imageId3])
        db.publishRelease(self.productShortName, releaseId2, True)

    def mockProddef(self):

        def newLoadFromRepository(*args, **kw):
            platdef = args[0]
            oldClient = args[1]
            label = args[2]
            return self.oldLoadFromRepository(platdef, self.cclient, label)

        self.oldLoadFromRepository = proddef.PlatformDefinition.loadFromRepository
        proddef.PlatformDefinition.loadFromRepository = newLoadFromRepository

    def unMockProddef(self):
        proddef.PlatformDefinition.loadFromRepository = \
            self.oldLoadFromRepository

    def setupPlatforms(self):
        repos = self.openRepository()
        self.cclient = self.getConaryClient()
        self.cclient.repos = repos

        platformLabel1 = self.mintCfg.availablePlatforms[0]
        pl1 = self.productDefinition.toPlatformDefinition()
        cst = pl1.newContentSourceType('RHN', 'RHN')
        pl1.setPlatformName('Crowbar Linux 1')
        pl1.saveToRepository(self.cclient, platformLabel1)

        platformLabel2 = self.mintCfg.availablePlatforms[1]
        pl2 = self.productDefinition.toPlatformDefinition()
        cst = pl1.newContentSourceType('RHN', 'RHN')
        pl2.setPlatformName('Crowbar Linux 2')
        pl2.saveToRepository(self.cclient, platformLabel2)

    def getConaryClient(self):
        return conaryclient.ConaryClient(self.cfg)

    def getRestClient(self, **kw):
        if 'db' in kw:
            db = kw.pop('db')
        else:
            subscribers = kw.pop('subscribers', None)
            db = self.openRestDatabase(subscribers = subscribers)

        admin = kw.pop('admin', None)
        if 'username' not in kw:
            if admin:
                kw['username'] = 'adminuser'
            else:
                kw['username'] = 'someuser'
        if 'password' not in kw:
            kw['password'] = kw['username']
            if kw['username']:
                try:
                    self.createUser(kw['username'], kw['password'], admin=admin)
                except UserAlreadyExists:
                    pass

        return self.ControllerFactory(self.mintCfg, db, **kw)

    def escapeURLQuotes(self, foo):
        """
        Escape URL quotes (%FF sequences) with an extra %
        so they can be used with the % operator.
        """
        return URLQUOTE_RE.sub('%%\\1', foo)

    def getTestProjectRepos(self):
        db = self.openMintDatabase()
        reposMgr = db.productMgr.reposMgr
        return reposMgr.getRepositoryClientForProduct('testproject')

    def assertBlobEquals(self, actual, expected):
        """
        Like assertEquals but prints a diff.
        """
        if actual != expected:
            if actual and expected:
                actualF = tempfile.NamedTemporaryFile(prefix='actual-')
                actualF.write(actual)
                actualF.flush()

                expectedF = tempfile.NamedTemporaryFile(prefix='expected-')
                expectedF.write(expected)
                expectedF.flush()

                proc = subprocess.Popen(['/usr/bin/diff', '-u',
                    expectedF.name, actualF.name],
                    shell=False, stdout=subprocess.PIPE)
                diff, _ = proc.communicate()

                raise self.failureException("Results do not match.\nDiff:\n%s"
                        % (diff,))
            elif actual:
                raise self.failureException("Expected no output, actual "
                        "result is:\n%s" % (actual,))
            else:
                raise self.failureException("Got no output, expected "
                        "result is:\n%s" % (expected,))


class Controller(object):
    RequestFactory = None
    HandlerFactory = None
    FormatCallbackFactory = None
    def __init__(self, cfg, restDb, username, password):
        self.server = 'localhost'
        self.port = '8000'
        self.controller = site.RbuilderRestServer(cfg, restDb)
        self.handler = self.HandlerFactory(self.controller)
        self.handler.addCallback(auth.AuthenticationCallback(cfg, restDb,
            self.controller))
        self.handler.addCallback(self.FormatCallbackFactory(self.controller))
        self.restDb = restDb
        self.username = username
        self.password = password

    def call(self, method, uri, body=None, convert=False, headers=None):
        request = self.RequestFactory(method, uri, body=body)
        request._convert = convert

        if self.username:
            request.headers['Authorization'] = 'Basic ' + base64.b64encode(
                    self.username + ':' + self.password)

        if headers:
            request.headers.update(headers)
        response = self.handler.handle(request)
        if convert:
            response = response.content
        return request, response

    def convert(self, type, request, object):
        return converter.toText(type, object, self.controller, request)

class MockRequest(request.Request):
    def __init__(self, method, uri, username=None, admin=False,
                 body=None):
        self.method = method
        self.uri = uri
        self.extension = None
        self.body = body
        request.Request.__init__(self, None, '/api')

    def _setProperties(self):
        self.headers = CaselessDict()
        # self.method is set in __init__
        self.remote = (None, None)

    def _getRawPath(self):
        return 'http://localhost:8000', ('/api/' + self.uri)

    def _getHeaders(self):
        return {}

    def read(self):
        return self.body

    def _getPostData(self):
        return {}

    def _getRemote(self):
        return (None, None)

    def getContentLength(self):
        return self.body and len(self.body) or 0


class MockHandler(handler.HttpHandler):
    def handle(self, request, pathPrefix=''):
        return self.getResponse(request)

    class exceptionCallback(object):
        def processException(self, request, excClass, exception, tb):
            raise excClass, exception, tb


class MockFormatCallback(formatter.FormatCallback):
    def processResponse(self, request, res):
        if request._convert:
            return formatter.FormatCallback.processResponse(self, request, res)
        else:
            return res

# Set defaults
# We do this here because I did not want to move classes around
BaseRestTest.ControllerFactory = Controller
Controller.FormatCallbackFactory = MockFormatCallback
Controller.HandlerFactory = MockHandler
Controller.RequestFactory = MockRequest
<commit_msg>RBL-5187 fix up unit tests for new platform defn changes<commit_after>#!/usr/bin/python
#
# Copyright (c) 2009 rPath, Inc.
#
# All Rights Reserved
#

from mint import buildtypes
import testsetup

import base64
import os
import re
import subprocess
import tempfile
import time
from conary.dbstore.sqllib import CaselessDict
from conary import conaryclient

from restlib.http import handler
from restlib.http import request
from mint import helperfuncs
from mint import users
from mint.mint_error import UserAlreadyExists
from mint.rest.api import site
from mint.rest.middleware import auth
from mint.rest.middleware import formatter
from mint.rest.modellib import converter
from rpath_proddef import api1 as proddef

from mint_test import mint_rephelp

URLQUOTE_RE = re.compile('%([A-Z0-9]{2})')

class BaseRestTest(mint_rephelp.MintDatabaseHelper):
    buildDefs = [
        ('Citrix XenServer 32-bit', 'xen', 'x86', 'xenOvaImage'),
        ('Citrix XenServer 64-bit', 'xen', 'x86_64', 'xenOvaImage'),
        ('VMware ESX 32-bit', 'vmware', 'x86', 'vmwareEsxImage'),
        ('VMware ESX 64-bit', 'vmware', 'x86_64', 'vmwareEsxImage'),
    ]
    buildTemplates = buildDefs
    architectures = [ ]
    containerTemplates = [ ]
    flavorSets = [ ]
    productVersion = '1.0'
    productName = 'Project 1'
    productShortName = 'testproject'
    productVersionDescription = 'Version description'
    productDomainName = mint_rephelp.MINT_PROJECT_DOMAIN
    productHostname = "%s.%s" % (productShortName, productDomainName)

    def setUp(self):
        mint_rephelp.MintDatabaseHelper.setUp(self)
        self.mockProddef()
        
    def tearDown(self):
        mint_rephelp.MintDatabaseHelper.tearDown(self)
        self.unMockProddef()

    ControllerFactory = None

    def setupProduct(self):
        self.setUpProductDefinition()

        pd = self._setupProduct()
        # used by other tests before setupProduct was fixturized.
        self.productDefinition = pd

    @mint_rephelp.restFixturize('apitest.setupProduct')
    def _setupProduct(self):
        version = self.productVersion
        projectName = self.productName
        shortName = self.productShortName
        domainName = self.productDomainName
        description = self.productVersionDescription
        db = self.openRestDatabase()
        self.createUser('adminuser', admin=True)
        self.setDbUser(db, 'adminuser')
        self.createProduct(shortName,
                           name=projectName,
                           domainname=domainName, db=db)
        self.createProductVersion(db, shortName,
                                  version,
                                  description=description,
                                  namespace=self.mintCfg.namespace)
        pd = helperfuncs.sanitizeProductDefinition(
            projectName, '', shortName, domainName,
            shortName, version, '', self.mintCfg.namespace)
        stageRefs = [ x.name for x in pd.getStages() ]
        for _name, displayName, _flavor in self.architectures:
            pd.addArchitecture(_name, displayName, _flavor)
        for _name, displayName, _flavor in self.flavorSets:
            pd.addFlavorSet(_name, displayName, _flavor)
        for _name, opts in self.containerTemplates:
            pd.addContainerTemplate(pd.imageType(_name, opts))
        for buildName, flavorSetRef, archRef, containerTemplateRef in \
                    self.buildTemplates:
            pd.addBuildTemplate(name = buildName,
                displayName = buildName,
                flavorSetRef = flavorSetRef,
                architectureRef = archRef,
                containerTemplateRef = containerTemplateRef)
        for buildName, flavorSetRef, archRef, containerTemplateRef in \
                    self.buildDefs:
            pd.addBuildDefinition(name = buildName,
                flavorSetRef = flavorSetRef,
                architectureRef = archRef,
                containerTemplateRef = containerTemplateRef,
                stages = stageRefs)
        client = db.productMgr.reposMgr.getConaryClientForProduct(shortName)
        pd.saveToRepository(client, 'Product Definition commit\n')
        return pd

    @mint_rephelp.restFixturize('apitest.setupReleases')
    def setupReleases(self):
        self.setupProduct()
        # Add a group
        label = self.productDefinition.getDefaultLabel()
        db = self.openRestDatabase()
        self.setDbUser(db, 'adminuser')
        client = db.productMgr.reposMgr.getConaryClientForProduct(self.productShortName)
        repos = client.getRepos()
        self.addComponent("foo:bin=%s" % label, repos=repos)
        self.addCollection("foo=%s" % label, ['foo:bin'], repos=repos)
        groupTrv = self.addCollection("group-foo=%s" % label, ['foo'],
                                      repos=repos)

        # Let's add some images
        images = [
            ('Image 1', buildtypes.INSTALLABLE_ISO),
            ('Image 2', buildtypes.TARBALL),
        ]
        groupName = groupTrv.getName()
        groupVer = groupTrv.getVersion().freeze()
        groupFlv = str(groupTrv.getFlavor())
        imageIds = []
        for imageName, imageType in images:
            imageId = self.createImage(db, self.productShortName,
                imageType, name = imageName,
                description = "Description for %s" % imageName,
                troveName = groupName,
                troveVersion = groupVer,
                troveFlavor = groupFlv)
            self.setImageFiles(db, self.productShortName, imageId)
            imageIds.append(imageId)

        # make sure the times are different fo
        imageId3 = self.createImage(db, self.productShortName,
                                   buildtypes.TARBALL,
                                   name = 'Image 3')
        self.setImageFiles(db, self.productShortName, imageId3)

        releaseId = db.createRelease(self.productShortName, 'Release Name', '',
                'v1', imageIds)
        releaseId2 = db.createRelease(self.productShortName, 'Release Name', '',
                'v1', [imageId3])
        db.publishRelease(self.productShortName, releaseId2, True)

    def mockProddef(self):

        def newLoadFromRepository(*args, **kw):
            platdef = args[0]
            oldClient = args[1]
            label = args[2]
            return self.oldLoadFromRepository(platdef, self.cclient, label)

        self.oldLoadFromRepository = proddef.PlatformDefinition.loadFromRepository
        proddef.PlatformDefinition.loadFromRepository = newLoadFromRepository

    def unMockProddef(self):
        proddef.PlatformDefinition.loadFromRepository = \
            self.oldLoadFromRepository

    def setupPlatforms(self):
        repos = self.openRepository()
        self.cclient = self.getConaryClient()
        self.cclient.repos = repos

        platformLabel1 = self.mintCfg.availablePlatforms[0]
        pl1 = self.productDefinition.toPlatformDefinition()
        cst = pl1.newContentSourceType('RHN', 'RHN')
        ds = pl1.newDataSource('Channel 1', 'Channel 1')
        pl1.setContentProvider('Crowbar', 'Crowbar', [cst], [ds])
        pl1.setPlatformName('Crowbar Linux 1')
        pl1.saveToRepository(self.cclient, platformLabel1)

        platformLabel2 = self.mintCfg.availablePlatforms[1]
        pl2 = self.productDefinition.toPlatformDefinition()
        cst = pl1.newContentSourceType('RHN', 'RHN')
        ds = pl1.newDataSource('Channel 1', 'Channel 1')
        pl1.setContentProvider('Crowbar', 'Crowbar', [cst], [ds])
        pl2.setPlatformName('Crowbar Linux 2')
        pl2.saveToRepository(self.cclient, platformLabel2)

    def getConaryClient(self):
        return conaryclient.ConaryClient(self.cfg)

    def getRestClient(self, **kw):
        if 'db' in kw:
            db = kw.pop('db')
        else:
            subscribers = kw.pop('subscribers', None)
            db = self.openRestDatabase(subscribers = subscribers)

        admin = kw.pop('admin', None)
        if 'username' not in kw:
            if admin:
                kw['username'] = 'adminuser'
            else:
                kw['username'] = 'someuser'
        if 'password' not in kw:
            kw['password'] = kw['username']
            if kw['username']:
                try:
                    self.createUser(kw['username'], kw['password'], admin=admin)
                except UserAlreadyExists:
                    pass

        return self.ControllerFactory(self.mintCfg, db, **kw)

    def escapeURLQuotes(self, foo):
        """
        Escape URL quotes (%FF sequences) with an extra %
        so they can be used with the % operator.
        """
        return URLQUOTE_RE.sub('%%\\1', foo)

    def getTestProjectRepos(self):
        db = self.openMintDatabase()
        reposMgr = db.productMgr.reposMgr
        return reposMgr.getRepositoryClientForProduct('testproject')

    def assertBlobEquals(self, actual, expected):
        """
        Like assertEquals but prints a diff.
        """
        if actual != expected:
            if actual and expected:
                actualF = tempfile.NamedTemporaryFile(prefix='actual-')
                actualF.write(actual)
                actualF.flush()

                expectedF = tempfile.NamedTemporaryFile(prefix='expected-')
                expectedF.write(expected)
                expectedF.flush()

                proc = subprocess.Popen(['/usr/bin/diff', '-u',
                    expectedF.name, actualF.name],
                    shell=False, stdout=subprocess.PIPE)
                diff, _ = proc.communicate()

                raise self.failureException("Results do not match.\nDiff:\n%s"
                        % (diff,))
            elif actual:
                raise self.failureException("Expected no output, actual "
                        "result is:\n%s" % (actual,))
            else:
                raise self.failureException("Got no output, expected "
                        "result is:\n%s" % (expected,))


class Controller(object):
    RequestFactory = None
    HandlerFactory = None
    FormatCallbackFactory = None
    def __init__(self, cfg, restDb, username, password):
        self.server = 'localhost'
        self.port = '8000'
        self.controller = site.RbuilderRestServer(cfg, restDb)
        self.handler = self.HandlerFactory(self.controller)
        self.handler.addCallback(auth.AuthenticationCallback(cfg, restDb,
            self.controller))
        self.handler.addCallback(self.FormatCallbackFactory(self.controller))
        self.restDb = restDb
        self.username = username
        self.password = password

    def call(self, method, uri, body=None, convert=False, headers=None):
        request = self.RequestFactory(method, uri, body=body)
        request._convert = convert

        if self.username:
            request.headers['Authorization'] = 'Basic ' + base64.b64encode(
                    self.username + ':' + self.password)

        if headers:
            request.headers.update(headers)
        response = self.handler.handle(request)
        if convert:
            response = response.content
        return request, response

    def convert(self, type, request, object):
        return converter.toText(type, object, self.controller, request)

class MockRequest(request.Request):
    def __init__(self, method, uri, username=None, admin=False,
                 body=None):
        self.method = method
        self.uri = uri
        self.extension = None
        self.body = body
        request.Request.__init__(self, None, '/api')

    def _setProperties(self):
        self.headers = CaselessDict()
        # self.method is set in __init__
        self.remote = (None, None)

    def _getRawPath(self):
        return 'http://localhost:8000', ('/api/' + self.uri)

    def _getHeaders(self):
        return {}

    def read(self):
        return self.body

    def _getPostData(self):
        return {}

    def _getRemote(self):
        return (None, None)

    def getContentLength(self):
        return self.body and len(self.body) or 0


class MockHandler(handler.HttpHandler):
    def handle(self, request, pathPrefix=''):
        return self.getResponse(request)

    class exceptionCallback(object):
        def processException(self, request, excClass, exception, tb):
            raise excClass, exception, tb


class MockFormatCallback(formatter.FormatCallback):
    def processResponse(self, request, res):
        if request._convert:
            return formatter.FormatCallback.processResponse(self, request, res)
        else:
            return res

# Set defaults
# We do this here because I did not want to move classes around
BaseRestTest.ControllerFactory = Controller
Controller.FormatCallbackFactory = MockFormatCallback
Controller.HandlerFactory = MockHandler
Controller.RequestFactory = MockRequest

{'fraction_non_alphanumeric': 0.09580016469942355, 'fraction_numerical': 0.006431120348221638, 'mean_word_length': 2.312378230939083, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '235277', 'n_tokens_mistral': 7151, 'n_tokens_neox': 6868, 'n_words': 1553}