<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import contextlib
import json
import os
import re
import tarfile
import tempfile
import urllib2
import urlparse

from devstack import downloader as down
from devstack import log
from devstack import shell as sh
from devstack import utils

from devstack.components import keystone

LOG = log.getLogger("devstack.image.creator")

# These are used when looking inside archives
KERNEL_FN_MATCH = re.compile(r"(.*)vmlinuz$", re.I)
RAMDISK_FN_MATCH = re.compile(r"(.*)initrd$", re.I)
IMAGE_FN_MATCH = re.compile(r"(.*)img$", re.I)

# Glance commands
KERNEL_ADD = ['glance', 'add', '-A', '%TOKEN%', '--silent-upload',
    'name="%IMAGE_NAME%-kernel"', 'is_public=true', 'container_format=aki',
    'disk_format=aki']
INITRD_ADD = ['glance', 'add', '-A', '%TOKEN%', '--silent-upload',
    'name="%IMAGE_NAME%-ramdisk"', 'is_public=true', 'container_format=ari',
    'disk_format=ari']
IMAGE_ADD = ['glance', 'add', '-A', '%TOKEN%', '--silent-upload',
    'name="%IMAGE_NAME%.img"',
    'is_public=true', 'container_format=ami', 'disk_format=ami',
    'kernel_id=%KERNEL_ID%', 'ramdisk_id=%INITRD_ID%']
DETAILS_SHOW = ['glance', '-A', '%TOKEN%', 'details']

# Extensions that tarfile knows how to work with
TAR_EXTS = ['.tgz', '.gzip', '.gz', '.bz2', '.tar']

# Used to attempt to produce a name for images (to see if we already have it)
# And to use as the final name...
# Reverse sorted so that .tar.gz replaces before .tar (and so on)
NAME_CLEANUPS = [
    '.tar.gz',
    '.img.gz',
    '.img',
] + TAR_EXTS
NAME_CLEANUPS.sort()
NAME_CLEANUPS.reverse()


class Unpacker(object):

    def __init__(self):
        pass

    def _unpack_tar(self, file_name, file_location, tmp_dir):
        (root_name, _) = os.path.splitext(file_name)
        kernel_fn = None
        ramdisk_fn = None
        root_img_fn = None
        with contextlib.closing(tarfile.open(file_location, 'r')) as tfh:
            for tmemb in tfh.getmembers():
                fn = tmemb.name
                if KERNEL_FN_MATCH.match(fn):
                    kernel_fn = fn
                    LOG.debug("Found kernel: %r" % (fn))
                elif RAMDISK_FN_MATCH.match(fn):
                    ramdisk_fn = fn
                    LOG.debug("Found ram disk: %r" % (fn))
                elif IMAGE_FN_MATCH.match(fn):
                    root_img_fn = fn
                    LOG.debug("Found root image: %r" % (fn))
                else:
                    LOG.debug("Unknown member %r - skipping" % (fn))
        if not root_img_fn:
            msg = "Image %r has no root image member" % (file_name)
            raise RuntimeError(msg)
        extract_dir = sh.joinpths(tmp_dir, root_name)
        sh.mkdir(extract_dir)
        LOG.info("Extracting to %r", extract_dir)
        with contextlib.closing(tarfile.open(file_location, 'r')) as tfh:
            tfh.extractall(extract_dir)
        locations = dict()
        if kernel_fn:
            locations['kernel'] = sh.joinpths(extract_dir, kernel_fn)
        if ramdisk_fn:
            locations['ramdisk'] = sh.joinpths(extract_dir, kernel_fn)
        locations['image'] = sh.joinpths(extract_dir, root_img_fn)
        return locations

    def _unpack_image(self, file_name, file_location, tmp_dir):
        locations = dict()
        locations['image'] = file_location
        return locations

    def unpack(self, file_name, file_location, tmp_dir):
        (_, fn_ext) = os.path.splitext(file_name)
        fn_ext = fn_ext.lower()
        if fn_ext in TAR_EXTS:
            return self._unpack_tar(file_name, file_location, tmp_dir)
        elif fn_ext in ['.img']:
            return self._unpack_image(file_name, file_location, tmp_dir)
        else:
            msg = "Currently we do not know how to unpack %r" % (file_name)
            raise NotImplementedError(msg)


class Image(object):

    def __init__(self, url, token):
        self.url = url
        self.token = token
        self._registry = Registry(token)
<commit_msg>Fixed variable for ramdisk filename<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import contextlib
import json
import os
import re
import tarfile
import tempfile
import urllib2
import urlparse

from devstack import downloader as down
from devstack import log
from devstack import shell as sh
from devstack import utils

from devstack.components import keystone

LOG = log.getLogger("devstack.image.creator")

# These are used when looking inside archives
KERNEL_FN_MATCH = re.compile(r"(.*)vmlinuz$", re.I)
RAMDISK_FN_MATCH = re.compile(r"(.*)initrd$", re.I)
IMAGE_FN_MATCH = re.compile(r"(.*)img$", re.I)

# Glance commands
KERNEL_ADD = ['glance', 'add', '-A', '%TOKEN%', '--silent-upload',
    'name="%IMAGE_NAME%-kernel"', 'is_public=true', 'container_format=aki',
    'disk_format=aki']
INITRD_ADD = ['glance', 'add', '-A', '%TOKEN%', '--silent-upload',
    'name="%IMAGE_NAME%-ramdisk"', 'is_public=true', 'container_format=ari',
    'disk_format=ari']
IMAGE_ADD = ['glance', 'add', '-A', '%TOKEN%', '--silent-upload',
    'name="%IMAGE_NAME%.img"',
    'is_public=true', 'container_format=ami', 'disk_format=ami',
    'kernel_id=%KERNEL_ID%', 'ramdisk_id=%INITRD_ID%']
DETAILS_SHOW = ['glance', '-A', '%TOKEN%', 'details']

# Extensions that tarfile knows how to work with
TAR_EXTS = ['.tgz', '.gzip', '.gz', '.bz2', '.tar']

# Used to attempt to produce a name for images (to see if we already have it)
# And to use as the final name...
# Reverse sorted so that .tar.gz replaces before .tar (and so on)
NAME_CLEANUPS = [
    '.tar.gz',
    '.img.gz',
    '.img',
] + TAR_EXTS
NAME_CLEANUPS.sort()
NAME_CLEANUPS.reverse()


class Unpacker(object):

    def __init__(self):
        pass

    def _unpack_tar(self, file_name, file_location, tmp_dir):
        (root_name, _) = os.path.splitext(file_name)
        kernel_fn = None
        ramdisk_fn = None
        root_img_fn = None
        with contextlib.closing(tarfile.open(file_location, 'r')) as tfh:
            for tmemb in tfh.getmembers():
                fn = tmemb.name
                if KERNEL_FN_MATCH.match(fn):
                    kernel_fn = fn
                    LOG.debug("Found kernel: %r" % (fn))
                elif RAMDISK_FN_MATCH.match(fn):
                    ramdisk_fn = fn
                    LOG.debug("Found ram disk: %r" % (fn))
                elif IMAGE_FN_MATCH.match(fn):
                    root_img_fn = fn
                    LOG.debug("Found root image: %r" % (fn))
                else:
                    LOG.debug("Unknown member %r - skipping" % (fn))
        if not root_img_fn:
            msg = "Image %r has no root image member" % (file_name)
            raise RuntimeError(msg)
        extract_dir = sh.joinpths(tmp_dir, root_name)
        sh.mkdir(extract_dir)
        LOG.info("Extracting to %r", extract_dir)
        with contextlib.closing(tarfile.open(file_location, 'r')) as tfh:
            tfh.extractall(extract_dir)
        locations = dict()
        if kernel_fn:
            locations['kernel'] = sh.joinpths(extract_dir, kernel_fn)
        if ramdisk_fn:
            locations['ramdisk'] = sh.joinpths(extract_dir, ramdisk_fn)
        locations['image'] = sh.joinpths(extract_dir, root_img_fn)
        return locations

    def _unpack_image(self, file_name, file_location, tmp_dir):
        locations = dict()
        locations['image'] = file_location
        return locations

    def unpack(self, file_name, file_location, tmp_dir):
        (_, fn_ext) = os.path.splitext(file_name)
        fn_ext = fn_ext.lower()
        if fn_ext in TAR_EXTS:
            return self._unpack_tar(file_name, file_location, tmp_dir)
        elif fn_ext in ['.img']:
            return self._unpack_image(file_name, file_location, tmp_dir)
        else:
            msg = "Currently we do not know how to unpack %r" % (file_name)
            raise NotImplementedError(msg)


class Image(object):

    def __init__(self, url, token):
        self.url = url
        self.token = token
        self._registry = Registry(token)

{'fraction_non_alphanumeric': 0.12027012541537142, 'fraction_numerical': 0.0027870082538321364, 'mean_word_length': 2.7666532095276546, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5364743', 'n_tokens_mistral': 3049, 'n_tokens_neox': 2950, 'n_words': 826}