<commit_before>/**
 * Copyright (C) 2012-2015 Dell, Inc.
 * See annotations for authorship information
 *
 * ====================================================================
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * ====================================================================
 */

package org.dasein.cloud.digitalocean.compute;

import org.apache.log4j.Logger;
import org.dasein.cloud.AsynchronousTask;
import org.dasein.cloud.CloudException;
import org.dasein.cloud.InternalException;
import org.dasein.cloud.OperationNotSupportedException;
import org.dasein.cloud.ProviderContext;
import org.dasein.cloud.ResourceStatus;
import org.dasein.cloud.Tag;
import org.dasein.cloud.compute.AbstractImageSupport;
import org.dasein.cloud.compute.Architecture;
import org.dasein.cloud.compute.ImageCapabilities;
import org.dasein.cloud.compute.ImageClass;
import org.dasein.cloud.compute.ImageCreateOptions;
import org.dasein.cloud.compute.ImageFilterOptions;
import org.dasein.cloud.compute.MachineImage;
import org.dasein.cloud.compute.MachineImageState;
import org.dasein.cloud.compute.Platform;
import org.dasein.cloud.compute.VirtualMachine;
import org.dasein.cloud.digitalocean.DigitalOcean;
import org.dasein.cloud.digitalocean.models.Action;
import org.dasein.cloud.digitalocean.models.Droplet;
import org.dasein.cloud.digitalocean.models.Image;
import org.dasein.cloud.digitalocean.models.Images;
import org.dasein.cloud.digitalocean.models.actions.droplet.Snapshot;
import org.dasein.cloud.digitalocean.models.rest.DigitalOceanModelFactory;
import org.dasein.cloud.identity.ServiceAction;
import org.dasein.cloud.util.APITrace;
import org.dasein.cloud.util.Cache;
import org.dasein.cloud.util.CacheLevel;
import org.dasein.util.Jiterator;
import org.dasein.util.JiteratorPopulator;
import org.dasein.util.PopulatorThread;
import org.dasein.util.uom.time.Minute;
import org.dasein.util.uom.time.TimePeriod;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;

import static org.dasein.cloud.digitalocean.models.rest.DigitalOceanModelFactory.*;

public class DOImage extends AbstractImageSupport<DigitalOcean> {
    static private final Logger logger = Logger.getLogger(DOImage.class);

    private DigitalOcean provider = null;
    private volatile transient ImageCapabilities capabilities;

    public DOImage(DigitalOcean provider) {
        super(provider);
        this.provider = provider;
    }

    @Override
    public void addImageShare(@Nonnull String providerImageId, @Nonnull String accountNumber) throws CloudException, InternalException {
        throw new OperationNotSupportedException("Image sharing is not supported by "+getProvider().getCloudName());
    }

    @Override
    public void addPublicShare(@Nonnull String providerImageId) throws CloudException, InternalException {
        throw new OperationNotSupportedException("Image sharing is not supported by "+getProvider().getCloudName());
    }

    @Override
    public ImageCapabilities getCapabilities() {
        if (capabilities == null) {
            capabilities = new DOImageCapabilities(provider);
        }
        return capabilities;
    }

    @Override
    protected MachineImage capture(@Nonnull ImageCreateOptions options, @Nullable AsynchronousTask<MachineImage> task) throws CloudException, InternalException {
        APITrace.begin(getProvider(), "Image.capture");
        try {
            // TODO: replace with HEAD request (checkDropletExists)
            Droplet droplet = DigitalOceanModelFactory.getDropletByInstance(getProvider(), options.getVirtualMachineId());
            if( droplet == null ) {
                throw new InternalException("Virtual machine "+options.getVirtualMachineId()+" does not exist, unable to capture");
            }
            getProvider().getComputeServices().getVirtualMachineSupport().waitForAllDropletEventsToComplete(options.getVirtualMachineId(), 5);
            List<String> previousSnapshotIds = Arrays.asList(droplet.getSnapshotIds());
            Action action = performAction(getProvider(), new Snapshot(options.getName()), options.getVirtualMachineId());
            while( !action.isComplete() ) {
                try {
                    Thread.sleep(2000L);
                }
                catch( InterruptedException e ) {
                }
                if( action.isError() ) {
                    throw new CloudException(action.getStatus());
                }
                action = DigitalOceanModelFactory.getEventById(getProvider(), action.getId());
            }
            droplet = DigitalOceanModelFactory.getDropletByInstance(getProvider(), options.getVirtualMachineId());
            // create a new list as Arrays.asList returns an unmodifiable list
            List<String> newSnapshotIds = new ArrayList<String>();
            newSnapshotIds.addAll(Arrays.asList(droplet.getSnapshotIds()));
            newSnapshotIds.removeAll(previousSnapshotIds);

            // there may be more than one new snapshots so have to traverse to find the one
            for( String snapshotId : newSnapshotIds ) {
                Image image = (Image) DigitalOceanModelFactory.getModelById(getProvider(), org.dasein.cloud.digitalocean.models.rest.DigitalOcean.IMAGE, snapshotId);
                if( options.getName().equalsIgnoreCase(image.getName()) ) {
                    // that's *probably* the one
                    return toImage(image);
                }
            }
            // not found any snapshot :-/
            throw new CloudException("Unable to create or find the captured image for VM "+options.getVirtualMachineId());
        }
        finally {
            APITrace.end();
        }
    }

    @Override
    public @Nonnull Iterable<MachineImage> searchImages(String accountNumber, String keyword, Platform platform, Architecture architecture, ImageClass... imageClasses) throws CloudException, InternalException {
        APITrace.begin(getProvider(), "Image.searchImages");
        try {
            List<MachineImage> results = new ArrayList<MachineImage>();
            Collection<MachineImage> images = new ArrayList<MachineImage>();
            if (accountNumber == null) {
                images.addAll((Collection<MachineImage>) searchPublicImages(ImageFilterOptions.getInstance()));
            }
            images.addAll((Collection<MachineImage>) listImages(ImageFilterOptions.getInstance()));

            for (MachineImage image : images) {
                if (image != null) {
                    if (keyword != null) {
                        if (!image.getProviderMachineImageId().contains(keyword) && !image.getName().contains(keyword) && !image.getDescription().contains(keyword)) {
                            continue;
                        }
                    }
                    if (platform != null) {
                        Platform p = image.getPlatform();

                        if (!platform.equals(p)) {
                            if (platform.isWindows()) {
                                if (!p.isWindows()) {
                                    continue;
                                }
                            } else if (platform.equals(Platform.UNIX)) {
                                if (!p.isUnix()) {
                                    continue;
                                }
                            } else {
                                continue;
                            }
                        }
                    }
                    if (architecture != null) {
                        if (architecture != image.getArchitecture()) {
                            continue;
                        }
                    }
                    results.add(image);
                }
            }

            return results;
        } finally {
            APITrace.end();
        }
    }

    private @Nonnull Iterable<MachineImage> executeImageSearch(boolean publicImagesOnly, @Nonnull ImageFilterOptions options) throws CloudException, InternalException {
        APITrace.begin(provider, "Image.executeImageSearch");
        try {
            final String regionId = getContext().getRegionId();
            if( regionId == null ) {
                throw new CloudException("No region was set for this request");
            }

            Architecture architecture = options.getArchitecture();
            if( architecture != null && !architecture.equals(Architecture.I32) && !architecture.equals(Architecture.I64) ) {
                if( !options.isMatchesAny() ) {
                    return Collections.emptyList();
                }
            }

            String cacheName = "ALL";
            if( !options.getWithAllRegions() ) {
                cacheName = regionId;
            }
            Cache<MachineImage> cache = Cache.getInstance(provider, "images" + ( publicImagesOnly ? "pub" : "prv" ) + "-" + cacheName, MachineImage.class, CacheLevel.REGION_ACCOUNT, new TimePeriod<Minute>(5, TimePeriod.MINUTE));
            Collection<MachineImage> cachedImages = ( Collection<MachineImage> ) cache.get(getContext());
            if( cachedImages != null ) {
                return cachedImages;
            }
            final List<MachineImage> results = new ArrayList<MachineImage>();
            final org.dasein.cloud.digitalocean.models.rest.DigitalOcean cmd = publicImagesOnly ? org.dasein.cloud.digitalocean.models.rest.DigitalOcean.IMAGES_PUBLIC : org.dasein.cloud.digitalocean.models.rest.DigitalOcean.IMAGES;

            Images images = (Images) getModel(getProvider(), cmd);
            int total = images.getTotal();
            int page = 1;
            while( true ) {
                for( Image image : images.getImages() ) {
                    MachineImage machineImage = toImage(image);
                    // check if image regions match the requested region if any
                    if( !options.getWithAllRegions() && image.getRegions().length > 0 && !Arrays.asList(image.getRegions()).contains(getContext().getRegionId()) ) {
                        total--;
                        continue;
                    }

                    if( machineImage != null && publicImagesOnly == machineImage.isPublic() ) {
                        if( options.getWithAllRegions() ) {
                            // explode image to all regions, update total count
                            int regions = image.getRegions().length;
                            total += regions - 1;
                            for( String region : image.getRegions() ) {<commit_msg>FB7157: image listing has been corrected and reworked<commit_after>/**
 * Copyright (C) 2012-2015 Dell, Inc.
 * See annotations for authorship information
 *
 * ====================================================================
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * ====================================================================
 */

package org.dasein.cloud.digitalocean.compute;

import org.apache.log4j.Logger;
import org.dasein.cloud.AsynchronousTask;
import org.dasein.cloud.CloudException;
import org.dasein.cloud.InternalException;
import org.dasein.cloud.OperationNotSupportedException;
import org.dasein.cloud.ProviderContext;
import org.dasein.cloud.ResourceStatus;
import org.dasein.cloud.Tag;
import org.dasein.cloud.compute.AbstractImageSupport;
import org.dasein.cloud.compute.Architecture;
import org.dasein.cloud.compute.ImageCapabilities;
import org.dasein.cloud.compute.ImageClass;
import org.dasein.cloud.compute.ImageCreateOptions;
import org.dasein.cloud.compute.ImageFilterOptions;
import org.dasein.cloud.compute.MachineImage;
import org.dasein.cloud.compute.MachineImageState;
import org.dasein.cloud.compute.Platform;
import org.dasein.cloud.compute.VirtualMachine;
import org.dasein.cloud.digitalocean.DigitalOcean;
import org.dasein.cloud.digitalocean.models.Action;
import org.dasein.cloud.digitalocean.models.Droplet;
import org.dasein.cloud.digitalocean.models.Image;
import org.dasein.cloud.digitalocean.models.Images;
import org.dasein.cloud.digitalocean.models.actions.droplet.Snapshot;
import org.dasein.cloud.digitalocean.models.rest.DigitalOceanModelFactory;
import org.dasein.cloud.identity.ServiceAction;
import org.dasein.cloud.util.APITrace;
import org.dasein.cloud.util.Cache;
import org.dasein.cloud.util.CacheLevel;
import org.dasein.util.Jiterator;
import org.dasein.util.JiteratorPopulator;
import org.dasein.util.PopulatorThread;
import org.dasein.util.uom.time.Minute;
import org.dasein.util.uom.time.TimePeriod;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;

import static org.dasein.cloud.digitalocean.models.rest.DigitalOceanModelFactory.*;

public class DOImage extends AbstractImageSupport<DigitalOcean> {
    static private final Logger logger = Logger.getLogger(DOImage.class);

    private DigitalOcean provider = null;
    private volatile transient ImageCapabilities capabilities;

    public DOImage(DigitalOcean provider) {
        super(provider);
        this.provider = provider;
    }

    @Override
    public void addImageShare(@Nonnull String providerImageId, @Nonnull String accountNumber) throws CloudException, InternalException {
        throw new OperationNotSupportedException("Image sharing is not supported by "+getProvider().getCloudName());
    }

    @Override
    public void addPublicShare(@Nonnull String providerImageId) throws CloudException, InternalException {
        throw new OperationNotSupportedException("Image sharing is not supported by "+getProvider().getCloudName());
    }

    @Override
    public ImageCapabilities getCapabilities() {
        if (capabilities == null) {
            capabilities = new DOImageCapabilities(provider);
        }
        return capabilities;
    }

    @Override
    protected MachineImage capture(@Nonnull ImageCreateOptions options, @Nullable AsynchronousTask<MachineImage> task) throws CloudException, InternalException {
        APITrace.begin(getProvider(), "Image.capture");
        try {
            // TODO: replace with HEAD request (checkDropletExists)
            Droplet droplet = DigitalOceanModelFactory.getDropletByInstance(getProvider(), options.getVirtualMachineId());
            if( droplet == null ) {
                throw new InternalException("Virtual machine "+options.getVirtualMachineId()+" does not exist, unable to capture");
            }
            getProvider().getComputeServices().getVirtualMachineSupport().waitForAllDropletEventsToComplete(options.getVirtualMachineId(), 5);
            List<String> previousSnapshotIds = Arrays.asList(droplet.getSnapshotIds());
            Action action = performAction(getProvider(), new Snapshot(options.getName()), options.getVirtualMachineId());
            while( !action.isComplete() ) {
                try {
                    Thread.sleep(2000L);
                }
                catch( InterruptedException e ) {
                }
                if( action.isError() ) {
                    throw new CloudException(action.getStatus());
                }
                action = DigitalOceanModelFactory.getEventById(getProvider(), action.getId());
            }
            droplet = DigitalOceanModelFactory.getDropletByInstance(getProvider(), options.getVirtualMachineId());
            // create a new list as Arrays.asList returns an unmodifiable list
            List<String> newSnapshotIds = new ArrayList<String>();
            newSnapshotIds.addAll(Arrays.asList(droplet.getSnapshotIds()));
            newSnapshotIds.removeAll(previousSnapshotIds);

            // there may be more than one new snapshots so have to traverse to find the one
            for( String snapshotId : newSnapshotIds ) {
                Image image = (Image) DigitalOceanModelFactory.getModelById(getProvider(), org.dasein.cloud.digitalocean.models.rest.DigitalOcean.IMAGE, snapshotId);
                if( options.getName().equalsIgnoreCase(image.getName()) ) {
                    // that's *probably* the one
                    return toImage(image);
                }
            }
            // not found any snapshot :-/
            throw new CloudException("Unable to create or find the captured image for VM "+options.getVirtualMachineId());
        }
        finally {
            APITrace.end();
        }
    }

    @Override
    public @Nonnull Iterable<MachineImage> searchImages(String accountNumber, String keyword, Platform platform, Architecture architecture, ImageClass... imageClasses) throws CloudException, InternalException {
        APITrace.begin(getProvider(), "Image.searchImages");
        try {
            List<MachineImage> results = new ArrayList<MachineImage>();
            Collection<MachineImage> images = new ArrayList<MachineImage>();
            if (accountNumber == null) {
                images.addAll((Collection<MachineImage>) searchPublicImages(ImageFilterOptions.getInstance()));
            }
            images.addAll((Collection<MachineImage>) listImages(ImageFilterOptions.getInstance()));

            for (MachineImage image : images) {
                if (image != null) {
                    if (keyword != null) {
                        if (!image.getProviderMachineImageId().contains(keyword) && !image.getName().contains(keyword) && !image.getDescription().contains(keyword)) {
                            continue;
                        }
                    }
                    if (platform != null) {
                        Platform p = image.getPlatform();

                        if (!platform.equals(p)) {
                            if (platform.isWindows()) {
                                if (!p.isWindows()) {
                                    continue;
                                }
                            } else if (platform.equals(Platform.UNIX)) {
                                if (!p.isUnix()) {
                                    continue;
                                }
                            } else {
                                continue;
                            }
                        }
                    }
                    if (architecture != null) {
                        if (architecture != image.getArchitecture()) {
                            continue;
                        }
                    }
                    results.add(image);
                }
            }

            return results;
        } finally {
            APITrace.end();
        }
    }

    private @Nonnull Iterable<MachineImage> executeImageSearch(boolean publicImagesOnly, @Nonnull ImageFilterOptions options) throws CloudException, InternalException {
        APITrace.begin(provider, "Image.executeImageSearch");
        try {
            final String regionId = getContext().getRegionId();
            if( regionId == null ) {
                throw new CloudException("No region was set for this request");
            }

            Architecture architecture = options.getArchitecture();
            if( architecture != null && !architecture.equals(Architecture.I32) && !architecture.equals(Architecture.I64) ) {
                if( !options.isMatchesAny() ) {
                    return Collections.emptyList();
                }
            }

            String cacheName = "ALL";
            if( !options.getWithAllRegions() ) {
                cacheName = regionId;
            }
            Cache<MachineImage> cache = Cache.getInstance(provider, "images" + ( publicImagesOnly ? "pub" : "prv" ) + "-" + cacheName, MachineImage.class, CacheLevel.REGION_ACCOUNT, new TimePeriod<Minute>(5, TimePeriod.MINUTE));
            Collection<MachineImage> cachedImages = ( Collection<MachineImage> ) cache.get(getContext());
            if( cachedImages != null ) {
                return cachedImages;
            }
            final List<MachineImage> results = new ArrayList<MachineImage>();
            final org.dasein.cloud.digitalocean.models.rest.DigitalOcean cmd = publicImagesOnly ? org.dasein.cloud.digitalocean.models.rest.DigitalOcean.IMAGES_PUBLIC : org.dasein.cloud.digitalocean.models.rest.DigitalOcean.IMAGES;

            Images images = (Images) getModel(getProvider(), cmd);
            int total = images.getTotal();
            int page = 1;
            while( images.getImages().size() > 0 ) { // let's check >0 just in case
                for( Image image : images.getImages() ) {
                    MachineImage machineImage = toImage(image);
                    // check if image regions match the requested region if any
                    if( !options.getWithAllRegions() && image.getRegions().length > 0 && !Arrays.asList(image.getRegions()).contains(getContext().getRegionId()) ) {
                        total--;
                        continue;
                    }

                    if( machineImage != null && publicImagesOnly == machineImage.isPublic() ) {
                        if( options.getWithAllRegions() ) {
                            // explode image to all regions, update total count
                            int regions = image.getRegions().length;
                            total += regions - 1;
                            for( String region : image.getRegions() ) {
{'fraction_non_alphanumeric': 0.1069060040675568, 'fraction_numerical': 0.0025643292952515695, 'mean_word_length': 2.319002201027146, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6271824', 'n_tokens_mistral': 5732, 'n_tokens_neox': 5458, 'n_words': 1252}