<commit_before># Factories to create scenes from geotiffs on disk somewhere
import logging
import os
import tempfile

import boto3

from .create_thumbnails import create_thumbnails
from .create_images import create_geotiff_image
from .create_scenes import create_geotiff_scene
from .io import s3_url, s3_bucket_and_key_from_url
from .utils import is_tif_too_large, split_tif, upload_split_files
from rf.utils.io import Visibility, IngestStatus
from rf.uploads.landsat8.io import get_tempdir

logger = logging.getLogger(__name__)


class GeoTiffS3SceneFactory(object):
    """A convenience class for creating Scenes from an S3 folder of multiband GeoTiffs.

    Example usage:
    ```
        from rf.utils.io import Visibility

        captureDate = datetime.date(YYYY, MM, DD)

        factory = GeoTiffS3SceneFactory('<Upload Here>')
        for scene in factory.generate_scenes():
            # do something with the created scenes
            # Note that this will download GeoTIFFs locally, so it is best run somewhere with a fast
            # connection to S3
    ```
    """
    def __init__(self, upload):
        """Args:
            upload (Upload): instance of upload model to create scenes for
        """
        self._upload = upload
        self.isProjectUpload = upload.projectId is not None
        self.files = self._upload.files
        self.owner = upload.owner
        self.organizationId = self._upload.organizationId
        self.visibility = Visibility.PRIVATE
        self.datasource = self._upload.datasource
        self.acquisitionDate = self._upload.metadata.get('acquisitionDate')
        self.cloudCover = self._upload.metadata.get('cloudCover', 0)
        self.tags = self._upload.metadata.get('tags') or ['']

    def generate_scenes(self):
        """Create a Scene and associated Image for each GeoTiff in self.s3_path
        Returns:
            Generator of Scenes
        """
        s3 = boto3.resource('s3')
        for infile in self.files:
            # We can't use the temp file as a context manager because it'll be opened/closed multiple
            # times and by default is deleted when it's closed. So we use try/finally to ensure that
            # it gets cleaned up.
            local_tif = tempfile.NamedTemporaryFile(delete=False, suffix='.tif')
            try:
                bucket_name, key = s3_bucket_and_key_from_url(infile)
                bucket = s3.Bucket(bucket_name)
                bucket.download_file(key, local_tif.name)
                logger.info('Downloading %s => %s', infile, local_tif.name)
                # We need to override the autodetected filename because we're loading into temp
                # files which don't preserve the file name that is on S3.
                filename = os.path.basename(key)
                scene = self.create_geotiff_scene(local_tif.name, os.path.splitext(filename)[0])

                if is_tif_too_large(local_tif.name):
                    with get_tempdir() as tempdir:
                        split_files = split_tif(local_tif.name, tempdir)
                        keys_and_filepaths = upload_split_files(key, bucket_name, split_files)
<commit_msg>Upload split files to platform bucket rather than source bucket<commit_after># Factories to create scenes from geotiffs on disk somewhere
import logging
import os
import tempfile

import boto3

from .create_thumbnails import create_thumbnails
from .create_images import create_geotiff_image
from .create_scenes import create_geotiff_scene
from .io import s3_url, s3_bucket_and_key_from_url
from .utils import is_tif_too_large, split_tif, upload_split_files
from rf.utils.io import Visibility, IngestStatus
from rf.uploads.landsat8.io import get_tempdir

logger = logging.getLogger(__name__)


class GeoTiffS3SceneFactory(object):
    """A convenience class for creating Scenes from an S3 folder of multiband GeoTiffs.

    Example usage:
    ```
        from rf.utils.io import Visibility

        captureDate = datetime.date(YYYY, MM, DD)

        factory = GeoTiffS3SceneFactory('<Upload Here>')
        for scene in factory.generate_scenes():
            # do something with the created scenes
            # Note that this will download GeoTIFFs locally, so it is best run somewhere with a fast
            # connection to S3
    ```
    """
    def __init__(self, upload):
        """Args:
            upload (Upload): instance of upload model to create scenes for
        """
        self._upload = upload
        self.isProjectUpload = upload.projectId is not None
        self.files = self._upload.files
        self.owner = upload.owner
        self.organizationId = self._upload.organizationId
        self.visibility = Visibility.PRIVATE
        self.datasource = self._upload.datasource
        self.acquisitionDate = self._upload.metadata.get('acquisitionDate')
        self.cloudCover = self._upload.metadata.get('cloudCover', 0)
        self.tags = self._upload.metadata.get('tags') or ['']

    def generate_scenes(self):
        """Create a Scene and associated Image for each GeoTiff in self.s3_path
        Returns:
            Generator of Scenes
        """
        s3 = boto3.resource('s3')
        for infile in self.files:
            # We can't use the temp file as a context manager because it'll be opened/closed multiple
            # times and by default is deleted when it's closed. So we use try/finally to ensure that
            # it gets cleaned up.
            local_tif = tempfile.NamedTemporaryFile(delete=False, suffix='.tif')
            try:
                bucket_name, key = s3_bucket_and_key_from_url(infile)
                bucket = s3.Bucket(bucket_name)
                bucket.download_file(key, local_tif.name)
                logger.info('Downloading %s => %s', infile, local_tif.name)
                # We need to override the autodetected filename because we're loading into temp
                # files which don't preserve the file name that is on S3.
                filename = os.path.basename(key)
                scene = self.create_geotiff_scene(local_tif.name, os.path.splitext(filename)[0])

                if is_tif_too_large(local_tif.name):
                    with get_tempdir() as tempdir:
                        split_files = split_tif(local_tif.name, tempdir)
                        target_key = 'user-uploads/{USER}/{UPLOAD}/'.format(USER=self.owner, UPLOAD=self._upload.id)
                        keys_and_filepaths = upload_split_files(target_key, os.getenv('DATA_BUCKET'), split_files)

{'fraction_non_alphanumeric': 0.0757923748277446, 'fraction_numerical': 0.005205940897259225, 'mean_word_length': 2.5212938005390835, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '927377', 'n_tokens_mistral': 1893, 'n_tokens_neox': 1783, 'n_words': 538}