<commit_before># USGS Landsat Imagery Util
#
#
# Author: developmentseed
# Contributer: scisco
#
# License: CC0 1.0 Universal

import os
import subprocess
from zipfile import ZipFile
import tarfile

from general_helper import check_create_folder


class GsHelper(object):

    def __init__(self, settings):
        self.scene_file_url = settings.SCENE_FILE_URL
        self.download_dir = settings.DOWNLOAD_DIR
        self.zip_dir = settings.ZIP_DIR
        self.unzip_dir = settings.UNZIP_DIR
        self.scene_file = settings.SCENE_FILE

        # Keep the number of images found on Google search
        # based on the search parameters
        self.found = 0

        check_create_folder(self.download_dir)

    #################
    # Public Methods
    #################

    def search(self, query, date_rng):
        files = []
        self.__fetch_gs_scence_list()
        file = open(self.scene_file, 'r')
# 		for q in query:
# 			print date_rng
        files.extend(
            self.__search_scene_list(scene=file,
                                     query=query,
                                     date_rng=date_rng))

        return files

    def download(self, image_list):
        self.__download_images(image_list)

    def unzip(self):
        self.__unzip_images()

    #################
    # Private Methods
    #################

    def __fetch_gs_scence_list(self):

        if not os.path.isfile(self.scene_file):
            # Download the file
            subprocess.call(
                ["gsutil", "cp", "-n",
                 self.scene_file_url, "%s.zip" % self.scene_file])

            # Unzip the file
            zip = ZipFile('%s.zip' % self.scene_file, 'r')
            zip.extractall(path=self.download_dir)
            zip.close()

            print "scene_file unziped"

# 		return open(self.scene_file, 'r')

    def __search_scene_list(self, scene, query, date_rng=None):<commit_msg>Added support for single image download<commit_after># USGS Landsat Imagery Util
#
#
# Author: developmentseed
# Contributer: scisco
#
# License: CC0 1.0 Universal

import os
import subprocess
from zipfile import ZipFile
import tarfile

from general_helper import check_create_folder


class GsHelper(object):

    def __init__(self, settings):
        self.scene_file_url = settings.SCENE_FILE_URL
        self.download_dir = settings.DOWNLOAD_DIR
        self.zip_dir = settings.ZIP_DIR
        self.unzip_dir = settings.UNZIP_DIR
        self.scene_file = settings.SCENE_FILE
        self.source_url = settings.SOURCE_URL

        # Keep the number of images found on Google search
        # based on the search parameters
        self.found = 0

        check_create_folder(self.download_dir)

    #################
    # Public Methods
    #################

    def search(self, query, date_rng=None):
        files = []
        self.__fetch_gs_scence_list()
        file = open(self.scene_file, 'r')
#       for q in query:
#           print date_rng
        files.extend(
            self.__search_scene_list(scene=file,
                                     query=query,
                                     date_rng=date_rng))

        return files

    def download_single(self, sat_type='L8', row, path, name):
        url = '%s/%s/%s/%s/%s.tar.bz' % (self.source_url,
                                         sat_type,
                                         row,
                                         path,
                                         name)

        subprocess.call(
            ["gsutil", "cp", "-n", url,
             "%s/%s" % (self.zip_dir, name)])

    def download(self, image_list):
        self.__download_images(image_list)

    def unzip(self):
        self.__unzip_images()

    #################
    # Private Methods
    #################

    def __fetch_gs_scence_list(self):

        if not os.path.isfile(self.scene_file):
            # Download the file
            subprocess.call(
                ["gsutil", "cp", "-n",
                 self.scene_file_url, "%s.zip" % self.scene_file])

            # Unzip the file
            zip = ZipFile('%s.zip' % self.scene_file, 'r')
            zip.extractall(path=self.download_dir)
            zip.close()

            print "scene_file unziped"

#       return open(self.scene_file, 'r')

    def __search_scene_list(self, scene, query, date_rng=None):
{'fraction_non_alphanumeric': 0.11829184745375657, 'fraction_numerical': 0.0020552637588490525, 'mean_word_length': 1.9514824797843666, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5354851', 'n_tokens_mistral': 1333, 'n_tokens_neox': 1259, 'n_words': 279}