<jupyter_start><jupyter_text>Global Wind Speed at 80 Meters	ene.018	http://www.vaisala.com/en/energy/Renewable-Energy-Consulting-Services/Wind-Energy-Assessment-Services/Pages/Wind-Maps-and-GIS-Layers.aspx Import libraries<jupyter_code># Libraries for downloading data from remote server (may be ftp)
import requests
from urllib.request import urlopen
from contextlib import closing
import shutil

# Library for uploading/downloading data to/from S3
import boto3

# Libraries for handling data
import rasterio as rio
import numpy as np
# from netCDF4 import Dataset
# import pandas as pd
# import scipy

# Libraries for various helper functions
# from datetime import datetime
import os
import threading
import sys
from glob import glob<jupyter_output><empty_output><jupyter_text>s3 tools<jupyter_code>s3_upload = boto3.client("s3")
s3_download = boto3.resource("s3")

s3_bucket = "wri-public-data"
s3_folder = "resourcewatch/raster/*"

s3_file = "*.tif"

s3_key_orig = s3_folder + s3_file
s3_key_edit = s3_key_orig[0:-4] + "_edit.tif"


class ProgressPercentage(object):
        def __init__(self, filename):
            self._filename = filename
            self._size = float(os.path.getsize(filename))
            self._seen_so_far = 0
            self._lock = threading.Lock()

        def __call__(self, bytes_amount):
            # To simplify we'll assume this is hooked up
            # to a single filename.
            with self._lock:
                self._seen_so_far += bytes_amount
                percentage = (self._seen_so_far / self._size) * 100
                sys.stdout.write("\r%s  %s / %s  (%.2f%%)"%(
                        self._filename, self._seen_so_far, self._size,
                        percentage))
                sys.stdout.flush()<jupyter_output><empty_output><jupyter_text>Potentially useful for online directories<jupyter_code># View files in a ftp, see cli.015
remote_path = "ftp://cidportal.jrc.ec.europa.eu/jrc-opendata/EDGAR/datasets/v431_v2"
remote_path_BC = remote_path + "/BC/TOTALS/"
file = req.urlopen(remote_path_BC).read().splitlines()

# Copy from https or ftp
with(closing(urlopen(online_folder + most_recent))) as r:
    with(open(local_orig, 'wb')) as f:
        shutil.copyfileobj(r, f)<jupyter_output><empty_output><jupyter_text>Define local file locations<jupyter_code>local_folder = "/Users/nathansuberi/Desktop/RW_Data/Rasters/*"
file_name = "*"]

local_orig = local_folder + file_name

orig_extension_length = 4 #4 for each char in .tif
local_edit = local_orig[:-orig_extension_length] + "edit.tif"<jupyter_output><empty_output><jupyter_text>Use rasterio to reproject and compress<jupyter_code># Note - this is the core of Vizz's netcdf2tif function

with rio.open(local_orig, 'r') as src:
    # This assumes data is readable by rasterio
    # May need to open instead with netcdf4.Dataset, for example
   
    data = src.read()[0]
    
    rows = data.shape[0]
    columns = data.shape[1]
    
    print(rows)
    print(columns)
    
    # Latitude bounds
    south_lat = -90
    north_lat = 90

    # Longitude bounds
    west_lon = -180
    east_lon = 180
    
    transform = rasterio.transform.from_bounds(west_lon, south_lat, east_lon, north_lat, columns, rows)
    
    # Profile
    
    no_data_val = *
    target_projection = 'EPSG:4326'
    target_data_type = np.float64
    
    profile = {
        'driver':'GTiff', 
        'height':rows, 
        'width':columns, 
        'count':1, 
        'dtype':target_data_type, 
        'crs':target_projection, 
        'transform':transform, 
        'compress':'lzw', 
        'nodata': no_data_val
    }
    
    with rio.open(local_edit, "w", **profile) as dst:
        dst.write(data.astype(profile["dtype"]), 1)<jupyter_output><empty_output><jupyter_text>Upload orig and edit files to s3<jupyter_code># Original
s3_upload.upload_file(local_orig, s3_bucket, s3_key_orig,
                         Callback=ProgressPercentage(local_orig))

# Edit
s3_upload.upload_file(local_edit, s3_bucket, s3_key_edit,
                         Callback=ProgressPercentage(local_edit))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10337296772628003, 'fraction_numerical': 0.016743508857073527, 'mean_word_length': 3.2582644628099175, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '15919', 'license': 'MIT', 'n_tokens_mistral': 1455, 'n_tokens_neox': 1384, 'n_words': 297, 'path': 'ene.018.ipynb', 'repo_name': 'nsuberi/RW_Raster_Getters', 'size': '4121'}