<commit_before>import calendar
import json
import os
import random
import requests
import shutil
import subprocess
import time
import unicodedata
import urllib
import xml.etree.ElementTree as ET

import numpy as np

import pyproj

from skimage import novice
from skimage import io
from skimage import transform
from skimage import exposure

import twitter


BASE_URL = "http://sentinel-s2-l1c.s3.amazonaws.com/"


def get_bands(flyby):
    """Get bands 2, 3 and 4 from AWS for this flyby"""
    directory_name = flyby.replace("/", "-")
    directory_name = '/tmp/%s' % directory_name

    # if the directory exists we assume the download was a success
    if os.path.exists(directory_name):
        return directory_name

    os.makedirs(directory_name)

    for i in (2, 3, 4):
        r = requests.get(BASE_URL + flyby + "B0%i.jp2" % i, stream=True)
        path = os.path.join(directory_name, "B0%i.jp2" % i)
        if r.status_code == 200:
            with open(path, 'wb') as f:
                r.raw.decode_content = True
                shutil.copyfileobj(r.raw, f)

    return directory_name


def process_bands(directory_name):
    output_image_fname = os.path.join(directory_name, "B.jpg")
    if os.path.exists(output_image_fname):
        return output_image_fname

    r = io.imread(directory_name + "/B04.jp2")
    g = io.imread(directory_name + "/B03.jp2")
    b = io.imread(directory_name + "/B02.jp2")

    r = transform.resize(r, (4000, 4000))
    r *= 0.93
    g = transform.resize(g, (4000, 4000))
    b = transform.resize(b, (4000, 4000))

    rgb = np.dstack((r,g,b))
    low, high = np.percentile(rgb, (1, 97))
    rgb = exposure.rescale_intensity(rgb, in_range=(low, high))
    #rgb = exposure.equalize_adapthist(rgb, clip_limit=0.03)
    rgb = transform.resize(rgb, (1098*2, 1098*2))

    io.imsave(output_image_fname, rgb, quality=90)
    return output_image_fname


def random_mgrs(seed=657):
    rng = random.Random(seed)
    gzd = rng.choice(range(1,61))
    sqid = rng.choice("CDEFGHJKLMNPQRSTUVWX")
    col = rng.choice("ABCDEFGHJKLMNPQRSTUVWXYZ")
    row = rng.choice("ABCDEFGHJKLMNPQRSTUV")
    return (gzd, sqid, "%s%s"% (col, row))


def get_listing(prefix):
    r = requests.get(BASE_URL + "?delimiter=/&prefix=%s" % prefix)
    text = r.text
    if 'CommonPrefixes' in text:
        root = ET.fromstring(r.text)
        prefixes = []
        for p in root.iter('{http://s3.amazonaws.com/doc/2006-03-01/}CommonPrefixes'):
            prefixes.append(p[0].text)
        return prefixes
    else:
        return []


def get_tileinfo(prefix):
    r = requests.get(BASE_URL + prefix + 'tileInfo.json')
    return json.loads(r.text)


def not_nan(x):
    return x[~np.isnan(x)]


def image_interestingness(prefix):
    """Calculate how visually interesting a picture is."""
    img = novice.open(BASE_URL + prefix).xy_array

    red = img[:,:,0].mean()
    green = img[:,:,1].mean()
    blue = img[:,:,2].mean()

    return (red + green) / 2 / blue


def get_position(geometry):
    """Extract coordinates of this image"""
    coords = geometry['coordinates'][0]
    crs = int(geometry['crs']['properties']['name'].split(":")[-1])
    projection = pyproj.Proj(init='epsg:%i' % crs)
    coords = [(projection(*c, inverse=True)[1], projection(*c, inverse=True)[0])
              for c in coords]
    return np.mean(coords, axis=0)


def get_address(lat, lng):
    """Convert latitude and longitude into an address using OSM"""
    def _norm_len(s):
        return len(unicodedata.normalize("NFC", s).encode('utf-8'))
    def _cut(s, max_len=72):
        if _norm_len(s) < max_len: return s
        while _norm_len(s) >= max_len:
            ss = s.split(",")
            s = ', '.join([x.strip() for x in ss[1:]])
        return s

    # otherwise we get unicode mixed with latin which often exceeds
    # the 140character limit of twitter :(
    headers = {'Accept-Language': "en-US,en;q=0.8"}
    nominatim_url = ("http://nominatim.openstreetmap.org/reverse?lat=%f&lon=%f&"
                     "addressdetails=0&format=json&zoom=6&extratags=0")
    info = json.loads(requests.get(nominatim_url % (lat, lng),
                                   headers=headers).text)
    if 'error' in info:
        return 'Unknown location, do you know it? Tell @openstreetmap'

    return _cut(info['display_name'])


def format_lat_lng(lat, lng):
    s = ""
    if lat < 0:
        s += "%.1f°S" % abs(lat)
    else:
        s += "%.1f°N" % abs(lat)

    s += " "

    if lng < 0:
        s += "%.1f°W" % abs(lng)
    else:
        s += "%.1f°E" % abs(lng)

    return s


def post_candidate(flyby, post=False, api=None):
    tile_info = get_tileinfo(flyby)

    lat, lng = get_position(tile_info['tileGeometry'])

    coverage = float(tile_info.get('dataCoveragePercentage', 0.))
    complete = coverage > 99
    cloudy_pixels = float(tile_info.get('cloudyPixelPercentage', 0.))
    cloudy =  cloudy_pixels > 35.
    interestingness = image_interestingness(flyby + "preview.jpg")

    MSG = "{location} ({lat_lng}), {date}"

    print('coverage:', coverage, 'clouds:', cloudy_pixels,
          '(R+G)/2/B %.1f' % interestingness)
    if (complete and not cloudy and (interestingness > 0.95)):
        print('Cloudy: %.2f Coverage: %.2f' % (cloudy_pixels, coverage))
        print('(R+G)/2/B %.1f' % interestingness)
        print(lat, lng, get_address(lat, lng))
        parts = flyby.split('/')
        day = parts[-3]
        month = calendar.month_name[int(parts[-4])]
        year = parts[-5]
        print(MSG.format(date="%s %s %s" %(day, month, year),
                         lat_lng=format_lat_lng(lat, lng),<commit_msg>Drop the interestingness threshold<commit_after>import calendar
import json
import os
import random
import requests
import shutil
import subprocess
import time
import unicodedata
import urllib
import xml.etree.ElementTree as ET

import numpy as np

import pyproj

from skimage import novice
from skimage import io
from skimage import transform
from skimage import exposure

import twitter


BASE_URL = "http://sentinel-s2-l1c.s3.amazonaws.com/"


def get_bands(flyby):
    """Get bands 2, 3 and 4 from AWS for this flyby"""
    directory_name = flyby.replace("/", "-")
    directory_name = '/tmp/%s' % directory_name

    # if the directory exists we assume the download was a success
    if os.path.exists(directory_name):
        return directory_name

    os.makedirs(directory_name)

    for i in (2, 3, 4):
        r = requests.get(BASE_URL + flyby + "B0%i.jp2" % i, stream=True)
        path = os.path.join(directory_name, "B0%i.jp2" % i)
        if r.status_code == 200:
            with open(path, 'wb') as f:
                r.raw.decode_content = True
                shutil.copyfileobj(r.raw, f)

    return directory_name


def process_bands(directory_name):
    output_image_fname = os.path.join(directory_name, "B.jpg")
    if os.path.exists(output_image_fname):
        return output_image_fname

    r = io.imread(directory_name + "/B04.jp2")
    g = io.imread(directory_name + "/B03.jp2")
    b = io.imread(directory_name + "/B02.jp2")

    r = transform.resize(r, (4000, 4000))
    r *= 0.93
    g = transform.resize(g, (4000, 4000))
    b = transform.resize(b, (4000, 4000))

    rgb = np.dstack((r,g,b))
    low, high = np.percentile(rgb, (1, 97))
    rgb = exposure.rescale_intensity(rgb, in_range=(low, high))
    #rgb = exposure.equalize_adapthist(rgb, clip_limit=0.03)
    rgb = transform.resize(rgb, (1098*2, 1098*2))

    io.imsave(output_image_fname, rgb, quality=90)
    return output_image_fname


def random_mgrs(seed=657):
    rng = random.Random(seed)
    gzd = rng.choice(range(1,61))
    sqid = rng.choice("CDEFGHJKLMNPQRSTUVWX")
    col = rng.choice("ABCDEFGHJKLMNPQRSTUVWXYZ")
    row = rng.choice("ABCDEFGHJKLMNPQRSTUV")
    return (gzd, sqid, "%s%s"% (col, row))


def get_listing(prefix):
    r = requests.get(BASE_URL + "?delimiter=/&prefix=%s" % prefix)
    text = r.text
    if 'CommonPrefixes' in text:
        root = ET.fromstring(r.text)
        prefixes = []
        for p in root.iter('{http://s3.amazonaws.com/doc/2006-03-01/}CommonPrefixes'):
            prefixes.append(p[0].text)
        return prefixes
    else:
        return []


def get_tileinfo(prefix):
    r = requests.get(BASE_URL + prefix + 'tileInfo.json')
    return json.loads(r.text)


def not_nan(x):
    return x[~np.isnan(x)]


def image_interestingness(prefix):
    """Calculate how visually interesting a picture is."""
    try:
        img = novice.open(BASE_URL + prefix).xy_array
    except (requests.HTTPError, urllib.error.HTTPError):
        return -1.

    red = img[:,:,0].mean()
    green = img[:,:,1].mean()
    blue = img[:,:,2].mean()

    return (red + green) / 2 / blue


def get_position(geometry):
    """Extract coordinates of this image"""
    coords = geometry['coordinates'][0]
    crs = int(geometry['crs']['properties']['name'].split(":")[-1])
    projection = pyproj.Proj(init='epsg:%i' % crs)
    coords = [(projection(*c, inverse=True)[1], projection(*c, inverse=True)[0])
              for c in coords]
    return np.mean(coords, axis=0)


def get_address(lat, lng):
    """Convert latitude and longitude into an address using OSM"""
    def _norm_len(s):
        return len(unicodedata.normalize("NFC", s).encode('utf-8'))
    def _cut(s, max_len=72):
        if _norm_len(s) < max_len: return s
        while _norm_len(s) >= max_len:
            ss = s.split(",")
            s = ', '.join([x.strip() for x in ss[1:]])
        return s

    # otherwise we get unicode mixed with latin which often exceeds
    # the 140character limit of twitter :(
    headers = {'Accept-Language': "en-US,en;q=0.8"}
    nominatim_url = ("http://nominatim.openstreetmap.org/reverse?lat=%f&lon=%f&"
                     "addressdetails=0&format=json&zoom=6&extratags=0")
    info = json.loads(requests.get(nominatim_url % (lat, lng),
                                   headers=headers).text)
    if 'error' in info:
        return 'Unknown location, do you know it? Tell @openstreetmap'

    return _cut(info['display_name'])


def format_lat_lng(lat, lng):
    s = ""
    if lat < 0:
        s += "%.1f°S" % abs(lat)
    else:
        s += "%.1f°N" % abs(lat)

    s += " "

    if lng < 0:
        s += "%.1f°W" % abs(lng)
    else:
        s += "%.1f°E" % abs(lng)

    return s


def post_candidate(flyby, post=False, api=None):
    tile_info = get_tileinfo(flyby)

    lat, lng = get_position(tile_info['tileGeometry'])

    coverage = float(tile_info.get('dataCoveragePercentage', 0.))
    complete = coverage > 99
    cloudy_pixels = float(tile_info.get('cloudyPixelPercentage', 0.))
    cloudy =  cloudy_pixels > 35.
    interestingness = image_interestingness(flyby + "preview.jpg")

    MSG = "{location} ({lat_lng}), {date}"

    print('coverage:', coverage, 'clouds:', cloudy_pixels,
          '(R+G)/2/B %.1f' % interestingness)
    if (complete and not cloudy and (interestingness > 0.8)):
        print('Cloudy: %.2f Coverage: %.2f' % (cloudy_pixels, coverage))
        print('(R+G)/2/B %.1f' % interestingness)
        print(lat, lng, get_address(lat, lng))
        parts = flyby.split('/')
        day = parts[-3]
        month = calendar.month_name[int(parts[-4])]
        year = parts[-5]
        print(MSG.format(date="%s %s %s" %(day, month, year),
                         lat_lng=format_lat_lng(lat, lng),
{'fraction_non_alphanumeric': 0.14745331813798546, 'fraction_numerical': 0.022968352765845534, 'mean_word_length': 2.908187735525865, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1010327', 'n_tokens_mistral': 4248, 'n_tokens_neox': 3998, 'n_words': 908}