<commit_before>"""
I convert raw GINI noaaport imagery into geo-referenced PNG files both in the
'native' projection and 4326.

Questions? <NAME> <EMAIL>
"""

# https://github.com/akrherz/pyIEM
from pyiem.nws import gini
import pytz
import cStringIO
import sys
from PIL import Image
import datetime
import logging
import os
import tempfile
import json
import numpy as np
import subprocess

FORMAT = "%(asctime)-15s:[" + str(os.getpid()) + "]: %(message)s"
LOG_FN = ('logs/gini2gis-%s.log'
          ) % (datetime.datetime.utcnow().strftime("%Y%m%d"), )
logging.basicConfig(filename=LOG_FN, filemode='a+', format=FORMAT)
logger = logging.getLogger()
logger.addHandler(logging.StreamHandler())
logger.setLevel(logging.INFO)

PQINSERT = "/home/ldm/bin/pqinsert"


def process_input():
    """
    Process what was provided to use by LDM on stdin
    @return GINIZFile instance
    """
    cstr = cStringIO.StringIO()
    cstr.write(sys.stdin.read())
    cstr.seek(0)
    sat = gini.GINIZFile(cstr)
    logger.info(str(sat))
    return sat


def do_legacy_ir(sat, tmpfn):
    """ since some are unable to process non-grayscale """
    print("Doing legacy IR junk...")
    png = Image.fromarray(sat.data[:-1, :])
    png.save('%s.png' % (tmpfn,))

    # World File
    out = open('%s.wld' % (tmpfn,), 'w')
    fmt = """%(dx).3f
0.0<commit_msg>fix legacy IR for PIL usage<commit_after>"""
I convert raw GINI noaaport imagery into geo-referenced PNG files both in the
'native' projection and 4326.

Questions? <NAME> <EMAIL>
"""

# https://github.com/akrherz/pyIEM
from pyiem.nws import gini
import pytz
import cStringIO
import sys
from PIL import Image
import datetime
import logging
import os
import tempfile
import json
import numpy as np
import subprocess

FORMAT = "%(asctime)-15s:[" + str(os.getpid()) + "]: %(message)s"
LOG_FN = ('logs/gini2gis-%s.log'
          ) % (datetime.datetime.utcnow().strftime("%Y%m%d"), )
logging.basicConfig(filename=LOG_FN, filemode='a+', format=FORMAT)
logger = logging.getLogger()
logger.addHandler(logging.StreamHandler())
logger.setLevel(logging.INFO)

PQINSERT = "/home/ldm/bin/pqinsert"


def process_input():
    """
    Process what was provided to use by LDM on stdin
    @return GINIZFile instance
    """
    cstr = cStringIO.StringIO()
    cstr.write(sys.stdin.read())
    cstr.seek(0)
    sat = gini.GINIZFile(cstr)
    logger.info(str(sat))
    return sat


def do_legacy_ir(sat, tmpfn):
    """ since some are unable to process non-grayscale """
    print("Doing legacy IR junk...")
    png = Image.fromarray(np.array(sat.data[:-1, :], np.uint8))
    png.save('%s.png' % (tmpfn,))

    # World File
    out = open('%s.wld' % (tmpfn,), 'w')
    fmt = """%(dx).3f
0.0
{'fraction_non_alphanumeric': 0.15166051660516605, 'fraction_numerical': 0.00922509225092251, 'mean_word_length': 3.849731663685152, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2598735', 'n_tokens_mistral': 1008, 'n_tokens_neox': 971, 'n_words': 243}