<commit_before>
import os
import re
from time import asctime

import raster
from ancillary import finder, run, dissolve
from spatial import haversine

# path = "/media/john/Data/DATA/Sentinel"
# path_tiling = "/media/john/Data/DATA/Sentinel/tiling_test"
# path = "/geonfs02_vol1/baci_fsu/02_Sentinel/03_FTS/02_Viterbo_Italy/04_output"
path = "/homes4/geoinf/ve39vem/RADAR/Sentinel/test_out"
path_tiling = "/homes4/geoinf/ve39vem/RADAR/Sentinel/tiling"
files = finder(path, ["*vv*.tif"])

for filename in files:
    print os.path.basename(filename), asctime()

    data = raster.Raster(filename)

    lat = [data.geo["ymin"], data.geo["ymax"]]
    lon = [data.geo["xmin"], data.geo["xmax"]]

    lat = [int(x//1) for x in lat]
    lon = [int(x//1) for x in lon]

    lat = range(min(lat), max(lat)+1)
    lon = range(min(lon), max(lon)+1)

    resampling = "bilinear"
    interleave = "BAND"

    # targetres = data.res
    warpoptions = ["gdalwarp", "-overwrite", "--config", "GDAL_CACHEMAX", 2000, "-wm", 6000, "-multi",
                   "-r", resampling, "-of", "GTiff", "-co", "INTERLEAVE="+interleave]

    for x in lon:
        for y in lat:
            tilename = "{0}{1}{2}{3}".format("N" if y > 0 else "S", str(y).zfill(2), "W" if x < 0 else "E", str(x).zfill(3))

            outpath = os.path.join(path_tiling, tilename)

            lat_center = y+.5
            lon_center = x+.5
            post_north = haversine(lat_center, lon_center, lat_center+data.res[1], lon_center)
            post_east = haversine(lat_center, lon_center, lat_center, lon_center+data.res[0])

            targetres = ["-tr", data.res[0]*(post_north/post_east), data.res[1]]

            if not os.path.isdir(outpath):
                os.makedirs(outpath)

            timestamp = re.search("[0-9]{8}_[0-9]{6}", filename).group().replace("_", "T")
            # dtime = datetime.strptime(timestamp, "%Y%m%dT%H%M%S")

            outname = os.path.join(outpath, tilename+"_"+timestamp+".tif")
            extent_arg = ["-te", x, y, x+1, y+1]

            # print "resample:", asctime()
            run(dissolve([warpoptions, extent_arg, targetres, filename, outname]))

            stats = raster.Raster(outname).allstats[0]

            # check whether resulting file contains valid pixels; if not delete it otherwise reduce it to lines and columns containing any number of valid data entries
            if stats is None:
                for item in finder(os.path.dirname(outname), [os.path.basename(outname)+"*"]):
                    os.remove(item)
            # else:
            #     # print "load raster:", asctime()
            #     rast = raster.Raster(outname)
            #     # print "reduce:", asctime()
            #     rast.reduce()
            #
            #     def fun(x):
            #         return x*1000
            #
            #     # print "rescale:", asctime()
            #     rast.rescale(fun)
            #     # print "write:", asctime()
            #     rast.write(outname=outname)
                # rast.write(outname=outname, dtype="Int16")
                # print "done:", asctime()
                # print "--------"

            if os.listdir(outpath) == []:
                os.rmdir(outpath)


def mosaic(directory, timestamp, outname, format="GTiff"):
    if format == "GTiff" and not outname.endswith(".tif"):
        outname += ".tif"
    files = finder(directory, [timestamp+"$"], regex=True)
    run(dissolve(["gdalwarp", "-overwrite", "--config", "GDAL_CACHEMAX", 2000, "-wm", 6000, "-multi", "-of", format, files, outname]))<commit_msg>changed raster dependency to package spatial syntactically simplified subprocess command handling<commit_after>import os
import re
from time import asctime

from ancillary import finder, run
from spatial import haversine, raster

# path = "/media/john/Data/DATA/Sentinel"
# path_tiling = "/media/john/Data/DATA/Sentinel/tiling_test"
# path = "/geonfs02_vol1/baci_fsu/02_Sentinel/03_FTS/02_Viterbo_Italy/04_output"
path = "/homes4/geoinf/ve39vem/RADAR/Sentinel/test_out"
path_tiling = "/homes4/geoinf/ve39vem/RADAR/Sentinel/tiling"
files = finder(path, ["*vv*.tif"])

for filename in files:
    print os.path.basename(filename), asctime()

    data = raster.Raster(filename)

    lat = [data.geo["ymin"], data.geo["ymax"]]
    lon = [data.geo["xmin"], data.geo["xmax"]]

    lat = [int(x//1) for x in lat]
    lon = [int(x//1) for x in lon]

    lat = range(min(lat), max(lat)+1)
    lon = range(min(lon), max(lon)+1)

    resampling = "bilinear"
    interleave = "BAND"

    # targetres = data.res
    warpoptions = ["gdalwarp", "-overwrite", "--config", "GDAL_CACHEMAX", 2000, "-wm", 6000, "-multi",
                   "-r", resampling, "-of", "GTiff", "-co", "INTERLEAVE="+interleave]

    for x in lon:
        for y in lat:
            tilename = "{0}{1}{2}{3}".format("N" if y > 0 else "S", str(y).zfill(2), "W" if x < 0 else "E", str(x).zfill(3))

            outpath = os.path.join(path_tiling, tilename)

            lat_center = y+.5
            lon_center = x+.5
            post_north = haversine(lat_center, lon_center, lat_center+data.res[1], lon_center)
            post_east = haversine(lat_center, lon_center, lat_center, lon_center+data.res[0])

            targetres = ["-tr", data.res[0]*(post_north/post_east), data.res[1]]

            if not os.path.isdir(outpath):
                os.makedirs(outpath)

            timestamp = re.search("[0-9]{8}_[0-9]{6}", filename).group().replace("_", "T")
            # dtime = datetime.strptime(timestamp, "%Y%m%dT%H%M%S")

            outname = os.path.join(outpath, tilename+"_"+timestamp+".tif")
            extent_arg = ["-te", x, y, x+1, y+1]

            # print "resample:", asctime()
            run([warpoptions, extent_arg, targetres, filename, outname])

            stats = raster.Raster(outname).allstats[0]

            # check whether resulting file contains valid pixels; if not delete it otherwise reduce it to lines and columns containing any number of valid data entries
            if stats is None:
                for item in finder(os.path.dirname(outname), [os.path.basename(outname)+"*"]):
                    os.remove(item)
            # else:
            #     # print "load raster:", asctime()
            #     rast = raster.Raster(outname)
            #     # print "reduce:", asctime()
            #     rast.reduce()
            #
            #     def fun(x):
            #         return x*1000
            #
            #     # print "rescale:", asctime()
            #     rast.rescale(fun)
            #     # print "write:", asctime()
            #     rast.write(outname=outname)
                # rast.write(outname=outname, dtype="Int16")
                # print "done:", asctime()
                # print "--------"

            if os.listdir(outpath) == []:
                os.rmdir(outpath)


def mosaic(directory, timestamp, outname, format="GTiff"):
    if format == "GTiff" and not outname.endswith(".tif"):
        outname += ".tif"
    files = finder(directory, [timestamp+"$"], regex=True)
    run(["gdalwarp", "-overwrite", "--config", "GDAL_CACHEMAX", 2000, "-wm", 6000, "-multi", "-of", format, files, outname])
{'fraction_non_alphanumeric': 0.15956708755376717, 'fraction_numerical': 0.018315526571388928, 'mean_word_length': 2.3186003683241254, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3898868', 'n_tokens_mistral': 2549, 'n_tokens_neox': 2382, 'n_words': 522}