<commit_before>#!/usr/bin/env python
# encoding: utf-8

DATA_DIR ="C:\\BigData\\defra-lidar\\LIDAR-DSM-2M-SJ46"
    
# Subdirectory
# LIDAR-DTM-2M-SJ46
# Filenames like: 
# sj4060_DTM_2m.asc, 
# sj4{0-9}{60-69}_DTM_2m.asc
#ncols        500
#nrows        500
#xllcorner    340000
#yllcorner    360000
#cellsize     2
#NODATA_value  -9999

from collections import OrderedDict

LOGLINE_TEMPLATE = OrderedDict([
    ('name', None),
    ('nrows', None),
    ('ncols', None),
    ('xllcorner', None),
    ('yllcorner', None),
    ('cellsize', None),
    ('NODATA_value', None),
])

from os import listdir
import os.path

import numpy as np

from matplotlib import pyplot as plt
import matplotlib

from mpl_toolkits.mplot3d import Axes3D
from matplotlib.ticker import LinearLocator, FormatStrFormatter


def main():
    datafiles = listdir(DATA_DIR)
    print("Found {} datafiles".format(len(datafiles)))

#    for item in datafiles:
        # Open file
        

            #print(log_line)

    # Select a file and then load it into an array
    # 8 - includes home!
    currentfile = datafiles[7]

    metadata = get_header_info(currentfile)
    print(metadata)

    data = get_image(currentfile)

    print("Number of NODATA_values: {}".format(np.sum(data == -9999)))
    data[data == -9999] = np.nan
    print("Minimum value found: {}".format(np.nanmin(data)))
    print("Maximum value found: {}".format(np.nanmax(data)))
    # Show the data
    # plot_image(data)
    plot_surface(data)

def plot_image(data):
    plt.imshow(data, interpolation='nearest', cmap=plt.gray())
    plt.show()

def plot_surface(data):
    # generate 3D sample data

    fig = plt.figure()
    ax = fig.gca(projection='3d')
    X = np.arange(0,500)
    Y = np.arange(0,500)
    X, Y = np.meshgrid(X, Y)
    surf = ax.plot_surface(X, Y, data, linewidth=0, antialiased=False)
    # ax.set_zlim(-1.01, 1.01)
<commit_msg>Start putting individual tiles into one big array, trivial case<commit_after>#!/usr/bin/env python
# encoding: utf-8

DATA_DIR ="C:\\BigData\\defra-lidar\\LIDAR-DSM-2M-SJ46"
    
# Subdirectory
# LIDAR-DTM-2M-SJ46
# Filenames like: 
# sj4060_DTM_2m.asc, 
# sj4{0-9}{60-69}_DTM_2m.asc
#ncols        500
#nrows        500
#xllcorner    340000
#yllcorner    360000
#cellsize     2
#NODATA_value  -9999

from collections import OrderedDict

LOGLINE_TEMPLATE = OrderedDict([
    ('name', None),
    ('nrows', None),
    ('ncols', None),
    ('xllcorner', None),
    ('yllcorner', None),
    ('cellsize', None),
    ('NODATA_value', None),
])

from os import listdir
import os.path

import numpy as np

from matplotlib import pyplot as plt
import matplotlib

from mpl_toolkits.mplot3d import Axes3D
from matplotlib.ticker import LinearLocator, FormatStrFormatter


def main():
    datafiles = listdir(DATA_DIR)
    print("Found {} datafiles".format(len(datafiles)))

    # Select a file and then load it into an array
    # 8 - includes home!
    currentfile = datafiles[7]

    metadata = get_header_info(currentfile)
    print(metadata)

    data = get_image(currentfile)

    print("Number of NODATA_values: {}".format(np.sum(data == -9999)))
    print("Minimum value found: {}".format(np.nanmin(data)))
    print("Maximum value found: {}".format(np.nanmax(data)))

    filelist = [0]

    bigdata = np.zeros((5000,5000), dtype=np.float)
    for idx in filelist:
        # Get data
        metadata = get_header_info(datafiles[idx])
        data = get_image(datafiles[idx])        
        data[data == -9999] = np.nan
        # Calculate x,y offset
        xoffset = 0
        yoffset = 5000
        width = 500
        height = 500
        # Write into array
        bigdata[yoffset - height:yoffset, xoffset:xoffset + width] = data 
    # Show the data
    plot_image(bigdata)
    #plot_surface(data)

def plot_image(data):
    plt.imshow(data, interpolation='nearest', cmap=plt.gray())
    plt.show()

def plot_surface(data):
    # generate 3D sample data

    fig = plt.figure()
    ax = fig.gca(projection='3d')
    X = np.arange(0,500)
    Y = np.arange(0,500)
    X, Y = np.meshgrid(X, Y)
    surf = ax.plot_surface(X, Y, data, linewidth=0, antialiased=False)
    # ax.set_zlim(-1.01, 1.01)

{'fraction_non_alphanumeric': 0.13293556085918853, 'fraction_numerical': 0.03914081145584725, 'mean_word_length': 3.1168958742632613, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1713454', 'n_tokens_mistral': 1639, 'n_tokens_neox': 1516, 'n_words': 350}