<commit_before>use tile::{Tile,Bbox};
use std::cmp::min;
use std::f32;

pub const TILE_SIZE: usize = 256;

/// Holds data and provides methods to regrid data into a 256 x 256 grid.
#[derive(Debug)]
pub struct TileData {
    /// Must be expressed in meters, in ascending order
    pub lat: Vec<f64>,
    /// Must be expressed in meters, in ascending order
    pub lon: Vec<f64>,
    /// Values must be a flattened array (lat, lon)
    pub values: Vec<f32>,
    pub bbox: Bbox,
    pub tile: Tile,
}
impl TileData {

    /**
     * regrid self.values into a TILE_SIZE x TILE_SIZE grid.
     */
    pub fn to_tile_grid(&self) -> [[f32; TILE_SIZE]; TILE_SIZE] {

        // fetch nearest latitudes indices:
        // loop on GRID latitudes, and for each of those, find the closest one in self.lat,
        // and store its index in an array (lat_ids)
        let mut lat_ids: [usize; TILE_SIZE] = [0; TILE_SIZE];
        let lat_inc: f64 = (self.bbox.north - self.bbox.south) / (TILE_SIZE as f64);

        let mut current_idx: usize = 0;
        let mut current_lat: f64 = self.bbox.south;
        for i in 0..TILE_SIZE {
            // If the closest lat value is the biggest one, 
            // it won't change over the next iteration
            if current_idx == self.lat.len() -1 {
                for following_i in i..TILE_SIZE {
                    lat_ids[following_i] = current_idx;
                }
                break;
            }
            while (self.lat[current_idx] - current_lat).abs() >  (self.lat[current_idx + 1] - current_lat).abs() {
                current_idx += 1;
                if current_idx == self.lat.len() -1 {
                    break;
                }
            }
            lat_ids[i] = current_idx;
            current_lat += lat_inc;
        }

        // fetch nearest longitude indices
        //   loop on the tile grid longitude , and for each of those, find the closest one in self.lon,
        //   and store its index in an array (lon_ids)
        let mut lon_ids: [usize; TILE_SIZE] = [0; TILE_SIZE];
        let lon_inc: f64 = (self.bbox.east - self.bbox.west) / (TILE_SIZE as f64);

        let mut current_idx: usize = 0;
        let mut current_lon: f64 = self.bbox.west;
        for i in 0..TILE_SIZE {
            if current_idx == self.lon.len() -1 {
                for following_i in i..TILE_SIZE {
                    lon_ids[following_i] = current_idx;
                }
                break;
            }
            while (self.lon[current_idx] - current_lon).abs() >  (self.lon[current_idx + 1] - current_lon).abs() {
                current_idx += 1;
                if current_idx == self.lon.len() -1 {
                    break;
                }
            }
            lon_ids[i] = current_idx;
            current_lon += lon_inc;
        }

        // Exclude coordinates outside the dataset bounding box
        //   (Check "how far" are the actual latitudes values from the requested ones (self.bbox))
        let mut data_lat_min: usize = 0;
        let mut boundary = self.bbox.south + lat_inc * 1.5;
        while self.lat[lat_ids[data_lat_min]] > boundary && data_lat_min < (lat_ids.len() - 1) {
            boundary += lat_inc;
            data_lat_min += 1;
        }
        let mut data_lat_max: usize = lat_ids.len() - 1;
        boundary = self.bbox.north - lat_inc * 1.5;
        while self.lat[lat_ids[data_lat_max]] < boundary && data_lat_max > 0 {
            boundary -= lat_inc;
            data_lat_max -= 1;
        }
        //   (Check "how far" are the actual longitude values from the requested ones (self.bbox))
        let mut data_lon_min: usize = 0;
        boundary = self.bbox.west + lon_inc * 1.5;
        while self.lon[lon_ids[data_lon_min]] > boundary && data_lon_min < (lon_ids.len() - 1) {
            boundary += lon_inc;
            data_lon_min += 1;
        }
        let mut data_lon_max: usize = lon_ids.len() - 1;
        boundary = self.bbox.east - lon_inc * 1.5;
        while self.lon[lon_ids[data_lon_max]] < boundary && data_lon_max > 0 {
            boundary -= lon_inc;
            data_lon_max -= 1;
        }

        // pick values using precomputed indices
        let mut values: [[f32; TILE_SIZE]; TILE_SIZE] = [[f32::NAN; TILE_SIZE]; TILE_SIZE];
        for i_lat in 0..TILE_SIZE {
            for i_lon in 0..TILE_SIZE {
                if i_lat < data_lat_min 
                    || i_lat > data_lat_max 
                    || i_lon < data_lon_min 
                    || i_lon > data_lon_max {
                    values[i_lat][i_lon] = f32::NAN;
                } else {
                    values[i_lat][i_lon] = self.value_at(lat_ids[i_lat], lon_ids[i_lon]);
                }
            }
        }
        values
    }

    #[inline]
    /// Return the value of self.values as if it was a bi-dimensional array.
    fn value_at(&self, lat_idx: usize, lon_idx: usize) -> f32 {
        return self.values[self.lon.len() * lat_idx + lon_idx];
    }

    /// Creates up to 4 tiles, representing the n+1 zoom level using self.values
    pub fn sub_tiledata(&self) -> Vec<Self> {
        let mut sub_tiledata: Vec<Self> = Vec::new();
        let base_x = self.tile.x * 2;
        let base_y = self.tile.y * 2;
        let z = self.tile.z + 1;
        for x in  base_x..(base_x + 2) {
            for y in base_y..(base_y + 2) {
                let tile = Tile { x: x, y: y, z: z };
                let xy = tile.xy_bounds();

                // get min latitude index
                let mut i_lat_min: usize = 0;
                for (i, lat) in self.lat.iter().enumerate() {
                    if *lat >= xy.south {
                        i_lat_min = if i > 0 { i - 1 } else { 0 };
                        break;
                    }
                }
                // get max latitude index
                let mut i_lat_max: usize = 0;
                for (i, lat) in self.lat.iter().rev().enumerate() {
                    if *lat <= xy.north {
                        i_lat_max = if i != (self.lat.len() - 1 ) { self.lat.len() - i } else { self.lat.len() -1 };
                        break;
                    }
                }
                
                // get min longitude index
                let mut i_lon_min: usize = 0;
                for (i, lon) in self.lon.iter().enumerate() {
                    if *lon >= xy.west {
                        i_lon_min = if i > 0 { i - 1 } else { 0 };
                        break;
                    }
                }
                // get max longitude index
                let mut i_lon_max: usize = 0;
                for (i, lon) in self.lon.iter().rev().enumerate() {
                    if *lon <= xy.east {
                        i_lon_max = if i != (self.lon.len() - 1 ) { self.lon.len() - i } else { self.lon.len() -1 };
                        break;
                    }
                }

                // Extract lat, lon and values using the computed indices
                let subset_lat: Vec<f64> = self.lat[i_lat_min..min(i_lat_max +1, self.lat.len() -1)].to_vec();
                let subset_lon: Vec<f64> = self.lon[i_lon_min..min(i_lon_max +1, self.lon.len() -1)].to_vec();
                let mut subset_values: Vec<f32> = Vec::with_capacity(subset_lat.len() * subset_lon.len());
                for i_lat in i_lat_min..min(i_lat_max +1, self.lat.len() -1) {
                    for i_lon in i_lon_min..min(i_lon_max +1, self.lon.len() -1) {
                        subset_values.push(self.value_at(i_lat, i_lon));
                    }
                }
                
                sub_tiledata.push(
                    Self {
                        lon: subset_lon,
                        lat: subset_lat,
                        values: subset_values,
                        bbox: xy,
                        tile: tile
                    }
                );
            }
        }
        sub_tiledata
    }
}
<commit_msg>Removed out of bound mask<commit_after>use tile::{Tile,Bbox};
use std::cmp::min;
use std::f32;

pub const TILE_SIZE: usize = 256;

/// Holds data and provides methods to regrid data into a 256 x 256 grid.
#[derive(Debug)]
pub struct TileData {
    /// Must be expressed in meters, in ascending order
    pub lat: Vec<f64>,
    /// Must be expressed in meters, in ascending order
    pub lon: Vec<f64>,
    /// Values must be a flattened array (lat, lon)
    pub values: Vec<f32>,
    pub bbox: Bbox,
    pub tile: Tile,
}
impl TileData {

    /**
     * regrid self.values into a TILE_SIZE x TILE_SIZE grid.
     */
    pub fn to_tile_grid(&self) -> [[f32; TILE_SIZE]; TILE_SIZE] {

        // fetch nearest latitudes indices:
        // loop on GRID latitudes, and for each of those, find the closest one in self.lat,
        // and store its index in an array (lat_ids)
        let mut lat_ids: [usize; TILE_SIZE] = [0; TILE_SIZE];
        let lat_inc: f64 = (self.bbox.north - self.bbox.south) / (TILE_SIZE as f64);

        let mut current_idx: usize = 0;
        let mut current_lat: f64 = self.bbox.south;
        for i in 0..TILE_SIZE {
            // If the closest lat value is the biggest one, 
            // it won't change over the next iteration
            if current_idx == self.lat.len() -1 {
                for following_i in i..TILE_SIZE {
                    lat_ids[following_i] = current_idx;
                }
                break;
            }
            while (self.lat[current_idx] - current_lat).abs() >  (self.lat[current_idx + 1] - current_lat).abs() {
                current_idx += 1;
                if current_idx == self.lat.len() -1 {
                    break;
                }
            }
            lat_ids[i] = current_idx;
            current_lat += lat_inc;
        }

        // fetch nearest longitude indices
        //   loop on the tile grid longitude , and for each of those, find the closest one in self.lon,
        //   and store its index in an array (lon_ids)
        let mut lon_ids: [usize; TILE_SIZE] = [0; TILE_SIZE];
        let lon_inc: f64 = (self.bbox.east - self.bbox.west) / (TILE_SIZE as f64);

        let mut current_idx: usize = 0;
        let mut current_lon: f64 = self.bbox.west;
        for i in 0..TILE_SIZE {
            if current_idx == self.lon.len() -1 {
                for following_i in i..TILE_SIZE {
                    lon_ids[following_i] = current_idx;
                }
                break;
            }
            while (self.lon[current_idx] - current_lon).abs() >  (self.lon[current_idx + 1] - current_lon).abs() {
                current_idx += 1;
                if current_idx == self.lon.len() -1 {
                    break;
                }
            }
            lon_ids[i] = current_idx;
            current_lon += lon_inc;
        }

        // pick values using precomputed indices
        let mut values: [[f32; TILE_SIZE]; TILE_SIZE] = [[f32::NAN; TILE_SIZE]; TILE_SIZE];
        for i_lat in 0..TILE_SIZE {
            for i_lon in 0..TILE_SIZE {
                values[i_lat][i_lon] = self.value_at(lat_ids[i_lat], lon_ids[i_lon]);
            }
        }
        values
    }

    #[inline]
    /// Return the value of self.values as if it was a bi-dimensional array.
    fn value_at(&self, lat_idx: usize, lon_idx: usize) -> f32 {
        return self.values[self.lon.len() * lat_idx + lon_idx];
    }

    /// Creates up to 4 tiles, representing the n+1 zoom level using self.values
    pub fn sub_tiledata(&self) -> Vec<Self> {
        let mut sub_tiledata: Vec<Self> = Vec::new();
        let base_x = self.tile.x * 2;
        let base_y = self.tile.y * 2;
        let z = self.tile.z + 1;
        for x in  base_x..(base_x + 2) {
            for y in base_y..(base_y + 2) {
                let tile = Tile { x: x, y: y, z: z };
                let xy = tile.xy_bounds();

                // get min latitude index
                let mut i_lat_min: usize = 0;
                for (i, lat) in self.lat.iter().enumerate() {
                    if *lat >= xy.south {
                        i_lat_min = if i > 0 { i - 1 } else { 0 };
                        break;
                    }
                }
                // get max latitude index
                let mut i_lat_max: usize = 0;
                for (i, lat) in self.lat.iter().rev().enumerate() {
                    if *lat <= xy.north {
                        i_lat_max = if i != (self.lat.len() - 1 ) { self.lat.len() - i } else { self.lat.len() -1 };
                        break;
                    }
                }
                
                // get min longitude index
                let mut i_lon_min: usize = 0;
                for (i, lon) in self.lon.iter().enumerate() {
                    if *lon >= xy.west {
                        i_lon_min = if i > 0 { i - 1 } else { 0 };
                        break;
                    }
                }
                // get max longitude index
                let mut i_lon_max: usize = 0;
                for (i, lon) in self.lon.iter().rev().enumerate() {
                    if *lon <= xy.east {
                        i_lon_max = if i != (self.lon.len() - 1 ) { self.lon.len() - i } else { self.lon.len() -1 };
                        break;
                    }
                }

                // Extract lat, lon and values using the computed indices
                let subset_lat: Vec<f64> = self.lat[i_lat_min..min(i_lat_max +1, self.lat.len() -1)].to_vec();
                let subset_lon: Vec<f64> = self.lon[i_lon_min..min(i_lon_max +1, self.lon.len() -1)].to_vec();
                let mut subset_values: Vec<f32> = Vec::with_capacity(subset_lat.len() * subset_lon.len());
                for i_lat in i_lat_min..min(i_lat_max +1, self.lat.len() -1) {
                    for i_lon in i_lon_min..min(i_lon_max +1, self.lon.len() -1) {
                        subset_values.push(self.value_at(i_lat, i_lon));
                    }
                }
                
                sub_tiledata.push(
                    Self {
                        lon: subset_lon,
                        lat: subset_lat,
                        values: subset_values,
                        bbox: xy,
                        tile: tile
                    }
                );
            }
        }
        sub_tiledata
    }
}

{'fraction_non_alphanumeric': 0.11214693196048421, 'fraction_numerical': 0.013774871295394462, 'mean_word_length': 1.314442118821446, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6151641', 'n_tokens_mistral': 4670, 'n_tokens_neox': 4363, 'n_words': 1209}