<commit_before>"""This module runs the w2besttrack algorithm with smart IO.

For more on w2besttrack, see best_tracks.py.

"Smart IO" means that, regardless of the time period to be processed, this
method holds only 3 SPC dates in memory at once.  Specifically, while working on
SPC date k, this method holds dates (k - 1)...(k + 1) in memory.  Holding
adjacent dates in memory prevents arbitrary cutoffs, where all tracks appear to
end when an SPC date ends.  Since no thunderstorm lasts > 24 hours (one day),
holding only dates (k - 1)...(k + 1) is sufficient.  If thunderstorms lasted up
to 48 hours, for example, we would need to hold dates (k - 2)...(k + 2).
"""

import os
import copy
import pickle
import tempfile
import numpy
import pandas
from gewittergefahr.gg_io import storm_tracking_io as tracking_io
from gewittergefahr.gg_utils import best_tracks
from gewittergefahr.gg_utils import projections
from gewittergefahr.gg_utils import time_conversion
from gewittergefahr.gg_utils import time_periods
from gewittergefahr.gg_utils import file_system_utils
from gewittergefahr.gg_utils import error_checking

DAYS_TO_SECONDS = 86400
TIME_FORMAT_FOR_MESSAGES = '%Y-%m-%d-%H%M%S'
PROJECTION_OBJECT = projections.init_azimuthal_equidistant_projection(35., 265.)

SPC_DATES_KEY = 'spc_dates_unix_sec'
TEMP_FILE_NAMES_KEY = 'temp_file_names_key'
INPUT_FILE_NAMES_KEY = 'input_file_names_by_spc_date'
OUTPUT_FILE_NAMES_KEY = 'output_file_names_by_spc_date'

INTERMEDIATE_COLUMNS = [
    tracking_io.STORM_ID_COLUMN, best_tracks.ORIG_STORM_ID_COLUMN,
    tracking_io.TIME_COLUMN, tracking_io.SPC_DATE_COLUMN,
    best_tracks.CENTROID_X_COLUMN, best_tracks.CENTROID_Y_COLUMN,
    best_tracks.FILE_INDEX_COLUMN]

<commit_msg>Had to fix one line in best_tracks_smart_io.py.<commit_after>"""This module runs the w2besttrack algorithm with smart IO.

For more on w2besttrack, see best_tracks.py.

"Smart IO" means that, regardless of the time period to be processed, this
method holds only 3 SPC dates in memory at once.  Specifically, while working on
SPC date k, this method holds dates (k - 1)...(k + 1) in memory.  Holding
adjacent dates in memory prevents arbitrary cutoffs, where all tracks appear to
end when an SPC date ends.  Since no thunderstorm lasts > 24 hours (one day),
holding only dates (k - 1)...(k + 1) is sufficient.  If thunderstorms lasted up
to 48 hours, for example, we would need to hold dates (k - 2)...(k + 2).
"""

import os
import copy
import pickle
import tempfile
import numpy
import pandas
from gewittergefahr.gg_io import storm_tracking_io as tracking_io
from gewittergefahr.gg_utils import best_tracks
from gewittergefahr.gg_utils import projections
from gewittergefahr.gg_utils import time_conversion
from gewittergefahr.gg_utils import time_periods
from gewittergefahr.gg_utils import file_system_utils
from gewittergefahr.gg_utils import error_checking

DAYS_TO_SECONDS = 86400
TIME_FORMAT_FOR_MESSAGES = '%Y-%m-%d-%H%M%S'
PROJECTION_OBJECT = projections.init_azimuthal_equidistant_projection(35., 265.)

SPC_DATES_KEY = 'spc_dates_unix_sec'
TEMP_FILE_NAMES_KEY = 'temp_file_names_key'
INPUT_FILE_NAMES_KEY = 'input_file_names_by_spc_date'
OUTPUT_FILE_NAMES_KEY = 'output_file_names_by_spc_date'

INTERMEDIATE_COLUMNS = [
    tracking_io.STORM_ID_COLUMN, tracking_io.ORIG_STORM_ID_COLUMN,
    tracking_io.TIME_COLUMN, tracking_io.SPC_DATE_COLUMN,
    best_tracks.CENTROID_X_COLUMN, best_tracks.CENTROID_Y_COLUMN,
    best_tracks.FILE_INDEX_COLUMN]


{'fraction_non_alphanumeric': 0.06547960941987363, 'fraction_numerical': 0.013210798391728892, 'mean_word_length': 5.685220729366603, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3513100', 'n_tokens_mistral': 1262, 'n_tokens_neox': 1199, 'n_words': 350}