<jupyter_start><jupyter_text>Removing invalid days, hours and years<jupyter_code>def is_in_range(range, value):
    return value in range
def is_not_in_range(range, value):
    return value not in range
_days_not_in_range_filter = data_frame[columns.JULIAN_DAY].map(lambda value: is_not_in_range(_DAYS_RANGE, value))
data_frame[_days_not_in_range_filter][columns.JULIAN_DAY].unique()
_days_in_range_filter = data_frame[columns.JULIAN_DAY].map(lambda value: is_in_range(_DAYS_RANGE, value))
data_frame = data_frame[_days_in_range_filter]
_years_not_in_range_filter = data_frame[columns.YEAR].map(lambda value: is_not_in_range(_YEARS_RANGE, value))
data_frame[_years_not_in_range_filter][columns.YEAR].unique()
_years_in_range_filter = data_frame[columns.YEAR].map(lambda value: is_in_range(_YEARS_RANGE, value))
data_frame = data_frame[_years_in_range_filter]
_hours_not_in_range_filter = data_frame[columns.HOUR_MINUTE].map(lambda value: is_not_in_range(_HOURS_RANGE, value))

data_frame[_hours_not_in_range_filter][columns.HOUR_MINUTE].unique()
_hours_in_range_filter = data_frame[columns.HOUR_MINUTE].map(lambda value: is_in_range(_HOURS_RANGE, value))
data_frame = data_frame[_hours_in_range_filter]
data_frame.shape<jupyter_output><empty_output><jupyter_text>Removing null values<jupyter_code>data_frame.isnull().sum()
data_frame.shape
data_frame.dropna().shape
data_frame = data_frame.dropna()
data_frame.isnull().sum()
data_frame.shape<jupyter_output><empty_output><jupyter_text>Removing days with value "0"<jupyter_code>data_frame[data_frame[columns.JULIAN_DAY]==0]
data_frame = data_frame[data_frame[columns.JULIAN_DAY]!=0]
data_frame.shape<jupyter_output><empty_output><jupyter_text>Removing data from 222<jupyter_code>data_frame[columns.CODE].unique()
data_frame[data_frame[columns.CODE]!=111].head()
data_frame = data_frame[data_frame[columns.CODE]==111]
data_frame.shape<jupyter_output><empty_output><jupyter_text>Removing invalid Air Temperature measurements<jupyter_code>_MIN_AIR_TEMPERATURE=-2
_MAX_AIR_TEMPERATURE=45
data_frame[columns.TEMPERATURE].describe()
data_frame_without_invalid_air_temp = data_frame.loc[(data_frame[columns.TEMPERATURE] > _MIN_AIR_TEMPERATURE) & (data_frame[columns.TEMPERATURE] < _MAX_AIR_TEMPERATURE), :]
data_frame_without_invalid_air_temp[columns.TEMPERATURE].describe()
data_frame.shape[0] - data_frame_without_invalid_air_temp.shape[0]
data_frame = data_frame_without_invalid_air_temp<jupyter_output><empty_output><jupyter_text>Removing invalid Air moisture measurements<jupyter_code>_MIN_AIR_MOISTURE=0
_MAX_AIR_MOISTURE=100
data_frame[columns.HUMIDITY].describe()
data_frame_without_invalid_air_mois = data_frame.loc[(data_frame[columns.HUMIDITY] >= _MIN_AIR_MOISTURE) & (data_frame[columns.HUMIDITY] <= _MAX_AIR_MOISTURE), :]
data_frame_without_invalid_air_mois[columns.HUMIDITY].describe()
data_frame.shape[0] - data_frame_without_invalid_air_mois.shape[0]
data_frame = data_frame_without_invalid_air_mois<jupyter_output><empty_output><jupyter_text>Removing invalid Air pressure measurements<jupyter_code>_MIN_AIR_PRESSURE=822
_MAX_AIR_PRESSURE=1030
data_frame[columns.PRESSURE].describe()
data_frame_without_invalid_air_press = data_frame.loc[(data_frame[columns.PRESSURE] >= _MIN_AIR_PRESSURE) & (data_frame[columns.PRESSURE] <= _MAX_AIR_PRESSURE), :]
data_frame_without_invalid_air_press[columns.PRESSURE].describe()
data_frame.shape[0] - data_frame_without_invalid_air_press.shape[0]
data_frame = data_frame_without_invalid_air_press<jupyter_output><empty_output><jupyter_text>Removing invalid Wind Directionmeasurements<jupyter_code>_MIN_WIND_DIRECTION=0
_MAX_WIND_DIRECTION=360
data_frame[columns.WIND_DIRECTION].describe()
data_frame_without_invalid_wind_dir = data_frame.loc[(data_frame[columns.WIND_DIRECTION] >= _MIN_WIND_DIRECTION) & (data_frame[columns.WIND_DIRECTION] <= _MAX_WIND_DIRECTION), :]
data_frame_without_invalid_wind_dir[columns.WIND_DIRECTION].describe()
data_frame.shape[0] - data_frame_without_invalid_wind_dir.shape[0]
# I'm commenting this for now to avoid removing valid temperature and pression measurements.
#data_frame = data_frame_without_invalid_wind_dir<jupyter_output><empty_output><jupyter_text>Results after cleaning<jupyter_code>original_amount_of_lines = original_shape[0]
amount_of_lines_after_cleaning = data_frame.shape[0]
amount_of_lines_removed = original_amount_of_lines - amount_of_lines_after_cleaning

percentage_of_lines_removed = 100 * amount_of_lines_removed / original_amount_of_lines

percentage_of_lines_removed = round(percentage_of_lines_removed, 2)

print(str(amount_of_lines_removed) + " observations were removed what is " + str(percentage_of_lines_removed) + " of the dataset")<jupyter_output>3804 observations were removed what is 0.0 of the dataset<jupyter_text>Creating the date column<jupyter_code>year_mapping = data_frame[columns.YEAR].map(lambda year : datetime(year, 1, 1))
time_delta_mapping = data_frame[columns.JULIAN_DAY].map(lambda day : timedelta(day-1))
data_frame[columns.DATE] = year_mapping + time_delta_mapping<jupyter_output>/home/felipe/.local/lib/python2.7/site-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  This is separate from the ipykernel package so we can avoid doing imports until<jupyter_text>Saving CSV<jupyter_code>directory_utils.create_if_doesnt_exist(datasets.FIRST_CLEAN_DIRECTORY_PATH)
data_frame.to_csv(datasets.FIRST_CLEAN_DATA_SET_111_PATH, index = False, encoding=datasets.DEFAULT_CSV_ENCODING)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09425087108013937, 'fraction_numerical': 0.010278745644599303, 'mean_word_length': 11.673289183222957, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '119089', 'license': 'MIT', 'n_tokens_mistral': 2127, 'n_tokens_neox': 2065, 'n_words': 255, 'path': '2. Cleaning 111 dataset.ipynb', 'repo_name': 'imdavi/climate-health-data-analysis', 'size': '5740'}