<jupyter_start><jupyter_text>Create xample data<jupyter_code># Create xample data
df = pd.DataFrame({'data':['cd_schufa_de201904_001',
                           'cd_schufa_de201904_002',
                           'cd_schufa_de201905_001',
                           'cd_schufa_de201905_002',
                           'rdm_20190701_de',
                           'rdm_20190701_se',
                           'rdm_20190702_de',
                           'rdm_20190702_se',
                           'rdm_20190703_de',
                           'rdm_20190703_se',
                           'cd_rdm_2017_all',
                           'cd_rdm_2018_all',
                           'old_one_1999',
                           'old_one_199912',
                           'old_one_19991230']})
# Display xample data
df<jupyter_output><empty_output><jupyter_text>Solution 1  (1) Apply regular expression<jupyter_code># Create column 'data_clean1' from 'data'
df['data_clean1'] = df['data']

# Perform regular expression on 'data_clean1'
df['data_clean1'] = df['data_clean1'].map(lambda x: re.sub("\d{8}","<ymd8>",x))
df['data_clean1'] = df['data_clean1'].map(lambda x: re.sub("\d{6}","<ym6>",x))
df['data_clean1'] = df['data_clean1'].map(lambda x: re.sub("\d{4}","<y4>",x))

# Display df
df<jupyter_output><empty_output><jupyter_text>(2) Filter data<jupyter_code># Create DataFrame 'df_solution1' as a sorted copy of 'df'
df_solution1 = df[['data', 'data_clean1']].copy().sort_values(['data_clean1', 'data'])

# Keep only the last occurence of the sorted df
# - keep='last': all duplicates EXCEPT LAST OCCURENCE will be marked as TRUE
df_solution1 = df_solution1[~df_solution1.duplicated('data_clean1', keep='last')]

# Display 'df_solution1'
df_solution1<jupyter_output><empty_output><jupyter_text>Solution 2  (1) Apply regular expression<jupyter_code># Create column 'data_clean2' from 'data'
df['data_clean2'] = df['data']

# Perform regular expression on 'data_clean2'
df['data_clean2'] = df['data_clean2'].map(lambda x: re.sub("20\d\d[0-1]\d[0-3]\d","<ymd8>",x))
df['data_clean2'] = df['data_clean2'].map(lambda x: re.sub("20\d\d[0-1]\d","<ym6>",x))
df['data_clean2'] = df['data_clean2'].map(lambda x: re.sub("20\d\d","<y4>",x))

# Display df
df<jupyter_output><empty_output><jupyter_text>(2) Filter data<jupyter_code># Create DataFrame 'df_solution2' as a sorted copy of 'df'
df_solution2 = df[['data', 'data_clean2']].copy().sort_values(['data_clean2', 'data'])

# Keep only the last occurence of the sorted df
# - keep='last': all duplicates EXCEPT LAST OCCURENCE will be marked as TRUE
df_solution2 = df_solution2[~df_solution2.duplicated('data_clean2', keep='last')]

# Display 'df_solution2'
df_solution2<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1513277555474718, 'fraction_numerical': 0.06365951255001819, 'mean_word_length': 3.4212218649517685, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '256984', 'license': 'MIT', 'n_tokens_mistral': 1091, 'n_tokens_neox': 991, 'n_words': 172, 'path': 'Python-techniques/RegEx_Find-and-Replace-Dates.ipynb', 'repo_name': 'j-honnacker/data-mgmt-Python', 'size': '2749'}