<jupyter_start><jupyter_text>OBSERVATION DATA SOURCES:TUSCON: 72274023160PHOENIX: 72278023183BURBAANK: 72288023152LOS ANGELES: 72295023174EL CENTRO: 72281023199BAKERSFIELD: 72384023155FRESNO: 72389093193VANDENBERG: 72393093214BRYCE CANYON: 72475623159SAN JOSE: 72494693232TRAVIS FIELD AFB: 74516023202NELLIS AFB:  74614023112SANTA BARBARA:  72392523190SANTA MARIA: 72394023273SAN DIEGO:  72290023188CAMP PENDLETON MCAS: 72292603154LONG BEACH: 72297023129<jupyter_code>stations={'72274023160','72278023183','72288023152','72295023174','72281023199','72384023155','72389093193','72393093214','72475623159','72494693232','74516023202',
'74614023112','72392523190','72394023273','72290023188','72292603154','72297023129'}
files = find_files_recursively(PATH, ('csv'))
found=set()

for file in files:
    file = Path(file)
    station = file.stem
    if station in stations:
        print('station was found! ' + station)
        found.add(station)
    else:
        file.unlink()
        
diffs = found.difference(stations)

for diff in diffs:
    print('station to remove from set: ' + diff)<jupyter_output>station was found! 72274023160
station was found! 72278023183
station was found! 72281023199
station was found! 72288023152
station was found! 72290023188
station was found! 72292603154
station was found! 72297023129
station was found! 72295023174
station was found! 72384023155
station was found! 72389093193
station was found! 72392523190
station was found! 72393093214
station was found! 72475623159
station was found! 72394023273
station was found! 72494693232
station was found! 74614023112
station was found! 74516023202
{'fraction_non_alphanumeric': 0.08466257668711656, 'fraction_numerical': 0.3441717791411043, 'mean_word_length': 6.91747572815534, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '605446', 'license': 'MIT', 'n_tokens_mistral': 961, 'n_tokens_neox': 615, 'n_words': 136, 'path': 'dive visibility/Preprocess.ipynb', 'repo_name': 'jantic/Deep-Learning-Projects', 'size': '1630'}