<jupyter_start><jupyter_text>notebook for changing filenames of dupe hires tilesA handful of tiles in the HIRES mosaic have duplicate pds3 product IDs.This goes through the data set and changes their filenames and pds4LIDs to make them unique.<jupyter_code>import fs

from clem_bulk import crude_time_log
from clem_conversion import *
standard_mosaic_products = pd.read_csv(
    './directories/clementine/standard_mosaic_products.csv'
)
duped_tiles = standard_mosaic_products.loc[
    standard_mosaic_products.duplicated(subset=['pds3_product_id'], keep=False)
]
# set root directories for your input and output data sets
input_root = '/home/ubuntu/buckets/clem_input/'
output_root = '/home/ubuntu/buckets/clem_reprocess/'
# this roundabout temp directory thing is only necessary to prevent gdal from breaking
# when it tries to write directly to a s3fs-fuse 'filesystem'. it would have
# no purpose in other configurations.
temp_output_directory = '/home/ubuntu/data_temp/'

# associate hires source index with an index of edr products that includes time
# in order to determine start and stop times for each hires mosaic tile
hires_source_groupby = pd.read_csv(
    './directories/clementine/hires_source_index.csv',
).groupby('tilename')
# convert all the files in this chunk 
for ix, tile in enumerate(duped_tiles.itertuples()):
    print("Converting " + fs.path.split(tile.file)[1])
    
    tile_start_time = dt.datetime.now() # just for logging
    
    source_path = tile.file
    source_groups = hires_source_groupby

    destination_path = tile.newpath
    # just a double-check to avoid any possible extra '/' in S3 object names --
    # in some cases it can cause really irritating problems.
    if output_root.endswith('/') and destination_path.startswith('/'):
        destination_path = destination_path[1:]
    sh.mkdir("-p", output_root + destination_path)
    volume = source_path[3:7]
    
    # initialize writer & convert product 
    writer = ClemMosaicConverter(
        input_root + source_path,
        source_groups = source_groups
    )
    
    # in case you wrote out unflagged versions before
    for extension in ['.xml', '.tif']:
        try:
            os.remove(
                output_root[:-1] + destination_path + writer.pds4_root + extension, 
            )
        except FileNotFoundError:
            print('no existing file')
            continue
    
    writer.pds4_root = writer.pds4_root + '_' + volume
    writer.write_pds4(temp_output_directory)
    

    for extension in ['.xml', '.tif']:
        sh.mv(
            temp_output_directory + writer.pds4_root + extension, 
            output_root + destination_path,
            _bg=True
        )

    # very simple logger
    elapsed = str((dt.datetime.now() - tile_start_time).total_seconds())
    crude_time_log(
        'mosaic_conversion_log_hires_dupes.csv',
        writer,
        elapsed
    )
    print("total seconds: " + elapsed)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.0848199259508583, 'fraction_numerical': 0.005048805116122518, 'mean_word_length': 3.2701149425287355, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '334409', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 923, 'n_tokens_neox': 883, 'n_words': 248, 'path': 'src/dupe_hires_tile_fix.ipynb', 'repo_name': 'MillionConcepts/clementine-conversion', 'size': '2971'}