processed_files_file = os.path.join(path, 'processed_files.txt')
processed_files = set(line.strip() for line in open(processed_files_file))
with open(processed_files_file, 'a') as pff:
    for root, dirs, files in os.walk(path):
        for file in files:
            if file.endswith('.gz'):
                if file not in processed_files:
                    pff.write('%s\n' % file)
How to process only new (unprocessed) files in linux