<jupyter_start><jupyter_text>read latex<jupyter_code>path = r'..\..\Paper\Tables\\'

converters = {'Genome ID': [astropy.io.ascii.convert_numpy(np.str)]}
tab = Table.read(path+'test_set_bacpacs.tex', converters=converters)
tab<jupyter_output><empty_output><jupyter_text>convert to pandas DataFrame<jupyter_code>bacpacs_tex_df = tab.to_pandas()
bacpacs_tex_df = bacpacs_tex_df.drop('col0', axis=1)
bacpacs_tex_df = bacpacs_tex_df.sort_values(['Balanced Test', 'Validated Label', 'Genome Name'], ascending=[False, True, True])
bacpacs_tex_df.index = range(1, len(bacpacs_tex_df)+1)
bacpacs_tex_df = bacpacs_tex_df.fillna('')

bacpacs_tex_df
GENOME_ID = 'Genome ID'

bacpacs_sup = pd.read_csv(bacpacs_dir_path + 'Bacpacs-Supplementary.csv', dtype='str')
bacpacs_sup
set(bacpacs_sup[GENOME_ID]) - set(bacpacs_tex_df[GENOME_ID])
bacpacs_metadata = bacpacs_tex_df.rename(columns={'Validated Label': 'Label'})
bacpacs_metadata = bacpacs_metadata[bacpacs_metadata['Label'] != 'INC']
bacpacs_metadata.to_csv(bacpacs_metadata_path)<jupyter_output><empty_output><jupyter_text>Download Bacpacs Genomes<jupyter_code>FILE_SUFFIX = '.PATRIC.features.tab'
logger = open('logger', 'w')

def read_genome_ids(input_file):
    with open(input_file) as f:
        ids = f.read().strip().split('\n')
    return ids

def grab_file(ftp, filename, local_path):

    local_file = open(local_path, 'wb')

    try:
        ftp.retrbinary('RETR ' + filename, local_file.write, 1024)
    except Exception as e:
        logger.write(e)

    local_file.close()


def download_genomes(genomes, output_dir_path):


    # connect to host, default port
    ftp = FTP(r'ftp.patricbrc.org')

    ftp.login()  # user anonymous, passwd anonymous@

    count = 0
    for i, genome_id in enumerate(genomes):

        try:
            ftp.cwd("/genomes/" + genome_id)
            local_file = output_dir_path + genome_id + FILE_SUFFIX
            remote_file = genome_id + FILE_SUFFIX
            grab_file(ftp, remote_file, local_file)

            print('downloaded genome: ' + genome_id + ' index: ' + str(i) + '\n')
            count += 1

        except Exception as e:
            # print('ERROR genome: ' + genome_id + ' index: ' + str(i) + '\n')
            logger.write(f'genome: {genome_id} index: {i} error: {e}\n')

    print('downloaded {} genomes'.format(count))
    ftp.quit()
    logger.close()
all_genomes = bacpacs_sup[GENOME_ID]
if download:
    download_genomes(all_genomes, bacpacs_patric_files_path)<jupyter_output>downloaded genome: 1041522.28 index: 0

downloaded genome: 1196162.3 index: 1

downloaded genome: 1196172.3 index: 2

downloaded genome: 1280.11681 index: 3

downloaded genome: 1280.12234 index: 4

downloaded genome: 1302.83 index: 5

downloaded genome: 1338.30 index: 6

downloaded genome: 1639.2624 index: 7

downloaded genome: 1773.8686 index: 8

downloaded genome: 1773.8714 index: 9

downloaded genome: 1773.8719 index: 10

downloaded genome: 210.2912 index: 11

downloaded genome: 2104.189 index: 12

downloaded genome: 2104.190 index: 13

downloaded genome: 28131.10 index: 14

downloaded genome: 28450.655 index: 15

downloaded genome: 287.4623 index: 16

downloaded genome: 37326.9 index: 17

downloaded genome: 487.1548 index: 18

downloaded genome: 520.529 index: 19

downloaded genome: 520.535 index: 20

downloaded genome: 520.537 index: 21

downloaded genome: 520.543 index: 22

downloaded genome: 520.546 index: 23

downloaded genome: 520.572 index: 24

downloaded genome: 520.629 inde[...]<jupyter_text>Create fasta file from PATRIC genomes files<jupyter_code>UNKNOWN_PGFAM = 'X'
def parse_patric_file(file_path):
    
    genome_df = pd.read_csv(file_path, sep='\t')
    pgfams = genome_df['pgfam_id'].fillna(UNKNOWN_PGFAM)
    
    return pgfams
def write_genomes_files(patric_files_folder, output_path):
    with os.scandir(patric_files_folder) as entries:
        with open(output_path, 'w') as out_f:
            for entry in entries:
                if entry.is_file():
                    genome_id = entry.name.split(FILE_SUFFIX)[0]
                    genome_pgfams = parse_patric_file(entry)

                    out_f.write(f'>{genome_id}\n')
                    out_f.write('\n'.join(genome_pgfams))
                    out_f.write('\n')
if write:
    write_genomes_files(bacpacs_patric_files_path, bacpacs_genomes_path)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11550777676120769, 'fraction_numerical': 0.053979871912168347, 'mean_word_length': 3.9580498866213154, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '611545', 'license': 'MIT', 'n_tokens_mistral': 1854, 'n_tokens_neox': 1586, 'n_words': 328, 'path': 'Notebooks/comp_analysis/Bacpacs/Bacpacs_download_genomes.ipynb', 'repo_name': 'shakedna1/wspc_rep', 'size': '4372'}