added also medrxiv

combine_databases.py

@@ -0,0 +1,136 @@
+import pandas as pd
+import numpy as np
+from pathlib import Path
+
+def combine_databases():
+    # Define paths
+    aggregated_data_path = Path("aggregated_data")
+    db_update_bio_path = Path("db_update")
+    biorxiv_embeddings_path = Path("biorxiv_ubin_embaddings.npy")
+    embed_update_bio_path = Path("embed_update")
+
+    db_update_med_path = Path("db_update_med")
+    embed_update_med_path = Path("embed_update_med")
+
+    # Load existing database and embeddings for BioRxiv
+    df_bio_existing = pd.read_parquet(aggregated_data_path)
+    bio_embeddings_existing = np.load(biorxiv_embeddings_path, allow_pickle=True)
+    print(f"Existing BioRxiv data shape: {df_bio_existing.shape}, Existing BioRxiv embeddings shape: {bio_embeddings_existing.shape}")
+
+    # Determine the embedding size from existing embeddings
+    embedding_size = bio_embeddings_existing.shape[1]
+
+    # Prepare lists to collect new updates
+    bio_dfs_list = []
+    bio_embeddings_list = []
+
+    # Helper function to process updates from a specified directory
+    def process_updates(new_data_directory, updated_embeddings_directory, dfs_list, embeddings_list):
+        new_data_files = sorted(Path(new_data_directory).glob("*.parquet"))
+        for data_file in new_data_files:
+            corresponding_embedding_file = Path(updated_embeddings_directory) / (data_file.stem + ".npy")
+
+            if corresponding_embedding_file.exists():
+                df = pd.read_parquet(data_file)
+                new_embeddings = np.load(corresponding_embedding_file, allow_pickle=True)
+
+                # Check if the number of rows in the DataFrame matches the number of rows in the embeddings
+                if df.shape[0] != new_embeddings.shape[0]:
+                    print(f"Shape mismatch for {data_file.name}: DataFrame has {df.shape[0]} rows, embeddings have {new_embeddings.shape[0]} rows. Skipping.")
+                    continue
+
+                # Check embedding size and adjust if necessary
+                if new_embeddings.shape[1] != embedding_size:
+                    print(f"Skipping {data_file.name} due to embedding size mismatch.")
+                    continue
+
+                dfs_list.append(df)
+                embeddings_list.append(new_embeddings)
+            else:
+                print(f"No corresponding embedding file found for {data_file.name}")
+
+    # Process updates from both BioRxiv and MedRxiv
+    process_updates(db_update_bio_path, embed_update_bio_path, bio_dfs_list, bio_embeddings_list)
+
+    # Concatenate all BioRxiv updates
+    if bio_dfs_list:
+        df_bio_updates = pd.concat(bio_dfs_list)
+    else:
+        df_bio_updates = pd.DataFrame()
+
+    if bio_embeddings_list:
+        bio_embeddings_updates = np.vstack(bio_embeddings_list)
+    else:
+        bio_embeddings_updates = np.array([])
+
+    # Append new BioRxiv data to existing, handling duplicates as needed
+    df_bio_combined = pd.concat([df_bio_existing, df_bio_updates])
+
+    # Create a mask for filtering unique titles
+    bio_mask = ~df_bio_combined.duplicated(subset=["title"], keep="last")
+    df_bio_combined = df_bio_combined[bio_mask]
+
+    # Combine BioRxiv embeddings, ensuring alignment with the DataFrame
+    bio_embeddings_combined = (
+        np.vstack([bio_embeddings_existing, bio_embeddings_updates])
+        if bio_embeddings_updates.size
+        else bio_embeddings_existing
+    )
+
+    # Filter the embeddings based on the DataFrame unique entries
+    bio_embeddings_combined = bio_embeddings_combined[bio_mask]
+
+    assert df_bio_combined.shape[0] == bio_embeddings_combined.shape[0], "Shape mismatch between BioRxiv DataFrame and embeddings"
+
+    print(f"Filtered BioRxiv DataFrame shape: {df_bio_combined.shape}")
+    print(f"Filtered BioRxiv embeddings shape: {bio_embeddings_combined.shape}")
+
+    # Save combined BioRxiv DataFrame and embeddings
+    combined_biorxiv_data_path = aggregated_data_path / "combined_biorxiv_data.parquet"
+    df_bio_combined.to_parquet(combined_biorxiv_data_path)
+    print(f"Saved combined BioRxiv DataFrame to {combined_biorxiv_data_path}")
+
+    combined_biorxiv_embeddings_path = "biorxiv_ubin_embaddings.npy"
+    np.save(combined_biorxiv_embeddings_path, bio_embeddings_combined)
+    print(f"Saved combined BioRxiv embeddings to {combined_biorxiv_embeddings_path}")
+
+    # Prepare lists to collect new MedRxiv updates
+    med_dfs_list = []
+    med_embeddings_list = []
+
+    process_updates(db_update_med_path, embed_update_med_path, med_dfs_list, med_embeddings_list)
+
+    # Concatenate all MedRxiv updates
+    if med_dfs_list:
+        df_med_combined = pd.concat(med_dfs_list)
+    else:
+        df_med_combined = pd.DataFrame()
+
+    if med_embeddings_list:
+        med_embeddings_combined = np.vstack(med_embeddings_list)
+    else:
+        med_embeddings_combined = np.array([])
+
+    last_date_in_med_database = df_med_combined['date'].max() if not df_med_combined.empty else "unknown"
+
+    # Create a mask for filtering unique titles
+    med_mask = ~df_med_combined.duplicated(subset=["title"], keep="last")
+    df_med_combined = df_med_combined[med_mask]
+    med_embeddings_combined = med_embeddings_combined[med_mask]
+
+    assert df_med_combined.shape[0] == med_embeddings_combined.shape[0], "Shape mismatch between MedRxiv DataFrame and embeddings"
+
+    print(f"Filtered MedRxiv DataFrame shape: {df_med_combined.shape}")
+    print(f"Filtered MedRxiv embeddings shape: {med_embeddings_combined.shape}")
+
+    # Save combined MedRxiv DataFrame and embeddings
+    combined_medrxiv_data_path = db_update_med_path / f"database_{last_date_in_med_database}.parquet"
+    df_med_combined.to_parquet(combined_medrxiv_data_path)
+    print(f"Saved combined MedRxiv DataFrame to {combined_medrxiv_data_path}")
+
+    combined_medrxiv_embeddings_path = embed_update_med_path / f"database_{last_date_in_med_database}.npy"
+    np.save(combined_medrxiv_embeddings_path, med_embeddings_combined)
+    print(f"Saved combined MedRxiv embeddings to {combined_medrxiv_embeddings_path}")
+
+if __name__ == "__main__":
+    combine_databases()

streamlit_app.py

@@ -263,32 +263,57 @@ def download_data_from_dropbox():
 @st.cache_resource(ttl="1d")
 def load_data_embeddings():
     existing_data_path = "aggregated_data"
-    new_data_directory = "db_update"
+    new_data_directory_bio = "db_update"
     existing_embeddings_path = "biorxiv_ubin_embaddings.npy"
-    updated_embeddings_directory = "embed_update"
+    updated_embeddings_directory_bio = "embed_update"
 
+    new_data_directory_med = "db_update_med"
+    updated_embeddings_directory_med = "embed_update_med"
+    
     # Load existing database and embeddings
     df_existing = pd.read_parquet(existing_data_path)
     embeddings_existing = np.load(existing_embeddings_path, allow_pickle=True)
 
+    print(f"Existing data shape: {df_existing.shape}, Existing embeddings shape: {embeddings_existing.shape}")
+
+    # Determine the embedding size from existing embeddings
+    embedding_size = embeddings_existing.shape[1]
+
     # Prepare lists to collect new updates
     df_updates_list = []
     embeddings_updates_list = []
 
-    # Ensure pairing of new data and embeddings by their matching filenames
-    new_data_files = sorted(Path(new_data_directory).glob("*.parquet"))
-    for data_file in new_data_files:
-        # Assuming naming convention allows direct correlation
-        corresponding_embedding_file = Path(updated_embeddings_directory) / (
-            data_file.stem + ".npy"
-        )
+    # Helper function to process updates from a specified directory
+    def process_updates(new_data_directory, updated_embeddings_directory):
+        new_data_files = sorted(Path(new_data_directory).glob("*.parquet"))
+        print(new_data_files)
+        for data_file in new_data_files:
+            corresponding_embedding_file = Path(updated_embeddings_directory) / (
+                data_file.stem + ".npy"
+            )
+
+            if corresponding_embedding_file.exists():
+                df = pd.read_parquet(data_file)
+                new_embeddings = np.load(corresponding_embedding_file, allow_pickle=True)
+                
+                # Check if the number of rows in the DataFrame matches the number of rows in the embeddings
+                if df.shape[0] != new_embeddings.shape[0]:
+                    print(f"Shape mismatch for {data_file.name}: DataFrame has {df.shape[0]} rows, embeddings have {new_embeddings.shape[0]} rows. Skipping.")
+                    continue
+                
+                # Check embedding size and adjust if necessary
+                if new_embeddings.shape[1] != embedding_size:
+                    print(f"Skipping {data_file.name} due to embedding size mismatch.")
+                    continue
+
+                df_updates_list.append(df)
+                embeddings_updates_list.append(new_embeddings)
+            else:
+                print(f"No corresponding embedding file found for {data_file.name}")
 
-        if corresponding_embedding_file.exists():
-            # Load and append DataFrame and embeddings
-            df_updates_list.append(pd.read_parquet(data_file))
-            embeddings_updates_list.append(np.load(corresponding_embedding_file))
-        else:
-            print(f"No corresponding embedding file found for {data_file.name}")
+    # Process updates from both BioRxiv and MedArXiv
+    process_updates(new_data_directory_bio, updated_embeddings_directory_bio)
+    process_updates(new_data_directory_med, updated_embeddings_directory_med)
 
     # Concatenate all updates
     if df_updates_list:
@@ -304,7 +329,7 @@ def load_data_embeddings():
     # Append new data to existing, handling duplicates as needed
     df_combined = pd.concat([df_existing, df_updates])
 
-    # create a mask for filtering
+    # Create a mask for filtering
     mask = ~df_combined.duplicated(subset=["title"], keep="last")
     df_combined = df_combined[mask]
 
@@ -315,20 +340,21 @@ def load_data_embeddings():
         else embeddings_existing
     )
 
-    # filter the embeddings based on dataframe unique entries
+    # Filter the embeddings based on the dataframe unique entries
     embeddings_combined = embeddings_combined[mask]
 
     return df_combined, embeddings_combined
 
+
 LLM_prompt = "Review the abstracts listed below and create a list and summary that captures their main themes and findings. Identify any commonalities across the abstracts and highlight these in your summary. Ensure your response is concise, avoids external links, and is formatted in markdown.\n\n"
 
-def summarize_abstract(abstract, llm_model="llama3-70b-8192", instructions=LLM_prompt, api_key=st.secrets["groq_token"]):
+def summarize_abstract(abstract, llm_model="llama-3.1-70b-versatile", instructions=LLM_prompt, api_key=st.secrets["groq_token"]):
     """
     Summarizes the provided abstract using a specified LLM model.
     
     Parameters:
     - abstract (str): The abstract text to be summarized.
-    - llm_model (str): The LLM model used for summarization. Defaults to "llama3-70b-8192".
+    - llm_model (str): The LLM model used for summarization. Defaults to "llama-3.1-70b-versatile".
     
     Returns:
     - str: A summary of the abstract, condensed into one to two sentences.
@@ -390,16 +416,25 @@ def define_style():
     )
 
 
-def logo(db_update_date, db_size):
+def logo(db_update_date, db_size_bio, db_size_med):
     # Initialize Streamlit app
-    image_path = "https://www.biorxiv.org/sites/default/files/biorxiv_logo_homepage.png"
+    biorxiv_logo = "https://www.biorxiv.org/sites/default/files/biorxiv_logo_homepage.png"
+    medarxiv_logo = "https://www.medrxiv.org/sites/default/files/medRxiv_homepage_logo.png"
     st.markdown(
         f"""
-        <div style='text-align: center;'>
-            <img src='{image_path}' alt='BioRxiv logo' style='max-height: 100px;'>
-            <h3 style='color: black;'>Manuscript Semantic Search [bMSS]</h1>
-            Last database update: {db_update_date}; Database size: {db_size} entries
+        <div style='display: flex; justify-content: center; align-items: center;'>
+            <div style='margin-right: 20px;'>
+                <img src='{biorxiv_logo}' alt='BioRxiv logo' style='max-height: 100px;'>
+            </div>
+            <div style='margin-left: 20px;'>
+                <img src='{medarxiv_logo}' alt='medRxiv logo' style='max-height: 100px;'>
+            </div>
         </div>
+        <div style='text-align: center; margin-top: 10px;'>
+            <h3 style='color: black;'>Manuscript Semantic Search [bMSS]</h3>
+            Last database update: {db_update_date}; Database size: bioRxiv: {db_size_bio} / medRxiv: {db_size_med} entries
+        </div>
+        <br>
         """,
         unsafe_allow_html=True,
     )
@@ -413,7 +448,7 @@ download_data_from_dropbox()
 define_style()
 
 df, embeddings_unique = load_data_embeddings()
-logo(df["date"].max(), df.shape[0])
+logo(df["date"].max(), df[df['server']=='biorxiv'].shape[0], df[df['server']=='medrxiv'].shape[0])
 
 # model = model_to_device()
 
@@ -474,7 +509,7 @@ if query:
         )
 
         # Prepare the results for plotting
-        plot_data = {"Date": [], "Title": [], "Score": [], "DOI": [], "category": []}
+        plot_data = {"Date": [], "Title": [], "Score": [], "DOI": [], "category": [], "server": []}
 
         search_df = pd.DataFrame(results[0])
 
@@ -503,11 +538,14 @@ if query:
             plot_data["Score"].append(search_df["score"][index])  # type: ignore
             plot_data["DOI"].append(row["doi"])
             plot_data["category"].append(row["category"])
+            plot_data["server"].append(row["server"])
 
             #summary_text = summarize_abstract(row['abstract'])
 
             with st.expander(f"{index+1}\. {row['title']}"): # type: ignore
-                st.markdown(f"**Score:** {entry['score']:.1f}")
+                col1, col2 = st.columns(2)
+                col1.markdown(f"**Score:** {entry['score']:.1f}")
+                col2.markdown(f"**Server:** [{row['server']}]")
                 st.markdown(f"**Authors:** {row['authors']}")
                 col1, col2 = st.columns(2)
                 col2.markdown(f"**Category:** {row['category']}")
@@ -556,6 +594,7 @@ if query:
             x="Date",
             y="Score",
             hover_data=["Title", "DOI"],
+            color='server',
             title="Publication Times and Scores",
         )
         fig.update_traces(marker=dict(size=10))

update_database_medarxiv.py

@@ -0,0 +1,289 @@
+import pandas as pd
+import numpy as np
+from pathlib import Path
+import datetime
+import requests
+import json
+import os
+from datetime import datetime
+from dateutil.relativedelta import relativedelta
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from sentence_transformers import SentenceTransformer
+import torch
+import shutil
+import dropbox
+import streamlit as st
+import time
+
+def retry_on_exception(exception, retries=5, delay=2):
+    def decorator(func):
+        def wrapper(*args, **kwargs):
+            last_exception = None
+            for _ in range(retries):
+                try:
+                    return func(*args, **kwargs)
+                except exception as e:
+                    last_exception = e
+                    print(f"Retrying due to: {str(e)}")
+                    time.sleep(delay)
+            raise last_exception
+        return wrapper
+    return decorator
+
+@retry_on_exception(requests.exceptions.ConnectionError)
+def fetch_and_save_data_block(endpoint, server, block_start, block_end, save_directory, format='json'):
+    base_url = f"https://api.medrxiv.org/details/{server}/"
+    block_interval = f"{block_start.strftime('%Y-%m-%d')}/{block_end.strftime('%Y-%m-%d')}"
+    block_data = []
+    cursor = 0
+    continue_fetching = True
+
+    while continue_fetching:
+        url = f"{base_url}{block_interval}/{cursor}/{format}"
+        response = requests.get(url)
+
+        if response.status_code != 200:
+            print(f"Failed to fetch data for block {block_interval} at cursor {cursor}. HTTP Status: {response.status_code}")
+            break
+
+        data = response.json()
+        fetched_papers = len(data['collection'])
+
+        if fetched_papers > 0:
+            block_data.extend(data['collection'])
+            cursor += fetched_papers
+            print(f"Fetched {fetched_papers} papers for block {block_interval}. Total fetched: {cursor}.")
+        else:
+            continue_fetching = False
+
+    if block_data:
+        save_data_block(block_data, block_start, block_end, endpoint, save_directory)
+
+def save_data_block(block_data, start_date, end_date, endpoint, save_directory):
+    start_yymmdd = start_date.strftime("%y%m%d")
+    end_yymmdd = end_date.strftime("%y%m%d")
+    filename = f"{save_directory}/{endpoint}_data_{start_yymmdd}_{end_yymmdd}.json"
+    
+    with open(filename, 'w') as file:
+        json.dump(block_data, file, indent=4)
+    
+    print(f"Saved data block to {filename}")
+
+def fetch_data(endpoint, server, interval, save_directory, format='json'):
+    os.makedirs(save_directory, exist_ok=True)
+    start_date, end_date = [datetime.strptime(date, "%Y-%m-%d") for date in interval.split('/')]
+    current_date = start_date
+    tasks = []
+
+    with ThreadPoolExecutor(max_workers=12) as executor:
+        while current_date <= end_date:
+            block_start = current_date
+            block_end = min(current_date + relativedelta(months=1) - relativedelta(days=1), end_date)
+            tasks.append(executor.submit(fetch_and_save_data_block, endpoint, server, block_start, block_end, save_directory, format))
+            current_date += relativedelta(months=1)
+        
+        for future in as_completed(tasks):
+            future.result()
+
+def load_json_to_dataframe(json_file):
+    with open(json_file, 'r') as file:
+        data = json.load(file)
+    return pd.DataFrame(data)
+
+def save_dataframe(df, save_path):
+    df.to_parquet(save_path)
+
+def process_json_files(directory, save_directory):
+    os.makedirs(save_directory, exist_ok=True)
+    
+    json_files = list(Path(directory).glob('*.json'))
+    print(f'json_files {type(json_files)}: {json_files}')
+    
+    for json_file in json_files:
+        df = load_json_to_dataframe(json_file)
+        
+        parquet_filename = f"{json_file.stem}.parquet"
+        save_path = os.path.join(save_directory, parquet_filename)
+        
+        if os.path.exists(save_path):
+            npy_file_path = save_path.replace('db_update', 'embed_update').replace('parquet', 'npy')
+            if os.path.exists(npy_file_path):
+                os.remove(npy_file_path)
+                print(f'Removed embedding file {npy_file_path} due to the dataframe update')
+                
+        save_dataframe(df, save_path)
+        print(f"Processed and saved {json_file.name} to {parquet_filename}")
+
+def load_unprocessed_parquets(db_update_directory, embed_update_directory):
+    db_update_directory = Path(db_update_directory)
+    embed_update_directory = Path(embed_update_directory)
+
+    parquet_files = list(db_update_directory.glob('*.parquet'))
+    npy_files = {f.stem for f in embed_update_directory.glob('*.npy')}
+    unprocessed_dataframes = []
+
+    for parquet_file in parquet_files:
+        if parquet_file.stem not in npy_files:
+            unprocessed_dataframes.append(parquet_file)
+            print(f"Loaded unprocessed Parquet file: {parquet_file.name}")
+        else:
+            print(f"Skipping processed Parquet file: {parquet_file.name}")
+
+    return unprocessed_dataframes
+
+def connect_to_dropbox():
+    dropbox_APP_KEY = st.secrets["dropbox_APP_KEY"]
+    dropbox_APP_SECRET = st.secrets["dropbox_APP_SECRET"]
+    dropbox_REFRESH_TOKEN = st.secrets["dropbox_REFRESH_TOKEN"]
+    
+    dbx = dropbox.Dropbox(
+        app_key=dropbox_APP_KEY,
+        app_secret=dropbox_APP_SECRET,
+        oauth2_refresh_token=dropbox_REFRESH_TOKEN
+    )
+    return dbx
+
+def upload_path(local_path, dropbox_path):
+    dbx = connect_to_dropbox()
+    local_path = Path(local_path)
+
+    if local_path.is_file():
+        relative_path = local_path.name
+        dropbox_file_path = os.path.join(dropbox_path, relative_path).replace('\\', '/').replace('//', '/')
+        upload_file(local_path, dropbox_file_path, dbx)
+    elif local_path.is_dir():
+        for local_file in local_path.rglob('*'):
+            if local_file.is_file():
+                relative_path = local_file.relative_to(local_path.parent)
+                dropbox_file_path = os.path.join(dropbox_path, relative_path).replace('\\', '/').replace('//', '/')
+                upload_file(local_file, dropbox_file_path, dbx)
+    else:
+        print("The provided path does not exist.")
+
+def upload_file(file_path, dropbox_file_path, dbx):
+    try:
+        dropbox_file_path = dropbox_file_path.replace('\\', '/')
+
+        try:
+            metadata = dbx.files_get_metadata(dropbox_file_path)
+            dropbox_mod_time = metadata.server_modified
+            local_mod_time = datetime.fromtimestamp(file_path.stat().st_mtime)
+
+            if dropbox_mod_time >= local_mod_time:
+                print(f"Skipped {dropbox_file_path}, Dropbox version is up-to-date.")
+                return
+        except dropbox.exceptions.ApiError as e:
+            if not isinstance(e.error, dropbox.files.GetMetadataError) or e.error.is_path() and e.error.get_path().is_not_found():
+                print(f"No existing file on Dropbox, proceeding with upload: {dropbox_file_path}")
+            else:
+                raise e
+
+        with file_path.open('rb') as f:
+            dbx.files_upload(f.read(), dropbox_file_path, mode=dropbox.files.WriteMode.overwrite)
+            print(f"Uploaded {dropbox_file_path}")
+    except Exception as e:
+        print(f"Failed to upload {dropbox_file_path}: {str(e)}")
+
+def load_data_embeddings():
+    new_data_directory = "db_update_med"
+    updated_embeddings_directory = "embed_update_med"
+    new_data_files = sorted(Path(new_data_directory).glob("*.parquet"))
+    
+    df_updates_list = []
+    embeddings_updates_list = []
+    
+    for data_file in new_data_files:
+        # Assuming naming convention allows direct correlation
+        corresponding_embedding_file = Path(updated_embeddings_directory) / (
+            data_file.stem + ".npy"
+        )
+
+        if corresponding_embedding_file.exists():
+            # Load and append DataFrame and embeddings
+            df_updates_list.append(pd.read_parquet(data_file))
+            embeddings_updates_list.append(np.load(corresponding_embedding_file))
+        else:
+            print(f"No corresponding embedding file found for {data_file.name}")
+
+    new_data_files = sorted(Path(new_data_directory).glob("*.parquet"))
+    for data_file in new_data_files:
+        corresponding_embedding_file = Path(updated_embeddings_directory) / (
+            data_file.stem + ".npy"
+        )
+
+        if corresponding_embedding_file.exists():
+            df_updates_list.append(pd.read_parquet(data_file))
+            embeddings_updates_list.append(np.load(corresponding_embedding_file))
+        else:
+            print(f"No corresponding embedding file found for {data_file.name}")
+
+    if df_updates_list:
+        df_updates = pd.concat(df_updates_list)
+    else:
+        df_updates = pd.DataFrame()
+
+    if embeddings_updates_list:
+        embeddings_updates = np.vstack(embeddings_updates_list)
+    else:
+        embeddings_updates = np.array([])
+
+    df_combined = df_updates
+    mask = ~df_combined.duplicated(subset=["title"], keep="last")
+    df_combined = df_combined[mask]
+
+    embeddings_combined = embeddings_updates
+
+    embeddings_combined = embeddings_combined[mask]
+
+    return df_combined, embeddings_combined
+
+endpoint = "details"
+server = "medrxiv"
+
+df, embeddings = load_data_embeddings()
+
+try:
+    start_date = df['date'].max()
+except:
+    start_date = '1990-01-01'
+last_date = datetime.today().strftime('%Y-%m-%d')
+
+interval = f'{start_date}/{last_date}'
+print(f'using interval: {interval}')
+
+save_directory = "db_update_json_med"
+fetch_data(endpoint, server, interval, save_directory)
+
+directory = r'db_update_json_med'
+save_directory = r'db_update_med'
+process_json_files(directory, save_directory)
+
+db_update_directory = 'db_update_med'
+embed_update_directory = 'embed_update_med'
+unprocessed_dataframes = load_unprocessed_parquets(db_update_directory, embed_update_directory)
+
+if unprocessed_dataframes:
+    for file in unprocessed_dataframes:
+        df = pd.read_parquet(file)
+        query = df['abstract'].tolist()
+
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        model = SentenceTransformer("mixedbread-ai/mxbai-embed-large-v1")
+        model.to(device)
+
+        query_embedding = model.encode(query, normalize_embeddings=True, precision='ubinary', show_progress_bar=True)
+        file_path = os.path.basename(file).split('.')[0]
+        os.makedirs('embed_update_med', exist_ok=True)
+        embeddings_path = f'embed_update_med/{file_path}'
+        np.save(embeddings_path, query_embedding)
+        print(f'Saved embeddings {embeddings_path}')
+
+    db_update_json = 'db_update_json_med'
+    shutil.rmtree(db_update_json)
+    print(f"Directory '{db_update_json}' and its contents have been removed.")
+    
+    for path in ['db_update_med', 'embed_update_med']:
+        upload_path(path, '/')
+
+else:
+    print('Nothing to do')