| ### This is example of the script that will be run in the test environment. | |
| ### Some parts of the code are compulsory and you should NOT CHANGE THEM. | |
| ### They are between '''---compulsory---''' comments. | |
| ### You can change the rest of the code to define and test your solution. | |
| ### However, you should not change the signature of the provided function. | |
| ### The script would save "submission.parquet" file in the current directory. | |
| ### The actual logic of the solution is implemented in the `handcrafted_solution.py` file. | |
| ### The `handcrafted_solution.py` file is a placeholder for your solution. | |
| ### You should implement the logic of your solution in that file. | |
| ### You can use any additional files and subdirectories to organize your code. | |
| '''---compulsory---''' | |
| # import subprocess | |
| # from pathlib import Path | |
| # def install_package_from_local_file(package_name, folder='packages'): | |
| # """ | |
| # Installs a package from a local .whl file or a directory containing .whl files using pip. | |
| # Parameters: | |
| # path_to_file_or_directory (str): The path to the .whl file or the directory containing .whl files. | |
| # """ | |
| # try: | |
| # pth = str(Path(folder) / package_name) | |
| # subprocess.check_call([subprocess.sys.executable, "-m", "pip", "install", | |
| # "--no-index", # Do not use package index | |
| # "--find-links", pth, # Look for packages in the specified directory or at the file | |
| # package_name]) # Specify the package to install | |
| # print(f"Package installed successfully from {pth}") | |
| # except subprocess.CalledProcessError as e: | |
| # print(f"Failed to install package from {pth}. Error: {e}") | |
| # install_package_from_local_file('hoho') | |
| import hoho; hoho.setup() # YOU MUST CALL hoho.setup() BEFORE ANYTHING ELSE | |
| # import subprocess | |
| # import importlib | |
| # from pathlib import Path | |
| # import subprocess | |
| # ### The function below is useful for installing additional python wheels. | |
| # def install_package_from_local_file(package_name, folder='packages'): | |
| # """ | |
| # Installs a package from a local .whl file or a directory containing .whl files using pip. | |
| # Parameters: | |
| # path_to_file_or_directory (str): The path to the .whl file or the directory containing .whl files. | |
| # """ | |
| # try: | |
| # pth = str(Path(folder) / package_name) | |
| # subprocess.check_call([subprocess.sys.executable, "-m", "pip", "install", | |
| # "--no-index", # Do not use package index | |
| # "--find-links", pth, # Look for packages in the specified directory or at the file | |
| # package_name]) # Specify the package to install | |
| # print(f"Package installed successfully from {pth}") | |
| # except subprocess.CalledProcessError as e: | |
| # print(f"Failed to install package from {pth}. Error: {e}") | |
| # pip download webdataset -d packages/webdataset --platform manylinux1_x86_64 --python-version 38 --only-binary=:all: | |
| # install_package_from_local_file('webdataset') | |
| # install_package_from_local_file('tqdm') | |
| ### Here you can import any library or module you want. | |
| ### The code below is used to read and parse the input dataset. | |
| ### Please, do not modify it. | |
| import webdataset as wds | |
| from tqdm import tqdm | |
| from typing import Dict | |
| import pandas as pd | |
| from transformers import AutoTokenizer | |
| import os | |
| import time | |
| import io | |
| from PIL import Image as PImage | |
| import numpy as np | |
| from hoho.read_write_colmap import read_cameras_binary, read_images_binary, read_points3D_binary | |
| from hoho import proc, Sample | |
| def convert_entry_to_human_readable(entry): | |
| out = {} | |
| already_good = ['__key__', 'wf_vertices', 'wf_edges', 'edge_semantics', 'mesh_vertices', 'mesh_faces', 'face_semantics', 'K', 'R', 't'] | |
| for k, v in entry.items(): | |
| if k in already_good: | |
| out[k] = v | |
| continue | |
| if k == 'points3d': | |
| out[k] = read_points3D_binary(fid=io.BytesIO(v)) | |
| if k == 'cameras': | |
| out[k] = read_cameras_binary(fid=io.BytesIO(v)) | |
| if k == 'images': | |
| out[k] = read_images_binary(fid=io.BytesIO(v)) | |
| if k in ['ade20k', 'gestalt']: | |
| out[k] = [PImage.open(io.BytesIO(x)).convert('RGB') for x in v] | |
| if k == 'depthcm': | |
| out[k] = [PImage.open(io.BytesIO(x)) for x in entry['depthcm']] | |
| return out | |
| '''---end of compulsory---''' | |
| ### The part below is used to define and test your solution. | |
| import subprocess | |
| import sys | |
| import os | |
| import numpy as np | |
| os.environ['MKL_THREADING_LAYER'] = 'GNU' | |
| os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' | |
| def uninstall_package(package_name): | |
| """ | |
| Uninstalls a package using pip. | |
| Parameters: | |
| package_name (str): The name of the package to uninstall. | |
| """ | |
| try: | |
| subprocess.check_call([sys.executable, "-m", "pip", "uninstall", "-y", package_name]) | |
| print(f"Package {package_name} uninstalled successfully") | |
| except subprocess.CalledProcessError as e: | |
| print(f"Failed to uninstall package {package_name}. Error: {e}") | |
| # def download_packages(packages, folder='packages/torch'): | |
| # """ | |
| # Downloads packages as .whl files into the specified folder using pip. | |
| # Parameters: | |
| # packages (list): List of packages to download with versions. | |
| # folder (str): The folder where the .whl files will be saved. | |
| # """ | |
| # Path(folder).mkdir(parents=True, exist_ok=True) | |
| # try: | |
| # subprocess.check_call([sys.executable, "-m", "pip", "download", | |
| # "--platform", "manylinux1_x86_64", | |
| # "--python-version", "38", | |
| # "--only-binary=:all:", | |
| # "-d", folder] + packages) | |
| # print(f"Packages downloaded successfully into {folder}") | |
| # except subprocess.CalledProcessError as e: | |
| # print(f"Failed to download packages. Error: {e}") | |
| def download_packages(packages, folder): | |
| # Create the directory if it doesn't exist | |
| if not os.path.exists(folder): | |
| os.makedirs(folder) | |
| try: | |
| subprocess.check_call([ | |
| 'pip', 'download', | |
| '--dest', folder, | |
| '-f', 'https://download.pytorch.org/whl/cu121' | |
| ] + packages) | |
| print(f"Packages downloaded successfully to {folder}") | |
| except subprocess.CalledProcessError as e: | |
| print(f"Failed to download packages. Error: {e}") | |
| # Set CUDA environment variables | |
| os.environ['CUDA_HOME'] = '/usr/local/cuda-12.1' | |
| os.environ['PATH'] = os.environ['CUDA_HOME'] + '/bin:' + os.environ['PATH'] | |
| os.environ['LD_LIBRARY_PATH'] = os.environ['CUDA_HOME'] + '/lib64:' + os.environ.get('LD_LIBRARY_PATH', '') | |
| def install_package_from_local_file(package_name, folder='packages'): | |
| """ | |
| Installs a package from a local .whl file or a directory containing .whl files using pip. | |
| Parameters: | |
| package_name (str): The name of the package to install. | |
| folder (str): The folder where the .whl files are located. | |
| """ | |
| try: | |
| pth = str(Path(folder) / package_name) | |
| subprocess.check_call([sys.executable, "-m", "pip", "install", | |
| "--no-index", # Do not use package index | |
| "--find-links", pth, # Look for packages in the specified directory or at the file | |
| package_name]) # Specify the package to install | |
| print(f"Package installed successfully from {pth}") | |
| except subprocess.CalledProcessError as e: | |
| print(f"Failed to install package from {pth}. Error: {e}") | |
| def install_which(): | |
| try: | |
| # Attempt to install which if it's not available | |
| subprocess.check_call(['sudo', 'apt-get', 'install', '-y', 'which']) | |
| print("Which installed successfully.") | |
| except subprocess.CalledProcessError as e: | |
| print(f"An error occurred while installing which: {e}") | |
| sys.exit(1) | |
| def setup_environment(): | |
| # Uninstall torch if it is already installed | |
| # packages_to_uninstall = ['torch', 'torchvision', 'torchaudio'] | |
| # for package in packages_to_uninstall: | |
| # uninstall_package(package) | |
| # Download required packages | |
| # pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116 | |
| # pip install torch==2.3.0 torchvision==0.18.0 torchaudio==2.3.0 --index-url https://download.pytorch.org/whl/cu121 | |
| # pip install torch==2.1.0 torchvision==0.16.0 torchaudio==2.1.0 --index-url https://download.pytorch.org/whl/cu121 | |
| # packages_to_download = ['torch==1.13.1', 'torchvision==0.14.1', 'torchaudio==0.13.1'] | |
| # packages_to_download = ['torch==2.1.0', 'torchvision==0.16.0', 'torchaudio==2.1.0'] | |
| # download_packages(packages_to_download, folder='packages/torch') | |
| # Install ninja | |
| # install_package_from_local_file('ninja', folder='packages') | |
| # packages_to_download = ['torch==2.1.0', 'torchvision==0.16.0', 'torchaudio==2.1.0'] | |
| # download_folder = 'packages/torch' | |
| # Download the packages | |
| # download_packages(packages_to_download, download_folder) | |
| # Install packages from local files | |
| # install_package_from_local_file('torch', folder='packages') | |
| # install_package_from_local_file('packages/torch/torchvision-0.16.0-cp38-cp38-manylinux1_x86_64.whl', folder='packages/torch') | |
| # install_package_from_local_file('packages/torch/torchaudio-2.1.0-cp38-cp38-manylinux1_x86_64.whl', folder='packages/torch') | |
| # install_package_from_local_file('scikit-learn', folder='packages') | |
| # install_package_from_local_file('open3d', folder='packages') | |
| # install_package_from_local_file('easydict', folder='packages') | |
| # install_package_from_local_file('setuptools', folder='packages') | |
| # install_package_from_local_file('ninja', folder='packages') | |
| # download_packages(['scikit-learn'], folder='packages/scikit-learn') | |
| # download_packages(['open3d'], folder='packages/open3d') | |
| # download_packages(['easydict'], folder='packages/easydict') | |
| # try: | |
| # subprocess.check_call(['which', 'which']) | |
| # except subprocess.CalledProcessError: | |
| # install_which() | |
| # Set environment variables for CUDA | |
| os.environ['CUDA_HOME'] = '/usr/local/cuda' | |
| os.environ['PATH'] = os.environ['CUDA_HOME'] + '/bin:' + os.environ['PATH'] | |
| os.environ['LD_LIBRARY_PATH'] = os.environ['CUDA_HOME'] + '/lib64:' + os.environ.get('LD_LIBRARY_PATH', '') | |
| os.environ['LIBRARY_PATH'] = os.environ['CUDA_HOME'] + '/lib64' | |
| # Print CUDA environment variables to verify | |
| # print("CUDA_HOME:", os.environ['CUDA_HOME']) | |
| # print("PATH:", os.environ['PATH']) | |
| # print("LD_LIBRARY_PATH:", os.environ['LD_LIBRARY_PATH']) | |
| # print("LIBRARY_PATH:", os.environ['LIBRARY_PATH']) | |
| # Verify CUDA headers are accessible | |
| cuda_include_path = os.path.join(os.environ['CUDA_HOME'], 'include') | |
| if not os.path.exists(os.path.join(cuda_include_path, 'cuda.h')): | |
| raise EnvironmentError(f"CUDA headers not found in {cuda_include_path}. Please check your CUDA installation.") | |
| pc_util_path = os.path.join(os.getcwd(), 'pc_util') | |
| if os.path.isdir(pc_util_path): | |
| os.chdir(pc_util_path) | |
| subprocess.check_call([sys.executable, "setup.py", "install"], cwd=pc_util_path) | |
| os.chdir("..") | |
| def setup_cuda_environment(): | |
| # cuda_home = '/usr/local/cuda' | |
| # if not os.path.exists(cuda_home): | |
| # raise EnvironmentError(f"CUDA_HOME directory {cuda_home} does not exist. Please install CUDA and set CUDA_HOME environment variable.") | |
| # os.environ['CUDA_HOME'] = cuda_home | |
| # os.environ['PATH'] = f"{cuda_home}/bin:{os.environ['PATH']}" | |
| # os.environ['LD_LIBRARY_PATH'] = f"{cuda_home}/lib64:{os.environ.get('LD_LIBRARY_PATH', '')}" | |
| os.environ['PATH'] = '/usr/local/cuda/bin' | |
| os.environ['LD_LIBRARY_PATH'] = '/usr/local/cuda/lib64' | |
| os.environ['LIBRARY_PATH'] = '/usr/local/cuda/lib64' | |
| # usr_local_contents = os.listdir('/usr/local') | |
| # # print("Items under /usr/local:") | |
| # for item in usr_local_contents: | |
| # print(item) | |
| from pathlib import Path | |
| def save_submission(submission, path): | |
| """ | |
| Saves the submission to a specified path. | |
| Parameters: | |
| submission (List[Dict[]]): The submission to save. | |
| path (str): The path to save the submission to. | |
| """ | |
| sub = pd.DataFrame(submission, columns=["__key__", "wf_vertices", "wf_edges"]) | |
| sub.to_parquet(path) | |
| print(f"Submission saved to {path}") | |
| if __name__ == "__main__": | |
| # setup_cuda_environment() | |
| setup_environment() | |
| from handcrafted_solution import predict | |
| print ("------------ Loading dataset------------ ") | |
| params = hoho.get_params() | |
| dataset = hoho.get_dataset(decode=None, split='all', dataset_type='webdataset') | |
| print('------------ Now you can do your solution ---------------') | |
| solution = [] | |
| from concurrent.futures import ProcessPoolExecutor | |
| with ProcessPoolExecutor(max_workers=1) as pool: | |
| results = [] | |
| for i, sample in enumerate(tqdm(dataset)): | |
| results.append(pool.submit(predict, sample, visualize=False)) | |
| for i, result in enumerate(tqdm(results)): | |
| key, pred_vertices, pred_edges = result.result() | |
| solution.append({ | |
| '__key__': key, | |
| 'wf_vertices': pred_vertices.tolist(), | |
| 'wf_edges': pred_edges | |
| }) | |
| if i % 100 == 0: | |
| # incrementally save the results in case we run out of time | |
| print(f"Processed {i} samples") | |
| # save_submission(solution, Path(params['output_path']) / "submission.parquet") | |
| print('------------ Saving results ---------------') | |
| save_submission(solution, Path(params['output_path']) / "submission.parquet") | |
| print("------------ Done ------------ ") |