from dataclasses import dataclass from enum import Enum @dataclass class Task: benchmark: str metric: str col_name: str # Select your tasks here # --------------------------------------------------- class Tasks(Enum): # task_key in the json file, metric_key in the json file, name to display in the leaderboard task0 = Task("anli_r1", "acc", "ANLI") task1 = Task("logiqa", "acc_norm", "LogiQA") NUM_FEWSHOT = 0 # Change with your few shot # --------------------------------------------------- # Your leaderboard name TITLE = """

UnlearnDiffAtk: Unlearned Diffusion Model Benchmark

""" # subtitle SUB_TITLE = """

Effective and efficient adversarial prompt generation approach for unlearned diffusion model evaluations.

""" # What does your leaderboard evaluate? INTRODUCTION_TEXT = """ This benchmark evaluates the robustness and utility retaining of safety-driven unlearned diffusion models (DMs) across a variety of tasks. For more details, please visit the [project](https://www.optml-group.com/posts/mu_attack). - The robustness of unlearned DM is evaluated through our proposed adversarial prompt attack, [UnlearnDiffAtk](https://github.com/OPTML-Group/Diffusion-MU-Attack), which has been accepted to ECCV 2024. - The utility retaining of unlearned DM is evaluated through FID and CLIP score on the generated images using [10K randomly sampled COCO caption prompts](https://github.com/OPTML-Group/Diffusion-MU-Attack/blob/main/prompts/coco_10k.csv). Demo of our offensive method: [UnlearnDiffAtk](https://huggingface.co/spaces/Intel/UnlearnDiffAtk)\\ Demo of our defensive method: [AdvUnlearn](https://huggingface.co/spaces/Intel/AdvUnlearn) """ EVALUATION_QUEUE_TEXT = """ \[Evaluation Metrics\]: - Pre-Attack Success Rate (Pre-ASR): lower is better; - Post-attack success rate (Post-ASR): lower is better; - Fréchet inception distance(FID): evaluate distributional quality of image generations, lower is better; - CLIP Score: measure contextual alignment with prompt descriptions, higher is better. \[DM Unlearning Tasks\]: - NSFW: Nudity - Style: Van Gogh - Objects: Church, Tench, Parachute, Garbage Truck """ # Which evaluations are you running? how can people reproduce what you have? LLM_BENCHMARKS_TEXT = f""" For more details of Unlearning Methods used in this benchmarks: - [Adversarial Unlearning (AdvUnlearn)](https://github.com/OPTML-Group/AdvUnlearn); - [Erased Stable Diffusion (ESD)](https://github.com/rohitgandikota/erasing); - [Forget-Me-Not (FMN)](https://github.com/SHI-Labs/Forget-Me-Not); - [Ablating Concepts (AC)](https://github.com/nupurkmr9/concept-ablation); - [Unified Concept Editing (UCE)](https://github.com/rohitgandikota/unified-concept-editing); - [concept-SemiPermeable Membrane (SPM)](https://github.com/Con6924/SPM); - [Saliency Unlearning (SalUn)](https://github.com/OPTML-Group/Unlearn-Saliency); - [EraseDiff (ED)](https://github.com/JingWu321/EraseDiff); - [ScissorHands (SH)](https://github.com/JingWu321/Scissorhands). We will evaluate your model on UnlearnDiffAtk Benchmark! \\ Open a [github issue](https://github.com/OPTML-Group/Diffusion-MU-Attack/issues) or email us at zhan1853@msu.edu! """ CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results" CITATION_BUTTON_TEXT = r""" @article{zhang2023generate, title={To Generate or Not? Safety-Driven Unlearned Diffusion Models Are Still Easy To Generate Unsafe Images... For Now}, author={Zhang, Yimeng and Jia, Jinghan and Chen, Xin and Chen, Aochuan and Zhang, Yihua and Liu, Jiancheng and Ding, Ke and Liu, Sijia}, journal={arXiv preprint arXiv:2310.11868}, year={2023} } @article{zhang2024defensive, title={Defensive Unlearning with Adversarial Training for Robust Concept Erasure in Diffusion Models}, author={Zhang, Yimeng and Chen, Xin and Jia, Jinghan and Zhang, Yihua and Fan, Chongyu and Liu, Jiancheng and Hong, Mingyi and Ding, Ke and Liu, Sijia}, journal={arXiv preprint arXiv:2405.15234}, year={2024} } """