first commit

.env.example

@@ -0,0 +1,3 @@
+DATASET_REPO_URL="https://huggingface.co/datasets/{DATASET_ID}"
+FORCE_PUSH="no"
+HF_TOKEN="hf_xxx"

.gitignore

@@ -0,0 +1,163 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Local development
+data/

Makefile

@@ -0,0 +1,10 @@
+.PHONY: style quality
+
+style:
+	python -m black --line-length 119 --target-version py38 .
+	python -m isort .
+
+quality:
+	python -m black --check --line-length 119 --target-version py38 .
+	python -m isort --check-only .
+	python -m flake8 --max-line-length 119 .

README.md

@@ -0,0 +1,70 @@
+---
+title: RLHF
+emoji: 🏢
+colorFrom: red
+colorTo: gray
+sdk: gradio
+sdk_version: 3.1
+app_file: app.py
+pinned: false
+---
+
+An RLHF interface for data collection with [Amazon Mechanical Turk](https://www.mturk.com) and Gradio.
+
+## Instructions for someone to use for their own project
+
+### Install dependencies
+
+First, create a Python virtual environment and install the project's dependencies as follows:
+
+```bash
+python -m pip install -r requirements.txt
+```
+
+### Setting up the Space
+
+1. Clone this repo and deploy it on your own Hugging Face space.
+2. Add the following secrets to your space:
+   - `HF_TOKEN`: One of your Hugging Face tokens.
+   - `DATASET_REPO_URL`: The url to an empty dataset that you created the hub. It
+    can be a private or public dataset.
+   - `FORCE_PUSH`: "yes"
+   When you run this space on mturk and when people visit your space on
+   huggingface.co, the app will use your token to automatically store new HITs
+   in your dataset. Setting `FORCE_PUSH` to "yes" ensures that your repo will
+   force push changes to the dataset during data collection. Otherwise,
+   accidental manual changes to your dataset could result in your space getting
+   merge conflicts as it automatically tries to push the dataset to the hub. For
+   local development, add these three keys to a `.env` file, and consider setting
+   `FORCE_PUSH` to "no".
+
+To launch the Space locally, run:
+
+```bash
+python app.py
+```
+
+The app will then be available at a local address, such as http://127.0.0.1:7860
+
+### Running data collection*
+
+1. On your local repo that you pulled, create a copy of `config.py.example`,
+   just called `config.py`. Now, put keys from your AWS account in `config.py`.
+   These keys should be for an AWS account that has the
+   AmazonMechanicalTurkFullAccess permission. You also need to
+   create an mturk requestor account associated with your AWS account.
+2. Run `python collect.py` locally.
+
+### Profit
+Now, you should be watching hits come into your Hugging Face dataset
+automatically!
+
+### Tips and tricks
+
+- Use caution while doing local development of your space and
+simultaneously running it on mturk. Consider setting `FORCE_PUSH` to "no" in
+your local `.env` file.
+- huggingface spaces have limited computational resources and memory. If you
+run too many HITs and/or assignments at once, then you could encounter issues.
+You could also encounter issues if you are trying to create a dataset that is
+very large. Check the log of your space for any errors that could be happening.

app.py

@@ -0,0 +1,236 @@
+# Basic example for doing model-in-the-loop dynamic adversarial data collection
+# using Gradio Blocks.
+import json
+import os
+import threading
+import uuid
+from pathlib import Path
+from urllib.parse import parse_qs
+from datasets import load_dataset
+import gradio as gr
+from dotenv import load_dotenv
+from huggingface_hub import Repository
+import random
+
+from utils import force_git_push
+
+
+# These variables are for storing the MTurk HITs in a Hugging Face dataset.
+if Path(".env").is_file():
+    load_dotenv(".env")
+DATASET_REPO_URL = os.getenv("DATASET_REPO_URL")
+FORCE_PUSH = os.getenv("FORCE_PUSH")
+HF_TOKEN = os.getenv("HF_TOKEN")
+PROMPT_TEMPLATES = Path("prompt_templates")
+
+DATA_FILENAME = "data.jsonl"
+DATA_FILE = os.path.join("data", DATA_FILENAME)
+repo = Repository(local_dir="data", clone_from=DATASET_REPO_URL, use_auth_token=HF_TOKEN)
+ds = load_dataset("HuggingFaceH4/instruction-pilot-outputs", split="train", use_auth_token=HF_TOKEN)
+
+TOTAL_CNT = 10  # How many user inputs per HIT
+
+# This function pushes the HIT data written in data.jsonl to our Hugging Face
+# dataset every minute. Adjust the frequency to suit your needs.
+PUSH_FREQUENCY = 60
+
+
+def asynchronous_push(f_stop):
+    if repo.is_repo_clean():
+        print("Repo currently clean. Ignoring push_to_hub")
+    else:
+        repo.git_add(auto_lfs_track=True)
+        repo.git_commit("Auto commit by space")
+        if FORCE_PUSH == "yes":
+            force_git_push(repo)
+        else:
+            repo.git_push()
+    if not f_stop.is_set():
+        # call again in 60 seconds
+        threading.Timer(PUSH_FREQUENCY, asynchronous_push, [f_stop]).start()
+
+
+f_stop = threading.Event()
+asynchronous_push(f_stop)
+
+demo = gr.Blocks()
+
+def random_sample_with_least_annotated_examples_first():
+    annotations = open(DATA_FILE, "r").readlines()
+    id_to_count = {}
+    for line in annotations:
+        annotation = json.loads(line)
+        # Only include annotations by actual turkers in the count.
+        if annotation["assignmentId"] != "":
+            example_id = annotation["id"]
+            id_to_count[example_id] = id_to_count.get(example_id, 0) + 1
+    ds_with_annotation_counts = ds.map(lambda example: {"annotation_count": id_to_count.get(example["id"], 0)})
+    ds_with_annotation_counts = ds_with_annotation_counts.shuffle()
+    ds_with_annotation_counts = ds_with_annotation_counts.sort("annotation_count")
+    example = ds_with_annotation_counts.select([0])[0]
+    # We only want to give the annotator 2 choices, so we sample 2 outputs without replacement.
+    example["outputs"] = random.sample(example["outputs"], 2)
+    return example
+
+
+with demo:
+    dummy = gr.Textbox(visible=False)  # dummy for passing assignmentId
+
+    initial_sample = random_sample_with_least_annotated_examples_first()
+
+    # We keep track of state as a JSON
+    state_dict = {
+        "taskId": str(uuid.uuid4()),
+        "assignmentId": "",
+        "cnt": 0,
+        "data": [initial_sample],
+    }
+    state = gr.JSON(state_dict, visible=False)
+
+    gr.Markdown("# Choose the most helpful and honest response.")
+
+    state_display = gr.Markdown(f"Your messages: 0/{TOTAL_CNT}")
+
+    def _select_response(selected_response, state, dummy):
+        if selected_response == "":
+            # Don't do anything if the worker didn't select anything yet.
+            return (
+                gr.update(),
+                gr.update(),
+                gr.update(),
+                gr.update(),
+                gr.update(),
+                gr.update(),
+                state,
+                dummy,
+            )
+        state["cnt"] += 1
+        state_display = f"Your messages: {state['cnt']}/{TOTAL_CNT}"
+        done = state["cnt"] == TOTAL_CNT
+        state["data"][-1]["selected_response"] = selected_response
+        if state["cnt"] == TOTAL_CNT:
+            # Write the HIT data to our local dataset because the worker has
+            # submitted everything now.
+            with open(DATA_FILE, "a") as jsonlfile:
+                json_data_with_assignment_id = [
+                    json.dumps(
+                        dict(
+                            {"assignmentId": state["assignmentId"], "taskId": state["taskId"]},
+                            **datum,
+                        )
+                    )
+                    for datum in state["data"]
+                ]
+                jsonlfile.write("\n".join(json_data_with_assignment_id) + "\n")
+        query = parse_qs(dummy[1:])
+        if "assignmentId" in query and query["assignmentId"][0] != "ASSIGNMENT_ID_NOT_AVAILABLE":
+            # It seems that someone is using this app on mturk. We need to
+            # store the assignmentId in the state before submit_hit_button
+            # is clicked. We can do this here in _predict. We need to save the
+            # assignmentId so that the turker can get credit for their HIT.
+            state["assignmentId"] = query["assignmentId"][0]
+            toggle_final_submit = gr.update(visible=done)
+            toggle_final_submit_preview = gr.update(visible=False)
+        else:
+            toggle_final_submit_preview = gr.update(visible=done)
+            toggle_final_submit = gr.update(visible=False)
+
+        toggle_select_response_button = gr.update(visible=not done)
+
+        new_sample = random_sample_with_least_annotated_examples_first()
+        new_outputs = [obj["output"] for obj in new_sample["outputs"]]
+        state["data"].append(new_sample)
+        past_conversation = gr.update(value=new_sample["prompt"])
+        select_response = gr.update(choices=new_outputs, value="")
+
+        return (
+            past_conversation,
+            select_response,
+            toggle_select_response_button,
+            toggle_final_submit,
+            toggle_final_submit_preview,
+            state_display,
+            state,
+            dummy,
+        )
+
+    # Input fields
+    past_conversation = gr.Markdown(value=initial_sample["prompt"])
+    initial_outputs = [obj["output"] for obj in initial_sample["outputs"]]
+    select_response = gr.Radio(
+        choices=initial_outputs, label="Choose the most helpful and honest response"
+    )
+    select_response_button = gr.Button("Submit Response")
+    submit_hit_button = gr.Button("Submit HIT", visible=False)
+    submit_hit_button_preview = gr.Button(
+        "Submit Work (preview mode; no MTurk HIT credit, but your examples will still be stored)",
+        visible=False,
+    )
+
+    # Button event handlers
+    get_window_location_search_js = """
+        function(select_response, state, dummy) {
+            return [select_response, state, window.location.search];
+        }
+        """
+
+    select_response_button.click(
+        _select_response,
+        inputs=[select_response, state, dummy],
+        outputs=[
+            past_conversation,
+            select_response,
+            select_response_button,
+            submit_hit_button,
+            submit_hit_button_preview,
+            state_display,
+            state,
+            dummy,
+        ],
+        _js=get_window_location_search_js,
+    )
+
+    post_hit_js = """
+        function(state) {
+            // If there is an assignmentId, then the submitter is on mturk
+            // and has accepted the HIT. So, we need to submit their HIT.
+            const form = document.createElement('form');
+            form.action = 'https://workersandbox.mturk.com/mturk/externalSubmit';
+            form.method = 'post';
+            for (const key in state) {
+                const hiddenField = document.createElement('input');
+                hiddenField.type = 'hidden';
+                hiddenField.name = key;
+                hiddenField.value = state[key];
+                form.appendChild(hiddenField);
+            };
+            document.body.appendChild(form);
+            form.submit();
+            return state;
+        }
+        """
+
+    submit_hit_button.click(
+        lambda state: state,
+        inputs=[state],
+        outputs=[state],
+        _js=post_hit_js,
+    )
+
+    refresh_app_js = """
+        function(state) {
+            // The following line here loads the app again so the user can
+            // enter in another preview-mode "HIT".
+            window.location.href = window.location.href;
+            return state;
+        }
+        """
+
+    submit_hit_button_preview.click(
+        lambda state: state,
+        inputs=[state],
+        outputs=[state],
+        _js=refresh_app_js,
+    )
+
+demo.launch()

collect.py

@@ -0,0 +1,133 @@
+import argparse
+from os import path
+
+import boto3
+from boto.mturk.question import ExternalQuestion
+from config import MTURK_KEY, MTURK_SECRET
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--mturk_region", default="us-east-1", help="The region for mturk (default: us-east-1)")
+parser.add_argument(
+    "--space_name",
+    default="Tristan/static-rlhf-interface",
+    help="Name of the accompanying Hugging Face space (default: huggingface/rlhf-interface)",
+)
+parser.add_argument("--num_hits", type=int, default=5, help="The number of HITs.")
+parser.add_argument(
+    "--num_assignments", type=int, default=1, help="The number of times that the HIT can be accepted and completed."
+)
+parser.add_argument(
+    "--live_mode",
+    action="store_true",
+    help="""
+    Whether to run in live mode with real turkers. This will charge your account money.
+    If you don't use this flag, the HITs will be deployed on the sandbox version of mturk,
+    which will not charge your account money.
+    """,
+)
+parser.add_argument(
+    "--refresh_qualification_test",
+    action="store_true",
+    help="""
+    Whether to refresh the qualification test. If you've made edits to the test
+    xml files, it is necessary to do this.
+    """,
+)
+parser.add_argument(
+    "--custom_qualification_test",
+    action="store_true",
+    help="""
+    Whether to require the custom qualification test.
+    """,
+)
+parser.add_argument(
+    "--master_turkers",
+    action="store_true",
+    help="""
+    Whether to only use turkers with the master qualification.
+    """,
+)
+parser.add_argument(
+    "--us_turkers",
+    action="store_true",
+    help="""
+    Whether to only use US-based turkers.
+    """,
+)
+
+args = parser.parse_args()
+
+MTURK_URL = f"https://mturk-requester{'' if args.live_mode else '-sandbox'}.{args.mturk_region}.amazonaws.com"
+
+mturk = boto3.client(
+    "mturk",
+    aws_access_key_id=MTURK_KEY,
+    aws_secret_access_key=MTURK_SECRET,
+    region_name=args.mturk_region,
+    endpoint_url=MTURK_URL,
+)
+
+# This is the URL that makes the space embeddable in an mturk iframe
+question = ExternalQuestion(f"https://hf.space/embed/{args.space_name}/+?__theme=light", frame_height=600)
+
+qualification_requirements=[]
+
+if args.master_turkers:
+    qualification_requirements.append({
+        QualificationTypeId: '2F1QJWKUDD8XADTFD2Q0G6UTO95ALH',
+        Comparator: 'Exists'
+      })
+
+if args.us_turkers:
+    qualification_requirements.append({
+        QualificationTypeId: '00000000000000000071',
+        Comparator: 'In',
+        LocaleValues: [
+          { Country: "US" },
+        ]
+      })
+
+if args.custom_qualification_test:
+    qualification_type_id = (
+        open("qualification_type_id.txt", "r").read() if path.exists("qualification_type_id.txt") else None
+    )
+    if args.refresh_qualification_test or qualification_type_id is None:
+        if qualification_type_id is not None:
+            mturk.delete_qualification_type(QualificationTypeId=qualification_type_id)
+        response = mturk.create_qualification_type(
+            Name="rlhf--qualification",
+            Keywords="RLHF qualification",
+            Description="Qualification test for RLHF task.",
+            QualificationTypeStatus="Active",
+            Test=open("qualification_questions.xml", mode="r").read(),
+            AnswerKey=open("qualification_answers.xml", mode="r").read(),
+            TestDurationInSeconds=3600,
+            AutoGranted=False,
+        )
+        qualification_type_id = response["QualificationType"]["QualificationTypeId"]
+        open("qualification_type_id.txt", "w+").write(qualification_type_id)
+    qualification_requirements.append({
+            "QualificationTypeId": qualification_type_id,
+            "Comparator": "Exists",
+            "RequiredToPreview": False,
+            "ActionsGuarded": "Accept",
+        })
+
+for i in range(args.num_hits):
+    new_hit = mturk.create_hit(
+        Title="RLHF HIT",
+        Description="Interact with an AI",
+        Keywords="chatbot",
+        Reward="0.25",
+        MaxAssignments=args.num_assignments,
+        LifetimeInSeconds=172800,
+        AssignmentDurationInSeconds=600,
+        AutoApprovalDelayInSeconds=14400,
+        Question=question.get_as_xml(),
+        QualificationRequirements=qualification_requirements,
+    )
+
+print(
+    f"HIT Group Link: https://worker{'' if args.live_mode else 'sandbox'}.mturk.com/mturk/preview?groupId="
+    + new_hit["HIT"]["HITGroupId"]
+)

config.py.example

@@ -0,0 +1,6 @@
+# Fill in the information and rename this file config.py
+# You can obtain the key and secret in the AWS Identity
+# and Access Management (IAM) panel.
+
+MTURK_KEY = ''
+MTURK_SECRET = ''

requirements.txt

@@ -0,0 +1,5 @@
+boto3==1.24.32
+boto==2.49.0
+huggingface_hub==0.8.1
+python-dotenv==0.20.0
+datasets==2.9.0

utils.py

@@ -0,0 +1,39 @@
+import subprocess
+
+from huggingface_hub.repository import _lfs_log_progress
+
+
+def force_git_push(
+    repo,
+):
+    """
+    force a simple git push
+    Blocking. Will return url to commit on remote
+    repo.
+    """
+    command = "git push --force"
+
+    try:
+        with _lfs_log_progress():
+            process = subprocess.Popen(
+                command.split(),
+                stderr=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                encoding="utf-8",
+                cwd=repo.local_dir,
+            )
+
+            stdout, stderr = process.communicate()
+            return_code = process.poll()
+            process.kill()
+
+            if len(stderr):
+                print(stderr)
+
+            if return_code:
+                raise subprocess.CalledProcessError(return_code, process.args, output=stdout, stderr=stderr)
+
+    except subprocess.CalledProcessError as exc:
+        raise EnvironmentError(exc.stderr)
+
+    return repo.git_head_commit_url()