Merge

.gitignore

@@ -0,0 +1,141 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# VSCode
+.vscode
+
+# IntelliJ
+.idea
+
+# Mac .DS_Store
+.DS_Store
+
+# More test things
+wandb

README.md

@@ -4,9 +4,84 @@ emoji: 🌖
 colorFrom: indigo
 colorTo: purple
 sdk: gradio
-sdk_version: 3.13.2
-app_file: app.py
+sdk_version: 3.14.0
+app_file: src/app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+## Accelerate Integration Examples
+
+This is an interactive utility to show users how to incorporate parts of 🤗 Accelerate into their code. 
+To use it select a feature to add and a github-like `diff` will be rendered showing what code to remove
+and add based on the initial template code.
+
+These are more simplified versions of examples that exist in the [accelerate](https://github.com/huggingface/accelerate) and [transformers](https://github.com/huggingface/transforms) repositories. 
+
+## How each example is made
+
+In the `code_examples` folder are basic text-like files which contain a much simplified version of some integration. For example:
+
+```
+<pre>
+from accelerate import Accelerator
+accelerator = Accelerator()
+train_dataloader, model, optimizer scheduler = accelerator.prepare(
+        dataloader, model, optimizer, scheduler
+)
+
+model.train()
+for batch in train_dataloader:
+&nbsp;&nbsp;&nbsp;&nbsp;optimizer.zero_grad()
+    inputs, targets = batch
+    outputs = model(inputs)
+    loss = loss_function(outputs, targets)
+    accelerator.backward(loss)
+    optimizer.step()
+    scheduler.step()
+</pre>
+```
+
+These are done in an HTML-like syntax, with `pre` being used instead of three back-ticks for formatting purposes. 
+
+## Creating a `diff`
+
+To create a diff, a similar `pre` tag should be wrapped around the code, and a single `+` or `-` (showing an addition or subtraction) should be added to the code with no extra spacing or formatting. The tool will automatically know how to render these properly. 
+
+For example:
+
+```
+<pre>
++from accelerate import Accelerator
++accelerator = Accelerator()
++dataloader, model, optimizer scheduler = accelerator.prepare(
++        dataloader, model, optimizer, scheduler
++)
+
+for batch in dataloader:
+    optimizer.zero_grad()
+    inputs, targets = batch
+-    inputs = inputs.to(device)
+-    targets = targets.to(device)
+    outputs = model(inputs)
+    loss = loss_function(outputs, targets)
+-    loss.backward()
++    accelerator.backward(loss)
+    optimizer.step()
+    scheduler.step()</pre>
+```
+
+Also note that the initial `pre` is on a newline, and the latter `</pre>` is not. 
+
+## Rendering the diff
+
+After a diff and starter (if needed) has been made, if a new template was created add it to `TEMPLATES` in `src/template.py`. Otherwise in `app.py` modify the `change` function to properly point to the new integration example to show on a particular selection:
+
+```
+def change(inp):
+    if inp == "Basic":
+        return (templates["initial"], highlight(inp), "## Accelerate Code (Base Integration)")
+    elif inp == "Calculating Metrics":
+        return (templates["initial_with_metrics"], highlight(inp), f"## Accelerate Code ({inp})")
+    else:
+        return (templates["accelerate"], highlight(inp), f"## Accelerate Code ({inp})")
+```

code_samples/accelerate

@@ -1,4 +1,4 @@
-```
+<pre>
 from accelerate import Accelerator
 accelerator = Accelerator()
 train_dataloader, model, optimizer scheduler = accelerator.prepare(
@@ -7,11 +7,11 @@ train_dataloader, model, optimizer scheduler = accelerator.prepare(
 
 model.train()
 for batch in train_dataloader:
-    optimizer.zero_grad()
+&nbsp;&nbsp;&nbsp;&nbsp;optimizer.zero_grad()
     inputs, targets = batch
     outputs = model(inputs)
     loss = loss_function(outputs, targets)
     accelerator.backward(loss)
     optimizer.step()
     scheduler.step()
-```
+</pre>

code_samples/basic

@@ -15,5 +15,4 @@ for batch in dataloader:
 -    loss.backward()
 +    accelerator.backward(loss)
     optimizer.step()
-    scheduler.step()
-</pre>
+    scheduler.step()</pre>

code_samples/calculating_metrics

@@ -32,5 +32,4 @@ for batch in eval_dataloader:
         predictions = predictions,
         references = references
     )
-print(metric.compute())
-<pre>
+print(metric.compute())</pre>

code_samples/checkpointing

@@ -13,5 +13,4 @@ for batch in dataloader:
     accelerator.backward(loss)
     optimizer.step()
     scheduler.step()
-+accelerator.save_state("checkpoint_dir")
-</pre>
++accelerator.save_state("checkpoint_dir")</pre>

code_samples/gradient_accumulation

@@ -15,5 +15,4 @@ for batch in dataloader:
       loss = loss_function(outputs, targets)
       accelerator.backward(loss)
       optimizer.step()
-      scheduler.step()
-</pre>
+      scheduler.step()</pre>

code_samples/initial

@@ -1,4 +1,4 @@
-```
+<pre>
 for batch in dataloader:
     optimizer.zero_grad()
     inputs, targets = batch
@@ -8,5 +8,4 @@ for batch in dataloader:
     loss = loss_function(outputs, targets)
     loss.backward()
     optimizer.step()
-    scheduler.step()
-```
+    scheduler.step()</pre>

code_samples/initial_with_metrics

@@ -1,4 +1,4 @@
-```
+<pre>
 import evaluate
 metric = evaluate.load("accuracy")
 for batch in train_dataloader:
@@ -24,5 +24,4 @@ for batch in eval_dataloader:
         predictions = predictions,
         references = references
     )
-print(metric.compute())
-```
+print(metric.compute())</pre>

requirements.txt

@@ -0,0 +1 @@
+gradio

src/app.py

@@ -0,0 +1,40 @@
+import gradio as gr
+from markup import highlight
+from template import get_templates
+
+templates = get_templates()
+
+def change(inp):
+    """Based on an `inp`, render and highlight the appropriate code sample.
+
+    Args:
+        inp (`str`):
+            The input button from the interface.
+
+    Returns:
+        `tuple`: A tuple of the initial code, the highlighted code, and the title for the section.
+    """
+    if inp == "Basic":
+        return (templates["initial"], highlight(inp), "## Accelerate Code (Base Integration)")
+    elif inp == "Calculating Metrics":
+        return (templates["initial_with_metrics"], highlight(inp), f"## Accelerate Code ({inp})")
+    else:
+        return (templates["accelerate"], highlight(inp), f"## Accelerate Code ({inp})")
+
+with gr.Blocks() as demo:
+    gr.Markdown(f'''# Accelerate Training Code Template Generator
+Here is a very basic Python training loop.
+Select how you would like to introduce an Accelerate capability to add to it.''')
+    inp = gr.Radio(
+        ["Basic", "Calculating Metrics", "Checkpointing", "Gradient Accumulation", ], 
+        label="Select a feature"
+    )
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown("## Initial Code")
+            code = gr.Markdown(templates["initial"])
+        with gr.Column():
+            feature = gr.Markdown("## Accelerate Code")
+            out = gr.Markdown()
+    inp.change(fn=change, inputs=inp, outputs=[code, out, feature])  
+demo.launch()

app.py → src/markup.py

@@ -1,18 +1,47 @@
-import gradio as gr
+# Copyright 2023 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 _remove_color = "rgb(103,6,12)"
 _addition_color = "rgb(6,103,12)"
 
 def mark_text(text, add=True):
+    """Marks text with a highlight color for addition or removal.
+
+    Args:
+        text (`str`):
+            Some code to be marked.
+        add (`bool`, *optional*, defaults to True):
+            Whether to mark the text as an addition or a removal.
+
+    Returns:
+        `str`: The marked text as an HTML `mark` element.
+    """
     if add:
         color = _addition_color
     else:
         color = _remove_color
     return f'<mark style="background-color:{color}!important;color:white!important">{text}</mark>'
 
-def highlight(option):
+def highlight(option:str):
+    """Takes an option and returns the respective highlighted code sample.
+
+    Args:
+        option (`str`):
+            A menu option chosen from the interface.
+    """
     filename = option.lower().replace(' ', '_')
-    with open(f"code_samples/{filename}") as f:
+    with open(f"../code_samples/{filename}") as f:
         output = f.read()
     lines = output.split("\n")
     for i,line in enumerate(lines):
@@ -25,38 +54,3 @@ def highlight(option):
         else:
             lines[i] = "  " + line
     return "\n".join(lines).rstrip()
-
-with open("code_samples/initial") as f:
-    template = f.read()
-
-with open("code_samples/accelerate") as f:
-    accelerated_template = f.read()
-
-with open("code_samples/initial_with_metrics") as f:
-    metrics_template = f.read()
-
-def change(inp):
-    if inp == "Basic":
-        return (template, highlight(inp), "## Accelerate Code (Base Integration)")
-    elif inp == "Calculating Metrics":
-        return (metrics_template, highlight(inp), f"## Accelerate Code ({inp})")
-    else:
-        return (accelerated_template, highlight(inp), f"## Accelerate Code ({inp})")
-
-with gr.Blocks() as demo:
-    gr.Markdown(f'''# Accelerate Template Generator
-Here is a very basic Python training loop.
-Select how you would like to introduce an Accelerate capability to add to it.''')
-    inp = gr.Radio(
-        ["Basic", "Calculating Metrics", "Checkpointing", "Gradient Accumulation", ], 
-        label="Select a feature"
-    )
-    with gr.Row():
-        with gr.Column():
-            gr.Markdown("## Initial Code")
-            code = gr.Markdown(template)
-        with gr.Column():
-            feature = gr.Markdown("## Accelerate Code")
-            out = gr.Markdown()
-    inp.change(fn=change, inputs=inp, outputs=[code, out, feature])  
-demo.launch()

src/template.py

@@ -0,0 +1,24 @@
+# Copyright 2023 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+TEMPLATES = ["initial", "initial_with_metrics", "accelerate"]
+
+def get_templates() -> dict:
+    """
+    Returns a dictionary of template type to code content
+    """
+    return {
+        template: open(f"../code_samples/{template}").read()
+        for template in TEMPLATES
+    }