Spaces:

MLCommons
/

croissant-editor

Running

App Files Files Community

marcenacp commited on Dec 5, 2023

Commit

7b9203f

•

1 Parent(s): c439062

Deploy (see actual commits on https://github.com/mlcommons/croissant).

Browse files

Files changed (17) hide show

components/safe_button.py +36 -0
components/tabs/frontend/build/asset-manifest.json +3 -3
components/tabs/frontend/build/index.html +1 -1
components/tabs/frontend/build/static/js/main.e6b754d8.js +0 -0
components/tabs/frontend/build/static/js/main.e6b754d8.js.LICENSE.txt +73 -0
components/tabs/frontend/build/static/js/main.e6b754d8.js.map +0 -0
components/tabs/frontend/src/Tabs.tsx +30 -9
core/constants.py +5 -0
core/files.py +4 -1
core/names.py +5 -0
core/names_test.py +1 -0
events/metadata.py +2 -1
views/files.py +25 -6
views/overview.py +11 -6
views/record_sets.py +32 -18
views/source.py +46 -11
views/splash.py +32 -2

components/safe_button.py ADDED Viewed

	@@ -0,0 +1,36 @@

+import streamlit as st
+HAS_CONFIRMED = "SAFELY_UPDATE"
+def handle_on_click(on_click):
+    """Handles on_click by waiting for the confirmation."""
+    if st.session_state.get(HAS_CONFIRMED):
+        return on_click
+    else:
+        def toggle_has_confirmed(*args, **kwargs):
+            del args, kwargs  # unused.
+            st.session_state[HAS_CONFIRMED] = not st.session_state.get(HAS_CONFIRMED)
+        return toggle_has_confirmed
+def button_with_confirmation(
+    label: str,
+    key: str = None,
+    on_click=None,
+    args=None,
+    kwargs=None,
+):
+    """Implements a safe button that asks for confirmation before executing on_click."""
+    st.button(
+        label,
+        on_click=handle_on_click(on_click),
+        args=args,
+        kwargs=kwargs,
+        key=key,
+        type="secondary",
+    )
+    if st.session_state.get(HAS_CONFIRMED):
+        st.error(f"Do you really want to {label.lower()}? Click again to confirm.")

components/tabs/frontend/build/asset-manifest.json CHANGED Viewed

@@ -1,10 +1,10 @@
 {
   "files": {
-    "main.js": "./static/js/main.716a0ab4.js",
     "index.html": "./index.html",
-    "main.716a0ab4.js.map": "./static/js/main.716a0ab4.js.map"
   },
   "entrypoints": [
-    "static/js/main.716a0ab4.js"
   ]
 }

 {
   "files": {
+    "main.js": "./static/js/main.e6b754d8.js",
     "index.html": "./index.html",
+    "main.e6b754d8.js.map": "./static/js/main.e6b754d8.js.map"
   },
   "entrypoints": [
+    "static/js/main.e6b754d8.js"
   ]
 }

components/tabs/frontend/build/index.html CHANGED Viewed

@@ -1 +1 @@

- <!doctype html><html lang="en"><head><title>Streamlit Tabs Component</title><meta charset="UTF-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Streamlit Tree Component"/><script defer="defer" src="./static/js/main.~~716a0ab4~~.js"></script></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

+ <!doctype html><html lang="en"><head><title>Streamlit Tabs Component</title><meta charset="UTF-8"/><meta name="viewport" content="width=device-width,initial-scale=1"/><meta name="theme-color" content="#000000"/><meta name="description" content="Streamlit Tree Component"/><script defer="defer" src="./static/js/main.e6b754d8.js"></script></head><body><noscript>You need to enable JavaScript to run this app.</noscript><div id="root"></div></body></html>

components/tabs/frontend/build/static/js/main.e6b754d8.js ADDED Viewed

The diff for this file is too large to render. See raw diff

components/tabs/frontend/build/static/js/main.e6b754d8.js.LICENSE.txt ADDED Viewed

	@@ -0,0 +1,73 @@

+/*
+object-assign
+(c) Sindre Sorhus
+@license MIT
+*/
+/**
+ * @license React
+ * react-dom.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+/**
+ * @license React
+ * react-is.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+/**
+ * @license React
+ * react-jsx-runtime.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+/**
+ * @license React
+ * react.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+/**
+ * @license React
+ * scheduler.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+/** @license React v16.13.1
+ * react-is.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+/** @license React v16.14.0
+ * react.production.min.js
+ *
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */

components/tabs/frontend/build/static/js/main.e6b754d8.js.map ADDED Viewed

The diff for this file is too large to render. See raw diff

components/tabs/frontend/src/Tabs.tsx CHANGED Viewed

@@ -10,6 +10,7 @@ import Tab from "@mui/material/Tab"
 import Box from "@mui/material/Box"
 import { ThemeProvider, createTheme } from "@mui/material"
 import { orange } from "@mui/material/colors"
 const theme = createTheme({
   palette: {
@@ -55,18 +56,38 @@ function BasicTabs({
           </Tabs>
         </Box>
       </Box>
-      <Button
-        disabled={!json}
-        variant="outlined"
-        href={
           json
-            ? `data:text/json;charset=utf-8,${encodeURIComponent(json.content)}`
-            : ""
         }
-        download={json ? json.name : ""}
       >
-        Export
-      </Button>
     </div>
   )
 }

 import Box from "@mui/material/Box"
 import { ThemeProvider, createTheme } from "@mui/material"
 import { orange } from "@mui/material/colors"
+import Tooltip from "@mui/material/Tooltip"
 const theme = createTheme({
   palette: {
           </Tabs>
         </Box>
       </Box>
+      <Tooltip
+        title={
           json
+            ? "Download the Croissant JSON-LD file."
+            : "Go to the overview to understand why the Croissant JSON-LD file cannot be generated."
         }
+        placement="left"
       >
+        <span>
+          <Button
+            disabled={!json}
+            disableElevation
+            variant="contained"
+            href={
+              json
+                ? `data:text/json;charset=utf-8,${encodeURIComponent(
+                    json.content
+                  )}`
+                : ""
+            }
+            download={json ? json.name : ""}
+            sx={{
+              color: "white",
+              padding: "6px 20px",
+              textAlign: "center",
+              whiteSpace: "nowrap",
+            }}
+          >
+            Download 🥐 file
+          </Button>
+        </span>
+      </Tooltip>
     </div>
   )
 }

core/constants.py CHANGED Viewed

@@ -35,3 +35,8 @@ METADATA = "Metadata"
 RESOURCES = "Resources"
 RECORD_SETS = "Record Sets"
 TABS = [OVERVIEW, METADATA, RESOURCES, RECORD_SETS]

 RESOURCES = "Resources"
 RECORD_SETS = "Record Sets"
 TABS = [OVERVIEW, METADATA, RESOURCES, RECORD_SETS]
+NAMES_INFO = (
+    "Names are used as identifiers. They are unique and cannot contain special"
+    " characters. The interface will replace any special characters."
+)

core/files.py CHANGED Viewed

@@ -142,7 +142,10 @@ def get_dataframe(file_type: FileType, file: io.BytesIO | epath.Path) -> pd.Data
     elif file_type == FileTypes.PARQUET:
         df = pd.read_parquet(file)
     else:
-        raise NotImplementedError()
     return df.infer_objects()

     elif file_type == FileTypes.PARQUET:
         df = pd.read_parquet(file)
     else:
+        raise NotImplementedError(
+            f"File type {file_type} is not supported. Please, open an issue on GitHub:"
+            " https://github.com/mlcommons/croissant/issues/new"
+        )
     return df.infer_objects()

core/names.py CHANGED Viewed

@@ -1,8 +1,13 @@
 """Module to handle naming of RecordSets and distribution."""
 def find_unique_name(names: set[str], name: str):
     """Find a unique UID."""
     while name in names:
         name = f"{name}_0"
     return name

 """Module to handle naming of RecordSets and distribution."""
+import re
+NAME_PATTERN_REGEX = "[^a-zA-Z0-9\\-_\\.]"
 def find_unique_name(names: set[str], name: str):
     """Find a unique UID."""
+    name = re.sub(NAME_PATTERN_REGEX, "_", name)
     while name in names:
         name = f"{name}_0"
     return name

core/names_test.py CHANGED Viewed

@@ -5,6 +5,7 @@ from .names import find_unique_name
 def test_find_unique_name():
     names = set(["first", "second", "first_0"])
     assert find_unique_name(names, "first") == "first_0_0"
     assert find_unique_name(names, "second") == "second_0"
     assert find_unique_name(names, "third") == "third"

 def test_find_unique_name():
     names = set(["first", "second", "first_0"])
+    assert find_unique_name(names, "are there spaces") == "are_there_spaces"
     assert find_unique_name(names, "first") == "first_0_0"
     assert find_unique_name(names, "second") == "second_0"
     assert find_unique_name(names, "third") == "third"

events/metadata.py CHANGED Viewed

@@ -2,6 +2,7 @@ import enum
 import streamlit as st
 from core.state import Metadata
 # List from:
@@ -97,7 +98,7 @@ class MetadataEvent(enum.Enum):
 def handle_metadata_change(event: MetadataEvent, metadata: Metadata, key: str):
     if event == MetadataEvent.NAME:
-        metadata.name = st.session_state[key]
     elif event == MetadataEvent.DESCRIPTION:
         metadata.description = st.session_state[key]
     elif event == MetadataEvent.LICENSE:

 import streamlit as st
+from core.names import find_unique_name
 from core.state import Metadata
 # List from:
 def handle_metadata_change(event: MetadataEvent, metadata: Metadata, key: str):
     if event == MetadataEvent.NAME:
+        metadata.name = find_unique_name(set(), st.session_state[key])
     elif event == MetadataEvent.DESCRIPTION:
         metadata.description = st.session_state[key]
     elif event == MetadataEvent.LICENSE:

views/files.py CHANGED Viewed

@@ -1,8 +1,9 @@
-from etils import epath
 import streamlit as st
 from components.tree import render_tree
 from core.constants import DF_HEIGHT
 from core.constants import OAUTH_CLIENT_ID
 from core.files import code_to_index
 from core.files import file_from_form
@@ -188,9 +189,10 @@ def _render_resource_details(selected_file: Resource):
             col1, col2 = st.columns([1, 1])
             col1.button("Close", key=f"{i}_close", on_click=close, type="primary")
-            col2.button(
-                "⚠️ Remove", key=f"{i}_remove", on_click=delete_line, type="secondary"
-            )
 def _render_resource(prefix: int, file: Resource, is_file_object: bool):
@@ -201,6 +203,11 @@ def _render_resource(prefix: int, file: Resource, is_file_object: bool):
         default=file.contained_in,
         options=parent_options,
         key=key,
         on_change=handle_resource_change,
         args=(ResourceEvent.CONTAINED_IN, file, key),
     )
@@ -209,6 +216,7 @@ def _render_resource(prefix: int, file: Resource, is_file_object: bool):
         needed_field("Name"),
         value=file.name,
         key=key,
         on_change=handle_resource_change,
         args=(ResourceEvent.NAME, file, key),
     )
@@ -224,9 +232,10 @@ def _render_resource(prefix: int, file: Resource, is_file_object: bool):
     if is_file_object:
         key = f"{prefix}_content_url"
         st.text_input(
-            needed_field("Content URL"),
             value=file.content_url,
             key=key,
             on_change=handle_resource_change,
             args=(ResourceEvent.CONTENT_URL, file, key),
         )
@@ -243,6 +252,7 @@ def _render_resource(prefix: int, file: Resource, is_file_object: bool):
             "Content size",
             value=file.content_size,
             key=key,
             on_change=handle_resource_change,
             args=(ResourceEvent.CONTENT_SIZE, file, key),
         )
@@ -261,6 +271,10 @@ def _render_resource(prefix: int, file: Resource, is_file_object: bool):
         index=code_to_index(file.encoding_format),
         options=FILE_TYPES.keys(),
         key=key,
         on_change=handle_resource_change,
         args=(ResourceEvent.ENCODING_FORMAT, file, key),
     )
@@ -269,4 +283,9 @@ def _render_resource(prefix: int, file: Resource, is_file_object: bool):
         if file.df is not None:
             st.dataframe(file.df, height=DF_HEIGHT)
         else:
-            st.button("Trigger download", on_click=trigger_download, args=(file,))

 import streamlit as st
+from components.safe_button import button_with_confirmation
 from components.tree import render_tree
 from core.constants import DF_HEIGHT
+from core.constants import NAMES_INFO
 from core.constants import OAUTH_CLIENT_ID
 from core.files import code_to_index
 from core.files import file_from_form
             col1, col2 = st.columns([1, 1])
             col1.button("Close", key=f"{i}_close", on_click=close, type="primary")
+            with col2:
+                button_with_confirmation(
+                    "Remove", key=f"{i}_remove", on_click=delete_line
+                )
 def _render_resource(prefix: int, file: Resource, is_file_object: bool):
         default=file.contained_in,
         options=parent_options,
         key=key,
+        help=(
+            "FileObjects and FileSets can be nested. Specifying `Parents` allows to"
+            " nest a FileObject/FileSet within another FileObject/FileSet. An example"
+            " of this is when images (FileSet) are nested within an archive (FileSet)."
+        ),
         on_change=handle_resource_change,
         args=(ResourceEvent.CONTAINED_IN, file, key),
     )
         needed_field("Name"),
         value=file.name,
         key=key,
+        help=f"The name of the resource. {NAMES_INFO}",
         on_change=handle_resource_change,
         args=(ResourceEvent.NAME, file, key),
     )
     if is_file_object:
         key = f"{prefix}_content_url"
         st.text_input(
+            needed_field("Content URL or local path"),
             value=file.content_url,
             key=key,
+            help="The URL or local file path pointing to the original FileObject.",
             on_change=handle_resource_change,
             args=(ResourceEvent.CONTENT_URL, file, key),
         )
             "Content size",
             value=file.content_size,
             key=key,
+            help="The size of the original FileObject in bytes.",
             on_change=handle_resource_change,
             args=(ResourceEvent.CONTENT_SIZE, file, key),
         )
         index=code_to_index(file.encoding_format),
         options=FILE_TYPES.keys(),
         key=key,
+        help=(
+            "MIME type corresponding to"
+            " ([sc:encodingFormat](https://schema.org/encodingFormat))."
+        ),
         on_change=handle_resource_change,
         args=(ResourceEvent.ENCODING_FORMAT, file, key),
     )
         if file.df is not None:
             st.dataframe(file.df, height=DF_HEIGHT)
         else:
+            st.button(
+                "Trigger download",
+                disabled=bool(file.content_url),
+                on_click=trigger_download,
+                args=(file,),
+            )

views/overview.py CHANGED Viewed

@@ -3,6 +3,7 @@ from typing import Any
 import streamlit as st
 from core.state import Metadata
 import mlcroissant as mlc
 from utils import needed_field
@@ -51,6 +52,7 @@ def render_overview():
             label=needed_field("Name"),
             key=key,
             value=metadata.name,
             placeholder="Dataset",
             on_change=handle_metadata_change,
             args=(MetadataEvent.NAME, metadata, key),
@@ -82,8 +84,15 @@ def render_overview():
             * 100
             / (3 * metadata_weight)
         )
-        col_a.metric("Completion", f"{completion}%")
-        col_b.metric("Number of metadata", fields)
         col_c.metric("Number of resources", len(metadata.distribution))
         col_d.metric("Number of RecordSets", len(metadata.record_sets))
     with col2:
@@ -96,10 +105,6 @@ def render_overview():
                     warning += "**Errors**\n"
                     for error in issues.errors:
                         warning += f"{error}\n"
-                if issues.warnings:
-                    warning += "**Warnings**\n"
-                    for warning in issues.warnings:
-                        warning += f"{warning}\n"
             except mlc.ValidationError as exception:
                 warning += "**Errors**\n"
                 warning += f"{str(exception)}\n"

 import streamlit as st
+from core.constants import NAMES_INFO
 from core.state import Metadata
 import mlcroissant as mlc
 from utils import needed_field
             label=needed_field("Name"),
             key=key,
             value=metadata.name,
+            help=f"The name of the dataset. {NAMES_INFO}",
             placeholder="Dataset",
             on_change=handle_metadata_change,
             args=(MetadataEvent.NAME, metadata, key),
             * 100
             / (3 * metadata_weight)
         )
+        col_a.metric(
+            "Completion",
+            f"{completion}%",
+            help=(
+                "Approximation of the total completion based on the number of fields"
+                " that are filled."
+            ),
+        )
+        col_b.metric("Number of metadata fields", fields)
         col_c.metric("Number of resources", len(metadata.distribution))
         col_d.metric("Number of RecordSets", len(metadata.record_sets))
     with col2:
                     warning += "**Errors**\n"
                     for error in issues.errors:
                         warning += f"{error}\n"
             except mlc.ValidationError as exception:
                 warning += "**Errors**\n"
                 warning += f"{str(exception)}\n"

views/record_sets.py CHANGED Viewed

@@ -9,6 +9,8 @@ import pandas as pd
 from rdflib import term
 import streamlit as st
 from core.data_types import MLC_DATA_TYPES
 from core.data_types import mlc_to_str_data_type
 from core.data_types import STR_DATA_TYPES
@@ -239,6 +241,7 @@ def _render_left_panel():
                 needed_field("Name"),
                 placeholder="Name without special character.",
                 key=key,
                 value=record_set.name,
                 on_change=handle_record_set_change,
                 args=(RecordSetEvent.NAME, record_set, key),
@@ -254,16 +257,24 @@ def _render_left_panel():
             )
             key = f"{prefix}-is-enumeration"
             st.checkbox(
-                "Whether the RecordSet is an enumeration",
                 key=key,
                 value=record_set.is_enumeration,
                 on_change=handle_record_set_change,
                 args=(RecordSetEvent.IS_ENUMERATION, record_set, key),
             )
             key = f"{prefix}-has-data"
             st.checkbox(
-                "Whether the RecordSet has in-line data",
                 key=key,
                 value=bool(record_set.data),
                 on_change=handle_record_set_change,
                 args=(RecordSetEvent.HAS_DATA, record_set, key),
@@ -271,7 +282,7 @@ def _render_left_panel():
             joins = _find_joins(record_set.fields)
             has_join = st.checkbox(
-                "Whether the RecordSet contains joins. To add a new join, add a field"
                 " with a source in `RecordSet`/`FileSet`/`FileObject` and a reference"
                 " to another `RecordSet`/`FileSet`/`FileObject`.",
                 key=f"{prefix}-has-joins",
@@ -323,8 +334,14 @@ def _render_left_panel():
             )
             data_editor_key = _data_editor_key(record_set_key, record_set)
             st.markdown(
-                f"{needed_field('Fields')} (add/delete fields by directly editing the"
-                " table)"
             )
             st.data_editor(
                 fields,
@@ -380,9 +397,8 @@ def _render_left_panel():
                 args=(record_set_key, record_set),
             )
             key = f"{prefix}-delete-record-set"
-            st.button(
-                "⚠️ Delete RecordSet",
-                type="primary",
                 key=key,
                 on_click=_handle_remove_record_set,
                 args=(record_set_key,),
@@ -437,6 +453,7 @@ def _render_right_panel():
                     needed_field("Name"),
                     placeholder="Name without special character.",
                     key=key,
                     value=field.name,
                     on_change=handle_field_change,
                     args=(FieldEvent.NAME, field, key),
@@ -450,32 +467,29 @@ def _render_right_panel():
                     value=field.description,
                     args=(FieldEvent.DESCRIPTION, field, key),
                 )
                 if field.data_types:
                     data_type = field.data_types[0]
                     if isinstance(data_type, str):
                         data_type = term.URIRef(data_type)
                     if data_type in MLC_DATA_TYPES:
                         data_type_index = MLC_DATA_TYPES.index(data_type)
-                    else:
-                        data_type_index = None
-                else:
-                    data_type_index = None
                 key = f"{prefix}-datatypes"
                 col3.selectbox(
                     needed_field("Data type"),
                     index=data_type_index,
                     options=STR_DATA_TYPES,
                     key=key,
                     on_change=handle_field_change,
                     args=(FieldEvent.DATA_TYPE, field, key),
                 )
                 possible_sources = _get_possible_sources(metadata)
-                render_source(
-                    record_set_key, record_set, field, field_key, possible_sources
-                )
-                render_references(
-                    record_set_key, record_set, field, field_key, possible_sources
-                )
                 st.divider()

 from rdflib import term
 import streamlit as st
+from components.safe_button import button_with_confirmation
+from core.constants import NAMES_INFO
 from core.data_types import MLC_DATA_TYPES
 from core.data_types import mlc_to_str_data_type
 from core.data_types import STR_DATA_TYPES
                 needed_field("Name"),
                 placeholder="Name without special character.",
                 key=key,
+                help=f"The name of the RecordSet. {NAMES_INFO}",
                 value=record_set.name,
                 on_change=handle_record_set_change,
                 args=(RecordSetEvent.NAME, record_set, key),
             )
             key = f"{prefix}-is-enumeration"
             st.checkbox(
+                "The RecordSet is an enumeration",
                 key=key,
+                help=(
+                    "Enumerations indicate that the RecordSet takes its values in a"
+                    " finite set. Similar to `ClassLabel` in TFDS or Hugging Face."
+                ),
                 value=record_set.is_enumeration,
                 on_change=handle_record_set_change,
                 args=(RecordSetEvent.IS_ENUMERATION, record_set, key),
             )
             key = f"{prefix}-has-data"
             st.checkbox(
+                "The RecordSet has in-line data",
                 key=key,
+                help=(
+                    "In-line data allows to embed data directly within the JSON-LD"
+                    " without referencing another data source."
+                ),
                 value=bool(record_set.data),
                 on_change=handle_record_set_change,
                 args=(RecordSetEvent.HAS_DATA, record_set, key),
             joins = _find_joins(record_set.fields)
             has_join = st.checkbox(
+                "The RecordSet contains joins. To add a new join, add a field"
                 " with a source in `RecordSet`/`FileSet`/`FileObject` and a reference"
                 " to another `RecordSet`/`FileSet`/`FileObject`.",
                 key=f"{prefix}-has-joins",
             )
             data_editor_key = _data_editor_key(record_set_key, record_set)
             st.markdown(
+                needed_field("Fields"),
+                help=(
+                    "Add/delete fields by directly editing the table. Warning: the"
+                    " table contains information about the fields--not the data"
+                    " directly. If you wish to embed data, select `The RecordSet is an"
+                    " enumeration` above. To edit fields details, click the button"
+                    " `Edit fields details` below."
+                ),
             )
             st.data_editor(
                 fields,
                 args=(record_set_key, record_set),
             )
             key = f"{prefix}-delete-record-set"
+            button_with_confirmation(
+                "Delete RecordSet",
                 key=key,
                 on_click=_handle_remove_record_set,
                 args=(record_set_key,),
                     needed_field("Name"),
                     placeholder="Name without special character.",
                     key=key,
+                    help=f"The name of the field. {NAMES_INFO}",
                     value=field.name,
                     on_change=handle_field_change,
                     args=(FieldEvent.NAME, field, key),
                     value=field.description,
                     args=(FieldEvent.DESCRIPTION, field, key),
                 )
+                data_type_index = None
                 if field.data_types:
                     data_type = field.data_types[0]
                     if isinstance(data_type, str):
                         data_type = term.URIRef(data_type)
                     if data_type in MLC_DATA_TYPES:
                         data_type_index = MLC_DATA_TYPES.index(data_type)
                 key = f"{prefix}-datatypes"
                 col3.selectbox(
                     needed_field("Data type"),
                     index=data_type_index,
                     options=STR_DATA_TYPES,
                     key=key,
+                    help=(
+                        "The type of the data. `Text` corresponds to"
+                        " https://schema.org/Text, etc."
+                    ),
                     on_change=handle_field_change,
                     args=(FieldEvent.DATA_TYPE, field, key),
                 )
                 possible_sources = _get_possible_sources(metadata)
+                render_source(record_set, field, possible_sources)
+                render_references(record_set, field, possible_sources)
                 st.divider()

views/source.py CHANGED Viewed

@@ -12,6 +12,15 @@ from events.fields import TransformType
 import mlcroissant as mlc
 from utils import needed_field
 class SourceType:
     """The type of the source (distribution or field)."""
@@ -105,10 +114,8 @@ def _handle_remove_reference(field):
 def render_source(
-    record_set_key: int,
     record_set: RecordSet,
     field: Field,
-    field_key: int,
     possible_sources: list[str],
 ):
     """Renders the form for the source."""
@@ -123,10 +130,13 @@ def render_source(
         index = None
     key = f"{prefix}-source"
     col1.selectbox(
-        needed_field("Source"),
         index=index,
         options=options,
         key=key,
         on_change=handle_field_change,
         args=(FieldEvent.SOURCE, field, key),
     )
@@ -135,6 +145,7 @@ def render_source(
             needed_field("Extract"),
             index=_get_extract_index(source),
             key=f"{prefix}-extract",
             options=EXTRACT_TYPES,
             on_change=handle_field_change,
             args=(FieldEvent.SOURCE_EXTRACT, field, key),
@@ -145,6 +156,7 @@ def render_source(
                 needed_field("Column name"),
                 value=source.extract.column,
                 key=key,
                 on_change=handle_field_change,
                 args=(FieldEvent.SOURCE_EXTRACT_COLUMN, field, key),
             )
@@ -154,6 +166,7 @@ def render_source(
                 needed_field("JSON path"),
                 value=source.extract.json_path,
                 key=key,
                 on_change=handle_field_change,
                 args=(FieldEvent.SOURCE_EXTRACT_JSON_PATH, field, key),
             )
@@ -170,18 +183,23 @@ def render_source(
                 key=key,
                 options=TRANSFORM_TYPES,
                 on_change=handle_field_change,
                 args=(FieldEvent.TRANSFORM, field, key),
                 kwargs={"number": number},
             )
             if selected == TransformType.FORMAT:
                 key = f"{prefix}-{number}-transform-format"
                 col3.text_input(
-                    needed_field("Format"),
                     value=transform.format,
                     key=key,
                     on_change=handle_field_change,
                     args=(selected, field, key),
-                    kwargs={"number": number, "type": "format"},
                 )
             elif selected == TransformType.JSON_PATH:
                 key = f"{prefix}-{number}-jsonpath"
@@ -190,8 +208,9 @@ def render_source(
                     value=transform.json_path,
                     key=key,
                     on_change=handle_field_change,
                     args=(selected, field, key),
-                    kwargs={"number": number, "type": "format"},
                 )
             elif selected == TransformType.REGEX:
                 key = f"{prefix}-{number}-regex"
@@ -200,8 +219,14 @@ def render_source(
                     value=transform.regex,
                     key=key,
                     on_change=handle_field_change,
                     args=(selected, field, key),
-                    kwargs={"number": number, "type": "format"},
                 )
             elif selected == TransformType.REPLACE:
                 key = f"{prefix}-{number}-replace"
@@ -210,8 +235,13 @@ def render_source(
                     value=transform.replace,
                     key=key,
                     on_change=handle_field_change,
                     args=(selected, field, key),
-                    kwargs={"number": number, "type": "format"},
                 )
             elif selected == TransformType.SEPARATOR:
                 key = f"{prefix}-{number}-separator"
@@ -220,8 +250,9 @@ def render_source(
                     value=transform.separator,
                     key=key,
                     on_change=handle_field_change,
                     args=(selected, field, key),
-                    kwargs={"number": number, "type": "format"},
                 )
             def _handle_remove_transform(field, number):
@@ -230,6 +261,7 @@ def render_source(
             col4.button(
                 "✖️",
                 key=f"{prefix}-{number}-remove-transform",
                 on_click=_handle_remove_transform,
                 args=(field, number),
             )
@@ -243,16 +275,15 @@ def render_source(
     col1.button(
         "Add transform on data",
         key=f"{prefix}-close-fields",
         on_click=_handle_add_transform,
         args=(field,),
     )
 def render_references(
-    record_set_key: int,
     record_set: RecordSet,
     field: Field,
-    field_key: int,
     possible_sources: list[str],
 ):
     """Renders the form for references."""
@@ -286,6 +317,7 @@ def render_references(
                 index=_get_extract_index(references),
                 key=key,
                 options=EXTRACT_TYPES,
                 on_change=handle_field_change,
                 args=(FieldEvent.REFERENCE_EXTRACT, field, key),
             )
@@ -295,6 +327,7 @@ def render_references(
                     needed_field("Column name"),
                     value=references.extract.column,
                     key=key,
                     on_change=handle_field_change,
                     args=(FieldEvent.REFERENCE_EXTRACT_COLUMN, field, key),
                 )
@@ -304,12 +337,14 @@ def render_references(
                     needed_field("JSON path"),
                     value=references.extract.json_path,
                     key=key,
                     on_change=handle_field_change,
                     args=(FieldEvent.REFERENCE_EXTRACT_JSON_PATH, field, key),
                 )
         col4.button(
             "✖️",
             key=f"{key}-remove-reference",
             on_click=_handle_remove_reference,
             args=(field,),
         )

 import mlcroissant as mlc
 from utils import needed_field
+_JSON_PATH_DOCUMENTATION = (
+    "The JSON path if the data source is a JSON (see"
+    " [documentation](https://www.ietf.org/archive/id/draft-goessner-dispatch-jsonpath-00.html))."
+)
+_EXTRACT_DOCUMENTATION = (
+    "The extraction method to get the value of the field (column in a CSV, etc)."
+)
+_COLUMN_NAME_DOCUMENTATION = "The name of the column if the data source is a CSV."
 class SourceType:
     """The type of the source (distribution or field)."""
 def render_source(
     record_set: RecordSet,
     field: Field,
     possible_sources: list[str],
 ):
     """Renders the form for the source."""
         index = None
     key = f"{prefix}-source"
     col1.selectbox(
+        needed_field("Data source"),
         index=index,
         options=options,
         key=key,
+        help=(
+            "Data sources can be other resources (FileObject, FileSet) or other fields."
+        ),
         on_change=handle_field_change,
         args=(FieldEvent.SOURCE, field, key),
     )
             needed_field("Extract"),
             index=_get_extract_index(source),
             key=f"{prefix}-extract",
+            help=_EXTRACT_DOCUMENTATION,
             options=EXTRACT_TYPES,
             on_change=handle_field_change,
             args=(FieldEvent.SOURCE_EXTRACT, field, key),
                 needed_field("Column name"),
                 value=source.extract.column,
                 key=key,
+                help=_COLUMN_NAME_DOCUMENTATION,
                 on_change=handle_field_change,
                 args=(FieldEvent.SOURCE_EXTRACT_COLUMN, field, key),
             )
                 needed_field("JSON path"),
                 value=source.extract.json_path,
                 key=key,
+                help=_JSON_PATH_DOCUMENTATION,
                 on_change=handle_field_change,
                 args=(FieldEvent.SOURCE_EXTRACT_JSON_PATH, field, key),
             )
                 key=key,
                 options=TRANSFORM_TYPES,
                 on_change=handle_field_change,
+                help="One or more transformations to apply after extracting the field.",
                 args=(FieldEvent.TRANSFORM, field, key),
                 kwargs={"number": number},
             )
             if selected == TransformType.FORMAT:
                 key = f"{prefix}-{number}-transform-format"
                 col3.text_input(
+                    needed_field("Format a date"),
                     value=transform.format,
                     key=key,
                     on_change=handle_field_change,
+                    help=(
+                        "For dates, use [`Python format"
+                        " codes`](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes)."
+                    ),
                     args=(selected, field, key),
+                    kwargs={"number": number},
                 )
             elif selected == TransformType.JSON_PATH:
                 key = f"{prefix}-{number}-jsonpath"
                     value=transform.json_path,
                     key=key,
                     on_change=handle_field_change,
+                    help=_JSON_PATH_DOCUMENTATION,
                     args=(selected, field, key),
+                    kwargs={"number": number},
                 )
             elif selected == TransformType.REGEX:
                 key = f"{prefix}-{number}-regex"
                     value=transform.regex,
                     key=key,
                     on_change=handle_field_change,
+                    help=(
+                        "A regular expression following [`re` Python"
+                        " convention](https://docs.python.org/3/library/re.html#regular-expression-syntax)"
+                        " with one capturing group. The result of the operation will be"
+                        " the last captured group."
+                    ),
                     args=(selected, field, key),
+                    kwargs={"number": number},
                 )
             elif selected == TransformType.REPLACE:
                 key = f"{prefix}-{number}-replace"
                     value=transform.replace,
                     key=key,
                     on_change=handle_field_change,
+                    help=(
+                        "A replace pattern separated by a `/`, i.e."
+                        " `string_to_replace/string_to_substitute` in order to replace"
+                        " `string_to_replace` by `string_to_substitute`."
+                    ),
                     args=(selected, field, key),
+                    kwargs={"number": number},
                 )
             elif selected == TransformType.SEPARATOR:
                 key = f"{prefix}-{number}-separator"
                     value=transform.separator,
                     key=key,
                     on_change=handle_field_change,
+                    help="A separator to split strings on, e.g. `|` to split `a|b|c`.",
                     args=(selected, field, key),
+                    kwargs={"number": number},
                 )
             def _handle_remove_transform(field, number):
             col4.button(
                 "✖️",
                 key=f"{prefix}-{number}-remove-transform",
+                help="Remove the transformation.",
                 on_click=_handle_remove_transform,
                 args=(field, number),
             )
     col1.button(
         "Add transform on data",
         key=f"{prefix}-close-fields",
+        help="Add a transformation.",
         on_click=_handle_add_transform,
         args=(field,),
     )
 def render_references(
     record_set: RecordSet,
     field: Field,
     possible_sources: list[str],
 ):
     """Renders the form for references."""
                 index=_get_extract_index(references),
                 key=key,
                 options=EXTRACT_TYPES,
+                help=_EXTRACT_DOCUMENTATION,
                 on_change=handle_field_change,
                 args=(FieldEvent.REFERENCE_EXTRACT, field, key),
             )
                     needed_field("Column name"),
                     value=references.extract.column,
                     key=key,
+                    help=_COLUMN_NAME_DOCUMENTATION,
                     on_change=handle_field_change,
                     args=(FieldEvent.REFERENCE_EXTRACT_COLUMN, field, key),
                 )
                     needed_field("JSON path"),
                     value=references.extract.json_path,
                     key=key,
+                    help=_JSON_PATH_DOCUMENTATION,
                     on_change=handle_field_change,
                     args=(FieldEvent.REFERENCE_EXTRACT_JSON_PATH, field, key),
                 )
         col4.button(
             "✖️",
             key=f"{key}-remove-reference",
+            help="Remove the join.",
             on_click=_handle_remove_reference,
             args=(field,),
         )

views/splash.py CHANGED Viewed

@@ -13,6 +13,8 @@ import mlcroissant as mlc
 from views.load import render_load
 from views.previous_files import render_previous_files
 _DATASETS = {
     "Titanic": ["data/embarkation_ports.csv", "data/genders.csv"],
     "FLORES-200": [],
@@ -34,8 +36,6 @@ def render_splash():
         )
     col1, col2 = st.columns([1, 1], gap="large")
     with col1:
-        with st.expander("**Load an existing Croissant JSON-LD file**", expanded=True):
-            render_load()
         with st.expander("**Create from scratch**", expanded=True):
             def create_new_croissant():
@@ -81,6 +81,36 @@ def render_splash():
                 type="primary",
                 args=(dataset,),
             )
     with col2:
         with st.expander("**Past projects**", expanded=True):
             render_previous_files()

 from views.load import render_load
 from views.previous_files import render_previous_files
+_HUGGING_FACE_URL = "https://huggingface.co/datasets/"
 _DATASETS = {
     "Titanic": ["data/embarkation_ports.csv", "data/genders.csv"],
     "FLORES-200": [],
         )
     col1, col2 = st.columns([1, 1], gap="large")
     with col1:
         with st.expander("**Create from scratch**", expanded=True):
             def create_new_croissant():
                 type="primary",
                 args=(dataset,),
             )
+        with st.expander("**Load a dataset from Hugging Face**", expanded=True):
+            url = st.text_input(
+                label="Hugging Face URL",
+            )
+            if url.startswith(_HUGGING_FACE_URL):
+                url = url.replace(_HUGGING_FACE_URL, "")
+                def download_huggingface_json(name: str):
+                    api_url = f"https://datasets-server.huggingface.co/croissant?dataset={name}"
+                    json = requests.get(api_url, headers=None).json()
+                    try:
+                        metadata = mlc.Metadata.from_json(mlc.Issues(), json, None)
+                        st.session_state[Metadata] = Metadata.from_canonical(metadata)
+                        save_current_project()
+                    except Exception:
+                        st.error(f"Malformed JSON: {json}")
+                st.button(
+                    f'Download "{url}"',
+                    on_click=download_huggingface_json,
+                    type="primary",
+                    args=(url,),
+                )
+            elif url:
+                st.error(
+                    f"Unknown URL {url}. Hugging Face URLS should look like"
+                    f" {_HUGGING_FACE_URL}somedataset."
+                )
+        with st.expander("**Load an existing Croissant JSON-LD file**", expanded=True):
+            render_load()
     with col2:
         with st.expander("**Past projects**", expanded=True):
             render_previous_files()