Deploy (see actual commits on https://github.com/mlcommons/croissant).

core/files.py

@@ -11,8 +11,8 @@ from .names import find_unique_name
 from .state import FileObject
 from .state import FileSet
 
-FILE_OBJECT = "File object"
-FILE_SET = "File set"
+FILE_OBJECT = "FileObject"
+FILE_SET = "FileSet"
 RESOURCE_TYPES = [FILE_OBJECT, FILE_SET]
 
 
@@ -131,32 +131,11 @@ def file_from_upload(
     )
 
 
-def file_from_form(
-    file_type: FileType,
-    type: str,
-    name,
-    description,
-    sha256: str,
-    contained_in: list[str],
-    names: set[str],
-) -> FileObject | FileSet:
+def file_from_form(type: str, names: set[str]) -> FileObject | FileSet:
     """Creates a file based on manually added fields."""
     if type == FILE_OBJECT:
-        return FileObject(
-            name=find_unique_name(names, name),
-            description=description,
-            content_url="",
-            encoding_format=file_type.encoding_format,
-            sha256=sha256,
-            df=None,
-            contained_in=contained_in,
-        )
+        return FileObject(name=find_unique_name(names, "file_object"))
     elif type == FILE_SET:
-        return FileSet(
-            name=find_unique_name(names, name),
-            description=description,
-            encoding_format=file_type.encoding_format,
-            contained_in=contained_in,
-        )
+        return FileSet(name=find_unique_name(names, "file_set"))
     else:
         raise ValueError("type has to be one of FILE_OBJECT, FILE_SET")

events/resources.py

@@ -1,7 +1,9 @@
+import dataclasses
 import enum
 
 import streamlit as st
 
+from core.files import FILE_OBJECT
 from core.state import FileObject
 from core.state import FileSet
 from core.state import Metadata
@@ -15,9 +17,12 @@ class ResourceEvent(enum.Enum):
     NAME = "NAME"
     DESCRIPTION = "DESCRIPTION"
     ENCODING_FORMAT = "ENCODING_FORMAT"
+    INCLUDES = "INCLUDES"
     SHA256 = "SHA256"
+    CONTAINED_IN = "CONTAINED_IN"
     CONTENT_SIZE = "CONTENT_SIZE"
     CONTENT_URL = "CONTENT_URL"
+    TYPE = "TYPE"
 
 
 def handle_resource_change(event: ResourceEvent, resource: Resource, key: str):
@@ -33,9 +38,33 @@ def handle_resource_change(event: ResourceEvent, resource: Resource, key: str):
         resource.description = value
     elif event == ResourceEvent.ENCODING_FORMAT:
         resource.encoding_format = value
+    elif event == ResourceEvent.INCLUDES:
+        resource.includes = value
     elif event == ResourceEvent.SHA256:
         resource.sha256 = value
+    elif event == ResourceEvent.CONTAINED_IN:
+        resource.contained_in = value
     elif event == ResourceEvent.CONTENT_SIZE:
         resource.content_size = value
     elif event == ResourceEvent.CONTENT_URL:
         resource.content_url = value
+    elif event == ResourceEvent.TYPE:
+        metadata: Metadata = st.session_state[Metadata]
+        index = metadata.distribution.index(resource)
+        # Changing type by trying to retain as many attributes as possible.
+        if value == FILE_OBJECT:
+            file_object = _create_instance1_from_instance2(resource, FileObject)
+            metadata.distribution[index] = file_object
+        else:
+            file_set = _create_instance1_from_instance2(resource, FileSet)
+            metadata.distribution[index] = file_set
+
+
+def _create_instance1_from_instance2(instance1: Resource, instance2: type):
+    """Creates instance2 by retaining as many common attributes as possible."""
+    attributes1 = set((field.name for field in dataclasses.fields(instance1)))
+    attributes2 = set((field.name for field in dataclasses.fields(instance2)))
+    common_attributes = attributes2.intersection(attributes1)
+    return instance2(**{
+        attribute: getattr(instance1, attribute) for attribute in common_attributes
+    })

events/resources_test.py

@@ -0,0 +1,19 @@
+from core.state import FileObject
+from core.state import FileSet
+
+from .resources import _create_instance1_from_instance2
+
+
+def test_create_instance1_from_instance2():
+    file_object = FileObject(
+        name="name",
+        description="description",
+        contained_in=["foo", "bar"],
+        content_url="mlcommons.com",
+    )
+    file_set = _create_instance1_from_instance2(file_object, FileSet)
+    assert isinstance(file_set, FileSet)
+    assert file_set.name == "name"
+    assert file_set.description == "description"
+    assert file_set.contained_in == ["foo", "bar"]
+    assert file_set.encoding_format == None

views/files.py

@@ -6,6 +6,7 @@ from core.files import file_from_form
 from core.files import file_from_upload
 from core.files import file_from_url
 from core.files import FILE_OBJECT
+from core.files import FILE_SET
 from core.files import FILE_TYPES
 from core.files import RESOURCE_TYPES
 from core.record_sets import infer_record_sets
@@ -27,14 +28,18 @@ _MANUAL_DESCRIPTION_KEY = "manual_object_description"
 _MANUAL_SHA256_KEY = "manual_object_sha256"
 _MANUAL_PARENT_KEY = "manual_object_parents"
 
+_INFO = """Resources can be `FileObjects` (single files) or `FileSets` (sets of files 
+with the same MIME type). On this page, you can upload `FileObjects`, point to external
+resources on the web or manually create new resources."""
+
 
 def render_files():
     col1, col2, col3 = st.columns([1, 1, 1], gap="small")
     with col1:
-        st.subheader("Upload more resources")
+        st.markdown("##### Upload more resources")
         _render_upload_panel()
     with col2:
-        st.subheader("Uploaded resources")
+        st.markdown("##### Uploaded resources")
         files = st.session_state[Metadata].distribution
         resource = _render_resources_panel(files)
         st.session_state[SelectedResource] = resource
@@ -86,30 +91,7 @@ def _render_upload_panel():
             st.text_input("URL:", key=_DISTANT_URL_KEY)
 
         with tab3:
-            resource_type = st.selectbox(
-                "Type", options=RESOURCE_TYPES, key=_MANUAL_RESOURCE_TYPE_KEY
-            )
-            st.text_input(
-                needed_field("File name"),
-                key=_MANUAL_NAME_KEY,
-            )
-            st.text_area(
-                "File description",
-                placeholder="Provide a clear description of the file.",
-                key=_MANUAL_DESCRIPTION_KEY,
-            )
-            st.text_input(
-                "SHA256",
-                key=_MANUAL_SHA256_KEY,
-            )
-            parent_options = [
-                file.name for file in st.session_state[Metadata].distribution
-            ]
-            st.multiselect(
-                "Parent",
-                options=parent_options,
-                key=_MANUAL_PARENT_KEY,
-            )
+            st.selectbox("Type", options=RESOURCE_TYPES, key=_MANUAL_RESOURCE_TYPE_KEY)
 
         def handle_on_click():
             url = st.session_state[_DISTANT_URL_KEY]
@@ -123,29 +105,7 @@ def _render_upload_panel():
                 file = file_from_upload(file_type, uploaded_file, names)
             else:
                 resource_type = st.session_state[_MANUAL_RESOURCE_TYPE_KEY]
-                needs_sha256 = resource_type == FILE_OBJECT
-
-                name = st.session_state[_MANUAL_NAME_KEY]
-                description = st.session_state[_MANUAL_DESCRIPTION_KEY]
-                sha256 = st.session_state[_MANUAL_SHA256_KEY] if needs_sha256 else None
-                parents = st.session_state[_MANUAL_PARENT_KEY]
-                errorMessage = (
-                    "Please import either a local file, provide a download URL or fill"
-                    " in all required fields: name"
-                )
-                if needs_sha256:
-                    errorMessage += " and SHA256"
-
-                if not name or (needs_sha256 and not sha256):
-                    # Some required fields are empty.
-                    st.toast(
-                        errorMessage,
-                        icon="❌",
-                    )
-                    return
-                file = file_from_form(
-                    file_type, resource_type, name, description, sha256, parents, names
-                )
+                file = file_from_form(resource_type, names)
 
             st.session_state[Metadata].add_distribution(file)
             record_sets = infer_record_sets(file, names)
@@ -161,70 +121,57 @@ def _render_right_panel():
     of the selected resource."""
     if st.session_state.get(SelectedResource):
         _render_resource_details(st.session_state[SelectedResource])
+    else:
+        st.info(_INFO, icon="💡")
 
 
 def _render_resource_details(selected_file: Resource):
     """Renders the details of the selected resource."""
     file: FileObject | FileSet
-    for key, file in enumerate(st.session_state[Metadata].distribution):
+    for i, file in enumerate(st.session_state[Metadata].distribution):
         if file.name == selected_file.name:
+            is_file_object = isinstance(file, FileObject)
+            index = (
+                RESOURCE_TYPES.index(FILE_OBJECT)
+                if is_file_object
+                else RESOURCE_TYPES.index(FILE_SET)
+            )
+            key = f"{i}-file-name"
+            st.selectbox(
+                "Type",
+                index=index,
+                options=RESOURCE_TYPES,
+                key=key,
+                on_change=handle_resource_change,
+                args=(ResourceEvent.TYPE, file, key),
+            )
 
-            if isinstance(file, FileObject):
-                _render_file_object(key, file)
-            else:
-                _render_file_set(key, file)
+            _render_resource(i, file, is_file_object)
 
             def delete_line():
-                st.session_state[Metadata].remove_distribution(key)
+                st.session_state[Metadata].remove_distribution(i)
 
-            _, col = st.columns([5, 1])
-            col.button("Remove", key=f"{key}_url", on_click=delete_line, type="primary")
+            def close():
+                st.session_state[SelectedResource] = None
 
+            col1, col2 = st.columns([1, 1])
+            col1.button("Close", key=f"{i}_close", on_click=close, type="primary")
+            col2.button(
+                "Remove", key=f"{i}_remove", on_click=delete_line, type="secondary"
+            )
 
-def _render_file_object(prefix: int, file: FileObject):
-    key = f"{prefix}_name"
-    st.text_input(
-        needed_field("Name"),
-        value=file.name,
-        key=key,
-        on_change=handle_resource_change,
-        args=(ResourceEvent.NAME, file, key),
-    )
-    key = f"{prefix}_description"
-    st.text_area(
-        "Description",
-        value=file.description,
-        placeholder="Provide a clear description of the file.",
-        key=key,
-        on_change=handle_resource_change,
-        args=(ResourceEvent.DESCRIPTION, file, key),
-    )
-    key = f"{prefix}_sha256"
-    st.text_input(
-        needed_field("SHA256"),
-        value=file.sha256,
-        disabled=True,
-        key=key,
-        on_change=handle_resource_change,
-        args=(ResourceEvent.SHA256, file, key),
-    )
-    key = f"{prefix}_encoding"
-    st.text_input(
-        needed_field("Encoding format"),
-        value=file.encoding_format,
-        disabled=True,
+
+def _render_resource(prefix: int, file: FileObject | FileSet, is_file_object: bool):
+    parent_options = [f.name for f in st.session_state[Metadata].distribution]
+    key = f"{prefix}_parents"
+    st.multiselect(
+        "Parents",
+        default=file.contained_in,
+        options=parent_options,
         key=key,
         on_change=handle_resource_change,
-        args=(ResourceEvent.ENCODING_FORMAT, file, key),
+        args=(ResourceEvent.CONTAINED_IN, file, key),
     )
-    st.markdown("First rows of data:")
-    if file.df is not None:
-        st.dataframe(file.df, height=DF_HEIGHT)
-    else:
-        st.text("No rendering possible.")
-
-
-def _render_file_set(prefix: int, file: FileSet):
     key = f"{prefix}_name"
     st.text_input(
         needed_field("Name"),
@@ -242,12 +189,54 @@ def _render_file_set(prefix: int, file: FileSet):
         on_change=handle_resource_change,
         args=(ResourceEvent.DESCRIPTION, file, key),
     )
+    if is_file_object:
+        key = f"{prefix}_content_url"
+        st.text_input(
+            needed_field("Content URL"),
+            value=file.content_url,
+            key=key,
+            on_change=handle_resource_change,
+            args=(ResourceEvent.CONTENT_URL, file, key),
+        )
+        key = f"{prefix}_sha256"
+        st.text_input(
+            needed_field("SHA256"),
+            value=file.sha256,
+            key=key,
+            on_change=handle_resource_change,
+            args=(ResourceEvent.SHA256, file, key),
+        )
+        key = f"{prefix}_content_size"
+        st.text_input(
+            "Content size",
+            value=file.content_size,
+            key=key,
+            on_change=handle_resource_change,
+            args=(ResourceEvent.CONTENT_SIZE, file, key),
+        )
+    else:
+        key = f"{prefix}_includes"
+        st.text_input(
+            needed_field("Glob pattern of files to include"),
+            value=file.includes,
+            key=key,
+            on_change=handle_resource_change,
+            args=(ResourceEvent.INCLUDES, file, key),
+        )
     key = f"{prefix}_encoding"
     st.text_input(
         needed_field("Encoding format"),
         value=file.encoding_format,
-        disabled=True,
         key=key,
         on_change=handle_resource_change,
         args=(ResourceEvent.ENCODING_FORMAT, file, key),
     )
+    if is_file_object:
+        st.markdown("First rows of data:")
+        is_url = file.content_url and file.content_url.startswith("http")
+        if file.df is not None:
+            st.dataframe(file.df, height=DF_HEIGHT)
+        elif is_url:
+            st.button("Trigger download")
+        else:
+            st.markdown("No rendering possible.")