[feature] add credentials form to retrieve data via inception API remote

.gitignore

@@ -1,2 +1,4 @@
 venv/
-.idea/
+.idea/
+__pycache__/
+LEGACY.py

app.py

@@ -1,45 +1,54 @@
 #!/usr/bin/env python3
 # -*- coding:utf-8 -*-
-
-
+import requests.exceptions
+import zipfile
 
 import streamlit as st
 from streamlit.components.v1 import html
 from n4a_analytics_lib.analytics import (GlobalStatistics, IaaStatistics)
+from n4a_analytics_lib.constants import (DESCRIPTION)
 
 
-TITLE = "NER4ARCHIVES Analytics"
-
 # Set application
 st.set_page_config(layout="wide")
-
 # sidebar: meta, inputs etc.
 sidebar = st.sidebar
 # cols: display results
 col1, col2 = st.columns(2)
 
 # description
-#sidebar.markdown(f"# 📏 {TITLE}")
-sidebar.markdown(f"""
-# 📏 {TITLE}
-
-A basic web application to display a dashboard for
-analyzing INCEpTION annotation project built in context
-of NER4Archives (Inria/Archives nationales).
- 
-- This tool provides two statistics levels:
-    - *Global project statistics*: Analyze named entities in overall curated documents in project;
-    - *Inter-Annotator Agreement results*: Analyze results of IAA experiment.  
-""")
+sidebar.markdown(DESCRIPTION)
+
+
+
+
+# to st components
+#def clear_cache():
+#    st.session_state = {}
+
+def check_login(username, password):
+    if (len(username) == 0) or (len(password) == 0):
+        return False
+    return True
+
+def logout():
+    pass
+
+
+
+
+
 
 # Level to analyze
-option = sidebar.selectbox('Which statistics level?', ('Inter-Annotator Agreement results', 'Global project statistics'))
+option = sidebar.selectbox('Which statistics level?', ('Inter-Annotator Agreement results',
+                                                       'Global project statistics'))
 
 # IAA results view
 if option == "Inter-Annotator Agreement results":
     annotations = sidebar.file_uploader("Upload IAA annotations (.zip format only): ")
     baseline_text = sidebar.file_uploader("Upload baseline text (.txt format only): ")
 
+
     if baseline_text is not None and annotations is not None:
         project_analyzed = IaaStatistics(zip_project=annotations, baseline_text=baseline_text.getvalue())
         baseline_analyzer = project_analyzed.analyze_text()
@@ -308,24 +317,135 @@ if option == "Inter-Annotator Agreement results":
         st.pyplot(f.figure)
 
 # global project results view
+# st_session = {"gs_local":True, "gs_remote":False, "gs_obj":<object>}
+
+def display_data():
+    col1.metric("Total curated annotations",
+                f"{st.session_state['gs_obj'].total_annotations_project} Named entities")
+    col1.dataframe(st.session_state['gs_obj'].df_i)
+    selected_data = col1.selectbox('Select specific data to display bar plot:',
+                                   st.session_state['gs_obj'].documents, key="selector_data")
+    col2.pyplot(st.session_state['gs_obj'].create_plot(selected_data))
+
+def init_session_statistics(remote: bool, local: bool, data: tuple) -> None:
+    # clear session
+    st.session_state = {}
+
+    # create a session variable
+    st.session_state["gs_local"] = local
+    st.session_state["gs_remote"] = remote
+
+    # create a new object:
+    # if remote fetch data from API Host first
+    if remote and not(local):
+        st.success('Fetch curated documents from host INCEpTION API in progress...')
+        fetch_curated_data_from_remote(
+                username=data[0],
+                password=data[1]
+        )
+
+    if local and not(remote):
+        st.session_state["gs_obj"] = GlobalStatistics(zip_project=data, remote=False)
+
+
+
+
+
+from pycaprio import Pycaprio, mappings
+from zipfile import ZipFile
+import io
+import requests
+
+def fetch_curated_data_from_remote(username: str,
+                                   password: str,
+                                   endpoint: str = "https://inception.dhlab.epfl.ch/prod",
+                                   project_title: str = "ner4archives-template"):
+    # open a client
+    try:
+        client = Pycaprio(inception_host=endpoint, authentication=(str(username), str(password)))
+    except requests.exceptions.JSONDecodeError:
+        # username / password incorrect
+        st.error('Username or Password is incorrect please retry.')
+
+    # get project object
+    project_name = [p for p in client.api.projects() if p.project_name == project_title]
+
+    # get all documents from project
+    documents = client.api.documents(project_name[0].project_id)
+
+    curations = []
+    zipfiles = []
+    count = 0
+    flag = "a"
+    # iterate over all documents and retrieve only curated into ZIP container
+    for document in documents:
+        if count > 0:
+            flag = "r"
+        if document.document_state == mappings.DocumentState.CURATION_COMPLETE:
+            curated_content = client.api.curation(project_name[0].project_id, document,
+                                                  curation_format=mappings.InceptionFormat.UIMA_CAS_XMI_XML_1_1)
+            curations.append(curated_content)
+            for curation in curations:
+                z = ZipFile(io.BytesIO(curation), mode=flag)
+                zipfiles.append(z)
+
+        count += 1
+
+    # Merge all zip in one
+    with zipfiles[0] as z1:
+        for fname in zipfiles[1:]:
+            zf = fname
+            # print(zf.namelist())
+            for n in zf.namelist():
+                if n not in z1.namelist():
+                    z1.writestr(n, zf.open(n).read())
+
+        # Create a new object
+        st.session_state["gs_obj"] = GlobalStatistics(zip_project=z1, remote=True)
+
 
 
-# to st components
-def clear_cache():
-    st.session_state["p_a"] = None
 
 if option == "Global project statistics":
-    project = sidebar.file_uploader("Project folder that contains curated annotations in XMI 1.1 (.zip format only) : ", on_change=clear_cache)
-    if project is not None:
-        if st.session_state["p_a"] is None:
-            st.session_state["p_a"] = GlobalStatistics(zip_project=project)
-        if st.session_state["p_a"] is not None:
-            with st.expander('Details on data'):
-                col1.metric("Total curated annotations",
-                            f"{st.session_state['p_a'].total_annotations_project} Named entities")
-                col1.dataframe(st.session_state['p_a'].df_i)
-                selected_data = col1.selectbox('Select specific data to display bar plot:',
-                                               st.session_state['p_a'].documents)
-                col2.pyplot(st.session_state['p_a'].create_plot(selected_data))
+    # User input controllers
+    mode = sidebar.radio("Choose mode to retrieve curated data: ", (
+        "Local directory", "INCEpTION API Host remote"
+    ))
+    data = None
+    if mode == "Local directory":
+        project = sidebar.file_uploader("Folder that contains curated annotations in XMI 1.1 (.zip format only): ",
+                                        type="zip")
+        data = project
+    if mode == "INCEpTION API Host remote":
+        username = sidebar.text_input("Username: ")
+        password = sidebar.text_input("Password: ", type="password")
+        data = (username, password)
+
+    # Validate inputs
+    btn_process = sidebar.button('Process', key='process')
+
+    # Access data with local ressources
+    if btn_process and mode == "Local directory":
+        if data is not None:
+            # create a new session
+            init_session_statistics(remote=False, local=True, data=data)
+
+    # Access data with remote ressources
+    if btn_process and mode == "API Host remote":
+        if data is not None:
+            if check_login(username=data[0], password=data[1]):
+                # create a new session
+                init_session_statistics(remote=True, local=False, data=data)
+            else:
+              st.error("Sorry! Username or Password is empty.")
+
+    # Change data values and visualize new plot
+    if "gs_obj" in st.session_state:
+        if st.session_state["gs_local"] or st.session_state["gs_remote"]:
+            display_data()
+
+
+
+
 
 

n4a_analytics_lib/analytics.py

@@ -15,9 +15,8 @@ from n4a_analytics_lib.project import Project
 
 
 class GlobalStatistics(Project):
-    def __init__(self, zip_project):
-        super().__init__(zip_project=zip_project, type="global")
-
+    def __init__(self, zip_project, remote=False):
+        super().__init__(zip_project=zip_project, remote=remote, type="global")
         self.data = [(src_file, ne_label) for src_file, ann in self.annotations.items() for ne_label in ann['labels']]
         self.df_base = pd.DataFrame(self.data, columns=["SOURCE_FILE", "LABEL"])
         self.df_i = self.df_base.groupby(["LABEL"])["LABEL"].count().reset_index(name="TOTAL")

n4a_analytics_lib/constants.py

@@ -0,0 +1,14 @@
+# -*- coding:utf-8 -*-
+
+TITLE = "NER4ARCHIVES Analytics"
+DESCRIPTION = f"""
+# 📏 {TITLE}
+
+A basic web application to display a dashboard for
+analyzing INCEpTION annotation project built in context
+of NER4Archives (Inria/Archives nationales).
+ 
+- This tool provides two statistics levels:
+    - *Global project statistics*: Analyze named entities in overall curated documents in project;
+    - *Inter-Annotator Agreement results*: Analyze results of IAA experiment.  
+"""

n4a_analytics_lib/project.py

@@ -1,5 +1,5 @@
 # -*- coding:utf-8 -*-
-
+import zipfile
 from io import BytesIO
 import re
 from zipfile import ZipFile
@@ -13,9 +13,12 @@ from n4a_analytics_lib.st_components import st_pb
 
 
 class Project:
-    def __init__(self, zip_project, type):
+    def __init__(self, zip_project, type, remote):
         # zip container that contains XMI and typesystem
         self.zip_project = zip_project
+
+        self.remote = remote
+
         # 'iaa' or 'global'
         self.type = type
 
@@ -42,36 +45,46 @@ class Project:
         self.annotations = {}
 
 
-        with ZipFile(self.zip_project) as project_zip:
-            if self.type == "global":
-                regex = re.compile('.*curation/.*/(?!\._).*zip$')
-            elif self.type == "iaa":
-                regex = re.compile('.*xm[il]$')
-
-            annotation_fps = (fp for fp in project_zip.namelist() if regex.match(fp))
-            for fp in annotation_fps:
-                if self.type == "global":
-                    with ZipFile(BytesIO(project_zip.read(fp))) as annotation_zip:
-                        if self.typesystem is None:
-                            self.typesystem = load_typesystem(BytesIO(annotation_zip.read('TypeSystem.xml')))
-                        for f in annotation_zip.namelist():
-                            if f.endswith('.xmi'):
-                                # store source filename
-                                self.documents.append(Path(fp).parent.name)
-                                # annotators = []
-                                # store XMI representation
-                                self.xmi_documents.append(str(annotation_zip.read(f).decode("utf-8")))
-                elif self.type == "iaa":
-                    if self.typesystem is None and fp.endswith('.xml'):
-                        self.typesystem = load_typesystem(BytesIO(project_zip.read('TypeSystem.xml')))
-                    else:
-                        if fp.endswith('.xmi'):
-                            # store source filename
-                            self.documents.append(fp)
-                            # set annotators
-                            self.annotators.append(os.path.splitext(fp)[0])
-                            # store XMI representation
-                            self.xmi_documents.append(str(project_zip.read(fp).decode("utf-8")))
+        if isinstance(self.zip_project, zipfile.ZipFile) and self.remote and self.type == "global":
+            for fp in self.zip_project.namelist():
+                if self.typesystem is None:
+                    self.typesystem = load_typesystem(BytesIO(self.zip_project.open('TypeSystem.xml').read()))
+                if fp.endswith('.xmi'):
+                    self.documents.append(fp)
+                    self.xmi_documents.append(str(self.zip_project.open(fp).read().decode("utf-8")))
+
+
+        else:
+         with ZipFile(self.zip_project) as project_zip:
+             if self.type == "global":
+                 regex = re.compile('.*curation/.*/(?!\._).*zip$')
+             elif self.type == "iaa":
+                 regex = re.compile('.*xm[il]$')
+
+             annotation_fps = (fp for fp in project_zip.namelist() if regex.match(fp))
+             for fp in annotation_fps:
+                 if self.type == "global":
+                     with ZipFile(BytesIO(project_zip.read(fp))) as annotation_zip:
+                         if self.typesystem is None:
+                             self.typesystem = load_typesystem(BytesIO(annotation_zip.read('TypeSystem.xml')))
+                         for f in annotation_zip.namelist():
+                             if f.endswith('.xmi'):
+                                 # store source filename
+                                 self.documents.append(Path(fp).parent.name)
+                                 # annotators = []
+                                 # store XMI representation
+                                 self.xmi_documents.append(str(annotation_zip.read(f).decode("utf-8")))
+                 elif self.type == "iaa":
+                     if self.typesystem is None and fp.endswith('.xml'):
+                         self.typesystem = load_typesystem(BytesIO(project_zip.read('TypeSystem.xml')))
+                     else:
+                         if fp.endswith('.xmi'):
+                             # store source filename
+                             self.documents.append(fp)
+                             # set annotators
+                             self.annotators.append(os.path.splitext(fp)[0])
+                             # store XMI representation
+                             self.xmi_documents.append(str(project_zip.read(fp).decode("utf-8")))
 
 
         self.extract_ne()