Change iframe interaction using postMessage

app.py

@@ -192,15 +192,8 @@ def load_js(name):
         return f"""
 function (x)
 {{ 
-    console.log(document.querySelector("gradio-app"));
-    console.log(document.querySelector("iframe"));
-    console.log(document.querySelector("#sdinfframe").contentWindow);
-    console.log(document.querySelector("#sdinfframe").contentWindow.document);
-    console.log(document.querySelector("#sdinfframe").contentWindow.document.querySelector("#{name}");
-    console.log(document.querySelector("#sdinfframe").contentDocument);
-    let frame=document.querySelector("gradio-app").shadowRoot.querySelector("#sdinfframe").contentWindow.document;
-    let button=frame.querySelector("#{name}");
-    button.click();
+    let frame=document.querySelector("gradio-app").shadowRoot.querySelector("#sdinfframe").contentWindow;
+    frame.postMessage(["click","{name}"], "*");
     return x;
 }}
 """

index.html

@@ -80,6 +80,7 @@ from js import (
     Uint8ClampedArray,
     CanvasRenderingContext2D as Context2d,
     requestAnimationFrame,
+    window
 )
 
 
@@ -129,7 +130,7 @@ async def export_func(event):
 
 async def outpaint_func(event):
     base=base_lst[0]
-    base64_str=parent.document.querySelector("gradio-app").shadowRoot.querySelector("#output textarea").value
+    base64_str=event.data[1]
     arr=base.base64_to_numpy(base64_str)
     base.fill_selection(arr)
     base.draw_selection_box()
@@ -151,12 +152,11 @@ async def transfer_func(event):
     base.read_selection_from_buffer()
     sel_buffer=base.sel_buffer
     sel_buffer_str=base.numpy_to_base64(sel_buffer)
-    parent.document.querySelector("gradio-app").shadowRoot.querySelector("#input textarea").value=sel_buffer_str
-    parent.document.querySelector("gradio-app").shadowRoot.querySelector("#proceed").click()
+    parent.postMessage(["transfer",sel_buffer_str],"*")
 
 async def upload_func(event):
     base=base_lst[0]
-    base64_str=parent.document.querySelector("gradio-app").shadowRoot.querySelector("#upload textarea").value
+    base64_str=event.data[1]
     arr=base.base64_to_numpy(base64_str)
     h,w,_=arr.shape
     yo=(base.height-h)//2
@@ -181,6 +181,24 @@ document.querySelector("#draw").addEventListener("click",create_proxy(draw_canva
 async def setup_func():
     document.querySelector("#setup").value="1"
 
+async def message_func(event):
+    if event.data[0]=="click":
+        if event.data[1]=="export":
+            export_func(event)
+        elif event.data[1]=="commit":
+            commit_func(event)
+        elif event.data[1]=="undo":
+            undo_func(event)
+    elif event.data[0]=="upload":
+        upload_func(event)
+    elif event.data[0]=="outpaint":
+        outpaint_func(event)
+    elif event.data[0]=="mode":
+        document.querySelector("#mode").value=event.data[1]
+    elif event.data[0]=="transfer":
+        transfer_func(event)
+    
+window.addEventListener("message",create_proxy(message_func))
 import asyncio
 
 _ = await asyncio.gather(

js/mode.js

@@ -1,6 +1,6 @@
 function(mode){
     let app=document.querySelector("gradio-app").shadowRoot;
-    let frame=app.querySelector("#sdinfframe").contentWindow.document;
-    frame.querySelector("#mode").value=mode;
+    let frame=app.querySelector("#sdinfframe").contentWindow;
+    frame.postMessage(["mode", mode], "*");
     return mode;
 }

js/outpaint.js

@@ -5,8 +5,9 @@ function(a){
         window.my_observe_outpaint = new MutationObserver(function (event) {
             console.log(event);
             let app=document.querySelector("gradio-app").shadowRoot;
-            let frame=app.querySelector("#sdinfframe").contentWindow.document;
-            frame.querySelector("#outpaint").click();
+            let frame=app.querySelector("#sdinfframe").contentWindow;
+            var str=document.querySelector("gradio-app").shadowRoot.querySelector("#output textarea").value;
+            frame.postMessage(["outpaint", str], "*");
         });
         window.my_observe_outpaint_target=document.querySelector("gradio-app").shadowRoot.querySelector("#output span")
         window.my_observe_outpaint.observe(window.my_observe_outpaint_target, {
@@ -15,10 +16,16 @@ function(a){
             childList: true, 
             characterData: true
         });
+        window.addEventListener("message", function(e){
+            if(e.data[0]=="transfer")
+            {
+                document.querySelector("gradio-app").shadowRoot.querySelector("#input textarea").value=e.data[1];
+                document.querySelector("gradio-app").shadowRoot.querySelector("#proceed").click();
+            }
+        });
     }
     let app=document.querySelector("gradio-app").shadowRoot;
-    let frame=app.querySelector("#sdinfframe").contentWindow.document;
-    let button=frame.querySelector("#transfer");
-    button.click();
+    let frame=app.querySelector("#sdinfframe").contentWindow;
+    frame.postMessage(["transfer"],"*")
     return a;
 }

js/upload.js

@@ -4,8 +4,9 @@ function(a,b){
         console.log("setup upload here");
         window.my_observe_upload = new MutationObserver(function (event) {
             console.log(event);
-            var frame=document.querySelector("gradio-app").shadowRoot.querySelector("#sdinfframe").contentWindow.document;
-            frame.querySelector("#upload").click();
+            var frame=document.querySelector("gradio-app").shadowRoot.querySelector("#sdinfframe").contentWindow;
+            var str=document.querySelector("gradio-app").shadowRoot.querySelector("#upload textarea").value;
+            frame.postMessage(["upload", str], "*");
         });
         window.my_observe_upload_target = document.querySelector("gradio-app").shadowRoot.querySelector("#upload span");
         window.my_observe_upload.observe(window.my_observe_upload_target, {