Refactoring `verbose` arg

app.py

@@ -172,13 +172,11 @@ def run_episode(goal, env, verbose=True):
                 page_type = Page.SEARCH
                 
             elif action == 'click[buy now]':
-                return_value = None
+                return_value = {}
                 if env == 'amazon':
-                    asin_url = f"https://www.amazon.com/dp/{asin}"
-                    return_value = "Product URL: " + asin_url
+                    return_value['Product URL'] = f"https://www.amazon.com/dp/{asin}"
                     if len(clicked_options) > 0:
-                        options_str = ', '.join(list(clicked_options))
-                        return_value += "\nSelected Options: " + options_str
+                        return_value['Selected Options'] = ', '.join(list(clicked_options))
                 if env == 'webshop':
                     query_str = "+".join(search_terms.split())
                     options_str = json.dumps(options)
@@ -186,13 +184,12 @@ def run_episode(goal, env, verbose=True):
                         f'{WEBSHOP_URL}/item_page/{WEBSHOP_SESSION}/'
                         f'{asin}/{query_str}/{page_num}/{options_str}'
                     )
-                    return_value = "Product URL: " + asin_url
+                    return_value['Product URL'] = asin_url
                 if env == 'ebay':
-                    asin_url = f"https:///www.ebay.com/itm/{asin}"
-                    return_value = "Product URL: " + asin_url
+                    asin_url = f"https://www.ebay.com/itm/{asin}"
+                    return_value['Product URL'] = asin_url
                     if len(clicked_options) > 0:
-                        options_str = ', '.join(list(clicked_options))
-                        return_value += "\nSelected Options: " + options_str
+                        return_value['Selected Options'] = ', '.join(list(clicked_options))
                 return return_value
             
             elif prev_page_type == Page.ITEM_PAGE:
@@ -216,16 +213,19 @@ def run_episode(goal, env, verbose=True):
         if page_type == Page.RESULTS:
             if search_terms in search_results_cache:
                 data = search_results_cache[search_terms]
+                if verbose:
+                    print(f"Loading cached results page for\"{search_terms}\"")
             else:
                 begin = time.time()
                 if env == 'amazon':
-                    data = parse_results_amz(search_terms, page_num)
+                    data = parse_results_amz(search_terms, page_num, verbose)
                 if env == 'webshop':
-                    data = parse_results_ws(search_terms, page_num)
+                    data = parse_results_ws(search_terms, page_num, verbose)
                 if env == 'ebay':
-                    data = parse_results_ebay(search_terms, page_num)
+                    data = parse_results_ebay(search_terms, page_num, verbose)
                 end = time.time()
-                print("Parsing search results took", end-begin, "seconds")
+                if verbose:
+                    print(f"Parsing search results took {end-begin} seconds")
 
                 search_results_cache[search_terms] = data
                 num_prods = len(data)
@@ -233,18 +233,20 @@ def run_episode(goal, env, verbose=True):
                     title_to_asin_map[d['Title']] = d['asin']
         elif page_type == Page.ITEM_PAGE or page_type == Page.SUB_PAGE:
             if asin in product_map:
-                print("Loading cached item page for", asin)
+                if verbose:
+                    print("Loading cached item page for", asin)
                 data = product_map[asin]
             else:
                 begin = time.time()
                 if env == 'amazon':
-                    data = parse_item_page_amz(asin)
+                    data = parse_item_page_amz(asin, verbose)
                 if env == 'webshop':
-                    data = parse_item_page_ws(asin, search_terms, page_num, options)
+                    data = parse_item_page_ws(asin, search_terms, page_num, options, verbose)
                 if env == 'ebay':
-                    data = parse_item_page_ebay(asin)
+                    data = parse_item_page_ebay(asin, verbose)
                 end = time.time()
-                print("Parsing item page took", end-begin, "seconds")
+                if verbose:
+                    print("Parsing item page took", end-begin, "seconds")
                 product_map[asin] = data
         elif page_type == Page.SEARCH:
             if verbose:
@@ -260,23 +262,23 @@ def run_episode(goal, env, verbose=True):
         html_str = dict_to_fake_html(data, page_type, asin, sub_page_type, options, product_map, goal)
         obs = convert_html_to_text(html_str, simple=False, clicked_options=clicked_options, visited_asins=visited_asins)
         end = time.time()
-        print("[Page Info -> WebShop HTML -> Observation] took", end-begin, "seconds")
+        if verbose:
+            print("[Page Info -> WebShop HTML -> Observation] took", end-begin, "seconds")
 
         # Dict of Info -> Valid Action State (Info)
         begin = time.time()
         prod_arg = product_map if page_type == Page.ITEM_PAGE else data
         info = convert_dict_to_actions(page_type, prod_arg, asin, page_num, num_prods)
         end = time.time()
-        print("Extracting available actions took", end-begin, "seconds")
+        if verbose:
+            print("Extracting available actions took", end-begin, "seconds")
         
-        if i == 99:
-            return_value = None
+        if i == 50:
+            return_value = {}
             if env == 'amazon':
-                asin_url = f"https://www.amazon.com/dp/{asin}"
-                return_value = "Product URL: " + asin_url
+                return_value['Product URL'] = f"https://www.amazon.com/dp/{asin}"
                 if len(clicked_options) > 0:
-                    options_str = ', '.join(list(clicked_options))
-                    return_value += "\nSelected Options: " + options_str
+                    return_value['Selected Options'] = ', '.join(list(clicked_options))
             if env == 'webshop':
                 query_str = "+".join(search_terms.split())
                 options_str = json.dumps(options)
@@ -284,13 +286,12 @@ def run_episode(goal, env, verbose=True):
                     f'{WEBSHOP_URL}/item_page/{WEBSHOP_SESSION}/'
                     f'{asin}/{query_str}/{page_num}/{options_str}'
                 )
-                return_value = "Product URL: " + asin_url
+                return_value['Product URL'] = asin_url
             if env == 'ebay':
-                asin_url = f"https:///www.ebay.com/itm/{asin}"
-                return_value = "Product URL: " + asin_url
+                asin_url = f"https://www.ebay.com/itm/{asin}"
+                return_value['Product URL'] = asin_url
                 if len(clicked_options) > 0:
-                    options_str = ', '.join(list(clicked_options))
-                    return_value += "\nSelected Options: " + options_str
+                    return_value['Selected Options'] = ', '.join(list(clicked_options))
             return return_value
 
 gr.Interface(fn=run_episode,\

predict_help.py

@@ -17,7 +17,6 @@ class Page(Enum):
 
 HEADER_ = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36'
 DEBUG_HTML = "temp.html"
-VERBOSE = True
 NUM_PROD_LIMIT = 10
 
 WEBSHOP_URL = "http://3.83.245.205:3000"
@@ -29,11 +28,11 @@ def get_url(url):
     proxy_url = 'http://api.scraperapi.com/?' + urlencode(payload)
     return proxy_url
 
-def parse_results_ebay(query, page_num=None):
+def parse_results_ebay(query, page_num=None, verbose=True):
     query_string = '+'.join(query.split())
     page_num = 1 if page_num is None else page_num
     url = f'https://www.ebay.com/sch/i.html?_nkw={query_string}&_pgn={page_num}'
-    if VERBOSE:
+    if verbose:
         print(f"Search Results URL: {url}")
     webpage = requests.get(url, headers={'User-Agent': HEADER_, 'Accept-Language': 'en-US, en;q=0.5'})
     soup = BeautifulSoup(webpage.text, 'html.parser')
@@ -61,21 +60,21 @@ def parse_results_ebay(query, page_num=None):
             "Title": title,
             "Price": price
         })
-    if VERBOSE:
+    if verbose:
         print(f"Scraped {len(results)} products")
     return results
 
-def parse_item_page_ebay(asin):
+def parse_item_page_ebay(asin, verbose=True):
     product_dict = {}
     product_dict["asin"] = asin
     
     url = f"https://www.ebay.com/itm/{asin}"
-    if VERBOSE:
+    if verbose:
         print(f"Item Page URL: {url}")
     begin = time.time()
     webpage = requests.get(url, headers={'User-Agent': HEADER_, 'Accept-Language': 'en-US, en;q=0.5'})
     end = time.time()
-    if VERBOSE:
+    if verbose:
         print(f"Item page scraping took {end-begin} seconds")
     soup = BeautifulSoup(webpage.content, "html.parser")
 
@@ -148,14 +147,14 @@ def parse_item_page_ebay(asin):
     return product_dict
     
 
-def parse_results_ws(query, page_num=None):
+def parse_results_ws(query, page_num=None, verbose=True):
     query_string = '+'.join(query.split())
     page_num = 1 if page_num is None else page_num
     url = (
         f'{WEBSHOP_URL}/search_results/{WEBSHOP_SESSION}/'
         f'{query_string}/{page_num}'
     )
-    if VERBOSE:
+    if verbose:
         print(f"Search Results URL: {url}")
     webpage = requests.get(url, headers={'User-Agent': HEADER_, 'Accept-Language': 'en-US, en;q=0.5'})
     soup = BeautifulSoup(webpage.content, 'html.parser')
@@ -185,11 +184,11 @@ def parse_results_ws(query, page_num=None):
             "Price": price
         })
 
-    if VERBOSE:
+    if verbose:
         print(f"Scraped {len(results)} products")
     return results
 
-def parse_item_page_ws(asin, query, page_num, options):
+def parse_item_page_ws(asin, query, page_num, options, verbose=True):
     product_dict = {}
     product_dict["asin"] = asin
 
@@ -199,7 +198,7 @@ def parse_item_page_ws(asin, query, page_num, options):
         f'{WEBSHOP_URL}/item_page/{WEBSHOP_SESSION}/'
         f'{asin}/{query_string}/{page_num}/{options_string}'
     )
-    if VERBOSE:
+    if verbose:
         print("Item Page URL: ", url)
     webpage = requests.get(url, headers={'User-Agent': HEADER_, 'Accept-Language': 'en-US, en;q=0.5'})
     soup = BeautifulSoup(webpage.content, 'html.parser')
@@ -258,11 +257,11 @@ def parse_item_page_ws(asin, query, page_num, options):
     return product_dict
 
 # Query -> Search Result ASINs
-def parse_results_amz(query, page_num=None):
+def parse_results_amz(query, page_num=None, verbose=True):
     url = 'https://www.amazon.com/s?k=' + query.replace(" ", "+")
     if page_num is not None:
         url += "&page=" + str(page_num)
-    if VERBOSE:
+    if verbose:
         print(f"Search Results URL: {url}")
     webpage = requests.get(url, headers={'User-Agent': HEADER_, 'Accept-Language': 'en-US, en;q=0.5'})
     soup = BeautifulSoup(webpage.content, 'html.parser')
@@ -286,22 +285,22 @@ def parse_results_amz(query, page_num=None):
             'Price': price.text.strip().strip("$")
         }
         results.append(result)
-    if VERBOSE:
+    if verbose:
         print("Scraped", len(results), "products")
     return results
 
 # Scrape information of each product
-def parse_item_page_amz(asin):
+def parse_item_page_amz(asin, verbose=True):
     product_dict = {}
     product_dict["asin"] = asin
 
     url = f"https://www.amazon.com/dp/{asin}"
-    if VERBOSE:
+    if verbose:
         print("Item Page URL:", url)
     begin = time.time()
     webpage = requests.get(url, headers={'User-Agent': HEADER_, 'Accept-Language': 'en-US, en;q=0.5'})
     end = time.time()
-    if VERBOSE:
+    if verbose:
         print(f"Item page scraping took {end-begin} seconds")
     soup = BeautifulSoup(webpage.content, "html.parser")