Update app.py

app.py

@@ -9,10 +9,10 @@ import pandas as pd
 import math
 import json
 
-#import sqlite3
-#import huggingface_hub
+import sqlite3
+import huggingface_hub
 #import pandas as pd
-#import shutil
+import shutil
 import os
 import datetime
 from apscheduler.schedulers.background import BackgroundScheduler
@@ -21,30 +21,27 @@ import random
 import time
 #import requests
 
-#from huggingface_hub import hf_hub_download
-#repo = huggingface_hub.HfRepository(repo_id="lysandre/test-model", token=token)
+from huggingface_hub import hf_hub_download
 
-# Clone the repository to a local directory
-#repo.clone_from_hub()
 #hf_hub_download(repo_id="CogSphere/aCogSphere", filename="./reviews.csv")
 
-#from huggingface_hub import login
+from huggingface_hub import login
 from datasets import load_dataset
 
 #dataset = load_dataset("csv", data_files="./data.csv")
 
 
-#DB_FILE = "./reviewsitr.db"
+DB_FILE = "./reviewsitr.db"
 
-#TOKEN = os.environ.get('HF_KEY')
+TOKEN = os.environ.get('HF_KEY')
 
-#repo = huggingface_hub.Repository(
-#    local_dir="data",
-#    repo_type="dataset",
-#    clone_from="CognitiveScience/csdhdata",
-#    use_auth_token=TOKEN
-#)
-#repo.git_pull()
+repo = huggingface_hub.Repository(
+    local_dir="data",
+    repo_type="dataset",
+    clone_from="CognitiveScience/csdhdata",
+    use_auth_token=TOKEN
+)
+repo.git_pull()
 
 #TOKEN2 = HF_TOKEN
 
@@ -56,47 +53,39 @@ from datasets import load_dataset
 
 # Create table if it doesn't already exist
 
-#db = sqlite3.connect(DB_FILE)
-#try:
-#    db.execute("SELECT * FROM reviews").fetchall()
-#    #db.execute("SELECT * FROM reviews2").fetchall()
-#
-#    db.close()
-#except sqlite3.OperationalError:
-#    db.execute(
-#        '''
-#        CREATE TABLE reviews (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
-#                              created_at TIMESTAMP DEFAULT (datetime('now', 'localtime', '+3 hours')) NOT NULL,
-#                              name TEXT, view TEXT, duration TEXT)
-#        ''')
-#    db.commit()
-#    db.close()
-
-#    db = sqlite3.connect(DB_FILE)
-# #created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
-def get_latest_reviews(celsci2): #db: sqlite3.Connection):
-    try:
-        celsci2=ecf(celsci2)
-        df=pd.DataFrame.from_dict(celsci2["videos"])
-        reviews = df.DataFrame(reviews, columns=["id", "date_created", "name", "view", "duration"])
-
-    except: # sqlite3.OperationalError:
-        df=pd.DataFrame()
-        print ("db ...")
-    #reviews = db.execute("SELECT * FROM reviews ORDER BY id DESC limit 100").fetchall()
-    #total_reviews = db.execute("Select COUNT(id) from reviews").fetchone()[0]
-    total_reviews=reviews.count()[0]
+db = sqlite3.connect(DB_FILE)
+try:
+    db.execute("SELECT * FROM reviews").fetchall()
+    #db.execute("SELECT * FROM reviews2").fetchall()
+
+    db.close()
+except sqlite3.OperationalError:
+    db.execute(
+        '''
+        CREATE TABLE reviews (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+                              created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
+                              name TEXT, view TEXT, duration TEXT)
+        ''')
+    db.commit()
+    db.close()
+
+    db = sqlite3.connect(DB_FILE)
+
+def get_latest_reviews(db: sqlite3.Connection):
+    reviews = db.execute("SELECT * FROM reviews ORDER BY id DESC limit 100").fetchall()
+    total_reviews = db.execute("Select COUNT(id) from reviews").fetchone()[0]
+    reviews = pd.DataFrame(reviews, columns=["id", "date_created", "name", "view", "duration"])
     return reviews, total_reviews
 
-#def get_latest_reviews2(db: sqlite3.Connection):
-#    reviews2 = db.execute("SELECT * FROM reviews2 ORDER BY id DESC limit 100").fetchall()
-#    total_reviews2 = db.execute("Select COUNT(id) from reviews2").fetchone()[0]
-#    reviews2 = pd.DataFrame(reviews2, columns=["id","title", "link","channel", "description", "views", "uploaded", "duration", "durationString"])
-#    return reviews2, total_reviews2
+def get_latest_reviews2(db: sqlite3.Connection):
+    reviews2 = db.execute("SELECT * FROM reviews2 ORDER BY id DESC limit 100").fetchall()
+    total_reviews2 = db.execute("Select COUNT(id) from reviews2").fetchone()[0]
+    reviews2 = pd.DataFrame(reviews2, columns=["id","title", "link","channel", "description", "views", "uploaded", "duration", "durationString"])
+    return reviews2, total_reviews2
     
 def ccogsphere(name: str, rate: int, celsci: str):
-    #db = sqlite3.connect(DB_FILE)
-    #cursor = db.cursor()
+    db = sqlite3.connect(DB_FILE)
+    cursor = db.cursor()
       
     #try:
     celsci2=celsci.split()
@@ -110,13 +99,11 @@ def ccogsphere(name: str, rate: int, celsci: str):
         duration = str(row["duration"])
         print(view, duration)
         #celsci=celsci+celsci2
-        
-        #cursor.execute("INSERT INTO reviews(name, view, duration) VALUES(?,?,?)", [celsci+" Video #"+str(index+1), view, duration])
-        #db.commit()
-    reviews=df
-    total_reviews=reviews.count()[0]
-    #reviews, total_reviews = get_latest_reviews(db)
-    #db.close()
+        cursor.execute("INSERT INTO reviews(name, view, duration) VALUES(?,?,?)", [celsci+str(index+1), view, duration])
+        db.commit()
+    
+    reviews, total_reviews = get_latest_reviews(db)
+    db.close()
     r = requests.post(url='https://ccml-persistent-data2.hf.space/api/predict/', json={"data": [celsci + " ", celsci2]}) 
 
     return reviews, total_reviews
@@ -143,37 +130,33 @@ def run_ecs(inp):
     try:
         result=ecf(inp)
         df=pd.DataFrame.from_dict(result["videos"])
-    except: # sqlite3.OperationalError:
-        df=pd.DataFrame()
-        print ("db ...")
+    except sqlite3.OperationalError:
+        print ("db error")
     
-    #df=df.drop(df.columns[4], axis=1)
+    df=df.drop(df.columns[4], axis=1)
 
-    #db = sqlite3.connect(DB_FILE)
+    db = sqlite3.connect(DB_FILE)
     #cursor = db.cursor()
     #cursor.execute("INSERT INTO reviews2(title, link, thumbnail,channel, description, views, uploaded, duration, durationString) VALUES(?,?,?,?,?,?,?,?,?)", [title, link, thumbnail,channel, description, views, uploaded, duration, durationString])
-    #df.to_sql('reviews2', db, if_exists='replace', index=False)
+    df.to_sql('reviews2', db, if_exists='replace', index=False)
 
     #db.commit()
-    #reviews2, total_reviews2 = get_latest_reviews(db)
-    #db.close()
+    reviews2, total_reviews2 = get_latest_reviews(db)
+    db.close()
     #print ("print000", total_reviews2,reviews2)
-    reviews2=df
-    total_reviews2=reviews2.count()[0]
-
     return reviews2, total_reviews2
     
     
-#def load_data():
-#    #db = sqlite3.connect(DB_FILE)
-#    reviews, total_reviews = get_latest_reviews()
-#    #db.close()
-#    return reviews, total_reviews
-#def load_data2():
-#    db = sqlite3.connect(DB_FILE)
-#    reviews2, total_reviews2 = get_latest_reviews2(db)
-#    db.close()
-#    return reviews2, total_reviews2
+def load_data():
+    db = sqlite3.connect(DB_FILE)
+    reviews, total_reviews = get_latest_reviews(db)
+    db.close()
+    return reviews, total_reviews
+def load_data2():
+    db = sqlite3.connect(DB_FILE)
+    reviews2, total_reviews2 = get_latest_reviews2(db)
+    db.close()
+    return reviews2, total_reviews2
     
 css="footer {visibility: hidden}"
 # Applying style to highlight the maximum value in each row
@@ -191,14 +174,61 @@ with gr.Blocks(css=css) as demo:
             #run_actr()
             submit = gr.Button(value=".")            
             submit.click(ccogsphere, [name, rate, celsci], [data, count])
-            demo.load(get_latest_reviews, celsci, [data, count])
-            #@name.change(inputs=name, outputs=celsci,_js="window.location.reload()")
-            #@rate.change(inputs=rate, outputs=name,_js="window.location.reload()")
-            #@celsci.change(inputs=celsci, outputs=rate,_js="window.location.reload()") 
+            demo.load(load_data, None, [data, count])
+            @name.change(inputs=name, outputs=celsci,_js="window.location.reload()")
+            @rate.change(inputs=rate, outputs=name,_js="window.location.reload()")
+            @celsci.change(inputs=celsci, outputs=rate,_js="window.location.reload()")  
             
-            #def secwork(celsci):
+            def secwork(name):
                 #if name=="abc":
                 #run_code()
-                #demo.load(get_latest_reviews, celsci, [data, count])
+                load_data()
                 #return "Hello " + name + "!"
+def backup_db():
+    shutil.copyfile(DB_FILE, "./reviews.db")
+    db = sqlite3.connect(DB_FILE)
+    reviews = db.execute("SELECT * FROM reviews").fetchall()
+    pd.DataFrame(reviews).to_csv("./reviews.csv", index=False)
+    print("updating db")
+    repo.push_to_hub(blocking=False, commit_message=f"Updating data at {datetime.datetime.now()}")
+    
+def backup_db_csv():
+    shutil.copyfile(DB_FILE, "./reviews2.db")
+    db = sqlite3.connect(DB_FILE)
+    reviews = db.execute("SELECT * FROM reviews").fetchall()
+    pd.DataFrame(reviews).to_csv("./reviews2.csv", index=False)
+    print("updating db csv")
+    dataset = load_dataset("csv", data_files="./reviews2.csv")
+    repo.push_to_hub("CognitiveScience/csdhdata", blocking=False) #, commit_message=f"Updating data-csv at {datetime.datetime.now()}")
+    #path1=hf_hub_url()
+    #print (path1)
+    #hf_hub_download(repo_id="CogSphere/aCogSphere", filename="./*.csv")
+    #hf_hub_download(repo_id="CognitiveScience/csdhdata", filename="./*.db")
+    #hf_hub_download(repo_id="CogSphere/aCogSphere", filename="./*.md")
+    #hf_hub_download(repo_id="CognitiveScience/csdhdata", filename="./*.md")
+
+
+#def load_data2():
+#    db = sqlite3.connect(DB_FILE)
+#    reviews, total_reviews = get_latest_reviews(db)
+#    #db.close()
+#    demo.load(load_data,None, [reviews, total_reviews])
+#    #return reviews, total_reviews
+    
+#scheduler1 = BackgroundScheduler()
+#scheduler1.add_job(func=run_actr, trigger="interval", seconds=10)
+#scheduler1.start()
+    
+scheduler1 = BackgroundScheduler()
+scheduler1.add_job(func=load_data, trigger="interval", seconds=5)
+scheduler1.start()
+
+scheduler2 = BackgroundScheduler()
+scheduler2.add_job(func=backup_db, trigger="interval", seconds=3633)
+scheduler2.start()
+
+scheduler3 = BackgroundScheduler()
+scheduler3.add_job(func=backup_db_csv, trigger="interval", seconds=3666)
+scheduler3.start()
+
 demo.launch()