mastodon migration

README.md

@@ -14,10 +14,10 @@ pinned: false
 
 ### Try it out on [🤗 Spaces](https://huggingface.co/spaces/deepset/should-i-follow)
 
-##### A simple app to get an overview of what the twitter user has been posting about and their tone
+##### A simple app to get an overview of what the Mastodon user has been posting about and their tone
 
 This is a demo just for fun 🥳
-This repo contains a streamlit application that given a Twitter username, tells you what type of things they've been posting about lately, their tone, and the languages they use. It uses the LLM by OpenAI `text-davinci-003`.
+This repo contains a streamlit application that given a Mastodon username, tells you what type of things they've been posting about lately, their tone, and the languages they use. It uses the LLM by OpenAI `text-davinci-003`.
 
 It's been built with [Haystack](https://haystack.deepset.ai) using the [`PromptNode`](https://docs.haystack.deepset.ai/docs/prompt_node) and by creating a custom [`PromptTemplate`](https://docs.haystack.deepset.ai/docs/prompt_node#templates)
 
@@ -40,8 +40,6 @@ To run the bare application which does _nothing_:
 `pip install -r requirements.txt`
 2. Run the streamlit app:
 `streamlit run app.py`
-3. Createa a `.env` and add your Twitter Bearer token:
-`TWITTER_BEARER_TOKEN` 
 
 This will start up the app on `localhost:8501` where you will dind a simple search bar
 

app.py

@@ -9,26 +9,25 @@ import streamlit as st
 
 from utils.haystack import query, start_haystack
 from utils.ui import reset_results, set_initial_state, sidebar
-from utils.config import TWITTER_BEARER
 
 set_initial_state()
 
 sidebar()
 
-st.write("# 🐤 What have they been tweeting about lately?")
+st.write("# 🐤 What have they been posting about lately on Mastodon?")
 
 if st.session_state.get("OPENAI_API_KEY"):
-    prompter, template = start_haystack(st.session_state.get("OPENAI_API_KEY"))
+    pipeline = start_haystack(st.session_state.get("OPENAI_API_KEY"))
     st.session_state["api_key_configured"] = True
     search_bar, button = st.columns(2)
     # Search bar
     with search_bar: 
-        username = st.text_input("Please provide a twitter username", on_change=reset_results)
+        username = st.text_input("Please provide a Mastodon username", on_change=reset_results)
 
     with button: 
         st.write("")
         st.write("")
-        run_pressed = st.button("Search tweets")
+        run_pressed = st.button("Search posts (toots)")
 else:
     st.write("Please provide your OpenAI Key to start using the application")
     st.write("If you are using a smaller screen, open the sidebar from the top left to provide your OpenAI Key 🙌")
@@ -44,7 +43,7 @@ if st.session_state.get("api_key_configured"):
         st.session_state.username = username
         with st.spinner("🔎"):
             try:
-                st.session_state.result = query(username, prompter, template)
+                st.session_state.result = query(username, pipeline)
             except JSONDecodeError as je:
                 st.error(
                     "👓    An error occurred reading the results. Is the document store working?"
@@ -55,5 +54,5 @@ if st.session_state.get("api_key_configured"):
                 
     if st.session_state.result:
         voice = st.session_state.result
-        st.write(voice[0])
+        st.write(voice['results'][0])
             

requirements.txt

@@ -1,5 +1,7 @@
-farm-haystack==1.18.1
+safetensors==0.3.3.post1
+farm-haystack==1.20.0
 streamlit==1.21.0
 markdown
 st-annotated-text
-python-dotenv
+python-dotenv
+mastodon-fetcher-haystack

utils/haystack.py

@@ -1,55 +1,51 @@
 import streamlit as st
-import requests
-from utils.config import TWITTER_BEARER
-
+from mastodon_fetcher_haystack.mastodon_fetcher import MastodonFetcher
+from haystack import Pipeline
 from haystack.nodes import PromptNode, PromptTemplate
 
 def start_haystack(openai_key):
     #Use this function to contruct a pipeline
-    prompt_node = PromptNode(model_name_or_path="text-davinci-003", api_key=openai_key)
-
+    fetcher = MastodonFetcher()
 
-    twitter_template = PromptTemplate(prompt="""You will be given a twitter stream belonging to a specific profile. Answer with a summary of what they've lately been tweeting about and in what languages.
-                                                You may go into some detail about what topics they tend to like tweeting about. Please also mention their overall tone, for example: positive,
+    mastodon_template = PromptTemplate(prompt="""You will be given a post stream belonging to a specific Mastodon profile. Answer with a summary of what they've lately been posting about and in what languages.
+                                                You may go into some detail about what topics they tend to like postint about. Please also mention their overall tone, for example: positive,
                                                 negative, political, sarcastic or something else.
 
                                                 Examples: 
                                                 
-                                                Twitter stream: RT @deepset_ai: Come join our Haystack server for our first Discord event tomorrow, a deepset AMA session with @rusic_milos @malte_pietsch…
-                                                RT @deepset_ai: Join us for a chat! On Thursday 25th we are hosting a 'deepset - Ask Me Anything' session on our brand new Discord. Come…
-                                                RT @deepset_ai: Curious about how you can use @OpenAI GPT3 in a Haystack pipeline? This week we released Haystack 1.7 with which we introdu…
-                                                RT @tuanacelik: So many updates from @deepset_ai today! 
+                                                Post stream: [@deepset_ai](https://mastodon.social/@deepset_ai): Come join our Haystack server for our first Discord event tomorrow, a deepset AMA session with @rusic_milos @malte_pietsch…
+                                                [@deepset_ai](https://mastodon.social/@deepset_ai): Join us for a chat! On Thursday 25th we are hosting a 'deepset - Ask Me Anything' session on our brand new Discord. Come…
+                                                [@deepset_ai](https://mastodon.social/@deepset_ai): Curious about how you can use @OpenAI GPT3 in a Haystack pipeline? This week we released Haystack 1.7 with which we introdu…
+                                                [@deepset_ai](https://mastodon.social/@deepset_ai): So many updates from @deepset_ai today! 
                                                 
-                                                Summary: This user has lately been retweeting tweets fomr @deepset_ai. The topics of the tweets have been around the Haystack community, NLP and GPT. They've
+                                                Summary: This user has lately been reposting posts from @deepset_ai. The topics of the posts have been around the Haystack community, NLP and GPT. They've
                                                 been posting in English, and have had a positive, informative tone.
                                                 
-                                                Twitter stream: I've directed my team to set sharper rules on how we deal with unidentified objects.\n\nWe will inventory, improve ca… 
+                                                Post stream: I've directed my team to set sharper rules on how we deal with unidentified objects.\n\nWe will inventory, improve ca… 
                                                 the incursion by China’s high-altitude balloon, we enhanced radar to pick up slower objects.\n \nBy doing so, w…
                                                 I gave an update on the United States’ response to recent aerial objects. 
 
-                                                Summary: This user has lately been tweeting about having sharper rules to deal with unidentified objects and an incursuin by China's high-altitude
-                                                baloon. Their tweets have mostly been neutral but determined in tone. They mostly post in English.
+                                                Summary: This user has lately been posting about having sharper rules to deal with unidentified objects and an incursuin by China's high-altitude
+                                                baloon. Their pots have mostly been neutral but determined in tone. They mostly post in English.
 
-                                                Twitter stream: {tweets}
+                                                Post stream: {join(documents)}
                                                                 
                                                 Summary: 
                                                 """)
+    prompt_node = PromptNode(model_name_or_path="gpt-4", default_prompt_template=mastodon_template, api_key=openai_key)
+
+    st.session_state["haystack_started"] = True   
 
-    st.session_state["haystack_started"] = True                                                     
-    return prompt_node, twitter_template
+    mastodon_pipeline = Pipeline()
+    mastodon_pipeline.add_node(component=fetcher, name="MastodonFetcher", inputs=["Query"])
+    mastodon_pipeline.add_node(component=prompt_node, name="PromptNode", inputs=["MastodonFetcher"])                                               
+    return mastodon_pipeline
 
 
 @st.cache_data(show_spinner=True)
-def query(username, _prompter, _template):
-    headers = {"Authorization": "Bearer {}".format(TWITTER_BEARER)}
-    print(username)
-    url = f"https://api.twitter.com/1.1/statuses/user_timeline.json?screen_name={username}&count={60}"
+def query(username, _pipeline):
     try:
-        response = requests.request("GET", url, headers = headers)
-        twitter_stream = ""
-        for tweet in response.json():
-            twitter_stream += tweet["text"]
-        result = _prompter.prompt(prompt_template=_template, tweets=twitter_stream)
+        result = _pipeline.run(query=username, params={"MastodonFetcher": {"last_k_posts": 20}})
     except Exception as e:
-        result = ["Please make sure you are providing a correct, public twitter account"]
+        result = ["Please make sure you are providing a correct, public Mastodon account"]
     return result

utils/ui.py

@@ -6,7 +6,7 @@ def set_state_if_absent(key, value):
         st.session_state[key] = value
 
 def set_initial_state():
-    set_state_if_absent("username", "Provide a Twitter username")
+    set_state_if_absent("username", "Provide a Mastodon username (e.g. [email protected])")
     set_state_if_absent("result", None)
     set_state_if_absent("haystack_started", False)
 
@@ -28,7 +28,7 @@ def sidebar():
         st.markdown(
             "## How to use\n"
             "1. Enter your [OpenAI API key](https://platform.openai.com/account/api-keys) below\n"
-            "2. Enter a Twitter username in the searchbar\n"
+            "2. Enter a Mastodon username in the searchbar\n"
             "3. Enjoy 🤗\n"
         )