Upload 3 files

constants.py

@@ -5,5 +5,5 @@
 #subject = 'Guerra entre Irã e Paquistão'
 
 #sites = ['https://www.cnnbrasil.com.br/']#, 'https://g1.globo.com/', 'https://www.metropoles.com/']
-num_sites = 5
+num_sites = 2 #5
 local_base = 'local_base'

main.py

@@ -45,11 +45,12 @@ def generate_news(subject, min_words, max_words, sites):
     print('\n\n' + '*' * 50 + '\n\n')
     print(result_news[0])
 
-    print('\n\nFontes: ')
-    print(result_news[1])
-
-    return result_news[0] + '\n\n' + 'Fontes: ' + '\n' + result_news[1]
-
+    if result_news[1]:
+        print('\n\nFontes: ')
+        print(result_news[1])
+        return result_news[0] + '\n\n' + 'Fontes: ' + '\n' + result_news[1]
+    else:
+        return result_news[0]
 
 def call_llm(context, prompt):
     print('\nChamando o modelo de linguagem...')

rag.py

@@ -28,7 +28,7 @@ def generate_embeddings_and_vectorstore(path):
         corpus = loader.load()
         print(f'    Total de documentos antes do text_split = {len(corpus)}')
 
-        text_splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=400)
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=4000, chunk_overlap=400)
         docs = text_splitter.split_documents(corpus)
         num_total_characters = sum([len(x.page_content) for x in docs])
         print(f"    Total de chunks depois do text_split = {len(docs)}")
@@ -57,10 +57,11 @@ class Rag:
         self.vectorstore = vectorstore
         self.memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True, output_key="answer")
 
-        #Do not make up any information, create the news just based on the given information on the pieces of texts delimited by <>.
+        #
+        #Do not use only your knowledge to make the news.
         prompt_template = """Your task is to create news for a newspaper based on pieces of text delimited by <> and a question delimited by <>.
-                    Do not use only your knowledge to make the news. Make the news based on the pieces of text delimited by <>.
-                    If the pieces of text don't have relevant information about the question, just say that you need more information to make the news.
+                    Do not use only your knowledge to make the news. Make the news based on the question, but using the pieces of text.
+                    If the pieces of text don't enough information about the question to create the news, just say that you need more sources of information, nothing more.
                     The news should have a title.
                     The news should be written in a formal language.
                     The news should have between {min_words} and {max_words} words and it should be in Portuguese language.
@@ -72,13 +73,13 @@ class Rag:
                                      partial_variables={"min_words": min_words, "max_words": max_words})
 
         self.qa = ConversationalRetrievalChain.from_llm(
-                    llm=ChatOpenAI(model_name="gpt-3.5-turbo-0125",
-                                   temperature=0.1,
+                    llm=ChatOpenAI(model_name="gpt-3.5-turbo-0125", #0125 #1106
+                                   temperature=0,
                                    openai_api_key=os.environ['OPENAI_KEY'],
                                    max_tokens=int(int(max_words) + (int(max_words) / 2))), #número máximo de tokens para a resposta
                     memory=self.memory,
-                    #retriever=vectorstore.as_retriever(search_type='similarity_score_threshold',
-                    #                                   search_kwargs={'k':4, 'score_threshold':0.5}), #search_kwargs={'k': 3}
+                    # retriever=vectorstore.as_retriever(search_type='similarity_score_threshold',
+                    #                                    search_kwargs={'k':4, 'score_threshold':0.8}), #search_kwargs={'k': 3}
                     retriever=vectorstore.as_retriever(),
                     combine_docs_chain_kwargs={"prompt": self.prompt},
                     chain_type="stuff",#map_reduce, refine, map_rerank