Adding v1 of rag-agent

Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.8-slim
+
+ENV APP_HOME=/app
+WORKDIR ${APP_HOME}
+
+COPY requirements.txt ./
+
+COPY ./agente ./
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+EXPOSE 7860
+
+CMD ["streamlit", "run", "interface/chatbot.py", "--server.port=7860"]

agente/.env-example

@@ -0,0 +1,18 @@
+OPENAI_API_KEY="SUA-KEY"
+OPENAI_MODEL_CHAT="gpt-3.5-turbo"
+OPENAI_MODEL_EMBEDDING="text-embedding-3-small"
+
+COLLECTION_NAME="sgd"
+
+EDGEDB_HOST="localhost"
+EDGEDB_USER="sgd"
+EDGEDB_PASSWORD="secret"
+EDGEDB_PORT="5656"
+
+QDRANT_HOST="localhost"
+QDRANT_PORT="6333"
+
+REDIS_HOST="localhost"
+REDIS_PORT="6379"
+
+HUGGINGFACE_API_KEY="SUA-KEY"

agente/README.md

@@ -0,0 +1,6 @@
+
+Para testar prototipo
+
+```
+streamlit run interface/main.py
+```

agente/agent.py

@@ -0,0 +1,91 @@
+import os
+from dotenv import load_dotenv
+from openai import OpenAI
+from qdrant_client import QdrantClient
+from pipelines.message import send_message
+import redis
+
+conversation_chat = []
+
+
+def run():
+    load_dotenv()
+
+    try:
+        oa_client = OpenAI(
+            api_key=os.environ.get("OPENAI_API_KEY")
+        )
+        print("✅ Conectado a OpenAI.")
+
+        qdrant_client = QdrantClient(
+            host=os.environ.get("QDRANT_HOST"),
+            port=os.environ.get("QDRANT_PORT")
+        )
+        print("✅ Conectado ao Qdrant.")
+
+        redis_client = redis.Redis(
+            host=os.environ.get("REDIS_HOST"),
+            port=os.environ.get("REDIS_PORT"),
+            decode_responses=True
+        )
+        print("✅ Conectado ao Redis.")
+
+        while True:
+            prompt = input("Digite sua pergunta: ")
+
+            embedding = oa_client.embeddings.create(
+                input=[prompt],
+                model=os.environ.get("OPENAI_MODEL_EMBEDDING")
+            ).data[0].embedding
+
+            child_texts = qdrant_client.search(
+                collection_name=os.environ.get("COLLECTION_NAME"), 
+                query_vector=embedding,
+                limit=3
+            )
+
+            print("--------- Child text ---------")
+            print(child_texts)
+
+            contexts = []
+
+            for child_text in child_texts:
+                parent_text = redis_client.hgetall(
+                    child_text[0].payload["parent_id"]
+                )
+                context = {
+                    "content": parent_text["content"],
+                    "url": parent_text["url"]
+                }
+                contexts.append(context)
+
+            print("--------- Contexts ---------")
+            print(contexts)
+
+            stream_response = send_message(
+                oa_client,
+                context,
+                prompt,
+                conversation_chat
+            )
+
+            print("--------- Response Agent ---------")
+            response = ""
+            for chunk in stream_response:
+                if chunk.choices[0].delta.content is not None:
+                    response += chunk.choices[0].delta.content
+                    print(chunk.choices[0].delta.content, end="")
+            conversation_chat.append({
+                "role": "assistant",
+                "content": response
+            })
+
+            is_exit = input("\nDeseja sair? (s/n): ")
+            if is_exit == "s":
+                break
+    except Exception as error:
+        print(f"❌ Erro: {error}")
+
+
+if __name__ == "__main__":
+    run()

agente/interface/chatbot.py

@@ -0,0 +1,150 @@
+import os
+import sys
+from dotenv import load_dotenv
+import edgedb
+from openai import OpenAI
+from qdrant_client import QdrantClient
+import streamlit as st
+
+# Add the parent directory to the Python path
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from pipelines.message import send_message # noqa
+
+st.set_page_config(page_title="Carlos AI Agent")
+
+
+load_dotenv()
+
+
+@st.cache_resource
+def connect_to_services():
+    oa_client = OpenAI(
+        api_key=os.environ.get("OPENAI_API_KEY")
+    )
+
+    qdrant_client = QdrantClient(
+        url=os.environ.get("QDRANT_URL"),
+        api_key=os.environ.get("QDRANT_KEY")
+    )
+
+    edgedb_client = edgedb.create_client()
+
+    return oa_client, qdrant_client, edgedb_client
+
+
+def send_message_for_ai(prompt, oa_client, qdrant_client, edgedb_client):
+    embedding = oa_client.embeddings.create(
+        input=[prompt],
+        model=os.environ.get("OPENAI_MODEL_EMBEDDING")
+    ).data[0].embedding
+
+    child_texts = qdrant_client.search(
+        collection_name=os.environ.get("COLLECTION_NAME"), 
+        query_vector=embedding,
+        limit=3
+    )
+
+    contexts = []
+
+    for child_text in child_texts:
+        parent_text = edgedb_client.query('''
+            SELECT Pattern {
+                content,
+                url,
+                parent_id
+            }
+            FILTER .id = <uuid>$parent_id
+            ''', parent_id=child_text.payload["parent_id"])[0]
+        context = {
+            "content": parent_text.content,
+            "url": parent_text.url,
+            "parent_id": parent_text.parent_id
+        }
+        contexts.append(context)
+
+    # system_msg.write(
+    #    f"""Contexto: {contexts}""")
+    
+    stream_response = send_message(
+        oa_client,
+        contexts,
+        prompt,
+        []
+    )
+
+    return stream_response
+
+
+oa_client, qdrant_client, edgedb_client = connect_to_services()
+
+
+def display_chat():
+    for msg in st.session_state.messages:
+        st.chat_message(msg["role"]).write(msg["content"])
+
+
+def sidebar_content():
+    st.image('https://www.gov.br/++theme++padrao_govbr/img/govbr-logo-large.png', width=200)
+    st.header("Tópicos frequentes")
+
+    # Botões de exemplo
+    topics = [
+        "Niveis da conta govbr.",
+        "Dúvidas no reconhecimento facial.",
+        "Dúvidas na autenticação dos bancos.",
+        "Dúvidas para aumentar o nível com a cin."
+    ]
+
+    for topic in topics:
+        st.button(topic)
+
+    # Espaços em branco para organização
+    for _ in range(5):
+        st.write("")
+
+    # Botão centralizado
+    col1, col2, col3 = st.columns([1, 1, 1])
+    with col2:
+        st.button("VOLTAR")
+
+
+# Função principal de processamento de input do chat
+def process_user_input():
+    if prompt := st.chat_input(placeholder="Digite sua mensagem"):
+        # Armazenar mensagem do usuário
+        st.session_state.messages.append({"role": "user", "content": prompt})
+        st.chat_message("user").write(prompt)
+
+        # Simulação de resposta do assistente
+        response = send_message_for_ai(
+        prompt, oa_client, qdrant_client, edgedb_client)
+
+        # Armazenar e exibir resposta do assistente
+        st.session_state.messages.append({"role": "assistant", "content": response})
+        st.chat_message("assistant").write(response)
+
+        
+#system_msg = st.chat_message("system")
+
+
+# Configuração inicial
+if "messages" not in st.session_state:
+    st.session_state["messages"] = [{"role": "assistant", "content": "Como eu posso ajudar?"}]
+
+# Exibição da barra lateral
+with st.sidebar:
+    sidebar_content()
+
+# Exibição do título e subtítulo
+st.title("Bem-vindo à ajuda do gov.br")
+st.caption("💬 Lorem ipsum dolor sit amet, consectetur adipiscing elit.")
+
+# Exibição do chat
+display_chat()
+
+# Processamento da entrada do usuário no chat
+process_user_input()
+
+
+

agente/interface/main.py

@@ -0,0 +1,91 @@
+import os
+import sys
+from dotenv import load_dotenv
+from openai import OpenAI
+from qdrant_client import QdrantClient
+import redis
+import streamlit as st
+
+# Add the parent directory to the Python path
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from pipelines.message import send_message # noqa
+
+st.set_page_config(page_title="Carlos AI Agent")
+
+
+load_dotenv()
+
+
+@st.cache_resource
+def connect_to_services():
+    oa_client = OpenAI(
+        api_key=os.environ.get("OPENAI_API_KEY")
+    )
+
+    qdrant_client = QdrantClient(
+        host=os.environ.get("QDRANT_HOST"),
+        port=os.environ.get("QDRANT_PORT")
+    )
+
+    redis_client = redis.Redis(
+        host=os.environ.get("REDIS_HOST"), 
+        port=os.environ.get("REDIS_PORT"), 
+        decode_responses=True
+    )
+
+    return oa_client, qdrant_client, redis_client
+
+
+def send_message_for_ai(prompt, oa_client, qdrant_client, redis_client):
+    embedding = oa_client.embeddings.create(
+        input=[prompt],
+        model=os.environ.get("OPENAI_MODEL_EMBEDDING")
+    ).data[0].embedding
+
+    child_texts = qdrant_client.search(
+        collection_name=os.environ.get("COLLECTION_NAME"), 
+        query_vector=embedding,
+        limit=3
+    )
+
+    contexts = []
+
+    for child_text in child_texts:
+        parent_text = redis_client.hgetall(
+            child_text.payload["parent_id"]
+        )
+        context = {
+            "content": parent_text["content"],
+            "url": parent_text["url"],
+            "parent_id": parent_text["parent_id"]
+        }
+        contexts.append(context)
+
+    system_msg.write(
+        f"""Contexto: {contexts}""")
+    
+    stream_response = send_message(
+        oa_client,
+        contexts,
+        prompt,
+        []
+    )
+
+    return stream_response
+
+
+oa_client, qdrant_client, redis_client = connect_to_services()
+
+st.write("# Carlos AI Agent")
+
+prompt = st.chat_input("Digite sua pergunta:")
+user_msg = st.chat_message("user")
+assistant_msg = st.chat_message("assistant")
+system_msg = st.chat_message("system")
+
+if prompt:
+    user_msg.write(prompt)
+    response = send_message_for_ai(
+        prompt, oa_client, qdrant_client, redis_client)
+    assistant_msg.write_stream(response)

agente/pipelines/message.py

@@ -0,0 +1,24 @@
+import os
+from openai import Client
+from utils.file import File
+
+
+def send_message(oa_client: Client, context, question, conversation_chat):
+    if len(conversation_chat) == 0:
+        system_message = File("prompts/system.md")
+        conversation_chat.append({
+            "role": "system",
+            "content": system_message
+        })
+
+    conversation_chat.append({
+        "role": "user",
+        "content": f"Context: {context}\nQuestion: {question}"
+    })
+
+    return oa_client.chat.completions.create(
+        model=os.environ.get("OPENAI_MODEL_CHAT"),
+        messages=conversation_chat,
+        stream=True
+    )
+

agente/pipelines/oac/oac.py

@@ -0,0 +1,33 @@
+from openai import Client
+
+
+def get_description_of_image(oa_client: Client, image_url):
+    completion = oa_client.chat.completions.create(
+                model="gpt-4o-mini",
+                messages=[
+                    {
+                        "role": "system",
+                        "content": """
+                            You are an image descriptor who will describe images using the Brazilian Portuguese language.
+                            This description should be short and to the point, with a maximum of 50 characters, to be placed in the alt tag of an HTML.
+                        """ # noqa
+                    },
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "text",
+                                "text": "What's in this image?"
+                            },
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": f"{image_url}"
+                                }
+                            }
+                        ]
+                    }
+                ],
+                max_tokens=300
+    )
+    return completion.choices[0].message.content

agente/pipelines/rag.py

@@ -0,0 +1,185 @@
+import os
+import sqlite3
+import time
+from typing import List
+
+from dotenv import load_dotenv
+from openai import OpenAI
+from qdrant_client import QdrantClient
+
+from utils.markdown import generate_image_description_with_empty_description
+from utils.cli import \
+    print_execution_time, start_loading_animation, stop_loading_animation
+from splitter import splitter
+from qdrant_client.models import Distance, VectorParams
+from qdrant_client.models import PointStruct
+
+
+HEADERS_TO_SPLIT_ON = [
+        ("#", "Header 1"),
+        ("##", "Header 2"),
+        ("###", "Header 3"),
+        ("####", "Header 4"),
+        ("#####", "Header 5"),
+        ("######", "Header 6"),
+]
+
+
+def safe_join(thread, done: List[bool]):
+    if thread is not None and thread.is_alive():
+        done[0] = True
+        thread.join()
+
+
+def run():
+    loading_split = None
+    loading_image_description = None
+    loading_embedding = None
+    done_split = [False]
+    done_image_description = [False]
+    done_embeding = [False]
+
+    try:
+        load_dotenv()
+
+        start_time = time.time()
+        print("➗ Começando o RAG...")
+
+        conn = sqlite3.connect('../meu_banco.db')
+        cursor = conn.cursor()
+        print("✅ Conectado ao sqlite.")
+
+        qdrant_client = QdrantClient(":memory:")
+        print("✅ Conectado ao qdrant.")
+
+        oa_client = OpenAI(
+            api_key=os.environ.get("OPENAI_API_KEY")
+        )
+        print("✅ Conectado a OpenAI.")
+
+        if qdrant_client.collection_exists(os.environ.get("COLLECTION_NAME")):
+            qdrant_client.delete_collection(
+                collection_name=os.environ.get("COLLECTION_NAME")
+            )
+        qdrant_client.create_collection(
+            collection_name=os.environ.get("COLLECTION_NAME"),
+            vectors_config=VectorParams(size=1536, distance=Distance.COSINE)
+        )
+        print("✅ Collection criada.")
+
+        cursor.execute('''
+            SELECT w.id, w.url, w.text_content, w.relative_path, w.hyperrefs, 
+                   i.name AS image_name, i.path AS image_path, i.url AS image_url, i.hyperlink AS image_hyperlink, i.alt AS image_alt,
+                   v.name AS video_name, v.url AS video_url, v.hyperlink AS video_hyperlink, v.alt AS video_alt
+            FROM Website w
+            LEFT JOIN Image i ON i.website_id = w.id
+            LEFT JOIN Video v ON v.website_id = w.id
+        ''')
+        web_sites_bruto = cursor.fetchall()
+        web_sites = []
+        for web_site_bruto in web_sites_bruto:
+            web_site = {
+                "text_content": web_site_bruto[2],
+                "text_id": web_site_bruto[0],
+                "url": web_site_bruto[1]
+            }
+            web_sites.append(web_site)
+
+        loading_image_description = start_loading_animation(
+            done_image_description,
+            "Gerando Descrições de imagens caso não tenha..."
+        )
+
+        texts_with_images_descriptions, error_images_description = \
+            generate_image_description_with_empty_description(
+                oa_client,
+                web_sites
+            )
+        stop_loading_animation(
+            done_image_description,
+            loading_image_description
+        )
+        print("✅ Descrição de imagens geradas.")
+
+        texts_parent_splitted, texts_child_splitted = splitter.split(
+            texts_with_images_descriptions,
+            HEADERS_TO_SPLIT_ON
+        )
+        print("✅ Textos dividos.")
+
+        loading_split = start_loading_animation(
+            done_split,
+            "Dividindo textos..."
+        )
+        for text_parent_splitted in texts_parent_splitted:
+            cursor.execute('''
+                CREATE TABLE IF NOT EXISTS ParentText
+                (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    content TEXT,
+                    parent_id INTEGER,
+                    url TEXT,
+                    FOREIGN KEY (parent_id) REFERENCES Website(id)
+                );
+            ''')
+            cursor.execute('''
+                INSERT INTO ParentText (content, parent_id, url)
+                VALUES (?, ?, ?);
+            ''', (
+                text_parent_splitted["content"].page_content,
+                text_parent_splitted["parent_id"],
+                text_parent_splitted["url"]
+            ))
+        stop_loading_animation(done_split, loading_split)
+        print("✅ Texto pai salvo no sqlite.")
+
+        count_embeddings = 0
+        total_child_splitted = len(texts_child_splitted)
+        for text_child_splitted in texts_child_splitted:
+            done_embeding = [False]
+            loading_embedding = start_loading_animation(
+                done_embeding,
+                f"""Gerando embeddings e salvando no qdrant: {count_embeddings} de {total_child_splitted}""" # noqa
+            )
+            embedding = oa_client.embeddings.create(
+                input=[text_child_splitted["content"].page_content],
+                model=os.environ.get("OPENAI_MODEL_EMBEDDING")
+            ).data[0].embedding
+            qdrant_client.upsert(
+                collection_name=os.environ.get("COLLECTION_NAME"),
+                points=[PointStruct(
+                    id=str(text_child_splitted["id"]),
+                    vector=embedding,
+                    payload={
+                        "content": text_child_splitted["content"].page_content,
+                        "parent_id": text_child_splitted["parent_id"],
+                        "type": "text"
+                    }
+                )]
+            )
+            count_embeddings += 1
+            stop_loading_animation(done_embeding, loading_embedding)
+        print("✅ Texto filho salvo no qdrant.")
+
+        print("✅ RAG finalizado.")
+
+        print(
+            f"""
+            📊 Relatório:\n
+            \t Tempo de execução: {print_execution_time(start_time)}\n
+            \t Textos Filhos e Embedding gerados: {len(texts_child_splitted)}\n
+            \t Textos pai gerados: {len(texts_parent_splitted)}\n
+            \t WebSites recuperados da base: {len(web_sites)}\n
+            \t Erros ao gerar descrição de imagens: {len(error_images_description)}\n
+        """) # noqa
+
+    except Exception as error:
+        print(f"❌ Erro: {error}")
+    finally:
+        safe_join(loading_split, done_split)
+        safe_join(loading_image_description, done_image_description)
+        safe_join(loading_embedding, done_embeding)
+
+
+if __name__ == "__main__":
+    run()

agente/pipelines/splitter/splitter.py

@@ -0,0 +1,61 @@
+from langchain_text_splitters import \
+    MarkdownHeaderTextSplitter, RecursiveCharacterTextSplitter
+import uuid
+
+
+def split(texts, headers_to_split_on):
+    """
+        Dividir os textos em chunks utilizando a lógica
+         "Parent Document Retriever",
+        retornando os chunks e os documentos pais.
+    """
+    texts_parent_splitted_list = []
+    texts_child_splitted_list = []
+
+    for text in texts:
+        texts_parent_splitted = markdown_split(
+            text["text_content"], headers_to_split_on)
+
+        for text_parent_splitted in texts_parent_splitted:
+            parent_id = uuid.uuid4()
+            text_parent_splitted_dict = {
+                "id": parent_id,
+                "parent_id": text["text_id"],
+                "content": text_parent_splitted,
+                "url": text["url"]
+            }
+
+            texts_parent_splitted_list.append(text_parent_splitted_dict)
+            texts_child_splitted = text_split(
+                text_parent_splitted.page_content)
+
+            for text_child_splitted in texts_child_splitted:
+                child_id = uuid.uuid4()
+                texts_child_splitted_dict = {
+                    "id": child_id,
+                    "parent_id": parent_id,
+                    "content": text_child_splitted
+                }
+
+                texts_child_splitted_list.append(texts_child_splitted_dict)
+
+    return [texts_parent_splitted_list, texts_child_splitted_list]
+
+
+def markdown_split(text, headers_to_split_on):
+    markdown_splitter = MarkdownHeaderTextSplitter(headers_to_split_on)
+
+    return markdown_splitter.split_text(text)
+
+
+def text_split(text):
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=500,
+        chunk_overlap=100,
+        length_function=len,
+        is_separator_regex=False,
+    )
+
+    texts = text_splitter.create_documents([text])
+
+    return texts

agente/pipelines/utils/cli.py

@@ -0,0 +1,42 @@
+from datetime import timedelta
+import itertools
+import sys
+import threading
+import time
+
+
+def loading_animation(done, text):
+    for c in itertools.cycle(['|', '/', '-', '\\']):
+        if done():
+            sys.stdout.write('\r')
+            sys.stdout.flush()
+            break
+        sys.stdout.write(f'\r{text} ' + c)
+        sys.stdout.flush()
+        time.sleep(0.1)
+
+
+def start_loading_animation(done, text):
+    t = threading.Thread(
+        target=loading_animation, args=(lambda: done[0], text))
+    t.start()
+    return t
+
+
+def stop_loading_animation(done, thread):
+    done[0] = True
+    thread.join()
+
+
+def print_execution_time(start_time):
+    end_time = time.time()
+    execution_time = end_time - start_time
+
+    execution_time_td = timedelta(seconds=execution_time)
+
+    hours, remainder = divmod(execution_time_td.total_seconds(), 3600)
+    minutes, remainder = divmod(remainder, 60)
+    seconds, milliseconds = divmod(remainder, 1)
+    milliseconds *= 1000
+
+    return f"{int(hours):02}:{int(minutes):02}:{int(seconds):02}:{int(milliseconds):03}" # noqa

agente/pipelines/utils/markdown.py

@@ -0,0 +1,37 @@
+import re
+from oac.oac import get_description_of_image
+
+
+def generate_image_description_with_empty_description(oa_client, websites):
+    # Expressão regular para encontrar o padrão ![](<link>)
+    pattern = r'!\[\]\(([^)]+)\)'
+
+    def replace_description_image(match):
+        link = match.group(1)
+        new_description = get_description_of_image(oa_client, link)
+        return f'![{new_description}]({link})'
+
+    texts_with_images_descriptions = []
+    error_images_upload = []
+    for website in websites:
+        try:
+            text_with_image_description = re.sub(
+                pattern,
+                replace_description_image,
+                website["text_content"]
+            )
+            texts_with_images_descriptions.append({
+                "id": website["text_id"],
+                "content": text_with_image_description,
+                "url": website["url"]
+            })
+        except Exception:
+            error_images_upload.append(
+                f"{website['text_id']}: {website['text_content']}")
+            texts_with_images_descriptions.append({
+                "id": website["text_id"],
+                "content": website["text_content"],
+                "url": website["url"]
+            })
+
+    return texts_with_images_descriptions, error_images_upload

agente/prompts/system.md

@@ -0,0 +1,22 @@
+You are an assistant designed to work as a FAQ for Brazilian Government Services. Be concise in your answers.
+
+Your natural language is Brazilian Portuguese.
+
+You will receive two labels in the user's question, one called 'Context' and the other 'Question'. Inside the 'Context', you will receive a JSON list in the following format, enclosed between three backticks.
+
+```
+[
+  {
+    content: here you will receive the content in markdown format of the context that you will use to answer the user's question,
+    url: here is the URL from which this content was taken, and you should reference it in your answer
+  }
+]
+```
+
+Since you will receive a list with these contexts, you will answer based on them what is inside the 'Question' label.
+
+The format of your response should be in markdown. You can include images in your response by interpreting them based on the description provided for each image and responding in markdown format.
+
+If you do not find the answer in the given context, simply respond to the user with "Não encontrei sua resposta."
+
+Do not use information that is not in the given context.

agente/utils/file.py

@@ -0,0 +1,3 @@
+def File(file_path):
+    with open(file_path, 'r', encoding='utf-8') as file:
+        return file.read()

edgedb/dbschema/initial.esdl

@@ -0,0 +1,36 @@
+module default {
+    type Image {
+         name: str;
+         path: str;
+         url: str;
+         hyperlink: str;
+         alt: str;
+    }
+
+    type Video {
+         name: str;
+         url: str;
+         hyperlink: str;
+         alt: str;
+    }
+
+    type Text {
+         content: str;
+    }
+
+    type Website {
+         url: str;
+         relative_path: str;
+         hyperrefs: array<str>;
+         multi images: Image;
+         multi videos: Video;
+         text: Text;
+         segmented_texts: array<str>;
+    }
+
+    type Pattern {
+          content: str;
+          parent_id: uuid;
+          url: str;
+    }
+}

edgedb/dbschema/migrations/00001-m14hvdp.edgeql

@@ -0,0 +1,29 @@
+CREATE MIGRATION m14hvdpwu2t2cxnf75otbhvkltgiwr5hqx7fh74t7rzysjw6fy34oa
+    ONTO initial
+{
+  CREATE TYPE default::Image {
+      CREATE PROPERTY alt: std::str;
+      CREATE PROPERTY hyperlink: std::str;
+      CREATE PROPERTY name: std::str;
+      CREATE PROPERTY path: std::str;
+      CREATE PROPERTY url: std::str;
+  };
+  CREATE TYPE default::Text {
+      CREATE PROPERTY content: std::str;
+  };
+  CREATE TYPE default::Video {
+      CREATE PROPERTY alt: std::str;
+      CREATE PROPERTY hyperlink: std::str;
+      CREATE PROPERTY name: std::str;
+      CREATE PROPERTY url: std::str;
+  };
+  CREATE TYPE default::Website {
+      CREATE MULTI LINK images: default::Image;
+      CREATE LINK text: default::Text;
+      CREATE MULTI LINK videos: default::Video;
+      CREATE PROPERTY hyperrefs: array<std::str>;
+      CREATE PROPERTY relative_path: std::str;
+      CREATE PROPERTY segmented_texts: array<std::str>;
+      CREATE PROPERTY url: std::str;
+  };
+};

edgedb/dbschema/migrations/00002-m1dsfjl.edgeql

@@ -0,0 +1,9 @@
+CREATE MIGRATION m1dsfjlp6m7m6zym3olrmjirgmulk4m4vk2er7x22ohmbskv2kgbxa
+    ONTO m14hvdpwu2t2cxnf75otbhvkltgiwr5hqx7fh74t7rzysjw6fy34oa
+{
+  CREATE TYPE default::Pattern {
+      CREATE PROPERTY content: std::str;
+      CREATE PROPERTY parent_id: std::uuid;
+      CREATE PROPERTY url: std::str;
+  };
+};

edgedb/depopulate_edgedb.py

@@ -0,0 +1,19 @@
+import edgedb
+from dotenv import load_dotenv
+
+load_dotenv()
+
+client = edgedb.create_client()
+
+resp = input('Are you sure you want to depopulate the database? (y/n) ')
+
+if resp.lower() == 'y':
+    client.query('''
+        DELETE Website;
+        DELETE Image;
+        DELETE Video;
+        DELETE Text;
+    ''')
+    print('Database depopulated.')
+else:
+    print('Operation canceled.')

edgedb/populate_edgedb.py

@@ -0,0 +1,77 @@
+import edgedb
+import json
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+client = edgedb.create_client()
+
+for root, dirs, files in os.walk("../Banco_de_Dados/Coleta/downloaded_files"):
+    for file in files:
+        if file.endswith(".json"):
+            with open(os.path.join(root, file), 'r') as f:
+                data = json.load(f)
+            data_path = './'+'/'.join(root.split('/')[5:])
+            # Insert Website
+            client.query('''
+                INSERT Website {
+                    url := <str>$url,
+                    relative_path := <str>$relative_path,
+                    hyperrefs := <array<str>>$gov_links,
+                    images := {},
+                    videos := {},
+                    text := (
+                        INSERT Text {
+                            content := <str>$content
+                        }
+                    )
+                };
+            ''', url=data['absolute_url'], \
+                relative_path=data_path, \
+                gov_links=data['gov_links'], \
+                content=data['text'])
+
+            
+            # Insert Images
+            for image in data.get('images', []):
+                client.query('''
+                    UPDATE Website
+                    FILTER .url = <str>$url
+                    SET {
+                        images += {
+                            (INSERT Image {
+                                name := <str>$name,
+                                path := <str>$path,
+                                url := <str>$image_url,
+                                hyperlink := <str>$hyperlink,
+                                alt := <str>$alt,
+                            })
+                        }
+                    };
+                ''', url=data['absolute_url'], \
+                    path=image['path'], \
+                    name=image['name'], \
+                    image_url=image['url'], \
+                    hyperlink=image['hyperlink'], \
+                    alt=image['alt'])
+
+            for video in data.get('videos', []):
+                client.query('''
+                    UPDATE Website
+                    FILTER .url = <str>$url
+                    SET {
+                        videos += {
+                            (INSERT Video {
+                                name := <str>$name,
+                                url := <str>$video_url,
+                                hyperlink := <str>$hyperlink,
+                                alt := <str>$alt,
+                            })
+                        }
+                    };
+                ''', url=data['absolute_url'], \
+                name=video['name'], \
+                video_url=video['url'], \
+                hyperlink=video['hyperlink'], \
+                alt=video['alt'])