update dockerfile

Dockerfile

@@ -0,0 +1,37 @@
+FROM python:3.9-slim
+
+# Exposing ports
+EXPOSE 6900
+
+# Environment variables
+ENV ARGILLA_LOCAL_AUTH_USERS_DB_FILE=/packages/users.yml
+ENV UVICORN_PORT=6900
+
+# Copying argilla distribution files
+COPY *.whl /packages/
+
+# Copy users db file along with execution script
+COPY start.sh /
+COPY load_data.py /
+COPY users.yml /packages/
+
+# Install packages
+RUN apt update
+RUN apt -y install python3.9-dev gcc gnupg apache2-utils systemctl curl sudo vim
+
+# Create new user for starting elasticsearch
+RUN useradd -ms /bin/bash user -p "$(openssl passwd -1 ubuntu)"
+RUN echo 'user ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
+
+# Install argilla
+RUN chmod +x /start.sh \
+    && for wheel in /packages/*.whl; do pip3 install "$wheel"[server]; done
+
+# Install Elasticsearch
+RUN curl -fsSL https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add -
+RUN echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | tee -a /etc/apt/sources.list.d/elastic-7.x.list
+RUN apt update
+RUN apt -y install elasticsearch
+
+# Executing argilla along with elasticsearch
+CMD /bin/bash /start.sh

load_data.py

@@ -0,0 +1,115 @@
+import os
+import sys
+import requests
+import time
+import pandas as pd
+import argilla as rg
+from datasets import load_dataset
+from argilla.labeling.text_classification import Rule, add_rules
+
+
+def load_datasets():
+    # This is the code that you want to execute when the endpoint is available
+    print("Argilla is available! Loading datasets")
+    api_key = sys.argv[-1]
+    rg.init(api_key=api_key, workspace="admin")
+
+    # load dataset from json
+    my_dataframe = pd.read_json(
+        "https://raw.githubusercontent.com/recognai/datasets/main/sst-sentimentclassification.json")
+
+    # convert pandas dataframe to DatasetForTextClassification
+    dataset_rg = rg.DatasetForTextClassification.from_pandas(my_dataframe)
+
+    # Define labeling schema to avoid UI user modification
+    settings = rg.TextClassificationSettings(label_schema=["POSITIVE", "NEGATIVE"])
+    rg.configure_dataset(name="sst-sentiment-explainability", settings=settings)
+
+    # log the dataset
+    rg.log(
+        dataset_rg,
+        name="sst-sentiment-explainability",
+        tags={
+            "description": "The sst2 sentiment dataset with predictions from a pretrained pipeline and explanations from Transformers Interpret."
+        }
+    )
+
+    dataset = load_dataset("argilla/news-summary", split="train").select(range(100))
+    dataset_rg = rg.read_datasets(dataset, task="Text2Text")
+
+    # log the dataset
+    rg.log(
+        dataset_rg,
+        name="news-text-summarization",
+        tags={
+            "description": "A text summarization dataset with news pieces and their predicted summaries."
+        }
+    )
+
+    # Read dataset from Hub
+    dataset_rg = rg.read_datasets(
+        load_dataset("argilla/agnews_weak_labeling", split="train"),
+        task="TextClassification",
+    )
+
+    # Define labeling schema to avoid UI user modification
+    settings = rg.TextClassificationSettings(label_schema=["World", "Sports", "Sci/Tech", "Business"])
+    rg.configure_dataset(name="news-programmatic-labeling", settings=settings)
+
+    # log the dataset
+    rg.log(
+        dataset_rg,
+        name="news-programmatic-labeling",
+        tags={
+            "description": "The AG News with programmatic labeling rules (see weak labeling mode in the UI)."
+        }
+    )
+
+    # define queries and patterns for each category (using ES DSL)
+    queries = [
+        (["money", "financ*", "dollar*"], "Business"),
+        (["war", "gov*", "minister*", "conflict"], "World"),
+        (["*ball", "sport*", "game", "play*"], "Sports"),
+        (["sci*", "techno*", "computer*", "software", "web"], "Sci/Tech"),
+    ]
+
+    # define rules
+    rules = [Rule(query=term, label=label) for terms, label in queries for term in terms]
+
+    # add rules to the dataset
+    add_rules(dataset="news-programmatic-labeling", rules=rules)
+
+    # load dataset from the hub
+    dataset = load_dataset("argilla/gutenberg_spacy-ner", split="train")
+
+    # read in dataset, assuming its a dataset for token classification
+    dataset_rg = rg.read_datasets(dataset, task="TokenClassification")
+
+    # Define labeling schema to avoid UI user modification
+    labels = ["CARDINAL", "DATE", "EVENT", "FAC", "GPE", "LANGUAGE", "LAW", "LOC", "MONEY", "NORP", "ORDINAL", "ORG",
+              "PERCENT", "PERSON", "PRODUCT", "QUANTITY", "TIME", "WORK_OF_ART"]
+    settings = rg.TokenClassificationSettings(label_schema=labels)
+    rg.configure_dataset(name="gutenberg_spacy-ner-monitoring", settings=settings)
+
+    # log the dataset
+    rg.log(
+        dataset_rg,
+        "gutenberg_spacy-ner-monitoring",
+        tags={
+            "description": "A dataset containing text from books with predictions from two spaCy NER pre-trained models."
+        }
+    )
+
+
+while True:
+    try:
+        response = requests.get("http://0.0.0.0:6900/")
+        if response.status_code == 200:
+            load_datasets()
+            break
+        else:
+            time.sleep(10)
+    except Exception as e:
+        print(e)
+        time.sleep(10)
+        pass

requirements.txt

@@ -0,0 +1,2 @@
+argilla[server]
+fastapi

start.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Changing user
+sudo -S su user
+
+# Disable security in elasticsearch configuration
+sudo sed -i "s/xpack.security.enabled: true/xpack.security.enabled: false/g" /etc/elasticsearch/elasticsearch.yml
+sudo sed -i "s/cluster.initial_master_nodes/#cluster.initial_master_nodes/g" /etc/elasticsearch/elasticsearch.yml
+echo "cluster.routing.allocation.disk.threshold_enabled: false" | sudo tee -a /etc/elasticsearch/elasticsearch.yml
+
+# Create elasticsearch directory and change ownerships
+sudo mkdir -p /var/run/elasticsearch
+sudo chown -R elasticsearch:elasticsearch /var/run/elasticsearch
+sudo chown -R user:user /load_data.py
+
+# Start elasticsearch
+sudo systemctl daemon-reload
+sudo systemctl enable elasticsearch.service
+sudo systemctl start elasticsearch.service
+
+# Update API_KEY and PASSWORD from users.yml
+sudo sed -i 's,API_KEY,'"$API_KEY"',g' /packages/users.yml
+sudo sed -i 's,ADMIN_PASSWORD,'"$ADMIN_PASSWORD"',g' /packages/users.yml
+sudo sed -i 's,ARGILLA_PASSWORD,'"$ARGILLA_PASSWORD"',g' /packages/users.yml
+
+# Load data
+pip3 install datasets
+python3.9 /load_data.py "$API_KEY" &
+
+# Start argilla
+uvicorn argilla:app --host "0.0.0.0"

start_test.sh

@@ -0,0 +1,12 @@
+set -e
+
+# Changing user
+sudo -S su user
+
+sudo systemctl start elasticsearch
+
+# Load data
+python3.9 /load_data.py &
+
+# Start argilla
+uvicorn argilla:app --host "0.0.0.0"

users.yml

@@ -0,0 +1,13 @@
+- username: "admin"
+  api_key: API_KEY
+  full_name: Hugging Face
+  email: [email protected]
+  hashed_password: ADMIN_PASSWORD
+  workspaces: [ ]
+
+- username: "argilla"
+  api_key: API_KEY
+  full_name: Hugging Face
+  email: [email protected]
+  hashed_password: ARGILLA_PASSWORD
+  workspaces: [ "admin" ]