rhoitjadhav commited on
Commit
2c80eb3
1 Parent(s): 37d81a4

update dockerfile

Browse files
Files changed (6) hide show
  1. Dockerfile +37 -0
  2. load_data.py +115 -0
  3. requirements.txt +2 -0
  4. start.sh +33 -0
  5. start_test.sh +12 -0
  6. users.yml +13 -0
Dockerfile ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.9-slim
2
+
3
+ # Exposing ports
4
+ EXPOSE 6900
5
+
6
+ # Environment variables
7
+ ENV ARGILLA_LOCAL_AUTH_USERS_DB_FILE=/packages/users.yml
8
+ ENV UVICORN_PORT=6900
9
+
10
+ # Copying argilla distribution files
11
+ COPY *.whl /packages/
12
+
13
+ # Copy users db file along with execution script
14
+ COPY start.sh /
15
+ COPY load_data.py /
16
+ COPY users.yml /packages/
17
+
18
+ # Install packages
19
+ RUN apt update
20
+ RUN apt -y install python3.9-dev gcc gnupg apache2-utils systemctl curl sudo vim
21
+
22
+ # Create new user for starting elasticsearch
23
+ RUN useradd -ms /bin/bash user -p "$(openssl passwd -1 ubuntu)"
24
+ RUN echo 'user ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
25
+
26
+ # Install argilla
27
+ RUN chmod +x /start.sh \
28
+ && for wheel in /packages/*.whl; do pip3 install "$wheel"[server]; done
29
+
30
+ # Install Elasticsearch
31
+ RUN curl -fsSL https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add -
32
+ RUN echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" | tee -a /etc/apt/sources.list.d/elastic-7.x.list
33
+ RUN apt update
34
+ RUN apt -y install elasticsearch
35
+
36
+ # Executing argilla along with elasticsearch
37
+ CMD /bin/bash /start.sh
load_data.py ADDED
@@ -0,0 +1,115 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import sys
3
+ import requests
4
+ import time
5
+ import pandas as pd
6
+ import argilla as rg
7
+ from datasets import load_dataset
8
+ from argilla.labeling.text_classification import Rule, add_rules
9
+
10
+
11
+ def load_datasets():
12
+ # This is the code that you want to execute when the endpoint is available
13
+ print("Argilla is available! Loading datasets")
14
+ api_key = sys.argv[-1]
15
+ rg.init(api_key=api_key, workspace="admin")
16
+
17
+ # load dataset from json
18
+ my_dataframe = pd.read_json(
19
+ "https://raw.githubusercontent.com/recognai/datasets/main/sst-sentimentclassification.json")
20
+
21
+ # convert pandas dataframe to DatasetForTextClassification
22
+ dataset_rg = rg.DatasetForTextClassification.from_pandas(my_dataframe)
23
+
24
+ # Define labeling schema to avoid UI user modification
25
+ settings = rg.TextClassificationSettings(label_schema=["POSITIVE", "NEGATIVE"])
26
+ rg.configure_dataset(name="sst-sentiment-explainability", settings=settings)
27
+
28
+ # log the dataset
29
+ rg.log(
30
+ dataset_rg,
31
+ name="sst-sentiment-explainability",
32
+ tags={
33
+ "description": "The sst2 sentiment dataset with predictions from a pretrained pipeline and explanations from Transformers Interpret."
34
+ }
35
+ )
36
+
37
+ dataset = load_dataset("argilla/news-summary", split="train").select(range(100))
38
+ dataset_rg = rg.read_datasets(dataset, task="Text2Text")
39
+
40
+ # log the dataset
41
+ rg.log(
42
+ dataset_rg,
43
+ name="news-text-summarization",
44
+ tags={
45
+ "description": "A text summarization dataset with news pieces and their predicted summaries."
46
+ }
47
+ )
48
+
49
+ # Read dataset from Hub
50
+ dataset_rg = rg.read_datasets(
51
+ load_dataset("argilla/agnews_weak_labeling", split="train"),
52
+ task="TextClassification",
53
+ )
54
+
55
+ # Define labeling schema to avoid UI user modification
56
+ settings = rg.TextClassificationSettings(label_schema=["World", "Sports", "Sci/Tech", "Business"])
57
+ rg.configure_dataset(name="news-programmatic-labeling", settings=settings)
58
+
59
+ # log the dataset
60
+ rg.log(
61
+ dataset_rg,
62
+ name="news-programmatic-labeling",
63
+ tags={
64
+ "description": "The AG News with programmatic labeling rules (see weak labeling mode in the UI)."
65
+ }
66
+ )
67
+
68
+ # define queries and patterns for each category (using ES DSL)
69
+ queries = [
70
+ (["money", "financ*", "dollar*"], "Business"),
71
+ (["war", "gov*", "minister*", "conflict"], "World"),
72
+ (["*ball", "sport*", "game", "play*"], "Sports"),
73
+ (["sci*", "techno*", "computer*", "software", "web"], "Sci/Tech"),
74
+ ]
75
+
76
+ # define rules
77
+ rules = [Rule(query=term, label=label) for terms, label in queries for term in terms]
78
+
79
+ # add rules to the dataset
80
+ add_rules(dataset="news-programmatic-labeling", rules=rules)
81
+
82
+ # load dataset from the hub
83
+ dataset = load_dataset("argilla/gutenberg_spacy-ner", split="train")
84
+
85
+ # read in dataset, assuming its a dataset for token classification
86
+ dataset_rg = rg.read_datasets(dataset, task="TokenClassification")
87
+
88
+ # Define labeling schema to avoid UI user modification
89
+ labels = ["CARDINAL", "DATE", "EVENT", "FAC", "GPE", "LANGUAGE", "LAW", "LOC", "MONEY", "NORP", "ORDINAL", "ORG",
90
+ "PERCENT", "PERSON", "PRODUCT", "QUANTITY", "TIME", "WORK_OF_ART"]
91
+ settings = rg.TokenClassificationSettings(label_schema=labels)
92
+ rg.configure_dataset(name="gutenberg_spacy-ner-monitoring", settings=settings)
93
+
94
+ # log the dataset
95
+ rg.log(
96
+ dataset_rg,
97
+ "gutenberg_spacy-ner-monitoring",
98
+ tags={
99
+ "description": "A dataset containing text from books with predictions from two spaCy NER pre-trained models."
100
+ }
101
+ )
102
+
103
+
104
+ while True:
105
+ try:
106
+ response = requests.get("http://0.0.0.0:6900/")
107
+ if response.status_code == 200:
108
+ load_datasets()
109
+ break
110
+ else:
111
+ time.sleep(10)
112
+ except Exception as e:
113
+ print(e)
114
+ time.sleep(10)
115
+ pass
requirements.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ argilla[server]
2
+ fastapi
start.sh ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+
3
+ set -e
4
+
5
+ # Changing user
6
+ sudo -S su user
7
+
8
+ # Disable security in elasticsearch configuration
9
+ sudo sed -i "s/xpack.security.enabled: true/xpack.security.enabled: false/g" /etc/elasticsearch/elasticsearch.yml
10
+ sudo sed -i "s/cluster.initial_master_nodes/#cluster.initial_master_nodes/g" /etc/elasticsearch/elasticsearch.yml
11
+ echo "cluster.routing.allocation.disk.threshold_enabled: false" | sudo tee -a /etc/elasticsearch/elasticsearch.yml
12
+
13
+ # Create elasticsearch directory and change ownerships
14
+ sudo mkdir -p /var/run/elasticsearch
15
+ sudo chown -R elasticsearch:elasticsearch /var/run/elasticsearch
16
+ sudo chown -R user:user /load_data.py
17
+
18
+ # Start elasticsearch
19
+ sudo systemctl daemon-reload
20
+ sudo systemctl enable elasticsearch.service
21
+ sudo systemctl start elasticsearch.service
22
+
23
+ # Update API_KEY and PASSWORD from users.yml
24
+ sudo sed -i 's,API_KEY,'"$API_KEY"',g' /packages/users.yml
25
+ sudo sed -i 's,ADMIN_PASSWORD,'"$ADMIN_PASSWORD"',g' /packages/users.yml
26
+ sudo sed -i 's,ARGILLA_PASSWORD,'"$ARGILLA_PASSWORD"',g' /packages/users.yml
27
+
28
+ # Load data
29
+ pip3 install datasets
30
+ python3.9 /load_data.py "$API_KEY" &
31
+
32
+ # Start argilla
33
+ uvicorn argilla:app --host "0.0.0.0"
start_test.sh ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ set -e
2
+
3
+ # Changing user
4
+ sudo -S su user
5
+
6
+ sudo systemctl start elasticsearch
7
+
8
+ # Load data
9
+ python3.9 /load_data.py &
10
+
11
+ # Start argilla
12
+ uvicorn argilla:app --host "0.0.0.0"
users.yml ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ - username: "admin"
2
+ api_key: API_KEY
3
+ full_name: Hugging Face
4
5
+ hashed_password: ADMIN_PASSWORD
6
+ workspaces: [ ]
7
+
8
+ - username: "argilla"
9
+ api_key: API_KEY
10
+ full_name: Hugging Face
11
12
+ hashed_password: ARGILLA_PASSWORD
13
+ workspaces: [ "admin" ]