<jupyter_start><jupyter_text>[x] Make NER microservice.[x] Make server_crypto.py and server_finance.py.[x] Make SA microservice.[x] Make server_crypto.py and server_finance.py.[x] Get Free Tier MongoDB Atlas.[x] NER from entities collection.[ ] Make script to download daily crypto chit-chat data.References:https://arxiv.org/pdf/1905.03423.pdf<jupyter_code>%reload_ext autoreload
%autoreload 2
import os
import sys
import grpc
import ner_pb2
import ner_pb2_grpc
if "../../ner" not in sys.path:
    sys.path.append("../../ner")
%load_ext autoreload
%autoreload 2
from server import NER
ner = NER()
ner_request =     ner_pb2.NerRequest(
        text="I love Bitcoin stocks and ETH.",
        metadata="{\"request\":{\"domains\":[\"crypto\"]}}"
    )
ner.ExtractEntities(request=ner_request,context = None)


ner_channel = grpc.insecure_channel(
    os.environ.get("NER_SERVER_ADDRESS")
)
ner_stub = ner_pb2_grpc.NERStub(ner_channel)

ner_response = ner_stub.ExtractEntities(
    ner_pb2.NerRequest(
        text="I love Stacks stocks.",
        metadata="kek"
    )
)

ner_response
import requests
import pandas as pd
from bs4 import BeautifulSoup
r = requests.get("https://coinmarketcap.com/all/views/all/")
soup = BeautifulSoup(r.content,'html.parser')
data = []

for row in soup.find_all('tr'):
    row_data = []
    for item in row.find_all('td')[:-1]:
        row_data.append(item.text)
    data.append(row_data)
data_cryptos = pd.DataFrame(data, columns = ["Rank","Name","Symbol","Market Cap","Price","Circulating Supply", "Volumne(24h)","%1h","%24h","%7d"])
data_cryptos = data_cryptos[["Name","Symbol"]].dropna()
crypto_entities = []
for _,row in data_cryptos.iterrows():
    crypto_entities.append({
        "entity":row["Symbol"],
        "domain":"crypto",
        "synonims":[row["Symbol"],row["Name"][len(row["Symbol"]):]]
    })
crypto_entities
import os
from pymongo import MongoClient
client = MongoClient(os.getenv("MONGO_CONNECTION_STRING"))
db = client[os.getenv("MONGO_DB_NAME")]
#db.entities.insert_many(crypto_entities)
sp500 = pd.read_csv("../../ner/s&p.csv")
sp_entities = []
for _, row in sp500.iterrows():
    sp_entities.append({
        "entity":row["Symbol"],
        "domain":"finance",
        "sector":row["Sector"],
        "synonims":[row["Symbol"],row["Name"]]
    })
db.entities.insert_many(sp_entities)
import os
import sys
if ".../../ner" not in sys.path:
    sys.path.append("../../ner")

from server_crypto import NER
ner_stub = NER()
ner_response = ner_stub.ExtractEntities(
    ner_pb2.NerRequest(
        text="I love btc stocks.",
        metadata="kek"
    ),
    context=None
)

ner_response
r = requests.get("https://www.gkgigs.com/list-of-top-cryptocurrencies/#List_Of_Top_1000_Cryptocurrencies_In_The_World")
soup = BeautifulSoup(r.content,'html.parser')

data = []
for row in soup.find_all('tr')[22:]:
    row_data = []
    for item in row.find_all('td')[1:]:
        row_data.append(item.text)
    data.append(row_data)
data = data[1:-1]
docs = []
for d in data:
    if d:
        docs.append({
            "entity":d[1],
            "synonims":d,
            "domain":"crypto"
        })
db.entities.delete_many({"domain":"crypto"})<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16370992841581078, 'fraction_numerical': 0.012760659819483348, 'mean_word_length': 3.975232198142415, 'pattern_counts': {'":': 13, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '340364', 'license': 'Apache-2.0', 'n_tokens_mistral': 1157, 'n_tokens_neox': 1130, 'n_words': 211, 'path': 'notebooks/grpc-services/ner.ipynb', 'repo_name': 'ilyocoris/dragonfly', 'size': '3213'}