<jupyter_start><jupyter_text>looking at neobi for adding data to leafly<jupyter_code>from os import getenv
import base64
from http import client
from urllib import request, parse, error

from os import path
from dotenv import load_dotenv
from bs4 import BeautifulSoup
# custom error to define that we can't co0nnect to the api
from app.utils.exceptions import APIError

class thirdpart_requester():
    def __init__(self):
        """sets the api key and fits it to the header
        """
        load_dotenv()
        # pass the api key to the api
        self.headers = {
            # Request headers
            'Ocp-Apim-Subscription-Key': getenv("API_KEY")}
        return
    def setEndPoint(self,APIEnpoint:str):
        """Set the endpoint for the api to use

        """
        self.ENDPOINT = APIEnpoint
    def setParams(self,params:dict)
        self.PARAMS = params
    def connect(self)
        pass
    def _encodeparams(self):
        params = parse.urlencode({
            # Request parameters
            #'page': '{integer}',
            #'count': '200',
            #'sort': '{string}',
            #'thc': '{string}',
            #'location': '{string}',
            #'htmlDescription': '{boolean}',
            #'active': '{string}',
        })
try:
    conn = client.HTTPSConnection('neobi.azure-api.net')
    conn.request("GET", "/stage/api/products?%s" % params, "{body}", headers)
    response = conn.getresponse()
    data = response.read()
    
    conn.close()
except Exception as e:
    print("[Errno {0}] {1}".format(e.errno, e.strerror))<jupyter_output>ova Vaporizer","ccc":"60556","link":"https://neobi.io/products?p=60556","qr":"https://api.qrserver.com/v1/create-qr-code/?size=150x150&data=https://neobi.io/products?p=60556","url":"https://cannabis.mynslc.com/en/products/accessories/vapourizers/1033372.aspx","image":"cannabis.mynslc.com/en/skins/cannabis/images/Unavailable.jpg","type":"Accessories","labTest":false,"thc":"","cbd":"","price":"$56.29","instockstatus":"In Stock","metaDescription":"","metaTitle":"","eqWeight":"","brand":"Premium 5","currency":"CAD","updatedAt":"2020-04-27T06:54:56","producer":{"name":"Cannabis.myNSLC","ccc":"248","link":"https://neobi.io/product?i=248-1033372","location":"","image":"https://neobi.io/images/coming-soon.jpg","phone":"","email":"","productCount":"500"}},{"name":"Pax3 Black Matte Complete","ccc":"46634","link":"https://neobi.io/products?p=46634","qr":"https://api.qrserver.com/v1/create-qr-code/?size=150x150&data=https://neobi.io/products?p=46634","url":"https://cannabis.mynslc.com/en/products/[...]<jupyter_text>Looking at the leafly dataset<jupyter_code>!pwd
!lsd --tree ../data/
%cd ../data
import pandas as pd
df = pd.read_csv('https://raw.githubusercontent.com/Build-Week-Med-Cabinet-6/DS/mark-dev/data/cannabis.csv')
df.head()
df.isnull().sum()
# drop na rows from dataset
df.dropna(inplace=True)
import numpy as np
df.replace('None',np.NaN,inplace=True)
for i in [1606,1607,1605,1698]:
    print(df.iloc[i].Description)
# make the flacors and effects into list's instead of strings
def split_strings(x):
    X = x.split(',')
    return X

df["Effects"] = df.Effects.apply(split_strings)
df ["Flavor"] = df.Flavor.apply(split_strings)
# convert to lowercase columns because ocd
df.columns = [x.lower() for x in df.columns]
df.columns = ['strain', 'species', 'rating', 'effects', 'flavor', 'description']
df.sample(15)
# seems good to save and make a base db with
# going to seperate the strain names out so that i can query api's with them and get more info
from models import db
from sqlalchemy import create_engine
engine = create_engine(getenv("DATABASE_URL"))
df.to_sql("strains",engine)
df.isnull().sum()
for i in df.flavor:
    for k in i:
        if k=='None':
            k=np.NaN

import models
help(models)
print("hello world")

from services.model_service import modelone
from os import path
PATH = path.join("..", "pickles", '')
PATH
PATH+"nn.pickle"
!pwd
m = modelone("pickles/nn.pickle","pickles/dtm_combined_tf.pickle")
ideal_strain = ['Creative,Energetic,Tingly,Euphoric,Relaxed,Earthy,Sweet,Citrus']
m.transform_predict(ideal_strain)
[0,1971,171] in m.getResults()[0]
from models import Strains

import numpy as np
str(np.array([1,2,3]).tolist()).replace('[','(').replace(']',')')
!python -m spacy download en_core_web_md<jupyter_output>Traceback (most recent call last):
  File "/home/joe/opt/conda/envs/ds-Flask/lib/python3.7/site-packages/urllib3/connection.py", line 157, in _new_conn
    (self._dns_host, self.port), self.timeout, **extra_kw
  File "/home/joe/opt/conda/envs/ds-Flask/lib/python3.7/site-packages/urllib3/util/connection.py", line 61, in create_connection
    for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
  File "/home/joe/opt/conda/envs/ds-Flask/lib/python3.7/socket.py", line 752, in getaddrinfo
    for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/joe/opt/conda/envs/ds-Flask/lib/python3.7/site-packages/urllib3/connectionpool.py", line 672, in urlopen
    chunked=chunked,
  File "/home/joe/opt/conda/envs/ds-Flask/lib/python3.7/site-packages/urllib3/connectionpool.py", line 376, in _make_r[...]
{'fraction_non_alphanumeric': 0.16722657708100036, 'fraction_numerical': 0.029115341545352745, 'mean_word_length': 4.290227048371174, 'pattern_counts': {'":': 31, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 11, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '113363', 'license': 'MIT', 'n_tokens_mistral': 1957, 'n_tokens_neox': 1827, 'n_words': 388, 'path': 'app/testing.ipynb', 'repo_name': 'Build-Week-Med-Cabinet-6/DS', 'size': '5358'}