<jupyter_start><jupyter_text>Update metadata script<jupyter_code># For pulling JSON from data.go.th
import requests
import json
from pandas.io.json import json_normalize
API_KEY="<KEY>"

url_meta = "https://opend.data.go.th/opend-search/vir_3253_1584588173/query?dsname=vir_3253_1584588173&path=vir_3253_1584588173&loadAll=1&type=json&limit=100&offset=0&api-key="+API_KEY
url_algo="https://opend.data.go.th/opend-search/vir_3250_1584586227/query?dsname=vir_3250_1584586227&path=vir_3250_1584586227&loadAll=1&type=json&limit=100&offset=0&api-key="+API_KEY

payload = {}
headers= {}

def obtain_json(url_in):
    response = requests.request("GET", url_in, headers=headers, data = payload)
    json_text=json.loads(response.text)
    print('Complete obtaining data from data_gov')
    return json_text['data']
json_data_meta=obtain_json(url_meta)
json_data_algo=obtain_json(url_algo)
mongo_url="mongodb://localhost:27017/"
import pymongo
myclient = pymongo.MongoClient(mongo_url)
mydb = myclient["Outbreak_casedef"]

#The function insert data and replace collection

def update_collection(json_data,col_name='metadata'):

    #Verify exisiting collection, and drop if exists
    collist=mydb.list_collection_names()
    if col_name in collist:
        mydb[col_name].drop()

    mycol = mydb[col_name]
    x=mycol.insert_many(json_data)

    print(x)
    print('Complete updating '+col_name)
    return

update_collection(json_data_meta,'metadata')
update_collection(json_data_algo,'COVID19_action_test')<jupyter_output><pymongo.results.InsertManyResult object at 0x000001E666865708>
Complete updating metadata
<pymongo.results.InsertManyResult object at 0x000001E666ACF2C8>
Complete updating COVID19_action_test<jupyter_text>Update metadata code<jupyter_code>METADATA=json_normalize(mydb['metadata'].find({}))
METADATA.columns=map(str.lower,METADATA.columns)
METADATA.columns=METADATA.columns.str.strip()

METADATA['data_type_mapped']=METADATA['data_type'].map(VAR_TYPE_MAP).fillna(2)
METADATA['mandatory_field_mapped']=METADATA['mandatory_field'].fillna(0)

METADATA[["data_type_mapped", "mandatory_field_mapped"]] = METADATA[["data_type_mapped", "mandatory_field_mapped"]].apply(pd.to_numeric)
list(METADATA[METADATA['mandatory_field_mapped']>0]['attribute'])<jupyter_output><empty_output><jupyter_text>Update data type<jupyter_code>ALLOWED_INPUT=list(METADATA[METADATA['mandatory_field_mapped']>0]['attribute'])

def db_type_update(db2):

    for i in ALLOWED_INPUT:
        signx="$"+i
        db2.update_many({i :{'$type':2}},[{'$set': {i: {'$convert': {'input': signx, 'to': 1}}}}])
    return
db_type_update(mydb['COVID19_action_test'])
.mydb['COVID19_action_test'].find({})
from flask import Flask,render_template,request,jsonify
from flask_pymongo import PyMongo

app=Flask(__name__)


#For local database
app.config["MONGO_URI"]="mongodb://localhost:27017/Outbreak_casedef"<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.132944005496393, 'fraction_numerical': 0.05084163517691515, 'mean_word_length': 8.706666666666667, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '340571', 'license': 'MIT', 'n_tokens_mistral': 1182, 'n_tokens_neox': 1074, 'n_words': 136, 'path': 'static/Testing_worksheet_Outbreak_API.ipynb', 'repo_name': 'kkurno/outbreak_api', 'size': '2938'}