<jupyter_start><jupyter_text>QueryCodes used to query databaseThis includes branchQuery validateInputQuery  import files<jupyter_code>#export
import pandas as pd
from datetime import datetime
from nicHelper.wrappers import add_class_method, add_method, add_static_method
from nicHelper.dictUtil import printDict
from dataclasses_json import dataclass_json, Undefined, CatchAll
from dataclasses import dataclass
import logging
from typing import List
from s3bz.s3bz import S3
import pickle, json , boto3, zlib, os
#export
try:
  INVENTORY_BUCKET_NAME = os.environ['INVENTORY_BUCKET_NAME']
except Exception as e:
  print(f'missing environment variable {e} in query NB')
  INVENTORY_BUCKET_NAME = None
#export
class Querier:
  
  @staticmethod
  def validateInputQuery(keys: list, input_:dict):
    '''
      check if input query contains the valid key
      data should have the following structure
      key is a list of keys to check

      ib_prcode: String?
      ib_brcode: String?

      option, one of or both of the ib_procde must be present
    '''
    for key in keys:
      if key not in input_.keys():
        raise ValueError(f"key {key} is missing from the input")
      if not type(input_.get(key))==int:
        raise ValueError(f'key is not convertable to in {input_.get(key)}')
    return True
from villaProductDatabase.database import ProductDatabase
class Tester(Querier, ProductDatabase):
  class Meta:
    table_name = os.environ['DATABASE_TABLE_NAME']
    region = os.environ['REGION']
    billing_mode='PAY_PER_REQUEST'
    dax_read_endpoints = [DAX_ENDPOINT] if DAX_ENDPOINT else None
    dax_write_endpoints = [DAX_ENDPOINT] if DAX_ENDPOINT else None
  pass<jupyter_output>dax endpoint missing 'DAX_ENDPOINT'
dax endpoint missing 'DAX_ENDPOINT'
None
dax endpoint missing 'DAX_ENDPOINT'
dax endpoint missing 'DAX_ENDPOINT'
None<jupyter_text>All product Query (get signedUrl)<jupyter_code>#export
@add_class_method(Querier)
def allQuery(cls, key = 'allData', bucket = os.environ.get('INVENTORY_BUCKET_NAME'), **kwargs):
  print(bucket)
  result = S3.presign(key, bucket = bucket, checkExist=False,  **kwargs)
  return result
import boto3
%time result = S3.presign(key='allData', bucket = os.environ.get('INVENTORY_BUCKET_NAME'))
%time boto3.client('s3').generate_presigned_url( 'get_object', Params={'Bucket': os.environ.get('INVENTORY_BUCKET_NAME'), 'Key': 'allData'}, ExpiresIn=86400)
%%time
result = Tester.allQuery()
result<jupyter_output>product-bucket-dev-manual
CPU times: user 5.7 ms, sys: 252 µs, total: 5.95 ms
Wall time: 5.63 ms<jupyter_text>list product query<jupyter_code>#export
@dataclass_json
@dataclass
class ProductsFromList:
  cprcodes: List[str]
  
#export
import sys
@add_class_method(Querier)
def productsFromList(cls,cprcodes:List[str])->pd.DataFrame:
  db = cls.loadFromCache()
  print(f'db shape is {db.shape}')
  try:
    products = db[db['cprcode'].isin(cprcodes)]
  except:
    print(db.head())
    raise Exception(f'{db.head()} is not a valid db')
  return products
%%time
result = Tester.productsFromList(['0217153','203915','0000009'])
assert result.shape[0] == 3, f'shape is not as intended {result}'<jupyter_output>loading hashkey allData-hash
loaded hash is 3b7e3d595a478355b8f7a9cb0219e561d63b795d
data is still in sync, using local cache
db shape is (43803, 50)
CPU times: user 241 ms, sys: 121 ms, total: 362 ms
Wall time: 437 ms<jupyter_text>SingleProductQuery<jupyter_code>#export
@add_class_method(Querier)
def singleProductQuery(cls, input_)->pd.Series:
#   if not cls.validateInputQuery(['iprcode'] , input_): return f"error input {input_}"
  iprcode = int(input_['iprcode'])
  if (result:=next(cls.query(iprcode),None)): return result.toSeries()
  else: raise Exception('product not found')
try:
  Tester.singleProductQuery({'iprcode':1234})
except Exception as e:
  print(e)
%%time
result = Tester.singleProductQuery({'iprcode':9})
assert type(result) == pd.Series, 'return type is not series'
assert result.shape[0] > 10, 'result is not the correct shape'
Tester.singleProductQuery({'iprcode':0}).to_dict()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10650743358518157, 'fraction_numerical': 0.024859858640019498, 'mean_word_length': 5.662337662337662, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '645673', 'license': 'Apache-2.0', 'n_tokens_mistral': 1461, 'n_tokens_neox': 1403, 'n_words': 376, 'path': 'query.ipynb', 'repo_name': 'thanakijwanavit/villa-product-database', 'size': '4103'}