<jupyter_start><jupyter_text>Directory Walkthrough ---><jupyter_code>from datetime import date
import pickle
import pathlib

try: # try to find directory of own path on deployment from .py, if not (on jupyter notebook) run directly through hardcoded path
  MY_DIR = pathlib.Path(__file__).parent
except NameError:
  MY_DIR = pathlib.Path(r"C:\Users\water\Desktop\WF\WholeFoods-Datascraping-Project-Deployment\Deployment")
  
#filename = "WF_Sales_" + str(date.today().strftime("%b_%d_%Y")) + '_' + str('_'.join(location.split()).replace(',','')) + ".pkl"
path = MY_DIR / 'scraped products dump' / filename
locpath = MY_DIR / 'scraped products dump' / 'location' / filename


#----------Path for most recent scraped dataset (.pkl) on local path by date time-----------------------
path = sorted([f for f in pathlib.Path('scraped products dump').glob("*.pkl")], key=lambda f: f.stat().st_mtime, reverse=True) # sorted path of scraped dataframes
locpath = sorted([f for f in pathlib.Path('scraped products dump/location').glob("*.pkl")], key=lambda f: f.stat().st_mtime, reverse=True) # sorted path of of scraped dataframes' locations
#----------Path for most recent scraped dataset (.pkl) on deployment path by date time------------------
path_deployment = sorted([f for f in pathlib.Path('Deployment/scraped products dump').glob("*.pkl")], key=lambda f: f.stat().st_ctime, reverse=True) # path adjusted for streamlit cloud deployment
locpath_deployment = sorted([f for f in pathlib.Path('Deployment/scraped products dump/location').glob("*.pkl")], key=lambda f: f.stat().st_ctime, reverse=True) # path adjusted for streamlit cloud deployment
'Deployment/'+str(path[0])
pathlib.Path('chromedriver.exe')
cwd = os.getcwd()
scraper_dir = cwd + '\wholefoods_scraper.py'
scraper_dir_deployment = cwd + '/Deployment/wholefoods_scraper.py'
os.path.isfile(scraper_dir_deployment)
os.path.isfile(scraper_dir)
!pip install webdriver_manager
if scraper_dir:
    print("True")
selections = []                                 
for x in locpath:                                             # iterate through local path of scraped data and append each item path to a list
    selections.append(str(x)[31:])                                         
queryselection = tuple(selections)[0]
a = 'Deployment/scraped products dump/location/WF_Sales_Feb_26_2022_Wayne_PA_19087.pkl'
len('Deployment/scraped products dump/location/')
queryselection
with open(str(pathlib.Path('scraped products dump\\location\\'+queryselection)), 'rb') as handle:             
    df = pickle.load(handle)
'Deployment/scraped products dump/location'+a[42:]
'scraped products dump\\location\\' + queryselection
if len(locpath_deployment) == 0:                              # if a selection from the select box is selected
  try:                                                        # try opening dataframe and location using the selected path
    with open(str(pathlib.Path(queryselection)).replace('\\location',''), 'rb') as handle:             
      df = pickle.load(handle)
    with open(str(queryselection), 'rb') as handle2:         
      location = pickle.load(handle2)
  except Exception as e:
    print(e)
else:
  try:
    with open(str(pathlib.Path(queryselection)).replace('/location',''), 'rb') as handle:             
      df = pickle.load(handle)
    with open(str(queryselection), 'rb') as handle2:         
      location = pickle.load(handle2)
  except Exception as e:
    print(e)
  # Get the current working directory (cwd)
files = os.listdir(cwd)
cwd = os.getcwd()
cwd + '\wholefoods_scraper.py'
driver_dir = cwd + "\chromedriver.exe"
driver_dir
files
cwd = os.getcwd()
driver_dir = cwd + "\chromedriver.exe"
df.hist(figsize=(20,15))
  try:                                         #
    MY_DIR = pathlib.Path(__file__).parent     #
  except NameError:                            #
    MY_DIR = pathlib.Path(os.getcwd()) # if not running as .py the directory is hardcoded locally
  #--------------------------------------------# 
  path = MY_DIR / 'scraped products dump' 
  locpath = MY_DIR / 'scraped products dump' / 'location' 
print(str(path))
print(sorted([f for f in pathlib.Path('scraped products dump\location').glob("*.pkl")], key=lambda f: f.stat().st_mtime, reverse=True)
import pathlib
locpath = sorted([f for f in pathlib.Path('scraped products dump/location').glob("*.pkl")], key=lambda f: f.stat().st_mtime, reverse=True)
len(locpath)
import pickle
df[df['regular'].str.contains('a|e|i|o|u', regex=True)]
browser.find_elements_by_class_name("wfm-search-bar--list_item")#[0].text#.split()
location = ' '.join(browser.find_elements_by_class_name("wfm-search-bar--list_item")[0].text.split()[-4:])
str(pathlib.Path(selections[0])).replace('/location','')
locpath
####################################################################  
path = sorted([f for f in pathlib.Path('scraped products dump').glob("*.pkl")], key=lambda f: f.stat().st_mtime, reverse=True)
locpath = sorted([f for f in pathlib.Path('scraped products dump/location').glob("*.pkl")], key=lambda f: f.stat().st_mtime, reverse=True)
####################################################################
path_deployment = sorted([f for f in pathlib.Path('Deployment/scraped products dump').glob("*.pkl")], key=lambda f: f.stat().st_ctime, reverse=True)
locpath_deployment = sorted([f for f in pathlib.Path('Deployment/scraped products dump/location').glob("*.pkl")], key=lambda f: f.stat().st_ctime, reverse=True)
####################################################################
# Load dataframe
try:                                                              # Tries to open through streamlit (using 'Deployment/' string addition)
  with open(str(path_deployment[0]), 'rb') as handle:             # loads .pkl back into variable
    df = pickle.load(handle)
  with open(str(locpath_deployment[0]), 'rb') as handle2:         # loads location data of dataframe
    location = pickle.load(handle2)
except:                                                           # Opens using local path if not using streamlit deployment
  with open(str(path[0]), 'rb') as handle:                        
    df = pickle.load(handle)
  with open(str(locpath[0]), 'rb') as handle2:                    
    location = pickle.load(handle2)
####################################################################
df[df['product'].str.contains('Cottage Cheese')]
list(df.columns)
pd.to_numeric(df["sale"])  
ct
df
pd.to_numeric(df["sale"]).round(2)

t1 = time()
# do something
t2 = time()
elapsed = round(t2 - t1, 2)
print('Elapsed time is %f seconds.' % elapsed)
def sec_to_hours(seconds):
    b=str((seconds%3600)//60)
    c=str((seconds%3600)%60)
    d=["{} mins {} seconds".format(b, c)]
    return d
sec_to_hours(round(3539.99, 2))
round(80.3423-40.349923,2)
t1 = 80.34293
t2 = 49.39923
elapsed = t2 - t1
import numpy as np
a = '\WF_Sales_Feb_26_2022_Wayne_PA_19087'
a[1:]
print('Elapsed time is %f seconds.' % elapsed)
def sec_to_min(raw_seconds):
    minutes = int(raw_seconds / 60) # whole number of times 60 seconds goes into the total
    seconds = raw_seconds % 60 # if you subtract 60 from total unless there is less than 60 remaining, the remainder is what is left
    d = minutes + seconds
    print(str(minutes) + " min" + str(seconds) + " sec Elapsed")
sec_to_min(elapsed)
!pip install pipdeptree
!pipdeptree --packages PyVirtualDisplay
rules = pd.read_csv(pathlib.Path('Deployment/rules.csv')).drop(columns='Unnamed: 0')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17990405117270789, 'fraction_numerical': 0.01585820895522388, 'mean_word_length': 3.4566508313539193, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '423576', 'license': 'MIT', 'n_tokens_mistral': 2413, 'n_tokens_neox': 2301, 'n_words': 587, 'path': 'Jupyter Notebooks/debug.ipynb', 'repo_name': 'YoussefSultan/WholeFoods-Datascraping-Project-Deployment', 'size': '7504'}