<jupyter_start><jupyter_text>New start......<jupyter_code>%cd /content/PyHelpers
!ls -a
!git add .
!git commit -m 'commit 1 from colabs'
# !cat '/content/PyHelpers/Libs/OptimalPrime.ipynb'

# !git clone https://github.com/bxck75/PyHelpers.git
import os
import subprocess
from IPython.display import clear_output
!python /content/PyHelpers/__main__.py 
# clear_output()
# help('__main__')
%%writefile /content/PyHelpers/__main__.py
import os
import subprocess
from Libs import BigHelp
global _ROOT_FOLDER_
global _LIB_
_ROOT_FOLDER_='/content/PyHelpers/'
_LIB_=_ROOT_FOLDER_+'Libs/'
import subprocess


import importlib.util as impylib
# print(len(H.Me(['globx',_LIB_,'*.py'])))

import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--dev', defaults=False, help='foo help')
args = parser.parse_args()

class Stimpy_Imp:
    
    def __init__(self,dev=False):
        self.dev = dev
        self.cmd_com = self.system_cmd()
        # main helper loading
        self.RootHelperFile ='BigHelp'
        self.H = self.load_lib(self.RootHelperFile)
        # module list loading
        self.H = self.load_lib('live_list', _ROOT_FOLDER_)
        self.H = self.load_lib('experimental_list', _ROOT_FOLDER_)
        # Live modules list
        self.libs_list = [
                            'ZipUp',
                            'RepCoList',
                            'send_mail'
                         ]
        # experimental modules list
        self.libs_experimental_list = self.libs_list +[
                                                        'GdriveD',
                                                        'RamGpu',
                                                        'GitAid',
                                                        'Fileview',
                                                        'custom_functions',
                                                        'FiFyFo',
                                                      ]
        self.list_to_load =self.libs_list
        if self.dev == True:
            self.list_to_load = self.libs_experimental_list   
        # load the modules into the globals
        for lib in self.list_to_load:
            globals()[lib] = self.load_lib(str(lib))
    
        
    def load_lib(self,lib_file,folder=_LIB_+'/'):
        '''
            1Get the specs from the file
            2Make a module outof the specs
            3init the module into the globals()[name] (same as import...)
        ''' 
        module_path = folder+lib_file+'.py'
        print('Module : '+ lib_file +' Loaded!')
        if self.dev==True:
            print(module_path)
            
        dummy = impylib.spec_from_file_location("module.name",module_path )
        Imp = impylib.module_from_spec(dummy)
        dummy.loader.exec_module(Imp)
        
        return Imp
        
    
    
    def system_cmd(cmd,args,vals):
        cmd = ['', '--arg', 'value']
        proc_out=[]
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
        for line in p.stdout:
            proc_out.append(line)
        # print(line)
        p.wait()
        print(p.returncode)
        print(proc_out)


        
         
# #Big Helpers Loading
# from pathlib import Path
# import os, inspect
# try:
#     os.system('rm -r /content/sample_data')
#     os.system('rm -r /content/ProjectPrimer.py')
# except:
#     print('No default garbage to remove')

# # lib_file=Path('/content/lib/Helpers.py')
# # if not lib_file.is_file():

# #     os.system('wget https://raw.githubusercontent.com/bxck75/PyHelpers/master/BigHelp.py -O /content/lib/BigHelp.py')
# # os.chdir('/content/')
# # import the biggest help
# from Libs.BigHelp import Helpers
# # installing done......bring in the helpers!
# global H
# H=Helpers()
# # bring in the helpers!
# H.Me(['cml','echo "Pull in the helpers!"'])
# H.prime=['bxck75/PyHelpers']
# H.Me(['inst_reps',H.prime,'/content/lib',True,True])
# H.Me(['cml','echo "All Done!"'])
# # os.chdir('/content/')
# os.system('rm -r /content/lib/BigHelp.py')

# check the new module
Stimpy_Imp()
# print(dir(experimental))
# help(sys.argv[2])

print(SI)
print(experimental_list)
%%writefile /content/PyHelpers/live_list.py
live_mods=[
        'ZipUp',
        'RepCoList',
        'send_mail'
       ]
%%writefile /content/PyHelpers/experimental_list.py
x_mods=[
        'GdriveD',
        'RamGpu',
        'GitAid',
        'Fileview',
        'custom_functions',
        'FiFyFo',
        ]
'''Own functions list'''
def get_gdrive_dataset(pack, DS_root='datasets',GD_root='datasets'):
    import google
    from google.colab import drive
    drive.mount('/content/drive', force_remount=True)
    H.GD_ROOT=GD_root+'/'
    H.DS_ROOT=DS_root+'/'
    os.chdir(H.gdrive_root+H.GD_ROOT)
    H.Me(['mkd',[DS_root,'models'],H.pix_root])
    H.Me(['cml','cp -r '+pack+' '+H.pix_root+DS_root])
    os.chdir(H.pix_root+DS_root)
    H.Me(['cml','unzip -q '+pack])
    H.Me(['cml','rm -r '+pack])
    os.chdir(H.pix_root)

def MethHelp(libs):
    os_help=H.Me(['vdir',libs])
    #make a list containing libs values of os_help
    listOfLibs = [x[0] for x in os_help]
    #make a list containing libs method values of os_help
    listOfMethods= [x[1] for x in os_help]
    # Create a zipped list of tuples from above lists
    zippedList =  list(zip(listOfLibs, listOfMethods[0:5]))
    zippedList
    # request help on method from list
    return zippedList
    
def loadTboard():
  '''load tensorboard'''
    import datetime, os
    # install tensorboard
#     H.Me(['cml','pip install -q tensorflow'])
    # Load the TensorBoard notebook extension
    try:
        %load_ext tensorboard
    except:
        %reload_ext tensorboard

# !wget https://raw.githubusercontent.com/bxck75/PyHelpers/master/ProjectPrimer.py
# !python ProjectPrimer.py
from lib.PyHelpers.ProjectPrimer import H
from IPython.display import clear_output
from lib import PyHelpers
from lib.PyHelpers import RepCoList,RamGpu,GdriveD,ZipUp
H.Me(['vdir',[ZipUp]])

clear_output()
from lib import PyHelpers
from lib.PyHelpers import RepCoList,RamGpu,GdriveD,ZipUp,BigHelp
H.zip_to_drive = ZipUp.ZipUp
H.Me(['vdir',[H.zip_to_drive]])
sheit_to = H.zip_to_drive('sample_data', # name of zipfile
                            '/content/drive/My Drive', # folder to push the zip to
                            '/content/sample_data') # folder to zip
print(sheit_to.ZipUp)
H.Me(['vdir',[H.zip_to_drive]])
# H.Me(['vdir',[BigHelp,GdriveD]])
%cd /content/
import os
os.system('pip install -U -q PyDrive')
from google.colab import files
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials
import zipfile
import os
import sys

class ZipUp:

    def __init__(self, zipname, foldername, target_dir):
        ''' init the details for the zip and push'''
        self.zipname = zipname
        self.foldername = foldername
        self.target_dir = target_dir

    @property
    def ZipUp(self):
        ''' define the zip_n_push property'''
        if( self.zipname !='' and self.foldername != '' and self.target_dir != '' ):
            self.zipfolder()
            self.g_login()
            self.make_push()
            self.status = self.status + self.get_id()
            return self.status
        
    def zipfolder(self):
        ''' zip the selected folder to the target dir with '''
        zipobj = zipfile.ZipFile(self.zipname + '.zip', 'w', zipfile.ZIP_DEFLATED)
        rootlen = len(self.target_dir) + 1
        for base, dirs, files in os.walk(self.target_dir):
            for file in files:
                fn = os.path.join(base, file)
                zipobj.write(fn, fn[rootlen:])
        self.status='zipped '
                
    def g_login(self):
        ''' Authenticate and create the PyDrive client.'''
        auth.authenticate_user()
        gauth = GoogleAuth()
        gauth.credentials = GoogleCredentials.get_application_default()
        self.drive = GoogleDrive(gauth)
        self.status= self.status+'and '
        
    def make_push(self):
        ''' Create & upload a file text file.'''
        file1 = self.drive.CreateFile({'id':self.get_id()})
        file1.SetContentFile(self.zipname+".zip")
        file1.Upload()
        self.status= self.status+'pushed! (id) '

    def get_id(self):
        query = "title = '"+self.zipname+".zip'"
        file_list = self.drive.ListFile({'q': query}).GetList()
        for file in file_list:
            if file['labels']['trashed'] ==False:
                return file['id']

if __name__ == "__main__":
    item_to = ZipUp('sample_data','/content/drive/My Drive','/content/lib/PyHelpers')
    print(item_to.ZipUp)

from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive

def g_login():
    ''' Authenticate and create the PyDrive client.'''
    auth.authenticate_user()
    gauth = GoogleAuth()
    gauth.credentials = GoogleCredentials.get_application_default()
    return GoogleDrive(gauth)

drive = g_login()

query = "title = 'sample_data.zip'"
file_list = drive.ListFile({'q': query}).GetList()

for file in file_list:
    if file['labels']['trashed'] ==False:
        print('-' * 10)
        print(file['id'])
#         print(file['downloadUrl'])

#         print('-' * 10)
        # first parent id
        parent_id = file['parents'][0]['id']
        print(parent_id)

#         print('-' * 10)
#         x = drive.CreateFile({'id': parent_id})
#         x.FetchMetadata()
#         print(x)
%cd /content/lib/PyHelpers

import tensorflow as tf
device_name = tf.test.gpu_device_name()
if device_name != '/device:GPU:0':
    raise SystemError('GPU device not found')
print('Found GPU at: {}'.format(device_name))

# Install Keras with pip
!pip install -q keras
import keras
# >>> Using TensorFlow backend.
#  Install GraphViz with apt
!apt-get install graphviz -y
# Here's the easiest way to do so, IMO, with a little direction from here.

# In a 3 step process, first invoke a file selector within your notebook with this:

from google.colab import files
uploaded = files.upload()


# After your file(s) is/are selected, use the following to iterate the uploaded files in order to find their key names, using:

for fn in uploaded.keys():
    print('User uploaded file "{name}" with length {length} bytes'.format(name=fn, length=len(uploaded[fn])))












import os
os.chdir('/content/lib')
from lib.ProjectPrimer import H as Tickle
import importlib.util
# print(len(H.Me(['globx','/content/lib/PyHelpers','*.py'])))
# Tickle.Me(['vdir',[importlib]])
# for mod in range(len(H.Me(['globx','/content/lib/PyHelpers','*.py']))):
#     print(mod)
# Get the specs from the file
spec = importlib.util.spec_from_file_location("module.name", "/content/lib/PyHelpers/ZipUp.py")
# Make a module outof the specs
ZipDrive = importlib.util.module_from_spec(spec)
# Load the new module(same as import...)
spec.loader.exec_module(ZipDrive)
# check the new module
Tickle.Me(['vdir',[Zip2Drive]])
# # item_to = foo.ZipUp('sample_data','/content/drive/My Drive','/content/lib/PyHelpers')
# # print(item_to.ZipUp)

# # foo.ZipUp()
# help(foo)
# print(foo.__dict__)
# foo.DICT=foo.__dict__
# __builtins__.locals()

H.Me<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13137585380998848, 'fraction_numerical': 0.0028386410006209528, 'mean_word_length': 2.5699810006333124, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '14251', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 3681, 'n_tokens_neox': 3606, 'n_words': 823, 'path': 'Helpers/OptimalPrime.ipynb', 'repo_name': 'bxck75/Python_Helpers', 'size': '11273'}