<jupyter_start><jupyter_text><jupyter_code>tmp['mag'].isna().sum() / len(tmp['mag']) > 0.9
tmp['mag'].count()
ls -lh
pd.concat(mag,axis=1)
from google.colab import drive
drive.mount('/content/drive')<jupyter_output>Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly

Enter your authorization code:
··········
Mounted at /content/drive<jupyter_text>UKIRT Confirmed Microlensing<jupyter_code># !wget https://exoplanetarchive.ipac.caltech.edu/cgi-bin/FileDownload/nph-download?ref=/work/TMP_GwWS7c_5969/IERDownload/5969//download_exoarch_5969.bat
# !wget https://exoplanetarchive.ipac.caltech.edu/cgi-bin/FileDownload/nph-download?ref=/work/TMP_ivHBn2_8695/IERDownload/8695//download_exoarch_8695.bat
# !wget https://exoplanetarchive.ipac.caltech.edu/cgi-bin/FileDownload/nph-download?ref=/work/TMP_J7dwYA_9871/IERDownload/9871//download_exoarch_9871.bat
# !wget https://exoplanetarchive.ipac.caltech.edu/cgi-bin/FileDownload/nph-download?ref=/work/TMP_eDiO1U_9805/IERDownload/9805//download_exoarch_9805.bat
# !wget https://exoplanetarchive.ipac.caltech.edu/cgi-bin/FileDownload/nph-download?ref=/work/TMP_PB9YU1_9739/IERDownload/9739//download_exoarch_9739.bat
# !wget https://exoplanetarchive.ipac.caltech.edu/cgi-bin/FileDownload/nph-download?ref=/work/TMP_ZBHfPx_9650/IERDownload/9650//download_exoarch_9650.bat
# !wget https://exoplanetarchive.ipac.caltech.edu/cgi-bin/FileDownload/nph-download?ref=/work/TMP_xGyjhD_9517/IERDownload/9517//download_exoarch_9517.bat
# !wget https://exoplanetarchive.ipac.caltech.edu/cgi-bin/FileDownload/nph-download?ref=/work/TMP_dK6gEg_9379/IERDownload/9379//download_exoarch_9379.bat
# !wget https://exoplanetarchive.ipac.caltech.edu/cgi-bin/FileDownload/nph-download?ref=/work/TMP_AM4LKo_9269/IERDownload/9269//download_exoarch_9269.bat
ls
!chmod +x *.bat
!for x in *.bat; do ./$x; done
# !./*.bat
ls -l
import matplotlib.pyplot as plt
from tqdm import tqdm
import re
import pickle

mag, emag = [], []
for filename in sorted(os.listdir(directory)):
    if filename.endswith(".tbl"): 
#         print(os.path.join(directory, filename))
        tmp=pd.read_table(os.path.join(directory, filename), skiprows=21, 
                 delim_whitespace=True, 
                 header=None,
                 index_col=0,                          
                 names=['hjd', 'mag', 'emag']
                )

        # Use Regex to get Index ID
        idx = re.findall("(\d{7})", filename)[0]
        # print(idx)
        mag.append(tmp['mag'].rename(idx))
        emag.append(tmp['emag'].rename(idx))


        print('{}/{}'.format(tmp['mag'].count(), len(tmp['mag'])), '\t{:.2%}'.format(tmp['mag'].count() / len(tmp['mag'])))

        # plt.plot(tmp['mag'])
        # plt.title('mag')
        # plt.show()
        # plt.plot(tmp['emag'])
        # plt.title('emag')
        # plt.show()
    else:
        continue
mag = pd.concat(mag,axis=1).T
emag = pd.concat(emag,axis=1).T
# pickle.dump(mag, open('/content/drive/My Drive/pTSA_microlensing/UKIRT/microlensing_mag.pkl', 'wb'))
# pickle.dump(emag, open('/content/drive/My Drive/pTSA_microlensing/UKIRT/microlensing_emag.pkl', 'wb'))

mag = pickle.load(open('/content/drive/My Drive/pTSA_microlensing/UKIRT/microlensing_mag.pkl', 'rb'))
emag = pickle.load(open('/content/drive/My Drive/pTSA_microlensing/UKIRT/microlensing_emag.pkl', 'rb'))
for i in range(mag.shape[0]):
  plt.plot(mag.iloc[i].T.dropna())
  plt.show()
for i in range(emag.shape[0]):
  plt.plot(emag.iloc[i].T.dropna())
  plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16798167472639347, 'fraction_numerical': 0.048103843217103585, 'mean_word_length': 4.527426160337553, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 10, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '504137', 'license': 'MIT', 'n_tokens_mistral': 1694, 'n_tokens_neox': 1568, 'n_words': 148, 'path': 'PTSA_UKIRTdata.ipynb', 'repo_name': 'dibyajit30/ExoplanetHunting', 'size': '3929'}