<jupyter_start><jupyter_text>Jason TracksThis notebook contains the code used to build the Jason track and image databases used in the exploratory maps.<jupyter_code>import numpy as np
import pandas as pd
import datetime
import requests
from tqdm import tqdm<jupyter_output><empty_output><jupyter_text>Utility Functions<jupyter_code>def exists(test_url):
    r = requests.head(test_url)
    return r.status_code == requests.codes.ok
def get_image_urls(line, image_dir):
    cruise = line.split(',')[25]
    
    #print(cruise)
    
    if cruise == 'RR1715' or cruise == 'RR1714' or cruise == 'RR1713':
        cruise_name = cruise.lower()
        cam1_name = 'SciCam'
        cam2_name = 'PilotCam'
        cam3_name = 'BrowCam'
        
    if cruise == 'KM1813' or cruise == 'RR1712':
        cruise_name = cruise.lower()[0:4] + '-' + cruise.lower()[4:6]
        cam1_name = 'SubSea1'
        cam2_name = 'SubSea2'
        cam3_name = 'SubSea3'
        
    if cruise == 'RR1712' or cruise == 'SKQ201610S' or cruise == 'TN327' or cruise == 'TN300':
        cruise_name = cruise.lower()
        cam1_name = 'SubSea1'
        cam2_name = 'SubSea2'
        cam3_name = 'SubSea3'
        
    if cruise == 'RB-14-03':
        cruise_name = cruise.lower()[0:2] + cruise.lower()[3:]
        cam1_name = 'SubSea1'
        cam2_name = 'SubSea2'
        cam3_name = 'SubSea3'
        
    
    timestamp = line.split(',')[2].split(' ')[0].replace('/','') \
        + '_' + line.split(',')[2].split(' ')[1].replace(':','') + '.jpg'

    
    #print('http://4dgeo.whoi.edu/DAQ/rr1715/Jason/Src1/Images0001/SciCam.20170821_182336.jpg')
    #print('http://4dgeo.whoi.edu/DAQ/km18-13/Jason/Src1/Images0006/SubSea1.20180824_025905.jpg')
    #print('http://4dgeo.whoi.edu/DAQ/rr1712/Jason/Src1/Images0003/SubSea1.20170717_212718.jpg')
    
    #print('http://4dgeo.whoi.edu/DAQ/rb14-03/Jason/Src1/Images0012/SubSea1.20140816_205125.jpg')
        
    #print('http://4dgeo.whoi.edu/DAQ/skq201610s/Jason/Src1/Images0009/SubSea1.20160716_025530.jpg')
    
    img1 = ('http://4dgeo.whoi.edu/DAQ/%s/Jason/Src1/Images%04.0f/%s.%s') % \
        (cruise_name, image_dir, cam1_name, timestamp)
    
    #print(img1)
        
    while not exists(img1):
        image_dir = image_dir + 1
        img1 = ('http://4dgeo.whoi.edu/DAQ/%s/Jason/Src1/Images%04.0f/%s.%s') % \
            (cruise_name, image_dir, cam1_name, timestamp)
        if image_dir == 100:
            raise ValueError
        
    img2 = ('http://4dgeo.whoi.edu/DAQ/%s/Jason/Src2/Images%04.0f/%s.%s') % \
        (cruise_name, image_dir, cam2_name, timestamp)
            #if not exists(img2):
            #    img2 = ''
#                raise FileNotFoundError(img2)
    img3 = ('http://4dgeo.whoi.edu/DAQ/%s/Jason/Src3/Images%04.0f/%s.%s') % \
        (cruise_name, image_dir, cam3_name, timestamp)
            #if not exists(img3):
            #    img3 = ''
#                raise FileNotFoundError(img3)

    return image_dir, img1, img2, img3
def read_jason(filename):
    event_list = []
    num_lines = sum(1 for line in open(filename, 'r', encoding='latin1'))
    i = 0
    image_dir = 1
    with open(filename, 'r', encoding='latin1') as f:
        for line in tqdm(f, total = num_lines):
            if i > 0:
                lowering = line.split(',')[0]
                if not lowering:
                    i = i+1
                    continue
                

                lat = np.float64(line.split(',')[3])
                lon = np.float64(line.split(',')[4])
                
                    
                depth = np.float64(line.split(',')[11])

                
                if depth > 1000 and lat > 45.85 and lat < 46 and lon > -130.1 and lon < -129.9:
                    total_depth = np.float64(line.split(',')[12])    
                    cruise = line.split(',')[25]   
                    comment = ','.join(line.split(',')[27:]).strip()
                    timestamp = datetime.datetime.strptime(line.split(',')[2], '%Y/%m/%d %H:%M:%S')
                    image_dir, img1, img2, img3 = get_image_urls(line, image_dir)
                    event_list.append([cruise, lowering, i, timestamp, lat, lon,
                        depth, total_depth, comment, img1, img2, img3])
            i = i+1
            
            
    df = pd.DataFrame(event_list, columns = ['cruise', 'lowering', 'event_number', 'timestamp',
        'lat', 'lon', 'depth', 'total_depth', 'event', 'img1', 'img2', 'img3'])
    return df<jupyter_output><empty_output><jupyter_text>RR1714<jupyter_code>data_dir = '/home/jovyan/github/ooicloud/axial-drilling/maps/data/jason/'
filename = 'rr1714.txt'
rr1714_df = read_jason(data_dir + filename)
rr1714_df.head()
rr1714_df.to_json(data_dir + filename.split('.')[0] + '.json', orient="records", lines=True)<jupyter_output><empty_output><jupyter_text>RR1715<jupyter_code>data_dir = '/home/jovyan/github/ooicloud/axial-drilling/maps/data/jason/'
filename = 'rr1715.txt'
rr1715_df = read_jason(data_dir + filename)
len(rr1715_df)
rr1715_df.to_json(data_dir + filename.split('.')[0] + '.json', orient="records", lines=True)<jupyter_output><empty_output><jupyter_text>KM1813<jupyter_code>data_dir = '/home/jovyan/github/ooicloud/axial-drilling/maps/data/jason/'
filename = 'km1813.txt'
km1813_df = read_jason(data_dir + filename)
len(km1813_df)
km1813_df.to_json(data_dir + filename.split('.')[0] + '.json', orient="records", lines=True)<jupyter_output><empty_output><jupyter_text>RR1712<jupyter_code>data_dir = '/home/jovyan/github/ooicloud/axial-drilling/maps/data/jason/'
filename = 'rr1712.txt'
rr1712_df = read_jason(data_dir + filename)
len(rr1712_df)
rr1712_df.to_json(data_dir + filename.split('.')[0] + '.json', orient="records", lines=True)<jupyter_output><empty_output><jupyter_text>RR1713<jupyter_code>data_dir = '/home/jovyan/github/ooicloud/axial-drilling/maps/data/jason/'
filename = 'rr1713.txt'
rr1713_df = read_jason(data_dir + filename)
len(rr1713_df)
rr1713_df.to_json(data_dir + filename.split('.')[0] + '.json', orient="records", lines=True)<jupyter_output><empty_output><jupyter_text>SKQ201610S<jupyter_code>data_dir = '/home/jovyan/github/ooicloud/axial-drilling/maps/data/jason/'
filename = 'skq201610s.txt'
skq201610s_df = read_jason(data_dir + filename)
len(skq201610s_df)
skq201610s_df.to_json(data_dir + filename.split('.')[0] + '.json', orient="records", lines=True)<jupyter_output><empty_output><jupyter_text>TN327<jupyter_code>data_dir = '/home/jovyan/github/ooicloud/axial-drilling/maps/data/jason/'
filename = 'tn327.txt'
tn327_df = read_jason(data_dir + filename)
len(tn327_df)
tn327_df.to_json(data_dir + filename.split('.')[0] + '.json', orient="records", lines=True)<jupyter_output><empty_output><jupyter_text>RB1403<jupyter_code>data_dir = '/home/jovyan/github/ooicloud/axial-drilling/maps/data/jason/'
filename = 'rb1403.txt'
rb1403_df = read_jason(data_dir + filename)
len(rb1403_df)
rb1403_df.to_json(data_dir + filename.split('.')[0] + '.json', orient="records", lines=True)<jupyter_output><empty_output><jupyter_text>TN300<jupyter_code>data_dir = '/home/jovyan/github/ooicloud/axial-drilling/maps/data/jason/'
filename = 'tn300.txt'
tn300_df = read_jason(data_dir + filename)
len(tn300_df)
tn300_df.to_json(data_dir + filename.split('.')[0] + '.json', orient="records", lines=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1602773246329527, 'fraction_numerical': 0.06620445894507884, 'mean_word_length': 3.187250996015936, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '164185', 'license': 'MIT', 'n_tokens_mistral': 3075, 'n_tokens_neox': 2781, 'n_words': 390, 'path': 'maps/notebooks/jason_tracks.ipynb', 'repo_name': 'ooicloud/axial-drilling', 'size': '7356'}