<jupyter_start><jupyter_text>-------------------- Signal to Binary Files (Train&Test) using SparkIn this notebook we read the Basic 4 dataset through Spark, and convert signals into a binary file.<jupyter_code>import requests
import json
#!pip install ibmseti
import ibmseti
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import tensorflow as tf
import os
from PIL import Image
from matplotlib import cm<jupyter_output><empty_output><jupyter_text>Get file listI have a list of simulated files stored in an OpenStack Object Storage container that is world-readable.Download that file, split by lines and parallelize it into an RDD<jupyter_code>import findspark
findspark.init()
import pyspark
sc = pyspark.SparkContext()
# Images
base_url = 'https://62a44.https.cdn.softlayer.net/seti'
container = 'Basic4'
# list of images
r = requests.get('{}/{}/{}'.format(base_url, container, 'index.csv'))
filelist_txt = r.text
fl_rdd = sc.parallelize(filelist_txt.split('\n')[1:-1], 20)
fl_rdd.count()<jupyter_output><empty_output><jupyter_text>Quick Inspection.Take a look at the first row. Each line is CSV with 'uuid' and 'signal_classification'. We'll convert each line to JSON.<jupyter_code>def csvtojson(row):
    uuid, sigclass = row.split(',')
    return {'file_name':uuid+'.dat', 'uuid':uuid, 'signal_classification':sigclass}

fl_rdd2 = fl_rdd.map(csvtojson)
fl_rdd2.cache()
print fl_rdd2.first()
fl_rdd2.cache()<jupyter_output><empty_output><jupyter_text>Here is the different "signal_classifications" we have.<jupyter_code>rdd_fname_lb = fl_rdd2.map(lambda row: (row['signal_classification'],row['file_name']))
classes = rdd_fname_lb.map(lambda row: row[0]).distinct().collect()
dictClass = dict(zip(classes, np.arange(4)))
dictClass<jupyter_output><empty_output><jupyter_text>Send request for data, and convert signals to spectogram<jupyter_code>def get_spectrogram(fname,h,w,lengthRatio=1.0):
    
    r = requests.get('{}/{}/{}'.format(base_url, container, fname), timeout=4.0)
    if r.status_code != 200:
        print 'Failed retrieving {}'.format(fname)
        print r
        return None
    else:
        aca = ibmseti.compamp.SimCompamp(r.content)
        com_data = aca.complex_data()
        ratio = int(np.sqrt(len(com_data) *lengthRatio / (h*w)))
        if ratio == 0: 
            raise ValueError, "The selected lenght of signal is less than (Height x Width), select bigger ratio"
        elif ratio == 1:
            sig_data = com_data[:h*w].reshape(h,w)
            spec = np.abs( np.fft.fftshift( np.fft.fft(sig_data), 1) )**2
            spec = np.log(spec)
            spec = spec/np.max(spec) # Convert to float (0-1)
            image = Image.fromarray(cm.jet(spec, bytes=True)) #convert to RGB
        elif ratio > 1: # resize using IPL image
            sig_data = com_data[:h*ratio*w*ratio].reshape(h*ratio,w*ratio)
            spec = np.abs( np.fft.fftshift( np.fft.fft(sig_data), 1) )**2
            spec = np.log(spec) # Convert to float (0-255)
            spec = spec/np.max(spec) # Convert to float (0-1)
            image = Image.fromarray(cm.jet(spec, bytes=True)) #convert to RGB  
            image = image.resize((int(w), int(h)), Image.ANTIALIAS)

        
        return image<jupyter_output><empty_output><jupyter_text>h and w are the hight and width of the images, and lengthRatio is the length of signal in ratio.<jupyter_code>h = 32 # The hight of output image (bins)
w = 32 # The witdh of output image
lengthRatio = 1.0  # the length-ration of signal to be read. The higher reatio, the better resolution. E.g. 0.5 means half of time sereis.
rdd_rgb_spec = rdd_fname_lb.map(lambda row: (row[1], dictClass[row[0]], get_spectrogram(row[1],h,w,lengthRatio)))


rdd_rgb_spec.cache()
#get_spectrogram('b1cc342f-eae4-442b-91de-10c9a444072e.dat',h,w,lengthRatio=1.0)<jupyter_output><empty_output><jupyter_text>verify<jupyter_code>a = rdd_rgb_spec.take(1)
a[0][2]
z=5
y= rdd_rgb_spec.take(z)
for i in range(z):
    img = y[i][2]
    fig, ax = plt.subplots(figsize=(10, 5))
    ax.imshow(img)
from array import array
from array import array
def toBinary(class_name,img):
    classnum = int(class_name)
    data = array('B')
    data.append(classnum)
    pix = img.load()
    for color in range(0,3):
        for x in range(0,32):
            for y in range(0,32):
                data.append(pix[x,y][color])
    return data
allfiles = rdd_rgb_spec.map(lambda row: toBinary(row[1],row[2])).take(1000)
data = array('B')
for f in allfiles:
    for byt in f:
        data.append(byt)
output_file = open('data_batch_1.bin', 'wb')
data.tofile(output_file)
output_file.close()<jupyter_output><empty_output><jupyter_text>Train/Test<jupyter_code>test, train = rdd_gray_spec.randomSplit(weights=[0.3, 0.7], seed=1)
train_data = train.map(lambda row: row[2]).collect()
train_img_data = np.array(train_data)
train_lbl = train.map(lambda row: row[1]).collect()
test_data = test.map(lambda row: row[2]).collect()
test_img_data = np.array(test_data)
test_lbl = test.map(lambda row: row[1]).collect()<jupyter_output><empty_output><jupyter_text>Write to Binary fileThis binary file is same as famouse __mnist__ dataset format to be read by different image processing algorithms, learning techniques and pattern recognition methods. There are 4 files:  - train-images-idx3-ubyte: training set images - train-labels-idx1-ubyte: training set labels - test-images-idx3-ubyte:  test set images - test-labels-idx1-ubyte:  test set labels<jupyter_code>from array import *
def wrtieToBinary(ds_directory, name , imgData , lblData,h,w):
    
    n = imgData.shape[0]
    imgData = imgData.reshape(-1,)
    
    data_image = array('B')
    data_label = array('B')
    
    data_image.extend(imgData)

    # number of files in HEX
    hexval = "{0:#0{1}x}".format(n,6) 
    
    # header for label array
    data_label.extend(lblData)
    header = array('B')
    header.extend([0,0,8,1,0,0])
    header.append(int('0x'+hexval[2:][:2],16))
    header.append(int('0x'+hexval[2:][2:],16))
    data_label = header + data_label
    print ('Label header:' )
    print(header)
    # additional header for images array
    if max([w,h]) <= 255:
        header.extend([0,0,0,h,0,0,0,w])
    else:
        hex_h = "{0:#0{1}x}".format(h,6)
        header.extend([0,0])
        header.append(int('0x'+hex_h[2:][:2],16))
        header.append(int('0x'+hex_h[2:][2:],16))
        hex_w = "{0:#0{1}x}".format(w,6)
        header.extend([0,0])
        header.append(int('0x'+hex_w[2:][:2],16))
        header.append(int('0x'+hex_w[2:][2:],16))
        #raise ValueError('Image exceeds maximum size: 256x256 pixels');
    header[3] = 3 # Changing MSB for image data (0x00000803)
        
    if not os.path.exists(ds_directory):
        os.makedirs(ds_directory)
            
        
    print ('Image header:' )
    print(header)
    data_image = header + data_image

    output_file = open(ds_directory + name+'-images-idx3-ubyte', 'wb')
    data_image.tofile(output_file)
    output_file.close()

    output_file = open(ds_directory+ name+'-labels-idx1-ubyte', 'wb')
    data_label.tofile(output_file)
    output_file.close()
    
    # gzip resulting files


    os.system('gzip '+ ds_directory + name +'-images-idx3-ubyte '+ name +'-images-idx3-ubyte.gz')
    os.system('gzip '+ ds_directory + name +'-labels-idx1-ubyte ')

ds_directory = 'SETI/SETI_ds_128x256/'  # The dataset directory to write the binary files
os.system('rm '+ds_directory+'*')
print os.popen("ls -lrt "+ ds_directory).read()
wrtieToBinary(ds_directory, 'train' , train_img_data , train_lbl, h, w)
wrtieToBinary(ds_directory, 'test' , test_img_data , test_lbl, h, w)<jupyter_output><empty_output><jupyter_text>Verify the binary filesLets read the binary file and plot an image<jupyter_code>print os.popen("ls -lrt "+ ds_directory).read()
import numpy as np
import gzip
def _read32(bytestream):
  dt = np.dtype(np.uint32).newbyteorder('>')
  return np.frombuffer(bytestream.read(4), dtype=dt)[0]

with open(ds_directory+'train-images-idx3-ubyte.gz', 'rb') as f:
    with gzip.GzipFile(fileobj=f ) as bytestream:
        magic = _read32(bytestream)
        if magic != 2051:
            raise ValueError('Invalid magic number %d in MNIST image file: %s' %(magic, f.name))
        num_images = _read32(bytestream)
        rows = _read32(bytestream)
        cols = _read32(bytestream)
        buf = bytestream.read(rows * cols * num_images)
        print(magic,num_images,rows,cols,)
        
        data = np.frombuffer(buf, dtype=np.uint8)
        data = data.reshape(num_images, rows, cols, 1)
# magic, num, rows, cols = struct.unpack(">IIII", bytestream.read(16))

gray_y = data[0].reshape(h,w)
img = Image.fromarray(np.float32(gray_y))
print (img.mode)
fig, ax = plt.subplots(figsize=(10, 5))
ax.imshow(img)
gray_y<jupyter_output><empty_output><jupyter_text>Verify the binary files by reader class__SETI.py__ is a helper class, identical to mnist dataset reader, to easily read dataset, one-hot coding, and read images as batch.<jupyter_code>!wget -q --output-document SETI.zip  https://ibm.box.com/shared/static/jhqdhcblhua5dx2t7ixwm88okitjrl6l.zip
!unzip -o SETI.zip
import SETI
SETIds = SETI.read_data_sets(ds_directory, one_hot=True, validation_size=0)
SETIds.train.num_examples
gray_y = SETIds.train.images[0].reshape(h,w)
img = Image.fromarray(gray_y*255)
fig, ax = plt.subplots(figsize=(10, 5))
ax.imshow(img)<jupyter_output><empty_output><jupyter_text>Export files to object storageYou have to move the binary file to __object storage__ if you need to download the binary file.<jupyter_code>print os.popen("ls -lrt "+ ds_directory).read()<jupyter_output><empty_output><jupyter_text>You need your credentials to move your files to object storage. How to get the credentials for object storage?Check this [link](https://github.com/ibm-cds-labs/seti_at_ibmobject-storage-credentials).<jupyter_code># 
credentials_1 = {
  'auth_url':'https://identity.open.softlayer.com',
  'project':'xx',
  'project_id':'xx',
  'region':'dallas',
  'user_id':'xx',
  'domain_id':'xx',
  'domain_name':'819515',
  'username':'xx',
  'password':"xx",
  'container':'x',
  'tenantId':'undefined',
  'filename':'SETI.py'
}
#!pip install --user --upgrade python-swiftclient

import swiftclient.client as swiftclient

conn = swiftclient.Connection(
    key=credentials_1['password'],
    authurl=credentials_1['auth_url']+"/v3",
    auth_version='3',
    os_options={
        "project_id": credentials_1['project_id'],
        "user_id": credentials_1['user_id'],
        "region_name": credentials_1['region']})<jupyter_output><empty_output><jupyter_text>Export the binary files as a single file to object storage<jupyter_code>import gzip
local_file = 'SETI128x256.tar.gz'
os.system('tar -zcvf '+local_file+' '+ds_directory)
print ('Moving '+ local_file + '...')
with open(local_file, 'rb') as f:
    with gzip.GzipFile(fileobj=f) as bytestream:
        etag = conn.put_object(credentials_1['container'], local_file , f.read() )<jupyter_output><empty_output><jupyter_text>Export the binary files as multiple files into object storage<jupyter_code>files = ['train-images-idx3-ubyte.gz','train-labels-idx1-ubyte.gz','test-images-idx3-ubyte.gz','test-labels-idx1-ubyte.gz']
for local_file in files:
    print ('Moving '+ local_file + '...')
    with open(ds_directory + local_file, 'rb') as f:
        with gzip.GzipFile(fileobj=f) as bytestream:
            etag = conn.put_object(credentials_1['container'], local_file , f.read() )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14118564742589704, 'fraction_numerical': 0.023227595770497485, 'mean_word_length': 4.2000901306894995, 'pattern_counts': {'":': 3, '<': 62, '<?xml version=': 0, '>': 64, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '213200', 'license': 'Apache-2.0', 'n_tokens_mistral': 4297, 'n_tokens_neox': 4135, 'n_words': 902, 'path': 'notebooks/SETI_img_to_binary_spark.ipynb', 'repo_name': 'kant/powerai-seti-signal-classification', 'size': '11538'}