<jupyter_start><jupyter_text><jupyter_code>!apt-get install openjdk-8-jdk-headless -qq > /dev/null
!wget -q https://archive.apache.org/dist/spark/spark-2.3.0/spark-2.3.0-bin-hadoop2.7.tgz
!tar xf spark-2.3.0-bin-hadoop2.7.tgz
!pip install -q findspark
!pip install pyspark
import os
os.environ["JAVA_HOME"] = "/usr/lib/jvm/java-8-openjdk-amd64"
os.environ["SPARK_HOME"] = "/content/spark-2.3.0-bin-hadoop2.7"
import findspark
findspark.init()

import pyspark # only run after findspark.init()
from pyspark.sql import SparkSession
spark = SparkSession.builder.getOrCreate()

sc = spark.sparkContext

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
# for google colab
# Load the Drive helper and mount
from google.colab import drive
drive.mount('/content/drive/')
import os
os.chdir("/content/drive/My Drive/Colab Notebooks/Skin Cancer MNIST HAM10000")
!ls

from pyspark.ml.image import ImageSchema
x = ImageSchema.readImages("/content/drive/My Drive/Colab Notebooks/Skin Cancer MNIST HAM10000/imagefolder_1/*.jpg")
type(x)
x.printSchema()
x.schema.names
x.show(4)
x.take(1)[0][0][0]
x.take(1)

## out> [Row(image=Row(origin='file:/content/drive/My Drive/Colab Notebooks/Skin Cancer MNIST HAM10000/imagefolder_1/ISIC_0028303.jpg', height=450, width=600, nChannels=3, mode=16, 
## data=bytearray(b'\xb9\xab\xc3\xb8\xaa\xc2......
# x.repartition(1)
x.take(1)[0][0][5]

## out> bytearray(b'\xb9\xab\xc3\xb8\xaa\xc2\xb8\xaa\xc2\xb6\xa8\xc0\xb7\xa9\xc1\xb7\xa9\xc1\xba\xaa\xc1......
z = x.select([x.image.origin] + [x.image.data])
z.show(4)
z = z.withColumnRenamed("image.origin", "path").withColumnRenamed("image.data", "array")
## or # z.selectExpr("image.origin as path", "image.data as array")
z.show(4)
z.count()
z.explain()
z.select("path", "array").write.save("pathandarray.csv")
z.select("path", "array").write.save("pathandarray.parquet")
z.select("path", "array").write.save("pathandarray.json", format = "json")
print(z.schema)
print(z.schema.names)<jupyter_output>StructType(List(StructField(path,StringType,true),StructField(array,BinaryType,true)))
['path', 'array']<jupyter_text>OR<jupyter_code># !pip install sparkdl
# !pip install tensorframes
# !pip install kafka
# !pip install tensorflowonspark
# from sparkdl import readImages
x = sc.binaryFiles('/content/drive/My Drive/Colab Notebooks/Skin Cancer MNIST HAM10000/imagefolder_1/*.jpg', minPartitions = 5).repartition(5)
x.take(1)

## out> [('file:/content/drive/My Drive/Colab Notebooks/Skin Cancer MNIST HAM10000/imagefolder_1/ISIC_0028342.jpg',
 ## b'\xff\xd8\xff\xdb\x00C\x00\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x.........
x_list = x.take(1)
type(x)
x_list[0][1].encode('iso-8859-1')
#x_list[0][1].decode('cp1250') 
#str(x_list[0][1], 'utf-8')
#"".join(map(chr, x_list[0][1]))
#np.array('b', x_list[0][1])
rawdata = x_list[0][1].decode("utf-8", errors="ignore")
rawdata#.encode('utf-8', errors="ignore")

## out> '\x00C\x00\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\.........
!pwd
try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO
from PIL import Image
from io import BytesIO
try:
  print(np.asarray(Image.open(StringIO(x_list[0][1]))))

except:
  print(np.asarray(Image.open(BytesIO(x_list[0][1]))))
image_to_array = lambda rawdata: np.asarray(Image.open(BytesIO(x_list[0][1])))
y = x.values().map(image_to_array)
y.take(1)
y.getNumPartitions()<jupyter_output><empty_output><jupyter_text>OR<jupyter_code>import cv2
image = cv2.imread("imagefolder_1/ISIC_0024306.jpg")
image2 = sc.parallelize(cv2.imread("imagefolder_1/ISIC_0024306.jpg"))
from tqdm import tqdm_notebook as tqdm
image_list = []
for image_name in tqdm(os.listdir("imagefolder_1")[0:20]):
  image_list.append(sc.parallelize(cv2.imread("imagefolder_1/" + image_name)))
plt.imshow(image_list[1].collect())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1854388635210553, 'fraction_numerical': 0.0700152207001522, 'mean_word_length': 5.538971807628524, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 17, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '342925', 'license': 'MIT', 'n_tokens_mistral': 1797, 'n_tokens_neox': 1650, 'n_words': 242, 'path': 'Spark/PySpark_Loading_Images/PySpark_Loading_Images.ipynb', 'repo_name': 'Nik8x/PySpark', 'size': '3942'}