<jupyter_start><jupyter_text>Notebook Setup<jupyter_code>from labelbox import Client
import databricks.koalas as pd
import labelspark

try: API_KEY
except NameError: 
  API_KEY = dbutils.notebook.run("api_key", 60)

client = Client(API_KEY)

projects = client.get_projects()
for project in projects:
    print(project.name, project.uid)
# can parse the directory and make a Spark table of image URLs

def create_unstructured_dataset(): 
  print("Creating table of unstructured image data")
  # Pull information from Data Lake or other storage  
  dataSet = client.get_dataset("ckolyi9ha7h800y7i5ppr3put") #Insert Dataset ID from Labelbox for a sample dataset 

  #creates a list of datarow dictionaries 
  df_list = [ {
          "external_id": dataRow.external_id,
          "row_data": dataRow.row_data
      } for dataRow in dataSet.data_rows()]

  # Create DataFrame 
  images = pd.DataFrame(df_list)
  df_images = images.to_spark()
#   display(df_images)
  df_images.registerTempTable("unstructured_data")
  # df_images = spark.createDataFrame(images) 

table_exists = False 
tblList = spark.catalog.listTables()
if len(tblList) == 0: 
  create_unstructured_dataset()
  table_exists = True

for table in tblList: 
    if table.name == "unstructured_data": 
      print("Unstructured data table exists")
      table_exists = True

if not table_exists: create_unstructured_dataset()<jupyter_output><empty_output><jupyter_text>Load Unstructured Data<jupyter_code>%sql 

select * from unstructured_data
from labelbox import Client
client = Client(API_KEY)<jupyter_output><empty_output><jupyter_text>LabelSpark expects a spark table with two columns; the first column "external_id" and second column "row_data"external_id is a filename, like "birds.jpg" or "my_video.mp4"row_data is the URL path to the file. Labelbox renders assets locally on your users' machines when they label, so your labeler will need permission to access that asset. Example: | external_id | row_data                             ||-------------|--------------------------------------|| image1.jpg  | https://url_to_your_asset/image1.jpg || image2.jpg  | https://url_to_your_asset/image2.jpg || image3.jpg  | https://url_to_your_asset/image3.jpg |<jupyter_code>import labelspark
unstructured_data = spark.table("unstructured_data")
dataSet_new = labelspark.create_dataset(client, unstructured_data, "Demo Dataset")<jupyter_output><empty_output><jupyter_text>You can use the labelbox SDK to build your ontology. An example is provided below. Please refer to documentation at https://docs.labelbox.com/python-sdk/en/index-en<jupyter_code>from labelbox.schema.ontology import OntologyBuilder, Tool, Classification, Option
# from labelbox import Client
# import os

ontology = OntologyBuilder()
tool_people = Tool(tool=Tool.Type.BBOX, name="People")
tool_car = Tool(tool=Tool.Type.SEGMENTATION, name="Car")
tool_umbrella = Tool(tool=Tool.Type.POLYGON, name="Umbrella")
Weather_Classification = Classification(class_type=Classification.Type.RADIO, instructions="Weather", 
                                       options=[Option(value="Clear"), 
                                                Option(value="Overcast"),
                                                Option(value="Rain"),
                                                Option(value="Other")])
Time_of_Day = Classification(class_type=Classification.Type.RADIO, instructions="Time of Day", 
                                       options=[Option(value="Day"),
                                                Option(value="Night"),
                                                Option(value="Unknown")])

ontology.add_tool(tool_people)
ontology.add_tool(tool_car)
ontology.add_tool(tool_umbrella)
ontology.add_classification(Weather_Classification)
ontology.add_classification(Time_of_Day)


project_demo2 = client.create_project(name="LabelSpark Demo Example", description = "Example description here.")
project_demo2.datasets.connect(dataSet_new)

# Setup frontends 
all_frontends = list(client.get_labeling_frontends())
for frontend in all_frontends:
    if frontend.name == 'Editor':
        project_frontend = frontend
        break

# Attach Frontends
project_demo2.labeling_frontend.connect(project_frontend) 
# Attach Project and Ontology
project_demo2.setup(project_frontend, ontology.asdict()) 


print("Project Setup is complete.")<jupyter_output><empty_output><jupyter_text>Bronze and Silver Annotation Tables Be sure to provide your Labelbox Project ID (a long string like "ckolzeshr7zsy0736w0usbxdy") to labelspark get_annotations method to pull in your labeled dataset. bronze_table = labelspark.get_annotations(client,"ckolzeshr7zsy0736w0usbxdy", spark, sc) *These other methods transform the bronze table and do not require a project ID.* flattened_bronze_table = labelspark.flatten_bronze_table(bronze_table)silver_table = labelspark.bronze_to_silver(bronze_table)<jupyter_code>client = Client(API_KEY) #refresh client 
bronze_table = labelspark.get_annotations(client,"ckolzeshr7zsy0736w0usbxdj", spark, sc) #insert your unique project ID here
bronze_table.registerTempTable("street_photo_demo")
display(bronze_table.limit(2))
client = Client(API_KEY) #refresh client 
bronze_table = spark.table("street_photo_demo")
flattened_bronze_table = labelspark.flatten_bronze_table(bronze_table)
display(flattened_bronze_table.limit(1))
client = Client(API_KEY) #refresh client 
silver_table = labelspark.bronze_to_silver(bronze_table)
silver_table.registerTempTable("silver_table")
display(silver_table)
%sql 

SELECT * FROM silver_table 
WHERE `People.count` > 0 
AND `Umbrella.count` > 0
AND `Car.count` > 0
AND Weather = "Rain"
%sql 

SELECT * FROM silver_table
WHERE `People.count` > 10
def cleanup(): 
  client = Client(API_KEY)
  dataSet_new.delete()
  project_demo2.delete()

cleanup()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09797239734196626, 'fraction_numerical': 0.008178565343329358, 'mean_word_length': 4.245755138516532, 'pattern_counts': {'":': 3, '<': 21, '<?xml version=': 0, '>': 25, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '579209', 'license': 'Apache-2.0', 'n_tokens_mistral': 1851, 'n_tokens_neox': 1742, 'n_words': 407, 'path': 'examples/integrations/databricks/labelbox_databricks_example.ipynb', 'repo_name': 'nickaustinlee/labelbox-python', 'size': '5869'}