<jupyter_start><jupyter_text>Set-up Import Datalogue librariesNote, you'll need to have downloaded and installed the Datalogue SDK before this step will work.Right now, to do so you will need to get access through Artifactory.<jupyter_code># Import Datalogue libraries 
from datalogue import *
from datalogue.version import __version__
from datalogue.models.ontology import *
from datalogue.models.datastore_collection import *
from datalogue.models.datastore import *
from datalogue.models.datastore import GCSDatastoreDef 
from datalogue.models.credentials import *
from datalogue.models.stream import *
from datalogue.models.transformations import *
from datalogue.models.transformations.structure import *
from datalogue.dtl import Dtl, DtlCredentials
from datalogue.models.training import DataRef

# Import Datalogue Bag of Tricks
from DTLBagOTricks import DTL as DTLHelper


# Import other useful libraries
from datetime import datetime, timedelta
from os import environ
import pandas
from IPython.display import Image

# Checks the version of the SDK is correct
# The expected version is 0.28.3
# If the SDK is not installed, run `! pip install datalogue` and restart the Jupyter Notebook kernel
# If the wrong versions is installed, run `! pip install datalogue --upgrade` and restart the Jupyter Notebook kernel
__version__
# Set host, username and password variables

datalogue_host = "https://internal.dtl.systems"  # for connecting to internal (note)

# datalogue_host = "https://internal.dtl.systems"  # for connecting to internal (note)
# datalogue_host = "http://10.2.161.119:3000"  # for connecting to Eric's DGX
#email = environ.get("DTL_EMAIL")
email = ""
#password = environ.get("DTL_PASSWORD")
password = ""

# Log in to Datalogue
BOT = DTLHelper(datalogue_host, email, password)
dtl = BOT.dtl

# Expected output Datalogue v0.28.3
# "Logged in '[host location]' with '[username]' account)"<jupyter_output>Datalogue v0.30.2
Logged in 'https://internal.dtl.systems/api' with '<EMAIL>' account.<jupyter_text>2. Read Source Files from S3 bucket<jupyter_code>from boto.s3.connection import S3Connection

conn = S3Connection('AKIAIXM6CXHGHC62R7GA','<KEY>')
bucket = conn.get_bucket('datalogue-demo')

keys = ["store_name", "URL"]
npl_data = []

for key in bucket.list():
    if 'telemetry_' in key.name:
        url="https://datalogue-demo.s3.amazonaws.com/" + key.name
        values = ["demo-"+key.name, url]
        npl_data.append(dict(zip(keys, values)))

print(type(npl_data))
print(type(npl_data[0]))
print(npl_data)


print("\nCSV Machine Sources to connect to:\n" "-------------------------")
for data_store in npl_data:
    print("➜ " + data_store["store_name"])
print("\n")<jupyter_output>CSV Machine Sources to connect to:
-------------------------
➜ demo-mobile/telemetry/telemetry_EU_MOVI.csv
➜ demo-mobile/telemetry/telemetry_EU_O2.csv
➜ demo-mobile/telemetry/telemetry_EU_ORNG.csv
➜ demo-mobile/telemetry/telemetry_US_ATT.csv
➜ demo-mobile/telemetry/telemetry_US_SPR.csv
➜ demo-mobile/telemetry/telemetry_US_TMOB.csv
➜ demo-mobile/telemetry/telemetry_US_VZW.csv<jupyter_text>3. Create datastore connections for each file in S3 bucket<jupyter_code>current_stores = []

for data_store in npl_data:
    data_store["datastore_object"] = dtl.datastore.create(
        Datastore(
            data_store["store_name"],
            HttpDatastoreDef(data_store["URL"], FileFormat.Csv),
        )
    )
    current_stores.append(data_store["datastore_object"])

print(type(current_stores))
print(type(current_stores[0]))

print(data_store)<jupyter_output><class 'list'>
<class 'datalogue.models.datastore.Datastore'>
{'store_name': 'demo-mobile/telemetry/telemetry_US_VZW.csv', 'URL': 'https://datalogue-demo.s3.amazonaws.com/mobile/telemetry/telemetry_US_VZW.csv', 'datastore_object': Datastore(id: da4074c8-5d50-439e-90f1-8eea01ed6831, name: 'demo-mobile/telemetry/telemetry_US_VZW.csv', alias: None, credential_id: None, definition: <datalogue.models.datastore.HttpDatastoreDef object at 0x0000024248EB8F60>, samples: None, schema_paths: [], schema_labels: [], schema_nodes: None)}<jupyter_text>3b. Create datastore for RDBMS target<jupyter_code># host: 172.16.58.3 (use jdbc:postgresql://172.16.58.3:5432/demo for creating target store)
# user: postgres
# pw: L8am0pO5zjJrFm2O

# bug in SDK for v<1.0; to be updated here but created in GUI for now<jupyter_output><empty_output><jupyter_text>4. Collecting data stores into a collectionThis is just used for organization, and uses the command `dtl.datastore_collection.create`.<jupyter_code>npl_collection = DatastoreCollection(
  name ="demo-Telemetry Collection",
  storeIds = [Datastore["datastore_object"].id for Datastore in npl_data],
  description = "Global Handset Telemetry Data"
)

npl_collection2 = dtl.datastore_collection.create(npl_collection)<jupyter_output><empty_output><jupyter_text>5. Creating a stream<jupyter_code># Check if any existing now 
#for stream in dtl.stream_collection.list():
#    print('Stream name: ', stream.name )
#    print('Stream ID: ', stream.id)
#print('-----------------------')
#print('Total of ', len(dtl.stream_collection.list()), ' stream(s).')


#set target of stream:

my_output_store = dtl.datastore.get("9c0b89e6-0e27-4f89-8c8d-e521b4b424cd")

print(my_output_store)
print(type(my_output_store))
print(my_output_store)<jupyter_output><class 'datalogue.models.datastore.Datastore'>
Datastore(id: 9c0b89e6-0e27-4f89-8c8d-e521b4b424cd, name: 'dtl-demo telemetry', alias: None, credential_id: <KEY>, definition: <datalogue.models.datastore.JdbcDatastoreDef object at 0x0000024248EFD4E0>, samples: None, schema_paths: [], schema_labels: [], schema_nodes: None)<jupyter_text>Sample pipeline<jupyter_code># Define the target output schema transformation using 'structure'

std_schema = Structure([
        ClassNodeDescription(
            path = ["Model"],
            tag = "Model",
            pick_strategy = PickStrategy.HighScore,
            data_type = DataType.String
        ),
        ClassNodeDescription(
            path = ["Manufacturer"],
            tag = "Manufacturer",
            pick_strategy = PickStrategy.HighScore,
            data_type = DataType.String
        ),
        ClassNodeDescription(
            path = ["OS"],
            tag = "OS",
            pick_strategy = PickStrategy.HighScore,
            data_type = DataType.String
        ),
        ClassNodeDescription(
            path = ["Generation"],
            tag = "Generation",
            pick_strategy = PickStrategy.HighScore,
            data_type = DataType.String
        ),
        ClassNodeDescription(
            path = ["Language"],
            tag = "Language",
            pick_strategy = PickStrategy.HighScore,
            data_type = DataType.String
        ),
        ClassNodeDescription(
            path = ["Timestamp"],
            tag = "Timestamp",
            pick_strategy = PickStrategy.HighScore,
            data_type = DataType.String
        ),
        ClassNodeDescription(
            path = ["Signal_Strength"],
            tag = "Signal_Strength",
            pick_strategy = PickStrategy.HighScore,
            data_type = DataType.String
        ),
        ClassNodeDescription(
            path = ["Latitude"],
            tag = "Latitude",
            pick_strategy = PickStrategy.HighScore,
            data_type = DataType.String
        ),
        ClassNodeDescription(
            path = ["Transmitted_Bytes"],
            tag = "Transmitted_Bytes",
            pick_strategy = PickStrategy.HighScore,
            data_type = DataType.String
        ),
        ClassNodeDescription(
            path = ["Longitude"],
            tag = "Longitude",
            pick_strategy = PickStrategy.HighScore,
            data_type = DataType.String
        ),
        ClassNodeDescription(
            path = ["Received_Bytes"],
            tag = "Received_Bytes",
            pick_strategy = PickStrategy.HighScore,
            data_type = DataType.String
        ),
        ClassNodeDescription(
            path = ["Region"],
            tag = "Region",
            pick_strategy = PickStrategy.HighScore,
            data_type = DataType.String
        ),
        ClassNodeDescription(
            path = ["City"],
            tag = "City",
            pick_strategy = PickStrategy.HighScore,
            data_type = DataType.String
        ),
        ClassNodeDescription(
            path = ["Country"],
            tag = "Country",
            pick_strategy = PickStrategy.HighScore,
            data_type = DataType.String
        ),
        ClassNodeDescription(
            path = ["Provider"],
            tag = "Provider",
            pick_strategy = PickStrategy.HighScore,
            data_type = DataType.String
        )
    ]
)
from datalogue.models.training import *
import uuid

modelUuid = dtl.training.get_trainings(uuid.UUID("9b39556e-d2d5-48c6-9a70-dec37579bf6e"))[0].id

print(type(modelUuid))
print(modelUuid)

# Define classify transformation

tx_definition = Definition(    # (List[Transformation], pipelines: List['Definition'], target_datastore )
            [
                Classify(training_id = modelUuid, use_context=True, include_classes=False, include_scores=False),
                std_schema
            ], # List of transformations
        [], # pipelines list
            my_output_store, # target_datastore
        )

print(type(tx_definition))
print(tx_definition)
# Define n stream(s), where n is number of datastore connections created from S3 bucket scan
n = len(current_stores)
i = 1

list_of_streams = []
for i in range(n):
    stream = Stream(current_stores[i], [tx_definition])
    i += 1
    list_of_streams.append(stream)

print(type(list_of_streams))
print(type(list_of_streams[0]))    
print(list_of_streams[0])
print(type(current_stores[0]))
print(current_stores[0])
print("\n")

print(len(list_of_streams))
# search the existing stream collections
#x=dtl.stream_collection.list()
#scid = ''
#for i in x:
#    if i.name == 'MFGDemoPipeline':
#        print(i, "\n")
#        scid = i.id
#print("deleting stream ID: ", scid)

# delete the existing stream collection if it exists
#dtl.stream_collection.delete(stream_collection_id = scid)
# print(type(stream1))
# Put the streams in a collection

stream_collection = dtl.stream_collection.create(
    list_of_streams,
    'Telemetry Demo Pipeline'
)

print(type(stream_collection))
print(stream_collection)
# Run the Collection

dtl.stream_collection.run(stream_collection.id)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11295366246564958, 'fraction_numerical': 0.017814839382166207, 'mean_word_length': 3.3254098360655737, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 33, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 3, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '346810', 'license': 'MIT', 'n_tokens_mistral': 3418, 'n_tokens_neox': 3322, 'n_words': 715, 'path': 'handset.ipynb', 'repo_name': 'clreimer/demo-data', 'size': '10633'}