<jupyter_start><jupyter_text>Load NER data into ElasticsearchThe Gretel Console (https://console.gretel.cloud) provides several views and tools to help explore your data.  For those who cannot or do not wish to use the Console, another option is to load the data into Elasticsearch for exploration and reporting.  This blueprint uses Gretel sample data to bootstrap a new project and shows a simple workflow for loading that data into an Elasticsearch cluster.  We run a sample query to verify that records were loaded and show how to look for an NER label in your records.<jupyter_code>!docker-compose up -d
%%capture

import sys

# Elasticsearch client version should match cluster version.  See https://elasticsearch-py.readthedocs.io/en/master/#compatibility
!{sys.executable} -m pip install -U gretel-client elasticsearch
# Be sure to use your Gretel API key here, which is available from the Profile menu in the Console

import getpass
import os

gretel_api_key = os.getenv("GRETEL_API_KEY") or getpass.getpass("Your Gretel API Key")
# Install Gretel SDKs and bootstrap the project

from gretel_client import project_from_uri, get_cloud_client

client = get_cloud_client("api", gretel_api_key)
client.install_packages()
project = client.get_project(create=True)

project.send_bulk(client.get_sample('bike-customer-orders'))
from copy import deepcopy

index_name = "gretel_ner_blueprint"

def trim_record(record):
    trim = {}
    # Keep the original record
    trim['record'] = deepcopy(record['record'])
    # Keep ingest_time for time series
    trim['ingest_time'] = record['ingest_time']
    # Keep just score_* lists for simplicity
    trim['score_high'] = record['metadata']['entities']['score_high']
    trim['score_med'] = record['metadata']['entities']['score_med']
    trim['score_low'] = record['metadata']['entities']['score_low']
    # Specify the Elasticsearch index for the record.
    trim['_index'] = index_name
    return trim

from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk

elasticsearch_host = 'localhost'
elasticsearch_port = 9200
es = Elasticsearch(
    hosts=[{'host': elasticsearch_host, 'port': elasticsearch_port}]
)

bulk(
    es, 
    project.iter_records(
        # NOTE that default direction is "forward".  Client will block and wait for new records.
        direction="backward",
        post_process=trim_record, 
        params={"flatten": "no"}))

# Find records tagged with the NER 'location' label.  They must also be in London with 3 or more cars.
# Include an aggregation showing the number of children.
aggregation_query = {
    "query": {
        "bool": {
            "must": [
                {"match": {"record.City": "London"}}, 
                {"range": {"record.NumberCarsOwned": {"gte": 3}}},
                {"match": {"score_high": "location"}}
            ]
        }
    },
    "size": 3,
    "aggs" : {
        "children": {
            "terms": {"field": "record.TotalChildren"}
        }
    }
}

es.search(index=index_name, body=aggregation_query)
# Clean up
!docker-compose down
project.delete()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10470701248799232, 'fraction_numerical': 0.002241434518091579, 'mean_word_length': 3.911949685534591, 'pattern_counts': {'":': 15, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '575145', 'license': 'Apache-2.0', 'n_tokens_mistral': 916, 'n_tokens_neox': 905, 'n_words': 300, 'path': 'gretel/gc-load_ner_to_elasticsearch/blueprint.ipynb', 'repo_name': 'Ab-Abdurrahman/gretel-blueprints', 'size': '3123'}