<jupyter_start><jupyter_text>Elastic engine demo<jupyter_code>import datalabframework as dlf
from datalabframework import utils
dlf.project.info()
metadata = dlf.params.metadata()
utils.pretty_print(metadata)
import datalabframework as dlf
engine = dlf.engines.get('spark-local')
res = engine.read(".keywords_search")
res.show()
dlf.project.repository()
type(res)
df = engine.read(".keywords")
df.show()
# import datalabframework as dlf
# from datalabframework import utils
# engine = dlf.engines.get('spark-local')
# df = engine.read(".keywords")
engine.write(df, ".keywords_elk", mode='overwrite')<jupyter_output>{'index': {'mapping': {'total_fields': {'limit': 1024}}, 'number_of_replicas': 3, 'number_of_shards': 1}}
{'api_result': {'type': 'double'}, 'count': {'type': 'integer'}, 'gg': {'type': 'integer'}, 'gg_clicks': {'type': 'double'}, 'gg_impressions': {'type': 'double'}, 'gg_position': {'type': 'double'}, 'pv': {'type': 'integer'}, 'pv_impressions': {'type': 'double'}, 'pv_position': {'type': 'double'}, 'pv_result': {'type': 'double'}, 'query': {'fields': {'keyword': {'ignore_above': 128, 'type': 'keyword'}}, 'type': 'text'}, 'query_wo_tones': {'fields': {'keyword': {'ignore_above': 128, 'type': 'keyword'}}, 'type': 'text'}}
2018-10-16 15:35:32,562 - INFO - f318bdd - datalabframework-demos.git - thuc - elastic.ipynb - engine.write - {'format': 'parquet', 'service': 'elastic', 'path': 'search_keywords_dev2', 'url': 'http://192.168.3.11:9200/search_keywords_dev2'}
{'fraction_non_alphanumeric': 0.22476446837146702, 'fraction_numerical': 0.031628532974428, 'mean_word_length': 7.545977011494253, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '460868', 'license': 'MIT', 'n_tokens_mistral': 557, 'n_tokens_neox': 513, 'n_words': 115, 'path': 'demos/elastic/elastic.ipynb', 'repo_name': 'ncthuc/datalabframework-demos', 'size': '1487'}