import time from functools import partial from typing import Iterator import gradio as gr import requests.exceptions from huggingface_hub import InferenceClient model_id = "microsoft/Phi-3-mini-4k-instruct" client = InferenceClient(model_id) GENERATE_DATASET_NAMES_FOR_SEARCH_QUERY = ( "A Machine Learning Practioner is looking for a dataset that matches '{search_query}'. " "Generate a list of 10 names of quality dataset that don't exist but sound plausible and would " "be helpful. Feel free to reuse words from the query '{search_query}' to name the datasets. " "Every dataset should be about '{search_query}' and have descriptive tags/keywords including the ML task name associated to the dataset (classification, regression, anomaly detection, etc.). Use the following format:\n1. DatasetName1 (tag1, tag2, tag3)\n1. DatasetName2 (tag1, tag2, tag3)" ) GENERATE_DATASET_CONTENT_FOR_SEARCH_QUERY_AND_NAME_AND_TAGS = ( "A ML practitioner is looking for a dataset CSV after the query '{search_query}'. " "Generate the first 5 rows of a plausible and quality CSV for the dataset '{dataset_name}'. " "You can get inspiration from related keywords '{tags}' but most importantly the dataset should correspond to the query '{search_query}'. " "Focus on quality text content and and use a 'label' or 'labels' column if it makes sense (invent labels, avoid reusing the keywords, be accurate while labelling texts). " "Reply using a short description of the dataset with title **Dataset Description:** followed by the CSV content in a code block and with title **CSV Content Preview:**." ) default_query = "various datasets on many different subjects and topics, from classification to language modeling, from science to sport to finance to news" def stream_reponse(msg: str, max_tokens=500) -> Iterator[str]: for _ in range(3): try: for message in client.chat_completion( messages=[{"role": "user", "content": msg}], max_tokens=max_tokens, stream=True, ): yield message.choices[0].delta.content except requests.exceptions.ConnectionError as e: print(e + "\n\nRetrying in 1sec") time.sleep(1) continue break def gen_datasets(search_query: str) -> Iterator[str]: search_query = search_query[:1000] if search_query.strip() else default_query generated_text = "" for token in stream_reponse(GENERATE_DATASET_NAMES_FOR_SEARCH_QUERY.format(search_query=search_query)): generated_text += token if generated_text.endswith("\n"): yield generated_text.strip() yield generated_text.strip() print("-----\n\n" + generated_text) def gen_dataset_content(search_query: str, dataset_name: str, tags: str) -> Iterator[str]: search_query = search_query[:1000] if search_query.strip() else default_query generated_text = "" for token in stream_reponse(GENERATE_DATASET_CONTENT_FOR_SEARCH_QUERY_AND_NAME_AND_TAGS.format( search_query=search_query, dataset_name=dataset_name, tags=tags, ), max_tokens=1500): generated_text += token yield generated_text print("-----\n\n" + generated_text) NB_ITEMS_PER_PAGE = 10 default_output = """ 1. NewsEventsPredict (classification, media, trend) 2. FinancialForecast (economy, stocks, regression) 3. HealthMonitor (science, real-time, anomaly detection) 4. SportsAnalysis (classification, performance, player tracking) 5. SciLiteracyTools (language modeling, science literacy, text classification) 6. RetailSalesAnalyzer (consumer behavior, sales trend, segmentation) 7. SocialSentimentEcho (social media, emotion analysis, clustering) 8. NewsEventTracker (classification, public awareness, topical clustering) 9. HealthVitalSigns (anomaly detection, biometrics, prediction) 10. GameStockPredict (classification, finance, sports contingency) """.strip().split("\n") assert len(default_output) == NB_ITEMS_PER_PAGE css = """ .datasetButton { justify-content: start; justify-content: left; } .tags { font-size: var(--button-small-text-size); color: var(--body-text-color-subdued); } a { color: var(--body-text-color); } .topButton { justify-content: start; justify-content: left; text-align: left; background: transparent; box-shadow: none; padding-bottom: 0; } .topButton::before { content: url("data:image/svg+xml,%3Csvg style='color: rgb(209 213 219)' xmlns='http://www.w3.org/2000/svg' xmlns:xlink='http://www.w3.org/1999/xlink' aria-hidden='true' focusable='false' role='img' width='1em' height='1em' preserveAspectRatio='xMidYMid meet' viewBox='0 0 25 25'%3E%3Cellipse cx='12.5' cy='5' fill='currentColor' fill-opacity='0.25' rx='7.5' ry='2'%3E%3C/ellipse%3E%3Cpath d='M12.5 15C16.6421 15 20 14.1046 20 13V20C20 21.1046 16.6421 22 12.5 22C8.35786 22 5 21.1046 5 20V13C5 14.1046 8.35786 15 12.5 15Z' fill='currentColor' opacity='0.5'%3E%3C/path%3E%3Cpath d='M12.5 7C16.6421 7 20 6.10457 20 5V11.5C20 12.6046 16.6421 13.5 12.5 13.5C8.35786 13.5 5 12.6046 5 11.5V5C5 6.10457 8.35786 7 12.5 7Z' fill='currentColor' opacity='0.5'%3E%3C/path%3E%3Cpath d='M5.23628 12C5.08204 12.1598 5 12.8273 5 13C5 14.1046 8.35786 15 12.5 15C16.6421 15 20 14.1046 20 13C20 12.8273 19.918 12.1598 19.7637 12C18.9311 12.8626 15.9947 13.5 12.5 13.5C9.0053 13.5 6.06886 12.8626 5.23628 12Z' fill='currentColor'%3E%3C/path%3E%3C/svg%3E"); margin-right: .25rem; margin-left: -.125rem; margin-top: .25rem; } .bottomButton { justify-content: start; justify-content: left; text-align: left; background: transparent; box-shadow: none; font-size: var(--button-small-text-size); color: var(--body-text-color-subdued); padding-top: 0; align-items: baseline; } .bottomButton::before { content: 'tags:'; margin-right: .25rem; } .buttonsGroup { background: transparent; } .buttonsGroup:hover { background: var(--input-background-fill); } .buttonsGroup div { background: transparent; } @keyframes placeHolderShimmer{ 0%{ background-position: -468px 0 } 100%{ background-position: 468px 0 } } .linear-background { animation-duration: 1s; animation-fill-mode: forwards; animation-iteration-count: infinite; animation-name: placeHolderShimmer; animation-timing-function: linear; background-image: linear-gradient(to right, var(--body-text-color-subdued) 8%, #dddddd11 18%, var(--body-text-color-subdued) 33%); background-size: 1000px 104px; color: transparent; background-clip: text; } """ def search_datasets(search_query): output_values = [ gr.Button("⬜⬜⬜⬜⬜⬜", elem_classes="topButton linear-background"), gr.Button("░░░░, ░░░░, ░░░░", elem_classes="bottomButton linear-background") ] * NB_ITEMS_PER_PAGE for generated_text in gen_datasets(search_query): if "I'm sorry" in generated_text: raise gr.Error("Error: inappropriate content") lines = [line for line in generated_text.split("\n") if line and line.split(".", 1)[0].isnumeric()][:NB_ITEMS_PER_PAGE] for i, line in enumerate(lines): dataset_name, tags = line.split(".", 1)[1].strip(" )").split(" (", 1) output_values[2 * i] = gr.Button(dataset_name, elem_classes="topButton") output_values[2 * i + 1] = gr.Button(tags, elem_classes="bottomButton") yield output_values def show_dataset(search_query, *buttons_values, i): dataset_name, tags = buttons_values[2 * i : 2 * i + 2] dataset_title = f"# {dataset_name}\n\n tags: {tags}\n\n _Note: This is an AI-generated dataset so its content may be inaccurate or false_" yield gr.Column(visible=False), gr.Column(visible=True), dataset_title, "" for generated_text in gen_dataset_content(search_query=search_query, dataset_name=dataset_name, tags=tags): yield gr.Column(), gr.Column(), dataset_title, generated_text def show_search_page(): return gr.Column(visible=True), gr.Column(visible=False) def generate_full_dataset(): raise gr.Error("Not implemented yet sorry ! Give me some feedbacks in the Community tab in the meantime ;)") with gr.Blocks(css=css) as demo: with gr.Row(): with gr.Column(scale=4, min_width=0): pass with gr.Column(scale=10): gr.Markdown( "# 🤗 Infinite Dataset Hub ♾️\n\n" "An endless catalog of datasets, created just for you.\n\n" ) with gr.Column(scale=4, min_width=0): pass with gr.Column() as search_page: with gr.Row(): with gr.Column(scale=4, min_width=0): pass with gr.Column(scale=9): search_bar = gr.Textbox(max_lines=1, placeholder="Search datasets, get infinite results", show_label=False, container=False) with gr.Column(min_width=64): search_button = gr.Button("🔍", variant="primary") with gr.Column(scale=4, min_width=0): pass inputs = [search_bar] show_dataset_outputs = [search_page] with gr.Row(): with gr.Column(scale=4, min_width=0): pass with gr.Column(scale=10): buttons = [] for i in range(10): line = default_output[i] dataset_name, tags = line.split(".", 1)[1].strip(" )").split(" (", 1) with gr.Group(elem_classes="buttonsGroup"): top = gr.Button(dataset_name, elem_classes="topButton") bottom = gr.Button(tags, elem_classes="bottomButton") buttons += [top, bottom] top.click(partial(show_dataset, i=i), inputs=inputs, outputs=show_dataset_outputs) bottom.click(partial(show_dataset, i=i), inputs=inputs, outputs=show_dataset_outputs) inputs += buttons gr.Markdown(f"_powered by [{model_id}](https://huggingface.co/{model_id})_") with gr.Column(scale=4, min_width=0): pass search_bar.submit(search_datasets, inputs=search_bar, outputs=buttons) search_button.click(search_datasets, inputs=search_bar, outputs=buttons) with gr.Column(visible=False) as dataset_page: with gr.Row(): with gr.Column(scale=4, min_width=0): pass with gr.Column(scale=10): dataset_title = gr.Markdown() dataset_content = gr.Markdown() with gr.Row(): with gr.Column(scale=4, min_width=0): pass with gr.Column(): generate_full_dataset_button = gr.Button("Generate Full Dataset", variant="primary") generate_full_dataset_button.click(generate_full_dataset) back_button = gr.Button("< Back", size="sm") back_button.click(show_search_page, inputs=[], outputs=[search_page, dataset_page]) with gr.Column(scale=4, min_width=0): pass with gr.Column(scale=4, min_width=0): pass show_dataset_outputs += [dataset_page, dataset_title, dataset_content] demo.launch()