import gradio as gr import spacy from spacy import displacy from cefrpy import CEFRSpaCyAnalyzer, CEFRLevel MODEL = "en_core_web_sm" ALL_ENTS = [ 'CARDINAL', 'DATE', 'EVENT', 'FAC', 'GPE', 'LANGUAGE', 'LAW', 'LOC', 'MONEY', 'NORP', 'ORDINAL', 'ORG', 'PERCENT', 'PERSON', 'PRODUCT', 'QUANTITY', 'TIME', 'WORK_OF_ART' ] DEFAULT_ENTITY_ITEMS_TO_SKIP = [ 'QUANTITY', 'MONEY', 'LANGUAGE', 'LAW', 'WORK_OF_ART', 'PRODUCT', 'GPE', 'ORG', 'FAC', 'PERSON' ] TOKEN_ATTRIBUTES = [ "Token", "POS", "Skipped", "Level", "Start", "End" ] WORDLIST_HEADER = ["Word", "Pos", "CEFR", "Level"] DEFAULT_WORDLIST_SLIDER_LEVEL = 4.0 DEFAULT_TEXT = """The world's oldest known recipe is for beer. It dates back to around 5,000 BC and was found in ancient Sumeria (modern-day Iraq). Due to thermal expansion, the iron structure of the Eiffel Tower can expand in hot weather, making the tower grow by up to 6 inches (15 centimeters) in height. Did you know that the word "antidisestablishmentarianism" is often cited as one of the longest non-technical words in the English language? It originated in the 19th century in Britain during debates over the disestablishment of the Church of England, and it refers to the opposition to the withdrawal of state support for an established church. This word has gained notoriety for its length and has been used as a challenge for spelling bees and word enthusiasts alike. In 2006, a Coca-Cola employee offered to sell Coca-Cola secrets to Pepsi. Pepsi responded by notifying Coca-Cola, and the FBI set up a sting operation to catch the culprit. Like humans, cows form strong social bonds and often have "best friends" within their herds. They display complex social behaviors, including grooming, playing, and even grieving when separated from their friends.""" DISPLACY_RENDER_OPTIONS = { "colors": { "A1": "#b0c4de", "A2": "#87ceeb", "B1": "#90ee90", "B2": "#adff2f", "C1": "#ffd700", "C2": "#ff9380", "SKIP": "#ffafed", "UNKNOWN": "#BCAAA4" } } ABBREVIATION_MAPPING = { "'m": "am", "'s": "is", "'re": "are", "'ve": "have", "'d": "had", "n't": "not", "'ll": "will" } LINKS_HTML = """

 Github: link
 Docs: link

""" CSS = """ h1 { padding-top: 5px; text-align: center; display:block; } """ nlp = spacy.load(MODEL) def get_dict_ents(text: str, tokens: list[tuple[str, str, bool, float, int, int]]) -> dict: ents = [] for token in tokens: if token[3]: ents.append({ "start": token[4], "end": token[5], "label": str(CEFRLevel(round(token[3]))) }) elif token[0].isalpha(): ents.append({ "start": token[4], "end": token[5], "label": "SKIP" if token[2] else "UNKNOWN" }) dict_ents = { "text": text, "ents": ents } return dict_ents def get_cefr_tokens(text: str, ents_to_skip: list[str]) -> list[tuple[str, str, bool, float, int, int]]: doc = nlp(text) text_analyzer = CEFRSpaCyAnalyzer(entity_types_to_skip=ents_to_skip, abbreviation_mapping=ABBREVIATION_MAPPING) tokens = text_analyzer.analize_doc(doc) return tokens def get_html_visualization(text: str, tokens: list[tuple[str, str, bool, float, int, int]]) -> str: dict_ents = get_dict_ents(text, tokens) html = displacy.render(dict_ents, manual=True, style="ent", options=DISPLACY_RENDER_OPTIONS) return html def get_wordlist_set(tokens: list[tuple[str, str, bool, float, int, int]], min_level: float) -> set[tuple[str, str, bool, float, int, int]]: filtered_tokens = set() for word, pos, _, level, _, _ in tokens: if level and level >= min_level: filtered_tokens.add((word.lower(), pos, str(CEFRLevel(round(level))), level)) return filtered_tokens def get_wordlist(tokens: list[tuple[str, str, bool, float, int, int]], min_level: float): wordlist_set = get_wordlist_set(tokens, min_level) wordlist = list(wordlist_set) wordlist.sort() return wordlist def get_wordlist_from_dataframe(dataframe, min_level: float): return get_wordlist(dataframe.values, min_level) def process_text(text: str, ents_to_skip: list[str] | None = DEFAULT_ENTITY_ITEMS_TO_SKIP, min_level: float = DEFAULT_WORDLIST_SLIDER_LEVEL) -> tuple[list[list], str]: tokens = get_cefr_tokens(text, ents_to_skip) html = get_html_visualization(text, tokens) wordlist = get_wordlist(tokens, min_level) return tokens, wordlist, html initial_tokens, initial_wordlist, initial_html = process_text(DEFAULT_TEXT) demo = gr.Blocks(css=CSS) with demo: with gr.Row(): with gr.Column(): with gr.Column(): with gr.Row(): gr.Markdown("# Gradio Demo: cefrpy") gr.HTML(LINKS_HTML) with gr.Row(): text_input = gr.TextArea( value=DEFAULT_TEXT, interactive=True, max_lines=500, label="Input Text", show_copy_button=True ) with gr.Row(): ent_input = gr.CheckboxGroup( ALL_ENTS, value=DEFAULT_ENTITY_ITEMS_TO_SKIP, label="Entity types to skip CEFR" ) with gr.Row(): clear_button = gr.ClearButton(text_input) render_button = gr.Button( "Render", variant="primary" ) with gr.Column(): with gr.Row(): gr.Markdown("# Words CEFR level visualization") with gr.Row(): rendered_html = gr.HTML(initial_html) with gr.Row(): with gr.Column(): with gr.Row(): tokens_output = gr.Dataframe(headers=TOKEN_ATTRIBUTES, value=initial_tokens, interactive=False) with gr.Column(): with gr.Row(): min_level_slider = gr.Slider( minimum=1.0, maximum=6.0, value=DEFAULT_WORDLIST_SLIDER_LEVEL, step=0.02, interactive=True, label="Min level to generate word list" ) with gr.Row(): wordlist = gr.Dataframe(headers=WORDLIST_HEADER, value=initial_wordlist, interactive=False) render_button.click( process_text, inputs=[text_input, ent_input], outputs=[tokens_output, wordlist, rendered_html], api_name="process_text" ) min_level_slider.release( get_wordlist_from_dataframe, inputs=[tokens_output, min_level_slider], outputs=[wordlist], api_name=False ) demo.launch(show_api=True)