initial commit
Browse files
app.py
CHANGED
@@ -44,143 +44,144 @@ def main(
|
|
44 |
"""
|
45 |
<div align="center">
|
46 |
<a href="https://sunglasses-ai.github.io/classy/">
|
47 |
-
<img alt="Python" style="height: 3em; margin:
|
48 |
</a>
|
49 |
<a href="https://spacy.io/" tyle="text-decoration: none">
|
50 |
-
<img alt="spaCy" style="height: 3em; margin:
|
51 |
</a>
|
52 |
</div>
|
53 |
""",
|
54 |
unsafe_allow_html=True,
|
55 |
)
|
56 |
|
57 |
-
#
|
58 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
59 |
"""
|
60 |
-
|
61 |
-
|
62 |
-
Link to full paper: https://www.researchgate.net/publication/359392427_ExtEnD_Extractive_Entity_Disambiguation
|
63 |
-
Link to GitHub paper: https://github.com/SapienzaNLP/extend
|
64 |
-
"""
|
65 |
-
)
|
66 |
-
st.markdown("""
|
67 |
-
## How it works
|
68 |
-
|
69 |
-
ExtEnD frames Entity Disambiguation as a text extraction problem:
|
70 |
-
""")
|
71 |
-
st.image(
|
72 |
-
"data/repo-assets/extend_formulation.png", caption="ExtEnD Formulation"
|
73 |
-
)
|
74 |
-
st.markdown(
|
75 |
-
"""
|
76 |
-
Given the sentence *After a long fight Superman saved Metropolis*, where *Superman* is the mention
|
77 |
-
to disambiguate, ExtEnD first concatenates the descriptions of all the possible candidates of *Superman* in the
|
78 |
-
inventory and then selects the span whose description best suits the mention in its context.
|
79 |
-
|
80 |
-
To convert this task to end2end entity linking, as we do in *Model demo*, we leverage spaCy
|
81 |
-
(more specifically, its NER) and run ExtEnD on each named entity spaCy identifies
|
82 |
-
(if the corresponding mention is contained in the inventory).
|
83 |
-
"""
|
84 |
-
)
|
85 |
|
86 |
# demo
|
87 |
-
|
88 |
-
|
89 |
-
|
90 |
-
|
91 |
-
|
92 |
-
|
93 |
-
|
94 |
-
|
95 |
-
|
96 |
-
|
97 |
-
|
98 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
99 |
)
|
100 |
-
|
101 |
|
102 |
-
#
|
103 |
-
|
104 |
-
|
105 |
-
|
106 |
-
|
107 |
-
|
108 |
-
# read input
|
109 |
-
placeholder = st.selectbox(
|
110 |
-
"Examples",
|
111 |
-
options=examples,
|
112 |
-
index=0,
|
113 |
-
)
|
114 |
-
input_text = st.text_area("Input text to entity-disambiguate", placeholder)
|
115 |
-
|
116 |
-
# custom inventory
|
117 |
-
uploaded_inventory_path = st.file_uploader(
|
118 |
-
"[Optional] Upload custom inventory (tsv file, mention \\t desc1 \\t desc2 \\t)",
|
119 |
-
accept_multiple_files=False,
|
120 |
-
type=["tsv"],
|
121 |
-
)
|
122 |
-
if uploaded_inventory_path is not None:
|
123 |
-
inventory_path = f"data/inventories/{uploaded_inventory_path.name}"
|
124 |
-
with open(inventory_path, "wb") as f:
|
125 |
-
f.write(uploaded_inventory_path.getbuffer())
|
126 |
-
else:
|
127 |
-
inventory_path = default_inventory_path
|
128 |
-
|
129 |
-
# load model and color generator
|
130 |
-
nlp = load_resources(inventory_path)
|
131 |
-
color_generator = get_md_200_random_color_generator()
|
132 |
-
|
133 |
-
if st.button("Disambiguate", key="classify"):
|
134 |
-
|
135 |
-
# tag sentence
|
136 |
-
time_start = time.perf_counter()
|
137 |
-
doc = nlp(input_text)
|
138 |
-
time_end = time.perf_counter()
|
139 |
-
|
140 |
-
# extract entities
|
141 |
-
entities = {}
|
142 |
-
for ent in doc.ents:
|
143 |
-
if ent._.disambiguated_entity is not None:
|
144 |
-
entities[ent.start_char] = (
|
145 |
-
ent.start_char,
|
146 |
-
ent.end_char,
|
147 |
-
ent.text,
|
148 |
-
ent._.disambiguated_entity,
|
149 |
-
)
|
150 |
-
|
151 |
-
# create annotated html components
|
152 |
-
|
153 |
-
annotated_html_components = []
|
154 |
-
|
155 |
-
assert all(any(t.idx == _s for t in doc) for _s in entities)
|
156 |
-
it = iter(list(doc))
|
157 |
-
while True:
|
158 |
-
try:
|
159 |
-
t = next(it)
|
160 |
-
except StopIteration:
|
161 |
-
break
|
162 |
-
if t.idx in entities:
|
163 |
-
_start, _end, _text, _entity = entities[t.idx]
|
164 |
-
while t.idx + len(t) != _end:
|
165 |
-
t = next(it)
|
166 |
-
annotated_html_components.append(
|
167 |
-
str(annotation(*(_text, _entity, color_generator())))
|
168 |
-
)
|
169 |
-
else:
|
170 |
-
annotated_html_components.append(str(html.escape(t.text)))
|
171 |
-
|
172 |
-
st.markdown(
|
173 |
-
"\n".join(
|
174 |
-
[
|
175 |
-
"<div>",
|
176 |
-
*annotated_html_components,
|
177 |
-
"<p></p>"
|
178 |
-
f'<div style="text-align: right"><p style="color: gray">Time: {(time_end - time_start):.2f}s</p></div>'
|
179 |
-
"</div>",
|
180 |
-
]
|
181 |
-
),
|
182 |
-
unsafe_allow_html=True,
|
183 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
184 |
|
185 |
|
186 |
|
|
|
44 |
"""
|
45 |
<div align="center">
|
46 |
<a href="https://sunglasses-ai.github.io/classy/">
|
47 |
+
<img alt="Python" style="height: 3em; margin: 0em 1em 2em 1em;" src="data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0idXRmLTgiPz4NCjwhLS0gR2VuZXJhdG9yOiBBZG9iZSBJbGx1c3RyYXRvciAxNy4wLjAsIFNWRyBFeHBvcnQgUGx1Zy1JbiAuIFNWRyBWZXJzaW9uOiA2LjAwIEJ1aWxkIDApICAtLT4NCjwhRE9DVFlQRSBzdmcgUFVCTElDICItLy9XM0MvL0RURCBTVkcgMS4xLy9FTiIgImh0dHA6Ly93d3cudzMub3JnL0dyYXBoaWNzL1NWRy8xLjEvRFREL3N2ZzExLmR0ZCI+DQo8c3ZnIHZlcnNpb249IjEuMSIgaWQ9IkxpdmVsbG9fMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayIgeD0iMHB4IiB5PSIwcHgiDQoJIHdpZHRoPSI0MzcuNnB4IiBoZWlnaHQ9IjQxMy45NzhweCIgdmlld0JveD0iMCAwIDQzNy42IDQxMy45NzgiIGVuYWJsZS1iYWNrZ3JvdW5kPSJuZXcgMCAwIDQzNy42IDQxMy45NzgiIHhtbDpzcGFjZT0icHJlc2VydmUiPg0KPGc+DQoJPHBhdGggZmlsbD0iI0ZGQ0MwMCIgZD0iTTM5NC42NjgsMTczLjAzOGMtMS4wMTgsMC0yLjAyLDAuMDY1LTMuMDE1LDAuMTUyQzM3NS42NjUsODIuODExLDI5Ni43OSwxNC4xNDYsMjAxLjgyMywxNC4xNDYNCgkJQzk1LjMxOSwxNC4xNDYsOC45OCwxMDAuNDg1LDguOTgsMjA2Ljk4OWMwLDEwNi41MDUsODYuMzM5LDE5Mi44NDMsMTkyLjg0MywxOTIuODQzYzk0Ljk2NywwLDE3My44NDItNjguNjY1LDE4OS44MjktMTU5LjA0NA0KCQljMC45OTUsMC4wODcsMS45OTcsMC4xNTIsMy4wMTUsMC4xNTJjMTguNzUxLDAsMzMuOTUyLTE1LjIsMzMuOTUyLTMzLjk1MkM0MjguNjIsMTg4LjIzOSw0MTMuNDE5LDE3My4wMzgsMzk0LjY2OCwxNzMuMDM4eg0KCQkgTTIwMS44MjMsMzQ2Ljg2OWMtNzcuMTMsMC0xMzkuODgtNjIuNzUtMTM5Ljg4LTEzOS44OGMwLTc3LjEyOSw2Mi43NS0xMzkuODc5LDEzOS44OC0xMzkuODc5czEzOS44OCw2Mi43NSwxMzkuODgsMTM5Ljg3OQ0KCQlDMzQxLjcwMywyODQuMTE5LDI3OC45NTMsMzQ2Ljg2OSwyMDEuODIzLDM0Ni44Njl6Ii8+DQoJPGc+DQoJCTxwYXRoIGZpbGw9IiNGRkNDMDAiIGQ9Ik0xMTQuOTA3LDIzMy40NzNjLTE0LjYyNiwwLTI2LjQ4My0xMS44NTYtMjYuNDgzLTI2LjQ4M2MwLTYyLjUyOCw1MC44NzEtMTEzLjQwMiwxMTMuMzk4LTExMy40MDINCgkJCWMxNC42MjYsMCwyNi40ODMsMTEuODU2LDI2LjQ4MywyNi40ODNzLTExLjg1NiwyNi40ODMtMjYuNDgzLDI2LjQ4M2MtMzMuMzI0LDAtNjAuNDMzLDI3LjExMi02MC40MzMsNjAuNDM2DQoJCQlDMTQxLjM5LDIyMS42MTcsMTI5LjUzNCwyMzMuNDczLDExNC45MDcsMjMzLjQ3M3oiLz4NCgk8L2c+DQo8L2c+DQo8L3N2Zz4NCg==">
|
48 |
</a>
|
49 |
<a href="https://spacy.io/" tyle="text-decoration: none">
|
50 |
+
<img alt="spaCy" style="height: 3em; margin: 0em 1em 2em 1em;" src="data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgOTAwIDUwMCAxNzUiIHdpZHRoPSIxNTAiIGhlaWdodD0iNTMiPjxwYXRoIGZpbGw9IiMwOUEzRDUiIGQ9Ik02NC44IDk3MC42Yy0xMS4zLTEuMy0xMi4yLTE2LjUtMjYuNy0xNS4yLTcgMC0xMy42IDIuOS0xMy42IDkuNCAwIDkuNyAxNSAxMC42IDI0LjEgMTMuMSAxNS40IDQuNyAzMC40IDcuOSAzMC40IDI0LjcgMCAyMS4zLTE2LjcgMjguNy0zOC43IDI4LjctMTguNCAwLTM3LjEtNi41LTM3LjEtMjMuNSAwLTQuNyA0LjUtOC40IDguOS04LjQgNS41IDAgNy41IDIuMyA5LjQgNi4yIDQuMyA3LjUgOS4xIDExLjYgMjEgMTEuNiA3LjUgMCAxNS4zLTIuOSAxNS4zLTkuNCAwLTkuMy05LjUtMTEuMy0xOS4zLTEzLjYtMTcuNC00LjktMzIuMy03LjQtMzQtMjYuNy0xLjgtMzIuOSA2Ni43LTM0LjEgNzAuNi01LjMtLjMgNS4yLTUuMiA4LjQtMTAuMyA4LjR6bTgxLjUtMjguOGMyNC4xIDAgMzcuNyAyMC4xIDM3LjcgNDQuOSAwIDI0LjktMTMuMiA0NC45LTM3LjcgNDQuOS0xMy42IDAtMjIuMS01LjgtMjguMi0xNC43djMyLjljMCA5LjktMy4yIDE0LjctMTAuNCAxNC43LTguOCAwLTEwLjQtNS42LTEwLjQtMTQuN3YtOTUuNmMwLTcuOCAzLjMtMTIuNiAxMC40LTEyLjYgNi43IDAgMTAuNCA1LjMgMTAuNCAxMi42djIuN2M2LjgtOC41IDE0LjYtMTUuMSAyOC4yLTE1LjF6bS01LjcgNzIuOGMxNC4xIDAgMjAuNC0xMyAyMC40LTI4LjIgMC0xNC44LTYuNC0yOC4yLTIwLjQtMjguMi0xNC43IDAtMjEuNSAxMi4xLTIxLjUgMjguMi4xIDE1LjcgNi45IDI4LjIgMjEuNSAyOC4yem01OS44LTQ5LjNjMC0xNy4zIDE5LjktMjMuNSAzOS4yLTIzLjUgMjcuMSAwIDM4LjIgNy45IDM4LjIgMzR2MjUuMmMwIDYgMy43IDE3LjkgMy43IDIxLjUgMCA1LjUtNSA4LjktMTAuNCA4LjktNiAwLTEwLjQtNy0xMy42LTEyLjEtOC44IDctMTguMSAxMi4xLTMyLjQgMTIuMS0xNS44IDAtMjguMi05LjMtMjguMi0yNC43IDAtMTMuNiA5LjctMjEuNCAyMS41LTI0LjEgMCAuMSAzNy43LTguOSAzNy43LTkgMC0xMS42LTQuMS0xNi43LTE2LjMtMTYuNy0xMC43IDAtMTYuMiAyLjktMjAuNCA5LjQtMy40IDQuOS0yLjkgNy44LTkuNCA3LjgtNS4xIDAtOS42LTMuNi05LjYtOC44em0zMi4yIDUxLjljMTYuNSAwIDIzLjUtOC43IDIzLjUtMjYuMXYtMy43Yy00LjQgMS41LTIyLjQgNi0yNy4zIDYuNy01LjIgMS0xMC40IDQuOS0xMC40IDExIC4yIDYuNyA3LjEgMTIuMSAxNC4yIDEyLjF6TTM1NCA5MDljMjMuMyAwIDQ4LjYgMTMuOSA0OC42IDM2LjEgMCA1LjctNC4zIDEwLjQtOS45IDEwLjQtNy42IDAtOC43LTQuMS0xMi4xLTkuOS01LjYtMTAuMy0xMi4yLTE3LjItMjYuNy0xNy4yLTIyLjMtLjItMzIuMyAxOS0zMi4zIDQyLjggMCAyNCA4LjMgNDEuMyAzMS40IDQxLjMgMTUuMyAwIDIzLjgtOC45IDI4LjItMjAuNCAxLjgtNS4zIDQuOS0xMC40IDExLjYtMTAuNCA1LjIgMCAxMC40IDUuMyAxMC40IDExIDAgMjMuNS0yNCAzOS43LTQ4LjYgMzkuNy0yNyAwLTQyLjMtMTEuNC01MC42LTMwLjQtNC4xLTkuMS02LjctMTguNC02LjctMzEuNC0uNC0zNi40IDIwLjgtNjEuNiA1Ni43LTYxLjZ6bTEzMy4zIDMyLjhjNiAwIDkuNCAzLjkgOS40IDkuOSAwIDIuNC0xLjkgNy4zLTIuNyA5LjlsLTI4LjcgNzUuNGMtNi40IDE2LjQtMTEuMiAyNy43LTMyLjkgMjcuNy0xMC4zIDAtMTkuMy0uOS0xOS4zLTkuOSAwLTUuMiAzLjktNy44IDkuNC03LjggMSAwIDIuNy41IDMuNy41IDEuNiAwIDIuNy41IDMuNy41IDEwLjkgMCAxMi40LTExLjIgMTYuMy0xOC45bC0yNy43LTY4LjVjLTEuNi0zLjctMi43LTYuMi0yLjctOC40IDAtNiA0LjctMTAuNCAxMS0xMC40IDcgMCA5LjggNS41IDExLjYgMTEuNmwxOC4zIDU0LjMgMTguMy01MC4yYzIuNy03LjggMy0xNS43IDEyLjMtMTUuN3oiIC8+IDwvc3ZnPg==">
|
51 |
</a>
|
52 |
</div>
|
53 |
""",
|
54 |
unsafe_allow_html=True,
|
55 |
)
|
56 |
|
57 |
+
# how it works
|
58 |
+
def hiw():
|
59 |
+
st.markdown("""
|
60 |
+
## How it works
|
61 |
+
|
62 |
+
ExtEnD frames Entity Disambiguation as a text extraction problem:
|
63 |
+
""")
|
64 |
+
st.image(
|
65 |
+
"data/repo-assets/extend_formulation.png", caption="ExtEnD Formulation"
|
66 |
+
)
|
67 |
+
st.markdown(
|
68 |
+
"""
|
69 |
+
Given the sentence *After a long fight Superman saved Metropolis*, where *Superman* is the mention
|
70 |
+
to disambiguate, ExtEnD first concatenates the descriptions of all the possible candidates of *Superman* in the
|
71 |
+
inventory and then selects the span whose description best suits the mention in its context.
|
72 |
+
|
73 |
+
To convert this task to end2end entity linking, as we do in *Model demo*, we leverage spaCy
|
74 |
+
(more specifically, its NER) and run ExtEnD on each named entity spaCy identifies
|
75 |
+
(if the corresponding mention is contained in the inventory).
|
76 |
+
|
77 |
+
Links:
|
78 |
+
* [full paper](https://www.researchgate.net/publication/359392427_ExtEnD_Extractive_Entity_Disambiguation)
|
79 |
+
* [GitHub](https://github.com/SapienzaNLP/extend)
|
80 |
"""
|
81 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
82 |
|
83 |
# demo
|
84 |
+
def demo():
|
85 |
+
st.markdown("## Demo")
|
86 |
+
|
87 |
+
@st.cache(allow_output_mutation=True)
|
88 |
+
def load_resources(inventory_path):
|
89 |
+
|
90 |
+
# load nlp
|
91 |
+
nlp = spacy.load("en_core_web_sm")
|
92 |
+
extend_config = dict(
|
93 |
+
checkpoint_path=model_checkpoint_path,
|
94 |
+
mentions_inventory_path=inventory_path,
|
95 |
+
device=cuda_device,
|
96 |
+
tokens_per_batch=10_000,
|
97 |
+
)
|
98 |
+
nlp.add_pipe("extend", after="ner", config=extend_config)
|
99 |
+
|
100 |
+
# mock call to load resources
|
101 |
+
nlp(examples[0])
|
102 |
+
|
103 |
+
# return
|
104 |
+
return nlp
|
105 |
+
|
106 |
+
# read input
|
107 |
+
placeholder = st.selectbox(
|
108 |
+
"Examples",
|
109 |
+
options=examples,
|
110 |
+
index=0,
|
111 |
)
|
112 |
+
input_text = st.text_area("Input text to entity-disambiguate", placeholder)
|
113 |
|
114 |
+
# custom inventory
|
115 |
+
uploaded_inventory_path = st.file_uploader(
|
116 |
+
"[Optional] Upload custom inventory (tsv file, mention \\t desc1 \\t desc2 \\t)",
|
117 |
+
accept_multiple_files=False,
|
118 |
+
type=["tsv"],
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
119 |
)
|
120 |
+
if uploaded_inventory_path is not None:
|
121 |
+
inventory_path = f"data/inventories/{uploaded_inventory_path.name}"
|
122 |
+
with open(inventory_path, "wb") as f:
|
123 |
+
f.write(uploaded_inventory_path.getbuffer())
|
124 |
+
else:
|
125 |
+
inventory_path = default_inventory_path
|
126 |
+
|
127 |
+
# load model and color generator
|
128 |
+
nlp = load_resources(inventory_path)
|
129 |
+
color_generator = get_md_200_random_color_generator()
|
130 |
+
|
131 |
+
if st.button("Disambiguate", key="classify"):
|
132 |
+
|
133 |
+
# tag sentence
|
134 |
+
time_start = time.perf_counter()
|
135 |
+
doc = nlp(input_text)
|
136 |
+
time_end = time.perf_counter()
|
137 |
+
|
138 |
+
# extract entities
|
139 |
+
entities = {}
|
140 |
+
for ent in doc.ents:
|
141 |
+
if ent._.disambiguated_entity is not None:
|
142 |
+
entities[ent.start_char] = (
|
143 |
+
ent.start_char,
|
144 |
+
ent.end_char,
|
145 |
+
ent.text,
|
146 |
+
ent._.disambiguated_entity,
|
147 |
+
)
|
148 |
+
|
149 |
+
# create annotated html components
|
150 |
+
|
151 |
+
annotated_html_components = []
|
152 |
+
|
153 |
+
assert all(any(t.idx == _s for t in doc) for _s in entities)
|
154 |
+
it = iter(list(doc))
|
155 |
+
while True:
|
156 |
+
try:
|
157 |
+
t = next(it)
|
158 |
+
except StopIteration:
|
159 |
+
break
|
160 |
+
if t.idx in entities:
|
161 |
+
_start, _end, _text, _entity = entities[t.idx]
|
162 |
+
while t.idx + len(t) != _end:
|
163 |
+
t = next(it)
|
164 |
+
annotated_html_components.append(
|
165 |
+
str(annotation(*(_text, _entity, color_generator())))
|
166 |
+
)
|
167 |
+
else:
|
168 |
+
annotated_html_components.append(str(html.escape(t.text)))
|
169 |
+
|
170 |
+
st.markdown(
|
171 |
+
"\n".join(
|
172 |
+
[
|
173 |
+
"<div>",
|
174 |
+
*annotated_html_components,
|
175 |
+
"<p></p>"
|
176 |
+
f'<div style="text-align: right"><p style="color: gray">Time: {(time_end - time_start):.2f}s</p></div>'
|
177 |
+
"</div>",
|
178 |
+
]
|
179 |
+
),
|
180 |
+
unsafe_allow_html=True,
|
181 |
+
)
|
182 |
+
|
183 |
+
demo()
|
184 |
+
hiw()
|
185 |
|
186 |
|
187 |
|