<jupyter_start><jupyter_text>POS datasets<jupyter_code>generate_tables(POS_FILES, display_df=False)<jupyter_output>Owoputi_2013-train: /datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/oct27.splits/oct27.train
1000 14619 Counter({'V': 2219, 'N': 2003, ',': 1715, 'P': 1252, '^': 890, 'D': 869, 'A': 755, '@': 713, 'R': 689, '~': 538, '!': 406, 'L': 252, '&': 239, 'U': 223, '$': 216, 'E': 148, '#': 141, 'G': 137, 'T': 92, 'Z': 21, 'S': 18, 'X': 15, 'M': 3, 'Y': 2})
Owoputi_2013-traindev: /datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/oct27.splits/oct27.traindev
1327 19442 Counter({'V': 2970, 'N': 2663, ',': 2215, 'P': 1692, '^': 1201, 'D': 1181, 'A': 994, '@': 956, 'R': 898, '~': 708, '!': 505, '&': 330, 'L': 317, 'U': 314, '$': 302, 'G': 202, 'E': 200, '#': 193, 'T': 128, 'Z': 30, 'S': 23, 'X': 19, 'M': 3, 'Y': 2})
Owoputi_2013-dev: /datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/oct27.splits/oct27.dev
327 4823 Counter({'V': 751, 'N': 660, ',': 500, 'P': 440, 'D': 312, '^': 311, '@': 243, 'A': 239, 'R[...]<jupyter_text>Chunking<jupyter_code>generate_tables(CHUNKING_FILES, label_processor=lambda x: get_ner_label(x, idx=1))<jupyter_output>Ritter-train: /datadrive/Datasets/Twitter/RitterNER/twitter_processed/chunk.train.conll
551 10584 Counter({'NP': 4681, 'VP': 1901, 'PP': 748, 'ADVP': 403, 'INTJ': 355, 'ADJP': 211, 'PRT': 75, 'SBAR': 73, 'CONJP': 4})
Ritter-dev: /datadrive/Datasets/Twitter/RitterNER/twitter_processed/chunk.dev.conll
118 2317 Counter({'NP': 1020, 'VP': 397, 'PP': 148, 'ADVP': 107, 'INTJ': 71, 'ADJP': 50, 'PRT': 22, 'SBAR': 10})
Ritter-test: /datadrive/Datasets/Twitter/RitterNER/twitter_processed/chunk.test.conll
119 2310 Counter({'NP': 977, 'VP': 428, 'PP': 141, 'ADVP': 91, 'INTJ': 82, 'ADJP': 66, 'SBAR': 25, 'PRT': 12})
\begin{tabular}{llrrlr}
\toprule
       &       &  total\_seq &  total\_tokens &                                      all\_labels &  num\_labels \\
datakey & datatype &            &               &                                                 &             \\
\midrule
Ritter & dev &  118 &  2317 &  ADJP, ADVP, INTJ, NP, PP, PRT, SBAR, VP &  8 \\
       & test &  119 &  2310 &  ADJP, [...]<jupyter_text>Supersense tagging<jupyter_code>generate_tables(SUPERSENSE_TAGGING_FILES, display_df=False)
generate_tables(SUPERSENSE_TAGGING_FILES, label_processor=lambda x: get_ner_label(x, idx=1))<jupyter_output>Ritter-train: /datadrive/Datasets/Twitter/supersense-data-twitter/ritter-train.tsv
551 10652 Counter({'NOUN.TIME': 575, 'VERB.STATIVE': 400, 'NOUN.PERSON': 345, 'NOUN.COMMUNICATION': 326, 'NOUN.ARTIFACT': 229, 'NOUN.LOCATION': 168, 'VERB.SOCIAL': 162, 'NOUN.ACT': 160, 'VERB.COMMUNICATION': 152, 'NOUN.GROUP': 151, 'VERB.MOTION': 136, 'VERB.COGNITION': 131, 'NOUN.EVENT': 122, 'VERB.POSSESSION': 120, 'VERB.CHANGE': 103, 'VERB.EMOTION': 94, 'VERB.PERCEPTION': 62, 'NOUN.COGNITION': 46, 'NOUN.FOOD': 45, 'NOUN.STATE': 42, 'VERB.CONTACT': 39, 'NOUN.ATTRIBUTE': 38, 'VERB.CREATION': 32, 'NOUN.QUANTITY': 32, 'VERB.BODY': 27, 'NOUN.BODY': 27, 'NOUN.POSSESSION': 26, 'VERB.COMPETITION': 23, 'NOUN.ANIMAL': 18, 'NOUN.RELATION': 16, 'NOUN.PHENOMENON': 12, 'VERB.CONSUMPTION': 12, 'NOUN.SUBSTANCE': 12, 'NOUN.TOPS': 11, 'NOUN.FEELING': 10, 'NOUN.OBJECT': 6, 'NOUN.PROCESS': 3, 'NOUN.MOTIVE': 3, 'NOUN.PLANT': 2, 'VERB.WEATHER': 1})
Ritter-dev: /datadrive/Datasets/Twitter/supersense-data-twitter/ritter-dev.t[...]<jupyter_text>DimSUM https://dimsum16.github.io/<jupyter_code>generate_tables(DIMSUM_FILES, label_col_id=7, label_processor=get_simple_label, skip_other=True)<jupyter_output>DiMSUM_2016-train: /datadrive/Datasets/Twitter/dimsum-data/conll/dimsum16.train
4799 73826 Counter({'v.stative': 3357, 'n.person': 1867, 'n.group': 1750, 'v.cognition': 1310, 'n.artifact': 1226, 'v.communication': 1225, 'n.time': 1173, 'v.social': 1073, 'n.act': 902, 'n.communication': 886, 'n.location': 845, 'n.cognition': 835, 'n.food': 830, 'v.motion': 765, 'n.event': 650, 'v.change': 539, 'v.emotion': 414, 'v.possession': 398, 'n.possession': 386, 'n.attribute': 311, 'v.perception': 241, 'n.quantity': 161, 'n.body': 153, 'v.body': 153, 'n.state': 122, 'n.animal': 122, 'v.creation': 117, 'v.consumption': 114, 'v.contact': 104, 'n.feeling': 68, 'n.natural_object': 62, 'n.substance': 52, 'v.competition': 51, 'n.phenomenon': 46, 'n.relation': 40, 'n.motive': 35, 'n.process': 29, 'n.other': 23, 'n.plant': 10, 'n.shape': 7, 'v.weather': 2})
DiMSUM_2016-test: /datadrive/Datasets/Twitter/dimsum-data/conll/dimsum16.test
1000 16500 Counter({'v.stative': 769, 'n.person': 413, 'n.artifact': 33[...]<jupyter_text>Frame Semantics```@paper{AAAI159349,	author = {<NAME> and <NAME> and <NAME>},	title = {Using Frame Semantics for Knowledge Extraction from Twitter},	conference = {AAAI Conference on Artificial Intelligence},	year = {2015},	keywords = {frame semantics; knowledge bases; twitter},	abstract = {Knowledge bases have the potential to advance artificial intelligence, but often suffer from recall problems, i.e., lack of knowledge of new entities and relations. On the contrary, social media such as Twitter provide abundance of data, in a timely manner: information spreads at an incredible pace and is posted long before it makes it into more commonly used resources for knowledge extraction. In this paper we address the question whether we can exploit social media to extract new facts, which may at first seem like finding needles in haystacks. We collect tweets about 60 entities in Freebase and compare four methods to extract binary relation candidates, based on syntactic and semantic parsing and simple mechanism for factuality scoring. The extracted facts are manually evaluated in terms of their correctness and relevance for search. We show that moving from bottom-up syntactic or semantic dependency parsing formalisms to top-down frame-semantic processing improves the robustness of knowledge extraction, producing more intelligible fact candidates of better quality. In order to evaluate the quality of frame semantic parsing on Twitter intrinsically, we make a multiply frame-annotated dataset of tweets publicly available.},	url = {https://www.aaai.org/ocs/index.php/AAAI/AAAI15/paper/view/9349}}```<jupyter_code>generate_tables(FRAME_SEMANTICS_FILE, show_labels=False, label_col_id=3, label_processor=get_simple_label, skip_other=True)<jupyter_output>Sogaard_2015-gavin: /datadrive/Datasets/lowlands-data/AAAI15/conll/all.gavin
236 4533 Counter({'Calendric_unit': 106, '{}': 33, 'Intentionally_act': 22, 'Measure_duration': 21, 'Have_associated': 21, 'Arriving': 20, 'Temporal_collocation': 19, 'Desirability': 19, 'Desiring': 18, 'Getting': 17, 'Motion': 17, 'Relative_time': 16, 'Quantity': 15, 'Observable_body_parts': 15, 'Locale_by_use': 13, 'Kinship': 12, 'Event_instance': 12, 'Becoming': 12, 'Experiencer_focus': 11, 'Age': 9, 'People': 9, 'Opinion': 8, 'Opportunity': 8, 'Social_event': 8, 'Capability': 8, 'Text': 8, 'Food': 8, 'Likelihood': 8, 'Perception_active': 8, 'Competition': 7, 'Performers_and_roles': 7, 'Make_noise': 7, 'Sounds': 7, 'Ordinal_numbers': 7, 'Personal_relationship': 7, 'Expensiveness': 7, 'Part_orientational': 7, 'Color': 7, 'Cotheme': 6, 'Location_in_time': 6, 'Contacting': 6, 'Communication': 6, 'Ingestion': 6, 'Political_locales': 6, 'Taking': 5, 'Hit_or_miss': 5, 'Size': 5, 'Emotion_directed': 5, 'Temporal_s[...]
{'fraction_non_alphanumeric': 0.18306451612903227, 'fraction_numerical': 0.09233870967741936, 'mean_word_length': 5.252941176470588, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '561972', 'license': 'Apache-2.0', 'n_tokens_mistral': 3451, 'n_tokens_neox': 2774, 'n_words': 713, 'path': 'notebooks/Twitter Multitask Sequence Prediction Datasets.ipynb', 'repo_name': 'vishalbelsare/SocialMediaIE', 'size': '7462'}