diff --git a/TextClassification/TextClassification/.DS_Store b/TextClassification/TextClassification/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..e1b4c45745a57119cc67b02925ff368048465621 Binary files /dev/null and b/TextClassification/TextClassification/.DS_Store differ diff --git a/TextClassification/TextClassification/app.py b/TextClassification/TextClassification/app.py new file mode 100644 index 0000000000000000000000000000000000000000..2b698004bdcb83d7e94b8dbf3503dd310467dc45 --- /dev/null +++ b/TextClassification/TextClassification/app.py @@ -0,0 +1,45 @@ +import string +import gradio as gr +import requests +import torch +from transformers import ( + AutoConfig, + AutoModelForSequenceClassification, + AutoTokenizer, +) + +model_dir = "my-bert-model" + +config = AutoConfig.from_pretrained(model_dir, num_labels=3, finetuning_task="text-classification") +tokenizer = AutoTokenizer.from_pretrained(model_dir) +model = AutoModelForSequenceClassification.from_pretrained(model_dir, config=config) + +def inference(input_text): + inputs = tokenizer.batch_encode_plus( + [input_text], + max_length=512, + pad_to_max_length=True, + truncation=True, + padding="max_length", + return_tensors="pt", + ) + + with torch.no_grad(): + logits = model(**inputs).logits + + predicted_class_id = logits.argmax().item() + output = model.config.id2label[predicted_class_id] + return output + +demo = gr.Interface( + fn=inference, + inputs=gr.Textbox(label="Input Text", scale=2, container=False), + outputs=gr.Textbox(label="Output Label"), + examples = [ + ["My last two weather pics from the storm on August 2nd. People packed up real fast after the temp dropped and winds picked up.", 1], + ["Lying Clinton sinking! Donald Trump singing: Let's Make America Great Again!", 0], + ], + title="Tutorial: BERT-based Text Classificatioin", + ) + +demo.launch(debug=True) \ No newline at end of file diff --git a/TextClassification/TextClassification/bert-base-uncased/config.json b/TextClassification/TextClassification/bert-base-uncased/config.json new file mode 100644 index 0000000000000000000000000000000000000000..45a2321a7ecfdaaf60a6c1fd7f5463994cc8907d --- /dev/null +++ b/TextClassification/TextClassification/bert-base-uncased/config.json @@ -0,0 +1,23 @@ +{ + "architectures": [ + "BertForMaskedLM" + ], + "attention_probs_dropout_prob": 0.1, + "gradient_checkpointing": false, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 768, + "initializer_range": 0.02, + "intermediate_size": 3072, + "layer_norm_eps": 1e-12, + "max_position_embeddings": 512, + "model_type": "bert", + "num_attention_heads": 12, + "num_hidden_layers": 12, + "pad_token_id": 0, + "position_embedding_type": "absolute", + "transformers_version": "4.6.0.dev0", + "type_vocab_size": 2, + "use_cache": true, + "vocab_size": 30522 +} diff --git a/TextClassification/TextClassification/bert-base-uncased/pytorch_model.bin b/TextClassification/TextClassification/bert-base-uncased/pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..ba5d19791be1dd7992e33bd61f20207b0f7f50a5 --- /dev/null +++ b/TextClassification/TextClassification/bert-base-uncased/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:097417381d6c7230bd9e3557456d726de6e83245ec8b24f529f60198a67b203a +size 440473133 diff --git a/TextClassification/TextClassification/bert-base-uncased/tokenizer.json b/TextClassification/TextClassification/bert-base-uncased/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..949a6f013d67eb8a5b4b5b46026217b888021b88 --- /dev/null +++ b/TextClassification/TextClassification/bert-base-uncased/tokenizer.json @@ -0,0 +1 @@ +{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":0,"special":true,"content":"[PAD]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":100,"special":true,"content":"[UNK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":101,"special":true,"content":"[CLS]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":102,"special":true,"content":"[SEP]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":103,"special":true,"content":"[MASK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],"normalizer":{"type":"BertNormalizer","clean_text":true,"handle_chinese_chars":true,"strip_accents":null,"lowercase":true},"pre_tokenizer":{"type":"BertPreTokenizer"},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}}],"pair":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}},{"Sequence":{"id":"B","type_id":1}},{"SpecialToken":{"id":"[SEP]","type_id":1}}],"special_tokens":{"[CLS]":{"id":"[CLS]","ids":[101],"tokens":["[CLS]"]},"[SEP]":{"id":"[SEP]","ids":[102],"tokens":["[SEP]"]}}},"decoder":{"type":"WordPiece","prefix":"##","cleanup":true},"model":{"unk_token":"[UNK]","continuing_subword_prefix":"##","max_input_chars_per_word":100,"vocab":{"[PAD]":0,"[unused0]":1,"[unused1]":2,"[unused2]":3,"[unused3]":4,"[unused4]":5,"[unused5]":6,"[unused6]":7,"[unused7]":8,"[unused8]":9,"[unused9]":10,"[unused10]":11,"[unused11]":12,"[unused12]":13,"[unused13]":14,"[unused14]":15,"[unused15]":16,"[unused16]":17,"[unused17]":18,"[unused18]":19,"[unused19]":20,"[unused20]":21,"[unused21]":22,"[unused22]":23,"[unused23]":24,"[unused24]":25,"[unused25]":26,"[unused26]":27,"[unused27]":28,"[unused28]":29,"[unused29]":30,"[unused30]":31,"[unused31]":32,"[unused32]":33,"[unused33]":34,"[unused34]":35,"[unused35]":36,"[unused36]":37,"[unused37]":38,"[unused38]":39,"[unused39]":40,"[unused40]":41,"[unused41]":42,"[unused42]":43,"[unused43]":44,"[unused44]":45,"[unused45]":46,"[unused46]":47,"[unused47]":48,"[unused48]":49,"[unused49]":50,"[unused50]":51,"[unused51]":52,"[unused52]":53,"[unused53]":54,"[unused54]":55,"[unused55]":56,"[unused56]":57,"[unused57]":58,"[unused58]":59,"[unused59]":60,"[unused60]":61,"[unused61]":62,"[unused62]":63,"[unused63]":64,"[unused64]":65,"[unused65]":66,"[unused66]":67,"[unused67]":68,"[unused68]":69,"[unused69]":70,"[unused70]":71,"[unused71]":72,"[unused72]":73,"[unused73]":74,"[unused74]":75,"[unused75]":76,"[unused76]":77,"[unused77]":78,"[unused78]":79,"[unused79]":80,"[unused80]":81,"[unused81]":82,"[unused82]":83,"[unused83]":84,"[unused84]":85,"[unused85]":86,"[unused86]":87,"[unused87]":88,"[unused88]":89,"[unused89]":90,"[unused90]":91,"[unused91]":92,"[unused92]":93,"[unused93]":94,"[unused94]":95,"[unused95]":96,"[unused96]":97,"[unused97]":98,"[unused98]":99,"[UNK]":100,"[CLS]":101,"[SEP]":102,"[MASK]":103,"[unused99]":104,"[unused100]":105,"[unused101]":106,"[unused102]":107,"[unused103]":108,"[unused104]":109,"[unused105]":110,"[unused106]":111,"[unused107]":112,"[unused108]":113,"[unused109]":114,"[unused110]":115,"[unused111]":116,"[unused112]":117,"[unused113]":118,"[unused114]":119,"[unused115]":120,"[unused116]":121,"[unused117]":122,"[unused118]":123,"[unused119]":124,"[unused120]":125,"[unused121]":126,"[unused122]":127,"[unused123]":128,"[unused124]":129,"[unused125]":130,"[unused126]":131,"[unused127]":132,"[unused128]":133,"[unused129]":134,"[unused130]":135,"[unused131]":136,"[unused132]":137,"[unused133]":138,"[unused134]":139,"[unused135]":140,"[unused136]":141,"[unused137]":142,"[unused138]":143,"[unused139]":144,"[unused140]":145,"[unused141]":146,"[unused142]":147,"[unused143]":148,"[unused144]":149,"[unused145]":150,"[unused146]":151,"[unused147]":152,"[unused148]":153,"[unused149]":154,"[unused150]":155,"[unused151]":156,"[unused152]":157,"[unused153]":158,"[unused154]":159,"[unused155]":160,"[unused156]":161,"[unused157]":162,"[unused158]":163,"[unused159]":164,"[unused160]":165,"[unused161]":166,"[unused162]":167,"[unused163]":168,"[unused164]":169,"[unused165]":170,"[unused166]":171,"[unused167]":172,"[unused168]":173,"[unused169]":174,"[unused170]":175,"[unused171]":176,"[unused172]":177,"[unused173]":178,"[unused174]":179,"[unused175]":180,"[unused176]":181,"[unused177]":182,"[unused178]":183,"[unused179]":184,"[unused180]":185,"[unused181]":186,"[unused182]":187,"[unused183]":188,"[unused184]":189,"[unused185]":190,"[unused186]":191,"[unused187]":192,"[unused188]":193,"[unused189]":194,"[unused190]":195,"[unused191]":196,"[unused192]":197,"[unused193]":198,"[unused194]":199,"[unused195]":200,"[unused196]":201,"[unused197]":202,"[unused198]":203,"[unused199]":204,"[unused200]":205,"[unused201]":206,"[unused202]":207,"[unused203]":208,"[unused204]":209,"[unused205]":210,"[unused206]":211,"[unused207]":212,"[unused208]":213,"[unused209]":214,"[unused210]":215,"[unused211]":216,"[unused212]":217,"[unused213]":218,"[unused214]":219,"[unused215]":220,"[unused216]":221,"[unused217]":222,"[unused218]":223,"[unused219]":224,"[unused220]":225,"[unused221]":226,"[unused222]":227,"[unused223]":228,"[unused224]":229,"[unused225]":230,"[unused226]":231,"[unused227]":232,"[unused228]":233,"[unused229]":234,"[unused230]":235,"[unused231]":236,"[unused232]":237,"[unused233]":238,"[unused234]":239,"[unused235]":240,"[unused236]":241,"[unused237]":242,"[unused238]":243,"[unused239]":244,"[unused240]":245,"[unused241]":246,"[unused242]":247,"[unused243]":248,"[unused244]":249,"[unused245]":250,"[unused246]":251,"[unused247]":252,"[unused248]":253,"[unused249]":254,"[unused250]":255,"[unused251]":256,"[unused252]":257,"[unused253]":258,"[unused254]":259,"[unused255]":260,"[unused256]":261,"[unused257]":262,"[unused258]":263,"[unused259]":264,"[unused260]":265,"[unused261]":266,"[unused262]":267,"[unused263]":268,"[unused264]":269,"[unused265]":270,"[unused266]":271,"[unused267]":272,"[unused268]":273,"[unused269]":274,"[unused270]":275,"[unused271]":276,"[unused272]":277,"[unused273]":278,"[unused274]":279,"[unused275]":280,"[unused276]":281,"[unused277]":282,"[unused278]":283,"[unused279]":284,"[unused280]":285,"[unused281]":286,"[unused282]":287,"[unused283]":288,"[unused284]":289,"[unused285]":290,"[unused286]":291,"[unused287]":292,"[unused288]":293,"[unused289]":294,"[unused290]":295,"[unused291]":296,"[unused292]":297,"[unused293]":298,"[unused294]":299,"[unused295]":300,"[unused296]":301,"[unused297]":302,"[unused298]":303,"[unused299]":304,"[unused300]":305,"[unused301]":306,"[unused302]":307,"[unused303]":308,"[unused304]":309,"[unused305]":310,"[unused306]":311,"[unused307]":312,"[unused308]":313,"[unused309]":314,"[unused310]":315,"[unused311]":316,"[unused312]":317,"[unused313]":318,"[unused314]":319,"[unused315]":320,"[unused316]":321,"[unused317]":322,"[unused318]":323,"[unused319]":324,"[unused320]":325,"[unused321]":326,"[unused322]":327,"[unused323]":328,"[unused324]":329,"[unused325]":330,"[unused326]":331,"[unused327]":332,"[unused328]":333,"[unused329]":334,"[unused330]":335,"[unused331]":336,"[unused332]":337,"[unused333]":338,"[unused334]":339,"[unused335]":340,"[unused336]":341,"[unused337]":342,"[unused338]":343,"[unused339]":344,"[unused340]":345,"[unused341]":346,"[unused342]":347,"[unused343]":348,"[unused344]":349,"[unused345]":350,"[unused346]":351,"[unused347]":352,"[unused348]":353,"[unused349]":354,"[unused350]":355,"[unused351]":356,"[unused352]":357,"[unused353]":358,"[unused354]":359,"[unused355]":360,"[unused356]":361,"[unused357]":362,"[unused358]":363,"[unused359]":364,"[unused360]":365,"[unused361]":366,"[unused362]":367,"[unused363]":368,"[unused364]":369,"[unused365]":370,"[unused366]":371,"[unused367]":372,"[unused368]":373,"[unused369]":374,"[unused370]":375,"[unused371]":376,"[unused372]":377,"[unused373]":378,"[unused374]":379,"[unused375]":380,"[unused376]":381,"[unused377]":382,"[unused378]":383,"[unused379]":384,"[unused380]":385,"[unused381]":386,"[unused382]":387,"[unused383]":388,"[unused384]":389,"[unused385]":390,"[unused386]":391,"[unused387]":392,"[unused388]":393,"[unused389]":394,"[unused390]":395,"[unused391]":396,"[unused392]":397,"[unused393]":398,"[unused394]":399,"[unused395]":400,"[unused396]":401,"[unused397]":402,"[unused398]":403,"[unused399]":404,"[unused400]":405,"[unused401]":406,"[unused402]":407,"[unused403]":408,"[unused404]":409,"[unused405]":410,"[unused406]":411,"[unused407]":412,"[unused408]":413,"[unused409]":414,"[unused410]":415,"[unused411]":416,"[unused412]":417,"[unused413]":418,"[unused414]":419,"[unused415]":420,"[unused416]":421,"[unused417]":422,"[unused418]":423,"[unused419]":424,"[unused420]":425,"[unused421]":426,"[unused422]":427,"[unused423]":428,"[unused424]":429,"[unused425]":430,"[unused426]":431,"[unused427]":432,"[unused428]":433,"[unused429]":434,"[unused430]":435,"[unused431]":436,"[unused432]":437,"[unused433]":438,"[unused434]":439,"[unused435]":440,"[unused436]":441,"[unused437]":442,"[unused438]":443,"[unused439]":444,"[unused440]":445,"[unused441]":446,"[unused442]":447,"[unused443]":448,"[unused444]":449,"[unused445]":450,"[unused446]":451,"[unused447]":452,"[unused448]":453,"[unused449]":454,"[unused450]":455,"[unused451]":456,"[unused452]":457,"[unused453]":458,"[unused454]":459,"[unused455]":460,"[unused456]":461,"[unused457]":462,"[unused458]":463,"[unused459]":464,"[unused460]":465,"[unused461]":466,"[unused462]":467,"[unused463]":468,"[unused464]":469,"[unused465]":470,"[unused466]":471,"[unused467]":472,"[unused468]":473,"[unused469]":474,"[unused470]":475,"[unused471]":476,"[unused472]":477,"[unused473]":478,"[unused474]":479,"[unused475]":480,"[unused476]":481,"[unused477]":482,"[unused478]":483,"[unused479]":484,"[unused480]":485,"[unused481]":486,"[unused482]":487,"[unused483]":488,"[unused484]":489,"[unused485]":490,"[unused486]":491,"[unused487]":492,"[unused488]":493,"[unused489]":494,"[unused490]":495,"[unused491]":496,"[unused492]":497,"[unused493]":498,"[unused494]":499,"[unused495]":500,"[unused496]":501,"[unused497]":502,"[unused498]":503,"[unused499]":504,"[unused500]":505,"[unused501]":506,"[unused502]":507,"[unused503]":508,"[unused504]":509,"[unused505]":510,"[unused506]":511,"[unused507]":512,"[unused508]":513,"[unused509]":514,"[unused510]":515,"[unused511]":516,"[unused512]":517,"[unused513]":518,"[unused514]":519,"[unused515]":520,"[unused516]":521,"[unused517]":522,"[unused518]":523,"[unused519]":524,"[unused520]":525,"[unused521]":526,"[unused522]":527,"[unused523]":528,"[unused524]":529,"[unused525]":530,"[unused526]":531,"[unused527]":532,"[unused528]":533,"[unused529]":534,"[unused530]":535,"[unused531]":536,"[unused532]":537,"[unused533]":538,"[unused534]":539,"[unused535]":540,"[unused536]":541,"[unused537]":542,"[unused538]":543,"[unused539]":544,"[unused540]":545,"[unused541]":546,"[unused542]":547,"[unused543]":548,"[unused544]":549,"[unused545]":550,"[unused546]":551,"[unused547]":552,"[unused548]":553,"[unused549]":554,"[unused550]":555,"[unused551]":556,"[unused552]":557,"[unused553]":558,"[unused554]":559,"[unused555]":560,"[unused556]":561,"[unused557]":562,"[unused558]":563,"[unused559]":564,"[unused560]":565,"[unused561]":566,"[unused562]":567,"[unused563]":568,"[unused564]":569,"[unused565]":570,"[unused566]":571,"[unused567]":572,"[unused568]":573,"[unused569]":574,"[unused570]":575,"[unused571]":576,"[unused572]":577,"[unused573]":578,"[unused574]":579,"[unused575]":580,"[unused576]":581,"[unused577]":582,"[unused578]":583,"[unused579]":584,"[unused580]":585,"[unused581]":586,"[unused582]":587,"[unused583]":588,"[unused584]":589,"[unused585]":590,"[unused586]":591,"[unused587]":592,"[unused588]":593,"[unused589]":594,"[unused590]":595,"[unused591]":596,"[unused592]":597,"[unused593]":598,"[unused594]":599,"[unused595]":600,"[unused596]":601,"[unused597]":602,"[unused598]":603,"[unused599]":604,"[unused600]":605,"[unused601]":606,"[unused602]":607,"[unused603]":608,"[unused604]":609,"[unused605]":610,"[unused606]":611,"[unused607]":612,"[unused608]":613,"[unused609]":614,"[unused610]":615,"[unused611]":616,"[unused612]":617,"[unused613]":618,"[unused614]":619,"[unused615]":620,"[unused616]":621,"[unused617]":622,"[unused618]":623,"[unused619]":624,"[unused620]":625,"[unused621]":626,"[unused622]":627,"[unused623]":628,"[unused624]":629,"[unused625]":630,"[unused626]":631,"[unused627]":632,"[unused628]":633,"[unused629]":634,"[unused630]":635,"[unused631]":636,"[unused632]":637,"[unused633]":638,"[unused634]":639,"[unused635]":640,"[unused636]":641,"[unused637]":642,"[unused638]":643,"[unused639]":644,"[unused640]":645,"[unused641]":646,"[unused642]":647,"[unused643]":648,"[unused644]":649,"[unused645]":650,"[unused646]":651,"[unused647]":652,"[unused648]":653,"[unused649]":654,"[unused650]":655,"[unused651]":656,"[unused652]":657,"[unused653]":658,"[unused654]":659,"[unused655]":660,"[unused656]":661,"[unused657]":662,"[unused658]":663,"[unused659]":664,"[unused660]":665,"[unused661]":666,"[unused662]":667,"[unused663]":668,"[unused664]":669,"[unused665]":670,"[unused666]":671,"[unused667]":672,"[unused668]":673,"[unused669]":674,"[unused670]":675,"[unused671]":676,"[unused672]":677,"[unused673]":678,"[unused674]":679,"[unused675]":680,"[unused676]":681,"[unused677]":682,"[unused678]":683,"[unused679]":684,"[unused680]":685,"[unused681]":686,"[unused682]":687,"[unused683]":688,"[unused684]":689,"[unused685]":690,"[unused686]":691,"[unused687]":692,"[unused688]":693,"[unused689]":694,"[unused690]":695,"[unused691]":696,"[unused692]":697,"[unused693]":698,"[unused694]":699,"[unused695]":700,"[unused696]":701,"[unused697]":702,"[unused698]":703,"[unused699]":704,"[unused700]":705,"[unused701]":706,"[unused702]":707,"[unused703]":708,"[unused704]":709,"[unused705]":710,"[unused706]":711,"[unused707]":712,"[unused708]":713,"[unused709]":714,"[unused710]":715,"[unused711]":716,"[unused712]":717,"[unused713]":718,"[unused714]":719,"[unused715]":720,"[unused716]":721,"[unused717]":722,"[unused718]":723,"[unused719]":724,"[unused720]":725,"[unused721]":726,"[unused722]":727,"[unused723]":728,"[unused724]":729,"[unused725]":730,"[unused726]":731,"[unused727]":732,"[unused728]":733,"[unused729]":734,"[unused730]":735,"[unused731]":736,"[unused732]":737,"[unused733]":738,"[unused734]":739,"[unused735]":740,"[unused736]":741,"[unused737]":742,"[unused738]":743,"[unused739]":744,"[unused740]":745,"[unused741]":746,"[unused742]":747,"[unused743]":748,"[unused744]":749,"[unused745]":750,"[unused746]":751,"[unused747]":752,"[unused748]":753,"[unused749]":754,"[unused750]":755,"[unused751]":756,"[unused752]":757,"[unused753]":758,"[unused754]":759,"[unused755]":760,"[unused756]":761,"[unused757]":762,"[unused758]":763,"[unused759]":764,"[unused760]":765,"[unused761]":766,"[unused762]":767,"[unused763]":768,"[unused764]":769,"[unused765]":770,"[unused766]":771,"[unused767]":772,"[unused768]":773,"[unused769]":774,"[unused770]":775,"[unused771]":776,"[unused772]":777,"[unused773]":778,"[unused774]":779,"[unused775]":780,"[unused776]":781,"[unused777]":782,"[unused778]":783,"[unused779]":784,"[unused780]":785,"[unused781]":786,"[unused782]":787,"[unused783]":788,"[unused784]":789,"[unused785]":790,"[unused786]":791,"[unused787]":792,"[unused788]":793,"[unused789]":794,"[unused790]":795,"[unused791]":796,"[unused792]":797,"[unused793]":798,"[unused794]":799,"[unused795]":800,"[unused796]":801,"[unused797]":802,"[unused798]":803,"[unused799]":804,"[unused800]":805,"[unused801]":806,"[unused802]":807,"[unused803]":808,"[unused804]":809,"[unused805]":810,"[unused806]":811,"[unused807]":812,"[unused808]":813,"[unused809]":814,"[unused810]":815,"[unused811]":816,"[unused812]":817,"[unused813]":818,"[unused814]":819,"[unused815]":820,"[unused816]":821,"[unused817]":822,"[unused818]":823,"[unused819]":824,"[unused820]":825,"[unused821]":826,"[unused822]":827,"[unused823]":828,"[unused824]":829,"[unused825]":830,"[unused826]":831,"[unused827]":832,"[unused828]":833,"[unused829]":834,"[unused830]":835,"[unused831]":836,"[unused832]":837,"[unused833]":838,"[unused834]":839,"[unused835]":840,"[unused836]":841,"[unused837]":842,"[unused838]":843,"[unused839]":844,"[unused840]":845,"[unused841]":846,"[unused842]":847,"[unused843]":848,"[unused844]":849,"[unused845]":850,"[unused846]":851,"[unused847]":852,"[unused848]":853,"[unused849]":854,"[unused850]":855,"[unused851]":856,"[unused852]":857,"[unused853]":858,"[unused854]":859,"[unused855]":860,"[unused856]":861,"[unused857]":862,"[unused858]":863,"[unused859]":864,"[unused860]":865,"[unused861]":866,"[unused862]":867,"[unused863]":868,"[unused864]":869,"[unused865]":870,"[unused866]":871,"[unused867]":872,"[unused868]":873,"[unused869]":874,"[unused870]":875,"[unused871]":876,"[unused872]":877,"[unused873]":878,"[unused874]":879,"[unused875]":880,"[unused876]":881,"[unused877]":882,"[unused878]":883,"[unused879]":884,"[unused880]":885,"[unused881]":886,"[unused882]":887,"[unused883]":888,"[unused884]":889,"[unused885]":890,"[unused886]":891,"[unused887]":892,"[unused888]":893,"[unused889]":894,"[unused890]":895,"[unused891]":896,"[unused892]":897,"[unused893]":898,"[unused894]":899,"[unused895]":900,"[unused896]":901,"[unused897]":902,"[unused898]":903,"[unused899]":904,"[unused900]":905,"[unused901]":906,"[unused902]":907,"[unused903]":908,"[unused904]":909,"[unused905]":910,"[unused906]":911,"[unused907]":912,"[unused908]":913,"[unused909]":914,"[unused910]":915,"[unused911]":916,"[unused912]":917,"[unused913]":918,"[unused914]":919,"[unused915]":920,"[unused916]":921,"[unused917]":922,"[unused918]":923,"[unused919]":924,"[unused920]":925,"[unused921]":926,"[unused922]":927,"[unused923]":928,"[unused924]":929,"[unused925]":930,"[unused926]":931,"[unused927]":932,"[unused928]":933,"[unused929]":934,"[unused930]":935,"[unused931]":936,"[unused932]":937,"[unused933]":938,"[unused934]":939,"[unused935]":940,"[unused936]":941,"[unused937]":942,"[unused938]":943,"[unused939]":944,"[unused940]":945,"[unused941]":946,"[unused942]":947,"[unused943]":948,"[unused944]":949,"[unused945]":950,"[unused946]":951,"[unused947]":952,"[unused948]":953,"[unused949]":954,"[unused950]":955,"[unused951]":956,"[unused952]":957,"[unused953]":958,"[unused954]":959,"[unused955]":960,"[unused956]":961,"[unused957]":962,"[unused958]":963,"[unused959]":964,"[unused960]":965,"[unused961]":966,"[unused962]":967,"[unused963]":968,"[unused964]":969,"[unused965]":970,"[unused966]":971,"[unused967]":972,"[unused968]":973,"[unused969]":974,"[unused970]":975,"[unused971]":976,"[unused972]":977,"[unused973]":978,"[unused974]":979,"[unused975]":980,"[unused976]":981,"[unused977]":982,"[unused978]":983,"[unused979]":984,"[unused980]":985,"[unused981]":986,"[unused982]":987,"[unused983]":988,"[unused984]":989,"[unused985]":990,"[unused986]":991,"[unused987]":992,"[unused988]":993,"[unused989]":994,"[unused990]":995,"[unused991]":996,"[unused992]":997,"[unused993]":998,"!":999,"\"":1000,"#":1001,"$":1002,"%":1003,"&":1004,"'":1005,"(":1006,")":1007,"*":1008,"+":1009,",":1010,"-":1011,".":1012,"/":1013,"0":1014,"1":1015,"2":1016,"3":1017,"4":1018,"5":1019,"6":1020,"7":1021,"8":1022,"9":1023,":":1024,";":1025,"<":1026,"=":1027,">":1028,"?":1029,"@":1030,"[":1031,"\\":1032,"]":1033,"^":1034,"_":1035,"`":1036,"a":1037,"b":1038,"c":1039,"d":1040,"e":1041,"f":1042,"g":1043,"h":1044,"i":1045,"j":1046,"k":1047,"l":1048,"m":1049,"n":1050,"o":1051,"p":1052,"q":1053,"r":1054,"s":1055,"t":1056,"u":1057,"v":1058,"w":1059,"x":1060,"y":1061,"z":1062,"{":1063,"|":1064,"}":1065,"~":1066,"¡":1067,"¢":1068,"£":1069,"¤":1070,"¥":1071,"¦":1072,"§":1073,"¨":1074,"©":1075,"ª":1076,"«":1077,"¬":1078,"®":1079,"°":1080,"±":1081,"²":1082,"³":1083,"´":1084,"µ":1085,"¶":1086,"·":1087,"¹":1088,"º":1089,"»":1090,"¼":1091,"½":1092,"¾":1093,"¿":1094,"×":1095,"ß":1096,"æ":1097,"ð":1098,"÷":1099,"ø":1100,"þ":1101,"đ":1102,"ħ":1103,"ı":1104,"ł":1105,"ŋ":1106,"œ":1107,"ƒ":1108,"ɐ":1109,"ɑ":1110,"ɒ":1111,"ɔ":1112,"ɕ":1113,"ə":1114,"ɛ":1115,"ɡ":1116,"ɣ":1117,"ɨ":1118,"ɪ":1119,"ɫ":1120,"ɬ":1121,"ɯ":1122,"ɲ":1123,"ɴ":1124,"ɹ":1125,"ɾ":1126,"ʀ":1127,"ʁ":1128,"ʂ":1129,"ʃ":1130,"ʉ":1131,"ʊ":1132,"ʋ":1133,"ʌ":1134,"ʎ":1135,"ʐ":1136,"ʑ":1137,"ʒ":1138,"ʔ":1139,"ʰ":1140,"ʲ":1141,"ʳ":1142,"ʷ":1143,"ʸ":1144,"ʻ":1145,"ʼ":1146,"ʾ":1147,"ʿ":1148,"ˈ":1149,"ː":1150,"ˡ":1151,"ˢ":1152,"ˣ":1153,"ˤ":1154,"α":1155,"β":1156,"γ":1157,"δ":1158,"ε":1159,"ζ":1160,"η":1161,"θ":1162,"ι":1163,"κ":1164,"λ":1165,"μ":1166,"ν":1167,"ξ":1168,"ο":1169,"π":1170,"ρ":1171,"ς":1172,"σ":1173,"τ":1174,"υ":1175,"φ":1176,"χ":1177,"ψ":1178,"ω":1179,"а":1180,"б":1181,"в":1182,"г":1183,"д":1184,"е":1185,"ж":1186,"з":1187,"и":1188,"к":1189,"л":1190,"м":1191,"н":1192,"о":1193,"п":1194,"р":1195,"с":1196,"т":1197,"у":1198,"ф":1199,"х":1200,"ц":1201,"ч":1202,"ш":1203,"щ":1204,"ъ":1205,"ы":1206,"ь":1207,"э":1208,"ю":1209,"я":1210,"ђ":1211,"є":1212,"і":1213,"ј":1214,"љ":1215,"њ":1216,"ћ":1217,"ӏ":1218,"ա":1219,"բ":1220,"գ":1221,"դ":1222,"ե":1223,"թ":1224,"ի":1225,"լ":1226,"կ":1227,"հ":1228,"մ":1229,"յ":1230,"ն":1231,"ո":1232,"պ":1233,"ս":1234,"վ":1235,"տ":1236,"ր":1237,"ւ":1238,"ք":1239,"־":1240,"א":1241,"ב":1242,"ג":1243,"ד":1244,"ה":1245,"ו":1246,"ז":1247,"ח":1248,"ט":1249,"י":1250,"ך":1251,"כ":1252,"ל":1253,"ם":1254,"מ":1255,"ן":1256,"נ":1257,"ס":1258,"ע":1259,"ף":1260,"פ":1261,"ץ":1262,"צ":1263,"ק":1264,"ר":1265,"ש":1266,"ת":1267,"،":1268,"ء":1269,"ا":1270,"ب":1271,"ة":1272,"ت":1273,"ث":1274,"ج":1275,"ح":1276,"خ":1277,"د":1278,"ذ":1279,"ر":1280,"ز":1281,"س":1282,"ش":1283,"ص":1284,"ض":1285,"ط":1286,"ظ":1287,"ع":1288,"غ":1289,"ـ":1290,"ف":1291,"ق":1292,"ك":1293,"ل":1294,"م":1295,"ن":1296,"ه":1297,"و":1298,"ى":1299,"ي":1300,"ٹ":1301,"پ":1302,"چ":1303,"ک":1304,"گ":1305,"ں":1306,"ھ":1307,"ہ":1308,"ی":1309,"ے":1310,"अ":1311,"आ":1312,"उ":1313,"ए":1314,"क":1315,"ख":1316,"ग":1317,"च":1318,"ज":1319,"ट":1320,"ड":1321,"ण":1322,"त":1323,"थ":1324,"द":1325,"ध":1326,"न":1327,"प":1328,"ब":1329,"भ":1330,"म":1331,"य":1332,"र":1333,"ल":1334,"व":1335,"श":1336,"ष":1337,"स":1338,"ह":1339,"ा":1340,"ि":1341,"ी":1342,"ो":1343,"।":1344,"॥":1345,"ং":1346,"অ":1347,"আ":1348,"ই":1349,"উ":1350,"এ":1351,"ও":1352,"ক":1353,"খ":1354,"গ":1355,"চ":1356,"ছ":1357,"জ":1358,"ট":1359,"ড":1360,"ণ":1361,"ত":1362,"থ":1363,"দ":1364,"ধ":1365,"ন":1366,"প":1367,"ব":1368,"ভ":1369,"ম":1370,"য":1371,"র":1372,"ল":1373,"শ":1374,"ষ":1375,"স":1376,"হ":1377,"া":1378,"ি":1379,"ী":1380,"ে":1381,"க":1382,"ச":1383,"ட":1384,"த":1385,"ந":1386,"ன":1387,"ப":1388,"ம":1389,"ய":1390,"ர":1391,"ல":1392,"ள":1393,"வ":1394,"ா":1395,"ி":1396,"ு":1397,"ே":1398,"ை":1399,"ನ":1400,"ರ":1401,"ಾ":1402,"ක":1403,"ය":1404,"ර":1405,"ල":1406,"ව":1407,"ා":1408,"ก":1409,"ง":1410,"ต":1411,"ท":1412,"น":1413,"พ":1414,"ม":1415,"ย":1416,"ร":1417,"ล":1418,"ว":1419,"ส":1420,"อ":1421,"า":1422,"เ":1423,"་":1424,"།":1425,"ག":1426,"ང":1427,"ད":1428,"ན":1429,"པ":1430,"བ":1431,"མ":1432,"འ":1433,"ར":1434,"ལ":1435,"ས":1436,"မ":1437,"ა":1438,"ბ":1439,"გ":1440,"დ":1441,"ე":1442,"ვ":1443,"თ":1444,"ი":1445,"კ":1446,"ლ":1447,"მ":1448,"ნ":1449,"ო":1450,"რ":1451,"ს":1452,"ტ":1453,"უ":1454,"ᄀ":1455,"ᄂ":1456,"ᄃ":1457,"ᄅ":1458,"ᄆ":1459,"ᄇ":1460,"ᄉ":1461,"ᄊ":1462,"ᄋ":1463,"ᄌ":1464,"ᄎ":1465,"ᄏ":1466,"ᄐ":1467,"ᄑ":1468,"ᄒ":1469,"ᅡ":1470,"ᅢ":1471,"ᅥ":1472,"ᅦ":1473,"ᅧ":1474,"ᅩ":1475,"ᅪ":1476,"ᅭ":1477,"ᅮ":1478,"ᅯ":1479,"ᅲ":1480,"ᅳ":1481,"ᅴ":1482,"ᅵ":1483,"ᆨ":1484,"ᆫ":1485,"ᆯ":1486,"ᆷ":1487,"ᆸ":1488,"ᆼ":1489,"ᴬ":1490,"ᴮ":1491,"ᴰ":1492,"ᴵ":1493,"ᴺ":1494,"ᵀ":1495,"ᵃ":1496,"ᵇ":1497,"ᵈ":1498,"ᵉ":1499,"ᵍ":1500,"ᵏ":1501,"ᵐ":1502,"ᵒ":1503,"ᵖ":1504,"ᵗ":1505,"ᵘ":1506,"ᵢ":1507,"ᵣ":1508,"ᵤ":1509,"ᵥ":1510,"ᶜ":1511,"ᶠ":1512,"‐":1513,"‑":1514,"‒":1515,"–":1516,"—":1517,"―":1518,"‖":1519,"‘":1520,"’":1521,"‚":1522,"“":1523,"”":1524,"„":1525,"†":1526,"‡":1527,"•":1528,"…":1529,"‰":1530,"′":1531,"″":1532,"›":1533,"‿":1534,"⁄":1535,"⁰":1536,"ⁱ":1537,"⁴":1538,"⁵":1539,"⁶":1540,"⁷":1541,"⁸":1542,"⁹":1543,"⁺":1544,"⁻":1545,"ⁿ":1546,"₀":1547,"₁":1548,"₂":1549,"₃":1550,"₄":1551,"₅":1552,"₆":1553,"₇":1554,"₈":1555,"₉":1556,"₊":1557,"₍":1558,"₎":1559,"ₐ":1560,"ₑ":1561,"ₒ":1562,"ₓ":1563,"ₕ":1564,"ₖ":1565,"ₗ":1566,"ₘ":1567,"ₙ":1568,"ₚ":1569,"ₛ":1570,"ₜ":1571,"₤":1572,"₩":1573,"€":1574,"₱":1575,"₹":1576,"ℓ":1577,"№":1578,"ℝ":1579,"™":1580,"⅓":1581,"⅔":1582,"←":1583,"↑":1584,"→":1585,"↓":1586,"↔":1587,"↦":1588,"⇄":1589,"⇌":1590,"⇒":1591,"∂":1592,"∅":1593,"∆":1594,"∇":1595,"∈":1596,"−":1597,"∗":1598,"∘":1599,"√":1600,"∞":1601,"∧":1602,"∨":1603,"∩":1604,"∪":1605,"≈":1606,"≡":1607,"≤":1608,"≥":1609,"⊂":1610,"⊆":1611,"⊕":1612,"⊗":1613,"⋅":1614,"─":1615,"│":1616,"■":1617,"▪":1618,"●":1619,"★":1620,"☆":1621,"☉":1622,"♠":1623,"♣":1624,"♥":1625,"♦":1626,"♭":1627,"♯":1628,"⟨":1629,"⟩":1630,"ⱼ":1631,"⺩":1632,"⺼":1633,"⽥":1634,"、":1635,"。":1636,"〈":1637,"〉":1638,"《":1639,"》":1640,"「":1641,"」":1642,"『":1643,"』":1644,"〜":1645,"あ":1646,"い":1647,"う":1648,"え":1649,"お":1650,"か":1651,"き":1652,"く":1653,"け":1654,"こ":1655,"さ":1656,"し":1657,"す":1658,"せ":1659,"そ":1660,"た":1661,"ち":1662,"っ":1663,"つ":1664,"て":1665,"と":1666,"な":1667,"に":1668,"ぬ":1669,"ね":1670,"の":1671,"は":1672,"ひ":1673,"ふ":1674,"へ":1675,"ほ":1676,"ま":1677,"み":1678,"む":1679,"め":1680,"も":1681,"や":1682,"ゆ":1683,"よ":1684,"ら":1685,"り":1686,"る":1687,"れ":1688,"ろ":1689,"を":1690,"ん":1691,"ァ":1692,"ア":1693,"ィ":1694,"イ":1695,"ウ":1696,"ェ":1697,"エ":1698,"オ":1699,"カ":1700,"キ":1701,"ク":1702,"ケ":1703,"コ":1704,"サ":1705,"シ":1706,"ス":1707,"セ":1708,"タ":1709,"チ":1710,"ッ":1711,"ツ":1712,"テ":1713,"ト":1714,"ナ":1715,"ニ":1716,"ノ":1717,"ハ":1718,"ヒ":1719,"フ":1720,"ヘ":1721,"ホ":1722,"マ":1723,"ミ":1724,"ム":1725,"メ":1726,"モ":1727,"ャ":1728,"ュ":1729,"ョ":1730,"ラ":1731,"リ":1732,"ル":1733,"レ":1734,"ロ":1735,"ワ":1736,"ン":1737,"・":1738,"ー":1739,"一":1740,"三":1741,"上":1742,"下":1743,"不":1744,"世":1745,"中":1746,"主":1747,"久":1748,"之":1749,"也":1750,"事":1751,"二":1752,"五":1753,"井":1754,"京":1755,"人":1756,"亻":1757,"仁":1758,"介":1759,"代":1760,"仮":1761,"伊":1762,"会":1763,"佐":1764,"侍":1765,"保":1766,"信":1767,"健":1768,"元":1769,"光":1770,"八":1771,"公":1772,"内":1773,"出":1774,"分":1775,"前":1776,"劉":1777,"力":1778,"加":1779,"勝":1780,"北":1781,"区":1782,"十":1783,"千":1784,"南":1785,"博":1786,"原":1787,"口":1788,"古":1789,"史":1790,"司":1791,"合":1792,"吉":1793,"同":1794,"名":1795,"和":1796,"囗":1797,"四":1798,"国":1799,"國":1800,"土":1801,"地":1802,"坂":1803,"城":1804,"堂":1805,"場":1806,"士":1807,"夏":1808,"外":1809,"大":1810,"天":1811,"太":1812,"夫":1813,"奈":1814,"女":1815,"子":1816,"学":1817,"宀":1818,"宇":1819,"安":1820,"宗":1821,"定":1822,"宣":1823,"宮":1824,"家":1825,"宿":1826,"寺":1827,"將":1828,"小":1829,"尚":1830,"山":1831,"岡":1832,"島":1833,"崎":1834,"川":1835,"州":1836,"巿":1837,"帝":1838,"平":1839,"年":1840,"幸":1841,"广":1842,"弘":1843,"張":1844,"彳":1845,"後":1846,"御":1847,"德":1848,"心":1849,"忄":1850,"志":1851,"忠":1852,"愛":1853,"成":1854,"我":1855,"戦":1856,"戸":1857,"手":1858,"扌":1859,"政":1860,"文":1861,"新":1862,"方":1863,"日":1864,"明":1865,"星":1866,"春":1867,"昭":1868,"智":1869,"曲":1870,"書":1871,"月":1872,"有":1873,"朝":1874,"木":1875,"本":1876,"李":1877,"村":1878,"東":1879,"松":1880,"林":1881,"森":1882,"楊":1883,"樹":1884,"橋":1885,"歌":1886,"止":1887,"正":1888,"武":1889,"比":1890,"氏":1891,"民":1892,"水":1893,"氵":1894,"氷":1895,"永":1896,"江":1897,"沢":1898,"河":1899,"治":1900,"法":1901,"海":1902,"清":1903,"漢":1904,"瀬":1905,"火":1906,"版":1907,"犬":1908,"王":1909,"生":1910,"田":1911,"男":1912,"疒":1913,"発":1914,"白":1915,"的":1916,"皇":1917,"目":1918,"相":1919,"省":1920,"真":1921,"石":1922,"示":1923,"社":1924,"神":1925,"福":1926,"禾":1927,"秀":1928,"秋":1929,"空":1930,"立":1931,"章":1932,"竹":1933,"糹":1934,"美":1935,"義":1936,"耳":1937,"良":1938,"艹":1939,"花":1940,"英":1941,"華":1942,"葉":1943,"藤":1944,"行":1945,"街":1946,"西":1947,"見":1948,"訁":1949,"語":1950,"谷":1951,"貝":1952,"貴":1953,"車":1954,"軍":1955,"辶":1956,"道":1957,"郎":1958,"郡":1959,"部":1960,"都":1961,"里":1962,"野":1963,"金":1964,"鈴":1965,"镇":1966,"長":1967,"門":1968,"間":1969,"阝":1970,"阿":1971,"陳":1972,"陽":1973,"雄":1974,"青":1975,"面":1976,"風":1977,"食":1978,"香":1979,"馬":1980,"高":1981,"龍":1982,"龸":1983,"fi":1984,"fl":1985,"！":1986,"（":1987,"）":1988,"，":1989,"－":1990,"．":1991,"／":1992,"：":1993,"？":1994,"～":1995,"the":1996,"of":1997,"and":1998,"in":1999,"to":2000,"was":2001,"he":2002,"is":2003,"as":2004,"for":2005,"on":2006,"with":2007,"that":2008,"it":2009,"his":2010,"by":2011,"at":2012,"from":2013,"her":2014,"##s":2015,"she":2016,"you":2017,"had":2018,"an":2019,"were":2020,"but":2021,"be":2022,"this":2023,"are":2024,"not":2025,"my":2026,"they":2027,"one":2028,"which":2029,"or":2030,"have":2031,"him":2032,"me":2033,"first":2034,"all":2035,"also":2036,"their":2037,"has":2038,"up":2039,"who":2040,"out":2041,"been":2042,"when":2043,"after":2044,"there":2045,"into":2046,"new":2047,"two":2048,"its":2049,"##a":2050,"time":2051,"would":2052,"no":2053,"what":2054,"about":2055,"said":2056,"we":2057,"over":2058,"then":2059,"other":2060,"so":2061,"more":2062,"##e":2063,"can":2064,"if":2065,"like":2066,"back":2067,"them":2068,"only":2069,"some":2070,"could":2071,"##i":2072,"where":2073,"just":2074,"##ing":2075,"during":2076,"before":2077,"##n":2078,"do":2079,"##o":2080,"made":2081,"school":2082,"through":2083,"than":2084,"now":2085,"years":2086,"most":2087,"world":2088,"may":2089,"between":2090,"down":2091,"well":2092,"three":2093,"##d":2094,"year":2095,"while":2096,"will":2097,"##ed":2098,"##r":2099,"##y":2100,"later":2101,"##t":2102,"city":2103,"under":2104,"around":2105,"did":2106,"such":2107,"being":2108,"used":2109,"state":2110,"people":2111,"part":2112,"know":2113,"against":2114,"your":2115,"many":2116,"second":2117,"university":2118,"both":2119,"national":2120,"##er":2121,"these":2122,"don":2123,"known":2124,"off":2125,"way":2126,"until":2127,"re":2128,"how":2129,"even":2130,"get":2131,"head":2132,"...":2133,"didn":2134,"##ly":2135,"team":2136,"american":2137,"because":2138,"de":2139,"##l":2140,"born":2141,"united":2142,"film":2143,"since":2144,"still":2145,"long":2146,"work":2147,"south":2148,"us":2149,"became":2150,"any":2151,"high":2152,"again":2153,"day":2154,"family":2155,"see":2156,"right":2157,"man":2158,"eyes":2159,"house":2160,"season":2161,"war":2162,"states":2163,"including":2164,"took":2165,"life":2166,"north":2167,"same":2168,"each":2169,"called":2170,"name":2171,"much":2172,"place":2173,"however":2174,"go":2175,"four":2176,"group":2177,"another":2178,"found":2179,"won":2180,"area":2181,"here":2182,"going":2183,"10":2184,"away":2185,"series":2186,"left":2187,"home":2188,"music":2189,"best":2190,"make":2191,"hand":2192,"number":2193,"company":2194,"several":2195,"never":2196,"last":2197,"john":2198,"000":2199,"very":2200,"album":2201,"take":2202,"end":2203,"good":2204,"too":2205,"following":2206,"released":2207,"game":2208,"played":2209,"little":2210,"began":2211,"district":2212,"##m":2213,"old":2214,"want":2215,"those":2216,"side":2217,"held":2218,"own":2219,"early":2220,"county":2221,"ll":2222,"league":2223,"use":2224,"west":2225,"##u":2226,"face":2227,"think":2228,"##es":2229,"2010":2230,"government":2231,"##h":2232,"march":2233,"came":2234,"small":2235,"general":2236,"town":2237,"june":2238,"##on":2239,"line":2240,"based":2241,"something":2242,"##k":2243,"september":2244,"thought":2245,"looked":2246,"along":2247,"international":2248,"2011":2249,"air":2250,"july":2251,"club":2252,"went":2253,"january":2254,"october":2255,"our":2256,"august":2257,"april":2258,"york":2259,"12":2260,"few":2261,"2012":2262,"2008":2263,"east":2264,"show":2265,"member":2266,"college":2267,"2009":2268,"father":2269,"public":2270,"##us":2271,"come":2272,"men":2273,"five":2274,"set":2275,"station":2276,"church":2277,"##c":2278,"next":2279,"former":2280,"november":2281,"room":2282,"party":2283,"located":2284,"december":2285,"2013":2286,"age":2287,"got":2288,"2007":2289,"##g":2290,"system":2291,"let":2292,"love":2293,"2006":2294,"though":2295,"every":2296,"2014":2297,"look":2298,"song":2299,"water":2300,"century":2301,"without":2302,"body":2303,"black":2304,"night":2305,"within":2306,"great":2307,"women":2308,"single":2309,"ve":2310,"building":2311,"large":2312,"population":2313,"river":2314,"named":2315,"band":2316,"white":2317,"started":2318,"##an":2319,"once":2320,"15":2321,"20":2322,"should":2323,"18":2324,"2015":2325,"service":2326,"top":2327,"built":2328,"british":2329,"open":2330,"death":2331,"king":2332,"moved":2333,"local":2334,"times":2335,"children":2336,"february":2337,"book":2338,"why":2339,"11":2340,"door":2341,"need":2342,"president":2343,"order":2344,"final":2345,"road":2346,"wasn":2347,"although":2348,"due":2349,"major":2350,"died":2351,"village":2352,"third":2353,"knew":2354,"2016":2355,"asked":2356,"turned":2357,"st":2358,"wanted":2359,"say":2360,"##p":2361,"together":2362,"received":2363,"main":2364,"son":2365,"served":2366,"different":2367,"##en":2368,"behind":2369,"himself":2370,"felt":2371,"members":2372,"power":2373,"football":2374,"law":2375,"voice":2376,"play":2377,"##in":2378,"near":2379,"park":2380,"history":2381,"30":2382,"having":2383,"2005":2384,"16":2385,"##man":2386,"saw":2387,"mother":2388,"##al":2389,"army":2390,"point":2391,"front":2392,"help":2393,"english":2394,"street":2395,"art":2396,"late":2397,"hands":2398,"games":2399,"award":2400,"##ia":2401,"young":2402,"14":2403,"put":2404,"published":2405,"country":2406,"division":2407,"across":2408,"told":2409,"13":2410,"often":2411,"ever":2412,"french":2413,"london":2414,"center":2415,"six":2416,"red":2417,"2017":2418,"led":2419,"days":2420,"include":2421,"light":2422,"25":2423,"find":2424,"tell":2425,"among":2426,"species":2427,"really":2428,"according":2429,"central":2430,"half":2431,"2004":2432,"form":2433,"original":2434,"gave":2435,"office":2436,"making":2437,"enough":2438,"lost":2439,"full":2440,"opened":2441,"must":2442,"included":2443,"live":2444,"given":2445,"german":2446,"player":2447,"run":2448,"business":2449,"woman":2450,"community":2451,"cup":2452,"might":2453,"million":2454,"land":2455,"2000":2456,"court":2457,"development":2458,"17":2459,"short":2460,"round":2461,"ii":2462,"km":2463,"seen":2464,"class":2465,"story":2466,"always":2467,"become":2468,"sure":2469,"research":2470,"almost":2471,"director":2472,"council":2473,"la":2474,"##2":2475,"career":2476,"things":2477,"using":2478,"island":2479,"##z":2480,"couldn":2481,"car":2482,"##is":2483,"24":2484,"close":2485,"force":2486,"##1":2487,"better":2488,"free":2489,"support":2490,"control":2491,"field":2492,"students":2493,"2003":2494,"education":2495,"married":2496,"##b":2497,"nothing":2498,"worked":2499,"others":2500,"record":2501,"big":2502,"inside":2503,"level":2504,"anything":2505,"continued":2506,"give":2507,"james":2508,"##3":2509,"military":2510,"established":2511,"non":2512,"returned":2513,"feel":2514,"does":2515,"title":2516,"written":2517,"thing":2518,"feet":2519,"william":2520,"far":2521,"co":2522,"association":2523,"hard":2524,"already":2525,"2002":2526,"##ra":2527,"championship":2528,"human":2529,"western":2530,"100":2531,"##na":2532,"department":2533,"hall":2534,"role":2535,"various":2536,"production":2537,"21":2538,"19":2539,"heart":2540,"2001":2541,"living":2542,"fire":2543,"version":2544,"##ers":2545,"##f":2546,"television":2547,"royal":2548,"##4":2549,"produced":2550,"working":2551,"act":2552,"case":2553,"society":2554,"region":2555,"present":2556,"radio":2557,"period":2558,"looking":2559,"least":2560,"total":2561,"keep":2562,"england":2563,"wife":2564,"program":2565,"per":2566,"brother":2567,"mind":2568,"special":2569,"22":2570,"##le":2571,"am":2572,"works":2573,"soon":2574,"##6":2575,"political":2576,"george":2577,"services":2578,"taken":2579,"created":2580,"##7":2581,"further":2582,"able":2583,"reached":2584,"david":2585,"union":2586,"joined":2587,"upon":2588,"done":2589,"important":2590,"social":2591,"information":2592,"either":2593,"##ic":2594,"##x":2595,"appeared":2596,"position":2597,"ground":2598,"lead":2599,"rock":2600,"dark":2601,"election":2602,"23":2603,"board":2604,"france":2605,"hair":2606,"course":2607,"arms":2608,"site":2609,"police":2610,"girl":2611,"instead":2612,"real":2613,"sound":2614,"##v":2615,"words":2616,"moment":2617,"##te":2618,"someone":2619,"##8":2620,"summer":2621,"project":2622,"announced":2623,"san":2624,"less":2625,"wrote":2626,"past":2627,"followed":2628,"##5":2629,"blue":2630,"founded":2631,"al":2632,"finally":2633,"india":2634,"taking":2635,"records":2636,"america":2637,"##ne":2638,"1999":2639,"design":2640,"considered":2641,"northern":2642,"god":2643,"stop":2644,"battle":2645,"toward":2646,"european":2647,"outside":2648,"described":2649,"track":2650,"today":2651,"playing":2652,"language":2653,"28":2654,"call":2655,"26":2656,"heard":2657,"professional":2658,"low":2659,"australia":2660,"miles":2661,"california":2662,"win":2663,"yet":2664,"green":2665,"##ie":2666,"trying":2667,"blood":2668,"##ton":2669,"southern":2670,"science":2671,"maybe":2672,"everything":2673,"match":2674,"square":2675,"27":2676,"mouth":2677,"video":2678,"race":2679,"recorded":2680,"leave":2681,"above":2682,"##9":2683,"daughter":2684,"points":2685,"space":2686,"1998":2687,"museum":2688,"change":2689,"middle":2690,"common":2691,"##0":2692,"move":2693,"tv":2694,"post":2695,"##ta":2696,"lake":2697,"seven":2698,"tried":2699,"elected":2700,"closed":2701,"ten":2702,"paul":2703,"minister":2704,"##th":2705,"months":2706,"start":2707,"chief":2708,"return":2709,"canada":2710,"person":2711,"sea":2712,"release":2713,"similar":2714,"modern":2715,"brought":2716,"rest":2717,"hit":2718,"formed":2719,"mr":2720,"##la":2721,"1997":2722,"floor":2723,"event":2724,"doing":2725,"thomas":2726,"1996":2727,"robert":2728,"care":2729,"killed":2730,"training":2731,"star":2732,"week":2733,"needed":2734,"turn":2735,"finished":2736,"railway":2737,"rather":2738,"news":2739,"health":2740,"sent":2741,"example":2742,"ran":2743,"term":2744,"michael":2745,"coming":2746,"currently":2747,"yes":2748,"forces":2749,"despite":2750,"gold":2751,"areas":2752,"50":2753,"stage":2754,"fact":2755,"29":2756,"dead":2757,"says":2758,"popular":2759,"2018":2760,"originally":2761,"germany":2762,"probably":2763,"developed":2764,"result":2765,"pulled":2766,"friend":2767,"stood":2768,"money":2769,"running":2770,"mi":2771,"signed":2772,"word":2773,"songs":2774,"child":2775,"eventually":2776,"met":2777,"tour":2778,"average":2779,"teams":2780,"minutes":2781,"festival":2782,"current":2783,"deep":2784,"kind":2785,"1995":2786,"decided":2787,"usually":2788,"eastern":2789,"seemed":2790,"##ness":2791,"episode":2792,"bed":2793,"added":2794,"table":2795,"indian":2796,"private":2797,"charles":2798,"route":2799,"available":2800,"idea":2801,"throughout":2802,"centre":2803,"addition":2804,"appointed":2805,"style":2806,"1994":2807,"books":2808,"eight":2809,"construction":2810,"press":2811,"mean":2812,"wall":2813,"friends":2814,"remained":2815,"schools":2816,"study":2817,"##ch":2818,"##um":2819,"institute":2820,"oh":2821,"chinese":2822,"sometimes":2823,"events":2824,"possible":2825,"1992":2826,"australian":2827,"type":2828,"brown":2829,"forward":2830,"talk":2831,"process":2832,"food":2833,"debut":2834,"seat":2835,"performance":2836,"committee":2837,"features":2838,"character":2839,"arts":2840,"herself":2841,"else":2842,"lot":2843,"strong":2844,"russian":2845,"range":2846,"hours":2847,"peter":2848,"arm":2849,"##da":2850,"morning":2851,"dr":2852,"sold":2853,"##ry":2854,"quickly":2855,"directed":2856,"1993":2857,"guitar":2858,"china":2859,"##w":2860,"31":2861,"list":2862,"##ma":2863,"performed":2864,"media":2865,"uk":2866,"players":2867,"smile":2868,"##rs":2869,"myself":2870,"40":2871,"placed":2872,"coach":2873,"province":2874,"towards":2875,"wouldn":2876,"leading":2877,"whole":2878,"boy":2879,"official":2880,"designed":2881,"grand":2882,"census":2883,"##el":2884,"europe":2885,"attack":2886,"japanese":2887,"henry":2888,"1991":2889,"##re":2890,"##os":2891,"cross":2892,"getting":2893,"alone":2894,"action":2895,"lower":2896,"network":2897,"wide":2898,"washington":2899,"japan":2900,"1990":2901,"hospital":2902,"believe":2903,"changed":2904,"sister":2905,"##ar":2906,"hold":2907,"gone":2908,"sir":2909,"hadn":2910,"ship":2911,"##ka":2912,"studies":2913,"academy":2914,"shot":2915,"rights":2916,"below":2917,"base":2918,"bad":2919,"involved":2920,"kept":2921,"largest":2922,"##ist":2923,"bank":2924,"future":2925,"especially":2926,"beginning":2927,"mark":2928,"movement":2929,"section":2930,"female":2931,"magazine":2932,"plan":2933,"professor":2934,"lord":2935,"longer":2936,"##ian":2937,"sat":2938,"walked":2939,"hill":2940,"actually":2941,"civil":2942,"energy":2943,"model":2944,"families":2945,"size":2946,"thus":2947,"aircraft":2948,"completed":2949,"includes":2950,"data":2951,"captain":2952,"##or":2953,"fight":2954,"vocals":2955,"featured":2956,"richard":2957,"bridge":2958,"fourth":2959,"1989":2960,"officer":2961,"stone":2962,"hear":2963,"##ism":2964,"means":2965,"medical":2966,"groups":2967,"management":2968,"self":2969,"lips":2970,"competition":2971,"entire":2972,"lived":2973,"technology":2974,"leaving":2975,"federal":2976,"tournament":2977,"bit":2978,"passed":2979,"hot":2980,"independent":2981,"awards":2982,"kingdom":2983,"mary":2984,"spent":2985,"fine":2986,"doesn":2987,"reported":2988,"##ling":2989,"jack":2990,"fall":2991,"raised":2992,"itself":2993,"stay":2994,"true":2995,"studio":2996,"1988":2997,"sports":2998,"replaced":2999,"paris":3000,"systems":3001,"saint":3002,"leader":3003,"theatre":3004,"whose":3005,"market":3006,"capital":3007,"parents":3008,"spanish":3009,"canadian":3010,"earth":3011,"##ity":3012,"cut":3013,"degree":3014,"writing":3015,"bay":3016,"christian":3017,"awarded":3018,"natural":3019,"higher":3020,"bill":3021,"##as":3022,"coast":3023,"provided":3024,"previous":3025,"senior":3026,"ft":3027,"valley":3028,"organization":3029,"stopped":3030,"onto":3031,"countries":3032,"parts":3033,"conference":3034,"queen":3035,"security":3036,"interest":3037,"saying":3038,"allowed":3039,"master":3040,"earlier":3041,"phone":3042,"matter":3043,"smith":3044,"winning":3045,"try":3046,"happened":3047,"moving":3048,"campaign":3049,"los":3050,"##ley":3051,"breath":3052,"nearly":3053,"mid":3054,"1987":3055,"certain":3056,"girls":3057,"date":3058,"italian":3059,"african":3060,"standing":3061,"fell":3062,"artist":3063,"##ted":3064,"shows":3065,"deal":3066,"mine":3067,"industry":3068,"1986":3069,"##ng":3070,"everyone":3071,"republic":3072,"provide":3073,"collection":3074,"library":3075,"student":3076,"##ville":3077,"primary":3078,"owned":3079,"older":3080,"via":3081,"heavy":3082,"1st":3083,"makes":3084,"##able":3085,"attention":3086,"anyone":3087,"africa":3088,"##ri":3089,"stated":3090,"length":3091,"ended":3092,"fingers":3093,"command":3094,"staff":3095,"skin":3096,"foreign":3097,"opening":3098,"governor":3099,"okay":3100,"medal":3101,"kill":3102,"sun":3103,"cover":3104,"job":3105,"1985":3106,"introduced":3107,"chest":3108,"hell":3109,"feeling":3110,"##ies":3111,"success":3112,"meet":3113,"reason":3114,"standard":3115,"meeting":3116,"novel":3117,"1984":3118,"trade":3119,"source":3120,"buildings":3121,"##land":3122,"rose":3123,"guy":3124,"goal":3125,"##ur":3126,"chapter":3127,"native":3128,"husband":3129,"previously":3130,"unit":3131,"limited":3132,"entered":3133,"weeks":3134,"producer":3135,"operations":3136,"mountain":3137,"takes":3138,"covered":3139,"forced":3140,"related":3141,"roman":3142,"complete":3143,"successful":3144,"key":3145,"texas":3146,"cold":3147,"##ya":3148,"channel":3149,"1980":3150,"traditional":3151,"films":3152,"dance":3153,"clear":3154,"approximately":3155,"500":3156,"nine":3157,"van":3158,"prince":3159,"question":3160,"active":3161,"tracks":3162,"ireland":3163,"regional":3164,"silver":3165,"author":3166,"personal":3167,"sense":3168,"operation":3169,"##ine":3170,"economic":3171,"1983":3172,"holding":3173,"twenty":3174,"isbn":3175,"additional":3176,"speed":3177,"hour":3178,"edition":3179,"regular":3180,"historic":3181,"places":3182,"whom":3183,"shook":3184,"movie":3185,"km²":3186,"secretary":3187,"prior":3188,"report":3189,"chicago":3190,"read":3191,"foundation":3192,"view":3193,"engine":3194,"scored":3195,"1982":3196,"units":3197,"ask":3198,"airport":3199,"property":3200,"ready":3201,"immediately":3202,"lady":3203,"month":3204,"listed":3205,"contract":3206,"##de":3207,"manager":3208,"themselves":3209,"lines":3210,"##ki":3211,"navy":3212,"writer":3213,"meant":3214,"##ts":3215,"runs":3216,"##ro":3217,"practice":3218,"championships":3219,"singer":3220,"glass":3221,"commission":3222,"required":3223,"forest":3224,"starting":3225,"culture":3226,"generally":3227,"giving":3228,"access":3229,"attended":3230,"test":3231,"couple":3232,"stand":3233,"catholic":3234,"martin":3235,"caught":3236,"executive":3237,"##less":3238,"eye":3239,"##ey":3240,"thinking":3241,"chair":3242,"quite":3243,"shoulder":3244,"1979":3245,"hope":3246,"decision":3247,"plays":3248,"defeated":3249,"municipality":3250,"whether":3251,"structure":3252,"offered":3253,"slowly":3254,"pain":3255,"ice":3256,"direction":3257,"##ion":3258,"paper":3259,"mission":3260,"1981":3261,"mostly":3262,"200":3263,"noted":3264,"individual":3265,"managed":3266,"nature":3267,"lives":3268,"plant":3269,"##ha":3270,"helped":3271,"except":3272,"studied":3273,"computer":3274,"figure":3275,"relationship":3276,"issue":3277,"significant":3278,"loss":3279,"die":3280,"smiled":3281,"gun":3282,"ago":3283,"highest":3284,"1972":3285,"##am":3286,"male":3287,"bring":3288,"goals":3289,"mexico":3290,"problem":3291,"distance":3292,"commercial":3293,"completely":3294,"location":3295,"annual":3296,"famous":3297,"drive":3298,"1976":3299,"neck":3300,"1978":3301,"surface":3302,"caused":3303,"italy":3304,"understand":3305,"greek":3306,"highway":3307,"wrong":3308,"hotel":3309,"comes":3310,"appearance":3311,"joseph":3312,"double":3313,"issues":3314,"musical":3315,"companies":3316,"castle":3317,"income":3318,"review":3319,"assembly":3320,"bass":3321,"initially":3322,"parliament":3323,"artists":3324,"experience":3325,"1974":3326,"particular":3327,"walk":3328,"foot":3329,"engineering":3330,"talking":3331,"window":3332,"dropped":3333,"##ter":3334,"miss":3335,"baby":3336,"boys":3337,"break":3338,"1975":3339,"stars":3340,"edge":3341,"remember":3342,"policy":3343,"carried":3344,"train":3345,"stadium":3346,"bar":3347,"sex":3348,"angeles":3349,"evidence":3350,"##ge":3351,"becoming":3352,"assistant":3353,"soviet":3354,"1977":3355,"upper":3356,"step":3357,"wing":3358,"1970":3359,"youth":3360,"financial":3361,"reach":3362,"##ll":3363,"actor":3364,"numerous":3365,"##se":3366,"##st":3367,"nodded":3368,"arrived":3369,"##ation":3370,"minute":3371,"##nt":3372,"believed":3373,"sorry":3374,"complex":3375,"beautiful":3376,"victory":3377,"associated":3378,"temple":3379,"1968":3380,"1973":3381,"chance":3382,"perhaps":3383,"metal":3384,"##son":3385,"1945":3386,"bishop":3387,"##et":3388,"lee":3389,"launched":3390,"particularly":3391,"tree":3392,"le":3393,"retired":3394,"subject":3395,"prize":3396,"contains":3397,"yeah":3398,"theory":3399,"empire":3400,"##ce":3401,"suddenly":3402,"waiting":3403,"trust":3404,"recording":3405,"##to":3406,"happy":3407,"terms":3408,"camp":3409,"champion":3410,"1971":3411,"religious":3412,"pass":3413,"zealand":3414,"names":3415,"2nd":3416,"port":3417,"ancient":3418,"tom":3419,"corner":3420,"represented":3421,"watch":3422,"legal":3423,"anti":3424,"justice":3425,"cause":3426,"watched":3427,"brothers":3428,"45":3429,"material":3430,"changes":3431,"simply":3432,"response":3433,"louis":3434,"fast":3435,"##ting":3436,"answer":3437,"60":3438,"historical":3439,"1969":3440,"stories":3441,"straight":3442,"create":3443,"feature":3444,"increased":3445,"rate":3446,"administration":3447,"virginia":3448,"el":3449,"activities":3450,"cultural":3451,"overall":3452,"winner":3453,"programs":3454,"basketball":3455,"legs":3456,"guard":3457,"beyond":3458,"cast":3459,"doctor":3460,"mm":3461,"flight":3462,"results":3463,"remains":3464,"cost":3465,"effect":3466,"winter":3467,"##ble":3468,"larger":3469,"islands":3470,"problems":3471,"chairman":3472,"grew":3473,"commander":3474,"isn":3475,"1967":3476,"pay":3477,"failed":3478,"selected":3479,"hurt":3480,"fort":3481,"box":3482,"regiment":3483,"majority":3484,"journal":3485,"35":3486,"edward":3487,"plans":3488,"##ke":3489,"##ni":3490,"shown":3491,"pretty":3492,"irish":3493,"characters":3494,"directly":3495,"scene":3496,"likely":3497,"operated":3498,"allow":3499,"spring":3500,"##j":3501,"junior":3502,"matches":3503,"looks":3504,"mike":3505,"houses":3506,"fellow":3507,"##tion":3508,"beach":3509,"marriage":3510,"##ham":3511,"##ive":3512,"rules":3513,"oil":3514,"65":3515,"florida":3516,"expected":3517,"nearby":3518,"congress":3519,"sam":3520,"peace":3521,"recent":3522,"iii":3523,"wait":3524,"subsequently":3525,"cell":3526,"##do":3527,"variety":3528,"serving":3529,"agreed":3530,"please":3531,"poor":3532,"joe":3533,"pacific":3534,"attempt":3535,"wood":3536,"democratic":3537,"piece":3538,"prime":3539,"##ca":3540,"rural":3541,"mile":3542,"touch":3543,"appears":3544,"township":3545,"1964":3546,"1966":3547,"soldiers":3548,"##men":3549,"##ized":3550,"1965":3551,"pennsylvania":3552,"closer":3553,"fighting":3554,"claimed":3555,"score":3556,"jones":3557,"physical":3558,"editor":3559,"##ous":3560,"filled":3561,"genus":3562,"specific":3563,"sitting":3564,"super":3565,"mom":3566,"##va":3567,"therefore":3568,"supported":3569,"status":3570,"fear":3571,"cases":3572,"store":3573,"meaning":3574,"wales":3575,"minor":3576,"spain":3577,"tower":3578,"focus":3579,"vice":3580,"frank":3581,"follow":3582,"parish":3583,"separate":3584,"golden":3585,"horse":3586,"fifth":3587,"remaining":3588,"branch":3589,"32":3590,"presented":3591,"stared":3592,"##id":3593,"uses":3594,"secret":3595,"forms":3596,"##co":3597,"baseball":3598,"exactly":3599,"##ck":3600,"choice":3601,"note":3602,"discovered":3603,"travel":3604,"composed":3605,"truth":3606,"russia":3607,"ball":3608,"color":3609,"kiss":3610,"dad":3611,"wind":3612,"continue":3613,"ring":3614,"referred":3615,"numbers":3616,"digital":3617,"greater":3618,"##ns":3619,"metres":3620,"slightly":3621,"direct":3622,"increase":3623,"1960":3624,"responsible":3625,"crew":3626,"rule":3627,"trees":3628,"troops":3629,"##no":3630,"broke":3631,"goes":3632,"individuals":3633,"hundred":3634,"weight":3635,"creek":3636,"sleep":3637,"memory":3638,"defense":3639,"provides":3640,"ordered":3641,"code":3642,"value":3643,"jewish":3644,"windows":3645,"1944":3646,"safe":3647,"judge":3648,"whatever":3649,"corps":3650,"realized":3651,"growing":3652,"pre":3653,"##ga":3654,"cities":3655,"alexander":3656,"gaze":3657,"lies":3658,"spread":3659,"scott":3660,"letter":3661,"showed":3662,"situation":3663,"mayor":3664,"transport":3665,"watching":3666,"workers":3667,"extended":3668,"##li":3669,"expression":3670,"normal":3671,"##ment":3672,"chart":3673,"multiple":3674,"border":3675,"##ba":3676,"host":3677,"##ner":3678,"daily":3679,"mrs":3680,"walls":3681,"piano":3682,"##ko":3683,"heat":3684,"cannot":3685,"##ate":3686,"earned":3687,"products":3688,"drama":3689,"era":3690,"authority":3691,"seasons":3692,"join":3693,"grade":3694,"##io":3695,"sign":3696,"difficult":3697,"machine":3698,"1963":3699,"territory":3700,"mainly":3701,"##wood":3702,"stations":3703,"squadron":3704,"1962":3705,"stepped":3706,"iron":3707,"19th":3708,"##led":3709,"serve":3710,"appear":3711,"sky":3712,"speak":3713,"broken":3714,"charge":3715,"knowledge":3716,"kilometres":3717,"removed":3718,"ships":3719,"article":3720,"campus":3721,"simple":3722,"##ty":3723,"pushed":3724,"britain":3725,"##ve":3726,"leaves":3727,"recently":3728,"cd":3729,"soft":3730,"boston":3731,"latter":3732,"easy":3733,"acquired":3734,"poland":3735,"##sa":3736,"quality":3737,"officers":3738,"presence":3739,"planned":3740,"nations":3741,"mass":3742,"broadcast":3743,"jean":3744,"share":3745,"image":3746,"influence":3747,"wild":3748,"offer":3749,"emperor":3750,"electric":3751,"reading":3752,"headed":3753,"ability":3754,"promoted":3755,"yellow":3756,"ministry":3757,"1942":3758,"throat":3759,"smaller":3760,"politician":3761,"##by":3762,"latin":3763,"spoke":3764,"cars":3765,"williams":3766,"males":3767,"lack":3768,"pop":3769,"80":3770,"##ier":3771,"acting":3772,"seeing":3773,"consists":3774,"##ti":3775,"estate":3776,"1961":3777,"pressure":3778,"johnson":3779,"newspaper":3780,"jr":3781,"chris":3782,"olympics":3783,"online":3784,"conditions":3785,"beat":3786,"elements":3787,"walking":3788,"vote":3789,"##field":3790,"needs":3791,"carolina":3792,"text":3793,"featuring":3794,"global":3795,"block":3796,"shirt":3797,"levels":3798,"francisco":3799,"purpose":3800,"females":3801,"et":3802,"dutch":3803,"duke":3804,"ahead":3805,"gas":3806,"twice":3807,"safety":3808,"serious":3809,"turning":3810,"highly":3811,"lieutenant":3812,"firm":3813,"maria":3814,"amount":3815,"mixed":3816,"daniel":3817,"proposed":3818,"perfect":3819,"agreement":3820,"affairs":3821,"3rd":3822,"seconds":3823,"contemporary":3824,"paid":3825,"1943":3826,"prison":3827,"save":3828,"kitchen":3829,"label":3830,"administrative":3831,"intended":3832,"constructed":3833,"academic":3834,"nice":3835,"teacher":3836,"races":3837,"1956":3838,"formerly":3839,"corporation":3840,"ben":3841,"nation":3842,"issued":3843,"shut":3844,"1958":3845,"drums":3846,"housing":3847,"victoria":3848,"seems":3849,"opera":3850,"1959":3851,"graduated":3852,"function":3853,"von":3854,"mentioned":3855,"picked":3856,"build":3857,"recognized":3858,"shortly":3859,"protection":3860,"picture":3861,"notable":3862,"exchange":3863,"elections":3864,"1980s":3865,"loved":3866,"percent":3867,"racing":3868,"fish":3869,"elizabeth":3870,"garden":3871,"volume":3872,"hockey":3873,"1941":3874,"beside":3875,"settled":3876,"##ford":3877,"1940":3878,"competed":3879,"replied":3880,"drew":3881,"1948":3882,"actress":3883,"marine":3884,"scotland":3885,"steel":3886,"glanced":3887,"farm":3888,"steve":3889,"1957":3890,"risk":3891,"tonight":3892,"positive":3893,"magic":3894,"singles":3895,"effects":3896,"gray":3897,"screen":3898,"dog":3899,"##ja":3900,"residents":3901,"bus":3902,"sides":3903,"none":3904,"secondary":3905,"literature":3906,"polish":3907,"destroyed":3908,"flying":3909,"founder":3910,"households":3911,"1939":3912,"lay":3913,"reserve":3914,"usa":3915,"gallery":3916,"##ler":3917,"1946":3918,"industrial":3919,"younger":3920,"approach":3921,"appearances":3922,"urban":3923,"ones":3924,"1950":3925,"finish":3926,"avenue":3927,"powerful":3928,"fully":3929,"growth":3930,"page":3931,"honor":3932,"jersey":3933,"projects":3934,"advanced":3935,"revealed":3936,"basic":3937,"90":3938,"infantry":3939,"pair":3940,"equipment":3941,"visit":3942,"33":3943,"evening":3944,"search":3945,"grant":3946,"effort":3947,"solo":3948,"treatment":3949,"buried":3950,"republican":3951,"primarily":3952,"bottom":3953,"owner":3954,"1970s":3955,"israel":3956,"gives":3957,"jim":3958,"dream":3959,"bob":3960,"remain":3961,"spot":3962,"70":3963,"notes":3964,"produce":3965,"champions":3966,"contact":3967,"ed":3968,"soul":3969,"accepted":3970,"ways":3971,"del":3972,"##ally":3973,"losing":3974,"split":3975,"price":3976,"capacity":3977,"basis":3978,"trial":3979,"questions":3980,"##ina":3981,"1955":3982,"20th":3983,"guess":3984,"officially":3985,"memorial":3986,"naval":3987,"initial":3988,"##ization":3989,"whispered":3990,"median":3991,"engineer":3992,"##ful":3993,"sydney":3994,"##go":3995,"columbia":3996,"strength":3997,"300":3998,"1952":3999,"tears":4000,"senate":4001,"00":4002,"card":4003,"asian":4004,"agent":4005,"1947":4006,"software":4007,"44":4008,"draw":4009,"warm":4010,"supposed":4011,"com":4012,"pro":4013,"##il":4014,"transferred":4015,"leaned":4016,"##at":4017,"candidate":4018,"escape":4019,"mountains":4020,"asia":4021,"potential":4022,"activity":4023,"entertainment":4024,"seem":4025,"traffic":4026,"jackson":4027,"murder":4028,"36":4029,"slow":4030,"product":4031,"orchestra":4032,"haven":4033,"agency":4034,"bbc":4035,"taught":4036,"website":4037,"comedy":4038,"unable":4039,"storm":4040,"planning":4041,"albums":4042,"rugby":4043,"environment":4044,"scientific":4045,"grabbed":4046,"protect":4047,"##hi":4048,"boat":4049,"typically":4050,"1954":4051,"1953":4052,"damage":4053,"principal":4054,"divided":4055,"dedicated":4056,"mount":4057,"ohio":4058,"##berg":4059,"pick":4060,"fought":4061,"driver":4062,"##der":4063,"empty":4064,"shoulders":4065,"sort":4066,"thank":4067,"berlin":4068,"prominent":4069,"account":4070,"freedom":4071,"necessary":4072,"efforts":4073,"alex":4074,"headquarters":4075,"follows":4076,"alongside":4077,"des":4078,"simon":4079,"andrew":4080,"suggested":4081,"operating":4082,"learning":4083,"steps":4084,"1949":4085,"sweet":4086,"technical":4087,"begin":4088,"easily":4089,"34":4090,"teeth":4091,"speaking":4092,"settlement":4093,"scale":4094,"##sh":4095,"renamed":4096,"ray":4097,"max":4098,"enemy":4099,"semi":4100,"joint":4101,"compared":4102,"##rd":4103,"scottish":4104,"leadership":4105,"analysis":4106,"offers":4107,"georgia":4108,"pieces":4109,"captured":4110,"animal":4111,"deputy":4112,"guest":4113,"organized":4114,"##lin":4115,"tony":4116,"combined":4117,"method":4118,"challenge":4119,"1960s":4120,"huge":4121,"wants":4122,"battalion":4123,"sons":4124,"rise":4125,"crime":4126,"types":4127,"facilities":4128,"telling":4129,"path":4130,"1951":4131,"platform":4132,"sit":4133,"1990s":4134,"##lo":4135,"tells":4136,"assigned":4137,"rich":4138,"pull":4139,"##ot":4140,"commonly":4141,"alive":4142,"##za":4143,"letters":4144,"concept":4145,"conducted":4146,"wearing":4147,"happen":4148,"bought":4149,"becomes":4150,"holy":4151,"gets":4152,"ocean":4153,"defeat":4154,"languages":4155,"purchased":4156,"coffee":4157,"occurred":4158,"titled":4159,"##q":4160,"declared":4161,"applied":4162,"sciences":4163,"concert":4164,"sounds":4165,"jazz":4166,"brain":4167,"##me":4168,"painting":4169,"fleet":4170,"tax":4171,"nick":4172,"##ius":4173,"michigan":4174,"count":4175,"animals":4176,"leaders":4177,"episodes":4178,"##line":4179,"content":4180,"##den":4181,"birth":4182,"##it":4183,"clubs":4184,"64":4185,"palace":4186,"critical":4187,"refused":4188,"fair":4189,"leg":4190,"laughed":4191,"returning":4192,"surrounding":4193,"participated":4194,"formation":4195,"lifted":4196,"pointed":4197,"connected":4198,"rome":4199,"medicine":4200,"laid":4201,"taylor":4202,"santa":4203,"powers":4204,"adam":4205,"tall":4206,"shared":4207,"focused":4208,"knowing":4209,"yards":4210,"entrance":4211,"falls":4212,"##wa":4213,"calling":4214,"##ad":4215,"sources":4216,"chosen":4217,"beneath":4218,"resources":4219,"yard":4220,"##ite":4221,"nominated":4222,"silence":4223,"zone":4224,"defined":4225,"##que":4226,"gained":4227,"thirty":4228,"38":4229,"bodies":4230,"moon":4231,"##ard":4232,"adopted":4233,"christmas":4234,"widely":4235,"register":4236,"apart":4237,"iran":4238,"premier":4239,"serves":4240,"du":4241,"unknown":4242,"parties":4243,"##les":4244,"generation":4245,"##ff":4246,"continues":4247,"quick":4248,"fields":4249,"brigade":4250,"quiet":4251,"teaching":4252,"clothes":4253,"impact":4254,"weapons":4255,"partner":4256,"flat":4257,"theater":4258,"supreme":4259,"1938":4260,"37":4261,"relations":4262,"##tor":4263,"plants":4264,"suffered":4265,"1936":4266,"wilson":4267,"kids":4268,"begins":4269,"##age":4270,"1918":4271,"seats":4272,"armed":4273,"internet":4274,"models":4275,"worth":4276,"laws":4277,"400":4278,"communities":4279,"classes":4280,"background":4281,"knows":4282,"thanks":4283,"quarter":4284,"reaching":4285,"humans":4286,"carry":4287,"killing":4288,"format":4289,"kong":4290,"hong":4291,"setting":4292,"75":4293,"architecture":4294,"disease":4295,"railroad":4296,"inc":4297,"possibly":4298,"wish":4299,"arthur":4300,"thoughts":4301,"harry":4302,"doors":4303,"density":4304,"##di":4305,"crowd":4306,"illinois":4307,"stomach":4308,"tone":4309,"unique":4310,"reports":4311,"anyway":4312,"##ir":4313,"liberal":4314,"der":4315,"vehicle":4316,"thick":4317,"dry":4318,"drug":4319,"faced":4320,"largely":4321,"facility":4322,"theme":4323,"holds":4324,"creation":4325,"strange":4326,"colonel":4327,"##mi":4328,"revolution":4329,"bell":4330,"politics":4331,"turns":4332,"silent":4333,"rail":4334,"relief":4335,"independence":4336,"combat":4337,"shape":4338,"write":4339,"determined":4340,"sales":4341,"learned":4342,"4th":4343,"finger":4344,"oxford":4345,"providing":4346,"1937":4347,"heritage":4348,"fiction":4349,"situated":4350,"designated":4351,"allowing":4352,"distribution":4353,"hosted":4354,"##est":4355,"sight":4356,"interview":4357,"estimated":4358,"reduced":4359,"##ria":4360,"toronto":4361,"footballer":4362,"keeping":4363,"guys":4364,"damn":4365,"claim":4366,"motion":4367,"sport":4368,"sixth":4369,"stayed":4370,"##ze":4371,"en":4372,"rear":4373,"receive":4374,"handed":4375,"twelve":4376,"dress":4377,"audience":4378,"granted":4379,"brazil":4380,"##well":4381,"spirit":4382,"##ated":4383,"noticed":4384,"etc":4385,"olympic":4386,"representative":4387,"eric":4388,"tight":4389,"trouble":4390,"reviews":4391,"drink":4392,"vampire":4393,"missing":4394,"roles":4395,"ranked":4396,"newly":4397,"household":4398,"finals":4399,"wave":4400,"critics":4401,"##ee":4402,"phase":4403,"massachusetts":4404,"pilot":4405,"unlike":4406,"philadelphia":4407,"bright":4408,"guns":4409,"crown":4410,"organizations":4411,"roof":4412,"42":4413,"respectively":4414,"clearly":4415,"tongue":4416,"marked":4417,"circle":4418,"fox":4419,"korea":4420,"bronze":4421,"brian":4422,"expanded":4423,"sexual":4424,"supply":4425,"yourself":4426,"inspired":4427,"labour":4428,"fc":4429,"##ah":4430,"reference":4431,"vision":4432,"draft":4433,"connection":4434,"brand":4435,"reasons":4436,"1935":4437,"classic":4438,"driving":4439,"trip":4440,"jesus":4441,"cells":4442,"entry":4443,"1920":4444,"neither":4445,"trail":4446,"claims":4447,"atlantic":4448,"orders":4449,"labor":4450,"nose":4451,"afraid":4452,"identified":4453,"intelligence":4454,"calls":4455,"cancer":4456,"attacked":4457,"passing":4458,"stephen":4459,"positions":4460,"imperial":4461,"grey":4462,"jason":4463,"39":4464,"sunday":4465,"48":4466,"swedish":4467,"avoid":4468,"extra":4469,"uncle":4470,"message":4471,"covers":4472,"allows":4473,"surprise":4474,"materials":4475,"fame":4476,"hunter":4477,"##ji":4478,"1930":4479,"citizens":4480,"figures":4481,"davis":4482,"environmental":4483,"confirmed":4484,"shit":4485,"titles":4486,"di":4487,"performing":4488,"difference":4489,"acts":4490,"attacks":4491,"##ov":4492,"existing":4493,"votes":4494,"opportunity":4495,"nor":4496,"shop":4497,"entirely":4498,"trains":4499,"opposite":4500,"pakistan":4501,"##pa":4502,"develop":4503,"resulted":4504,"representatives":4505,"actions":4506,"reality":4507,"pressed":4508,"##ish":4509,"barely":4510,"wine":4511,"conversation":4512,"faculty":4513,"northwest":4514,"ends":4515,"documentary":4516,"nuclear":4517,"stock":4518,"grace":4519,"sets":4520,"eat":4521,"alternative":4522,"##ps":4523,"bag":4524,"resulting":4525,"creating":4526,"surprised":4527,"cemetery":4528,"1919":4529,"drop":4530,"finding":4531,"sarah":4532,"cricket":4533,"streets":4534,"tradition":4535,"ride":4536,"1933":4537,"exhibition":4538,"target":4539,"ear":4540,"explained":4541,"rain":4542,"composer":4543,"injury":4544,"apartment":4545,"municipal":4546,"educational":4547,"occupied":4548,"netherlands":4549,"clean":4550,"billion":4551,"constitution":4552,"learn":4553,"1914":4554,"maximum":4555,"classical":4556,"francis":4557,"lose":4558,"opposition":4559,"jose":4560,"ontario":4561,"bear":4562,"core":4563,"hills":4564,"rolled":4565,"ending":4566,"drawn":4567,"permanent":4568,"fun":4569,"##tes":4570,"##lla":4571,"lewis":4572,"sites":4573,"chamber":4574,"ryan":4575,"##way":4576,"scoring":4577,"height":4578,"1934":4579,"##house":4580,"lyrics":4581,"staring":4582,"55":4583,"officials":4584,"1917":4585,"snow":4586,"oldest":4587,"##tic":4588,"orange":4589,"##ger":4590,"qualified":4591,"interior":4592,"apparently":4593,"succeeded":4594,"thousand":4595,"dinner":4596,"lights":4597,"existence":4598,"fans":4599,"heavily":4600,"41":4601,"greatest":4602,"conservative":4603,"send":4604,"bowl":4605,"plus":4606,"enter":4607,"catch":4608,"##un":4609,"economy":4610,"duty":4611,"1929":4612,"speech":4613,"authorities":4614,"princess":4615,"performances":4616,"versions":4617,"shall":4618,"graduate":4619,"pictures":4620,"effective":4621,"remembered":4622,"poetry":4623,"desk":4624,"crossed":4625,"starring":4626,"starts":4627,"passenger":4628,"sharp":4629,"##ant":4630,"acres":4631,"ass":4632,"weather":4633,"falling":4634,"rank":4635,"fund":4636,"supporting":4637,"check":4638,"adult":4639,"publishing":4640,"heads":4641,"cm":4642,"southeast":4643,"lane":4644,"##burg":4645,"application":4646,"bc":4647,"##ura":4648,"les":4649,"condition":4650,"transfer":4651,"prevent":4652,"display":4653,"ex":4654,"regions":4655,"earl":4656,"federation":4657,"cool":4658,"relatively":4659,"answered":4660,"besides":4661,"1928":4662,"obtained":4663,"portion":4664,"##town":4665,"mix":4666,"##ding":4667,"reaction":4668,"liked":4669,"dean":4670,"express":4671,"peak":4672,"1932":4673,"##tte":4674,"counter":4675,"religion":4676,"chain":4677,"rare":4678,"miller":4679,"convention":4680,"aid":4681,"lie":4682,"vehicles":4683,"mobile":4684,"perform":4685,"squad":4686,"wonder":4687,"lying":4688,"crazy":4689,"sword":4690,"##ping":4691,"attempted":4692,"centuries":4693,"weren":4694,"philosophy":4695,"category":4696,"##ize":4697,"anna":4698,"interested":4699,"47":4700,"sweden":4701,"wolf":4702,"frequently":4703,"abandoned":4704,"kg":4705,"literary":4706,"alliance":4707,"task":4708,"entitled":4709,"##ay":4710,"threw":4711,"promotion":4712,"factory":4713,"tiny":4714,"soccer":4715,"visited":4716,"matt":4717,"fm":4718,"achieved":4719,"52":4720,"defence":4721,"internal":4722,"persian":4723,"43":4724,"methods":4725,"##ging":4726,"arrested":4727,"otherwise":4728,"cambridge":4729,"programming":4730,"villages":4731,"elementary":4732,"districts":4733,"rooms":4734,"criminal":4735,"conflict":4736,"worry":4737,"trained":4738,"1931":4739,"attempts":4740,"waited":4741,"signal":4742,"bird":4743,"truck":4744,"subsequent":4745,"programme":4746,"##ol":4747,"ad":4748,"49":4749,"communist":4750,"details":4751,"faith":4752,"sector":4753,"patrick":4754,"carrying":4755,"laugh":4756,"##ss":4757,"controlled":4758,"korean":4759,"showing":4760,"origin":4761,"fuel":4762,"evil":4763,"1927":4764,"##ent":4765,"brief":4766,"identity":4767,"darkness":4768,"address":4769,"pool":4770,"missed":4771,"publication":4772,"web":4773,"planet":4774,"ian":4775,"anne":4776,"wings":4777,"invited":4778,"##tt":4779,"briefly":4780,"standards":4781,"kissed":4782,"##be":4783,"ideas":4784,"climate":4785,"causing":4786,"walter":4787,"worse":4788,"albert":4789,"articles":4790,"winners":4791,"desire":4792,"aged":4793,"northeast":4794,"dangerous":4795,"gate":4796,"doubt":4797,"1922":4798,"wooden":4799,"multi":4800,"##ky":4801,"poet":4802,"rising":4803,"funding":4804,"46":4805,"communications":4806,"communication":4807,"violence":4808,"copies":4809,"prepared":4810,"ford":4811,"investigation":4812,"skills":4813,"1924":4814,"pulling":4815,"electronic":4816,"##ak":4817,"##ial":4818,"##han":4819,"containing":4820,"ultimately":4821,"offices":4822,"singing":4823,"understanding":4824,"restaurant":4825,"tomorrow":4826,"fashion":4827,"christ":4828,"ward":4829,"da":4830,"pope":4831,"stands":4832,"5th":4833,"flow":4834,"studios":4835,"aired":4836,"commissioned":4837,"contained":4838,"exist":4839,"fresh":4840,"americans":4841,"##per":4842,"wrestling":4843,"approved":4844,"kid":4845,"employed":4846,"respect":4847,"suit":4848,"1925":4849,"angel":4850,"asking":4851,"increasing":4852,"frame":4853,"angry":4854,"selling":4855,"1950s":4856,"thin":4857,"finds":4858,"##nd":4859,"temperature":4860,"statement":4861,"ali":4862,"explain":4863,"inhabitants":4864,"towns":4865,"extensive":4866,"narrow":4867,"51":4868,"jane":4869,"flowers":4870,"images":4871,"promise":4872,"somewhere":4873,"object":4874,"fly":4875,"closely":4876,"##ls":4877,"1912":4878,"bureau":4879,"cape":4880,"1926":4881,"weekly":4882,"presidential":4883,"legislative":4884,"1921":4885,"##ai":4886,"##au":4887,"launch":4888,"founding":4889,"##ny":4890,"978":4891,"##ring":4892,"artillery":4893,"strike":4894,"un":4895,"institutions":4896,"roll":4897,"writers":4898,"landing":4899,"chose":4900,"kevin":4901,"anymore":4902,"pp":4903,"##ut":4904,"attorney":4905,"fit":4906,"dan":4907,"billboard":4908,"receiving":4909,"agricultural":4910,"breaking":4911,"sought":4912,"dave":4913,"admitted":4914,"lands":4915,"mexican":4916,"##bury":4917,"charlie":4918,"specifically":4919,"hole":4920,"iv":4921,"howard":4922,"credit":4923,"moscow":4924,"roads":4925,"accident":4926,"1923":4927,"proved":4928,"wear":4929,"struck":4930,"hey":4931,"guards":4932,"stuff":4933,"slid":4934,"expansion":4935,"1915":4936,"cat":4937,"anthony":4938,"##kin":4939,"melbourne":4940,"opposed":4941,"sub":4942,"southwest":4943,"architect":4944,"failure":4945,"plane":4946,"1916":4947,"##ron":4948,"map":4949,"camera":4950,"tank":4951,"listen":4952,"regarding":4953,"wet":4954,"introduction":4955,"metropolitan":4956,"link":4957,"ep":4958,"fighter":4959,"inch":4960,"grown":4961,"gene":4962,"anger":4963,"fixed":4964,"buy":4965,"dvd":4966,"khan":4967,"domestic":4968,"worldwide":4969,"chapel":4970,"mill":4971,"functions":4972,"examples":4973,"##head":4974,"developing":4975,"1910":4976,"turkey":4977,"hits":4978,"pocket":4979,"antonio":4980,"papers":4981,"grow":4982,"unless":4983,"circuit":4984,"18th":4985,"concerned":4986,"attached":4987,"journalist":4988,"selection":4989,"journey":4990,"converted":4991,"provincial":4992,"painted":4993,"hearing":4994,"aren":4995,"bands":4996,"negative":4997,"aside":4998,"wondered":4999,"knight":5000,"lap":5001,"survey":5002,"ma":5003,"##ow":5004,"noise":5005,"billy":5006,"##ium":5007,"shooting":5008,"guide":5009,"bedroom":5010,"priest":5011,"resistance":5012,"motor":5013,"homes":5014,"sounded":5015,"giant":5016,"##mer":5017,"150":5018,"scenes":5019,"equal":5020,"comic":5021,"patients":5022,"hidden":5023,"solid":5024,"actual":5025,"bringing":5026,"afternoon":5027,"touched":5028,"funds":5029,"wedding":5030,"consisted":5031,"marie":5032,"canal":5033,"sr":5034,"kim":5035,"treaty":5036,"turkish":5037,"recognition":5038,"residence":5039,"cathedral":5040,"broad":5041,"knees":5042,"incident":5043,"shaped":5044,"fired":5045,"norwegian":5046,"handle":5047,"cheek":5048,"contest":5049,"represent":5050,"##pe":5051,"representing":5052,"beauty":5053,"##sen":5054,"birds":5055,"advantage":5056,"emergency":5057,"wrapped":5058,"drawing":5059,"notice":5060,"pink":5061,"broadcasting":5062,"##ong":5063,"somehow":5064,"bachelor":5065,"seventh":5066,"collected":5067,"registered":5068,"establishment":5069,"alan":5070,"assumed":5071,"chemical":5072,"personnel":5073,"roger":5074,"retirement":5075,"jeff":5076,"portuguese":5077,"wore":5078,"tied":5079,"device":5080,"threat":5081,"progress":5082,"advance":5083,"##ised":5084,"banks":5085,"hired":5086,"manchester":5087,"nfl":5088,"teachers":5089,"structures":5090,"forever":5091,"##bo":5092,"tennis":5093,"helping":5094,"saturday":5095,"sale":5096,"applications":5097,"junction":5098,"hip":5099,"incorporated":5100,"neighborhood":5101,"dressed":5102,"ceremony":5103,"##ds":5104,"influenced":5105,"hers":5106,"visual":5107,"stairs":5108,"decades":5109,"inner":5110,"kansas":5111,"hung":5112,"hoped":5113,"gain":5114,"scheduled":5115,"downtown":5116,"engaged":5117,"austria":5118,"clock":5119,"norway":5120,"certainly":5121,"pale":5122,"protected":5123,"1913":5124,"victor":5125,"employees":5126,"plate":5127,"putting":5128,"surrounded":5129,"##ists":5130,"finishing":5131,"blues":5132,"tropical":5133,"##ries":5134,"minnesota":5135,"consider":5136,"philippines":5137,"accept":5138,"54":5139,"retrieved":5140,"1900":5141,"concern":5142,"anderson":5143,"properties":5144,"institution":5145,"gordon":5146,"successfully":5147,"vietnam":5148,"##dy":5149,"backing":5150,"outstanding":5151,"muslim":5152,"crossing":5153,"folk":5154,"producing":5155,"usual":5156,"demand":5157,"occurs":5158,"observed":5159,"lawyer":5160,"educated":5161,"##ana":5162,"kelly":5163,"string":5164,"pleasure":5165,"budget":5166,"items":5167,"quietly":5168,"colorado":5169,"philip":5170,"typical":5171,"##worth":5172,"derived":5173,"600":5174,"survived":5175,"asks":5176,"mental":5177,"##ide":5178,"56":5179,"jake":5180,"jews":5181,"distinguished":5182,"ltd":5183,"1911":5184,"sri":5185,"extremely":5186,"53":5187,"athletic":5188,"loud":5189,"thousands":5190,"worried":5191,"shadow":5192,"transportation":5193,"horses":5194,"weapon":5195,"arena":5196,"importance":5197,"users":5198,"tim":5199,"objects":5200,"contributed":5201,"dragon":5202,"douglas":5203,"aware":5204,"senator":5205,"johnny":5206,"jordan":5207,"sisters":5208,"engines":5209,"flag":5210,"investment":5211,"samuel":5212,"shock":5213,"capable":5214,"clark":5215,"row":5216,"wheel":5217,"refers":5218,"session":5219,"familiar":5220,"biggest":5221,"wins":5222,"hate":5223,"maintained":5224,"drove":5225,"hamilton":5226,"request":5227,"expressed":5228,"injured":5229,"underground":5230,"churches":5231,"walker":5232,"wars":5233,"tunnel":5234,"passes":5235,"stupid":5236,"agriculture":5237,"softly":5238,"cabinet":5239,"regarded":5240,"joining":5241,"indiana":5242,"##ea":5243,"##ms":5244,"push":5245,"dates":5246,"spend":5247,"behavior":5248,"woods":5249,"protein":5250,"gently":5251,"chase":5252,"morgan":5253,"mention":5254,"burning":5255,"wake":5256,"combination":5257,"occur":5258,"mirror":5259,"leads":5260,"jimmy":5261,"indeed":5262,"impossible":5263,"singapore":5264,"paintings":5265,"covering":5266,"##nes":5267,"soldier":5268,"locations":5269,"attendance":5270,"sell":5271,"historian":5272,"wisconsin":5273,"invasion":5274,"argued":5275,"painter":5276,"diego":5277,"changing":5278,"egypt":5279,"##don":5280,"experienced":5281,"inches":5282,"##ku":5283,"missouri":5284,"vol":5285,"grounds":5286,"spoken":5287,"switzerland":5288,"##gan":5289,"reform":5290,"rolling":5291,"ha":5292,"forget":5293,"massive":5294,"resigned":5295,"burned":5296,"allen":5297,"tennessee":5298,"locked":5299,"values":5300,"improved":5301,"##mo":5302,"wounded":5303,"universe":5304,"sick":5305,"dating":5306,"facing":5307,"pack":5308,"purchase":5309,"user":5310,"##pur":5311,"moments":5312,"##ul":5313,"merged":5314,"anniversary":5315,"1908":5316,"coal":5317,"brick":5318,"understood":5319,"causes":5320,"dynasty":5321,"queensland":5322,"establish":5323,"stores":5324,"crisis":5325,"promote":5326,"hoping":5327,"views":5328,"cards":5329,"referee":5330,"extension":5331,"##si":5332,"raise":5333,"arizona":5334,"improve":5335,"colonial":5336,"formal":5337,"charged":5338,"##rt":5339,"palm":5340,"lucky":5341,"hide":5342,"rescue":5343,"faces":5344,"95":5345,"feelings":5346,"candidates":5347,"juan":5348,"##ell":5349,"goods":5350,"6th":5351,"courses":5352,"weekend":5353,"59":5354,"luke":5355,"cash":5356,"fallen":5357,"##om":5358,"delivered":5359,"affected":5360,"installed":5361,"carefully":5362,"tries":5363,"swiss":5364,"hollywood":5365,"costs":5366,"lincoln":5367,"responsibility":5368,"##he":5369,"shore":5370,"file":5371,"proper":5372,"normally":5373,"maryland":5374,"assistance":5375,"jump":5376,"constant":5377,"offering":5378,"friendly":5379,"waters":5380,"persons":5381,"realize":5382,"contain":5383,"trophy":5384,"800":5385,"partnership":5386,"factor":5387,"58":5388,"musicians":5389,"cry":5390,"bound":5391,"oregon":5392,"indicated":5393,"hero":5394,"houston":5395,"medium":5396,"##ure":5397,"consisting":5398,"somewhat":5399,"##ara":5400,"57":5401,"cycle":5402,"##che":5403,"beer":5404,"moore":5405,"frederick":5406,"gotten":5407,"eleven":5408,"worst":5409,"weak":5410,"approached":5411,"arranged":5412,"chin":5413,"loan":5414,"universal":5415,"bond":5416,"fifteen":5417,"pattern":5418,"disappeared":5419,"##ney":5420,"translated":5421,"##zed":5422,"lip":5423,"arab":5424,"capture":5425,"interests":5426,"insurance":5427,"##chi":5428,"shifted":5429,"cave":5430,"prix":5431,"warning":5432,"sections":5433,"courts":5434,"coat":5435,"plot":5436,"smell":5437,"feed":5438,"golf":5439,"favorite":5440,"maintain":5441,"knife":5442,"vs":5443,"voted":5444,"degrees":5445,"finance":5446,"quebec":5447,"opinion":5448,"translation":5449,"manner":5450,"ruled":5451,"operate":5452,"productions":5453,"choose":5454,"musician":5455,"discovery":5456,"confused":5457,"tired":5458,"separated":5459,"stream":5460,"techniques":5461,"committed":5462,"attend":5463,"ranking":5464,"kings":5465,"throw":5466,"passengers":5467,"measure":5468,"horror":5469,"fan":5470,"mining":5471,"sand":5472,"danger":5473,"salt":5474,"calm":5475,"decade":5476,"dam":5477,"require":5478,"runner":5479,"##ik":5480,"rush":5481,"associate":5482,"greece":5483,"##ker":5484,"rivers":5485,"consecutive":5486,"matthew":5487,"##ski":5488,"sighed":5489,"sq":5490,"documents":5491,"steam":5492,"edited":5493,"closing":5494,"tie":5495,"accused":5496,"1905":5497,"##ini":5498,"islamic":5499,"distributed":5500,"directors":5501,"organisation":5502,"bruce":5503,"7th":5504,"breathing":5505,"mad":5506,"lit":5507,"arrival":5508,"concrete":5509,"taste":5510,"08":5511,"composition":5512,"shaking":5513,"faster":5514,"amateur":5515,"adjacent":5516,"stating":5517,"1906":5518,"twin":5519,"flew":5520,"##ran":5521,"tokyo":5522,"publications":5523,"##tone":5524,"obviously":5525,"ridge":5526,"storage":5527,"1907":5528,"carl":5529,"pages":5530,"concluded":5531,"desert":5532,"driven":5533,"universities":5534,"ages":5535,"terminal":5536,"sequence":5537,"borough":5538,"250":5539,"constituency":5540,"creative":5541,"cousin":5542,"economics":5543,"dreams":5544,"margaret":5545,"notably":5546,"reduce":5547,"montreal":5548,"mode":5549,"17th":5550,"ears":5551,"saved":5552,"jan":5553,"vocal":5554,"##ica":5555,"1909":5556,"andy":5557,"##jo":5558,"riding":5559,"roughly":5560,"threatened":5561,"##ise":5562,"meters":5563,"meanwhile":5564,"landed":5565,"compete":5566,"repeated":5567,"grass":5568,"czech":5569,"regularly":5570,"charges":5571,"tea":5572,"sudden":5573,"appeal":5574,"##ung":5575,"solution":5576,"describes":5577,"pierre":5578,"classification":5579,"glad":5580,"parking":5581,"##ning":5582,"belt":5583,"physics":5584,"99":5585,"rachel":5586,"add":5587,"hungarian":5588,"participate":5589,"expedition":5590,"damaged":5591,"gift":5592,"childhood":5593,"85":5594,"fifty":5595,"##red":5596,"mathematics":5597,"jumped":5598,"letting":5599,"defensive":5600,"mph":5601,"##ux":5602,"##gh":5603,"testing":5604,"##hip":5605,"hundreds":5606,"shoot":5607,"owners":5608,"matters":5609,"smoke":5610,"israeli":5611,"kentucky":5612,"dancing":5613,"mounted":5614,"grandfather":5615,"emma":5616,"designs":5617,"profit":5618,"argentina":5619,"##gs":5620,"truly":5621,"li":5622,"lawrence":5623,"cole":5624,"begun":5625,"detroit":5626,"willing":5627,"branches":5628,"smiling":5629,"decide":5630,"miami":5631,"enjoyed":5632,"recordings":5633,"##dale":5634,"poverty":5635,"ethnic":5636,"gay":5637,"##bi":5638,"gary":5639,"arabic":5640,"09":5641,"accompanied":5642,"##one":5643,"##ons":5644,"fishing":5645,"determine":5646,"residential":5647,"acid":5648,"##ary":5649,"alice":5650,"returns":5651,"starred":5652,"mail":5653,"##ang":5654,"jonathan":5655,"strategy":5656,"##ue":5657,"net":5658,"forty":5659,"cook":5660,"businesses":5661,"equivalent":5662,"commonwealth":5663,"distinct":5664,"ill":5665,"##cy":5666,"seriously":5667,"##ors":5668,"##ped":5669,"shift":5670,"harris":5671,"replace":5672,"rio":5673,"imagine":5674,"formula":5675,"ensure":5676,"##ber":5677,"additionally":5678,"scheme":5679,"conservation":5680,"occasionally":5681,"purposes":5682,"feels":5683,"favor":5684,"##and":5685,"##ore":5686,"1930s":5687,"contrast":5688,"hanging":5689,"hunt":5690,"movies":5691,"1904":5692,"instruments":5693,"victims":5694,"danish":5695,"christopher":5696,"busy":5697,"demon":5698,"sugar":5699,"earliest":5700,"colony":5701,"studying":5702,"balance":5703,"duties":5704,"##ks":5705,"belgium":5706,"slipped":5707,"carter":5708,"05":5709,"visible":5710,"stages":5711,"iraq":5712,"fifa":5713,"##im":5714,"commune":5715,"forming":5716,"zero":5717,"07":5718,"continuing":5719,"talked":5720,"counties":5721,"legend":5722,"bathroom":5723,"option":5724,"tail":5725,"clay":5726,"daughters":5727,"afterwards":5728,"severe":5729,"jaw":5730,"visitors":5731,"##ded":5732,"devices":5733,"aviation":5734,"russell":5735,"kate":5736,"##vi":5737,"entering":5738,"subjects":5739,"##ino":5740,"temporary":5741,"swimming":5742,"forth":5743,"smooth":5744,"ghost":5745,"audio":5746,"bush":5747,"operates":5748,"rocks":5749,"movements":5750,"signs":5751,"eddie":5752,"##tz":5753,"ann":5754,"voices":5755,"honorary":5756,"06":5757,"memories":5758,"dallas":5759,"pure":5760,"measures":5761,"racial":5762,"promised":5763,"66":5764,"harvard":5765,"ceo":5766,"16th":5767,"parliamentary":5768,"indicate":5769,"benefit":5770,"flesh":5771,"dublin":5772,"louisiana":5773,"1902":5774,"1901":5775,"patient":5776,"sleeping":5777,"1903":5778,"membership":5779,"coastal":5780,"medieval":5781,"wanting":5782,"element":5783,"scholars":5784,"rice":5785,"62":5786,"limit":5787,"survive":5788,"makeup":5789,"rating":5790,"definitely":5791,"collaboration":5792,"obvious":5793,"##tan":5794,"boss":5795,"ms":5796,"baron":5797,"birthday":5798,"linked":5799,"soil":5800,"diocese":5801,"##lan":5802,"ncaa":5803,"##mann":5804,"offensive":5805,"shell":5806,"shouldn":5807,"waist":5808,"##tus":5809,"plain":5810,"ross":5811,"organ":5812,"resolution":5813,"manufacturing":5814,"adding":5815,"relative":5816,"kennedy":5817,"98":5818,"whilst":5819,"moth":5820,"marketing":5821,"gardens":5822,"crash":5823,"72":5824,"heading":5825,"partners":5826,"credited":5827,"carlos":5828,"moves":5829,"cable":5830,"##zi":5831,"marshall":5832,"##out":5833,"depending":5834,"bottle":5835,"represents":5836,"rejected":5837,"responded":5838,"existed":5839,"04":5840,"jobs":5841,"denmark":5842,"lock":5843,"##ating":5844,"treated":5845,"graham":5846,"routes":5847,"talent":5848,"commissioner":5849,"drugs":5850,"secure":5851,"tests":5852,"reign":5853,"restored":5854,"photography":5855,"##gi":5856,"contributions":5857,"oklahoma":5858,"designer":5859,"disc":5860,"grin":5861,"seattle":5862,"robin":5863,"paused":5864,"atlanta":5865,"unusual":5866,"##gate":5867,"praised":5868,"las":5869,"laughing":5870,"satellite":5871,"hungary":5872,"visiting":5873,"##sky":5874,"interesting":5875,"factors":5876,"deck":5877,"poems":5878,"norman":5879,"##water":5880,"stuck":5881,"speaker":5882,"rifle":5883,"domain":5884,"premiered":5885,"##her":5886,"dc":5887,"comics":5888,"actors":5889,"01":5890,"reputation":5891,"eliminated":5892,"8th":5893,"ceiling":5894,"prisoners":5895,"script":5896,"##nce":5897,"leather":5898,"austin":5899,"mississippi":5900,"rapidly":5901,"admiral":5902,"parallel":5903,"charlotte":5904,"guilty":5905,"tools":5906,"gender":5907,"divisions":5908,"fruit":5909,"##bs":5910,"laboratory":5911,"nelson":5912,"fantasy":5913,"marry":5914,"rapid":5915,"aunt":5916,"tribe":5917,"requirements":5918,"aspects":5919,"suicide":5920,"amongst":5921,"adams":5922,"bone":5923,"ukraine":5924,"abc":5925,"kick":5926,"sees":5927,"edinburgh":5928,"clothing":5929,"column":5930,"rough":5931,"gods":5932,"hunting":5933,"broadway":5934,"gathered":5935,"concerns":5936,"##ek":5937,"spending":5938,"ty":5939,"12th":5940,"snapped":5941,"requires":5942,"solar":5943,"bones":5944,"cavalry":5945,"##tta":5946,"iowa":5947,"drinking":5948,"waste":5949,"index":5950,"franklin":5951,"charity":5952,"thompson":5953,"stewart":5954,"tip":5955,"flash":5956,"landscape":5957,"friday":5958,"enjoy":5959,"singh":5960,"poem":5961,"listening":5962,"##back":5963,"eighth":5964,"fred":5965,"differences":5966,"adapted":5967,"bomb":5968,"ukrainian":5969,"surgery":5970,"corporate":5971,"masters":5972,"anywhere":5973,"##more":5974,"waves":5975,"odd":5976,"sean":5977,"portugal":5978,"orleans":5979,"dick":5980,"debate":5981,"kent":5982,"eating":5983,"puerto":5984,"cleared":5985,"96":5986,"expect":5987,"cinema":5988,"97":5989,"guitarist":5990,"blocks":5991,"electrical":5992,"agree":5993,"involving":5994,"depth":5995,"dying":5996,"panel":5997,"struggle":5998,"##ged":5999,"peninsula":6000,"adults":6001,"novels":6002,"emerged":6003,"vienna":6004,"metro":6005,"debuted":6006,"shoes":6007,"tamil":6008,"songwriter":6009,"meets":6010,"prove":6011,"beating":6012,"instance":6013,"heaven":6014,"scared":6015,"sending":6016,"marks":6017,"artistic":6018,"passage":6019,"superior":6020,"03":6021,"significantly":6022,"shopping":6023,"##tive":6024,"retained":6025,"##izing":6026,"malaysia":6027,"technique":6028,"cheeks":6029,"##ola":6030,"warren":6031,"maintenance":6032,"destroy":6033,"extreme":6034,"allied":6035,"120":6036,"appearing":6037,"##yn":6038,"fill":6039,"advice":6040,"alabama":6041,"qualifying":6042,"policies":6043,"cleveland":6044,"hat":6045,"battery":6046,"smart":6047,"authors":6048,"10th":6049,"soundtrack":6050,"acted":6051,"dated":6052,"lb":6053,"glance":6054,"equipped":6055,"coalition":6056,"funny":6057,"outer":6058,"ambassador":6059,"roy":6060,"possibility":6061,"couples":6062,"campbell":6063,"dna":6064,"loose":6065,"ethan":6066,"supplies":6067,"1898":6068,"gonna":6069,"88":6070,"monster":6071,"##res":6072,"shake":6073,"agents":6074,"frequency":6075,"springs":6076,"dogs":6077,"practices":6078,"61":6079,"gang":6080,"plastic":6081,"easier":6082,"suggests":6083,"gulf":6084,"blade":6085,"exposed":6086,"colors":6087,"industries":6088,"markets":6089,"pan":6090,"nervous":6091,"electoral":6092,"charts":6093,"legislation":6094,"ownership":6095,"##idae":6096,"mac":6097,"appointment":6098,"shield":6099,"copy":6100,"assault":6101,"socialist":6102,"abbey":6103,"monument":6104,"license":6105,"throne":6106,"employment":6107,"jay":6108,"93":6109,"replacement":6110,"charter":6111,"cloud":6112,"powered":6113,"suffering":6114,"accounts":6115,"oak":6116,"connecticut":6117,"strongly":6118,"wright":6119,"colour":6120,"crystal":6121,"13th":6122,"context":6123,"welsh":6124,"networks":6125,"voiced":6126,"gabriel":6127,"jerry":6128,"##cing":6129,"forehead":6130,"mp":6131,"##ens":6132,"manage":6133,"schedule":6134,"totally":6135,"remix":6136,"##ii":6137,"forests":6138,"occupation":6139,"print":6140,"nicholas":6141,"brazilian":6142,"strategic":6143,"vampires":6144,"engineers":6145,"76":6146,"roots":6147,"seek":6148,"correct":6149,"instrumental":6150,"und":6151,"alfred":6152,"backed":6153,"hop":6154,"##des":6155,"stanley":6156,"robinson":6157,"traveled":6158,"wayne":6159,"welcome":6160,"austrian":6161,"achieve":6162,"67":6163,"exit":6164,"rates":6165,"1899":6166,"strip":6167,"whereas":6168,"##cs":6169,"sing":6170,"deeply":6171,"adventure":6172,"bobby":6173,"rick":6174,"jamie":6175,"careful":6176,"components":6177,"cap":6178,"useful":6179,"personality":6180,"knee":6181,"##shi":6182,"pushing":6183,"hosts":6184,"02":6185,"protest":6186,"ca":6187,"ottoman":6188,"symphony":6189,"##sis":6190,"63":6191,"boundary":6192,"1890":6193,"processes":6194,"considering":6195,"considerable":6196,"tons":6197,"##work":6198,"##ft":6199,"##nia":6200,"cooper":6201,"trading":6202,"dear":6203,"conduct":6204,"91":6205,"illegal":6206,"apple":6207,"revolutionary":6208,"holiday":6209,"definition":6210,"harder":6211,"##van":6212,"jacob":6213,"circumstances":6214,"destruction":6215,"##lle":6216,"popularity":6217,"grip":6218,"classified":6219,"liverpool":6220,"donald":6221,"baltimore":6222,"flows":6223,"seeking":6224,"honour":6225,"approval":6226,"92":6227,"mechanical":6228,"till":6229,"happening":6230,"statue":6231,"critic":6232,"increasingly":6233,"immediate":6234,"describe":6235,"commerce":6236,"stare":6237,"##ster":6238,"indonesia":6239,"meat":6240,"rounds":6241,"boats":6242,"baker":6243,"orthodox":6244,"depression":6245,"formally":6246,"worn":6247,"naked":6248,"claire":6249,"muttered":6250,"sentence":6251,"11th":6252,"emily":6253,"document":6254,"77":6255,"criticism":6256,"wished":6257,"vessel":6258,"spiritual":6259,"bent":6260,"virgin":6261,"parker":6262,"minimum":6263,"murray":6264,"lunch":6265,"danny":6266,"printed":6267,"compilation":6268,"keyboards":6269,"false":6270,"blow":6271,"belonged":6272,"68":6273,"raising":6274,"78":6275,"cutting":6276,"##board":6277,"pittsburgh":6278,"##up":6279,"9th":6280,"shadows":6281,"81":6282,"hated":6283,"indigenous":6284,"jon":6285,"15th":6286,"barry":6287,"scholar":6288,"ah":6289,"##zer":6290,"oliver":6291,"##gy":6292,"stick":6293,"susan":6294,"meetings":6295,"attracted":6296,"spell":6297,"romantic":6298,"##ver":6299,"ye":6300,"1895":6301,"photo":6302,"demanded":6303,"customers":6304,"##ac":6305,"1896":6306,"logan":6307,"revival":6308,"keys":6309,"modified":6310,"commanded":6311,"jeans":6312,"##ious":6313,"upset":6314,"raw":6315,"phil":6316,"detective":6317,"hiding":6318,"resident":6319,"vincent":6320,"##bly":6321,"experiences":6322,"diamond":6323,"defeating":6324,"coverage":6325,"lucas":6326,"external":6327,"parks":6328,"franchise":6329,"helen":6330,"bible":6331,"successor":6332,"percussion":6333,"celebrated":6334,"il":6335,"lift":6336,"profile":6337,"clan":6338,"romania":6339,"##ied":6340,"mills":6341,"##su":6342,"nobody":6343,"achievement":6344,"shrugged":6345,"fault":6346,"1897":6347,"rhythm":6348,"initiative":6349,"breakfast":6350,"carbon":6351,"700":6352,"69":6353,"lasted":6354,"violent":6355,"74":6356,"wound":6357,"ken":6358,"killer":6359,"gradually":6360,"filmed":6361,"°c":6362,"dollars":6363,"processing":6364,"94":6365,"remove":6366,"criticized":6367,"guests":6368,"sang":6369,"chemistry":6370,"##vin":6371,"legislature":6372,"disney":6373,"##bridge":6374,"uniform":6375,"escaped":6376,"integrated":6377,"proposal":6378,"purple":6379,"denied":6380,"liquid":6381,"karl":6382,"influential":6383,"morris":6384,"nights":6385,"stones":6386,"intense":6387,"experimental":6388,"twisted":6389,"71":6390,"84":6391,"##ld":6392,"pace":6393,"nazi":6394,"mitchell":6395,"ny":6396,"blind":6397,"reporter":6398,"newspapers":6399,"14th":6400,"centers":6401,"burn":6402,"basin":6403,"forgotten":6404,"surviving":6405,"filed":6406,"collections":6407,"monastery":6408,"losses":6409,"manual":6410,"couch":6411,"description":6412,"appropriate":6413,"merely":6414,"tag":6415,"missions":6416,"sebastian":6417,"restoration":6418,"replacing":6419,"triple":6420,"73":6421,"elder":6422,"julia":6423,"warriors":6424,"benjamin":6425,"julian":6426,"convinced":6427,"stronger":6428,"amazing":6429,"declined":6430,"versus":6431,"merchant":6432,"happens":6433,"output":6434,"finland":6435,"bare":6436,"barbara":6437,"absence":6438,"ignored":6439,"dawn":6440,"injuries":6441,"##port":6442,"producers":6443,"##ram":6444,"82":6445,"luis":6446,"##ities":6447,"kw":6448,"admit":6449,"expensive":6450,"electricity":6451,"nba":6452,"exception":6453,"symbol":6454,"##ving":6455,"ladies":6456,"shower":6457,"sheriff":6458,"characteristics":6459,"##je":6460,"aimed":6461,"button":6462,"ratio":6463,"effectively":6464,"summit":6465,"angle":6466,"jury":6467,"bears":6468,"foster":6469,"vessels":6470,"pants":6471,"executed":6472,"evans":6473,"dozen":6474,"advertising":6475,"kicked":6476,"patrol":6477,"1889":6478,"competitions":6479,"lifetime":6480,"principles":6481,"athletics":6482,"##logy":6483,"birmingham":6484,"sponsored":6485,"89":6486,"rob":6487,"nomination":6488,"1893":6489,"acoustic":6490,"##sm":6491,"creature":6492,"longest":6493,"##tra":6494,"credits":6495,"harbor":6496,"dust":6497,"josh":6498,"##so":6499,"territories":6500,"milk":6501,"infrastructure":6502,"completion":6503,"thailand":6504,"indians":6505,"leon":6506,"archbishop":6507,"##sy":6508,"assist":6509,"pitch":6510,"blake":6511,"arrangement":6512,"girlfriend":6513,"serbian":6514,"operational":6515,"hence":6516,"sad":6517,"scent":6518,"fur":6519,"dj":6520,"sessions":6521,"hp":6522,"refer":6523,"rarely":6524,"##ora":6525,"exists":6526,"1892":6527,"##ten":6528,"scientists":6529,"dirty":6530,"penalty":6531,"burst":6532,"portrait":6533,"seed":6534,"79":6535,"pole":6536,"limits":6537,"rival":6538,"1894":6539,"stable":6540,"alpha":6541,"grave":6542,"constitutional":6543,"alcohol":6544,"arrest":6545,"flower":6546,"mystery":6547,"devil":6548,"architectural":6549,"relationships":6550,"greatly":6551,"habitat":6552,"##istic":6553,"larry":6554,"progressive":6555,"remote":6556,"cotton":6557,"##ics":6558,"##ok":6559,"preserved":6560,"reaches":6561,"##ming":6562,"cited":6563,"86":6564,"vast":6565,"scholarship":6566,"decisions":6567,"cbs":6568,"joy":6569,"teach":6570,"1885":6571,"editions":6572,"knocked":6573,"eve":6574,"searching":6575,"partly":6576,"participation":6577,"gap":6578,"animated":6579,"fate":6580,"excellent":6581,"##ett":6582,"na":6583,"87":6584,"alternate":6585,"saints":6586,"youngest":6587,"##ily":6588,"climbed":6589,"##ita":6590,"##tors":6591,"suggest":6592,"##ct":6593,"discussion":6594,"staying":6595,"choir":6596,"lakes":6597,"jacket":6598,"revenue":6599,"nevertheless":6600,"peaked":6601,"instrument":6602,"wondering":6603,"annually":6604,"managing":6605,"neil":6606,"1891":6607,"signing":6608,"terry":6609,"##ice":6610,"apply":6611,"clinical":6612,"brooklyn":6613,"aim":6614,"catherine":6615,"fuck":6616,"farmers":6617,"figured":6618,"ninth":6619,"pride":6620,"hugh":6621,"evolution":6622,"ordinary":6623,"involvement":6624,"comfortable":6625,"shouted":6626,"tech":6627,"encouraged":6628,"taiwan":6629,"representation":6630,"sharing":6631,"##lia":6632,"##em":6633,"panic":6634,"exact":6635,"cargo":6636,"competing":6637,"fat":6638,"cried":6639,"83":6640,"1920s":6641,"occasions":6642,"pa":6643,"cabin":6644,"borders":6645,"utah":6646,"marcus":6647,"##isation":6648,"badly":6649,"muscles":6650,"##ance":6651,"victorian":6652,"transition":6653,"warner":6654,"bet":6655,"permission":6656,"##rin":6657,"slave":6658,"terrible":6659,"similarly":6660,"shares":6661,"seth":6662,"uefa":6663,"possession":6664,"medals":6665,"benefits":6666,"colleges":6667,"lowered":6668,"perfectly":6669,"mall":6670,"transit":6671,"##ye":6672,"##kar":6673,"publisher":6674,"##ened":6675,"harrison":6676,"deaths":6677,"elevation":6678,"##ae":6679,"asleep":6680,"machines":6681,"sigh":6682,"ash":6683,"hardly":6684,"argument":6685,"occasion":6686,"parent":6687,"leo":6688,"decline":6689,"1888":6690,"contribution":6691,"##ua":6692,"concentration":6693,"1000":6694,"opportunities":6695,"hispanic":6696,"guardian":6697,"extent":6698,"emotions":6699,"hips":6700,"mason":6701,"volumes":6702,"bloody":6703,"controversy":6704,"diameter":6705,"steady":6706,"mistake":6707,"phoenix":6708,"identify":6709,"violin":6710,"##sk":6711,"departure":6712,"richmond":6713,"spin":6714,"funeral":6715,"enemies":6716,"1864":6717,"gear":6718,"literally":6719,"connor":6720,"random":6721,"sergeant":6722,"grab":6723,"confusion":6724,"1865":6725,"transmission":6726,"informed":6727,"op":6728,"leaning":6729,"sacred":6730,"suspended":6731,"thinks":6732,"gates":6733,"portland":6734,"luck":6735,"agencies":6736,"yours":6737,"hull":6738,"expert":6739,"muscle":6740,"layer":6741,"practical":6742,"sculpture":6743,"jerusalem":6744,"latest":6745,"lloyd":6746,"statistics":6747,"deeper":6748,"recommended":6749,"warrior":6750,"arkansas":6751,"mess":6752,"supports":6753,"greg":6754,"eagle":6755,"1880":6756,"recovered":6757,"rated":6758,"concerts":6759,"rushed":6760,"##ano":6761,"stops":6762,"eggs":6763,"files":6764,"premiere":6765,"keith":6766,"##vo":6767,"delhi":6768,"turner":6769,"pit":6770,"affair":6771,"belief":6772,"paint":6773,"##zing":6774,"mate":6775,"##ach":6776,"##ev":6777,"victim":6778,"##ology":6779,"withdrew":6780,"bonus":6781,"styles":6782,"fled":6783,"##ud":6784,"glasgow":6785,"technologies":6786,"funded":6787,"nbc":6788,"adaptation":6789,"##ata":6790,"portrayed":6791,"cooperation":6792,"supporters":6793,"judges":6794,"bernard":6795,"justin":6796,"hallway":6797,"ralph":6798,"##ick":6799,"graduating":6800,"controversial":6801,"distant":6802,"continental":6803,"spider":6804,"bite":6805,"##ho":6806,"recognize":6807,"intention":6808,"mixing":6809,"##ese":6810,"egyptian":6811,"bow":6812,"tourism":6813,"suppose":6814,"claiming":6815,"tiger":6816,"dominated":6817,"participants":6818,"vi":6819,"##ru":6820,"nurse":6821,"partially":6822,"tape":6823,"##rum":6824,"psychology":6825,"##rn":6826,"essential":6827,"touring":6828,"duo":6829,"voting":6830,"civilian":6831,"emotional":6832,"channels":6833,"##king":6834,"apparent":6835,"hebrew":6836,"1887":6837,"tommy":6838,"carrier":6839,"intersection":6840,"beast":6841,"hudson":6842,"##gar":6843,"##zo":6844,"lab":6845,"nova":6846,"bench":6847,"discuss":6848,"costa":6849,"##ered":6850,"detailed":6851,"behalf":6852,"drivers":6853,"unfortunately":6854,"obtain":6855,"##lis":6856,"rocky":6857,"##dae":6858,"siege":6859,"friendship":6860,"honey":6861,"##rian":6862,"1861":6863,"amy":6864,"hang":6865,"posted":6866,"governments":6867,"collins":6868,"respond":6869,"wildlife":6870,"preferred":6871,"operator":6872,"##po":6873,"laura":6874,"pregnant":6875,"videos":6876,"dennis":6877,"suspected":6878,"boots":6879,"instantly":6880,"weird":6881,"automatic":6882,"businessman":6883,"alleged":6884,"placing":6885,"throwing":6886,"ph":6887,"mood":6888,"1862":6889,"perry":6890,"venue":6891,"jet":6892,"remainder":6893,"##lli":6894,"##ci":6895,"passion":6896,"biological":6897,"boyfriend":6898,"1863":6899,"dirt":6900,"buffalo":6901,"ron":6902,"segment":6903,"fa":6904,"abuse":6905,"##era":6906,"genre":6907,"thrown":6908,"stroke":6909,"colored":6910,"stress":6911,"exercise":6912,"displayed":6913,"##gen":6914,"struggled":6915,"##tti":6916,"abroad":6917,"dramatic":6918,"wonderful":6919,"thereafter":6920,"madrid":6921,"component":6922,"widespread":6923,"##sed":6924,"tale":6925,"citizen":6926,"todd":6927,"monday":6928,"1886":6929,"vancouver":6930,"overseas":6931,"forcing":6932,"crying":6933,"descent":6934,"##ris":6935,"discussed":6936,"substantial":6937,"ranks":6938,"regime":6939,"1870":6940,"provinces":6941,"switch":6942,"drum":6943,"zane":6944,"ted":6945,"tribes":6946,"proof":6947,"lp":6948,"cream":6949,"researchers":6950,"volunteer":6951,"manor":6952,"silk":6953,"milan":6954,"donated":6955,"allies":6956,"venture":6957,"principle":6958,"delivery":6959,"enterprise":6960,"##ves":6961,"##ans":6962,"bars":6963,"traditionally":6964,"witch":6965,"reminded":6966,"copper":6967,"##uk":6968,"pete":6969,"inter":6970,"links":6971,"colin":6972,"grinned":6973,"elsewhere":6974,"competitive":6975,"frequent":6976,"##oy":6977,"scream":6978,"##hu":6979,"tension":6980,"texts":6981,"submarine":6982,"finnish":6983,"defending":6984,"defend":6985,"pat":6986,"detail":6987,"1884":6988,"affiliated":6989,"stuart":6990,"themes":6991,"villa":6992,"periods":6993,"tool":6994,"belgian":6995,"ruling":6996,"crimes":6997,"answers":6998,"folded":6999,"licensed":7000,"resort":7001,"demolished":7002,"hans":7003,"lucy":7004,"1881":7005,"lion":7006,"traded":7007,"photographs":7008,"writes":7009,"craig":7010,"##fa":7011,"trials":7012,"generated":7013,"beth":7014,"noble":7015,"debt":7016,"percentage":7017,"yorkshire":7018,"erected":7019,"ss":7020,"viewed":7021,"grades":7022,"confidence":7023,"ceased":7024,"islam":7025,"telephone":7026,"retail":7027,"##ible":7028,"chile":7029,"m²":7030,"roberts":7031,"sixteen":7032,"##ich":7033,"commented":7034,"hampshire":7035,"innocent":7036,"dual":7037,"pounds":7038,"checked":7039,"regulations":7040,"afghanistan":7041,"sung":7042,"rico":7043,"liberty":7044,"assets":7045,"bigger":7046,"options":7047,"angels":7048,"relegated":7049,"tribute":7050,"wells":7051,"attending":7052,"leaf":7053,"##yan":7054,"butler":7055,"romanian":7056,"forum":7057,"monthly":7058,"lisa":7059,"patterns":7060,"gmina":7061,"##tory":7062,"madison":7063,"hurricane":7064,"rev":7065,"##ians":7066,"bristol":7067,"##ula":7068,"elite":7069,"valuable":7070,"disaster":7071,"democracy":7072,"awareness":7073,"germans":7074,"freyja":7075,"##ins":7076,"loop":7077,"absolutely":7078,"paying":7079,"populations":7080,"maine":7081,"sole":7082,"prayer":7083,"spencer":7084,"releases":7085,"doorway":7086,"bull":7087,"##ani":7088,"lover":7089,"midnight":7090,"conclusion":7091,"##sson":7092,"thirteen":7093,"lily":7094,"mediterranean":7095,"##lt":7096,"nhl":7097,"proud":7098,"sample":7099,"##hill":7100,"drummer":7101,"guinea":7102,"##ova":7103,"murphy":7104,"climb":7105,"##ston":7106,"instant":7107,"attributed":7108,"horn":7109,"ain":7110,"railways":7111,"steven":7112,"##ao":7113,"autumn":7114,"ferry":7115,"opponent":7116,"root":7117,"traveling":7118,"secured":7119,"corridor":7120,"stretched":7121,"tales":7122,"sheet":7123,"trinity":7124,"cattle":7125,"helps":7126,"indicates":7127,"manhattan":7128,"murdered":7129,"fitted":7130,"1882":7131,"gentle":7132,"grandmother":7133,"mines":7134,"shocked":7135,"vegas":7136,"produces":7137,"##light":7138,"caribbean":7139,"##ou":7140,"belong":7141,"continuous":7142,"desperate":7143,"drunk":7144,"historically":7145,"trio":7146,"waved":7147,"raf":7148,"dealing":7149,"nathan":7150,"bat":7151,"murmured":7152,"interrupted":7153,"residing":7154,"scientist":7155,"pioneer":7156,"harold":7157,"aaron":7158,"##net":7159,"delta":7160,"attempting":7161,"minority":7162,"mini":7163,"believes":7164,"chorus":7165,"tend":7166,"lots":7167,"eyed":7168,"indoor":7169,"load":7170,"shots":7171,"updated":7172,"jail":7173,"##llo":7174,"concerning":7175,"connecting":7176,"wealth":7177,"##ved":7178,"slaves":7179,"arrive":7180,"rangers":7181,"sufficient":7182,"rebuilt":7183,"##wick":7184,"cardinal":7185,"flood":7186,"muhammad":7187,"whenever":7188,"relation":7189,"runners":7190,"moral":7191,"repair":7192,"viewers":7193,"arriving":7194,"revenge":7195,"punk":7196,"assisted":7197,"bath":7198,"fairly":7199,"breathe":7200,"lists":7201,"innings":7202,"illustrated":7203,"whisper":7204,"nearest":7205,"voters":7206,"clinton":7207,"ties":7208,"ultimate":7209,"screamed":7210,"beijing":7211,"lions":7212,"andre":7213,"fictional":7214,"gathering":7215,"comfort":7216,"radar":7217,"suitable":7218,"dismissed":7219,"hms":7220,"ban":7221,"pine":7222,"wrist":7223,"atmosphere":7224,"voivodeship":7225,"bid":7226,"timber":7227,"##ned":7228,"##nan":7229,"giants":7230,"##ane":7231,"cameron":7232,"recovery":7233,"uss":7234,"identical":7235,"categories":7236,"switched":7237,"serbia":7238,"laughter":7239,"noah":7240,"ensemble":7241,"therapy":7242,"peoples":7243,"touching":7244,"##off":7245,"locally":7246,"pearl":7247,"platforms":7248,"everywhere":7249,"ballet":7250,"tables":7251,"lanka":7252,"herbert":7253,"outdoor":7254,"toured":7255,"derek":7256,"1883":7257,"spaces":7258,"contested":7259,"swept":7260,"1878":7261,"exclusive":7262,"slight":7263,"connections":7264,"##dra":7265,"winds":7266,"prisoner":7267,"collective":7268,"bangladesh":7269,"tube":7270,"publicly":7271,"wealthy":7272,"thai":7273,"##ys":7274,"isolated":7275,"select":7276,"##ric":7277,"insisted":7278,"pen":7279,"fortune":7280,"ticket":7281,"spotted":7282,"reportedly":7283,"animation":7284,"enforcement":7285,"tanks":7286,"110":7287,"decides":7288,"wider":7289,"lowest":7290,"owen":7291,"##time":7292,"nod":7293,"hitting":7294,"##hn":7295,"gregory":7296,"furthermore":7297,"magazines":7298,"fighters":7299,"solutions":7300,"##ery":7301,"pointing":7302,"requested":7303,"peru":7304,"reed":7305,"chancellor":7306,"knights":7307,"mask":7308,"worker":7309,"eldest":7310,"flames":7311,"reduction":7312,"1860":7313,"volunteers":7314,"##tis":7315,"reporting":7316,"##hl":7317,"wire":7318,"advisory":7319,"endemic":7320,"origins":7321,"settlers":7322,"pursue":7323,"knock":7324,"consumer":7325,"1876":7326,"eu":7327,"compound":7328,"creatures":7329,"mansion":7330,"sentenced":7331,"ivan":7332,"deployed":7333,"guitars":7334,"frowned":7335,"involves":7336,"mechanism":7337,"kilometers":7338,"perspective":7339,"shops":7340,"maps":7341,"terminus":7342,"duncan":7343,"alien":7344,"fist":7345,"bridges":7346,"##pers":7347,"heroes":7348,"fed":7349,"derby":7350,"swallowed":7351,"##ros":7352,"patent":7353,"sara":7354,"illness":7355,"characterized":7356,"adventures":7357,"slide":7358,"hawaii":7359,"jurisdiction":7360,"##op":7361,"organised":7362,"##side":7363,"adelaide":7364,"walks":7365,"biology":7366,"se":7367,"##ties":7368,"rogers":7369,"swing":7370,"tightly":7371,"boundaries":7372,"##rie":7373,"prepare":7374,"implementation":7375,"stolen":7376,"##sha":7377,"certified":7378,"colombia":7379,"edwards":7380,"garage":7381,"##mm":7382,"recalled":7383,"##ball":7384,"rage":7385,"harm":7386,"nigeria":7387,"breast":7388,"##ren":7389,"furniture":7390,"pupils":7391,"settle":7392,"##lus":7393,"cuba":7394,"balls":7395,"client":7396,"alaska":7397,"21st":7398,"linear":7399,"thrust":7400,"celebration":7401,"latino":7402,"genetic":7403,"terror":7404,"##cia":7405,"##ening":7406,"lightning":7407,"fee":7408,"witness":7409,"lodge":7410,"establishing":7411,"skull":7412,"##ique":7413,"earning":7414,"hood":7415,"##ei":7416,"rebellion":7417,"wang":7418,"sporting":7419,"warned":7420,"missile":7421,"devoted":7422,"activist":7423,"porch":7424,"worship":7425,"fourteen":7426,"package":7427,"1871":7428,"decorated":7429,"##shire":7430,"housed":7431,"##ock":7432,"chess":7433,"sailed":7434,"doctors":7435,"oscar":7436,"joan":7437,"treat":7438,"garcia":7439,"harbour":7440,"jeremy":7441,"##ire":7442,"traditions":7443,"dominant":7444,"jacques":7445,"##gon":7446,"##wan":7447,"relocated":7448,"1879":7449,"amendment":7450,"sized":7451,"companion":7452,"simultaneously":7453,"volleyball":7454,"spun":7455,"acre":7456,"increases":7457,"stopping":7458,"loves":7459,"belongs":7460,"affect":7461,"drafted":7462,"tossed":7463,"scout":7464,"battles":7465,"1875":7466,"filming":7467,"shoved":7468,"munich":7469,"tenure":7470,"vertical":7471,"romance":7472,"pc":7473,"##cher":7474,"argue":7475,"##ical":7476,"craft":7477,"ranging":7478,"www":7479,"opens":7480,"honest":7481,"tyler":7482,"yesterday":7483,"virtual":7484,"##let":7485,"muslims":7486,"reveal":7487,"snake":7488,"immigrants":7489,"radical":7490,"screaming":7491,"speakers":7492,"firing":7493,"saving":7494,"belonging":7495,"ease":7496,"lighting":7497,"prefecture":7498,"blame":7499,"farmer":7500,"hungry":7501,"grows":7502,"rubbed":7503,"beam":7504,"sur":7505,"subsidiary":7506,"##cha":7507,"armenian":7508,"sao":7509,"dropping":7510,"conventional":7511,"##fer":7512,"microsoft":7513,"reply":7514,"qualify":7515,"spots":7516,"1867":7517,"sweat":7518,"festivals":7519,"##ken":7520,"immigration":7521,"physician":7522,"discover":7523,"exposure":7524,"sandy":7525,"explanation":7526,"isaac":7527,"implemented":7528,"##fish":7529,"hart":7530,"initiated":7531,"connect":7532,"stakes":7533,"presents":7534,"heights":7535,"householder":7536,"pleased":7537,"tourist":7538,"regardless":7539,"slip":7540,"closest":7541,"##ction":7542,"surely":7543,"sultan":7544,"brings":7545,"riley":7546,"preparation":7547,"aboard":7548,"slammed":7549,"baptist":7550,"experiment":7551,"ongoing":7552,"interstate":7553,"organic":7554,"playoffs":7555,"##ika":7556,"1877":7557,"130":7558,"##tar":7559,"hindu":7560,"error":7561,"tours":7562,"tier":7563,"plenty":7564,"arrangements":7565,"talks":7566,"trapped":7567,"excited":7568,"sank":7569,"ho":7570,"athens":7571,"1872":7572,"denver":7573,"welfare":7574,"suburb":7575,"athletes":7576,"trick":7577,"diverse":7578,"belly":7579,"exclusively":7580,"yelled":7581,"1868":7582,"##med":7583,"conversion":7584,"##ette":7585,"1874":7586,"internationally":7587,"computers":7588,"conductor":7589,"abilities":7590,"sensitive":7591,"hello":7592,"dispute":7593,"measured":7594,"globe":7595,"rocket":7596,"prices":7597,"amsterdam":7598,"flights":7599,"tigers":7600,"inn":7601,"municipalities":7602,"emotion":7603,"references":7604,"3d":7605,"##mus":7606,"explains":7607,"airlines":7608,"manufactured":7609,"pm":7610,"archaeological":7611,"1873":7612,"interpretation":7613,"devon":7614,"comment":7615,"##ites":7616,"settlements":7617,"kissing":7618,"absolute":7619,"improvement":7620,"suite":7621,"impressed":7622,"barcelona":7623,"sullivan":7624,"jefferson":7625,"towers":7626,"jesse":7627,"julie":7628,"##tin":7629,"##lu":7630,"grandson":7631,"hi":7632,"gauge":7633,"regard":7634,"rings":7635,"interviews":7636,"trace":7637,"raymond":7638,"thumb":7639,"departments":7640,"burns":7641,"serial":7642,"bulgarian":7643,"scores":7644,"demonstrated":7645,"##ix":7646,"1866":7647,"kyle":7648,"alberta":7649,"underneath":7650,"romanized":7651,"##ward":7652,"relieved":7653,"acquisition":7654,"phrase":7655,"cliff":7656,"reveals":7657,"han":7658,"cuts":7659,"merger":7660,"custom":7661,"##dar":7662,"nee":7663,"gilbert":7664,"graduation":7665,"##nts":7666,"assessment":7667,"cafe":7668,"difficulty":7669,"demands":7670,"swung":7671,"democrat":7672,"jennifer":7673,"commons":7674,"1940s":7675,"grove":7676,"##yo":7677,"completing":7678,"focuses":7679,"sum":7680,"substitute":7681,"bearing":7682,"stretch":7683,"reception":7684,"##py":7685,"reflected":7686,"essentially":7687,"destination":7688,"pairs":7689,"##ched":7690,"survival":7691,"resource":7692,"##bach":7693,"promoting":7694,"doubles":7695,"messages":7696,"tear":7697,"##down":7698,"##fully":7699,"parade":7700,"florence":7701,"harvey":7702,"incumbent":7703,"partial":7704,"framework":7705,"900":7706,"pedro":7707,"frozen":7708,"procedure":7709,"olivia":7710,"controls":7711,"##mic":7712,"shelter":7713,"personally":7714,"temperatures":7715,"##od":7716,"brisbane":7717,"tested":7718,"sits":7719,"marble":7720,"comprehensive":7721,"oxygen":7722,"leonard":7723,"##kov":7724,"inaugural":7725,"iranian":7726,"referring":7727,"quarters":7728,"attitude":7729,"##ivity":7730,"mainstream":7731,"lined":7732,"mars":7733,"dakota":7734,"norfolk":7735,"unsuccessful":7736,"##°":7737,"explosion":7738,"helicopter":7739,"congressional":7740,"##sing":7741,"inspector":7742,"bitch":7743,"seal":7744,"departed":7745,"divine":7746,"##ters":7747,"coaching":7748,"examination":7749,"punishment":7750,"manufacturer":7751,"sink":7752,"columns":7753,"unincorporated":7754,"signals":7755,"nevada":7756,"squeezed":7757,"dylan":7758,"dining":7759,"photos":7760,"martial":7761,"manuel":7762,"eighteen":7763,"elevator":7764,"brushed":7765,"plates":7766,"ministers":7767,"ivy":7768,"congregation":7769,"##len":7770,"slept":7771,"specialized":7772,"taxes":7773,"curve":7774,"restricted":7775,"negotiations":7776,"likes":7777,"statistical":7778,"arnold":7779,"inspiration":7780,"execution":7781,"bold":7782,"intermediate":7783,"significance":7784,"margin":7785,"ruler":7786,"wheels":7787,"gothic":7788,"intellectual":7789,"dependent":7790,"listened":7791,"eligible":7792,"buses":7793,"widow":7794,"syria":7795,"earn":7796,"cincinnati":7797,"collapsed":7798,"recipient":7799,"secrets":7800,"accessible":7801,"philippine":7802,"maritime":7803,"goddess":7804,"clerk":7805,"surrender":7806,"breaks":7807,"playoff":7808,"database":7809,"##ified":7810,"##lon":7811,"ideal":7812,"beetle":7813,"aspect":7814,"soap":7815,"regulation":7816,"strings":7817,"expand":7818,"anglo":7819,"shorter":7820,"crosses":7821,"retreat":7822,"tough":7823,"coins":7824,"wallace":7825,"directions":7826,"pressing":7827,"##oon":7828,"shipping":7829,"locomotives":7830,"comparison":7831,"topics":7832,"nephew":7833,"##mes":7834,"distinction":7835,"honors":7836,"travelled":7837,"sierra":7838,"ibn":7839,"##over":7840,"fortress":7841,"sa":7842,"recognised":7843,"carved":7844,"1869":7845,"clients":7846,"##dan":7847,"intent":7848,"##mar":7849,"coaches":7850,"describing":7851,"bread":7852,"##ington":7853,"beaten":7854,"northwestern":7855,"##ona":7856,"merit":7857,"youtube":7858,"collapse":7859,"challenges":7860,"em":7861,"historians":7862,"objective":7863,"submitted":7864,"virus":7865,"attacking":7866,"drake":7867,"assume":7868,"##ere":7869,"diseases":7870,"marc":7871,"stem":7872,"leeds":7873,"##cus":7874,"##ab":7875,"farming":7876,"glasses":7877,"##lock":7878,"visits":7879,"nowhere":7880,"fellowship":7881,"relevant":7882,"carries":7883,"restaurants":7884,"experiments":7885,"101":7886,"constantly":7887,"bases":7888,"targets":7889,"shah":7890,"tenth":7891,"opponents":7892,"verse":7893,"territorial":7894,"##ira":7895,"writings":7896,"corruption":7897,"##hs":7898,"instruction":7899,"inherited":7900,"reverse":7901,"emphasis":7902,"##vic":7903,"employee":7904,"arch":7905,"keeps":7906,"rabbi":7907,"watson":7908,"payment":7909,"uh":7910,"##ala":7911,"nancy":7912,"##tre":7913,"venice":7914,"fastest":7915,"sexy":7916,"banned":7917,"adrian":7918,"properly":7919,"ruth":7920,"touchdown":7921,"dollar":7922,"boards":7923,"metre":7924,"circles":7925,"edges":7926,"favour":7927,"comments":7928,"ok":7929,"travels":7930,"liberation":7931,"scattered":7932,"firmly":7933,"##ular":7934,"holland":7935,"permitted":7936,"diesel":7937,"kenya":7938,"den":7939,"originated":7940,"##ral":7941,"demons":7942,"resumed":7943,"dragged":7944,"rider":7945,"##rus":7946,"servant":7947,"blinked":7948,"extend":7949,"torn":7950,"##ias":7951,"##sey":7952,"input":7953,"meal":7954,"everybody":7955,"cylinder":7956,"kinds":7957,"camps":7958,"##fe":7959,"bullet":7960,"logic":7961,"##wn":7962,"croatian":7963,"evolved":7964,"healthy":7965,"fool":7966,"chocolate":7967,"wise":7968,"preserve":7969,"pradesh":7970,"##ess":7971,"respective":7972,"1850":7973,"##ew":7974,"chicken":7975,"artificial":7976,"gross":7977,"corresponding":7978,"convicted":7979,"cage":7980,"caroline":7981,"dialogue":7982,"##dor":7983,"narrative":7984,"stranger":7985,"mario":7986,"br":7987,"christianity":7988,"failing":7989,"trent":7990,"commanding":7991,"buddhist":7992,"1848":7993,"maurice":7994,"focusing":7995,"yale":7996,"bike":7997,"altitude":7998,"##ering":7999,"mouse":8000,"revised":8001,"##sley":8002,"veteran":8003,"##ig":8004,"pulls":8005,"theology":8006,"crashed":8007,"campaigns":8008,"legion":8009,"##ability":8010,"drag":8011,"excellence":8012,"customer":8013,"cancelled":8014,"intensity":8015,"excuse":8016,"##lar":8017,"liga":8018,"participating":8019,"contributing":8020,"printing":8021,"##burn":8022,"variable":8023,"##rk":8024,"curious":8025,"bin":8026,"legacy":8027,"renaissance":8028,"##my":8029,"symptoms":8030,"binding":8031,"vocalist":8032,"dancer":8033,"##nie":8034,"grammar":8035,"gospel":8036,"democrats":8037,"ya":8038,"enters":8039,"sc":8040,"diplomatic":8041,"hitler":8042,"##ser":8043,"clouds":8044,"mathematical":8045,"quit":8046,"defended":8047,"oriented":8048,"##heim":8049,"fundamental":8050,"hardware":8051,"impressive":8052,"equally":8053,"convince":8054,"confederate":8055,"guilt":8056,"chuck":8057,"sliding":8058,"##ware":8059,"magnetic":8060,"narrowed":8061,"petersburg":8062,"bulgaria":8063,"otto":8064,"phd":8065,"skill":8066,"##ama":8067,"reader":8068,"hopes":8069,"pitcher":8070,"reservoir":8071,"hearts":8072,"automatically":8073,"expecting":8074,"mysterious":8075,"bennett":8076,"extensively":8077,"imagined":8078,"seeds":8079,"monitor":8080,"fix":8081,"##ative":8082,"journalism":8083,"struggling":8084,"signature":8085,"ranch":8086,"encounter":8087,"photographer":8088,"observation":8089,"protests":8090,"##pin":8091,"influences":8092,"##hr":8093,"calendar":8094,"##all":8095,"cruz":8096,"croatia":8097,"locomotive":8098,"hughes":8099,"naturally":8100,"shakespeare":8101,"basement":8102,"hook":8103,"uncredited":8104,"faded":8105,"theories":8106,"approaches":8107,"dare":8108,"phillips":8109,"filling":8110,"fury":8111,"obama":8112,"##ain":8113,"efficient":8114,"arc":8115,"deliver":8116,"min":8117,"raid":8118,"breeding":8119,"inducted":8120,"leagues":8121,"efficiency":8122,"axis":8123,"montana":8124,"eagles":8125,"##ked":8126,"supplied":8127,"instructions":8128,"karen":8129,"picking":8130,"indicating":8131,"trap":8132,"anchor":8133,"practically":8134,"christians":8135,"tomb":8136,"vary":8137,"occasional":8138,"electronics":8139,"lords":8140,"readers":8141,"newcastle":8142,"faint":8143,"innovation":8144,"collect":8145,"situations":8146,"engagement":8147,"160":8148,"claude":8149,"mixture":8150,"##feld":8151,"peer":8152,"tissue":8153,"logo":8154,"lean":8155,"##ration":8156,"°f":8157,"floors":8158,"##ven":8159,"architects":8160,"reducing":8161,"##our":8162,"##ments":8163,"rope":8164,"1859":8165,"ottawa":8166,"##har":8167,"samples":8168,"banking":8169,"declaration":8170,"proteins":8171,"resignation":8172,"francois":8173,"saudi":8174,"advocate":8175,"exhibited":8176,"armor":8177,"twins":8178,"divorce":8179,"##ras":8180,"abraham":8181,"reviewed":8182,"jo":8183,"temporarily":8184,"matrix":8185,"physically":8186,"pulse":8187,"curled":8188,"##ena":8189,"difficulties":8190,"bengal":8191,"usage":8192,"##ban":8193,"annie":8194,"riders":8195,"certificate":8196,"##pi":8197,"holes":8198,"warsaw":8199,"distinctive":8200,"jessica":8201,"##mon":8202,"mutual":8203,"1857":8204,"customs":8205,"circular":8206,"eugene":8207,"removal":8208,"loaded":8209,"mere":8210,"vulnerable":8211,"depicted":8212,"generations":8213,"dame":8214,"heir":8215,"enormous":8216,"lightly":8217,"climbing":8218,"pitched":8219,"lessons":8220,"pilots":8221,"nepal":8222,"ram":8223,"google":8224,"preparing":8225,"brad":8226,"louise":8227,"renowned":8228,"##₂":8229,"liam":8230,"##ably":8231,"plaza":8232,"shaw":8233,"sophie":8234,"brilliant":8235,"bills":8236,"##bar":8237,"##nik":8238,"fucking":8239,"mainland":8240,"server":8241,"pleasant":8242,"seized":8243,"veterans":8244,"jerked":8245,"fail":8246,"beta":8247,"brush":8248,"radiation":8249,"stored":8250,"warmth":8251,"southeastern":8252,"nate":8253,"sin":8254,"raced":8255,"berkeley":8256,"joke":8257,"athlete":8258,"designation":8259,"trunk":8260,"##low":8261,"roland":8262,"qualification":8263,"archives":8264,"heels":8265,"artwork":8266,"receives":8267,"judicial":8268,"reserves":8269,"##bed":8270,"woke":8271,"installation":8272,"abu":8273,"floating":8274,"fake":8275,"lesser":8276,"excitement":8277,"interface":8278,"concentrated":8279,"addressed":8280,"characteristic":8281,"amanda":8282,"saxophone":8283,"monk":8284,"auto":8285,"##bus":8286,"releasing":8287,"egg":8288,"dies":8289,"interaction":8290,"defender":8291,"ce":8292,"outbreak":8293,"glory":8294,"loving":8295,"##bert":8296,"sequel":8297,"consciousness":8298,"http":8299,"awake":8300,"ski":8301,"enrolled":8302,"##ress":8303,"handling":8304,"rookie":8305,"brow":8306,"somebody":8307,"biography":8308,"warfare":8309,"amounts":8310,"contracts":8311,"presentation":8312,"fabric":8313,"dissolved":8314,"challenged":8315,"meter":8316,"psychological":8317,"lt":8318,"elevated":8319,"rally":8320,"accurate":8321,"##tha":8322,"hospitals":8323,"undergraduate":8324,"specialist":8325,"venezuela":8326,"exhibit":8327,"shed":8328,"nursing":8329,"protestant":8330,"fluid":8331,"structural":8332,"footage":8333,"jared":8334,"consistent":8335,"prey":8336,"##ska":8337,"succession":8338,"reflect":8339,"exile":8340,"lebanon":8341,"wiped":8342,"suspect":8343,"shanghai":8344,"resting":8345,"integration":8346,"preservation":8347,"marvel":8348,"variant":8349,"pirates":8350,"sheep":8351,"rounded":8352,"capita":8353,"sailing":8354,"colonies":8355,"manuscript":8356,"deemed":8357,"variations":8358,"clarke":8359,"functional":8360,"emerging":8361,"boxing":8362,"relaxed":8363,"curse":8364,"azerbaijan":8365,"heavyweight":8366,"nickname":8367,"editorial":8368,"rang":8369,"grid":8370,"tightened":8371,"earthquake":8372,"flashed":8373,"miguel":8374,"rushing":8375,"##ches":8376,"improvements":8377,"boxes":8378,"brooks":8379,"180":8380,"consumption":8381,"molecular":8382,"felix":8383,"societies":8384,"repeatedly":8385,"variation":8386,"aids":8387,"civic":8388,"graphics":8389,"professionals":8390,"realm":8391,"autonomous":8392,"receiver":8393,"delayed":8394,"workshop":8395,"militia":8396,"chairs":8397,"trump":8398,"canyon":8399,"##point":8400,"harsh":8401,"extending":8402,"lovely":8403,"happiness":8404,"##jan":8405,"stake":8406,"eyebrows":8407,"embassy":8408,"wellington":8409,"hannah":8410,"##ella":8411,"sony":8412,"corners":8413,"bishops":8414,"swear":8415,"cloth":8416,"contents":8417,"xi":8418,"namely":8419,"commenced":8420,"1854":8421,"stanford":8422,"nashville":8423,"courage":8424,"graphic":8425,"commitment":8426,"garrison":8427,"##bin":8428,"hamlet":8429,"clearing":8430,"rebels":8431,"attraction":8432,"literacy":8433,"cooking":8434,"ruins":8435,"temples":8436,"jenny":8437,"humanity":8438,"celebrate":8439,"hasn":8440,"freight":8441,"sixty":8442,"rebel":8443,"bastard":8444,"##art":8445,"newton":8446,"##ada":8447,"deer":8448,"##ges":8449,"##ching":8450,"smiles":8451,"delaware":8452,"singers":8453,"##ets":8454,"approaching":8455,"assists":8456,"flame":8457,"##ph":8458,"boulevard":8459,"barrel":8460,"planted":8461,"##ome":8462,"pursuit":8463,"##sia":8464,"consequences":8465,"posts":8466,"shallow":8467,"invitation":8468,"rode":8469,"depot":8470,"ernest":8471,"kane":8472,"rod":8473,"concepts":8474,"preston":8475,"topic":8476,"chambers":8477,"striking":8478,"blast":8479,"arrives":8480,"descendants":8481,"montgomery":8482,"ranges":8483,"worlds":8484,"##lay":8485,"##ari":8486,"span":8487,"chaos":8488,"praise":8489,"##ag":8490,"fewer":8491,"1855":8492,"sanctuary":8493,"mud":8494,"fbi":8495,"##ions":8496,"programmes":8497,"maintaining":8498,"unity":8499,"harper":8500,"bore":8501,"handsome":8502,"closure":8503,"tournaments":8504,"thunder":8505,"nebraska":8506,"linda":8507,"facade":8508,"puts":8509,"satisfied":8510,"argentine":8511,"dale":8512,"cork":8513,"dome":8514,"panama":8515,"##yl":8516,"1858":8517,"tasks":8518,"experts":8519,"##ates":8520,"feeding":8521,"equation":8522,"##las":8523,"##ida":8524,"##tu":8525,"engage":8526,"bryan":8527,"##ax":8528,"um":8529,"quartet":8530,"melody":8531,"disbanded":8532,"sheffield":8533,"blocked":8534,"gasped":8535,"delay":8536,"kisses":8537,"maggie":8538,"connects":8539,"##non":8540,"sts":8541,"poured":8542,"creator":8543,"publishers":8544,"##we":8545,"guided":8546,"ellis":8547,"extinct":8548,"hug":8549,"gaining":8550,"##ord":8551,"complicated":8552,"##bility":8553,"poll":8554,"clenched":8555,"investigate":8556,"##use":8557,"thereby":8558,"quantum":8559,"spine":8560,"cdp":8561,"humor":8562,"kills":8563,"administered":8564,"semifinals":8565,"##du":8566,"encountered":8567,"ignore":8568,"##bu":8569,"commentary":8570,"##maker":8571,"bother":8572,"roosevelt":8573,"140":8574,"plains":8575,"halfway":8576,"flowing":8577,"cultures":8578,"crack":8579,"imprisoned":8580,"neighboring":8581,"airline":8582,"##ses":8583,"##view":8584,"##mate":8585,"##ec":8586,"gather":8587,"wolves":8588,"marathon":8589,"transformed":8590,"##ill":8591,"cruise":8592,"organisations":8593,"carol":8594,"punch":8595,"exhibitions":8596,"numbered":8597,"alarm":8598,"ratings":8599,"daddy":8600,"silently":8601,"##stein":8602,"queens":8603,"colours":8604,"impression":8605,"guidance":8606,"liu":8607,"tactical":8608,"##rat":8609,"marshal":8610,"della":8611,"arrow":8612,"##ings":8613,"rested":8614,"feared":8615,"tender":8616,"owns":8617,"bitter":8618,"advisor":8619,"escort":8620,"##ides":8621,"spare":8622,"farms":8623,"grants":8624,"##ene":8625,"dragons":8626,"encourage":8627,"colleagues":8628,"cameras":8629,"##und":8630,"sucked":8631,"pile":8632,"spirits":8633,"prague":8634,"statements":8635,"suspension":8636,"landmark":8637,"fence":8638,"torture":8639,"recreation":8640,"bags":8641,"permanently":8642,"survivors":8643,"pond":8644,"spy":8645,"predecessor":8646,"bombing":8647,"coup":8648,"##og":8649,"protecting":8650,"transformation":8651,"glow":8652,"##lands":8653,"##book":8654,"dug":8655,"priests":8656,"andrea":8657,"feat":8658,"barn":8659,"jumping":8660,"##chen":8661,"##ologist":8662,"##con":8663,"casualties":8664,"stern":8665,"auckland":8666,"pipe":8667,"serie":8668,"revealing":8669,"ba":8670,"##bel":8671,"trevor":8672,"mercy":8673,"spectrum":8674,"yang":8675,"consist":8676,"governing":8677,"collaborated":8678,"possessed":8679,"epic":8680,"comprises":8681,"blew":8682,"shane":8683,"##ack":8684,"lopez":8685,"honored":8686,"magical":8687,"sacrifice":8688,"judgment":8689,"perceived":8690,"hammer":8691,"mtv":8692,"baronet":8693,"tune":8694,"das":8695,"missionary":8696,"sheets":8697,"350":8698,"neutral":8699,"oral":8700,"threatening":8701,"attractive":8702,"shade":8703,"aims":8704,"seminary":8705,"##master":8706,"estates":8707,"1856":8708,"michel":8709,"wounds":8710,"refugees":8711,"manufacturers":8712,"##nic":8713,"mercury":8714,"syndrome":8715,"porter":8716,"##iya":8717,"##din":8718,"hamburg":8719,"identification":8720,"upstairs":8721,"purse":8722,"widened":8723,"pause":8724,"cared":8725,"breathed":8726,"affiliate":8727,"santiago":8728,"prevented":8729,"celtic":8730,"fisher":8731,"125":8732,"recruited":8733,"byzantine":8734,"reconstruction":8735,"farther":8736,"##mp":8737,"diet":8738,"sake":8739,"au":8740,"spite":8741,"sensation":8742,"##ert":8743,"blank":8744,"separation":8745,"105":8746,"##hon":8747,"vladimir":8748,"armies":8749,"anime":8750,"##lie":8751,"accommodate":8752,"orbit":8753,"cult":8754,"sofia":8755,"archive":8756,"##ify":8757,"##box":8758,"founders":8759,"sustained":8760,"disorder":8761,"honours":8762,"northeastern":8763,"mia":8764,"crops":8765,"violet":8766,"threats":8767,"blanket":8768,"fires":8769,"canton":8770,"followers":8771,"southwestern":8772,"prototype":8773,"voyage":8774,"assignment":8775,"altered":8776,"moderate":8777,"protocol":8778,"pistol":8779,"##eo":8780,"questioned":8781,"brass":8782,"lifting":8783,"1852":8784,"math":8785,"authored":8786,"##ual":8787,"doug":8788,"dimensional":8789,"dynamic":8790,"##san":8791,"1851":8792,"pronounced":8793,"grateful":8794,"quest":8795,"uncomfortable":8796,"boom":8797,"presidency":8798,"stevens":8799,"relating":8800,"politicians":8801,"chen":8802,"barrier":8803,"quinn":8804,"diana":8805,"mosque":8806,"tribal":8807,"cheese":8808,"palmer":8809,"portions":8810,"sometime":8811,"chester":8812,"treasure":8813,"wu":8814,"bend":8815,"download":8816,"millions":8817,"reforms":8818,"registration":8819,"##osa":8820,"consequently":8821,"monitoring":8822,"ate":8823,"preliminary":8824,"brandon":8825,"invented":8826,"ps":8827,"eaten":8828,"exterior":8829,"intervention":8830,"ports":8831,"documented":8832,"log":8833,"displays":8834,"lecture":8835,"sally":8836,"favourite":8837,"##itz":8838,"vermont":8839,"lo":8840,"invisible":8841,"isle":8842,"breed":8843,"##ator":8844,"journalists":8845,"relay":8846,"speaks":8847,"backward":8848,"explore":8849,"midfielder":8850,"actively":8851,"stefan":8852,"procedures":8853,"cannon":8854,"blond":8855,"kenneth":8856,"centered":8857,"servants":8858,"chains":8859,"libraries":8860,"malcolm":8861,"essex":8862,"henri":8863,"slavery":8864,"##hal":8865,"facts":8866,"fairy":8867,"coached":8868,"cassie":8869,"cats":8870,"washed":8871,"cop":8872,"##fi":8873,"announcement":8874,"item":8875,"2000s":8876,"vinyl":8877,"activated":8878,"marco":8879,"frontier":8880,"growled":8881,"curriculum":8882,"##das":8883,"loyal":8884,"accomplished":8885,"leslie":8886,"ritual":8887,"kenny":8888,"##00":8889,"vii":8890,"napoleon":8891,"hollow":8892,"hybrid":8893,"jungle":8894,"stationed":8895,"friedrich":8896,"counted":8897,"##ulated":8898,"platinum":8899,"theatrical":8900,"seated":8901,"col":8902,"rubber":8903,"glen":8904,"1840":8905,"diversity":8906,"healing":8907,"extends":8908,"id":8909,"provisions":8910,"administrator":8911,"columbus":8912,"##oe":8913,"tributary":8914,"te":8915,"assured":8916,"org":8917,"##uous":8918,"prestigious":8919,"examined":8920,"lectures":8921,"grammy":8922,"ronald":8923,"associations":8924,"bailey":8925,"allan":8926,"essays":8927,"flute":8928,"believing":8929,"consultant":8930,"proceedings":8931,"travelling":8932,"1853":8933,"kit":8934,"kerala":8935,"yugoslavia":8936,"buddy":8937,"methodist":8938,"##ith":8939,"burial":8940,"centres":8941,"batman":8942,"##nda":8943,"discontinued":8944,"bo":8945,"dock":8946,"stockholm":8947,"lungs":8948,"severely":8949,"##nk":8950,"citing":8951,"manga":8952,"##ugh":8953,"steal":8954,"mumbai":8955,"iraqi":8956,"robot":8957,"celebrity":8958,"bride":8959,"broadcasts":8960,"abolished":8961,"pot":8962,"joel":8963,"overhead":8964,"franz":8965,"packed":8966,"reconnaissance":8967,"johann":8968,"acknowledged":8969,"introduce":8970,"handled":8971,"doctorate":8972,"developments":8973,"drinks":8974,"alley":8975,"palestine":8976,"##nis":8977,"##aki":8978,"proceeded":8979,"recover":8980,"bradley":8981,"grain":8982,"patch":8983,"afford":8984,"infection":8985,"nationalist":8986,"legendary":8987,"##ath":8988,"interchange":8989,"virtually":8990,"gen":8991,"gravity":8992,"exploration":8993,"amber":8994,"vital":8995,"wishes":8996,"powell":8997,"doctrine":8998,"elbow":8999,"screenplay":9000,"##bird":9001,"contribute":9002,"indonesian":9003,"pet":9004,"creates":9005,"##com":9006,"enzyme":9007,"kylie":9008,"discipline":9009,"drops":9010,"manila":9011,"hunger":9012,"##ien":9013,"layers":9014,"suffer":9015,"fever":9016,"bits":9017,"monica":9018,"keyboard":9019,"manages":9020,"##hood":9021,"searched":9022,"appeals":9023,"##bad":9024,"testament":9025,"grande":9026,"reid":9027,"##war":9028,"beliefs":9029,"congo":9030,"##ification":9031,"##dia":9032,"si":9033,"requiring":9034,"##via":9035,"casey":9036,"1849":9037,"regret":9038,"streak":9039,"rape":9040,"depends":9041,"syrian":9042,"sprint":9043,"pound":9044,"tourists":9045,"upcoming":9046,"pub":9047,"##xi":9048,"tense":9049,"##els":9050,"practiced":9051,"echo":9052,"nationwide":9053,"guild":9054,"motorcycle":9055,"liz":9056,"##zar":9057,"chiefs":9058,"desired":9059,"elena":9060,"bye":9061,"precious":9062,"absorbed":9063,"relatives":9064,"booth":9065,"pianist":9066,"##mal":9067,"citizenship":9068,"exhausted":9069,"wilhelm":9070,"##ceae":9071,"##hed":9072,"noting":9073,"quarterback":9074,"urge":9075,"hectares":9076,"##gue":9077,"ace":9078,"holly":9079,"##tal":9080,"blonde":9081,"davies":9082,"parked":9083,"sustainable":9084,"stepping":9085,"twentieth":9086,"airfield":9087,"galaxy":9088,"nest":9089,"chip":9090,"##nell":9091,"tan":9092,"shaft":9093,"paulo":9094,"requirement":9095,"##zy":9096,"paradise":9097,"tobacco":9098,"trans":9099,"renewed":9100,"vietnamese":9101,"##cker":9102,"##ju":9103,"suggesting":9104,"catching":9105,"holmes":9106,"enjoying":9107,"md":9108,"trips":9109,"colt":9110,"holder":9111,"butterfly":9112,"nerve":9113,"reformed":9114,"cherry":9115,"bowling":9116,"trailer":9117,"carriage":9118,"goodbye":9119,"appreciate":9120,"toy":9121,"joshua":9122,"interactive":9123,"enabled":9124,"involve":9125,"##kan":9126,"collar":9127,"determination":9128,"bunch":9129,"facebook":9130,"recall":9131,"shorts":9132,"superintendent":9133,"episcopal":9134,"frustration":9135,"giovanni":9136,"nineteenth":9137,"laser":9138,"privately":9139,"array":9140,"circulation":9141,"##ovic":9142,"armstrong":9143,"deals":9144,"painful":9145,"permit":9146,"discrimination":9147,"##wi":9148,"aires":9149,"retiring":9150,"cottage":9151,"ni":9152,"##sta":9153,"horizon":9154,"ellen":9155,"jamaica":9156,"ripped":9157,"fernando":9158,"chapters":9159,"playstation":9160,"patron":9161,"lecturer":9162,"navigation":9163,"behaviour":9164,"genes":9165,"georgian":9166,"export":9167,"solomon":9168,"rivals":9169,"swift":9170,"seventeen":9171,"rodriguez":9172,"princeton":9173,"independently":9174,"sox":9175,"1847":9176,"arguing":9177,"entity":9178,"casting":9179,"hank":9180,"criteria":9181,"oakland":9182,"geographic":9183,"milwaukee":9184,"reflection":9185,"expanding":9186,"conquest":9187,"dubbed":9188,"##tv":9189,"halt":9190,"brave":9191,"brunswick":9192,"doi":9193,"arched":9194,"curtis":9195,"divorced":9196,"predominantly":9197,"somerset":9198,"streams":9199,"ugly":9200,"zoo":9201,"horrible":9202,"curved":9203,"buenos":9204,"fierce":9205,"dictionary":9206,"vector":9207,"theological":9208,"unions":9209,"handful":9210,"stability":9211,"chan":9212,"punjab":9213,"segments":9214,"##lly":9215,"altar":9216,"ignoring":9217,"gesture":9218,"monsters":9219,"pastor":9220,"##stone":9221,"thighs":9222,"unexpected":9223,"operators":9224,"abruptly":9225,"coin":9226,"compiled":9227,"associates":9228,"improving":9229,"migration":9230,"pin":9231,"##ose":9232,"compact":9233,"collegiate":9234,"reserved":9235,"##urs":9236,"quarterfinals":9237,"roster":9238,"restore":9239,"assembled":9240,"hurry":9241,"oval":9242,"##cies":9243,"1846":9244,"flags":9245,"martha":9246,"##del":9247,"victories":9248,"sharply":9249,"##rated":9250,"argues":9251,"deadly":9252,"neo":9253,"drawings":9254,"symbols":9255,"performer":9256,"##iel":9257,"griffin":9258,"restrictions":9259,"editing":9260,"andrews":9261,"java":9262,"journals":9263,"arabia":9264,"compositions":9265,"dee":9266,"pierce":9267,"removing":9268,"hindi":9269,"casino":9270,"runway":9271,"civilians":9272,"minds":9273,"nasa":9274,"hotels":9275,"##zation":9276,"refuge":9277,"rent":9278,"retain":9279,"potentially":9280,"conferences":9281,"suburban":9282,"conducting":9283,"##tto":9284,"##tions":9285,"##tle":9286,"descended":9287,"massacre":9288,"##cal":9289,"ammunition":9290,"terrain":9291,"fork":9292,"souls":9293,"counts":9294,"chelsea":9295,"durham":9296,"drives":9297,"cab":9298,"##bank":9299,"perth":9300,"realizing":9301,"palestinian":9302,"finn":9303,"simpson":9304,"##dal":9305,"betty":9306,"##ule":9307,"moreover":9308,"particles":9309,"cardinals":9310,"tent":9311,"evaluation":9312,"extraordinary":9313,"##oid":9314,"inscription":9315,"##works":9316,"wednesday":9317,"chloe":9318,"maintains":9319,"panels":9320,"ashley":9321,"trucks":9322,"##nation":9323,"cluster":9324,"sunlight":9325,"strikes":9326,"zhang":9327,"##wing":9328,"dialect":9329,"canon":9330,"##ap":9331,"tucked":9332,"##ws":9333,"collecting":9334,"##mas":9335,"##can":9336,"##sville":9337,"maker":9338,"quoted":9339,"evan":9340,"franco":9341,"aria":9342,"buying":9343,"cleaning":9344,"eva":9345,"closet":9346,"provision":9347,"apollo":9348,"clinic":9349,"rat":9350,"##ez":9351,"necessarily":9352,"ac":9353,"##gle":9354,"##ising":9355,"venues":9356,"flipped":9357,"cent":9358,"spreading":9359,"trustees":9360,"checking":9361,"authorized":9362,"##sco":9363,"disappointed":9364,"##ado":9365,"notion":9366,"duration":9367,"trumpet":9368,"hesitated":9369,"topped":9370,"brussels":9371,"rolls":9372,"theoretical":9373,"hint":9374,"define":9375,"aggressive":9376,"repeat":9377,"wash":9378,"peaceful":9379,"optical":9380,"width":9381,"allegedly":9382,"mcdonald":9383,"strict":9384,"copyright":9385,"##illa":9386,"investors":9387,"mar":9388,"jam":9389,"witnesses":9390,"sounding":9391,"miranda":9392,"michelle":9393,"privacy":9394,"hugo":9395,"harmony":9396,"##pp":9397,"valid":9398,"lynn":9399,"glared":9400,"nina":9401,"102":9402,"headquartered":9403,"diving":9404,"boarding":9405,"gibson":9406,"##ncy":9407,"albanian":9408,"marsh":9409,"routine":9410,"dealt":9411,"enhanced":9412,"er":9413,"intelligent":9414,"substance":9415,"targeted":9416,"enlisted":9417,"discovers":9418,"spinning":9419,"observations":9420,"pissed":9421,"smoking":9422,"rebecca":9423,"capitol":9424,"visa":9425,"varied":9426,"costume":9427,"seemingly":9428,"indies":9429,"compensation":9430,"surgeon":9431,"thursday":9432,"arsenal":9433,"westminster":9434,"suburbs":9435,"rid":9436,"anglican":9437,"##ridge":9438,"knots":9439,"foods":9440,"alumni":9441,"lighter":9442,"fraser":9443,"whoever":9444,"portal":9445,"scandal":9446,"##ray":9447,"gavin":9448,"advised":9449,"instructor":9450,"flooding":9451,"terrorist":9452,"##ale":9453,"teenage":9454,"interim":9455,"senses":9456,"duck":9457,"teen":9458,"thesis":9459,"abby":9460,"eager":9461,"overcome":9462,"##ile":9463,"newport":9464,"glenn":9465,"rises":9466,"shame":9467,"##cc":9468,"prompted":9469,"priority":9470,"forgot":9471,"bomber":9472,"nicolas":9473,"protective":9474,"360":9475,"cartoon":9476,"katherine":9477,"breeze":9478,"lonely":9479,"trusted":9480,"henderson":9481,"richardson":9482,"relax":9483,"banner":9484,"candy":9485,"palms":9486,"remarkable":9487,"##rio":9488,"legends":9489,"cricketer":9490,"essay":9491,"ordained":9492,"edmund":9493,"rifles":9494,"trigger":9495,"##uri":9496,"##away":9497,"sail":9498,"alert":9499,"1830":9500,"audiences":9501,"penn":9502,"sussex":9503,"siblings":9504,"pursued":9505,"indianapolis":9506,"resist":9507,"rosa":9508,"consequence":9509,"succeed":9510,"avoided":9511,"1845":9512,"##ulation":9513,"inland":9514,"##tie":9515,"##nna":9516,"counsel":9517,"profession":9518,"chronicle":9519,"hurried":9520,"##una":9521,"eyebrow":9522,"eventual":9523,"bleeding":9524,"innovative":9525,"cure":9526,"##dom":9527,"committees":9528,"accounting":9529,"con":9530,"scope":9531,"hardy":9532,"heather":9533,"tenor":9534,"gut":9535,"herald":9536,"codes":9537,"tore":9538,"scales":9539,"wagon":9540,"##oo":9541,"luxury":9542,"tin":9543,"prefer":9544,"fountain":9545,"triangle":9546,"bonds":9547,"darling":9548,"convoy":9549,"dried":9550,"traced":9551,"beings":9552,"troy":9553,"accidentally":9554,"slam":9555,"findings":9556,"smelled":9557,"joey":9558,"lawyers":9559,"outcome":9560,"steep":9561,"bosnia":9562,"configuration":9563,"shifting":9564,"toll":9565,"brook":9566,"performers":9567,"lobby":9568,"philosophical":9569,"construct":9570,"shrine":9571,"aggregate":9572,"boot":9573,"cox":9574,"phenomenon":9575,"savage":9576,"insane":9577,"solely":9578,"reynolds":9579,"lifestyle":9580,"##ima":9581,"nationally":9582,"holdings":9583,"consideration":9584,"enable":9585,"edgar":9586,"mo":9587,"mama":9588,"##tein":9589,"fights":9590,"relegation":9591,"chances":9592,"atomic":9593,"hub":9594,"conjunction":9595,"awkward":9596,"reactions":9597,"currency":9598,"finale":9599,"kumar":9600,"underwent":9601,"steering":9602,"elaborate":9603,"gifts":9604,"comprising":9605,"melissa":9606,"veins":9607,"reasonable":9608,"sunshine":9609,"chi":9610,"solve":9611,"trails":9612,"inhabited":9613,"elimination":9614,"ethics":9615,"huh":9616,"ana":9617,"molly":9618,"consent":9619,"apartments":9620,"layout":9621,"marines":9622,"##ces":9623,"hunters":9624,"bulk":9625,"##oma":9626,"hometown":9627,"##wall":9628,"##mont":9629,"cracked":9630,"reads":9631,"neighbouring":9632,"withdrawn":9633,"admission":9634,"wingspan":9635,"damned":9636,"anthology":9637,"lancashire":9638,"brands":9639,"batting":9640,"forgive":9641,"cuban":9642,"awful":9643,"##lyn":9644,"104":9645,"dimensions":9646,"imagination":9647,"##ade":9648,"dante":9649,"##ship":9650,"tracking":9651,"desperately":9652,"goalkeeper":9653,"##yne":9654,"groaned":9655,"workshops":9656,"confident":9657,"burton":9658,"gerald":9659,"milton":9660,"circus":9661,"uncertain":9662,"slope":9663,"copenhagen":9664,"sophia":9665,"fog":9666,"philosopher":9667,"portraits":9668,"accent":9669,"cycling":9670,"varying":9671,"gripped":9672,"larvae":9673,"garrett":9674,"specified":9675,"scotia":9676,"mature":9677,"luther":9678,"kurt":9679,"rap":9680,"##kes":9681,"aerial":9682,"750":9683,"ferdinand":9684,"heated":9685,"es":9686,"transported":9687,"##shan":9688,"safely":9689,"nonetheless":9690,"##orn":9691,"##gal":9692,"motors":9693,"demanding":9694,"##sburg":9695,"startled":9696,"##brook":9697,"ally":9698,"generate":9699,"caps":9700,"ghana":9701,"stained":9702,"demo":9703,"mentions":9704,"beds":9705,"ap":9706,"afterward":9707,"diary":9708,"##bling":9709,"utility":9710,"##iro":9711,"richards":9712,"1837":9713,"conspiracy":9714,"conscious":9715,"shining":9716,"footsteps":9717,"observer":9718,"cyprus":9719,"urged":9720,"loyalty":9721,"developer":9722,"probability":9723,"olive":9724,"upgraded":9725,"gym":9726,"miracle":9727,"insects":9728,"graves":9729,"1844":9730,"ourselves":9731,"hydrogen":9732,"amazon":9733,"katie":9734,"tickets":9735,"poets":9736,"##pm":9737,"planes":9738,"##pan":9739,"prevention":9740,"witnessed":9741,"dense":9742,"jin":9743,"randy":9744,"tang":9745,"warehouse":9746,"monroe":9747,"bang":9748,"archived":9749,"elderly":9750,"investigations":9751,"alec":9752,"granite":9753,"mineral":9754,"conflicts":9755,"controlling":9756,"aboriginal":9757,"carlo":9758,"##zu":9759,"mechanics":9760,"stan":9761,"stark":9762,"rhode":9763,"skirt":9764,"est":9765,"##berry":9766,"bombs":9767,"respected":9768,"##horn":9769,"imposed":9770,"limestone":9771,"deny":9772,"nominee":9773,"memphis":9774,"grabbing":9775,"disabled":9776,"##als":9777,"amusement":9778,"aa":9779,"frankfurt":9780,"corn":9781,"referendum":9782,"varies":9783,"slowed":9784,"disk":9785,"firms":9786,"unconscious":9787,"incredible":9788,"clue":9789,"sue":9790,"##zhou":9791,"twist":9792,"##cio":9793,"joins":9794,"idaho":9795,"chad":9796,"developers":9797,"computing":9798,"destroyer":9799,"103":9800,"mortal":9801,"tucker":9802,"kingston":9803,"choices":9804,"yu":9805,"carson":9806,"1800":9807,"os":9808,"whitney":9809,"geneva":9810,"pretend":9811,"dimension":9812,"staged":9813,"plateau":9814,"maya":9815,"##une":9816,"freestyle":9817,"##bc":9818,"rovers":9819,"hiv":9820,"##ids":9821,"tristan":9822,"classroom":9823,"prospect":9824,"##hus":9825,"honestly":9826,"diploma":9827,"lied":9828,"thermal":9829,"auxiliary":9830,"feast":9831,"unlikely":9832,"iata":9833,"##tel":9834,"morocco":9835,"pounding":9836,"treasury":9837,"lithuania":9838,"considerably":9839,"1841":9840,"dish":9841,"1812":9842,"geological":9843,"matching":9844,"stumbled":9845,"destroying":9846,"marched":9847,"brien":9848,"advances":9849,"cake":9850,"nicole":9851,"belle":9852,"settling":9853,"measuring":9854,"directing":9855,"##mie":9856,"tuesday":9857,"bassist":9858,"capabilities":9859,"stunned":9860,"fraud":9861,"torpedo":9862,"##list":9863,"##phone":9864,"anton":9865,"wisdom":9866,"surveillance":9867,"ruined":9868,"##ulate":9869,"lawsuit":9870,"healthcare":9871,"theorem":9872,"halls":9873,"trend":9874,"aka":9875,"horizontal":9876,"dozens":9877,"acquire":9878,"lasting":9879,"swim":9880,"hawk":9881,"gorgeous":9882,"fees":9883,"vicinity":9884,"decrease":9885,"adoption":9886,"tactics":9887,"##ography":9888,"pakistani":9889,"##ole":9890,"draws":9891,"##hall":9892,"willie":9893,"burke":9894,"heath":9895,"algorithm":9896,"integral":9897,"powder":9898,"elliott":9899,"brigadier":9900,"jackie":9901,"tate":9902,"varieties":9903,"darker":9904,"##cho":9905,"lately":9906,"cigarette":9907,"specimens":9908,"adds":9909,"##ree":9910,"##ensis":9911,"##inger":9912,"exploded":9913,"finalist":9914,"cia":9915,"murders":9916,"wilderness":9917,"arguments":9918,"nicknamed":9919,"acceptance":9920,"onwards":9921,"manufacture":9922,"robertson":9923,"jets":9924,"tampa":9925,"enterprises":9926,"blog":9927,"loudly":9928,"composers":9929,"nominations":9930,"1838":9931,"ai":9932,"malta":9933,"inquiry":9934,"automobile":9935,"hosting":9936,"viii":9937,"rays":9938,"tilted":9939,"grief":9940,"museums":9941,"strategies":9942,"furious":9943,"euro":9944,"equality":9945,"cohen":9946,"poison":9947,"surrey":9948,"wireless":9949,"governed":9950,"ridiculous":9951,"moses":9952,"##esh":9953,"##room":9954,"vanished":9955,"##ito":9956,"barnes":9957,"attract":9958,"morrison":9959,"istanbul":9960,"##iness":9961,"absent":9962,"rotation":9963,"petition":9964,"janet":9965,"##logical":9966,"satisfaction":9967,"custody":9968,"deliberately":9969,"observatory":9970,"comedian":9971,"surfaces":9972,"pinyin":9973,"novelist":9974,"strictly":9975,"canterbury":9976,"oslo":9977,"monks":9978,"embrace":9979,"ibm":9980,"jealous":9981,"photograph":9982,"continent":9983,"dorothy":9984,"marina":9985,"doc":9986,"excess":9987,"holden":9988,"allegations":9989,"explaining":9990,"stack":9991,"avoiding":9992,"lance":9993,"storyline":9994,"majesty":9995,"poorly":9996,"spike":9997,"dos":9998,"bradford":9999,"raven":10000,"travis":10001,"classics":10002,"proven":10003,"voltage":10004,"pillow":10005,"fists":10006,"butt":10007,"1842":10008,"interpreted":10009,"##car":10010,"1839":10011,"gage":10012,"telegraph":10013,"lens":10014,"promising":10015,"expelled":10016,"casual":10017,"collector":10018,"zones":10019,"##min":10020,"silly":10021,"nintendo":10022,"##kh":10023,"##bra":10024,"downstairs":10025,"chef":10026,"suspicious":10027,"afl":10028,"flies":10029,"vacant":10030,"uganda":10031,"pregnancy":10032,"condemned":10033,"lutheran":10034,"estimates":10035,"cheap":10036,"decree":10037,"saxon":10038,"proximity":10039,"stripped":10040,"idiot":10041,"deposits":10042,"contrary":10043,"presenter":10044,"magnus":10045,"glacier":10046,"im":10047,"offense":10048,"edwin":10049,"##ori":10050,"upright":10051,"##long":10052,"bolt":10053,"##ois":10054,"toss":10055,"geographical":10056,"##izes":10057,"environments":10058,"delicate":10059,"marking":10060,"abstract":10061,"xavier":10062,"nails":10063,"windsor":10064,"plantation":10065,"occurring":10066,"equity":10067,"saskatchewan":10068,"fears":10069,"drifted":10070,"sequences":10071,"vegetation":10072,"revolt":10073,"##stic":10074,"1843":10075,"sooner":10076,"fusion":10077,"opposing":10078,"nato":10079,"skating":10080,"1836":10081,"secretly":10082,"ruin":10083,"lease":10084,"##oc":10085,"edit":10086,"##nne":10087,"flora":10088,"anxiety":10089,"ruby":10090,"##ological":10091,"##mia":10092,"tel":10093,"bout":10094,"taxi":10095,"emmy":10096,"frost":10097,"rainbow":10098,"compounds":10099,"foundations":10100,"rainfall":10101,"assassination":10102,"nightmare":10103,"dominican":10104,"##win":10105,"achievements":10106,"deserve":10107,"orlando":10108,"intact":10109,"armenia":10110,"##nte":10111,"calgary":10112,"valentine":10113,"106":10114,"marion":10115,"proclaimed":10116,"theodore":10117,"bells":10118,"courtyard":10119,"thigh":10120,"gonzalez":10121,"console":10122,"troop":10123,"minimal":10124,"monte":10125,"everyday":10126,"##ence":10127,"##if":10128,"supporter":10129,"terrorism":10130,"buck":10131,"openly":10132,"presbyterian":10133,"activists":10134,"carpet":10135,"##iers":10136,"rubbing":10137,"uprising":10138,"##yi":10139,"cute":10140,"conceived":10141,"legally":10142,"##cht":10143,"millennium":10144,"cello":10145,"velocity":10146,"ji":10147,"rescued":10148,"cardiff":10149,"1835":10150,"rex":10151,"concentrate":10152,"senators":10153,"beard":10154,"rendered":10155,"glowing":10156,"battalions":10157,"scouts":10158,"competitors":10159,"sculptor":10160,"catalogue":10161,"arctic":10162,"ion":10163,"raja":10164,"bicycle":10165,"wow":10166,"glancing":10167,"lawn":10168,"##woman":10169,"gentleman":10170,"lighthouse":10171,"publish":10172,"predicted":10173,"calculated":10174,"##val":10175,"variants":10176,"##gne":10177,"strain":10178,"##ui":10179,"winston":10180,"deceased":10181,"##nus":10182,"touchdowns":10183,"brady":10184,"caleb":10185,"sinking":10186,"echoed":10187,"crush":10188,"hon":10189,"blessed":10190,"protagonist":10191,"hayes":10192,"endangered":10193,"magnitude":10194,"editors":10195,"##tine":10196,"estimate":10197,"responsibilities":10198,"##mel":10199,"backup":10200,"laying":10201,"consumed":10202,"sealed":10203,"zurich":10204,"lovers":10205,"frustrated":10206,"##eau":10207,"ahmed":10208,"kicking":10209,"mit":10210,"treasurer":10211,"1832":10212,"biblical":10213,"refuse":10214,"terrified":10215,"pump":10216,"agrees":10217,"genuine":10218,"imprisonment":10219,"refuses":10220,"plymouth":10221,"##hen":10222,"lou":10223,"##nen":10224,"tara":10225,"trembling":10226,"antarctic":10227,"ton":10228,"learns":10229,"##tas":10230,"crap":10231,"crucial":10232,"faction":10233,"atop":10234,"##borough":10235,"wrap":10236,"lancaster":10237,"odds":10238,"hopkins":10239,"erik":10240,"lyon":10241,"##eon":10242,"bros":10243,"##ode":10244,"snap":10245,"locality":10246,"tips":10247,"empress":10248,"crowned":10249,"cal":10250,"acclaimed":10251,"chuckled":10252,"##ory":10253,"clara":10254,"sends":10255,"mild":10256,"towel":10257,"##fl":10258,"##day":10259,"##а":10260,"wishing":10261,"assuming":10262,"interviewed":10263,"##bal":10264,"##die":10265,"interactions":10266,"eden":10267,"cups":10268,"helena":10269,"##lf":10270,"indie":10271,"beck":10272,"##fire":10273,"batteries":10274,"filipino":10275,"wizard":10276,"parted":10277,"##lam":10278,"traces":10279,"##born":10280,"rows":10281,"idol":10282,"albany":10283,"delegates":10284,"##ees":10285,"##sar":10286,"discussions":10287,"##ex":10288,"notre":10289,"instructed":10290,"belgrade":10291,"highways":10292,"suggestion":10293,"lauren":10294,"possess":10295,"orientation":10296,"alexandria":10297,"abdul":10298,"beats":10299,"salary":10300,"reunion":10301,"ludwig":10302,"alright":10303,"wagner":10304,"intimate":10305,"pockets":10306,"slovenia":10307,"hugged":10308,"brighton":10309,"merchants":10310,"cruel":10311,"stole":10312,"trek":10313,"slopes":10314,"repairs":10315,"enrollment":10316,"politically":10317,"underlying":10318,"promotional":10319,"counting":10320,"boeing":10321,"##bb":10322,"isabella":10323,"naming":10324,"##и":10325,"keen":10326,"bacteria":10327,"listing":10328,"separately":10329,"belfast":10330,"ussr":10331,"450":10332,"lithuanian":10333,"anybody":10334,"ribs":10335,"sphere":10336,"martinez":10337,"cock":10338,"embarrassed":10339,"proposals":10340,"fragments":10341,"nationals":10342,"##fs":10343,"##wski":10344,"premises":10345,"fin":10346,"1500":10347,"alpine":10348,"matched":10349,"freely":10350,"bounded":10351,"jace":10352,"sleeve":10353,"##af":10354,"gaming":10355,"pier":10356,"populated":10357,"evident":10358,"##like":10359,"frances":10360,"flooded":10361,"##dle":10362,"frightened":10363,"pour":10364,"trainer":10365,"framed":10366,"visitor":10367,"challenging":10368,"pig":10369,"wickets":10370,"##fold":10371,"infected":10372,"email":10373,"##pes":10374,"arose":10375,"##aw":10376,"reward":10377,"ecuador":10378,"oblast":10379,"vale":10380,"ch":10381,"shuttle":10382,"##usa":10383,"bach":10384,"rankings":10385,"forbidden":10386,"cornwall":10387,"accordance":10388,"salem":10389,"consumers":10390,"bruno":10391,"fantastic":10392,"toes":10393,"machinery":10394,"resolved":10395,"julius":10396,"remembering":10397,"propaganda":10398,"iceland":10399,"bombardment":10400,"tide":10401,"contacts":10402,"wives":10403,"##rah":10404,"concerto":10405,"macdonald":10406,"albania":10407,"implement":10408,"daisy":10409,"tapped":10410,"sudan":10411,"helmet":10412,"angela":10413,"mistress":10414,"##lic":10415,"crop":10416,"sunk":10417,"finest":10418,"##craft":10419,"hostile":10420,"##ute":10421,"##tsu":10422,"boxer":10423,"fr":10424,"paths":10425,"adjusted":10426,"habit":10427,"ballot":10428,"supervision":10429,"soprano":10430,"##zen":10431,"bullets":10432,"wicked":10433,"sunset":10434,"regiments":10435,"disappear":10436,"lamp":10437,"performs":10438,"app":10439,"##gia":10440,"##oa":10441,"rabbit":10442,"digging":10443,"incidents":10444,"entries":10445,"##cion":10446,"dishes":10447,"##oi":10448,"introducing":10449,"##ati":10450,"##fied":10451,"freshman":10452,"slot":10453,"jill":10454,"tackles":10455,"baroque":10456,"backs":10457,"##iest":10458,"lone":10459,"sponsor":10460,"destiny":10461,"altogether":10462,"convert":10463,"##aro":10464,"consensus":10465,"shapes":10466,"demonstration":10467,"basically":10468,"feminist":10469,"auction":10470,"artifacts":10471,"##bing":10472,"strongest":10473,"twitter":10474,"halifax":10475,"2019":10476,"allmusic":10477,"mighty":10478,"smallest":10479,"precise":10480,"alexandra":10481,"viola":10482,"##los":10483,"##ille":10484,"manuscripts":10485,"##illo":10486,"dancers":10487,"ari":10488,"managers":10489,"monuments":10490,"blades":10491,"barracks":10492,"springfield":10493,"maiden":10494,"consolidated":10495,"electron":10496,"##end":10497,"berry":10498,"airing":10499,"wheat":10500,"nobel":10501,"inclusion":10502,"blair":10503,"payments":10504,"geography":10505,"bee":10506,"cc":10507,"eleanor":10508,"react":10509,"##hurst":10510,"afc":10511,"manitoba":10512,"##yu":10513,"su":10514,"lineup":10515,"fitness":10516,"recreational":10517,"investments":10518,"airborne":10519,"disappointment":10520,"##dis":10521,"edmonton":10522,"viewing":10523,"##row":10524,"renovation":10525,"##cast":10526,"infant":10527,"bankruptcy":10528,"roses":10529,"aftermath":10530,"pavilion":10531,"##yer":10532,"carpenter":10533,"withdrawal":10534,"ladder":10535,"##hy":10536,"discussing":10537,"popped":10538,"reliable":10539,"agreements":10540,"rochester":10541,"##abad":10542,"curves":10543,"bombers":10544,"220":10545,"rao":10546,"reverend":10547,"decreased":10548,"choosing":10549,"107":10550,"stiff":10551,"consulting":10552,"naples":10553,"crawford":10554,"tracy":10555,"ka":10556,"ribbon":10557,"cops":10558,"##lee":10559,"crushed":10560,"deciding":10561,"unified":10562,"teenager":10563,"accepting":10564,"flagship":10565,"explorer":10566,"poles":10567,"sanchez":10568,"inspection":10569,"revived":10570,"skilled":10571,"induced":10572,"exchanged":10573,"flee":10574,"locals":10575,"tragedy":10576,"swallow":10577,"loading":10578,"hanna":10579,"demonstrate":10580,"##ela":10581,"salvador":10582,"flown":10583,"contestants":10584,"civilization":10585,"##ines":10586,"wanna":10587,"rhodes":10588,"fletcher":10589,"hector":10590,"knocking":10591,"considers":10592,"##ough":10593,"nash":10594,"mechanisms":10595,"sensed":10596,"mentally":10597,"walt":10598,"unclear":10599,"##eus":10600,"renovated":10601,"madame":10602,"##cks":10603,"crews":10604,"governmental":10605,"##hin":10606,"undertaken":10607,"monkey":10608,"##ben":10609,"##ato":10610,"fatal":10611,"armored":10612,"copa":10613,"caves":10614,"governance":10615,"grasp":10616,"perception":10617,"certification":10618,"froze":10619,"damp":10620,"tugged":10621,"wyoming":10622,"##rg":10623,"##ero":10624,"newman":10625,"##lor":10626,"nerves":10627,"curiosity":10628,"graph":10629,"115":10630,"##ami":10631,"withdraw":10632,"tunnels":10633,"dull":10634,"meredith":10635,"moss":10636,"exhibits":10637,"neighbors":10638,"communicate":10639,"accuracy":10640,"explored":10641,"raiders":10642,"republicans":10643,"secular":10644,"kat":10645,"superman":10646,"penny":10647,"criticised":10648,"##tch":10649,"freed":10650,"update":10651,"conviction":10652,"wade":10653,"ham":10654,"likewise":10655,"delegation":10656,"gotta":10657,"doll":10658,"promises":10659,"technological":10660,"myth":10661,"nationality":10662,"resolve":10663,"convent":10664,"##mark":10665,"sharon":10666,"dig":10667,"sip":10668,"coordinator":10669,"entrepreneur":10670,"fold":10671,"##dine":10672,"capability":10673,"councillor":10674,"synonym":10675,"blown":10676,"swan":10677,"cursed":10678,"1815":10679,"jonas":10680,"haired":10681,"sofa":10682,"canvas":10683,"keeper":10684,"rivalry":10685,"##hart":10686,"rapper":10687,"speedway":10688,"swords":10689,"postal":10690,"maxwell":10691,"estonia":10692,"potter":10693,"recurring":10694,"##nn":10695,"##ave":10696,"errors":10697,"##oni":10698,"cognitive":10699,"1834":10700,"##²":10701,"claws":10702,"nadu":10703,"roberto":10704,"bce":10705,"wrestler":10706,"ellie":10707,"##ations":10708,"infinite":10709,"ink":10710,"##tia":10711,"presumably":10712,"finite":10713,"staircase":10714,"108":10715,"noel":10716,"patricia":10717,"nacional":10718,"##cation":10719,"chill":10720,"eternal":10721,"tu":10722,"preventing":10723,"prussia":10724,"fossil":10725,"limbs":10726,"##logist":10727,"ernst":10728,"frog":10729,"perez":10730,"rene":10731,"##ace":10732,"pizza":10733,"prussian":10734,"##ios":10735,"##vy":10736,"molecules":10737,"regulatory":10738,"answering":10739,"opinions":10740,"sworn":10741,"lengths":10742,"supposedly":10743,"hypothesis":10744,"upward":10745,"habitats":10746,"seating":10747,"ancestors":10748,"drank":10749,"yield":10750,"hd":10751,"synthesis":10752,"researcher":10753,"modest":10754,"##var":10755,"mothers":10756,"peered":10757,"voluntary":10758,"homeland":10759,"##the":10760,"acclaim":10761,"##igan":10762,"static":10763,"valve":10764,"luxembourg":10765,"alto":10766,"carroll":10767,"fe":10768,"receptor":10769,"norton":10770,"ambulance":10771,"##tian":10772,"johnston":10773,"catholics":10774,"depicting":10775,"jointly":10776,"elephant":10777,"gloria":10778,"mentor":10779,"badge":10780,"ahmad":10781,"distinguish":10782,"remarked":10783,"councils":10784,"precisely":10785,"allison":10786,"advancing":10787,"detection":10788,"crowded":10789,"##10":10790,"cooperative":10791,"ankle":10792,"mercedes":10793,"dagger":10794,"surrendered":10795,"pollution":10796,"commit":10797,"subway":10798,"jeffrey":10799,"lesson":10800,"sculptures":10801,"provider":10802,"##fication":10803,"membrane":10804,"timothy":10805,"rectangular":10806,"fiscal":10807,"heating":10808,"teammate":10809,"basket":10810,"particle":10811,"anonymous":10812,"deployment":10813,"##ple":10814,"missiles":10815,"courthouse":10816,"proportion":10817,"shoe":10818,"sec":10819,"##ller":10820,"complaints":10821,"forbes":10822,"blacks":10823,"abandon":10824,"remind":10825,"sizes":10826,"overwhelming":10827,"autobiography":10828,"natalie":10829,"##awa":10830,"risks":10831,"contestant":10832,"countryside":10833,"babies":10834,"scorer":10835,"invaded":10836,"enclosed":10837,"proceed":10838,"hurling":10839,"disorders":10840,"##cu":10841,"reflecting":10842,"continuously":10843,"cruiser":10844,"graduates":10845,"freeway":10846,"investigated":10847,"ore":10848,"deserved":10849,"maid":10850,"blocking":10851,"phillip":10852,"jorge":10853,"shakes":10854,"dove":10855,"mann":10856,"variables":10857,"lacked":10858,"burden":10859,"accompanying":10860,"que":10861,"consistently":10862,"organizing":10863,"provisional":10864,"complained":10865,"endless":10866,"##rm":10867,"tubes":10868,"juice":10869,"georges":10870,"krishna":10871,"mick":10872,"labels":10873,"thriller":10874,"##uch":10875,"laps":10876,"arcade":10877,"sage":10878,"snail":10879,"##table":10880,"shannon":10881,"fi":10882,"laurence":10883,"seoul":10884,"vacation":10885,"presenting":10886,"hire":10887,"churchill":10888,"surprisingly":10889,"prohibited":10890,"savannah":10891,"technically":10892,"##oli":10893,"170":10894,"##lessly":10895,"testimony":10896,"suited":10897,"speeds":10898,"toys":10899,"romans":10900,"mlb":10901,"flowering":10902,"measurement":10903,"talented":10904,"kay":10905,"settings":10906,"charleston":10907,"expectations":10908,"shattered":10909,"achieving":10910,"triumph":10911,"ceremonies":10912,"portsmouth":10913,"lanes":10914,"mandatory":10915,"loser":10916,"stretching":10917,"cologne":10918,"realizes":10919,"seventy":10920,"cornell":10921,"careers":10922,"webb":10923,"##ulating":10924,"americas":10925,"budapest":10926,"ava":10927,"suspicion":10928,"##ison":10929,"yo":10930,"conrad":10931,"##hai":10932,"sterling":10933,"jessie":10934,"rector":10935,"##az":10936,"1831":10937,"transform":10938,"organize":10939,"loans":10940,"christine":10941,"volcanic":10942,"warrant":10943,"slender":10944,"summers":10945,"subfamily":10946,"newer":10947,"danced":10948,"dynamics":10949,"rhine":10950,"proceeds":10951,"heinrich":10952,"gastropod":10953,"commands":10954,"sings":10955,"facilitate":10956,"easter":10957,"ra":10958,"positioned":10959,"responses":10960,"expense":10961,"fruits":10962,"yanked":10963,"imported":10964,"25th":10965,"velvet":10966,"vic":10967,"primitive":10968,"tribune":10969,"baldwin":10970,"neighbourhood":10971,"donna":10972,"rip":10973,"hay":10974,"pr":10975,"##uro":10976,"1814":10977,"espn":10978,"welcomed":10979,"##aria":10980,"qualifier":10981,"glare":10982,"highland":10983,"timing":10984,"##cted":10985,"shells":10986,"eased":10987,"geometry":10988,"louder":10989,"exciting":10990,"slovakia":10991,"##sion":10992,"##iz":10993,"##lot":10994,"savings":10995,"prairie":10996,"##ques":10997,"marching":10998,"rafael":10999,"tonnes":11000,"##lled":11001,"curtain":11002,"preceding":11003,"shy":11004,"heal":11005,"greene":11006,"worthy":11007,"##pot":11008,"detachment":11009,"bury":11010,"sherman":11011,"##eck":11012,"reinforced":11013,"seeks":11014,"bottles":11015,"contracted":11016,"duchess":11017,"outfit":11018,"walsh":11019,"##sc":11020,"mickey":11021,"##ase":11022,"geoffrey":11023,"archer":11024,"squeeze":11025,"dawson":11026,"eliminate":11027,"invention":11028,"##enberg":11029,"neal":11030,"##eth":11031,"stance":11032,"dealer":11033,"coral":11034,"maple":11035,"retire":11036,"polo":11037,"simplified":11038,"##ht":11039,"1833":11040,"hid":11041,"watts":11042,"backwards":11043,"jules":11044,"##oke":11045,"genesis":11046,"mt":11047,"frames":11048,"rebounds":11049,"burma":11050,"woodland":11051,"moist":11052,"santos":11053,"whispers":11054,"drained":11055,"subspecies":11056,"##aa":11057,"streaming":11058,"ulster":11059,"burnt":11060,"correspondence":11061,"maternal":11062,"gerard":11063,"denis":11064,"stealing":11065,"##load":11066,"genius":11067,"duchy":11068,"##oria":11069,"inaugurated":11070,"momentum":11071,"suits":11072,"placement":11073,"sovereign":11074,"clause":11075,"thames":11076,"##hara":11077,"confederation":11078,"reservation":11079,"sketch":11080,"yankees":11081,"lets":11082,"rotten":11083,"charm":11084,"hal":11085,"verses":11086,"ultra":11087,"commercially":11088,"dot":11089,"salon":11090,"citation":11091,"adopt":11092,"winnipeg":11093,"mist":11094,"allocated":11095,"cairo":11096,"##boy":11097,"jenkins":11098,"interference":11099,"objectives":11100,"##wind":11101,"1820":11102,"portfolio":11103,"armoured":11104,"sectors":11105,"##eh":11106,"initiatives":11107,"##world":11108,"integrity":11109,"exercises":11110,"robe":11111,"tap":11112,"ab":11113,"gazed":11114,"##tones":11115,"distracted":11116,"rulers":11117,"111":11118,"favorable":11119,"jerome":11120,"tended":11121,"cart":11122,"factories":11123,"##eri":11124,"diplomat":11125,"valued":11126,"gravel":11127,"charitable":11128,"##try":11129,"calvin":11130,"exploring":11131,"chang":11132,"shepherd":11133,"terrace":11134,"pdf":11135,"pupil":11136,"##ural":11137,"reflects":11138,"ups":11139,"##rch":11140,"governors":11141,"shelf":11142,"depths":11143,"##nberg":11144,"trailed":11145,"crest":11146,"tackle":11147,"##nian":11148,"##ats":11149,"hatred":11150,"##kai":11151,"clare":11152,"makers":11153,"ethiopia":11154,"longtime":11155,"detected":11156,"embedded":11157,"lacking":11158,"slapped":11159,"rely":11160,"thomson":11161,"anticipation":11162,"iso":11163,"morton":11164,"successive":11165,"agnes":11166,"screenwriter":11167,"straightened":11168,"philippe":11169,"playwright":11170,"haunted":11171,"licence":11172,"iris":11173,"intentions":11174,"sutton":11175,"112":11176,"logical":11177,"correctly":11178,"##weight":11179,"branded":11180,"licked":11181,"tipped":11182,"silva":11183,"ricky":11184,"narrator":11185,"requests":11186,"##ents":11187,"greeted":11188,"supernatural":11189,"cow":11190,"##wald":11191,"lung":11192,"refusing":11193,"employer":11194,"strait":11195,"gaelic":11196,"liner":11197,"##piece":11198,"zoe":11199,"sabha":11200,"##mba":11201,"driveway":11202,"harvest":11203,"prints":11204,"bates":11205,"reluctantly":11206,"threshold":11207,"algebra":11208,"ira":11209,"wherever":11210,"coupled":11211,"240":11212,"assumption":11213,"picks":11214,"##air":11215,"designers":11216,"raids":11217,"gentlemen":11218,"##ean":11219,"roller":11220,"blowing":11221,"leipzig":11222,"locks":11223,"screw":11224,"dressing":11225,"strand":11226,"##lings":11227,"scar":11228,"dwarf":11229,"depicts":11230,"##nu":11231,"nods":11232,"##mine":11233,"differ":11234,"boris":11235,"##eur":11236,"yuan":11237,"flip":11238,"##gie":11239,"mob":11240,"invested":11241,"questioning":11242,"applying":11243,"##ture":11244,"shout":11245,"##sel":11246,"gameplay":11247,"blamed":11248,"illustrations":11249,"bothered":11250,"weakness":11251,"rehabilitation":11252,"##of":11253,"##zes":11254,"envelope":11255,"rumors":11256,"miners":11257,"leicester":11258,"subtle":11259,"kerry":11260,"##ico":11261,"ferguson":11262,"##fu":11263,"premiership":11264,"ne":11265,"##cat":11266,"bengali":11267,"prof":11268,"catches":11269,"remnants":11270,"dana":11271,"##rily":11272,"shouting":11273,"presidents":11274,"baltic":11275,"ought":11276,"ghosts":11277,"dances":11278,"sailors":11279,"shirley":11280,"fancy":11281,"dominic":11282,"##bie":11283,"madonna":11284,"##rick":11285,"bark":11286,"buttons":11287,"gymnasium":11288,"ashes":11289,"liver":11290,"toby":11291,"oath":11292,"providence":11293,"doyle":11294,"evangelical":11295,"nixon":11296,"cement":11297,"carnegie":11298,"embarked":11299,"hatch":11300,"surroundings":11301,"guarantee":11302,"needing":11303,"pirate":11304,"essence":11305,"##bee":11306,"filter":11307,"crane":11308,"hammond":11309,"projected":11310,"immune":11311,"percy":11312,"twelfth":11313,"##ult":11314,"regent":11315,"doctoral":11316,"damon":11317,"mikhail":11318,"##ichi":11319,"lu":11320,"critically":11321,"elect":11322,"realised":11323,"abortion":11324,"acute":11325,"screening":11326,"mythology":11327,"steadily":11328,"##fc":11329,"frown":11330,"nottingham":11331,"kirk":11332,"wa":11333,"minneapolis":11334,"##rra":11335,"module":11336,"algeria":11337,"mc":11338,"nautical":11339,"encounters":11340,"surprising":11341,"statues":11342,"availability":11343,"shirts":11344,"pie":11345,"alma":11346,"brows":11347,"munster":11348,"mack":11349,"soup":11350,"crater":11351,"tornado":11352,"sanskrit":11353,"cedar":11354,"explosive":11355,"bordered":11356,"dixon":11357,"planets":11358,"stamp":11359,"exam":11360,"happily":11361,"##bble":11362,"carriers":11363,"kidnapped":11364,"##vis":11365,"accommodation":11366,"emigrated":11367,"##met":11368,"knockout":11369,"correspondent":11370,"violation":11371,"profits":11372,"peaks":11373,"lang":11374,"specimen":11375,"agenda":11376,"ancestry":11377,"pottery":11378,"spelling":11379,"equations":11380,"obtaining":11381,"ki":11382,"linking":11383,"1825":11384,"debris":11385,"asylum":11386,"##20":11387,"buddhism":11388,"teddy":11389,"##ants":11390,"gazette":11391,"##nger":11392,"##sse":11393,"dental":11394,"eligibility":11395,"utc":11396,"fathers":11397,"averaged":11398,"zimbabwe":11399,"francesco":11400,"coloured":11401,"hissed":11402,"translator":11403,"lynch":11404,"mandate":11405,"humanities":11406,"mackenzie":11407,"uniforms":11408,"lin":11409,"##iana":11410,"##gio":11411,"asset":11412,"mhz":11413,"fitting":11414,"samantha":11415,"genera":11416,"wei":11417,"rim":11418,"beloved":11419,"shark":11420,"riot":11421,"entities":11422,"expressions":11423,"indo":11424,"carmen":11425,"slipping":11426,"owing":11427,"abbot":11428,"neighbor":11429,"sidney":11430,"##av":11431,"rats":11432,"recommendations":11433,"encouraging":11434,"squadrons":11435,"anticipated":11436,"commanders":11437,"conquered":11438,"##oto":11439,"donations":11440,"diagnosed":11441,"##mond":11442,"divide":11443,"##iva":11444,"guessed":11445,"decoration":11446,"vernon":11447,"auditorium":11448,"revelation":11449,"conversations":11450,"##kers":11451,"##power":11452,"herzegovina":11453,"dash":11454,"alike":11455,"protested":11456,"lateral":11457,"herman":11458,"accredited":11459,"mg":11460,"##gent":11461,"freeman":11462,"mel":11463,"fiji":11464,"crow":11465,"crimson":11466,"##rine":11467,"livestock":11468,"##pped":11469,"humanitarian":11470,"bored":11471,"oz":11472,"whip":11473,"##lene":11474,"##ali":11475,"legitimate":11476,"alter":11477,"grinning":11478,"spelled":11479,"anxious":11480,"oriental":11481,"wesley":11482,"##nin":11483,"##hole":11484,"carnival":11485,"controller":11486,"detect":11487,"##ssa":11488,"bowed":11489,"educator":11490,"kosovo":11491,"macedonia":11492,"##sin":11493,"occupy":11494,"mastering":11495,"stephanie":11496,"janeiro":11497,"para":11498,"unaware":11499,"nurses":11500,"noon":11501,"135":11502,"cam":11503,"hopefully":11504,"ranger":11505,"combine":11506,"sociology":11507,"polar":11508,"rica":11509,"##eer":11510,"neill":11511,"##sman":11512,"holocaust":11513,"##ip":11514,"doubled":11515,"lust":11516,"1828":11517,"109":11518,"decent":11519,"cooling":11520,"unveiled":11521,"##card":11522,"1829":11523,"nsw":11524,"homer":11525,"chapman":11526,"meyer":11527,"##gin":11528,"dive":11529,"mae":11530,"reagan":11531,"expertise":11532,"##gled":11533,"darwin":11534,"brooke":11535,"sided":11536,"prosecution":11537,"investigating":11538,"comprised":11539,"petroleum":11540,"genres":11541,"reluctant":11542,"differently":11543,"trilogy":11544,"johns":11545,"vegetables":11546,"corpse":11547,"highlighted":11548,"lounge":11549,"pension":11550,"unsuccessfully":11551,"elegant":11552,"aided":11553,"ivory":11554,"beatles":11555,"amelia":11556,"cain":11557,"dubai":11558,"sunny":11559,"immigrant":11560,"babe":11561,"click":11562,"##nder":11563,"underwater":11564,"pepper":11565,"combining":11566,"mumbled":11567,"atlas":11568,"horns":11569,"accessed":11570,"ballad":11571,"physicians":11572,"homeless":11573,"gestured":11574,"rpm":11575,"freak":11576,"louisville":11577,"corporations":11578,"patriots":11579,"prizes":11580,"rational":11581,"warn":11582,"modes":11583,"decorative":11584,"overnight":11585,"din":11586,"troubled":11587,"phantom":11588,"##ort":11589,"monarch":11590,"sheer":11591,"##dorf":11592,"generals":11593,"guidelines":11594,"organs":11595,"addresses":11596,"##zon":11597,"enhance":11598,"curling":11599,"parishes":11600,"cord":11601,"##kie":11602,"linux":11603,"caesar":11604,"deutsche":11605,"bavaria":11606,"##bia":11607,"coleman":11608,"cyclone":11609,"##eria":11610,"bacon":11611,"petty":11612,"##yama":11613,"##old":11614,"hampton":11615,"diagnosis":11616,"1824":11617,"throws":11618,"complexity":11619,"rita":11620,"disputed":11621,"##₃":11622,"pablo":11623,"##sch":11624,"marketed":11625,"trafficking":11626,"##ulus":11627,"examine":11628,"plague":11629,"formats":11630,"##oh":11631,"vault":11632,"faithful":11633,"##bourne":11634,"webster":11635,"##ox":11636,"highlights":11637,"##ient":11638,"##ann":11639,"phones":11640,"vacuum":11641,"sandwich":11642,"modeling":11643,"##gated":11644,"bolivia":11645,"clergy":11646,"qualities":11647,"isabel":11648,"##nas":11649,"##ars":11650,"wears":11651,"screams":11652,"reunited":11653,"annoyed":11654,"bra":11655,"##ancy":11656,"##rate":11657,"differential":11658,"transmitter":11659,"tattoo":11660,"container":11661,"poker":11662,"##och":11663,"excessive":11664,"resides":11665,"cowboys":11666,"##tum":11667,"augustus":11668,"trash":11669,"providers":11670,"statute":11671,"retreated":11672,"balcony":11673,"reversed":11674,"void":11675,"storey":11676,"preceded":11677,"masses":11678,"leap":11679,"laughs":11680,"neighborhoods":11681,"wards":11682,"schemes":11683,"falcon":11684,"santo":11685,"battlefield":11686,"pad":11687,"ronnie":11688,"thread":11689,"lesbian":11690,"venus":11691,"##dian":11692,"beg":11693,"sandstone":11694,"daylight":11695,"punched":11696,"gwen":11697,"analog":11698,"stroked":11699,"wwe":11700,"acceptable":11701,"measurements":11702,"dec":11703,"toxic":11704,"##kel":11705,"adequate":11706,"surgical":11707,"economist":11708,"parameters":11709,"varsity":11710,"##sberg":11711,"quantity":11712,"ella":11713,"##chy":11714,"##rton":11715,"countess":11716,"generating":11717,"precision":11718,"diamonds":11719,"expressway":11720,"ga":11721,"##ı":11722,"1821":11723,"uruguay":11724,"talents":11725,"galleries":11726,"expenses":11727,"scanned":11728,"colleague":11729,"outlets":11730,"ryder":11731,"lucien":11732,"##ila":11733,"paramount":11734,"##bon":11735,"syracuse":11736,"dim":11737,"fangs":11738,"gown":11739,"sweep":11740,"##sie":11741,"toyota":11742,"missionaries":11743,"websites":11744,"##nsis":11745,"sentences":11746,"adviser":11747,"val":11748,"trademark":11749,"spells":11750,"##plane":11751,"patience":11752,"starter":11753,"slim":11754,"##borg":11755,"toe":11756,"incredibly":11757,"shoots":11758,"elliot":11759,"nobility":11760,"##wyn":11761,"cowboy":11762,"endorsed":11763,"gardner":11764,"tendency":11765,"persuaded":11766,"organisms":11767,"emissions":11768,"kazakhstan":11769,"amused":11770,"boring":11771,"chips":11772,"themed":11773,"##hand":11774,"llc":11775,"constantinople":11776,"chasing":11777,"systematic":11778,"guatemala":11779,"borrowed":11780,"erin":11781,"carey":11782,"##hard":11783,"highlands":11784,"struggles":11785,"1810":11786,"##ifying":11787,"##ced":11788,"wong":11789,"exceptions":11790,"develops":11791,"enlarged":11792,"kindergarten":11793,"castro":11794,"##ern":11795,"##rina":11796,"leigh":11797,"zombie":11798,"juvenile":11799,"##most":11800,"consul":11801,"##nar":11802,"sailor":11803,"hyde":11804,"clarence":11805,"intensive":11806,"pinned":11807,"nasty":11808,"useless":11809,"jung":11810,"clayton":11811,"stuffed":11812,"exceptional":11813,"ix":11814,"apostolic":11815,"230":11816,"transactions":11817,"##dge":11818,"exempt":11819,"swinging":11820,"cove":11821,"religions":11822,"##ash":11823,"shields":11824,"dairy":11825,"bypass":11826,"190":11827,"pursuing":11828,"bug":11829,"joyce":11830,"bombay":11831,"chassis":11832,"southampton":11833,"chat":11834,"interact":11835,"redesignated":11836,"##pen":11837,"nascar":11838,"pray":11839,"salmon":11840,"rigid":11841,"regained":11842,"malaysian":11843,"grim":11844,"publicity":11845,"constituted":11846,"capturing":11847,"toilet":11848,"delegate":11849,"purely":11850,"tray":11851,"drift":11852,"loosely":11853,"striker":11854,"weakened":11855,"trinidad":11856,"mitch":11857,"itv":11858,"defines":11859,"transmitted":11860,"ming":11861,"scarlet":11862,"nodding":11863,"fitzgerald":11864,"fu":11865,"narrowly":11866,"sp":11867,"tooth":11868,"standings":11869,"virtue":11870,"##₁":11871,"##wara":11872,"##cting":11873,"chateau":11874,"gloves":11875,"lid":11876,"##nel":11877,"hurting":11878,"conservatory":11879,"##pel":11880,"sinclair":11881,"reopened":11882,"sympathy":11883,"nigerian":11884,"strode":11885,"advocated":11886,"optional":11887,"chronic":11888,"discharge":11889,"##rc":11890,"suck":11891,"compatible":11892,"laurel":11893,"stella":11894,"shi":11895,"fails":11896,"wage":11897,"dodge":11898,"128":11899,"informal":11900,"sorts":11901,"levi":11902,"buddha":11903,"villagers":11904,"##aka":11905,"chronicles":11906,"heavier":11907,"summoned":11908,"gateway":11909,"3000":11910,"eleventh":11911,"jewelry":11912,"translations":11913,"accordingly":11914,"seas":11915,"##ency":11916,"fiber":11917,"pyramid":11918,"cubic":11919,"dragging":11920,"##ista":11921,"caring":11922,"##ops":11923,"android":11924,"contacted":11925,"lunar":11926,"##dt":11927,"kai":11928,"lisbon":11929,"patted":11930,"1826":11931,"sacramento":11932,"theft":11933,"madagascar":11934,"subtropical":11935,"disputes":11936,"ta":11937,"holidays":11938,"piper":11939,"willow":11940,"mare":11941,"cane":11942,"itunes":11943,"newfoundland":11944,"benny":11945,"companions":11946,"dong":11947,"raj":11948,"observe":11949,"roar":11950,"charming":11951,"plaque":11952,"tibetan":11953,"fossils":11954,"enacted":11955,"manning":11956,"bubble":11957,"tina":11958,"tanzania":11959,"##eda":11960,"##hir":11961,"funk":11962,"swamp":11963,"deputies":11964,"cloak":11965,"ufc":11966,"scenario":11967,"par":11968,"scratch":11969,"metals":11970,"anthem":11971,"guru":11972,"engaging":11973,"specially":11974,"##boat":11975,"dialects":11976,"nineteen":11977,"cecil":11978,"duet":11979,"disability":11980,"messenger":11981,"unofficial":11982,"##lies":11983,"defunct":11984,"eds":11985,"moonlight":11986,"drainage":11987,"surname":11988,"puzzle":11989,"honda":11990,"switching":11991,"conservatives":11992,"mammals":11993,"knox":11994,"broadcaster":11995,"sidewalk":11996,"cope":11997,"##ried":11998,"benson":11999,"princes":12000,"peterson":12001,"##sal":12002,"bedford":12003,"sharks":12004,"eli":12005,"wreck":12006,"alberto":12007,"gasp":12008,"archaeology":12009,"lgbt":12010,"teaches":12011,"securities":12012,"madness":12013,"compromise":12014,"waving":12015,"coordination":12016,"davidson":12017,"visions":12018,"leased":12019,"possibilities":12020,"eighty":12021,"jun":12022,"fernandez":12023,"enthusiasm":12024,"assassin":12025,"sponsorship":12026,"reviewer":12027,"kingdoms":12028,"estonian":12029,"laboratories":12030,"##fy":12031,"##nal":12032,"applies":12033,"verb":12034,"celebrations":12035,"##zzo":12036,"rowing":12037,"lightweight":12038,"sadness":12039,"submit":12040,"mvp":12041,"balanced":12042,"dude":12043,"##vas":12044,"explicitly":12045,"metric":12046,"magnificent":12047,"mound":12048,"brett":12049,"mohammad":12050,"mistakes":12051,"irregular":12052,"##hing":12053,"##ass":12054,"sanders":12055,"betrayed":12056,"shipped":12057,"surge":12058,"##enburg":12059,"reporters":12060,"termed":12061,"georg":12062,"pity":12063,"verbal":12064,"bulls":12065,"abbreviated":12066,"enabling":12067,"appealed":12068,"##are":12069,"##atic":12070,"sicily":12071,"sting":12072,"heel":12073,"sweetheart":12074,"bart":12075,"spacecraft":12076,"brutal":12077,"monarchy":12078,"##tter":12079,"aberdeen":12080,"cameo":12081,"diane":12082,"##ub":12083,"survivor":12084,"clyde":12085,"##aries":12086,"complaint":12087,"##makers":12088,"clarinet":12089,"delicious":12090,"chilean":12091,"karnataka":12092,"coordinates":12093,"1818":12094,"panties":12095,"##rst":12096,"pretending":12097,"ar":12098,"dramatically":12099,"kiev":12100,"bella":12101,"tends":12102,"distances":12103,"113":12104,"catalog":12105,"launching":12106,"instances":12107,"telecommunications":12108,"portable":12109,"lindsay":12110,"vatican":12111,"##eim":12112,"angles":12113,"aliens":12114,"marker":12115,"stint":12116,"screens":12117,"bolton":12118,"##rne":12119,"judy":12120,"wool":12121,"benedict":12122,"plasma":12123,"europa":12124,"spark":12125,"imaging":12126,"filmmaker":12127,"swiftly":12128,"##een":12129,"contributor":12130,"##nor":12131,"opted":12132,"stamps":12133,"apologize":12134,"financing":12135,"butter":12136,"gideon":12137,"sophisticated":12138,"alignment":12139,"avery":12140,"chemicals":12141,"yearly":12142,"speculation":12143,"prominence":12144,"professionally":12145,"##ils":12146,"immortal":12147,"institutional":12148,"inception":12149,"wrists":12150,"identifying":12151,"tribunal":12152,"derives":12153,"gains":12154,"##wo":12155,"papal":12156,"preference":12157,"linguistic":12158,"vince":12159,"operative":12160,"brewery":12161,"##ont":12162,"unemployment":12163,"boyd":12164,"##ured":12165,"##outs":12166,"albeit":12167,"prophet":12168,"1813":12169,"bi":12170,"##rr":12171,"##face":12172,"##rad":12173,"quarterly":12174,"asteroid":12175,"cleaned":12176,"radius":12177,"temper":12178,"##llen":12179,"telugu":12180,"jerk":12181,"viscount":12182,"menu":12183,"##ote":12184,"glimpse":12185,"##aya":12186,"yacht":12187,"hawaiian":12188,"baden":12189,"##rl":12190,"laptop":12191,"readily":12192,"##gu":12193,"monetary":12194,"offshore":12195,"scots":12196,"watches":12197,"##yang":12198,"##arian":12199,"upgrade":12200,"needle":12201,"xbox":12202,"lea":12203,"encyclopedia":12204,"flank":12205,"fingertips":12206,"##pus":12207,"delight":12208,"teachings":12209,"confirm":12210,"roth":12211,"beaches":12212,"midway":12213,"winters":12214,"##iah":12215,"teasing":12216,"daytime":12217,"beverly":12218,"gambling":12219,"bonnie":12220,"##backs":12221,"regulated":12222,"clement":12223,"hermann":12224,"tricks":12225,"knot":12226,"##shing":12227,"##uring":12228,"##vre":12229,"detached":12230,"ecological":12231,"owed":12232,"specialty":12233,"byron":12234,"inventor":12235,"bats":12236,"stays":12237,"screened":12238,"unesco":12239,"midland":12240,"trim":12241,"affection":12242,"##ander":12243,"##rry":12244,"jess":12245,"thoroughly":12246,"feedback":12247,"##uma":12248,"chennai":12249,"strained":12250,"heartbeat":12251,"wrapping":12252,"overtime":12253,"pleaded":12254,"##sworth":12255,"mon":12256,"leisure":12257,"oclc":12258,"##tate":12259,"##ele":12260,"feathers":12261,"angelo":12262,"thirds":12263,"nuts":12264,"surveys":12265,"clever":12266,"gill":12267,"commentator":12268,"##dos":12269,"darren":12270,"rides":12271,"gibraltar":12272,"##nc":12273,"##mu":12274,"dissolution":12275,"dedication":12276,"shin":12277,"meals":12278,"saddle":12279,"elvis":12280,"reds":12281,"chaired":12282,"taller":12283,"appreciation":12284,"functioning":12285,"niece":12286,"favored":12287,"advocacy":12288,"robbie":12289,"criminals":12290,"suffolk":12291,"yugoslav":12292,"passport":12293,"constable":12294,"congressman":12295,"hastings":12296,"vera":12297,"##rov":12298,"consecrated":12299,"sparks":12300,"ecclesiastical":12301,"confined":12302,"##ovich":12303,"muller":12304,"floyd":12305,"nora":12306,"1822":12307,"paved":12308,"1827":12309,"cumberland":12310,"ned":12311,"saga":12312,"spiral":12313,"##flow":12314,"appreciated":12315,"yi":12316,"collaborative":12317,"treating":12318,"similarities":12319,"feminine":12320,"finishes":12321,"##ib":12322,"jade":12323,"import":12324,"##nse":12325,"##hot":12326,"champagne":12327,"mice":12328,"securing":12329,"celebrities":12330,"helsinki":12331,"attributes":12332,"##gos":12333,"cousins":12334,"phases":12335,"ache":12336,"lucia":12337,"gandhi":12338,"submission":12339,"vicar":12340,"spear":12341,"shine":12342,"tasmania":12343,"biting":12344,"detention":12345,"constitute":12346,"tighter":12347,"seasonal":12348,"##gus":12349,"terrestrial":12350,"matthews":12351,"##oka":12352,"effectiveness":12353,"parody":12354,"philharmonic":12355,"##onic":12356,"1816":12357,"strangers":12358,"encoded":12359,"consortium":12360,"guaranteed":12361,"regards":12362,"shifts":12363,"tortured":12364,"collision":12365,"supervisor":12366,"inform":12367,"broader":12368,"insight":12369,"theaters":12370,"armour":12371,"emeritus":12372,"blink":12373,"incorporates":12374,"mapping":12375,"##50":12376,"##ein":12377,"handball":12378,"flexible":12379,"##nta":12380,"substantially":12381,"generous":12382,"thief":12383,"##own":12384,"carr":12385,"loses":12386,"1793":12387,"prose":12388,"ucla":12389,"romeo":12390,"generic":12391,"metallic":12392,"realization":12393,"damages":12394,"mk":12395,"commissioners":12396,"zach":12397,"default":12398,"##ther":12399,"helicopters":12400,"lengthy":12401,"stems":12402,"spa":12403,"partnered":12404,"spectators":12405,"rogue":12406,"indication":12407,"penalties":12408,"teresa":12409,"1801":12410,"sen":12411,"##tric":12412,"dalton":12413,"##wich":12414,"irving":12415,"photographic":12416,"##vey":12417,"dell":12418,"deaf":12419,"peters":12420,"excluded":12421,"unsure":12422,"##vable":12423,"patterson":12424,"crawled":12425,"##zio":12426,"resided":12427,"whipped":12428,"latvia":12429,"slower":12430,"ecole":12431,"pipes":12432,"employers":12433,"maharashtra":12434,"comparable":12435,"va":12436,"textile":12437,"pageant":12438,"##gel":12439,"alphabet":12440,"binary":12441,"irrigation":12442,"chartered":12443,"choked":12444,"antoine":12445,"offs":12446,"waking":12447,"supplement":12448,"##wen":12449,"quantities":12450,"demolition":12451,"regain":12452,"locate":12453,"urdu":12454,"folks":12455,"alt":12456,"114":12457,"##mc":12458,"scary":12459,"andreas":12460,"whites":12461,"##ava":12462,"classrooms":12463,"mw":12464,"aesthetic":12465,"publishes":12466,"valleys":12467,"guides":12468,"cubs":12469,"johannes":12470,"bryant":12471,"conventions":12472,"affecting":12473,"##itt":12474,"drain":12475,"awesome":12476,"isolation":12477,"prosecutor":12478,"ambitious":12479,"apology":12480,"captive":12481,"downs":12482,"atmospheric":12483,"lorenzo":12484,"aisle":12485,"beef":12486,"foul":12487,"##onia":12488,"kidding":12489,"composite":12490,"disturbed":12491,"illusion":12492,"natives":12493,"##ffer":12494,"emi":12495,"rockets":12496,"riverside":12497,"wartime":12498,"painters":12499,"adolf":12500,"melted":12501,"##ail":12502,"uncertainty":12503,"simulation":12504,"hawks":12505,"progressed":12506,"meantime":12507,"builder":12508,"spray":12509,"breach":12510,"unhappy":12511,"regina":12512,"russians":12513,"##urg":12514,"determining":12515,"##tation":12516,"tram":12517,"1806":12518,"##quin":12519,"aging":12520,"##12":12521,"1823":12522,"garion":12523,"rented":12524,"mister":12525,"diaz":12526,"terminated":12527,"clip":12528,"1817":12529,"depend":12530,"nervously":12531,"disco":12532,"owe":12533,"defenders":12534,"shiva":12535,"notorious":12536,"disbelief":12537,"shiny":12538,"worcester":12539,"##gation":12540,"##yr":12541,"trailing":12542,"undertook":12543,"islander":12544,"belarus":12545,"limitations":12546,"watershed":12547,"fuller":12548,"overlooking":12549,"utilized":12550,"raphael":12551,"1819":12552,"synthetic":12553,"breakdown":12554,"klein":12555,"##nate":12556,"moaned":12557,"memoir":12558,"lamb":12559,"practicing":12560,"##erly":12561,"cellular":12562,"arrows":12563,"exotic":12564,"##graphy":12565,"witches":12566,"117":12567,"charted":12568,"rey":12569,"hut":12570,"hierarchy":12571,"subdivision":12572,"freshwater":12573,"giuseppe":12574,"aloud":12575,"reyes":12576,"qatar":12577,"marty":12578,"sideways":12579,"utterly":12580,"sexually":12581,"jude":12582,"prayers":12583,"mccarthy":12584,"softball":12585,"blend":12586,"damien":12587,"##gging":12588,"##metric":12589,"wholly":12590,"erupted":12591,"lebanese":12592,"negro":12593,"revenues":12594,"tasted":12595,"comparative":12596,"teamed":12597,"transaction":12598,"labeled":12599,"maori":12600,"sovereignty":12601,"parkway":12602,"trauma":12603,"gran":12604,"malay":12605,"121":12606,"advancement":12607,"descendant":12608,"2020":12609,"buzz":12610,"salvation":12611,"inventory":12612,"symbolic":12613,"##making":12614,"antarctica":12615,"mps":12616,"##gas":12617,"##bro":12618,"mohammed":12619,"myanmar":12620,"holt":12621,"submarines":12622,"tones":12623,"##lman":12624,"locker":12625,"patriarch":12626,"bangkok":12627,"emerson":12628,"remarks":12629,"predators":12630,"kin":12631,"afghan":12632,"confession":12633,"norwich":12634,"rental":12635,"emerge":12636,"advantages":12637,"##zel":12638,"rca":12639,"##hold":12640,"shortened":12641,"storms":12642,"aidan":12643,"##matic":12644,"autonomy":12645,"compliance":12646,"##quet":12647,"dudley":12648,"atp":12649,"##osis":12650,"1803":12651,"motto":12652,"documentation":12653,"summary":12654,"professors":12655,"spectacular":12656,"christina":12657,"archdiocese":12658,"flashing":12659,"innocence":12660,"remake":12661,"##dell":12662,"psychic":12663,"reef":12664,"scare":12665,"employ":12666,"rs":12667,"sticks":12668,"meg":12669,"gus":12670,"leans":12671,"##ude":12672,"accompany":12673,"bergen":12674,"tomas":12675,"##iko":12676,"doom":12677,"wages":12678,"pools":12679,"##nch":12680,"##bes":12681,"breasts":12682,"scholarly":12683,"alison":12684,"outline":12685,"brittany":12686,"breakthrough":12687,"willis":12688,"realistic":12689,"##cut":12690,"##boro":12691,"competitor":12692,"##stan":12693,"pike":12694,"picnic":12695,"icon":12696,"designing":12697,"commercials":12698,"washing":12699,"villain":12700,"skiing":12701,"micro":12702,"costumes":12703,"auburn":12704,"halted":12705,"executives":12706,"##hat":12707,"logistics":12708,"cycles":12709,"vowel":12710,"applicable":12711,"barrett":12712,"exclaimed":12713,"eurovision":12714,"eternity":12715,"ramon":12716,"##umi":12717,"##lls":12718,"modifications":12719,"sweeping":12720,"disgust":12721,"##uck":12722,"torch":12723,"aviv":12724,"ensuring":12725,"rude":12726,"dusty":12727,"sonic":12728,"donovan":12729,"outskirts":12730,"cu":12731,"pathway":12732,"##band":12733,"##gun":12734,"##lines":12735,"disciplines":12736,"acids":12737,"cadet":12738,"paired":12739,"##40":12740,"sketches":12741,"##sive":12742,"marriages":12743,"##⁺":12744,"folding":12745,"peers":12746,"slovak":12747,"implies":12748,"admired":12749,"##beck":12750,"1880s":12751,"leopold":12752,"instinct":12753,"attained":12754,"weston":12755,"megan":12756,"horace":12757,"##ination":12758,"dorsal":12759,"ingredients":12760,"evolutionary":12761,"##its":12762,"complications":12763,"deity":12764,"lethal":12765,"brushing":12766,"levy":12767,"deserted":12768,"institutes":12769,"posthumously":12770,"delivering":12771,"telescope":12772,"coronation":12773,"motivated":12774,"rapids":12775,"luc":12776,"flicked":12777,"pays":12778,"volcano":12779,"tanner":12780,"weighed":12781,"##nica":12782,"crowds":12783,"frankie":12784,"gifted":12785,"addressing":12786,"granddaughter":12787,"winding":12788,"##rna":12789,"constantine":12790,"gomez":12791,"##front":12792,"landscapes":12793,"rudolf":12794,"anthropology":12795,"slate":12796,"werewolf":12797,"##lio":12798,"astronomy":12799,"circa":12800,"rouge":12801,"dreaming":12802,"sack":12803,"knelt":12804,"drowned":12805,"naomi":12806,"prolific":12807,"tracked":12808,"freezing":12809,"herb":12810,"##dium":12811,"agony":12812,"randall":12813,"twisting":12814,"wendy":12815,"deposit":12816,"touches":12817,"vein":12818,"wheeler":12819,"##bbled":12820,"##bor":12821,"batted":12822,"retaining":12823,"tire":12824,"presently":12825,"compare":12826,"specification":12827,"daemon":12828,"nigel":12829,"##grave":12830,"merry":12831,"recommendation":12832,"czechoslovakia":12833,"sandra":12834,"ng":12835,"roma":12836,"##sts":12837,"lambert":12838,"inheritance":12839,"sheikh":12840,"winchester":12841,"cries":12842,"examining":12843,"##yle":12844,"comeback":12845,"cuisine":12846,"nave":12847,"##iv":12848,"ko":12849,"retrieve":12850,"tomatoes":12851,"barker":12852,"polished":12853,"defining":12854,"irene":12855,"lantern":12856,"personalities":12857,"begging":12858,"tract":12859,"swore":12860,"1809":12861,"175":12862,"##gic":12863,"omaha":12864,"brotherhood":12865,"##rley":12866,"haiti":12867,"##ots":12868,"exeter":12869,"##ete":12870,"##zia":12871,"steele":12872,"dumb":12873,"pearson":12874,"210":12875,"surveyed":12876,"elisabeth":12877,"trends":12878,"##ef":12879,"fritz":12880,"##rf":12881,"premium":12882,"bugs":12883,"fraction":12884,"calmly":12885,"viking":12886,"##birds":12887,"tug":12888,"inserted":12889,"unusually":12890,"##ield":12891,"confronted":12892,"distress":12893,"crashing":12894,"brent":12895,"turks":12896,"resign":12897,"##olo":12898,"cambodia":12899,"gabe":12900,"sauce":12901,"##kal":12902,"evelyn":12903,"116":12904,"extant":12905,"clusters":12906,"quarry":12907,"teenagers":12908,"luna":12909,"##lers":12910,"##ister":12911,"affiliation":12912,"drill":12913,"##ashi":12914,"panthers":12915,"scenic":12916,"libya":12917,"anita":12918,"strengthen":12919,"inscriptions":12920,"##cated":12921,"lace":12922,"sued":12923,"judith":12924,"riots":12925,"##uted":12926,"mint":12927,"##eta":12928,"preparations":12929,"midst":12930,"dub":12931,"challenger":12932,"##vich":12933,"mock":12934,"cf":12935,"displaced":12936,"wicket":12937,"breaths":12938,"enables":12939,"schmidt":12940,"analyst":12941,"##lum":12942,"ag":12943,"highlight":12944,"automotive":12945,"axe":12946,"josef":12947,"newark":12948,"sufficiently":12949,"resembles":12950,"50th":12951,"##pal":12952,"flushed":12953,"mum":12954,"traits":12955,"##ante":12956,"commodore":12957,"incomplete":12958,"warming":12959,"titular":12960,"ceremonial":12961,"ethical":12962,"118":12963,"celebrating":12964,"eighteenth":12965,"cao":12966,"lima":12967,"medalist":12968,"mobility":12969,"strips":12970,"snakes":12971,"##city":12972,"miniature":12973,"zagreb":12974,"barton":12975,"escapes":12976,"umbrella":12977,"automated":12978,"doubted":12979,"differs":12980,"cooled":12981,"georgetown":12982,"dresden":12983,"cooked":12984,"fade":12985,"wyatt":12986,"rna":12987,"jacobs":12988,"carlton":12989,"abundant":12990,"stereo":12991,"boost":12992,"madras":12993,"inning":12994,"##hia":12995,"spur":12996,"ip":12997,"malayalam":12998,"begged":12999,"osaka":13000,"groan":13001,"escaping":13002,"charging":13003,"dose":13004,"vista":13005,"##aj":13006,"bud":13007,"papa":13008,"communists":13009,"advocates":13010,"edged":13011,"tri":13012,"##cent":13013,"resemble":13014,"peaking":13015,"necklace":13016,"fried":13017,"montenegro":13018,"saxony":13019,"goose":13020,"glances":13021,"stuttgart":13022,"curator":13023,"recruit":13024,"grocery":13025,"sympathetic":13026,"##tting":13027,"##fort":13028,"127":13029,"lotus":13030,"randolph":13031,"ancestor":13032,"##rand":13033,"succeeding":13034,"jupiter":13035,"1798":13036,"macedonian":13037,"##heads":13038,"hiking":13039,"1808":13040,"handing":13041,"fischer":13042,"##itive":13043,"garbage":13044,"node":13045,"##pies":13046,"prone":13047,"singular":13048,"papua":13049,"inclined":13050,"attractions":13051,"italia":13052,"pouring":13053,"motioned":13054,"grandma":13055,"garnered":13056,"jacksonville":13057,"corp":13058,"ego":13059,"ringing":13060,"aluminum":13061,"##hausen":13062,"ordering":13063,"##foot":13064,"drawer":13065,"traders":13066,"synagogue":13067,"##play":13068,"##kawa":13069,"resistant":13070,"wandering":13071,"fragile":13072,"fiona":13073,"teased":13074,"var":13075,"hardcore":13076,"soaked":13077,"jubilee":13078,"decisive":13079,"exposition":13080,"mercer":13081,"poster":13082,"valencia":13083,"hale":13084,"kuwait":13085,"1811":13086,"##ises":13087,"##wr":13088,"##eed":13089,"tavern":13090,"gamma":13091,"122":13092,"johan":13093,"##uer":13094,"airways":13095,"amino":13096,"gil":13097,"##ury":13098,"vocational":13099,"domains":13100,"torres":13101,"##sp":13102,"generator":13103,"folklore":13104,"outcomes":13105,"##keeper":13106,"canberra":13107,"shooter":13108,"fl":13109,"beams":13110,"confrontation":13111,"##lling":13112,"##gram":13113,"feb":13114,"aligned":13115,"forestry":13116,"pipeline":13117,"jax":13118,"motorway":13119,"conception":13120,"decay":13121,"##tos":13122,"coffin":13123,"##cott":13124,"stalin":13125,"1805":13126,"escorted":13127,"minded":13128,"##nam":13129,"sitcom":13130,"purchasing":13131,"twilight":13132,"veronica":13133,"additions":13134,"passive":13135,"tensions":13136,"straw":13137,"123":13138,"frequencies":13139,"1804":13140,"refugee":13141,"cultivation":13142,"##iate":13143,"christie":13144,"clary":13145,"bulletin":13146,"crept":13147,"disposal":13148,"##rich":13149,"##zong":13150,"processor":13151,"crescent":13152,"##rol":13153,"bmw":13154,"emphasized":13155,"whale":13156,"nazis":13157,"aurora":13158,"##eng":13159,"dwelling":13160,"hauled":13161,"sponsors":13162,"toledo":13163,"mega":13164,"ideology":13165,"theatres":13166,"tessa":13167,"cerambycidae":13168,"saves":13169,"turtle":13170,"cone":13171,"suspects":13172,"kara":13173,"rusty":13174,"yelling":13175,"greeks":13176,"mozart":13177,"shades":13178,"cocked":13179,"participant":13180,"##tro":13181,"shire":13182,"spit":13183,"freeze":13184,"necessity":13185,"##cos":13186,"inmates":13187,"nielsen":13188,"councillors":13189,"loaned":13190,"uncommon":13191,"omar":13192,"peasants":13193,"botanical":13194,"offspring":13195,"daniels":13196,"formations":13197,"jokes":13198,"1794":13199,"pioneers":13200,"sigma":13201,"licensing":13202,"##sus":13203,"wheelchair":13204,"polite":13205,"1807":13206,"liquor":13207,"pratt":13208,"trustee":13209,"##uta":13210,"forewings":13211,"balloon":13212,"##zz":13213,"kilometre":13214,"camping":13215,"explicit":13216,"casually":13217,"shawn":13218,"foolish":13219,"teammates":13220,"nm":13221,"hassan":13222,"carrie":13223,"judged":13224,"satisfy":13225,"vanessa":13226,"knives":13227,"selective":13228,"cnn":13229,"flowed":13230,"##lice":13231,"eclipse":13232,"stressed":13233,"eliza":13234,"mathematician":13235,"cease":13236,"cultivated":13237,"##roy":13238,"commissions":13239,"browns":13240,"##ania":13241,"destroyers":13242,"sheridan":13243,"meadow":13244,"##rius":13245,"minerals":13246,"##cial":13247,"downstream":13248,"clash":13249,"gram":13250,"memoirs":13251,"ventures":13252,"baha":13253,"seymour":13254,"archie":13255,"midlands":13256,"edith":13257,"fare":13258,"flynn":13259,"invite":13260,"canceled":13261,"tiles":13262,"stabbed":13263,"boulder":13264,"incorporate":13265,"amended":13266,"camden":13267,"facial":13268,"mollusk":13269,"unreleased":13270,"descriptions":13271,"yoga":13272,"grabs":13273,"550":13274,"raises":13275,"ramp":13276,"shiver":13277,"##rose":13278,"coined":13279,"pioneering":13280,"tunes":13281,"qing":13282,"warwick":13283,"tops":13284,"119":13285,"melanie":13286,"giles":13287,"##rous":13288,"wandered":13289,"##inal":13290,"annexed":13291,"nov":13292,"30th":13293,"unnamed":13294,"##ished":13295,"organizational":13296,"airplane":13297,"normandy":13298,"stoke":13299,"whistle":13300,"blessing":13301,"violations":13302,"chased":13303,"holders":13304,"shotgun":13305,"##ctic":13306,"outlet":13307,"reactor":13308,"##vik":13309,"tires":13310,"tearing":13311,"shores":13312,"fortified":13313,"mascot":13314,"constituencies":13315,"nc":13316,"columnist":13317,"productive":13318,"tibet":13319,"##rta":13320,"lineage":13321,"hooked":13322,"oct":13323,"tapes":13324,"judging":13325,"cody":13326,"##gger":13327,"hansen":13328,"kashmir":13329,"triggered":13330,"##eva":13331,"solved":13332,"cliffs":13333,"##tree":13334,"resisted":13335,"anatomy":13336,"protesters":13337,"transparent":13338,"implied":13339,"##iga":13340,"injection":13341,"mattress":13342,"excluding":13343,"##mbo":13344,"defenses":13345,"helpless":13346,"devotion":13347,"##elli":13348,"growl":13349,"liberals":13350,"weber":13351,"phenomena":13352,"atoms":13353,"plug":13354,"##iff":13355,"mortality":13356,"apprentice":13357,"howe":13358,"convincing":13359,"aaa":13360,"swimmer":13361,"barber":13362,"leone":13363,"promptly":13364,"sodium":13365,"def":13366,"nowadays":13367,"arise":13368,"##oning":13369,"gloucester":13370,"corrected":13371,"dignity":13372,"norm":13373,"erie":13374,"##ders":13375,"elders":13376,"evacuated":13377,"sylvia":13378,"compression":13379,"##yar":13380,"hartford":13381,"pose":13382,"backpack":13383,"reasoning":13384,"accepts":13385,"24th":13386,"wipe":13387,"millimetres":13388,"marcel":13389,"##oda":13390,"dodgers":13391,"albion":13392,"1790":13393,"overwhelmed":13394,"aerospace":13395,"oaks":13396,"1795":13397,"showcase":13398,"acknowledge":13399,"recovering":13400,"nolan":13401,"ashe":13402,"hurts":13403,"geology":13404,"fashioned":13405,"disappearance":13406,"farewell":13407,"swollen":13408,"shrug":13409,"marquis":13410,"wimbledon":13411,"124":13412,"rue":13413,"1792":13414,"commemorate":13415,"reduces":13416,"experiencing":13417,"inevitable":13418,"calcutta":13419,"intel":13420,"##court":13421,"murderer":13422,"sticking":13423,"fisheries":13424,"imagery":13425,"bloom":13426,"280":13427,"brake":13428,"##inus":13429,"gustav":13430,"hesitation":13431,"memorable":13432,"po":13433,"viral":13434,"beans":13435,"accidents":13436,"tunisia":13437,"antenna":13438,"spilled":13439,"consort":13440,"treatments":13441,"aye":13442,"perimeter":13443,"##gard":13444,"donation":13445,"hostage":13446,"migrated":13447,"banker":13448,"addiction":13449,"apex":13450,"lil":13451,"trout":13452,"##ously":13453,"conscience":13454,"##nova":13455,"rams":13456,"sands":13457,"genome":13458,"passionate":13459,"troubles":13460,"##lets":13461,"##set":13462,"amid":13463,"##ibility":13464,"##ret":13465,"higgins":13466,"exceed":13467,"vikings":13468,"##vie":13469,"payne":13470,"##zan":13471,"muscular":13472,"##ste":13473,"defendant":13474,"sucking":13475,"##wal":13476,"ibrahim":13477,"fuselage":13478,"claudia":13479,"vfl":13480,"europeans":13481,"snails":13482,"interval":13483,"##garh":13484,"preparatory":13485,"statewide":13486,"tasked":13487,"lacrosse":13488,"viktor":13489,"##lation":13490,"angola":13491,"##hra":13492,"flint":13493,"implications":13494,"employs":13495,"teens":13496,"patrons":13497,"stall":13498,"weekends":13499,"barriers":13500,"scrambled":13501,"nucleus":13502,"tehran":13503,"jenna":13504,"parsons":13505,"lifelong":13506,"robots":13507,"displacement":13508,"5000":13509,"##bles":13510,"precipitation":13511,"##gt":13512,"knuckles":13513,"clutched":13514,"1802":13515,"marrying":13516,"ecology":13517,"marx":13518,"accusations":13519,"declare":13520,"scars":13521,"kolkata":13522,"mat":13523,"meadows":13524,"bermuda":13525,"skeleton":13526,"finalists":13527,"vintage":13528,"crawl":13529,"coordinate":13530,"affects":13531,"subjected":13532,"orchestral":13533,"mistaken":13534,"##tc":13535,"mirrors":13536,"dipped":13537,"relied":13538,"260":13539,"arches":13540,"candle":13541,"##nick":13542,"incorporating":13543,"wildly":13544,"fond":13545,"basilica":13546,"owl":13547,"fringe":13548,"rituals":13549,"whispering":13550,"stirred":13551,"feud":13552,"tertiary":13553,"slick":13554,"goat":13555,"honorable":13556,"whereby":13557,"skip":13558,"ricardo":13559,"stripes":13560,"parachute":13561,"adjoining":13562,"submerged":13563,"synthesizer":13564,"##gren":13565,"intend":13566,"positively":13567,"ninety":13568,"phi":13569,"beaver":13570,"partition":13571,"fellows":13572,"alexis":13573,"prohibition":13574,"carlisle":13575,"bizarre":13576,"fraternity":13577,"##bre":13578,"doubts":13579,"icy":13580,"cbc":13581,"aquatic":13582,"sneak":13583,"sonny":13584,"combines":13585,"airports":13586,"crude":13587,"supervised":13588,"spatial":13589,"merge":13590,"alfonso":13591,"##bic":13592,"corrupt":13593,"scan":13594,"undergo":13595,"##ams":13596,"disabilities":13597,"colombian":13598,"comparing":13599,"dolphins":13600,"perkins":13601,"##lish":13602,"reprinted":13603,"unanimous":13604,"bounced":13605,"hairs":13606,"underworld":13607,"midwest":13608,"semester":13609,"bucket":13610,"paperback":13611,"miniseries":13612,"coventry":13613,"demise":13614,"##leigh":13615,"demonstrations":13616,"sensor":13617,"rotating":13618,"yan":13619,"##hler":13620,"arrange":13621,"soils":13622,"##idge":13623,"hyderabad":13624,"labs":13625,"##dr":13626,"brakes":13627,"grandchildren":13628,"##nde":13629,"negotiated":13630,"rover":13631,"ferrari":13632,"continuation":13633,"directorate":13634,"augusta":13635,"stevenson":13636,"counterpart":13637,"gore":13638,"##rda":13639,"nursery":13640,"rican":13641,"ave":13642,"collectively":13643,"broadly":13644,"pastoral":13645,"repertoire":13646,"asserted":13647,"discovering":13648,"nordic":13649,"styled":13650,"fiba":13651,"cunningham":13652,"harley":13653,"middlesex":13654,"survives":13655,"tumor":13656,"tempo":13657,"zack":13658,"aiming":13659,"lok":13660,"urgent":13661,"##rade":13662,"##nto":13663,"devils":13664,"##ement":13665,"contractor":13666,"turin":13667,"##wl":13668,"##ool":13669,"bliss":13670,"repaired":13671,"simmons":13672,"moan":13673,"astronomical":13674,"cr":13675,"negotiate":13676,"lyric":13677,"1890s":13678,"lara":13679,"bred":13680,"clad":13681,"angus":13682,"pbs":13683,"##ience":13684,"engineered":13685,"posed":13686,"##lk":13687,"hernandez":13688,"possessions":13689,"elbows":13690,"psychiatric":13691,"strokes":13692,"confluence":13693,"electorate":13694,"lifts":13695,"campuses":13696,"lava":13697,"alps":13698,"##ep":13699,"##ution":13700,"##date":13701,"physicist":13702,"woody":13703,"##page":13704,"##ographic":13705,"##itis":13706,"juliet":13707,"reformation":13708,"sparhawk":13709,"320":13710,"complement":13711,"suppressed":13712,"jewel":13713,"##½":13714,"floated":13715,"##kas":13716,"continuity":13717,"sadly":13718,"##ische":13719,"inability":13720,"melting":13721,"scanning":13722,"paula":13723,"flour":13724,"judaism":13725,"safer":13726,"vague":13727,"##lm":13728,"solving":13729,"curb":13730,"##stown":13731,"financially":13732,"gable":13733,"bees":13734,"expired":13735,"miserable":13736,"cassidy":13737,"dominion":13738,"1789":13739,"cupped":13740,"145":13741,"robbery":13742,"facto":13743,"amos":13744,"warden":13745,"resume":13746,"tallest":13747,"marvin":13748,"ing":13749,"pounded":13750,"usd":13751,"declaring":13752,"gasoline":13753,"##aux":13754,"darkened":13755,"270":13756,"650":13757,"sophomore":13758,"##mere":13759,"erection":13760,"gossip":13761,"televised":13762,"risen":13763,"dial":13764,"##eu":13765,"pillars":13766,"##link":13767,"passages":13768,"profound":13769,"##tina":13770,"arabian":13771,"ashton":13772,"silicon":13773,"nail":13774,"##ead":13775,"##lated":13776,"##wer":13777,"##hardt":13778,"fleming":13779,"firearms":13780,"ducked":13781,"circuits":13782,"blows":13783,"waterloo":13784,"titans":13785,"##lina":13786,"atom":13787,"fireplace":13788,"cheshire":13789,"financed":13790,"activation":13791,"algorithms":13792,"##zzi":13793,"constituent":13794,"catcher":13795,"cherokee":13796,"partnerships":13797,"sexuality":13798,"platoon":13799,"tragic":13800,"vivian":13801,"guarded":13802,"whiskey":13803,"meditation":13804,"poetic":13805,"##late":13806,"##nga":13807,"##ake":13808,"porto":13809,"listeners":13810,"dominance":13811,"kendra":13812,"mona":13813,"chandler":13814,"factions":13815,"22nd":13816,"salisbury":13817,"attitudes":13818,"derivative":13819,"##ido":13820,"##haus":13821,"intake":13822,"paced":13823,"javier":13824,"illustrator":13825,"barrels":13826,"bias":13827,"cockpit":13828,"burnett":13829,"dreamed":13830,"ensuing":13831,"##anda":13832,"receptors":13833,"someday":13834,"hawkins":13835,"mattered":13836,"##lal":13837,"slavic":13838,"1799":13839,"jesuit":13840,"cameroon":13841,"wasted":13842,"tai":13843,"wax":13844,"lowering":13845,"victorious":13846,"freaking":13847,"outright":13848,"hancock":13849,"librarian":13850,"sensing":13851,"bald":13852,"calcium":13853,"myers":13854,"tablet":13855,"announcing":13856,"barack":13857,"shipyard":13858,"pharmaceutical":13859,"##uan":13860,"greenwich":13861,"flush":13862,"medley":13863,"patches":13864,"wolfgang":13865,"pt":13866,"speeches":13867,"acquiring":13868,"exams":13869,"nikolai":13870,"##gg":13871,"hayden":13872,"kannada":13873,"##type":13874,"reilly":13875,"##pt":13876,"waitress":13877,"abdomen":13878,"devastated":13879,"capped":13880,"pseudonym":13881,"pharmacy":13882,"fulfill":13883,"paraguay":13884,"1796":13885,"clicked":13886,"##trom":13887,"archipelago":13888,"syndicated":13889,"##hman":13890,"lumber":13891,"orgasm":13892,"rejection":13893,"clifford":13894,"lorraine":13895,"advent":13896,"mafia":13897,"rodney":13898,"brock":13899,"##ght":13900,"##used":13901,"##elia":13902,"cassette":13903,"chamberlain":13904,"despair":13905,"mongolia":13906,"sensors":13907,"developmental":13908,"upstream":13909,"##eg":13910,"##alis":13911,"spanning":13912,"165":13913,"trombone":13914,"basque":13915,"seeded":13916,"interred":13917,"renewable":13918,"rhys":13919,"leapt":13920,"revision":13921,"molecule":13922,"##ages":13923,"chord":13924,"vicious":13925,"nord":13926,"shivered":13927,"23rd":13928,"arlington":13929,"debts":13930,"corpus":13931,"sunrise":13932,"bays":13933,"blackburn":13934,"centimetres":13935,"##uded":13936,"shuddered":13937,"gm":13938,"strangely":13939,"gripping":13940,"cartoons":13941,"isabelle":13942,"orbital":13943,"##ppa":13944,"seals":13945,"proving":13946,"##lton":13947,"refusal":13948,"strengthened":13949,"bust":13950,"assisting":13951,"baghdad":13952,"batsman":13953,"portrayal":13954,"mara":13955,"pushes":13956,"spears":13957,"og":13958,"##cock":13959,"reside":13960,"nathaniel":13961,"brennan":13962,"1776":13963,"confirmation":13964,"caucus":13965,"##worthy":13966,"markings":13967,"yemen":13968,"nobles":13969,"ku":13970,"lazy":13971,"viewer":13972,"catalan":13973,"encompasses":13974,"sawyer":13975,"##fall":13976,"sparked":13977,"substances":13978,"patents":13979,"braves":13980,"arranger":13981,"evacuation":13982,"sergio":13983,"persuade":13984,"dover":13985,"tolerance":13986,"penguin":13987,"cum":13988,"jockey":13989,"insufficient":13990,"townships":13991,"occupying":13992,"declining":13993,"plural":13994,"processed":13995,"projection":13996,"puppet":13997,"flanders":13998,"introduces":13999,"liability":14000,"##yon":14001,"gymnastics":14002,"antwerp":14003,"taipei":14004,"hobart":14005,"candles":14006,"jeep":14007,"wes":14008,"observers":14009,"126":14010,"chaplain":14011,"bundle":14012,"glorious":14013,"##hine":14014,"hazel":14015,"flung":14016,"sol":14017,"excavations":14018,"dumped":14019,"stares":14020,"sh":14021,"bangalore":14022,"triangular":14023,"icelandic":14024,"intervals":14025,"expressing":14026,"turbine":14027,"##vers":14028,"songwriting":14029,"crafts":14030,"##igo":14031,"jasmine":14032,"ditch":14033,"rite":14034,"##ways":14035,"entertaining":14036,"comply":14037,"sorrow":14038,"wrestlers":14039,"basel":14040,"emirates":14041,"marian":14042,"rivera":14043,"helpful":14044,"##some":14045,"caution":14046,"downward":14047,"networking":14048,"##atory":14049,"##tered":14050,"darted":14051,"genocide":14052,"emergence":14053,"replies":14054,"specializing":14055,"spokesman":14056,"convenient":14057,"unlocked":14058,"fading":14059,"augustine":14060,"concentrations":14061,"resemblance":14062,"elijah":14063,"investigator":14064,"andhra":14065,"##uda":14066,"promotes":14067,"bean":14068,"##rrell":14069,"fleeing":14070,"wan":14071,"simone":14072,"announcer":14073,"##ame":14074,"##bby":14075,"lydia":14076,"weaver":14077,"132":14078,"residency":14079,"modification":14080,"##fest":14081,"stretches":14082,"##ast":14083,"alternatively":14084,"nat":14085,"lowe":14086,"lacks":14087,"##ented":14088,"pam":14089,"tile":14090,"concealed":14091,"inferior":14092,"abdullah":14093,"residences":14094,"tissues":14095,"vengeance":14096,"##ided":14097,"moisture":14098,"peculiar":14099,"groove":14100,"zip":14101,"bologna":14102,"jennings":14103,"ninja":14104,"oversaw":14105,"zombies":14106,"pumping":14107,"batch":14108,"livingston":14109,"emerald":14110,"installations":14111,"1797":14112,"peel":14113,"nitrogen":14114,"rama":14115,"##fying":14116,"##star":14117,"schooling":14118,"strands":14119,"responding":14120,"werner":14121,"##ost":14122,"lime":14123,"casa":14124,"accurately":14125,"targeting":14126,"##rod":14127,"underway":14128,"##uru":14129,"hemisphere":14130,"lester":14131,"##yard":14132,"occupies":14133,"2d":14134,"griffith":14135,"angrily":14136,"reorganized":14137,"##owing":14138,"courtney":14139,"deposited":14140,"##dd":14141,"##30":14142,"estadio":14143,"##ifies":14144,"dunn":14145,"exiled":14146,"##ying":14147,"checks":14148,"##combe":14149,"##о":14150,"##fly":14151,"successes":14152,"unexpectedly":14153,"blu":14154,"assessed":14155,"##flower":14156,"##ه":14157,"observing":14158,"sacked":14159,"spiders":14160,"kn":14161,"##tail":14162,"mu":14163,"nodes":14164,"prosperity":14165,"audrey":14166,"divisional":14167,"155":14168,"broncos":14169,"tangled":14170,"adjust":14171,"feeds":14172,"erosion":14173,"paolo":14174,"surf":14175,"directory":14176,"snatched":14177,"humid":14178,"admiralty":14179,"screwed":14180,"gt":14181,"reddish":14182,"##nese":14183,"modules":14184,"trench":14185,"lamps":14186,"bind":14187,"leah":14188,"bucks":14189,"competes":14190,"##nz":14191,"##form":14192,"transcription":14193,"##uc":14194,"isles":14195,"violently":14196,"clutching":14197,"pga":14198,"cyclist":14199,"inflation":14200,"flats":14201,"ragged":14202,"unnecessary":14203,"##hian":14204,"stubborn":14205,"coordinated":14206,"harriet":14207,"baba":14208,"disqualified":14209,"330":14210,"insect":14211,"wolfe":14212,"##fies":14213,"reinforcements":14214,"rocked":14215,"duel":14216,"winked":14217,"embraced":14218,"bricks":14219,"##raj":14220,"hiatus":14221,"defeats":14222,"pending":14223,"brightly":14224,"jealousy":14225,"##xton":14226,"##hm":14227,"##uki":14228,"lena":14229,"gdp":14230,"colorful":14231,"##dley":14232,"stein":14233,"kidney":14234,"##shu":14235,"underwear":14236,"wanderers":14237,"##haw":14238,"##icus":14239,"guardians":14240,"m³":14241,"roared":14242,"habits":14243,"##wise":14244,"permits":14245,"gp":14246,"uranium":14247,"punished":14248,"disguise":14249,"bundesliga":14250,"elise":14251,"dundee":14252,"erotic":14253,"partisan":14254,"pi":14255,"collectors":14256,"float":14257,"individually":14258,"rendering":14259,"behavioral":14260,"bucharest":14261,"ser":14262,"hare":14263,"valerie":14264,"corporal":14265,"nutrition":14266,"proportional":14267,"##isa":14268,"immense":14269,"##kis":14270,"pavement":14271,"##zie":14272,"##eld":14273,"sutherland":14274,"crouched":14275,"1775":14276,"##lp":14277,"suzuki":14278,"trades":14279,"endurance":14280,"operas":14281,"crosby":14282,"prayed":14283,"priory":14284,"rory":14285,"socially":14286,"##urn":14287,"gujarat":14288,"##pu":14289,"walton":14290,"cube":14291,"pasha":14292,"privilege":14293,"lennon":14294,"floods":14295,"thorne":14296,"waterfall":14297,"nipple":14298,"scouting":14299,"approve":14300,"##lov":14301,"minorities":14302,"voter":14303,"dwight":14304,"extensions":14305,"assure":14306,"ballroom":14307,"slap":14308,"dripping":14309,"privileges":14310,"rejoined":14311,"confessed":14312,"demonstrating":14313,"patriotic":14314,"yell":14315,"investor":14316,"##uth":14317,"pagan":14318,"slumped":14319,"squares":14320,"##cle":14321,"##kins":14322,"confront":14323,"bert":14324,"embarrassment":14325,"##aid":14326,"aston":14327,"urging":14328,"sweater":14329,"starr":14330,"yuri":14331,"brains":14332,"williamson":14333,"commuter":14334,"mortar":14335,"structured":14336,"selfish":14337,"exports":14338,"##jon":14339,"cds":14340,"##him":14341,"unfinished":14342,"##rre":14343,"mortgage":14344,"destinations":14345,"##nagar":14346,"canoe":14347,"solitary":14348,"buchanan":14349,"delays":14350,"magistrate":14351,"fk":14352,"##pling":14353,"motivation":14354,"##lier":14355,"##vier":14356,"recruiting":14357,"assess":14358,"##mouth":14359,"malik":14360,"antique":14361,"1791":14362,"pius":14363,"rahman":14364,"reich":14365,"tub":14366,"zhou":14367,"smashed":14368,"airs":14369,"galway":14370,"xii":14371,"conditioning":14372,"honduras":14373,"discharged":14374,"dexter":14375,"##pf":14376,"lionel":14377,"129":14378,"debates":14379,"lemon":14380,"tiffany":14381,"volunteered":14382,"dom":14383,"dioxide":14384,"procession":14385,"devi":14386,"sic":14387,"tremendous":14388,"advertisements":14389,"colts":14390,"transferring":14391,"verdict":14392,"hanover":14393,"decommissioned":14394,"utter":14395,"relate":14396,"pac":14397,"racism":14398,"##top":14399,"beacon":14400,"limp":14401,"similarity":14402,"terra":14403,"occurrence":14404,"ant":14405,"##how":14406,"becky":14407,"capt":14408,"updates":14409,"armament":14410,"richie":14411,"pal":14412,"##graph":14413,"halloween":14414,"mayo":14415,"##ssen":14416,"##bone":14417,"cara":14418,"serena":14419,"fcc":14420,"dolls":14421,"obligations":14422,"##dling":14423,"violated":14424,"lafayette":14425,"jakarta":14426,"exploitation":14427,"##ime":14428,"infamous":14429,"iconic":14430,"##lah":14431,"##park":14432,"kitty":14433,"moody":14434,"reginald":14435,"dread":14436,"spill":14437,"crystals":14438,"olivier":14439,"modeled":14440,"bluff":14441,"equilibrium":14442,"separating":14443,"notices":14444,"ordnance":14445,"extinction":14446,"onset":14447,"cosmic":14448,"attachment":14449,"sammy":14450,"expose":14451,"privy":14452,"anchored":14453,"##bil":14454,"abbott":14455,"admits":14456,"bending":14457,"baritone":14458,"emmanuel":14459,"policeman":14460,"vaughan":14461,"winged":14462,"climax":14463,"dresses":14464,"denny":14465,"polytechnic":14466,"mohamed":14467,"burmese":14468,"authentic":14469,"nikki":14470,"genetics":14471,"grandparents":14472,"homestead":14473,"gaza":14474,"postponed":14475,"metacritic":14476,"una":14477,"##sby":14478,"##bat":14479,"unstable":14480,"dissertation":14481,"##rial":14482,"##cian":14483,"curls":14484,"obscure":14485,"uncovered":14486,"bronx":14487,"praying":14488,"disappearing":14489,"##hoe":14490,"prehistoric":14491,"coke":14492,"turret":14493,"mutations":14494,"nonprofit":14495,"pits":14496,"monaco":14497,"##ي":14498,"##usion":14499,"prominently":14500,"dispatched":14501,"podium":14502,"##mir":14503,"uci":14504,"##uation":14505,"133":14506,"fortifications":14507,"birthplace":14508,"kendall":14509,"##lby":14510,"##oll":14511,"preacher":14512,"rack":14513,"goodman":14514,"##rman":14515,"persistent":14516,"##ott":14517,"countless":14518,"jaime":14519,"recorder":14520,"lexington":14521,"persecution":14522,"jumps":14523,"renewal":14524,"wagons":14525,"##11":14526,"crushing":14527,"##holder":14528,"decorations":14529,"##lake":14530,"abundance":14531,"wrath":14532,"laundry":14533,"£1":14534,"garde":14535,"##rp":14536,"jeanne":14537,"beetles":14538,"peasant":14539,"##sl":14540,"splitting":14541,"caste":14542,"sergei":14543,"##rer":14544,"##ema":14545,"scripts":14546,"##ively":14547,"rub":14548,"satellites":14549,"##vor":14550,"inscribed":14551,"verlag":14552,"scrapped":14553,"gale":14554,"packages":14555,"chick":14556,"potato":14557,"slogan":14558,"kathleen":14559,"arabs":14560,"##culture":14561,"counterparts":14562,"reminiscent":14563,"choral":14564,"##tead":14565,"rand":14566,"retains":14567,"bushes":14568,"dane":14569,"accomplish":14570,"courtesy":14571,"closes":14572,"##oth":14573,"slaughter":14574,"hague":14575,"krakow":14576,"lawson":14577,"tailed":14578,"elias":14579,"ginger":14580,"##ttes":14581,"canopy":14582,"betrayal":14583,"rebuilding":14584,"turf":14585,"##hof":14586,"frowning":14587,"allegiance":14588,"brigades":14589,"kicks":14590,"rebuild":14591,"polls":14592,"alias":14593,"nationalism":14594,"td":14595,"rowan":14596,"audition":14597,"bowie":14598,"fortunately":14599,"recognizes":14600,"harp":14601,"dillon":14602,"horrified":14603,"##oro":14604,"renault":14605,"##tics":14606,"ropes":14607,"##α":14608,"presumed":14609,"rewarded":14610,"infrared":14611,"wiping":14612,"accelerated":14613,"illustration":14614,"##rid":14615,"presses":14616,"practitioners":14617,"badminton":14618,"##iard":14619,"detained":14620,"##tera":14621,"recognizing":14622,"relates":14623,"misery":14624,"##sies":14625,"##tly":14626,"reproduction":14627,"piercing":14628,"potatoes":14629,"thornton":14630,"esther":14631,"manners":14632,"hbo":14633,"##aan":14634,"ours":14635,"bullshit":14636,"ernie":14637,"perennial":14638,"sensitivity":14639,"illuminated":14640,"rupert":14641,"##jin":14642,"##iss":14643,"##ear":14644,"rfc":14645,"nassau":14646,"##dock":14647,"staggered":14648,"socialism":14649,"##haven":14650,"appointments":14651,"nonsense":14652,"prestige":14653,"sharma":14654,"haul":14655,"##tical":14656,"solidarity":14657,"gps":14658,"##ook":14659,"##rata":14660,"igor":14661,"pedestrian":14662,"##uit":14663,"baxter":14664,"tenants":14665,"wires":14666,"medication":14667,"unlimited":14668,"guiding":14669,"impacts":14670,"diabetes":14671,"##rama":14672,"sasha":14673,"pas":14674,"clive":14675,"extraction":14676,"131":14677,"continually":14678,"constraints":14679,"##bilities":14680,"sonata":14681,"hunted":14682,"sixteenth":14683,"chu":14684,"planting":14685,"quote":14686,"mayer":14687,"pretended":14688,"abs":14689,"spat":14690,"##hua":14691,"ceramic":14692,"##cci":14693,"curtains":14694,"pigs":14695,"pitching":14696,"##dad":14697,"latvian":14698,"sore":14699,"dayton":14700,"##sted":14701,"##qi":14702,"patrols":14703,"slice":14704,"playground":14705,"##nted":14706,"shone":14707,"stool":14708,"apparatus":14709,"inadequate":14710,"mates":14711,"treason":14712,"##ija":14713,"desires":14714,"##liga":14715,"##croft":14716,"somalia":14717,"laurent":14718,"mir":14719,"leonardo":14720,"oracle":14721,"grape":14722,"obliged":14723,"chevrolet":14724,"thirteenth":14725,"stunning":14726,"enthusiastic":14727,"##ede":14728,"accounted":14729,"concludes":14730,"currents":14731,"basil":14732,"##kovic":14733,"drought":14734,"##rica":14735,"mai":14736,"##aire":14737,"shove":14738,"posting":14739,"##shed":14740,"pilgrimage":14741,"humorous":14742,"packing":14743,"fry":14744,"pencil":14745,"wines":14746,"smells":14747,"144":14748,"marilyn":14749,"aching":14750,"newest":14751,"clung":14752,"bon":14753,"neighbours":14754,"sanctioned":14755,"##pie":14756,"mug":14757,"##stock":14758,"drowning":14759,"##mma":14760,"hydraulic":14761,"##vil":14762,"hiring":14763,"reminder":14764,"lilly":14765,"investigators":14766,"##ncies":14767,"sour":14768,"##eous":14769,"compulsory":14770,"packet":14771,"##rion":14772,"##graphic":14773,"##elle":14774,"cannes":14775,"##inate":14776,"depressed":14777,"##rit":14778,"heroic":14779,"importantly":14780,"theresa":14781,"##tled":14782,"conway":14783,"saturn":14784,"marginal":14785,"rae":14786,"##xia":14787,"corresponds":14788,"royce":14789,"pact":14790,"jasper":14791,"explosives":14792,"packaging":14793,"aluminium":14794,"##ttered":14795,"denotes":14796,"rhythmic":14797,"spans":14798,"assignments":14799,"hereditary":14800,"outlined":14801,"originating":14802,"sundays":14803,"lad":14804,"reissued":14805,"greeting":14806,"beatrice":14807,"##dic":14808,"pillar":14809,"marcos":14810,"plots":14811,"handbook":14812,"alcoholic":14813,"judiciary":14814,"avant":14815,"slides":14816,"extract":14817,"masculine":14818,"blur":14819,"##eum":14820,"##force":14821,"homage":14822,"trembled":14823,"owens":14824,"hymn":14825,"trey":14826,"omega":14827,"signaling":14828,"socks":14829,"accumulated":14830,"reacted":14831,"attic":14832,"theo":14833,"lining":14834,"angie":14835,"distraction":14836,"primera":14837,"talbot":14838,"##key":14839,"1200":14840,"ti":14841,"creativity":14842,"billed":14843,"##hey":14844,"deacon":14845,"eduardo":14846,"identifies":14847,"proposition":14848,"dizzy":14849,"gunner":14850,"hogan":14851,"##yam":14852,"##pping":14853,"##hol":14854,"ja":14855,"##chan":14856,"jensen":14857,"reconstructed":14858,"##berger":14859,"clearance":14860,"darius":14861,"##nier":14862,"abe":14863,"harlem":14864,"plea":14865,"dei":14866,"circled":14867,"emotionally":14868,"notation":14869,"fascist":14870,"neville":14871,"exceeded":14872,"upwards":14873,"viable":14874,"ducks":14875,"##fo":14876,"workforce":14877,"racer":14878,"limiting":14879,"shri":14880,"##lson":14881,"possesses":14882,"1600":14883,"kerr":14884,"moths":14885,"devastating":14886,"laden":14887,"disturbing":14888,"locking":14889,"##cture":14890,"gal":14891,"fearing":14892,"accreditation":14893,"flavor":14894,"aide":14895,"1870s":14896,"mountainous":14897,"##baum":14898,"melt":14899,"##ures":14900,"motel":14901,"texture":14902,"servers":14903,"soda":14904,"##mb":14905,"herd":14906,"##nium":14907,"erect":14908,"puzzled":14909,"hum":14910,"peggy":14911,"examinations":14912,"gould":14913,"testified":14914,"geoff":14915,"ren":14916,"devised":14917,"sacks":14918,"##law":14919,"denial":14920,"posters":14921,"grunted":14922,"cesar":14923,"tutor":14924,"ec":14925,"gerry":14926,"offerings":14927,"byrne":14928,"falcons":14929,"combinations":14930,"ct":14931,"incoming":14932,"pardon":14933,"rocking":14934,"26th":14935,"avengers":14936,"flared":14937,"mankind":14938,"seller":14939,"uttar":14940,"loch":14941,"nadia":14942,"stroking":14943,"exposing":14944,"##hd":14945,"fertile":14946,"ancestral":14947,"instituted":14948,"##has":14949,"noises":14950,"prophecy":14951,"taxation":14952,"eminent":14953,"vivid":14954,"pol":14955,"##bol":14956,"dart":14957,"indirect":14958,"multimedia":14959,"notebook":14960,"upside":14961,"displaying":14962,"adrenaline":14963,"referenced":14964,"geometric":14965,"##iving":14966,"progression":14967,"##ddy":14968,"blunt":14969,"announce":14970,"##far":14971,"implementing":14972,"##lav":14973,"aggression":14974,"liaison":14975,"cooler":14976,"cares":14977,"headache":14978,"plantations":14979,"gorge":14980,"dots":14981,"impulse":14982,"thickness":14983,"ashamed":14984,"averaging":14985,"kathy":14986,"obligation":14987,"precursor":14988,"137":14989,"fowler":14990,"symmetry":14991,"thee":14992,"225":14993,"hears":14994,"##rai":14995,"undergoing":14996,"ads":14997,"butcher":14998,"bowler":14999,"##lip":15000,"cigarettes":15001,"subscription":15002,"goodness":15003,"##ically":15004,"browne":15005,"##hos":15006,"##tech":15007,"kyoto":15008,"donor":15009,"##erty":15010,"damaging":15011,"friction":15012,"drifting":15013,"expeditions":15014,"hardened":15015,"prostitution":15016,"152":15017,"fauna":15018,"blankets":15019,"claw":15020,"tossing":15021,"snarled":15022,"butterflies":15023,"recruits":15024,"investigative":15025,"coated":15026,"healed":15027,"138":15028,"communal":15029,"hai":15030,"xiii":15031,"academics":15032,"boone":15033,"psychologist":15034,"restless":15035,"lahore":15036,"stephens":15037,"mba":15038,"brendan":15039,"foreigners":15040,"printer":15041,"##pc":15042,"ached":15043,"explode":15044,"27th":15045,"deed":15046,"scratched":15047,"dared":15048,"##pole":15049,"cardiac":15050,"1780":15051,"okinawa":15052,"proto":15053,"commando":15054,"compelled":15055,"oddly":15056,"electrons":15057,"##base":15058,"replica":15059,"thanksgiving":15060,"##rist":15061,"sheila":15062,"deliberate":15063,"stafford":15064,"tidal":15065,"representations":15066,"hercules":15067,"ou":15068,"##path":15069,"##iated":15070,"kidnapping":15071,"lenses":15072,"##tling":15073,"deficit":15074,"samoa":15075,"mouths":15076,"consuming":15077,"computational":15078,"maze":15079,"granting":15080,"smirk":15081,"razor":15082,"fixture":15083,"ideals":15084,"inviting":15085,"aiden":15086,"nominal":15087,"##vs":15088,"issuing":15089,"julio":15090,"pitt":15091,"ramsey":15092,"docks":15093,"##oss":15094,"exhaust":15095,"##owed":15096,"bavarian":15097,"draped":15098,"anterior":15099,"mating":15100,"ethiopian":15101,"explores":15102,"noticing":15103,"##nton":15104,"discarded":15105,"convenience":15106,"hoffman":15107,"endowment":15108,"beasts":15109,"cartridge":15110,"mormon":15111,"paternal":15112,"probe":15113,"sleeves":15114,"interfere":15115,"lump":15116,"deadline":15117,"##rail":15118,"jenks":15119,"bulldogs":15120,"scrap":15121,"alternating":15122,"justified":15123,"reproductive":15124,"nam":15125,"seize":15126,"descending":15127,"secretariat":15128,"kirby":15129,"coupe":15130,"grouped":15131,"smash":15132,"panther":15133,"sedan":15134,"tapping":15135,"##18":15136,"lola":15137,"cheer":15138,"germanic":15139,"unfortunate":15140,"##eter":15141,"unrelated":15142,"##fan":15143,"subordinate":15144,"##sdale":15145,"suzanne":15146,"advertisement":15147,"##ility":15148,"horsepower":15149,"##lda":15150,"cautiously":15151,"discourse":15152,"luigi":15153,"##mans":15154,"##fields":15155,"noun":15156,"prevalent":15157,"mao":15158,"schneider":15159,"everett":15160,"surround":15161,"governorate":15162,"kira":15163,"##avia":15164,"westward":15165,"##take":15166,"misty":15167,"rails":15168,"sustainability":15169,"134":15170,"unused":15171,"##rating":15172,"packs":15173,"toast":15174,"unwilling":15175,"regulate":15176,"thy":15177,"suffrage":15178,"nile":15179,"awe":15180,"assam":15181,"definitions":15182,"travelers":15183,"affordable":15184,"##rb":15185,"conferred":15186,"sells":15187,"undefeated":15188,"beneficial":15189,"torso":15190,"basal":15191,"repeating":15192,"remixes":15193,"##pass":15194,"bahrain":15195,"cables":15196,"fang":15197,"##itated":15198,"excavated":15199,"numbering":15200,"statutory":15201,"##rey":15202,"deluxe":15203,"##lian":15204,"forested":15205,"ramirez":15206,"derbyshire":15207,"zeus":15208,"slamming":15209,"transfers":15210,"astronomer":15211,"banana":15212,"lottery":15213,"berg":15214,"histories":15215,"bamboo":15216,"##uchi":15217,"resurrection":15218,"posterior":15219,"bowls":15220,"vaguely":15221,"##thi":15222,"thou":15223,"preserving":15224,"tensed":15225,"offence":15226,"##inas":15227,"meyrick":15228,"callum":15229,"ridden":15230,"watt":15231,"langdon":15232,"tying":15233,"lowland":15234,"snorted":15235,"daring":15236,"truman":15237,"##hale":15238,"##girl":15239,"aura":15240,"overly":15241,"filing":15242,"weighing":15243,"goa":15244,"infections":15245,"philanthropist":15246,"saunders":15247,"eponymous":15248,"##owski":15249,"latitude":15250,"perspectives":15251,"reviewing":15252,"mets":15253,"commandant":15254,"radial":15255,"##kha":15256,"flashlight":15257,"reliability":15258,"koch":15259,"vowels":15260,"amazed":15261,"ada":15262,"elaine":15263,"supper":15264,"##rth":15265,"##encies":15266,"predator":15267,"debated":15268,"soviets":15269,"cola":15270,"##boards":15271,"##nah":15272,"compartment":15273,"crooked":15274,"arbitrary":15275,"fourteenth":15276,"##ctive":15277,"havana":15278,"majors":15279,"steelers":15280,"clips":15281,"profitable":15282,"ambush":15283,"exited":15284,"packers":15285,"##tile":15286,"nude":15287,"cracks":15288,"fungi":15289,"##е":15290,"limb":15291,"trousers":15292,"josie":15293,"shelby":15294,"tens":15295,"frederic":15296,"##ος":15297,"definite":15298,"smoothly":15299,"constellation":15300,"insult":15301,"baton":15302,"discs":15303,"lingering":15304,"##nco":15305,"conclusions":15306,"lent":15307,"staging":15308,"becker":15309,"grandpa":15310,"shaky":15311,"##tron":15312,"einstein":15313,"obstacles":15314,"sk":15315,"adverse":15316,"elle":15317,"economically":15318,"##moto":15319,"mccartney":15320,"thor":15321,"dismissal":15322,"motions":15323,"readings":15324,"nostrils":15325,"treatise":15326,"##pace":15327,"squeezing":15328,"evidently":15329,"prolonged":15330,"1783":15331,"venezuelan":15332,"je":15333,"marguerite":15334,"beirut":15335,"takeover":15336,"shareholders":15337,"##vent":15338,"denise":15339,"digit":15340,"airplay":15341,"norse":15342,"##bbling":15343,"imaginary":15344,"pills":15345,"hubert":15346,"blaze":15347,"vacated":15348,"eliminating":15349,"##ello":15350,"vine":15351,"mansfield":15352,"##tty":15353,"retrospective":15354,"barrow":15355,"borne":15356,"clutch":15357,"bail":15358,"forensic":15359,"weaving":15360,"##nett":15361,"##witz":15362,"desktop":15363,"citadel":15364,"promotions":15365,"worrying":15366,"dorset":15367,"ieee":15368,"subdivided":15369,"##iating":15370,"manned":15371,"expeditionary":15372,"pickup":15373,"synod":15374,"chuckle":15375,"185":15376,"barney":15377,"##rz":15378,"##ffin":15379,"functionality":15380,"karachi":15381,"litigation":15382,"meanings":15383,"uc":15384,"lick":15385,"turbo":15386,"anders":15387,"##ffed":15388,"execute":15389,"curl":15390,"oppose":15391,"ankles":15392,"typhoon":15393,"##د":15394,"##ache":15395,"##asia":15396,"linguistics":15397,"compassion":15398,"pressures":15399,"grazing":15400,"perfection":15401,"##iting":15402,"immunity":15403,"monopoly":15404,"muddy":15405,"backgrounds":15406,"136":15407,"namibia":15408,"francesca":15409,"monitors":15410,"attracting":15411,"stunt":15412,"tuition":15413,"##ии":15414,"vegetable":15415,"##mates":15416,"##quent":15417,"mgm":15418,"jen":15419,"complexes":15420,"forts":15421,"##ond":15422,"cellar":15423,"bites":15424,"seventeenth":15425,"royals":15426,"flemish":15427,"failures":15428,"mast":15429,"charities":15430,"##cular":15431,"peruvian":15432,"capitals":15433,"macmillan":15434,"ipswich":15435,"outward":15436,"frigate":15437,"postgraduate":15438,"folds":15439,"employing":15440,"##ouse":15441,"concurrently":15442,"fiery":15443,"##tai":15444,"contingent":15445,"nightmares":15446,"monumental":15447,"nicaragua":15448,"##kowski":15449,"lizard":15450,"mal":15451,"fielding":15452,"gig":15453,"reject":15454,"##pad":15455,"harding":15456,"##ipe":15457,"coastline":15458,"##cin":15459,"##nos":15460,"beethoven":15461,"humphrey":15462,"innovations":15463,"##tam":15464,"##nge":15465,"norris":15466,"doris":15467,"solicitor":15468,"huang":15469,"obey":15470,"141":15471,"##lc":15472,"niagara":15473,"##tton":15474,"shelves":15475,"aug":15476,"bourbon":15477,"curry":15478,"nightclub":15479,"specifications":15480,"hilton":15481,"##ndo":15482,"centennial":15483,"dispersed":15484,"worm":15485,"neglected":15486,"briggs":15487,"sm":15488,"font":15489,"kuala":15490,"uneasy":15491,"plc":15492,"##nstein":15493,"##bound":15494,"##aking":15495,"##burgh":15496,"awaiting":15497,"pronunciation":15498,"##bbed":15499,"##quest":15500,"eh":15501,"optimal":15502,"zhu":15503,"raped":15504,"greens":15505,"presided":15506,"brenda":15507,"worries":15508,"##life":15509,"venetian":15510,"marxist":15511,"turnout":15512,"##lius":15513,"refined":15514,"braced":15515,"sins":15516,"grasped":15517,"sunderland":15518,"nickel":15519,"speculated":15520,"lowell":15521,"cyrillic":15522,"communism":15523,"fundraising":15524,"resembling":15525,"colonists":15526,"mutant":15527,"freddie":15528,"usc":15529,"##mos":15530,"gratitude":15531,"##run":15532,"mural":15533,"##lous":15534,"chemist":15535,"wi":15536,"reminds":15537,"28th":15538,"steals":15539,"tess":15540,"pietro":15541,"##ingen":15542,"promoter":15543,"ri":15544,"microphone":15545,"honoured":15546,"rai":15547,"sant":15548,"##qui":15549,"feather":15550,"##nson":15551,"burlington":15552,"kurdish":15553,"terrorists":15554,"deborah":15555,"sickness":15556,"##wed":15557,"##eet":15558,"hazard":15559,"irritated":15560,"desperation":15561,"veil":15562,"clarity":15563,"##rik":15564,"jewels":15565,"xv":15566,"##gged":15567,"##ows":15568,"##cup":15569,"berkshire":15570,"unfair":15571,"mysteries":15572,"orchid":15573,"winced":15574,"exhaustion":15575,"renovations":15576,"stranded":15577,"obe":15578,"infinity":15579,"##nies":15580,"adapt":15581,"redevelopment":15582,"thanked":15583,"registry":15584,"olga":15585,"domingo":15586,"noir":15587,"tudor":15588,"ole":15589,"##atus":15590,"commenting":15591,"behaviors":15592,"##ais":15593,"crisp":15594,"pauline":15595,"probable":15596,"stirling":15597,"wigan":15598,"##bian":15599,"paralympics":15600,"panting":15601,"surpassed":15602,"##rew":15603,"luca":15604,"barred":15605,"pony":15606,"famed":15607,"##sters":15608,"cassandra":15609,"waiter":15610,"carolyn":15611,"exported":15612,"##orted":15613,"andres":15614,"destructive":15615,"deeds":15616,"jonah":15617,"castles":15618,"vacancy":15619,"suv":15620,"##glass":15621,"1788":15622,"orchard":15623,"yep":15624,"famine":15625,"belarusian":15626,"sprang":15627,"##forth":15628,"skinny":15629,"##mis":15630,"administrators":15631,"rotterdam":15632,"zambia":15633,"zhao":15634,"boiler":15635,"discoveries":15636,"##ride":15637,"##physics":15638,"lucius":15639,"disappointing":15640,"outreach":15641,"spoon":15642,"##frame":15643,"qualifications":15644,"unanimously":15645,"enjoys":15646,"regency":15647,"##iidae":15648,"stade":15649,"realism":15650,"veterinary":15651,"rodgers":15652,"dump":15653,"alain":15654,"chestnut":15655,"castile":15656,"censorship":15657,"rumble":15658,"gibbs":15659,"##itor":15660,"communion":15661,"reggae":15662,"inactivated":15663,"logs":15664,"loads":15665,"##houses":15666,"homosexual":15667,"##iano":15668,"ale":15669,"informs":15670,"##cas":15671,"phrases":15672,"plaster":15673,"linebacker":15674,"ambrose":15675,"kaiser":15676,"fascinated":15677,"850":15678,"limerick":15679,"recruitment":15680,"forge":15681,"mastered":15682,"##nding":15683,"leinster":15684,"rooted":15685,"threaten":15686,"##strom":15687,"borneo":15688,"##hes":15689,"suggestions":15690,"scholarships":15691,"propeller":15692,"documentaries":15693,"patronage":15694,"coats":15695,"constructing":15696,"invest":15697,"neurons":15698,"comet":15699,"entirety":15700,"shouts":15701,"identities":15702,"annoying":15703,"unchanged":15704,"wary":15705,"##antly":15706,"##ogy":15707,"neat":15708,"oversight":15709,"##kos":15710,"phillies":15711,"replay":15712,"constance":15713,"##kka":15714,"incarnation":15715,"humble":15716,"skies":15717,"minus":15718,"##acy":15719,"smithsonian":15720,"##chel":15721,"guerrilla":15722,"jar":15723,"cadets":15724,"##plate":15725,"surplus":15726,"audit":15727,"##aru":15728,"cracking":15729,"joanna":15730,"louisa":15731,"pacing":15732,"##lights":15733,"intentionally":15734,"##iri":15735,"diner":15736,"nwa":15737,"imprint":15738,"australians":15739,"tong":15740,"unprecedented":15741,"bunker":15742,"naive":15743,"specialists":15744,"ark":15745,"nichols":15746,"railing":15747,"leaked":15748,"pedal":15749,"##uka":15750,"shrub":15751,"longing":15752,"roofs":15753,"v8":15754,"captains":15755,"neural":15756,"tuned":15757,"##ntal":15758,"##jet":15759,"emission":15760,"medina":15761,"frantic":15762,"codex":15763,"definitive":15764,"sid":15765,"abolition":15766,"intensified":15767,"stocks":15768,"enrique":15769,"sustain":15770,"genoa":15771,"oxide":15772,"##written":15773,"clues":15774,"cha":15775,"##gers":15776,"tributaries":15777,"fragment":15778,"venom":15779,"##rity":15780,"##ente":15781,"##sca":15782,"muffled":15783,"vain":15784,"sire":15785,"laos":15786,"##ingly":15787,"##hana":15788,"hastily":15789,"snapping":15790,"surfaced":15791,"sentiment":15792,"motive":15793,"##oft":15794,"contests":15795,"approximate":15796,"mesa":15797,"luckily":15798,"dinosaur":15799,"exchanges":15800,"propelled":15801,"accord":15802,"bourne":15803,"relieve":15804,"tow":15805,"masks":15806,"offended":15807,"##ues":15808,"cynthia":15809,"##mmer":15810,"rains":15811,"bartender":15812,"zinc":15813,"reviewers":15814,"lois":15815,"##sai":15816,"legged":15817,"arrogant":15818,"rafe":15819,"rosie":15820,"comprise":15821,"handicap":15822,"blockade":15823,"inlet":15824,"lagoon":15825,"copied":15826,"drilling":15827,"shelley":15828,"petals":15829,"##inian":15830,"mandarin":15831,"obsolete":15832,"##inated":15833,"onward":15834,"arguably":15835,"productivity":15836,"cindy":15837,"praising":15838,"seldom":15839,"busch":15840,"discusses":15841,"raleigh":15842,"shortage":15843,"ranged":15844,"stanton":15845,"encouragement":15846,"firstly":15847,"conceded":15848,"overs":15849,"temporal":15850,"##uke":15851,"cbe":15852,"##bos":15853,"woo":15854,"certainty":15855,"pumps":15856,"##pton":15857,"stalked":15858,"##uli":15859,"lizzie":15860,"periodic":15861,"thieves":15862,"weaker":15863,"##night":15864,"gases":15865,"shoving":15866,"chooses":15867,"wc":15868,"##chemical":15869,"prompting":15870,"weights":15871,"##kill":15872,"robust":15873,"flanked":15874,"sticky":15875,"hu":15876,"tuberculosis":15877,"##eb":15878,"##eal":15879,"christchurch":15880,"resembled":15881,"wallet":15882,"reese":15883,"inappropriate":15884,"pictured":15885,"distract":15886,"fixing":15887,"fiddle":15888,"giggled":15889,"burger":15890,"heirs":15891,"hairy":15892,"mechanic":15893,"torque":15894,"apache":15895,"obsessed":15896,"chiefly":15897,"cheng":15898,"logging":15899,"##tag":15900,"extracted":15901,"meaningful":15902,"numb":15903,"##vsky":15904,"gloucestershire":15905,"reminding":15906,"##bay":15907,"unite":15908,"##lit":15909,"breeds":15910,"diminished":15911,"clown":15912,"glove":15913,"1860s":15914,"##ن":15915,"##ug":15916,"archibald":15917,"focal":15918,"freelance":15919,"sliced":15920,"depiction":15921,"##yk":15922,"organism":15923,"switches":15924,"sights":15925,"stray":15926,"crawling":15927,"##ril":15928,"lever":15929,"leningrad":15930,"interpretations":15931,"loops":15932,"anytime":15933,"reel":15934,"alicia":15935,"delighted":15936,"##ech":15937,"inhaled":15938,"xiv":15939,"suitcase":15940,"bernie":15941,"vega":15942,"licenses":15943,"northampton":15944,"exclusion":15945,"induction":15946,"monasteries":15947,"racecourse":15948,"homosexuality":15949,"##right":15950,"##sfield":15951,"##rky":15952,"dimitri":15953,"michele":15954,"alternatives":15955,"ions":15956,"commentators":15957,"genuinely":15958,"objected":15959,"pork":15960,"hospitality":15961,"fencing":15962,"stephan":15963,"warships":15964,"peripheral":15965,"wit":15966,"drunken":15967,"wrinkled":15968,"quentin":15969,"spends":15970,"departing":15971,"chung":15972,"numerical":15973,"spokesperson":15974,"##zone":15975,"johannesburg":15976,"caliber":15977,"killers":15978,"##udge":15979,"assumes":15980,"neatly":15981,"demographic":15982,"abigail":15983,"bloc":15984,"##vel":15985,"mounting":15986,"##lain":15987,"bentley":15988,"slightest":15989,"xu":15990,"recipients":15991,"##jk":15992,"merlin":15993,"##writer":15994,"seniors":15995,"prisons":15996,"blinking":15997,"hindwings":15998,"flickered":15999,"kappa":16000,"##hel":16001,"80s":16002,"strengthening":16003,"appealing":16004,"brewing":16005,"gypsy":16006,"mali":16007,"lashes":16008,"hulk":16009,"unpleasant":16010,"harassment":16011,"bio":16012,"treaties":16013,"predict":16014,"instrumentation":16015,"pulp":16016,"troupe":16017,"boiling":16018,"mantle":16019,"##ffe":16020,"ins":16021,"##vn":16022,"dividing":16023,"handles":16024,"verbs":16025,"##onal":16026,"coconut":16027,"senegal":16028,"340":16029,"thorough":16030,"gum":16031,"momentarily":16032,"##sto":16033,"cocaine":16034,"panicked":16035,"destined":16036,"##turing":16037,"teatro":16038,"denying":16039,"weary":16040,"captained":16041,"mans":16042,"##hawks":16043,"##code":16044,"wakefield":16045,"bollywood":16046,"thankfully":16047,"##16":16048,"cyril":16049,"##wu":16050,"amendments":16051,"##bahn":16052,"consultation":16053,"stud":16054,"reflections":16055,"kindness":16056,"1787":16057,"internally":16058,"##ovo":16059,"tex":16060,"mosaic":16061,"distribute":16062,"paddy":16063,"seeming":16064,"143":16065,"##hic":16066,"piers":16067,"##15":16068,"##mura":16069,"##verse":16070,"popularly":16071,"winger":16072,"kang":16073,"sentinel":16074,"mccoy":16075,"##anza":16076,"covenant":16077,"##bag":16078,"verge":16079,"fireworks":16080,"suppress":16081,"thrilled":16082,"dominate":16083,"##jar":16084,"swansea":16085,"##60":16086,"142":16087,"reconciliation":16088,"##ndi":16089,"stiffened":16090,"cue":16091,"dorian":16092,"##uf":16093,"damascus":16094,"amor":16095,"ida":16096,"foremost":16097,"##aga":16098,"porsche":16099,"unseen":16100,"dir":16101,"##had":16102,"##azi":16103,"stony":16104,"lexi":16105,"melodies":16106,"##nko":16107,"angular":16108,"integer":16109,"podcast":16110,"ants":16111,"inherent":16112,"jaws":16113,"justify":16114,"persona":16115,"##olved":16116,"josephine":16117,"##nr":16118,"##ressed":16119,"customary":16120,"flashes":16121,"gala":16122,"cyrus":16123,"glaring":16124,"backyard":16125,"ariel":16126,"physiology":16127,"greenland":16128,"html":16129,"stir":16130,"avon":16131,"atletico":16132,"finch":16133,"methodology":16134,"ked":16135,"##lent":16136,"mas":16137,"catholicism":16138,"townsend":16139,"branding":16140,"quincy":16141,"fits":16142,"containers":16143,"1777":16144,"ashore":16145,"aragon":16146,"##19":16147,"forearm":16148,"poisoning":16149,"##sd":16150,"adopting":16151,"conquer":16152,"grinding":16153,"amnesty":16154,"keller":16155,"finances":16156,"evaluate":16157,"forged":16158,"lankan":16159,"instincts":16160,"##uto":16161,"guam":16162,"bosnian":16163,"photographed":16164,"workplace":16165,"desirable":16166,"protector":16167,"##dog":16168,"allocation":16169,"intently":16170,"encourages":16171,"willy":16172,"##sten":16173,"bodyguard":16174,"electro":16175,"brighter":16176,"##ν":16177,"bihar":16178,"##chev":16179,"lasts":16180,"opener":16181,"amphibious":16182,"sal":16183,"verde":16184,"arte":16185,"##cope":16186,"captivity":16187,"vocabulary":16188,"yields":16189,"##tted":16190,"agreeing":16191,"desmond":16192,"pioneered":16193,"##chus":16194,"strap":16195,"campaigned":16196,"railroads":16197,"##ович":16198,"emblem":16199,"##dre":16200,"stormed":16201,"501":16202,"##ulous":16203,"marijuana":16204,"northumberland":16205,"##gn":16206,"##nath":16207,"bowen":16208,"landmarks":16209,"beaumont":16210,"##qua":16211,"danube":16212,"##bler":16213,"attorneys":16214,"th":16215,"ge":16216,"flyers":16217,"critique":16218,"villains":16219,"cass":16220,"mutation":16221,"acc":16222,"##0s":16223,"colombo":16224,"mckay":16225,"motif":16226,"sampling":16227,"concluding":16228,"syndicate":16229,"##rell":16230,"neon":16231,"stables":16232,"ds":16233,"warnings":16234,"clint":16235,"mourning":16236,"wilkinson":16237,"##tated":16238,"merrill":16239,"leopard":16240,"evenings":16241,"exhaled":16242,"emil":16243,"sonia":16244,"ezra":16245,"discrete":16246,"stove":16247,"farrell":16248,"fifteenth":16249,"prescribed":16250,"superhero":16251,"##rier":16252,"worms":16253,"helm":16254,"wren":16255,"##duction":16256,"##hc":16257,"expo":16258,"##rator":16259,"hq":16260,"unfamiliar":16261,"antony":16262,"prevents":16263,"acceleration":16264,"fiercely":16265,"mari":16266,"painfully":16267,"calculations":16268,"cheaper":16269,"ign":16270,"clifton":16271,"irvine":16272,"davenport":16273,"mozambique":16274,"##np":16275,"pierced":16276,"##evich":16277,"wonders":16278,"##wig":16279,"##cate":16280,"##iling":16281,"crusade":16282,"ware":16283,"##uel":16284,"enzymes":16285,"reasonably":16286,"mls":16287,"##coe":16288,"mater":16289,"ambition":16290,"bunny":16291,"eliot":16292,"kernel":16293,"##fin":16294,"asphalt":16295,"headmaster":16296,"torah":16297,"aden":16298,"lush":16299,"pins":16300,"waived":16301,"##care":16302,"##yas":16303,"joao":16304,"substrate":16305,"enforce":16306,"##grad":16307,"##ules":16308,"alvarez":16309,"selections":16310,"epidemic":16311,"tempted":16312,"##bit":16313,"bremen":16314,"translates":16315,"ensured":16316,"waterfront":16317,"29th":16318,"forrest":16319,"manny":16320,"malone":16321,"kramer":16322,"reigning":16323,"cookies":16324,"simpler":16325,"absorption":16326,"205":16327,"engraved":16328,"##ffy":16329,"evaluated":16330,"1778":16331,"haze":16332,"146":16333,"comforting":16334,"crossover":16335,"##abe":16336,"thorn":16337,"##rift":16338,"##imo":16339,"##pop":16340,"suppression":16341,"fatigue":16342,"cutter":16343,"##tr":16344,"201":16345,"wurttemberg":16346,"##orf":16347,"enforced":16348,"hovering":16349,"proprietary":16350,"gb":16351,"samurai":16352,"syllable":16353,"ascent":16354,"lacey":16355,"tick":16356,"lars":16357,"tractor":16358,"merchandise":16359,"rep":16360,"bouncing":16361,"defendants":16362,"##yre":16363,"huntington":16364,"##ground":16365,"##oko":16366,"standardized":16367,"##hor":16368,"##hima":16369,"assassinated":16370,"nu":16371,"predecessors":16372,"rainy":16373,"liar":16374,"assurance":16375,"lyrical":16376,"##uga":16377,"secondly":16378,"flattened":16379,"ios":16380,"parameter":16381,"undercover":16382,"##mity":16383,"bordeaux":16384,"punish":16385,"ridges":16386,"markers":16387,"exodus":16388,"inactive":16389,"hesitate":16390,"debbie":16391,"nyc":16392,"pledge":16393,"savoy":16394,"nagar":16395,"offset":16396,"organist":16397,"##tium":16398,"hesse":16399,"marin":16400,"converting":16401,"##iver":16402,"diagram":16403,"propulsion":16404,"pu":16405,"validity":16406,"reverted":16407,"supportive":16408,"##dc":16409,"ministries":16410,"clans":16411,"responds":16412,"proclamation":16413,"##inae":16414,"##ø":16415,"##rea":16416,"ein":16417,"pleading":16418,"patriot":16419,"sf":16420,"birch":16421,"islanders":16422,"strauss":16423,"hates":16424,"##dh":16425,"brandenburg":16426,"concession":16427,"rd":16428,"##ob":16429,"1900s":16430,"killings":16431,"textbook":16432,"antiquity":16433,"cinematography":16434,"wharf":16435,"embarrassing":16436,"setup":16437,"creed":16438,"farmland":16439,"inequality":16440,"centred":16441,"signatures":16442,"fallon":16443,"370":16444,"##ingham":16445,"##uts":16446,"ceylon":16447,"gazing":16448,"directive":16449,"laurie":16450,"##tern":16451,"globally":16452,"##uated":16453,"##dent":16454,"allah":16455,"excavation":16456,"threads":16457,"##cross":16458,"148":16459,"frantically":16460,"icc":16461,"utilize":16462,"determines":16463,"respiratory":16464,"thoughtful":16465,"receptions":16466,"##dicate":16467,"merging":16468,"chandra":16469,"seine":16470,"147":16471,"builders":16472,"builds":16473,"diagnostic":16474,"dev":16475,"visibility":16476,"goddamn":16477,"analyses":16478,"dhaka":16479,"cho":16480,"proves":16481,"chancel":16482,"concurrent":16483,"curiously":16484,"canadians":16485,"pumped":16486,"restoring":16487,"1850s":16488,"turtles":16489,"jaguar":16490,"sinister":16491,"spinal":16492,"traction":16493,"declan":16494,"vows":16495,"1784":16496,"glowed":16497,"capitalism":16498,"swirling":16499,"install":16500,"universidad":16501,"##lder":16502,"##oat":16503,"soloist":16504,"##genic":16505,"##oor":16506,"coincidence":16507,"beginnings":16508,"nissan":16509,"dip":16510,"resorts":16511,"caucasus":16512,"combustion":16513,"infectious":16514,"##eno":16515,"pigeon":16516,"serpent":16517,"##itating":16518,"conclude":16519,"masked":16520,"salad":16521,"jew":16522,"##gr":16523,"surreal":16524,"toni":16525,"##wc":16526,"harmonica":16527,"151":16528,"##gins":16529,"##etic":16530,"##coat":16531,"fishermen":16532,"intending":16533,"bravery":16534,"##wave":16535,"klaus":16536,"titan":16537,"wembley":16538,"taiwanese":16539,"ransom":16540,"40th":16541,"incorrect":16542,"hussein":16543,"eyelids":16544,"jp":16545,"cooke":16546,"dramas":16547,"utilities":16548,"##etta":16549,"##print":16550,"eisenhower":16551,"principally":16552,"granada":16553,"lana":16554,"##rak":16555,"openings":16556,"concord":16557,"##bl":16558,"bethany":16559,"connie":16560,"morality":16561,"sega":16562,"##mons":16563,"##nard":16564,"earnings":16565,"##kara":16566,"##cine":16567,"wii":16568,"communes":16569,"##rel":16570,"coma":16571,"composing":16572,"softened":16573,"severed":16574,"grapes":16575,"##17":16576,"nguyen":16577,"analyzed":16578,"warlord":16579,"hubbard":16580,"heavenly":16581,"behave":16582,"slovenian":16583,"##hit":16584,"##ony":16585,"hailed":16586,"filmmakers":16587,"trance":16588,"caldwell":16589,"skye":16590,"unrest":16591,"coward":16592,"likelihood":16593,"##aging":16594,"bern":16595,"sci":16596,"taliban":16597,"honolulu":16598,"propose":16599,"##wang":16600,"1700":16601,"browser":16602,"imagining":16603,"cobra":16604,"contributes":16605,"dukes":16606,"instinctively":16607,"conan":16608,"violinist":16609,"##ores":16610,"accessories":16611,"gradual":16612,"##amp":16613,"quotes":16614,"sioux":16615,"##dating":16616,"undertake":16617,"intercepted":16618,"sparkling":16619,"compressed":16620,"139":16621,"fungus":16622,"tombs":16623,"haley":16624,"imposing":16625,"rests":16626,"degradation":16627,"lincolnshire":16628,"retailers":16629,"wetlands":16630,"tulsa":16631,"distributor":16632,"dungeon":16633,"nun":16634,"greenhouse":16635,"convey":16636,"atlantis":16637,"aft":16638,"exits":16639,"oman":16640,"dresser":16641,"lyons":16642,"##sti":16643,"joking":16644,"eddy":16645,"judgement":16646,"omitted":16647,"digits":16648,"##cts":16649,"##game":16650,"juniors":16651,"##rae":16652,"cents":16653,"stricken":16654,"une":16655,"##ngo":16656,"wizards":16657,"weir":16658,"breton":16659,"nan":16660,"technician":16661,"fibers":16662,"liking":16663,"royalty":16664,"##cca":16665,"154":16666,"persia":16667,"terribly":16668,"magician":16669,"##rable":16670,"##unt":16671,"vance":16672,"cafeteria":16673,"booker":16674,"camille":16675,"warmer":16676,"##static":16677,"consume":16678,"cavern":16679,"gaps":16680,"compass":16681,"contemporaries":16682,"foyer":16683,"soothing":16684,"graveyard":16685,"maj":16686,"plunged":16687,"blush":16688,"##wear":16689,"cascade":16690,"demonstrates":16691,"ordinance":16692,"##nov":16693,"boyle":16694,"##lana":16695,"rockefeller":16696,"shaken":16697,"banjo":16698,"izzy":16699,"##ense":16700,"breathless":16701,"vines":16702,"##32":16703,"##eman":16704,"alterations":16705,"chromosome":16706,"dwellings":16707,"feudal":16708,"mole":16709,"153":16710,"catalonia":16711,"relics":16712,"tenant":16713,"mandated":16714,"##fm":16715,"fridge":16716,"hats":16717,"honesty":16718,"patented":16719,"raul":16720,"heap":16721,"cruisers":16722,"accusing":16723,"enlightenment":16724,"infants":16725,"wherein":16726,"chatham":16727,"contractors":16728,"zen":16729,"affinity":16730,"hc":16731,"osborne":16732,"piston":16733,"156":16734,"traps":16735,"maturity":16736,"##rana":16737,"lagos":16738,"##zal":16739,"peering":16740,"##nay":16741,"attendant":16742,"dealers":16743,"protocols":16744,"subset":16745,"prospects":16746,"biographical":16747,"##cre":16748,"artery":16749,"##zers":16750,"insignia":16751,"nuns":16752,"endured":16753,"##eration":16754,"recommend":16755,"schwartz":16756,"serbs":16757,"berger":16758,"cromwell":16759,"crossroads":16760,"##ctor":16761,"enduring":16762,"clasped":16763,"grounded":16764,"##bine":16765,"marseille":16766,"twitched":16767,"abel":16768,"choke":16769,"https":16770,"catalyst":16771,"moldova":16772,"italians":16773,"##tist":16774,"disastrous":16775,"wee":16776,"##oured":16777,"##nti":16778,"wwf":16779,"nope":16780,"##piration":16781,"##asa":16782,"expresses":16783,"thumbs":16784,"167":16785,"##nza":16786,"coca":16787,"1781":16788,"cheating":16789,"##ption":16790,"skipped":16791,"sensory":16792,"heidelberg":16793,"spies":16794,"satan":16795,"dangers":16796,"semifinal":16797,"202":16798,"bohemia":16799,"whitish":16800,"confusing":16801,"shipbuilding":16802,"relies":16803,"surgeons":16804,"landings":16805,"ravi":16806,"baku":16807,"moor":16808,"suffix":16809,"alejandro":16810,"##yana":16811,"litre":16812,"upheld":16813,"##unk":16814,"rajasthan":16815,"##rek":16816,"coaster":16817,"insists":16818,"posture":16819,"scenarios":16820,"etienne":16821,"favoured":16822,"appoint":16823,"transgender":16824,"elephants":16825,"poked":16826,"greenwood":16827,"defences":16828,"fulfilled":16829,"militant":16830,"somali":16831,"1758":16832,"chalk":16833,"potent":16834,"##ucci":16835,"migrants":16836,"wink":16837,"assistants":16838,"nos":16839,"restriction":16840,"activism":16841,"niger":16842,"##ario":16843,"colon":16844,"shaun":16845,"##sat":16846,"daphne":16847,"##erated":16848,"swam":16849,"congregations":16850,"reprise":16851,"considerations":16852,"magnet":16853,"playable":16854,"xvi":16855,"##р":16856,"overthrow":16857,"tobias":16858,"knob":16859,"chavez":16860,"coding":16861,"##mers":16862,"propped":16863,"katrina":16864,"orient":16865,"newcomer":16866,"##suke":16867,"temperate":16868,"##pool":16869,"farmhouse":16870,"interrogation":16871,"##vd":16872,"committing":16873,"##vert":16874,"forthcoming":16875,"strawberry":16876,"joaquin":16877,"macau":16878,"ponds":16879,"shocking":16880,"siberia":16881,"##cellular":16882,"chant":16883,"contributors":16884,"##nant":16885,"##ologists":16886,"sped":16887,"absorb":16888,"hail":16889,"1782":16890,"spared":16891,"##hore":16892,"barbados":16893,"karate":16894,"opus":16895,"originates":16896,"saul":16897,"##xie":16898,"evergreen":16899,"leaped":16900,"##rock":16901,"correlation":16902,"exaggerated":16903,"weekday":16904,"unification":16905,"bump":16906,"tracing":16907,"brig":16908,"afb":16909,"pathways":16910,"utilizing":16911,"##ners":16912,"mod":16913,"mb":16914,"disturbance":16915,"kneeling":16916,"##stad":16917,"##guchi":16918,"100th":16919,"pune":16920,"##thy":16921,"decreasing":16922,"168":16923,"manipulation":16924,"miriam":16925,"academia":16926,"ecosystem":16927,"occupational":16928,"rbi":16929,"##lem":16930,"rift":16931,"##14":16932,"rotary":16933,"stacked":16934,"incorporation":16935,"awakening":16936,"generators":16937,"guerrero":16938,"racist":16939,"##omy":16940,"cyber":16941,"derivatives":16942,"culminated":16943,"allie":16944,"annals":16945,"panzer":16946,"sainte":16947,"wikipedia":16948,"pops":16949,"zu":16950,"austro":16951,"##vate":16952,"algerian":16953,"politely":16954,"nicholson":16955,"mornings":16956,"educate":16957,"tastes":16958,"thrill":16959,"dartmouth":16960,"##gating":16961,"db":16962,"##jee":16963,"regan":16964,"differing":16965,"concentrating":16966,"choreography":16967,"divinity":16968,"##media":16969,"pledged":16970,"alexandre":16971,"routing":16972,"gregor":16973,"madeline":16974,"##idal":16975,"apocalypse":16976,"##hora":16977,"gunfire":16978,"culminating":16979,"elves":16980,"fined":16981,"liang":16982,"lam":16983,"programmed":16984,"tar":16985,"guessing":16986,"transparency":16987,"gabrielle":16988,"##gna":16989,"cancellation":16990,"flexibility":16991,"##lining":16992,"accession":16993,"shea":16994,"stronghold":16995,"nets":16996,"specializes":16997,"##rgan":16998,"abused":16999,"hasan":17000,"sgt":17001,"ling":17002,"exceeding":17003,"##₄":17004,"admiration":17005,"supermarket":17006,"##ark":17007,"photographers":17008,"specialised":17009,"tilt":17010,"resonance":17011,"hmm":17012,"perfume":17013,"380":17014,"sami":17015,"threatens":17016,"garland":17017,"botany":17018,"guarding":17019,"boiled":17020,"greet":17021,"puppy":17022,"russo":17023,"supplier":17024,"wilmington":17025,"vibrant":17026,"vijay":17027,"##bius":17028,"paralympic":17029,"grumbled":17030,"paige":17031,"faa":17032,"licking":17033,"margins":17034,"hurricanes":17035,"##gong":17036,"fest":17037,"grenade":17038,"ripping":17039,"##uz":17040,"counseling":17041,"weigh":17042,"##sian":17043,"needles":17044,"wiltshire":17045,"edison":17046,"costly":17047,"##not":17048,"fulton":17049,"tramway":17050,"redesigned":17051,"staffordshire":17052,"cache":17053,"gasping":17054,"watkins":17055,"sleepy":17056,"candidacy":17057,"##group":17058,"monkeys":17059,"timeline":17060,"throbbing":17061,"##bid":17062,"##sos":17063,"berth":17064,"uzbekistan":17065,"vanderbilt":17066,"bothering":17067,"overturned":17068,"ballots":17069,"gem":17070,"##iger":17071,"sunglasses":17072,"subscribers":17073,"hooker":17074,"compelling":17075,"ang":17076,"exceptionally":17077,"saloon":17078,"stab":17079,"##rdi":17080,"carla":17081,"terrifying":17082,"rom":17083,"##vision":17084,"coil":17085,"##oids":17086,"satisfying":17087,"vendors":17088,"31st":17089,"mackay":17090,"deities":17091,"overlooked":17092,"ambient":17093,"bahamas":17094,"felipe":17095,"olympia":17096,"whirled":17097,"botanist":17098,"advertised":17099,"tugging":17100,"##dden":17101,"disciples":17102,"morales":17103,"unionist":17104,"rites":17105,"foley":17106,"morse":17107,"motives":17108,"creepy":17109,"##₀":17110,"soo":17111,"##sz":17112,"bargain":17113,"highness":17114,"frightening":17115,"turnpike":17116,"tory":17117,"reorganization":17118,"##cer":17119,"depict":17120,"biographer":17121,"##walk":17122,"unopposed":17123,"manifesto":17124,"##gles":17125,"institut":17126,"emile":17127,"accidental":17128,"kapoor":17129,"##dam":17130,"kilkenny":17131,"cortex":17132,"lively":17133,"##13":17134,"romanesque":17135,"jain":17136,"shan":17137,"cannons":17138,"##ood":17139,"##ske":17140,"petrol":17141,"echoing":17142,"amalgamated":17143,"disappears":17144,"cautious":17145,"proposes":17146,"sanctions":17147,"trenton":17148,"##ر":17149,"flotilla":17150,"aus":17151,"contempt":17152,"tor":17153,"canary":17154,"cote":17155,"theirs":17156,"##hun":17157,"conceptual":17158,"deleted":17159,"fascinating":17160,"paso":17161,"blazing":17162,"elf":17163,"honourable":17164,"hutchinson":17165,"##eiro":17166,"##outh":17167,"##zin":17168,"surveyor":17169,"tee":17170,"amidst":17171,"wooded":17172,"reissue":17173,"intro":17174,"##ono":17175,"cobb":17176,"shelters":17177,"newsletter":17178,"hanson":17179,"brace":17180,"encoding":17181,"confiscated":17182,"dem":17183,"caravan":17184,"marino":17185,"scroll":17186,"melodic":17187,"cows":17188,"imam":17189,"##adi":17190,"##aneous":17191,"northward":17192,"searches":17193,"biodiversity":17194,"cora":17195,"310":17196,"roaring":17197,"##bers":17198,"connell":17199,"theologian":17200,"halo":17201,"compose":17202,"pathetic":17203,"unmarried":17204,"dynamo":17205,"##oot":17206,"az":17207,"calculation":17208,"toulouse":17209,"deserves":17210,"humour":17211,"nr":17212,"forgiveness":17213,"tam":17214,"undergone":17215,"martyr":17216,"pamela":17217,"myths":17218,"whore":17219,"counselor":17220,"hicks":17221,"290":17222,"heavens":17223,"battleship":17224,"electromagnetic":17225,"##bbs":17226,"stellar":17227,"establishments":17228,"presley":17229,"hopped":17230,"##chin":17231,"temptation":17232,"90s":17233,"wills":17234,"nas":17235,"##yuan":17236,"nhs":17237,"##nya":17238,"seminars":17239,"##yev":17240,"adaptations":17241,"gong":17242,"asher":17243,"lex":17244,"indicator":17245,"sikh":17246,"tobago":17247,"cites":17248,"goin":17249,"##yte":17250,"satirical":17251,"##gies":17252,"characterised":17253,"correspond":17254,"bubbles":17255,"lure":17256,"participates":17257,"##vid":17258,"eruption":17259,"skate":17260,"therapeutic":17261,"1785":17262,"canals":17263,"wholesale":17264,"defaulted":17265,"sac":17266,"460":17267,"petit":17268,"##zzled":17269,"virgil":17270,"leak":17271,"ravens":17272,"256":17273,"portraying":17274,"##yx":17275,"ghetto":17276,"creators":17277,"dams":17278,"portray":17279,"vicente":17280,"##rington":17281,"fae":17282,"namesake":17283,"bounty":17284,"##arium":17285,"joachim":17286,"##ota":17287,"##iser":17288,"aforementioned":17289,"axle":17290,"snout":17291,"depended":17292,"dismantled":17293,"reuben":17294,"480":17295,"##ibly":17296,"gallagher":17297,"##lau":17298,"##pd":17299,"earnest":17300,"##ieu":17301,"##iary":17302,"inflicted":17303,"objections":17304,"##llar":17305,"asa":17306,"gritted":17307,"##athy":17308,"jericho":17309,"##sea":17310,"##was":17311,"flick":17312,"underside":17313,"ceramics":17314,"undead":17315,"substituted":17316,"195":17317,"eastward":17318,"undoubtedly":17319,"wheeled":17320,"chimney":17321,"##iche":17322,"guinness":17323,"cb":17324,"##ager":17325,"siding":17326,"##bell":17327,"traitor":17328,"baptiste":17329,"disguised":17330,"inauguration":17331,"149":17332,"tipperary":17333,"choreographer":17334,"perched":17335,"warmed":17336,"stationary":17337,"eco":17338,"##ike":17339,"##ntes":17340,"bacterial":17341,"##aurus":17342,"flores":17343,"phosphate":17344,"##core":17345,"attacker":17346,"invaders":17347,"alvin":17348,"intersects":17349,"a1":17350,"indirectly":17351,"immigrated":17352,"businessmen":17353,"cornelius":17354,"valves":17355,"narrated":17356,"pill":17357,"sober":17358,"ul":17359,"nationale":17360,"monastic":17361,"applicants":17362,"scenery":17363,"##jack":17364,"161":17365,"motifs":17366,"constitutes":17367,"cpu":17368,"##osh":17369,"jurisdictions":17370,"sd":17371,"tuning":17372,"irritation":17373,"woven":17374,"##uddin":17375,"fertility":17376,"gao":17377,"##erie":17378,"antagonist":17379,"impatient":17380,"glacial":17381,"hides":17382,"boarded":17383,"denominations":17384,"interception":17385,"##jas":17386,"cookie":17387,"nicola":17388,"##tee":17389,"algebraic":17390,"marquess":17391,"bahn":17392,"parole":17393,"buyers":17394,"bait":17395,"turbines":17396,"paperwork":17397,"bestowed":17398,"natasha":17399,"renee":17400,"oceans":17401,"purchases":17402,"157":17403,"vaccine":17404,"215":17405,"##tock":17406,"fixtures":17407,"playhouse":17408,"integrate":17409,"jai":17410,"oswald":17411,"intellectuals":17412,"##cky":17413,"booked":17414,"nests":17415,"mortimer":17416,"##isi":17417,"obsession":17418,"sept":17419,"##gler":17420,"##sum":17421,"440":17422,"scrutiny":17423,"simultaneous":17424,"squinted":17425,"##shin":17426,"collects":17427,"oven":17428,"shankar":17429,"penned":17430,"remarkably":17431,"##я":17432,"slips":17433,"luggage":17434,"spectral":17435,"1786":17436,"collaborations":17437,"louie":17438,"consolidation":17439,"##ailed":17440,"##ivating":17441,"420":17442,"hoover":17443,"blackpool":17444,"harness":17445,"ignition":17446,"vest":17447,"tails":17448,"belmont":17449,"mongol":17450,"skinner":17451,"##nae":17452,"visually":17453,"mage":17454,"derry":17455,"##tism":17456,"##unce":17457,"stevie":17458,"transitional":17459,"##rdy":17460,"redskins":17461,"drying":17462,"prep":17463,"prospective":17464,"##21":17465,"annoyance":17466,"oversee":17467,"##loaded":17468,"fills":17469,"##books":17470,"##iki":17471,"announces":17472,"fda":17473,"scowled":17474,"respects":17475,"prasad":17476,"mystic":17477,"tucson":17478,"##vale":17479,"revue":17480,"springer":17481,"bankrupt":17482,"1772":17483,"aristotle":17484,"salvatore":17485,"habsburg":17486,"##geny":17487,"dal":17488,"natal":17489,"nut":17490,"pod":17491,"chewing":17492,"darts":17493,"moroccan":17494,"walkover":17495,"rosario":17496,"lenin":17497,"punjabi":17498,"##ße":17499,"grossed":17500,"scattering":17501,"wired":17502,"invasive":17503,"hui":17504,"polynomial":17505,"corridors":17506,"wakes":17507,"gina":17508,"portrays":17509,"##cratic":17510,"arid":17511,"retreating":17512,"erich":17513,"irwin":17514,"sniper":17515,"##dha":17516,"linen":17517,"lindsey":17518,"maneuver":17519,"butch":17520,"shutting":17521,"socio":17522,"bounce":17523,"commemorative":17524,"postseason":17525,"jeremiah":17526,"pines":17527,"275":17528,"mystical":17529,"beads":17530,"bp":17531,"abbas":17532,"furnace":17533,"bidding":17534,"consulted":17535,"assaulted":17536,"empirical":17537,"rubble":17538,"enclosure":17539,"sob":17540,"weakly":17541,"cancel":17542,"polly":17543,"yielded":17544,"##emann":17545,"curly":17546,"prediction":17547,"battered":17548,"70s":17549,"vhs":17550,"jacqueline":17551,"render":17552,"sails":17553,"barked":17554,"detailing":17555,"grayson":17556,"riga":17557,"sloane":17558,"raging":17559,"##yah":17560,"herbs":17561,"bravo":17562,"##athlon":17563,"alloy":17564,"giggle":17565,"imminent":17566,"suffers":17567,"assumptions":17568,"waltz":17569,"##itate":17570,"accomplishments":17571,"##ited":17572,"bathing":17573,"remixed":17574,"deception":17575,"prefix":17576,"##emia":17577,"deepest":17578,"##tier":17579,"##eis":17580,"balkan":17581,"frogs":17582,"##rong":17583,"slab":17584,"##pate":17585,"philosophers":17586,"peterborough":17587,"grains":17588,"imports":17589,"dickinson":17590,"rwanda":17591,"##atics":17592,"1774":17593,"dirk":17594,"lan":17595,"tablets":17596,"##rove":17597,"clone":17598,"##rice":17599,"caretaker":17600,"hostilities":17601,"mclean":17602,"##gre":17603,"regimental":17604,"treasures":17605,"norms":17606,"impose":17607,"tsar":17608,"tango":17609,"diplomacy":17610,"variously":17611,"complain":17612,"192":17613,"recognise":17614,"arrests":17615,"1779":17616,"celestial":17617,"pulitzer":17618,"##dus":17619,"bing":17620,"libretto":17621,"##moor":17622,"adele":17623,"splash":17624,"##rite":17625,"expectation":17626,"lds":17627,"confronts":17628,"##izer":17629,"spontaneous":17630,"harmful":17631,"wedge":17632,"entrepreneurs":17633,"buyer":17634,"##ope":17635,"bilingual":17636,"translate":17637,"rugged":17638,"conner":17639,"circulated":17640,"uae":17641,"eaton":17642,"##gra":17643,"##zzle":17644,"lingered":17645,"lockheed":17646,"vishnu":17647,"reelection":17648,"alonso":17649,"##oom":17650,"joints":17651,"yankee":17652,"headline":17653,"cooperate":17654,"heinz":17655,"laureate":17656,"invading":17657,"##sford":17658,"echoes":17659,"scandinavian":17660,"##dham":17661,"hugging":17662,"vitamin":17663,"salute":17664,"micah":17665,"hind":17666,"trader":17667,"##sper":17668,"radioactive":17669,"##ndra":17670,"militants":17671,"poisoned":17672,"ratified":17673,"remark":17674,"campeonato":17675,"deprived":17676,"wander":17677,"prop":17678,"##dong":17679,"outlook":17680,"##tani":17681,"##rix":17682,"##eye":17683,"chiang":17684,"darcy":17685,"##oping":17686,"mandolin":17687,"spice":17688,"statesman":17689,"babylon":17690,"182":17691,"walled":17692,"forgetting":17693,"afro":17694,"##cap":17695,"158":17696,"giorgio":17697,"buffer":17698,"##polis":17699,"planetary":17700,"##gis":17701,"overlap":17702,"terminals":17703,"kinda":17704,"centenary":17705,"##bir":17706,"arising":17707,"manipulate":17708,"elm":17709,"ke":17710,"1770":17711,"ak":17712,"##tad":17713,"chrysler":17714,"mapped":17715,"moose":17716,"pomeranian":17717,"quad":17718,"macarthur":17719,"assemblies":17720,"shoreline":17721,"recalls":17722,"stratford":17723,"##rted":17724,"noticeable":17725,"##evic":17726,"imp":17727,"##rita":17728,"##sque":17729,"accustomed":17730,"supplying":17731,"tents":17732,"disgusted":17733,"vogue":17734,"sipped":17735,"filters":17736,"khz":17737,"reno":17738,"selecting":17739,"luftwaffe":17740,"mcmahon":17741,"tyne":17742,"masterpiece":17743,"carriages":17744,"collided":17745,"dunes":17746,"exercised":17747,"flare":17748,"remembers":17749,"muzzle":17750,"##mobile":17751,"heck":17752,"##rson":17753,"burgess":17754,"lunged":17755,"middleton":17756,"boycott":17757,"bilateral":17758,"##sity":17759,"hazardous":17760,"lumpur":17761,"multiplayer":17762,"spotlight":17763,"jackets":17764,"goldman":17765,"liege":17766,"porcelain":17767,"rag":17768,"waterford":17769,"benz":17770,"attracts":17771,"hopeful":17772,"battling":17773,"ottomans":17774,"kensington":17775,"baked":17776,"hymns":17777,"cheyenne":17778,"lattice":17779,"levine":17780,"borrow":17781,"polymer":17782,"clashes":17783,"michaels":17784,"monitored":17785,"commitments":17786,"denounced":17787,"##25":17788,"##von":17789,"cavity":17790,"##oney":17791,"hobby":17792,"akin":17793,"##holders":17794,"futures":17795,"intricate":17796,"cornish":17797,"patty":17798,"##oned":17799,"illegally":17800,"dolphin":17801,"##lag":17802,"barlow":17803,"yellowish":17804,"maddie":17805,"apologized":17806,"luton":17807,"plagued":17808,"##puram":17809,"nana":17810,"##rds":17811,"sway":17812,"fanny":17813,"łodz":17814,"##rino":17815,"psi":17816,"suspicions":17817,"hanged":17818,"##eding":17819,"initiate":17820,"charlton":17821,"##por":17822,"nak":17823,"competent":17824,"235":17825,"analytical":17826,"annex":17827,"wardrobe":17828,"reservations":17829,"##rma":17830,"sect":17831,"162":17832,"fairfax":17833,"hedge":17834,"piled":17835,"buckingham":17836,"uneven":17837,"bauer":17838,"simplicity":17839,"snyder":17840,"interpret":17841,"accountability":17842,"donors":17843,"moderately":17844,"byrd":17845,"continents":17846,"##cite":17847,"##max":17848,"disciple":17849,"hr":17850,"jamaican":17851,"ping":17852,"nominees":17853,"##uss":17854,"mongolian":17855,"diver":17856,"attackers":17857,"eagerly":17858,"ideological":17859,"pillows":17860,"miracles":17861,"apartheid":17862,"revolver":17863,"sulfur":17864,"clinics":17865,"moran":17866,"163":17867,"##enko":17868,"ile":17869,"katy":17870,"rhetoric":17871,"##icated":17872,"chronology":17873,"recycling":17874,"##hrer":17875,"elongated":17876,"mughal":17877,"pascal":17878,"profiles":17879,"vibration":17880,"databases":17881,"domination":17882,"##fare":17883,"##rant":17884,"matthias":17885,"digest":17886,"rehearsal":17887,"polling":17888,"weiss":17889,"initiation":17890,"reeves":17891,"clinging":17892,"flourished":17893,"impress":17894,"ngo":17895,"##hoff":17896,"##ume":17897,"buckley":17898,"symposium":17899,"rhythms":17900,"weed":17901,"emphasize":17902,"transforming":17903,"##taking":17904,"##gence":17905,"##yman":17906,"accountant":17907,"analyze":17908,"flicker":17909,"foil":17910,"priesthood":17911,"voluntarily":17912,"decreases":17913,"##80":17914,"##hya":17915,"slater":17916,"sv":17917,"charting":17918,"mcgill":17919,"##lde":17920,"moreno":17921,"##iu":17922,"besieged":17923,"zur":17924,"robes":17925,"##phic":17926,"admitting":17927,"api":17928,"deported":17929,"turmoil":17930,"peyton":17931,"earthquakes":17932,"##ares":17933,"nationalists":17934,"beau":17935,"clair":17936,"brethren":17937,"interrupt":17938,"welch":17939,"curated":17940,"galerie":17941,"requesting":17942,"164":17943,"##ested":17944,"impending":17945,"steward":17946,"viper":17947,"##vina":17948,"complaining":17949,"beautifully":17950,"brandy":17951,"foam":17952,"nl":17953,"1660":17954,"##cake":17955,"alessandro":17956,"punches":17957,"laced":17958,"explanations":17959,"##lim":17960,"attribute":17961,"clit":17962,"reggie":17963,"discomfort":17964,"##cards":17965,"smoothed":17966,"whales":17967,"##cene":17968,"adler":17969,"countered":17970,"duffy":17971,"disciplinary":17972,"widening":17973,"recipe":17974,"reliance":17975,"conducts":17976,"goats":17977,"gradient":17978,"preaching":17979,"##shaw":17980,"matilda":17981,"quasi":17982,"striped":17983,"meridian":17984,"cannabis":17985,"cordoba":17986,"certificates":17987,"##agh":17988,"##tering":17989,"graffiti":17990,"hangs":17991,"pilgrims":17992,"repeats":17993,"##ych":17994,"revive":17995,"urine":17996,"etat":17997,"##hawk":17998,"fueled":17999,"belts":18000,"fuzzy":18001,"susceptible":18002,"##hang":18003,"mauritius":18004,"salle":18005,"sincere":18006,"beers":18007,"hooks":18008,"##cki":18009,"arbitration":18010,"entrusted":18011,"advise":18012,"sniffed":18013,"seminar":18014,"junk":18015,"donnell":18016,"processors":18017,"principality":18018,"strapped":18019,"celia":18020,"mendoza":18021,"everton":18022,"fortunes":18023,"prejudice":18024,"starving":18025,"reassigned":18026,"steamer":18027,"##lund":18028,"tuck":18029,"evenly":18030,"foreman":18031,"##ffen":18032,"dans":18033,"375":18034,"envisioned":18035,"slit":18036,"##xy":18037,"baseman":18038,"liberia":18039,"rosemary":18040,"##weed":18041,"electrified":18042,"periodically":18043,"potassium":18044,"stride":18045,"contexts":18046,"sperm":18047,"slade":18048,"mariners":18049,"influx":18050,"bianca":18051,"subcommittee":18052,"##rane":18053,"spilling":18054,"icao":18055,"estuary":18056,"##nock":18057,"delivers":18058,"iphone":18059,"##ulata":18060,"isa":18061,"mira":18062,"bohemian":18063,"dessert":18064,"##sbury":18065,"welcoming":18066,"proudly":18067,"slowing":18068,"##chs":18069,"musee":18070,"ascension":18071,"russ":18072,"##vian":18073,"waits":18074,"##psy":18075,"africans":18076,"exploit":18077,"##morphic":18078,"gov":18079,"eccentric":18080,"crab":18081,"peck":18082,"##ull":18083,"entrances":18084,"formidable":18085,"marketplace":18086,"groom":18087,"bolted":18088,"metabolism":18089,"patton":18090,"robbins":18091,"courier":18092,"payload":18093,"endure":18094,"##ifier":18095,"andes":18096,"refrigerator":18097,"##pr":18098,"ornate":18099,"##uca":18100,"ruthless":18101,"illegitimate":18102,"masonry":18103,"strasbourg":18104,"bikes":18105,"adobe":18106,"##³":18107,"apples":18108,"quintet":18109,"willingly":18110,"niche":18111,"bakery":18112,"corpses":18113,"energetic":18114,"##cliffe":18115,"##sser":18116,"##ards":18117,"177":18118,"centimeters":18119,"centro":18120,"fuscous":18121,"cretaceous":18122,"rancho":18123,"##yde":18124,"andrei":18125,"telecom":18126,"tottenham":18127,"oasis":18128,"ordination":18129,"vulnerability":18130,"presiding":18131,"corey":18132,"cp":18133,"penguins":18134,"sims":18135,"##pis":18136,"malawi":18137,"piss":18138,"##48":18139,"correction":18140,"##cked":18141,"##ffle":18142,"##ryn":18143,"countdown":18144,"detectives":18145,"psychiatrist":18146,"psychedelic":18147,"dinosaurs":18148,"blouse":18149,"##get":18150,"choi":18151,"vowed":18152,"##oz":18153,"randomly":18154,"##pol":18155,"49ers":18156,"scrub":18157,"blanche":18158,"bruins":18159,"dusseldorf":18160,"##using":18161,"unwanted":18162,"##ums":18163,"212":18164,"dominique":18165,"elevations":18166,"headlights":18167,"om":18168,"laguna":18169,"##oga":18170,"1750":18171,"famously":18172,"ignorance":18173,"shrewsbury":18174,"##aine":18175,"ajax":18176,"breuning":18177,"che":18178,"confederacy":18179,"greco":18180,"overhaul":18181,"##screen":18182,"paz":18183,"skirts":18184,"disagreement":18185,"cruelty":18186,"jagged":18187,"phoebe":18188,"shifter":18189,"hovered":18190,"viruses":18191,"##wes":18192,"mandy":18193,"##lined":18194,"##gc":18195,"landlord":18196,"squirrel":18197,"dashed":18198,"##ι":18199,"ornamental":18200,"gag":18201,"wally":18202,"grange":18203,"literal":18204,"spurs":18205,"undisclosed":18206,"proceeding":18207,"yin":18208,"##text":18209,"billie":18210,"orphan":18211,"spanned":18212,"humidity":18213,"indy":18214,"weighted":18215,"presentations":18216,"explosions":18217,"lucian":18218,"##tary":18219,"vaughn":18220,"hindus":18221,"##anga":18222,"##hell":18223,"psycho":18224,"171":18225,"daytona":18226,"protects":18227,"efficiently":18228,"rematch":18229,"sly":18230,"tandem":18231,"##oya":18232,"rebranded":18233,"impaired":18234,"hee":18235,"metropolis":18236,"peach":18237,"godfrey":18238,"diaspora":18239,"ethnicity":18240,"prosperous":18241,"gleaming":18242,"dar":18243,"grossing":18244,"playback":18245,"##rden":18246,"stripe":18247,"pistols":18248,"##tain":18249,"births":18250,"labelled":18251,"##cating":18252,"172":18253,"rudy":18254,"alba":18255,"##onne":18256,"aquarium":18257,"hostility":18258,"##gb":18259,"##tase":18260,"shudder":18261,"sumatra":18262,"hardest":18263,"lakers":18264,"consonant":18265,"creeping":18266,"demos":18267,"homicide":18268,"capsule":18269,"zeke":18270,"liberties":18271,"expulsion":18272,"pueblo":18273,"##comb":18274,"trait":18275,"transporting":18276,"##ddin":18277,"##neck":18278,"##yna":18279,"depart":18280,"gregg":18281,"mold":18282,"ledge":18283,"hangar":18284,"oldham":18285,"playboy":18286,"termination":18287,"analysts":18288,"gmbh":18289,"romero":18290,"##itic":18291,"insist":18292,"cradle":18293,"filthy":18294,"brightness":18295,"slash":18296,"shootout":18297,"deposed":18298,"bordering":18299,"##truct":18300,"isis":18301,"microwave":18302,"tumbled":18303,"sheltered":18304,"cathy":18305,"werewolves":18306,"messy":18307,"andersen":18308,"convex":18309,"clapped":18310,"clinched":18311,"satire":18312,"wasting":18313,"edo":18314,"vc":18315,"rufus":18316,"##jak":18317,"mont":18318,"##etti":18319,"poznan":18320,"##keeping":18321,"restructuring":18322,"transverse":18323,"##rland":18324,"azerbaijani":18325,"slovene":18326,"gestures":18327,"roommate":18328,"choking":18329,"shear":18330,"##quist":18331,"vanguard":18332,"oblivious":18333,"##hiro":18334,"disagreed":18335,"baptism":18336,"##lich":18337,"coliseum":18338,"##aceae":18339,"salvage":18340,"societe":18341,"cory":18342,"locke":18343,"relocation":18344,"relying":18345,"versailles":18346,"ahl":18347,"swelling":18348,"##elo":18349,"cheerful":18350,"##word":18351,"##edes":18352,"gin":18353,"sarajevo":18354,"obstacle":18355,"diverted":18356,"##nac":18357,"messed":18358,"thoroughbred":18359,"fluttered":18360,"utrecht":18361,"chewed":18362,"acquaintance":18363,"assassins":18364,"dispatch":18365,"mirza":18366,"##wart":18367,"nike":18368,"salzburg":18369,"swell":18370,"yen":18371,"##gee":18372,"idle":18373,"ligue":18374,"samson":18375,"##nds":18376,"##igh":18377,"playful":18378,"spawned":18379,"##cise":18380,"tease":18381,"##case":18382,"burgundy":18383,"##bot":18384,"stirring":18385,"skeptical":18386,"interceptions":18387,"marathi":18388,"##dies":18389,"bedrooms":18390,"aroused":18391,"pinch":18392,"##lik":18393,"preferences":18394,"tattoos":18395,"buster":18396,"digitally":18397,"projecting":18398,"rust":18399,"##ital":18400,"kitten":18401,"priorities":18402,"addison":18403,"pseudo":18404,"##guard":18405,"dusk":18406,"icons":18407,"sermon":18408,"##psis":18409,"##iba":18410,"bt":18411,"##lift":18412,"##xt":18413,"ju":18414,"truce":18415,"rink":18416,"##dah":18417,"##wy":18418,"defects":18419,"psychiatry":18420,"offences":18421,"calculate":18422,"glucose":18423,"##iful":18424,"##rized":18425,"##unda":18426,"francaise":18427,"##hari":18428,"richest":18429,"warwickshire":18430,"carly":18431,"1763":18432,"purity":18433,"redemption":18434,"lending":18435,"##cious":18436,"muse":18437,"bruises":18438,"cerebral":18439,"aero":18440,"carving":18441,"##name":18442,"preface":18443,"terminology":18444,"invade":18445,"monty":18446,"##int":18447,"anarchist":18448,"blurred":18449,"##iled":18450,"rossi":18451,"treats":18452,"guts":18453,"shu":18454,"foothills":18455,"ballads":18456,"undertaking":18457,"premise":18458,"cecilia":18459,"affiliates":18460,"blasted":18461,"conditional":18462,"wilder":18463,"minors":18464,"drone":18465,"rudolph":18466,"buffy":18467,"swallowing":18468,"horton":18469,"attested":18470,"##hop":18471,"rutherford":18472,"howell":18473,"primetime":18474,"livery":18475,"penal":18476,"##bis":18477,"minimize":18478,"hydro":18479,"wrecked":18480,"wrought":18481,"palazzo":18482,"##gling":18483,"cans":18484,"vernacular":18485,"friedman":18486,"nobleman":18487,"shale":18488,"walnut":18489,"danielle":18490,"##ection":18491,"##tley":18492,"sears":18493,"##kumar":18494,"chords":18495,"lend":18496,"flipping":18497,"streamed":18498,"por":18499,"dracula":18500,"gallons":18501,"sacrifices":18502,"gamble":18503,"orphanage":18504,"##iman":18505,"mckenzie":18506,"##gible":18507,"boxers":18508,"daly":18509,"##balls":18510,"##ان":18511,"208":18512,"##ific":18513,"##rative":18514,"##iq":18515,"exploited":18516,"slated":18517,"##uity":18518,"circling":18519,"hillary":18520,"pinched":18521,"goldberg":18522,"provost":18523,"campaigning":18524,"lim":18525,"piles":18526,"ironically":18527,"jong":18528,"mohan":18529,"successors":18530,"usaf":18531,"##tem":18532,"##ught":18533,"autobiographical":18534,"haute":18535,"preserves":18536,"##ending":18537,"acquitted":18538,"comparisons":18539,"203":18540,"hydroelectric":18541,"gangs":18542,"cypriot":18543,"torpedoes":18544,"rushes":18545,"chrome":18546,"derive":18547,"bumps":18548,"instability":18549,"fiat":18550,"pets":18551,"##mbe":18552,"silas":18553,"dye":18554,"reckless":18555,"settler":18556,"##itation":18557,"info":18558,"heats":18559,"##writing":18560,"176":18561,"canonical":18562,"maltese":18563,"fins":18564,"mushroom":18565,"stacy":18566,"aspen":18567,"avid":18568,"##kur":18569,"##loading":18570,"vickers":18571,"gaston":18572,"hillside":18573,"statutes":18574,"wilde":18575,"gail":18576,"kung":18577,"sabine":18578,"comfortably":18579,"motorcycles":18580,"##rgo":18581,"169":18582,"pneumonia":18583,"fetch":18584,"##sonic":18585,"axel":18586,"faintly":18587,"parallels":18588,"##oop":18589,"mclaren":18590,"spouse":18591,"compton":18592,"interdisciplinary":18593,"miner":18594,"##eni":18595,"181":18596,"clamped":18597,"##chal":18598,"##llah":18599,"separates":18600,"versa":18601,"##mler":18602,"scarborough":18603,"labrador":18604,"##lity":18605,"##osing":18606,"rutgers":18607,"hurdles":18608,"como":18609,"166":18610,"burt":18611,"divers":18612,"##100":18613,"wichita":18614,"cade":18615,"coincided":18616,"##erson":18617,"bruised":18618,"mla":18619,"##pper":18620,"vineyard":18621,"##ili":18622,"##brush":18623,"notch":18624,"mentioning":18625,"jase":18626,"hearted":18627,"kits":18628,"doe":18629,"##acle":18630,"pomerania":18631,"##ady":18632,"ronan":18633,"seizure":18634,"pavel":18635,"problematic":18636,"##zaki":18637,"domenico":18638,"##ulin":18639,"catering":18640,"penelope":18641,"dependence":18642,"parental":18643,"emilio":18644,"ministerial":18645,"atkinson":18646,"##bolic":18647,"clarkson":18648,"chargers":18649,"colby":18650,"grill":18651,"peeked":18652,"arises":18653,"summon":18654,"##aged":18655,"fools":18656,"##grapher":18657,"faculties":18658,"qaeda":18659,"##vial":18660,"garner":18661,"refurbished":18662,"##hwa":18663,"geelong":18664,"disasters":18665,"nudged":18666,"bs":18667,"shareholder":18668,"lori":18669,"algae":18670,"reinstated":18671,"rot":18672,"##ades":18673,"##nous":18674,"invites":18675,"stainless":18676,"183":18677,"inclusive":18678,"##itude":18679,"diocesan":18680,"til":18681,"##icz":18682,"denomination":18683,"##xa":18684,"benton":18685,"floral":18686,"registers":18687,"##ider":18688,"##erman":18689,"##kell":18690,"absurd":18691,"brunei":18692,"guangzhou":18693,"hitter":18694,"retaliation":18695,"##uled":18696,"##eve":18697,"blanc":18698,"nh":18699,"consistency":18700,"contamination":18701,"##eres":18702,"##rner":18703,"dire":18704,"palermo":18705,"broadcasters":18706,"diaries":18707,"inspire":18708,"vols":18709,"brewer":18710,"tightening":18711,"ky":18712,"mixtape":18713,"hormone":18714,"##tok":18715,"stokes":18716,"##color":18717,"##dly":18718,"##ssi":18719,"pg":18720,"##ometer":18721,"##lington":18722,"sanitation":18723,"##tility":18724,"intercontinental":18725,"apps":18726,"##adt":18727,"¹⁄₂":18728,"cylinders":18729,"economies":18730,"favourable":18731,"unison":18732,"croix":18733,"gertrude":18734,"odyssey":18735,"vanity":18736,"dangling":18737,"##logists":18738,"upgrades":18739,"dice":18740,"middleweight":18741,"practitioner":18742,"##ight":18743,"206":18744,"henrik":18745,"parlor":18746,"orion":18747,"angered":18748,"lac":18749,"python":18750,"blurted":18751,"##rri":18752,"sensual":18753,"intends":18754,"swings":18755,"angled":18756,"##phs":18757,"husky":18758,"attain":18759,"peerage":18760,"precinct":18761,"textiles":18762,"cheltenham":18763,"shuffled":18764,"dai":18765,"confess":18766,"tasting":18767,"bhutan":18768,"##riation":18769,"tyrone":18770,"segregation":18771,"abrupt":18772,"ruiz":18773,"##rish":18774,"smirked":18775,"blackwell":18776,"confidential":18777,"browning":18778,"amounted":18779,"##put":18780,"vase":18781,"scarce":18782,"fabulous":18783,"raided":18784,"staple":18785,"guyana":18786,"unemployed":18787,"glider":18788,"shay":18789,"##tow":18790,"carmine":18791,"troll":18792,"intervene":18793,"squash":18794,"superstar":18795,"##uce":18796,"cylindrical":18797,"len":18798,"roadway":18799,"researched":18800,"handy":18801,"##rium":18802,"##jana":18803,"meta":18804,"lao":18805,"declares":18806,"##rring":18807,"##tadt":18808,"##elin":18809,"##kova":18810,"willem":18811,"shrubs":18812,"napoleonic":18813,"realms":18814,"skater":18815,"qi":18816,"volkswagen":18817,"##ł":18818,"tad":18819,"hara":18820,"archaeologist":18821,"awkwardly":18822,"eerie":18823,"##kind":18824,"wiley":18825,"##heimer":18826,"##24":18827,"titus":18828,"organizers":18829,"cfl":18830,"crusaders":18831,"lama":18832,"usb":18833,"vent":18834,"enraged":18835,"thankful":18836,"occupants":18837,"maximilian":18838,"##gaard":18839,"possessing":18840,"textbooks":18841,"##oran":18842,"collaborator":18843,"quaker":18844,"##ulo":18845,"avalanche":18846,"mono":18847,"silky":18848,"straits":18849,"isaiah":18850,"mustang":18851,"surged":18852,"resolutions":18853,"potomac":18854,"descend":18855,"cl":18856,"kilograms":18857,"plato":18858,"strains":18859,"saturdays":18860,"##olin":18861,"bernstein":18862,"##ype":18863,"holstein":18864,"ponytail":18865,"##watch":18866,"belize":18867,"conversely":18868,"heroine":18869,"perpetual":18870,"##ylus":18871,"charcoal":18872,"piedmont":18873,"glee":18874,"negotiating":18875,"backdrop":18876,"prologue":18877,"##jah":18878,"##mmy":18879,"pasadena":18880,"climbs":18881,"ramos":18882,"sunni":18883,"##holm":18884,"##tner":18885,"##tri":18886,"anand":18887,"deficiency":18888,"hertfordshire":18889,"stout":18890,"##avi":18891,"aperture":18892,"orioles":18893,"##irs":18894,"doncaster":18895,"intrigued":18896,"bombed":18897,"coating":18898,"otis":18899,"##mat":18900,"cocktail":18901,"##jit":18902,"##eto":18903,"amir":18904,"arousal":18905,"sar":18906,"##proof":18907,"##act":18908,"##ories":18909,"dixie":18910,"pots":18911,"##bow":18912,"whereabouts":18913,"159":18914,"##fted":18915,"drains":18916,"bullying":18917,"cottages":18918,"scripture":18919,"coherent":18920,"fore":18921,"poe":18922,"appetite":18923,"##uration":18924,"sampled":18925,"##ators":18926,"##dp":18927,"derrick":18928,"rotor":18929,"jays":18930,"peacock":18931,"installment":18932,"##rro":18933,"advisors":18934,"##coming":18935,"rodeo":18936,"scotch":18937,"##mot":18938,"##db":18939,"##fen":18940,"##vant":18941,"ensued":18942,"rodrigo":18943,"dictatorship":18944,"martyrs":18945,"twenties":18946,"##н":18947,"towed":18948,"incidence":18949,"marta":18950,"rainforest":18951,"sai":18952,"scaled":18953,"##cles":18954,"oceanic":18955,"qualifiers":18956,"symphonic":18957,"mcbride":18958,"dislike":18959,"generalized":18960,"aubrey":18961,"colonization":18962,"##iation":18963,"##lion":18964,"##ssing":18965,"disliked":18966,"lublin":18967,"salesman":18968,"##ulates":18969,"spherical":18970,"whatsoever":18971,"sweating":18972,"avalon":18973,"contention":18974,"punt":18975,"severity":18976,"alderman":18977,"atari":18978,"##dina":18979,"##grant":18980,"##rop":18981,"scarf":18982,"seville":18983,"vertices":18984,"annexation":18985,"fairfield":18986,"fascination":18987,"inspiring":18988,"launches":18989,"palatinate":18990,"regretted":18991,"##rca":18992,"feral":18993,"##iom":18994,"elk":18995,"nap":18996,"olsen":18997,"reddy":18998,"yong":18999,"##leader":19000,"##iae":19001,"garment":19002,"transports":19003,"feng":19004,"gracie":19005,"outrage":19006,"viceroy":19007,"insides":19008,"##esis":19009,"breakup":19010,"grady":19011,"organizer":19012,"softer":19013,"grimaced":19014,"222":19015,"murals":19016,"galicia":19017,"arranging":19018,"vectors":19019,"##rsten":19020,"bas":19021,"##sb":19022,"##cens":19023,"sloan":19024,"##eka":19025,"bitten":19026,"ara":19027,"fender":19028,"nausea":19029,"bumped":19030,"kris":19031,"banquet":19032,"comrades":19033,"detector":19034,"persisted":19035,"##llan":19036,"adjustment":19037,"endowed":19038,"cinemas":19039,"##shot":19040,"sellers":19041,"##uman":19042,"peek":19043,"epa":19044,"kindly":19045,"neglect":19046,"simpsons":19047,"talon":19048,"mausoleum":19049,"runaway":19050,"hangul":19051,"lookout":19052,"##cic":19053,"rewards":19054,"coughed":19055,"acquainted":19056,"chloride":19057,"##ald":19058,"quicker":19059,"accordion":19060,"neolithic":19061,"##qa":19062,"artemis":19063,"coefficient":19064,"lenny":19065,"pandora":19066,"tx":19067,"##xed":19068,"ecstasy":19069,"litter":19070,"segunda":19071,"chairperson":19072,"gemma":19073,"hiss":19074,"rumor":19075,"vow":19076,"nasal":19077,"antioch":19078,"compensate":19079,"patiently":19080,"transformers":19081,"##eded":19082,"judo":19083,"morrow":19084,"penis":19085,"posthumous":19086,"philips":19087,"bandits":19088,"husbands":19089,"denote":19090,"flaming":19091,"##any":19092,"##phones":19093,"langley":19094,"yorker":19095,"1760":19096,"walters":19097,"##uo":19098,"##kle":19099,"gubernatorial":19100,"fatty":19101,"samsung":19102,"leroy":19103,"outlaw":19104,"##nine":19105,"unpublished":19106,"poole":19107,"jakob":19108,"##ᵢ":19109,"##ₙ":19110,"crete":19111,"distorted":19112,"superiority":19113,"##dhi":19114,"intercept":19115,"crust":19116,"mig":19117,"claus":19118,"crashes":19119,"positioning":19120,"188":19121,"stallion":19122,"301":19123,"frontal":19124,"armistice":19125,"##estinal":19126,"elton":19127,"aj":19128,"encompassing":19129,"camel":19130,"commemorated":19131,"malaria":19132,"woodward":19133,"calf":19134,"cigar":19135,"penetrate":19136,"##oso":19137,"willard":19138,"##rno":19139,"##uche":19140,"illustrate":19141,"amusing":19142,"convergence":19143,"noteworthy":19144,"##lma":19145,"##rva":19146,"journeys":19147,"realise":19148,"manfred":19149,"##sable":19150,"410":19151,"##vocation":19152,"hearings":19153,"fiance":19154,"##posed":19155,"educators":19156,"provoked":19157,"adjusting":19158,"##cturing":19159,"modular":19160,"stockton":19161,"paterson":19162,"vlad":19163,"rejects":19164,"electors":19165,"selena":19166,"maureen":19167,"##tres":19168,"uber":19169,"##rce":19170,"swirled":19171,"##num":19172,"proportions":19173,"nanny":19174,"pawn":19175,"naturalist":19176,"parma":19177,"apostles":19178,"awoke":19179,"ethel":19180,"wen":19181,"##bey":19182,"monsoon":19183,"overview":19184,"##inating":19185,"mccain":19186,"rendition":19187,"risky":19188,"adorned":19189,"##ih":19190,"equestrian":19191,"germain":19192,"nj":19193,"conspicuous":19194,"confirming":19195,"##yoshi":19196,"shivering":19197,"##imeter":19198,"milestone":19199,"rumours":19200,"flinched":19201,"bounds":19202,"smacked":19203,"token":19204,"##bei":19205,"lectured":19206,"automobiles":19207,"##shore":19208,"impacted":19209,"##iable":19210,"nouns":19211,"nero":19212,"##leaf":19213,"ismail":19214,"prostitute":19215,"trams":19216,"##lace":19217,"bridget":19218,"sud":19219,"stimulus":19220,"impressions":19221,"reins":19222,"revolves":19223,"##oud":19224,"##gned":19225,"giro":19226,"honeymoon":19227,"##swell":19228,"criterion":19229,"##sms":19230,"##uil":19231,"libyan":19232,"prefers":19233,"##osition":19234,"211":19235,"preview":19236,"sucks":19237,"accusation":19238,"bursts":19239,"metaphor":19240,"diffusion":19241,"tolerate":19242,"faye":19243,"betting":19244,"cinematographer":19245,"liturgical":19246,"specials":19247,"bitterly":19248,"humboldt":19249,"##ckle":19250,"flux":19251,"rattled":19252,"##itzer":19253,"archaeologists":19254,"odor":19255,"authorised":19256,"marshes":19257,"discretion":19258,"##ов":19259,"alarmed":19260,"archaic":19261,"inverse":19262,"##leton":19263,"explorers":19264,"##pine":19265,"drummond":19266,"tsunami":19267,"woodlands":19268,"##minate":19269,"##tland":19270,"booklet":19271,"insanity":19272,"owning":19273,"insert":19274,"crafted":19275,"calculus":19276,"##tore":19277,"receivers":19278,"##bt":19279,"stung":19280,"##eca":19281,"##nched":19282,"prevailing":19283,"travellers":19284,"eyeing":19285,"lila":19286,"graphs":19287,"##borne":19288,"178":19289,"julien":19290,"##won":19291,"morale":19292,"adaptive":19293,"therapist":19294,"erica":19295,"cw":19296,"libertarian":19297,"bowman":19298,"pitches":19299,"vita":19300,"##ional":19301,"crook":19302,"##ads":19303,"##entation":19304,"caledonia":19305,"mutiny":19306,"##sible":19307,"1840s":19308,"automation":19309,"##ß":19310,"flock":19311,"##pia":19312,"ironic":19313,"pathology":19314,"##imus":19315,"remarried":19316,"##22":19317,"joker":19318,"withstand":19319,"energies":19320,"##att":19321,"shropshire":19322,"hostages":19323,"madeleine":19324,"tentatively":19325,"conflicting":19326,"mateo":19327,"recipes":19328,"euros":19329,"ol":19330,"mercenaries":19331,"nico":19332,"##ndon":19333,"albuquerque":19334,"augmented":19335,"mythical":19336,"bel":19337,"freud":19338,"##child":19339,"cough":19340,"##lica":19341,"365":19342,"freddy":19343,"lillian":19344,"genetically":19345,"nuremberg":19346,"calder":19347,"209":19348,"bonn":19349,"outdoors":19350,"paste":19351,"suns":19352,"urgency":19353,"vin":19354,"restraint":19355,"tyson":19356,"##cera":19357,"##selle":19358,"barrage":19359,"bethlehem":19360,"kahn":19361,"##par":19362,"mounts":19363,"nippon":19364,"barony":19365,"happier":19366,"ryu":19367,"makeshift":19368,"sheldon":19369,"blushed":19370,"castillo":19371,"barking":19372,"listener":19373,"taped":19374,"bethel":19375,"fluent":19376,"headlines":19377,"pornography":19378,"rum":19379,"disclosure":19380,"sighing":19381,"mace":19382,"doubling":19383,"gunther":19384,"manly":19385,"##plex":19386,"rt":19387,"interventions":19388,"physiological":19389,"forwards":19390,"emerges":19391,"##tooth":19392,"##gny":19393,"compliment":19394,"rib":19395,"recession":19396,"visibly":19397,"barge":19398,"faults":19399,"connector":19400,"exquisite":19401,"prefect":19402,"##rlin":19403,"patio":19404,"##cured":19405,"elevators":19406,"brandt":19407,"italics":19408,"pena":19409,"173":19410,"wasp":19411,"satin":19412,"ea":19413,"botswana":19414,"graceful":19415,"respectable":19416,"##jima":19417,"##rter":19418,"##oic":19419,"franciscan":19420,"generates":19421,"##dl":19422,"alfredo":19423,"disgusting":19424,"##olate":19425,"##iously":19426,"sherwood":19427,"warns":19428,"cod":19429,"promo":19430,"cheryl":19431,"sino":19432,"##ة":19433,"##escu":19434,"twitch":19435,"##zhi":19436,"brownish":19437,"thom":19438,"ortiz":19439,"##dron":19440,"densely":19441,"##beat":19442,"carmel":19443,"reinforce":19444,"##bana":19445,"187":19446,"anastasia":19447,"downhill":19448,"vertex":19449,"contaminated":19450,"remembrance":19451,"harmonic":19452,"homework":19453,"##sol":19454,"fiancee":19455,"gears":19456,"olds":19457,"angelica":19458,"loft":19459,"ramsay":19460,"quiz":19461,"colliery":19462,"sevens":19463,"##cape":19464,"autism":19465,"##hil":19466,"walkway":19467,"##boats":19468,"ruben":19469,"abnormal":19470,"ounce":19471,"khmer":19472,"##bbe":19473,"zachary":19474,"bedside":19475,"morphology":19476,"punching":19477,"##olar":19478,"sparrow":19479,"convinces":19480,"##35":19481,"hewitt":19482,"queer":19483,"remastered":19484,"rods":19485,"mabel":19486,"solemn":19487,"notified":19488,"lyricist":19489,"symmetric":19490,"##xide":19491,"174":19492,"encore":19493,"passports":19494,"wildcats":19495,"##uni":19496,"baja":19497,"##pac":19498,"mildly":19499,"##ease":19500,"bleed":19501,"commodity":19502,"mounds":19503,"glossy":19504,"orchestras":19505,"##omo":19506,"damian":19507,"prelude":19508,"ambitions":19509,"##vet":19510,"awhile":19511,"remotely":19512,"##aud":19513,"asserts":19514,"imply":19515,"##iques":19516,"distinctly":19517,"modelling":19518,"remedy":19519,"##dded":19520,"windshield":19521,"dani":19522,"xiao":19523,"##endra":19524,"audible":19525,"powerplant":19526,"1300":19527,"invalid":19528,"elemental":19529,"acquisitions":19530,"##hala":19531,"immaculate":19532,"libby":19533,"plata":19534,"smuggling":19535,"ventilation":19536,"denoted":19537,"minh":19538,"##morphism":19539,"430":19540,"differed":19541,"dion":19542,"kelley":19543,"lore":19544,"mocking":19545,"sabbath":19546,"spikes":19547,"hygiene":19548,"drown":19549,"runoff":19550,"stylized":19551,"tally":19552,"liberated":19553,"aux":19554,"interpreter":19555,"righteous":19556,"aba":19557,"siren":19558,"reaper":19559,"pearce":19560,"millie":19561,"##cier":19562,"##yra":19563,"gaius":19564,"##iso":19565,"captures":19566,"##ttering":19567,"dorm":19568,"claudio":19569,"##sic":19570,"benches":19571,"knighted":19572,"blackness":19573,"##ored":19574,"discount":19575,"fumble":19576,"oxidation":19577,"routed":19578,"##ς":19579,"novak":19580,"perpendicular":19581,"spoiled":19582,"fracture":19583,"splits":19584,"##urt":19585,"pads":19586,"topology":19587,"##cats":19588,"axes":19589,"fortunate":19590,"offenders":19591,"protestants":19592,"esteem":19593,"221":19594,"broadband":19595,"convened":19596,"frankly":19597,"hound":19598,"prototypes":19599,"isil":19600,"facilitated":19601,"keel":19602,"##sher":19603,"sahara":19604,"awaited":19605,"bubba":19606,"orb":19607,"prosecutors":19608,"186":19609,"hem":19610,"520":19611,"##xing":19612,"relaxing":19613,"remnant":19614,"romney":19615,"sorted":19616,"slalom":19617,"stefano":19618,"ulrich":19619,"##active":19620,"exemption":19621,"folder":19622,"pauses":19623,"foliage":19624,"hitchcock":19625,"epithet":19626,"204":19627,"criticisms":19628,"##aca":19629,"ballistic":19630,"brody":19631,"hinduism":19632,"chaotic":19633,"youths":19634,"equals":19635,"##pala":19636,"pts":19637,"thicker":19638,"analogous":19639,"capitalist":19640,"improvised":19641,"overseeing":19642,"sinatra":19643,"ascended":19644,"beverage":19645,"##tl":19646,"straightforward":19647,"##kon":19648,"curran":19649,"##west":19650,"bois":19651,"325":19652,"induce":19653,"surveying":19654,"emperors":19655,"sax":19656,"unpopular":19657,"##kk":19658,"cartoonist":19659,"fused":19660,"##mble":19661,"unto":19662,"##yuki":19663,"localities":19664,"##cko":19665,"##ln":19666,"darlington":19667,"slain":19668,"academie":19669,"lobbying":19670,"sediment":19671,"puzzles":19672,"##grass":19673,"defiance":19674,"dickens":19675,"manifest":19676,"tongues":19677,"alumnus":19678,"arbor":19679,"coincide":19680,"184":19681,"appalachian":19682,"mustafa":19683,"examiner":19684,"cabaret":19685,"traumatic":19686,"yves":19687,"bracelet":19688,"draining":19689,"heroin":19690,"magnum":19691,"baths":19692,"odessa":19693,"consonants":19694,"mitsubishi":19695,"##gua":19696,"kellan":19697,"vaudeville":19698,"##fr":19699,"joked":19700,"null":19701,"straps":19702,"probation":19703,"##ław":19704,"ceded":19705,"interfaces":19706,"##pas":19707,"##zawa":19708,"blinding":19709,"viet":19710,"224":19711,"rothschild":19712,"museo":19713,"640":19714,"huddersfield":19715,"##vr":19716,"tactic":19717,"##storm":19718,"brackets":19719,"dazed":19720,"incorrectly":19721,"##vu":19722,"reg":19723,"glazed":19724,"fearful":19725,"manifold":19726,"benefited":19727,"irony":19728,"##sun":19729,"stumbling":19730,"##rte":19731,"willingness":19732,"balkans":19733,"mei":19734,"wraps":19735,"##aba":19736,"injected":19737,"##lea":19738,"gu":19739,"syed":19740,"harmless":19741,"##hammer":19742,"bray":19743,"takeoff":19744,"poppy":19745,"timor":19746,"cardboard":19747,"astronaut":19748,"purdue":19749,"weeping":19750,"southbound":19751,"cursing":19752,"stalls":19753,"diagonal":19754,"##neer":19755,"lamar":19756,"bryce":19757,"comte":19758,"weekdays":19759,"harrington":19760,"##uba":19761,"negatively":19762,"##see":19763,"lays":19764,"grouping":19765,"##cken":19766,"##henko":19767,"affirmed":19768,"halle":19769,"modernist":19770,"##lai":19771,"hodges":19772,"smelling":19773,"aristocratic":19774,"baptized":19775,"dismiss":19776,"justification":19777,"oilers":19778,"##now":19779,"coupling":19780,"qin":19781,"snack":19782,"healer":19783,"##qing":19784,"gardener":19785,"layla":19786,"battled":19787,"formulated":19788,"stephenson":19789,"gravitational":19790,"##gill":19791,"##jun":19792,"1768":19793,"granny":19794,"coordinating":19795,"suites":19796,"##cd":19797,"##ioned":19798,"monarchs":19799,"##cote":19800,"##hips":19801,"sep":19802,"blended":19803,"apr":19804,"barrister":19805,"deposition":19806,"fia":19807,"mina":19808,"policemen":19809,"paranoid":19810,"##pressed":19811,"churchyard":19812,"covert":19813,"crumpled":19814,"creep":19815,"abandoning":19816,"tr":19817,"transmit":19818,"conceal":19819,"barr":19820,"understands":19821,"readiness":19822,"spire":19823,"##cology":19824,"##enia":19825,"##erry":19826,"610":19827,"startling":19828,"unlock":19829,"vida":19830,"bowled":19831,"slots":19832,"##nat":19833,"##islav":19834,"spaced":19835,"trusting":19836,"admire":19837,"rig":19838,"##ink":19839,"slack":19840,"##70":19841,"mv":19842,"207":19843,"casualty":19844,"##wei":19845,"classmates":19846,"##odes":19847,"##rar":19848,"##rked":19849,"amherst":19850,"furnished":19851,"evolve":19852,"foundry":19853,"menace":19854,"mead":19855,"##lein":19856,"flu":19857,"wesleyan":19858,"##kled":19859,"monterey":19860,"webber":19861,"##vos":19862,"wil":19863,"##mith":19864,"##на":19865,"bartholomew":19866,"justices":19867,"restrained":19868,"##cke":19869,"amenities":19870,"191":19871,"mediated":19872,"sewage":19873,"trenches":19874,"ml":19875,"mainz":19876,"##thus":19877,"1800s":19878,"##cula":19879,"##inski":19880,"caine":19881,"bonding":19882,"213":19883,"converts":19884,"spheres":19885,"superseded":19886,"marianne":19887,"crypt":19888,"sweaty":19889,"ensign":19890,"historia":19891,"##br":19892,"spruce":19893,"##post":19894,"##ask":19895,"forks":19896,"thoughtfully":19897,"yukon":19898,"pamphlet":19899,"ames":19900,"##uter":19901,"karma":19902,"##yya":19903,"bryn":19904,"negotiation":19905,"sighs":19906,"incapable":19907,"##mbre":19908,"##ntial":19909,"actresses":19910,"taft":19911,"##mill":19912,"luce":19913,"prevailed":19914,"##amine":19915,"1773":19916,"motionless":19917,"envoy":19918,"testify":19919,"investing":19920,"sculpted":19921,"instructors":19922,"provence":19923,"kali":19924,"cullen":19925,"horseback":19926,"##while":19927,"goodwin":19928,"##jos":19929,"gaa":19930,"norte":19931,"##ldon":19932,"modify":19933,"wavelength":19934,"abd":19935,"214":19936,"skinned":19937,"sprinter":19938,"forecast":19939,"scheduling":19940,"marries":19941,"squared":19942,"tentative":19943,"##chman":19944,"boer":19945,"##isch":19946,"bolts":19947,"swap":19948,"fisherman":19949,"assyrian":19950,"impatiently":19951,"guthrie":19952,"martins":19953,"murdoch":19954,"194":19955,"tanya":19956,"nicely":19957,"dolly":19958,"lacy":19959,"med":19960,"##45":19961,"syn":19962,"decks":19963,"fashionable":19964,"millionaire":19965,"##ust":19966,"surfing":19967,"##ml":19968,"##ision":19969,"heaved":19970,"tammy":19971,"consulate":19972,"attendees":19973,"routinely":19974,"197":19975,"fuse":19976,"saxophonist":19977,"backseat":19978,"malaya":19979,"##lord":19980,"scowl":19981,"tau":19982,"##ishly":19983,"193":19984,"sighted":19985,"steaming":19986,"##rks":19987,"303":19988,"911":19989,"##holes":19990,"##hong":19991,"ching":19992,"##wife":19993,"bless":19994,"conserved":19995,"jurassic":19996,"stacey":19997,"unix":19998,"zion":19999,"chunk":20000,"rigorous":20001,"blaine":20002,"198":20003,"peabody":20004,"slayer":20005,"dismay":20006,"brewers":20007,"nz":20008,"##jer":20009,"det":20010,"##glia":20011,"glover":20012,"postwar":20013,"int":20014,"penetration":20015,"sylvester":20016,"imitation":20017,"vertically":20018,"airlift":20019,"heiress":20020,"knoxville":20021,"viva":20022,"##uin":20023,"390":20024,"macon":20025,"##rim":20026,"##fighter":20027,"##gonal":20028,"janice":20029,"##orescence":20030,"##wari":20031,"marius":20032,"belongings":20033,"leicestershire":20034,"196":20035,"blanco":20036,"inverted":20037,"preseason":20038,"sanity":20039,"sobbing":20040,"##due":20041,"##elt":20042,"##dled":20043,"collingwood":20044,"regeneration":20045,"flickering":20046,"shortest":20047,"##mount":20048,"##osi":20049,"feminism":20050,"##lat":20051,"sherlock":20052,"cabinets":20053,"fumbled":20054,"northbound":20055,"precedent":20056,"snaps":20057,"##mme":20058,"researching":20059,"##akes":20060,"guillaume":20061,"insights":20062,"manipulated":20063,"vapor":20064,"neighbour":20065,"sap":20066,"gangster":20067,"frey":20068,"f1":20069,"stalking":20070,"scarcely":20071,"callie":20072,"barnett":20073,"tendencies":20074,"audi":20075,"doomed":20076,"assessing":20077,"slung":20078,"panchayat":20079,"ambiguous":20080,"bartlett":20081,"##etto":20082,"distributing":20083,"violating":20084,"wolverhampton":20085,"##hetic":20086,"swami":20087,"histoire":20088,"##urus":20089,"liable":20090,"pounder":20091,"groin":20092,"hussain":20093,"larsen":20094,"popping":20095,"surprises":20096,"##atter":20097,"vie":20098,"curt":20099,"##station":20100,"mute":20101,"relocate":20102,"musicals":20103,"authorization":20104,"richter":20105,"##sef":20106,"immortality":20107,"tna":20108,"bombings":20109,"##press":20110,"deteriorated":20111,"yiddish":20112,"##acious":20113,"robbed":20114,"colchester":20115,"cs":20116,"pmid":20117,"ao":20118,"verified":20119,"balancing":20120,"apostle":20121,"swayed":20122,"recognizable":20123,"oxfordshire":20124,"retention":20125,"nottinghamshire":20126,"contender":20127,"judd":20128,"invitational":20129,"shrimp":20130,"uhf":20131,"##icient":20132,"cleaner":20133,"longitudinal":20134,"tanker":20135,"##mur":20136,"acronym":20137,"broker":20138,"koppen":20139,"sundance":20140,"suppliers":20141,"##gil":20142,"4000":20143,"clipped":20144,"fuels":20145,"petite":20146,"##anne":20147,"landslide":20148,"helene":20149,"diversion":20150,"populous":20151,"landowners":20152,"auspices":20153,"melville":20154,"quantitative":20155,"##xes":20156,"ferries":20157,"nicky":20158,"##llus":20159,"doo":20160,"haunting":20161,"roche":20162,"carver":20163,"downed":20164,"unavailable":20165,"##pathy":20166,"approximation":20167,"hiroshima":20168,"##hue":20169,"garfield":20170,"valle":20171,"comparatively":20172,"keyboardist":20173,"traveler":20174,"##eit":20175,"congestion":20176,"calculating":20177,"subsidiaries":20178,"##bate":20179,"serb":20180,"modernization":20181,"fairies":20182,"deepened":20183,"ville":20184,"averages":20185,"##lore":20186,"inflammatory":20187,"tonga":20188,"##itch":20189,"co₂":20190,"squads":20191,"##hea":20192,"gigantic":20193,"serum":20194,"enjoyment":20195,"retailer":20196,"verona":20197,"35th":20198,"cis":20199,"##phobic":20200,"magna":20201,"technicians":20202,"##vati":20203,"arithmetic":20204,"##sport":20205,"levin":20206,"##dation":20207,"amtrak":20208,"chow":20209,"sienna":20210,"##eyer":20211,"backstage":20212,"entrepreneurship":20213,"##otic":20214,"learnt":20215,"tao":20216,"##udy":20217,"worcestershire":20218,"formulation":20219,"baggage":20220,"hesitant":20221,"bali":20222,"sabotage":20223,"##kari":20224,"barren":20225,"enhancing":20226,"murmur":20227,"pl":20228,"freshly":20229,"putnam":20230,"syntax":20231,"aces":20232,"medicines":20233,"resentment":20234,"bandwidth":20235,"##sier":20236,"grins":20237,"chili":20238,"guido":20239,"##sei":20240,"framing":20241,"implying":20242,"gareth":20243,"lissa":20244,"genevieve":20245,"pertaining":20246,"admissions":20247,"geo":20248,"thorpe":20249,"proliferation":20250,"sato":20251,"bela":20252,"analyzing":20253,"parting":20254,"##gor":20255,"awakened":20256,"##isman":20257,"huddled":20258,"secrecy":20259,"##kling":20260,"hush":20261,"gentry":20262,"540":20263,"dungeons":20264,"##ego":20265,"coasts":20266,"##utz":20267,"sacrificed":20268,"##chule":20269,"landowner":20270,"mutually":20271,"prevalence":20272,"programmer":20273,"adolescent":20274,"disrupted":20275,"seaside":20276,"gee":20277,"trusts":20278,"vamp":20279,"georgie":20280,"##nesian":20281,"##iol":20282,"schedules":20283,"sindh":20284,"##market":20285,"etched":20286,"hm":20287,"sparse":20288,"bey":20289,"beaux":20290,"scratching":20291,"gliding":20292,"unidentified":20293,"216":20294,"collaborating":20295,"gems":20296,"jesuits":20297,"oro":20298,"accumulation":20299,"shaping":20300,"mbe":20301,"anal":20302,"##xin":20303,"231":20304,"enthusiasts":20305,"newscast":20306,"##egan":20307,"janata":20308,"dewey":20309,"parkinson":20310,"179":20311,"ankara":20312,"biennial":20313,"towering":20314,"dd":20315,"inconsistent":20316,"950":20317,"##chet":20318,"thriving":20319,"terminate":20320,"cabins":20321,"furiously":20322,"eats":20323,"advocating":20324,"donkey":20325,"marley":20326,"muster":20327,"phyllis":20328,"leiden":20329,"##user":20330,"grassland":20331,"glittering":20332,"iucn":20333,"loneliness":20334,"217":20335,"memorandum":20336,"armenians":20337,"##ddle":20338,"popularized":20339,"rhodesia":20340,"60s":20341,"lame":20342,"##illon":20343,"sans":20344,"bikini":20345,"header":20346,"orbits":20347,"##xx":20348,"##finger":20349,"##ulator":20350,"sharif":20351,"spines":20352,"biotechnology":20353,"strolled":20354,"naughty":20355,"yates":20356,"##wire":20357,"fremantle":20358,"milo":20359,"##mour":20360,"abducted":20361,"removes":20362,"##atin":20363,"humming":20364,"wonderland":20365,"##chrome":20366,"##ester":20367,"hume":20368,"pivotal":20369,"##rates":20370,"armand":20371,"grams":20372,"believers":20373,"elector":20374,"rte":20375,"apron":20376,"bis":20377,"scraped":20378,"##yria":20379,"endorsement":20380,"initials":20381,"##llation":20382,"eps":20383,"dotted":20384,"hints":20385,"buzzing":20386,"emigration":20387,"nearer":20388,"##tom":20389,"indicators":20390,"##ulu":20391,"coarse":20392,"neutron":20393,"protectorate":20394,"##uze":20395,"directional":20396,"exploits":20397,"pains":20398,"loire":20399,"1830s":20400,"proponents":20401,"guggenheim":20402,"rabbits":20403,"ritchie":20404,"305":20405,"hectare":20406,"inputs":20407,"hutton":20408,"##raz":20409,"verify":20410,"##ako":20411,"boilers":20412,"longitude":20413,"##lev":20414,"skeletal":20415,"yer":20416,"emilia":20417,"citrus":20418,"compromised":20419,"##gau":20420,"pokemon":20421,"prescription":20422,"paragraph":20423,"eduard":20424,"cadillac":20425,"attire":20426,"categorized":20427,"kenyan":20428,"weddings":20429,"charley":20430,"##bourg":20431,"entertain":20432,"monmouth":20433,"##lles":20434,"nutrients":20435,"davey":20436,"mesh":20437,"incentive":20438,"practised":20439,"ecosystems":20440,"kemp":20441,"subdued":20442,"overheard":20443,"##rya":20444,"bodily":20445,"maxim":20446,"##nius":20447,"apprenticeship":20448,"ursula":20449,"##fight":20450,"lodged":20451,"rug":20452,"silesian":20453,"unconstitutional":20454,"patel":20455,"inspected":20456,"coyote":20457,"unbeaten":20458,"##hak":20459,"34th":20460,"disruption":20461,"convict":20462,"parcel":20463,"##cl":20464,"##nham":20465,"collier":20466,"implicated":20467,"mallory":20468,"##iac":20469,"##lab":20470,"susannah":20471,"winkler":20472,"##rber":20473,"shia":20474,"phelps":20475,"sediments":20476,"graphical":20477,"robotic":20478,"##sner":20479,"adulthood":20480,"mart":20481,"smoked":20482,"##isto":20483,"kathryn":20484,"clarified":20485,"##aran":20486,"divides":20487,"convictions":20488,"oppression":20489,"pausing":20490,"burying":20491,"##mt":20492,"federico":20493,"mathias":20494,"eileen":20495,"##tana":20496,"kite":20497,"hunched":20498,"##acies":20499,"189":20500,"##atz":20501,"disadvantage":20502,"liza":20503,"kinetic":20504,"greedy":20505,"paradox":20506,"yokohama":20507,"dowager":20508,"trunks":20509,"ventured":20510,"##gement":20511,"gupta":20512,"vilnius":20513,"olaf":20514,"##thest":20515,"crimean":20516,"hopper":20517,"##ej":20518,"progressively":20519,"arturo":20520,"mouthed":20521,"arrondissement":20522,"##fusion":20523,"rubin":20524,"simulcast":20525,"oceania":20526,"##orum":20527,"##stra":20528,"##rred":20529,"busiest":20530,"intensely":20531,"navigator":20532,"cary":20533,"##vine":20534,"##hini":20535,"##bies":20536,"fife":20537,"rowe":20538,"rowland":20539,"posing":20540,"insurgents":20541,"shafts":20542,"lawsuits":20543,"activate":20544,"conor":20545,"inward":20546,"culturally":20547,"garlic":20548,"265":20549,"##eering":20550,"eclectic":20551,"##hui":20552,"##kee":20553,"##nl":20554,"furrowed":20555,"vargas":20556,"meteorological":20557,"rendezvous":20558,"##aus":20559,"culinary":20560,"commencement":20561,"##dition":20562,"quota":20563,"##notes":20564,"mommy":20565,"salaries":20566,"overlapping":20567,"mule":20568,"##iology":20569,"##mology":20570,"sums":20571,"wentworth":20572,"##isk":20573,"##zione":20574,"mainline":20575,"subgroup":20576,"##illy":20577,"hack":20578,"plaintiff":20579,"verdi":20580,"bulb":20581,"differentiation":20582,"engagements":20583,"multinational":20584,"supplemented":20585,"bertrand":20586,"caller":20587,"regis":20588,"##naire":20589,"##sler":20590,"##arts":20591,"##imated":20592,"blossom":20593,"propagation":20594,"kilometer":20595,"viaduct":20596,"vineyards":20597,"##uate":20598,"beckett":20599,"optimization":20600,"golfer":20601,"songwriters":20602,"seminal":20603,"semitic":20604,"thud":20605,"volatile":20606,"evolving":20607,"ridley":20608,"##wley":20609,"trivial":20610,"distributions":20611,"scandinavia":20612,"jiang":20613,"##ject":20614,"wrestled":20615,"insistence":20616,"##dio":20617,"emphasizes":20618,"napkin":20619,"##ods":20620,"adjunct":20621,"rhyme":20622,"##ricted":20623,"##eti":20624,"hopeless":20625,"surrounds":20626,"tremble":20627,"32nd":20628,"smoky":20629,"##ntly":20630,"oils":20631,"medicinal":20632,"padded":20633,"steer":20634,"wilkes":20635,"219":20636,"255":20637,"concessions":20638,"hue":20639,"uniquely":20640,"blinded":20641,"landon":20642,"yahoo":20643,"##lane":20644,"hendrix":20645,"commemorating":20646,"dex":20647,"specify":20648,"chicks":20649,"##ggio":20650,"intercity":20651,"1400":20652,"morley":20653,"##torm":20654,"highlighting":20655,"##oting":20656,"pang":20657,"oblique":20658,"stalled":20659,"##liner":20660,"flirting":20661,"newborn":20662,"1769":20663,"bishopric":20664,"shaved":20665,"232":20666,"currie":20667,"##ush":20668,"dharma":20669,"spartan":20670,"##ooped":20671,"favorites":20672,"smug":20673,"novella":20674,"sirens":20675,"abusive":20676,"creations":20677,"espana":20678,"##lage":20679,"paradigm":20680,"semiconductor":20681,"sheen":20682,"##rdo":20683,"##yen":20684,"##zak":20685,"nrl":20686,"renew":20687,"##pose":20688,"##tur":20689,"adjutant":20690,"marches":20691,"norma":20692,"##enity":20693,"ineffective":20694,"weimar":20695,"grunt":20696,"##gat":20697,"lordship":20698,"plotting":20699,"expenditure":20700,"infringement":20701,"lbs":20702,"refrain":20703,"av":20704,"mimi":20705,"mistakenly":20706,"postmaster":20707,"1771":20708,"##bara":20709,"ras":20710,"motorsports":20711,"tito":20712,"199":20713,"subjective":20714,"##zza":20715,"bully":20716,"stew":20717,"##kaya":20718,"prescott":20719,"1a":20720,"##raphic":20721,"##zam":20722,"bids":20723,"styling":20724,"paranormal":20725,"reeve":20726,"sneaking":20727,"exploding":20728,"katz":20729,"akbar":20730,"migrant":20731,"syllables":20732,"indefinitely":20733,"##ogical":20734,"destroys":20735,"replaces":20736,"applause":20737,"##phine":20738,"pest":20739,"##fide":20740,"218":20741,"articulated":20742,"bertie":20743,"##thing":20744,"##cars":20745,"##ptic":20746,"courtroom":20747,"crowley":20748,"aesthetics":20749,"cummings":20750,"tehsil":20751,"hormones":20752,"titanic":20753,"dangerously":20754,"##ibe":20755,"stadion":20756,"jaenelle":20757,"auguste":20758,"ciudad":20759,"##chu":20760,"mysore":20761,"partisans":20762,"##sio":20763,"lucan":20764,"philipp":20765,"##aly":20766,"debating":20767,"henley":20768,"interiors":20769,"##rano":20770,"##tious":20771,"homecoming":20772,"beyonce":20773,"usher":20774,"henrietta":20775,"prepares":20776,"weeds":20777,"##oman":20778,"ely":20779,"plucked":20780,"##pire":20781,"##dable":20782,"luxurious":20783,"##aq":20784,"artifact":20785,"password":20786,"pasture":20787,"juno":20788,"maddy":20789,"minsk":20790,"##dder":20791,"##ologies":20792,"##rone":20793,"assessments":20794,"martian":20795,"royalist":20796,"1765":20797,"examines":20798,"##mani":20799,"##rge":20800,"nino":20801,"223":20802,"parry":20803,"scooped":20804,"relativity":20805,"##eli":20806,"##uting":20807,"##cao":20808,"congregational":20809,"noisy":20810,"traverse":20811,"##agawa":20812,"strikeouts":20813,"nickelodeon":20814,"obituary":20815,"transylvania":20816,"binds":20817,"depictions":20818,"polk":20819,"trolley":20820,"##yed":20821,"##lard":20822,"breeders":20823,"##under":20824,"dryly":20825,"hokkaido":20826,"1762":20827,"strengths":20828,"stacks":20829,"bonaparte":20830,"connectivity":20831,"neared":20832,"prostitutes":20833,"stamped":20834,"anaheim":20835,"gutierrez":20836,"sinai":20837,"##zzling":20838,"bram":20839,"fresno":20840,"madhya":20841,"##86":20842,"proton":20843,"##lena":20844,"##llum":20845,"##phon":20846,"reelected":20847,"wanda":20848,"##anus":20849,"##lb":20850,"ample":20851,"distinguishing":20852,"##yler":20853,"grasping":20854,"sermons":20855,"tomato":20856,"bland":20857,"stimulation":20858,"avenues":20859,"##eux":20860,"spreads":20861,"scarlett":20862,"fern":20863,"pentagon":20864,"assert":20865,"baird":20866,"chesapeake":20867,"ir":20868,"calmed":20869,"distortion":20870,"fatalities":20871,"##olis":20872,"correctional":20873,"pricing":20874,"##astic":20875,"##gina":20876,"prom":20877,"dammit":20878,"ying":20879,"collaborate":20880,"##chia":20881,"welterweight":20882,"33rd":20883,"pointer":20884,"substitution":20885,"bonded":20886,"umpire":20887,"communicating":20888,"multitude":20889,"paddle":20890,"##obe":20891,"federally":20892,"intimacy":20893,"##insky":20894,"betray":20895,"ssr":20896,"##lett":20897,"##lean":20898,"##lves":20899,"##therapy":20900,"airbus":20901,"##tery":20902,"functioned":20903,"ud":20904,"bearer":20905,"biomedical":20906,"netflix":20907,"##hire":20908,"##nca":20909,"condom":20910,"brink":20911,"ik":20912,"##nical":20913,"macy":20914,"##bet":20915,"flap":20916,"gma":20917,"experimented":20918,"jelly":20919,"lavender":20920,"##icles":20921,"##ulia":20922,"munro":20923,"##mian":20924,"##tial":20925,"rye":20926,"##rle":20927,"60th":20928,"gigs":20929,"hottest":20930,"rotated":20931,"predictions":20932,"fuji":20933,"bu":20934,"##erence":20935,"##omi":20936,"barangay":20937,"##fulness":20938,"##sas":20939,"clocks":20940,"##rwood":20941,"##liness":20942,"cereal":20943,"roe":20944,"wight":20945,"decker":20946,"uttered":20947,"babu":20948,"onion":20949,"xml":20950,"forcibly":20951,"##df":20952,"petra":20953,"sarcasm":20954,"hartley":20955,"peeled":20956,"storytelling":20957,"##42":20958,"##xley":20959,"##ysis":20960,"##ffa":20961,"fibre":20962,"kiel":20963,"auditor":20964,"fig":20965,"harald":20966,"greenville":20967,"##berries":20968,"geographically":20969,"nell":20970,"quartz":20971,"##athic":20972,"cemeteries":20973,"##lr":20974,"crossings":20975,"nah":20976,"holloway":20977,"reptiles":20978,"chun":20979,"sichuan":20980,"snowy":20981,"660":20982,"corrections":20983,"##ivo":20984,"zheng":20985,"ambassadors":20986,"blacksmith":20987,"fielded":20988,"fluids":20989,"hardcover":20990,"turnover":20991,"medications":20992,"melvin":20993,"academies":20994,"##erton":20995,"ro":20996,"roach":20997,"absorbing":20998,"spaniards":20999,"colton":21000,"##founded":21001,"outsider":21002,"espionage":21003,"kelsey":21004,"245":21005,"edible":21006,"##ulf":21007,"dora":21008,"establishes":21009,"##sham":21010,"##tries":21011,"contracting":21012,"##tania":21013,"cinematic":21014,"costello":21015,"nesting":21016,"##uron":21017,"connolly":21018,"duff":21019,"##nology":21020,"mma":21021,"##mata":21022,"fergus":21023,"sexes":21024,"gi":21025,"optics":21026,"spectator":21027,"woodstock":21028,"banning":21029,"##hee":21030,"##fle":21031,"differentiate":21032,"outfielder":21033,"refinery":21034,"226":21035,"312":21036,"gerhard":21037,"horde":21038,"lair":21039,"drastically":21040,"##udi":21041,"landfall":21042,"##cheng":21043,"motorsport":21044,"odi":21045,"##achi":21046,"predominant":21047,"quay":21048,"skins":21049,"##ental":21050,"edna":21051,"harshly":21052,"complementary":21053,"murdering":21054,"##aves":21055,"wreckage":21056,"##90":21057,"ono":21058,"outstretched":21059,"lennox":21060,"munitions":21061,"galen":21062,"reconcile":21063,"470":21064,"scalp":21065,"bicycles":21066,"gillespie":21067,"questionable":21068,"rosenberg":21069,"guillermo":21070,"hostel":21071,"jarvis":21072,"kabul":21073,"volvo":21074,"opium":21075,"yd":21076,"##twined":21077,"abuses":21078,"decca":21079,"outpost":21080,"##cino":21081,"sensible":21082,"neutrality":21083,"##64":21084,"ponce":21085,"anchorage":21086,"atkins":21087,"turrets":21088,"inadvertently":21089,"disagree":21090,"libre":21091,"vodka":21092,"reassuring":21093,"weighs":21094,"##yal":21095,"glide":21096,"jumper":21097,"ceilings":21098,"repertory":21099,"outs":21100,"stain":21101,"##bial":21102,"envy":21103,"##ucible":21104,"smashing":21105,"heightened":21106,"policing":21107,"hyun":21108,"mixes":21109,"lai":21110,"prima":21111,"##ples":21112,"celeste":21113,"##bina":21114,"lucrative":21115,"intervened":21116,"kc":21117,"manually":21118,"##rned":21119,"stature":21120,"staffed":21121,"bun":21122,"bastards":21123,"nairobi":21124,"priced":21125,"##auer":21126,"thatcher":21127,"##kia":21128,"tripped":21129,"comune":21130,"##ogan":21131,"##pled":21132,"brasil":21133,"incentives":21134,"emanuel":21135,"hereford":21136,"musica":21137,"##kim":21138,"benedictine":21139,"biennale":21140,"##lani":21141,"eureka":21142,"gardiner":21143,"rb":21144,"knocks":21145,"sha":21146,"##ael":21147,"##elled":21148,"##onate":21149,"efficacy":21150,"ventura":21151,"masonic":21152,"sanford":21153,"maize":21154,"leverage":21155,"##feit":21156,"capacities":21157,"santana":21158,"##aur":21159,"novelty":21160,"vanilla":21161,"##cter":21162,"##tour":21163,"benin":21164,"##oir":21165,"##rain":21166,"neptune":21167,"drafting":21168,"tallinn":21169,"##cable":21170,"humiliation":21171,"##boarding":21172,"schleswig":21173,"fabian":21174,"bernardo":21175,"liturgy":21176,"spectacle":21177,"sweeney":21178,"pont":21179,"routledge":21180,"##tment":21181,"cosmos":21182,"ut":21183,"hilt":21184,"sleek":21185,"universally":21186,"##eville":21187,"##gawa":21188,"typed":21189,"##dry":21190,"favors":21191,"allegheny":21192,"glaciers":21193,"##rly":21194,"recalling":21195,"aziz":21196,"##log":21197,"parasite":21198,"requiem":21199,"auf":21200,"##berto":21201,"##llin":21202,"illumination":21203,"##breaker":21204,"##issa":21205,"festivities":21206,"bows":21207,"govern":21208,"vibe":21209,"vp":21210,"333":21211,"sprawled":21212,"larson":21213,"pilgrim":21214,"bwf":21215,"leaping":21216,"##rts":21217,"##ssel":21218,"alexei":21219,"greyhound":21220,"hoarse":21221,"##dler":21222,"##oration":21223,"seneca":21224,"##cule":21225,"gaping":21226,"##ulously":21227,"##pura":21228,"cinnamon":21229,"##gens":21230,"##rricular":21231,"craven":21232,"fantasies":21233,"houghton":21234,"engined":21235,"reigned":21236,"dictator":21237,"supervising":21238,"##oris":21239,"bogota":21240,"commentaries":21241,"unnatural":21242,"fingernails":21243,"spirituality":21244,"tighten":21245,"##tm":21246,"canadiens":21247,"protesting":21248,"intentional":21249,"cheers":21250,"sparta":21251,"##ytic":21252,"##iere":21253,"##zine":21254,"widen":21255,"belgarath":21256,"controllers":21257,"dodd":21258,"iaaf":21259,"navarre":21260,"##ication":21261,"defect":21262,"squire":21263,"steiner":21264,"whisky":21265,"##mins":21266,"560":21267,"inevitably":21268,"tome":21269,"##gold":21270,"chew":21271,"##uid":21272,"##lid":21273,"elastic":21274,"##aby":21275,"streaked":21276,"alliances":21277,"jailed":21278,"regal":21279,"##ined":21280,"##phy":21281,"czechoslovak":21282,"narration":21283,"absently":21284,"##uld":21285,"bluegrass":21286,"guangdong":21287,"quran":21288,"criticizing":21289,"hose":21290,"hari":21291,"##liest":21292,"##owa":21293,"skier":21294,"streaks":21295,"deploy":21296,"##lom":21297,"raft":21298,"bose":21299,"dialed":21300,"huff":21301,"##eira":21302,"haifa":21303,"simplest":21304,"bursting":21305,"endings":21306,"ib":21307,"sultanate":21308,"##titled":21309,"franks":21310,"whitman":21311,"ensures":21312,"sven":21313,"##ggs":21314,"collaborators":21315,"forster":21316,"organising":21317,"ui":21318,"banished":21319,"napier":21320,"injustice":21321,"teller":21322,"layered":21323,"thump":21324,"##otti":21325,"roc":21326,"battleships":21327,"evidenced":21328,"fugitive":21329,"sadie":21330,"robotics":21331,"##roud":21332,"equatorial":21333,"geologist":21334,"##iza":21335,"yielding":21336,"##bron":21337,"##sr":21338,"internationale":21339,"mecca":21340,"##diment":21341,"sbs":21342,"skyline":21343,"toad":21344,"uploaded":21345,"reflective":21346,"undrafted":21347,"lal":21348,"leafs":21349,"bayern":21350,"##dai":21351,"lakshmi":21352,"shortlisted":21353,"##stick":21354,"##wicz":21355,"camouflage":21356,"donate":21357,"af":21358,"christi":21359,"lau":21360,"##acio":21361,"disclosed":21362,"nemesis":21363,"1761":21364,"assemble":21365,"straining":21366,"northamptonshire":21367,"tal":21368,"##asi":21369,"bernardino":21370,"premature":21371,"heidi":21372,"42nd":21373,"coefficients":21374,"galactic":21375,"reproduce":21376,"buzzed":21377,"sensations":21378,"zionist":21379,"monsieur":21380,"myrtle":21381,"##eme":21382,"archery":21383,"strangled":21384,"musically":21385,"viewpoint":21386,"antiquities":21387,"bei":21388,"trailers":21389,"seahawks":21390,"cured":21391,"pee":21392,"preferring":21393,"tasmanian":21394,"lange":21395,"sul":21396,"##mail":21397,"##working":21398,"colder":21399,"overland":21400,"lucivar":21401,"massey":21402,"gatherings":21403,"haitian":21404,"##smith":21405,"disapproval":21406,"flaws":21407,"##cco":21408,"##enbach":21409,"1766":21410,"npr":21411,"##icular":21412,"boroughs":21413,"creole":21414,"forums":21415,"techno":21416,"1755":21417,"dent":21418,"abdominal":21419,"streetcar":21420,"##eson":21421,"##stream":21422,"procurement":21423,"gemini":21424,"predictable":21425,"##tya":21426,"acheron":21427,"christoph":21428,"feeder":21429,"fronts":21430,"vendor":21431,"bernhard":21432,"jammu":21433,"tumors":21434,"slang":21435,"##uber":21436,"goaltender":21437,"twists":21438,"curving":21439,"manson":21440,"vuelta":21441,"mer":21442,"peanut":21443,"confessions":21444,"pouch":21445,"unpredictable":21446,"allowance":21447,"theodor":21448,"vascular":21449,"##factory":21450,"bala":21451,"authenticity":21452,"metabolic":21453,"coughing":21454,"nanjing":21455,"##cea":21456,"pembroke":21457,"##bard":21458,"splendid":21459,"36th":21460,"ff":21461,"hourly":21462,"##ahu":21463,"elmer":21464,"handel":21465,"##ivate":21466,"awarding":21467,"thrusting":21468,"dl":21469,"experimentation":21470,"##hesion":21471,"##46":21472,"caressed":21473,"entertained":21474,"steak":21475,"##rangle":21476,"biologist":21477,"orphans":21478,"baroness":21479,"oyster":21480,"stepfather":21481,"##dridge":21482,"mirage":21483,"reefs":21484,"speeding":21485,"##31":21486,"barons":21487,"1764":21488,"227":21489,"inhabit":21490,"preached":21491,"repealed":21492,"##tral":21493,"honoring":21494,"boogie":21495,"captives":21496,"administer":21497,"johanna":21498,"##imate":21499,"gel":21500,"suspiciously":21501,"1767":21502,"sobs":21503,"##dington":21504,"backbone":21505,"hayward":21506,"garry":21507,"##folding":21508,"##nesia":21509,"maxi":21510,"##oof":21511,"##ppe":21512,"ellison":21513,"galileo":21514,"##stand":21515,"crimea":21516,"frenzy":21517,"amour":21518,"bumper":21519,"matrices":21520,"natalia":21521,"baking":21522,"garth":21523,"palestinians":21524,"##grove":21525,"smack":21526,"conveyed":21527,"ensembles":21528,"gardening":21529,"##manship":21530,"##rup":21531,"##stituting":21532,"1640":21533,"harvesting":21534,"topography":21535,"jing":21536,"shifters":21537,"dormitory":21538,"##carriage":21539,"##lston":21540,"ist":21541,"skulls":21542,"##stadt":21543,"dolores":21544,"jewellery":21545,"sarawak":21546,"##wai":21547,"##zier":21548,"fences":21549,"christy":21550,"confinement":21551,"tumbling":21552,"credibility":21553,"fir":21554,"stench":21555,"##bria":21556,"##plication":21557,"##nged":21558,"##sam":21559,"virtues":21560,"##belt":21561,"marjorie":21562,"pba":21563,"##eem":21564,"##made":21565,"celebrates":21566,"schooner":21567,"agitated":21568,"barley":21569,"fulfilling":21570,"anthropologist":21571,"##pro":21572,"restrict":21573,"novi":21574,"regulating":21575,"##nent":21576,"padres":21577,"##rani":21578,"##hesive":21579,"loyola":21580,"tabitha":21581,"milky":21582,"olson":21583,"proprietor":21584,"crambidae":21585,"guarantees":21586,"intercollegiate":21587,"ljubljana":21588,"hilda":21589,"##sko":21590,"ignorant":21591,"hooded":21592,"##lts":21593,"sardinia":21594,"##lidae":21595,"##vation":21596,"frontman":21597,"privileged":21598,"witchcraft":21599,"##gp":21600,"jammed":21601,"laude":21602,"poking":21603,"##than":21604,"bracket":21605,"amazement":21606,"yunnan":21607,"##erus":21608,"maharaja":21609,"linnaeus":21610,"264":21611,"commissioning":21612,"milano":21613,"peacefully":21614,"##logies":21615,"akira":21616,"rani":21617,"regulator":21618,"##36":21619,"grasses":21620,"##rance":21621,"luzon":21622,"crows":21623,"compiler":21624,"gretchen":21625,"seaman":21626,"edouard":21627,"tab":21628,"buccaneers":21629,"ellington":21630,"hamlets":21631,"whig":21632,"socialists":21633,"##anto":21634,"directorial":21635,"easton":21636,"mythological":21637,"##kr":21638,"##vary":21639,"rhineland":21640,"semantic":21641,"taut":21642,"dune":21643,"inventions":21644,"succeeds":21645,"##iter":21646,"replication":21647,"branched":21648,"##pired":21649,"jul":21650,"prosecuted":21651,"kangaroo":21652,"penetrated":21653,"##avian":21654,"middlesbrough":21655,"doses":21656,"bleak":21657,"madam":21658,"predatory":21659,"relentless":21660,"##vili":21661,"reluctance":21662,"##vir":21663,"hailey":21664,"crore":21665,"silvery":21666,"1759":21667,"monstrous":21668,"swimmers":21669,"transmissions":21670,"hawthorn":21671,"informing":21672,"##eral":21673,"toilets":21674,"caracas":21675,"crouch":21676,"kb":21677,"##sett":21678,"295":21679,"cartel":21680,"hadley":21681,"##aling":21682,"alexia":21683,"yvonne":21684,"##biology":21685,"cinderella":21686,"eton":21687,"superb":21688,"blizzard":21689,"stabbing":21690,"industrialist":21691,"maximus":21692,"##gm":21693,"##orus":21694,"groves":21695,"maud":21696,"clade":21697,"oversized":21698,"comedic":21699,"##bella":21700,"rosen":21701,"nomadic":21702,"fulham":21703,"montane":21704,"beverages":21705,"galaxies":21706,"redundant":21707,"swarm":21708,"##rot":21709,"##folia":21710,"##llis":21711,"buckinghamshire":21712,"fen":21713,"bearings":21714,"bahadur":21715,"##rom":21716,"gilles":21717,"phased":21718,"dynamite":21719,"faber":21720,"benoit":21721,"vip":21722,"##ount":21723,"##wd":21724,"booking":21725,"fractured":21726,"tailored":21727,"anya":21728,"spices":21729,"westwood":21730,"cairns":21731,"auditions":21732,"inflammation":21733,"steamed":21734,"##rocity":21735,"##acion":21736,"##urne":21737,"skyla":21738,"thereof":21739,"watford":21740,"torment":21741,"archdeacon":21742,"transforms":21743,"lulu":21744,"demeanor":21745,"fucked":21746,"serge":21747,"##sor":21748,"mckenna":21749,"minas":21750,"entertainer":21751,"##icide":21752,"caress":21753,"originate":21754,"residue":21755,"##sty":21756,"1740":21757,"##ilised":21758,"##org":21759,"beech":21760,"##wana":21761,"subsidies":21762,"##ghton":21763,"emptied":21764,"gladstone":21765,"ru":21766,"firefighters":21767,"voodoo":21768,"##rcle":21769,"het":21770,"nightingale":21771,"tamara":21772,"edmond":21773,"ingredient":21774,"weaknesses":21775,"silhouette":21776,"285":21777,"compatibility":21778,"withdrawing":21779,"hampson":21780,"##mona":21781,"anguish":21782,"giggling":21783,"##mber":21784,"bookstore":21785,"##jiang":21786,"southernmost":21787,"tilting":21788,"##vance":21789,"bai":21790,"economical":21791,"rf":21792,"briefcase":21793,"dreadful":21794,"hinted":21795,"projections":21796,"shattering":21797,"totaling":21798,"##rogate":21799,"analogue":21800,"indicted":21801,"periodical":21802,"fullback":21803,"##dman":21804,"haynes":21805,"##tenberg":21806,"##ffs":21807,"##ishment":21808,"1745":21809,"thirst":21810,"stumble":21811,"penang":21812,"vigorous":21813,"##ddling":21814,"##kor":21815,"##lium":21816,"octave":21817,"##ove":21818,"##enstein":21819,"##inen":21820,"##ones":21821,"siberian":21822,"##uti":21823,"cbn":21824,"repeal":21825,"swaying":21826,"##vington":21827,"khalid":21828,"tanaka":21829,"unicorn":21830,"otago":21831,"plastered":21832,"lobe":21833,"riddle":21834,"##rella":21835,"perch":21836,"##ishing":21837,"croydon":21838,"filtered":21839,"graeme":21840,"tripoli":21841,"##ossa":21842,"crocodile":21843,"##chers":21844,"sufi":21845,"mined":21846,"##tung":21847,"inferno":21848,"lsu":21849,"##phi":21850,"swelled":21851,"utilizes":21852,"£2":21853,"cale":21854,"periodicals":21855,"styx":21856,"hike":21857,"informally":21858,"coop":21859,"lund":21860,"##tidae":21861,"ala":21862,"hen":21863,"qui":21864,"transformations":21865,"disposed":21866,"sheath":21867,"chickens":21868,"##cade":21869,"fitzroy":21870,"sas":21871,"silesia":21872,"unacceptable":21873,"odisha":21874,"1650":21875,"sabrina":21876,"pe":21877,"spokane":21878,"ratios":21879,"athena":21880,"massage":21881,"shen":21882,"dilemma":21883,"##drum":21884,"##riz":21885,"##hul":21886,"corona":21887,"doubtful":21888,"niall":21889,"##pha":21890,"##bino":21891,"fines":21892,"cite":21893,"acknowledging":21894,"bangor":21895,"ballard":21896,"bathurst":21897,"##resh":21898,"huron":21899,"mustered":21900,"alzheimer":21901,"garments":21902,"kinase":21903,"tyre":21904,"warship":21905,"##cp":21906,"flashback":21907,"pulmonary":21908,"braun":21909,"cheat":21910,"kamal":21911,"cyclists":21912,"constructions":21913,"grenades":21914,"ndp":21915,"traveller":21916,"excuses":21917,"stomped":21918,"signalling":21919,"trimmed":21920,"futsal":21921,"mosques":21922,"relevance":21923,"##wine":21924,"wta":21925,"##23":21926,"##vah":21927,"##lter":21928,"hoc":21929,"##riding":21930,"optimistic":21931,"##´s":21932,"deco":21933,"sim":21934,"interacting":21935,"rejecting":21936,"moniker":21937,"waterways":21938,"##ieri":21939,"##oku":21940,"mayors":21941,"gdansk":21942,"outnumbered":21943,"pearls":21944,"##ended":21945,"##hampton":21946,"fairs":21947,"totals":21948,"dominating":21949,"262":21950,"notions":21951,"stairway":21952,"compiling":21953,"pursed":21954,"commodities":21955,"grease":21956,"yeast":21957,"##jong":21958,"carthage":21959,"griffiths":21960,"residual":21961,"amc":21962,"contraction":21963,"laird":21964,"sapphire":21965,"##marine":21966,"##ivated":21967,"amalgamation":21968,"dissolve":21969,"inclination":21970,"lyle":21971,"packaged":21972,"altitudes":21973,"suez":21974,"canons":21975,"graded":21976,"lurched":21977,"narrowing":21978,"boasts":21979,"guise":21980,"wed":21981,"enrico":21982,"##ovsky":21983,"rower":21984,"scarred":21985,"bree":21986,"cub":21987,"iberian":21988,"protagonists":21989,"bargaining":21990,"proposing":21991,"trainers":21992,"voyages":21993,"vans":21994,"fishes":21995,"##aea":21996,"##ivist":21997,"##verance":21998,"encryption":21999,"artworks":22000,"kazan":22001,"sabre":22002,"cleopatra":22003,"hepburn":22004,"rotting":22005,"supremacy":22006,"mecklenburg":22007,"##brate":22008,"burrows":22009,"hazards":22010,"outgoing":22011,"flair":22012,"organizes":22013,"##ctions":22014,"scorpion":22015,"##usions":22016,"boo":22017,"234":22018,"chevalier":22019,"dunedin":22020,"slapping":22021,"##34":22022,"ineligible":22023,"pensions":22024,"##38":22025,"##omic":22026,"manufactures":22027,"emails":22028,"bismarck":22029,"238":22030,"weakening":22031,"blackish":22032,"ding":22033,"mcgee":22034,"quo":22035,"##rling":22036,"northernmost":22037,"xx":22038,"manpower":22039,"greed":22040,"sampson":22041,"clicking":22042,"##ange":22043,"##horpe":22044,"##inations":22045,"##roving":22046,"torre":22047,"##eptive":22048,"##moral":22049,"symbolism":22050,"38th":22051,"asshole":22052,"meritorious":22053,"outfits":22054,"splashed":22055,"biographies":22056,"sprung":22057,"astros":22058,"##tale":22059,"302":22060,"737":22061,"filly":22062,"raoul":22063,"nw":22064,"tokugawa":22065,"linden":22066,"clubhouse":22067,"##apa":22068,"tracts":22069,"romano":22070,"##pio":22071,"putin":22072,"tags":22073,"##note":22074,"chained":22075,"dickson":22076,"gunshot":22077,"moe":22078,"gunn":22079,"rashid":22080,"##tails":22081,"zipper":22082,"##bas":22083,"##nea":22084,"contrasted":22085,"##ply":22086,"##udes":22087,"plum":22088,"pharaoh":22089,"##pile":22090,"aw":22091,"comedies":22092,"ingrid":22093,"sandwiches":22094,"subdivisions":22095,"1100":22096,"mariana":22097,"nokia":22098,"kamen":22099,"hz":22100,"delaney":22101,"veto":22102,"herring":22103,"##words":22104,"possessive":22105,"outlines":22106,"##roup":22107,"siemens":22108,"stairwell":22109,"rc":22110,"gallantry":22111,"messiah":22112,"palais":22113,"yells":22114,"233":22115,"zeppelin":22116,"##dm":22117,"bolivar":22118,"##cede":22119,"smackdown":22120,"mckinley":22121,"##mora":22122,"##yt":22123,"muted":22124,"geologic":22125,"finely":22126,"unitary":22127,"avatar":22128,"hamas":22129,"maynard":22130,"rees":22131,"bog":22132,"contrasting":22133,"##rut":22134,"liv":22135,"chico":22136,"disposition":22137,"pixel":22138,"##erate":22139,"becca":22140,"dmitry":22141,"yeshiva":22142,"narratives":22143,"##lva":22144,"##ulton":22145,"mercenary":22146,"sharpe":22147,"tempered":22148,"navigate":22149,"stealth":22150,"amassed":22151,"keynes":22152,"##lini":22153,"untouched":22154,"##rrie":22155,"havoc":22156,"lithium":22157,"##fighting":22158,"abyss":22159,"graf":22160,"southward":22161,"wolverine":22162,"balloons":22163,"implements":22164,"ngos":22165,"transitions":22166,"##icum":22167,"ambushed":22168,"concacaf":22169,"dormant":22170,"economists":22171,"##dim":22172,"costing":22173,"csi":22174,"rana":22175,"universite":22176,"boulders":22177,"verity":22178,"##llon":22179,"collin":22180,"mellon":22181,"misses":22182,"cypress":22183,"fluorescent":22184,"lifeless":22185,"spence":22186,"##ulla":22187,"crewe":22188,"shepard":22189,"pak":22190,"revelations":22191,"##م":22192,"jolly":22193,"gibbons":22194,"paw":22195,"##dro":22196,"##quel":22197,"freeing":22198,"##test":22199,"shack":22200,"fries":22201,"palatine":22202,"##51":22203,"##hiko":22204,"accompaniment":22205,"cruising":22206,"recycled":22207,"##aver":22208,"erwin":22209,"sorting":22210,"synthesizers":22211,"dyke":22212,"realities":22213,"sg":22214,"strides":22215,"enslaved":22216,"wetland":22217,"##ghan":22218,"competence":22219,"gunpowder":22220,"grassy":22221,"maroon":22222,"reactors":22223,"objection":22224,"##oms":22225,"carlson":22226,"gearbox":22227,"macintosh":22228,"radios":22229,"shelton":22230,"##sho":22231,"clergyman":22232,"prakash":22233,"254":22234,"mongols":22235,"trophies":22236,"oricon":22237,"228":22238,"stimuli":22239,"twenty20":22240,"cantonese":22241,"cortes":22242,"mirrored":22243,"##saurus":22244,"bhp":22245,"cristina":22246,"melancholy":22247,"##lating":22248,"enjoyable":22249,"nuevo":22250,"##wny":22251,"downfall":22252,"schumacher":22253,"##ind":22254,"banging":22255,"lausanne":22256,"rumbled":22257,"paramilitary":22258,"reflex":22259,"ax":22260,"amplitude":22261,"migratory":22262,"##gall":22263,"##ups":22264,"midi":22265,"barnard":22266,"lastly":22267,"sherry":22268,"##hp":22269,"##nall":22270,"keystone":22271,"##kra":22272,"carleton":22273,"slippery":22274,"##53":22275,"coloring":22276,"foe":22277,"socket":22278,"otter":22279,"##rgos":22280,"mats":22281,"##tose":22282,"consultants":22283,"bafta":22284,"bison":22285,"topping":22286,"##km":22287,"490":22288,"primal":22289,"abandonment":22290,"transplant":22291,"atoll":22292,"hideous":22293,"mort":22294,"pained":22295,"reproduced":22296,"tae":22297,"howling":22298,"##turn":22299,"unlawful":22300,"billionaire":22301,"hotter":22302,"poised":22303,"lansing":22304,"##chang":22305,"dinamo":22306,"retro":22307,"messing":22308,"nfc":22309,"domesday":22310,"##mina":22311,"blitz":22312,"timed":22313,"##athing":22314,"##kley":22315,"ascending":22316,"gesturing":22317,"##izations":22318,"signaled":22319,"tis":22320,"chinatown":22321,"mermaid":22322,"savanna":22323,"jameson":22324,"##aint":22325,"catalina":22326,"##pet":22327,"##hers":22328,"cochrane":22329,"cy":22330,"chatting":22331,"##kus":22332,"alerted":22333,"computation":22334,"mused":22335,"noelle":22336,"majestic":22337,"mohawk":22338,"campo":22339,"octagonal":22340,"##sant":22341,"##hend":22342,"241":22343,"aspiring":22344,"##mart":22345,"comprehend":22346,"iona":22347,"paralyzed":22348,"shimmering":22349,"swindon":22350,"rhone":22351,"##eley":22352,"reputed":22353,"configurations":22354,"pitchfork":22355,"agitation":22356,"francais":22357,"gillian":22358,"lipstick":22359,"##ilo":22360,"outsiders":22361,"pontifical":22362,"resisting":22363,"bitterness":22364,"sewer":22365,"rockies":22366,"##edd":22367,"##ucher":22368,"misleading":22369,"1756":22370,"exiting":22371,"galloway":22372,"##nging":22373,"risked":22374,"##heart":22375,"246":22376,"commemoration":22377,"schultz":22378,"##rka":22379,"integrating":22380,"##rsa":22381,"poses":22382,"shrieked":22383,"##weiler":22384,"guineas":22385,"gladys":22386,"jerking":22387,"owls":22388,"goldsmith":22389,"nightly":22390,"penetrating":22391,"##unced":22392,"lia":22393,"##33":22394,"ignited":22395,"betsy":22396,"##aring":22397,"##thorpe":22398,"follower":22399,"vigorously":22400,"##rave":22401,"coded":22402,"kiran":22403,"knit":22404,"zoology":22405,"tbilisi":22406,"##28":22407,"##bered":22408,"repository":22409,"govt":22410,"deciduous":22411,"dino":22412,"growling":22413,"##bba":22414,"enhancement":22415,"unleashed":22416,"chanting":22417,"pussy":22418,"biochemistry":22419,"##eric":22420,"kettle":22421,"repression":22422,"toxicity":22423,"nrhp":22424,"##arth":22425,"##kko":22426,"##bush":22427,"ernesto":22428,"commended":22429,"outspoken":22430,"242":22431,"mca":22432,"parchment":22433,"sms":22434,"kristen":22435,"##aton":22436,"bisexual":22437,"raked":22438,"glamour":22439,"navajo":22440,"a2":22441,"conditioned":22442,"showcased":22443,"##hma":22444,"spacious":22445,"youthful":22446,"##esa":22447,"usl":22448,"appliances":22449,"junta":22450,"brest":22451,"layne":22452,"conglomerate":22453,"enchanted":22454,"chao":22455,"loosened":22456,"picasso":22457,"circulating":22458,"inspect":22459,"montevideo":22460,"##centric":22461,"##kti":22462,"piazza":22463,"spurred":22464,"##aith":22465,"bari":22466,"freedoms":22467,"poultry":22468,"stamford":22469,"lieu":22470,"##ect":22471,"indigo":22472,"sarcastic":22473,"bahia":22474,"stump":22475,"attach":22476,"dvds":22477,"frankenstein":22478,"lille":22479,"approx":22480,"scriptures":22481,"pollen":22482,"##script":22483,"nmi":22484,"overseen":22485,"##ivism":22486,"tides":22487,"proponent":22488,"newmarket":22489,"inherit":22490,"milling":22491,"##erland":22492,"centralized":22493,"##rou":22494,"distributors":22495,"credentials":22496,"drawers":22497,"abbreviation":22498,"##lco":22499,"##xon":22500,"downing":22501,"uncomfortably":22502,"ripe":22503,"##oes":22504,"erase":22505,"franchises":22506,"##ever":22507,"populace":22508,"##bery":22509,"##khar":22510,"decomposition":22511,"pleas":22512,"##tet":22513,"daryl":22514,"sabah":22515,"##stle":22516,"##wide":22517,"fearless":22518,"genie":22519,"lesions":22520,"annette":22521,"##ogist":22522,"oboe":22523,"appendix":22524,"nair":22525,"dripped":22526,"petitioned":22527,"maclean":22528,"mosquito":22529,"parrot":22530,"rpg":22531,"hampered":22532,"1648":22533,"operatic":22534,"reservoirs":22535,"##tham":22536,"irrelevant":22537,"jolt":22538,"summarized":22539,"##fp":22540,"medallion":22541,"##taff":22542,"##−":22543,"clawed":22544,"harlow":22545,"narrower":22546,"goddard":22547,"marcia":22548,"bodied":22549,"fremont":22550,"suarez":22551,"altering":22552,"tempest":22553,"mussolini":22554,"porn":22555,"##isms":22556,"sweetly":22557,"oversees":22558,"walkers":22559,"solitude":22560,"grimly":22561,"shrines":22562,"hk":22563,"ich":22564,"supervisors":22565,"hostess":22566,"dietrich":22567,"legitimacy":22568,"brushes":22569,"expressive":22570,"##yp":22571,"dissipated":22572,"##rse":22573,"localized":22574,"systemic":22575,"##nikov":22576,"gettysburg":22577,"##js":22578,"##uaries":22579,"dialogues":22580,"muttering":22581,"251":22582,"housekeeper":22583,"sicilian":22584,"discouraged":22585,"##frey":22586,"beamed":22587,"kaladin":22588,"halftime":22589,"kidnap":22590,"##amo":22591,"##llet":22592,"1754":22593,"synonymous":22594,"depleted":22595,"instituto":22596,"insulin":22597,"reprised":22598,"##opsis":22599,"clashed":22600,"##ctric":22601,"interrupting":22602,"radcliffe":22603,"insisting":22604,"medici":22605,"1715":22606,"ejected":22607,"playfully":22608,"turbulent":22609,"##47":22610,"starvation":22611,"##rini":22612,"shipment":22613,"rebellious":22614,"petersen":22615,"verification":22616,"merits":22617,"##rified":22618,"cakes":22619,"##charged":22620,"1757":22621,"milford":22622,"shortages":22623,"spying":22624,"fidelity":22625,"##aker":22626,"emitted":22627,"storylines":22628,"harvested":22629,"seismic":22630,"##iform":22631,"cheung":22632,"kilda":22633,"theoretically":22634,"barbie":22635,"lynx":22636,"##rgy":22637,"##tius":22638,"goblin":22639,"mata":22640,"poisonous":22641,"##nburg":22642,"reactive":22643,"residues":22644,"obedience":22645,"##евич":22646,"conjecture":22647,"##rac":22648,"401":22649,"hating":22650,"sixties":22651,"kicker":22652,"moaning":22653,"motown":22654,"##bha":22655,"emancipation":22656,"neoclassical":22657,"##hering":22658,"consoles":22659,"ebert":22660,"professorship":22661,"##tures":22662,"sustaining":22663,"assaults":22664,"obeyed":22665,"affluent":22666,"incurred":22667,"tornadoes":22668,"##eber":22669,"##zow":22670,"emphasizing":22671,"highlanders":22672,"cheated":22673,"helmets":22674,"##ctus":22675,"internship":22676,"terence":22677,"bony":22678,"executions":22679,"legislators":22680,"berries":22681,"peninsular":22682,"tinged":22683,"##aco":22684,"1689":22685,"amplifier":22686,"corvette":22687,"ribbons":22688,"lavish":22689,"pennant":22690,"##lander":22691,"worthless":22692,"##chfield":22693,"##forms":22694,"mariano":22695,"pyrenees":22696,"expenditures":22697,"##icides":22698,"chesterfield":22699,"mandir":22700,"tailor":22701,"39th":22702,"sergey":22703,"nestled":22704,"willed":22705,"aristocracy":22706,"devotees":22707,"goodnight":22708,"raaf":22709,"rumored":22710,"weaponry":22711,"remy":22712,"appropriations":22713,"harcourt":22714,"burr":22715,"riaa":22716,"##lence":22717,"limitation":22718,"unnoticed":22719,"guo":22720,"soaking":22721,"swamps":22722,"##tica":22723,"collapsing":22724,"tatiana":22725,"descriptive":22726,"brigham":22727,"psalm":22728,"##chment":22729,"maddox":22730,"##lization":22731,"patti":22732,"caliph":22733,"##aja":22734,"akron":22735,"injuring":22736,"serra":22737,"##ganj":22738,"basins":22739,"##sari":22740,"astonished":22741,"launcher":22742,"##church":22743,"hilary":22744,"wilkins":22745,"sewing":22746,"##sf":22747,"stinging":22748,"##fia":22749,"##ncia":22750,"underwood":22751,"startup":22752,"##ition":22753,"compilations":22754,"vibrations":22755,"embankment":22756,"jurist":22757,"##nity":22758,"bard":22759,"juventus":22760,"groundwater":22761,"kern":22762,"palaces":22763,"helium":22764,"boca":22765,"cramped":22766,"marissa":22767,"soto":22768,"##worm":22769,"jae":22770,"princely":22771,"##ggy":22772,"faso":22773,"bazaar":22774,"warmly":22775,"##voking":22776,"229":22777,"pairing":22778,"##lite":22779,"##grate":22780,"##nets":22781,"wien":22782,"freaked":22783,"ulysses":22784,"rebirth":22785,"##alia":22786,"##rent":22787,"mummy":22788,"guzman":22789,"jimenez":22790,"stilled":22791,"##nitz":22792,"trajectory":22793,"tha":22794,"woken":22795,"archival":22796,"professions":22797,"##pts":22798,"##pta":22799,"hilly":22800,"shadowy":22801,"shrink":22802,"##bolt":22803,"norwood":22804,"glued":22805,"migrate":22806,"stereotypes":22807,"devoid":22808,"##pheus":22809,"625":22810,"evacuate":22811,"horrors":22812,"infancy":22813,"gotham":22814,"knowles":22815,"optic":22816,"downloaded":22817,"sachs":22818,"kingsley":22819,"parramatta":22820,"darryl":22821,"mor":22822,"##onale":22823,"shady":22824,"commence":22825,"confesses":22826,"kan":22827,"##meter":22828,"##placed":22829,"marlborough":22830,"roundabout":22831,"regents":22832,"frigates":22833,"io":22834,"##imating":22835,"gothenburg":22836,"revoked":22837,"carvings":22838,"clockwise":22839,"convertible":22840,"intruder":22841,"##sche":22842,"banged":22843,"##ogo":22844,"vicky":22845,"bourgeois":22846,"##mony":22847,"dupont":22848,"footing":22849,"##gum":22850,"pd":22851,"##real":22852,"buckle":22853,"yun":22854,"penthouse":22855,"sane":22856,"720":22857,"serviced":22858,"stakeholders":22859,"neumann":22860,"bb":22861,"##eers":22862,"comb":22863,"##gam":22864,"catchment":22865,"pinning":22866,"rallies":22867,"typing":22868,"##elles":22869,"forefront":22870,"freiburg":22871,"sweetie":22872,"giacomo":22873,"widowed":22874,"goodwill":22875,"worshipped":22876,"aspirations":22877,"midday":22878,"##vat":22879,"fishery":22880,"##trick":22881,"bournemouth":22882,"turk":22883,"243":22884,"hearth":22885,"ethanol":22886,"guadalajara":22887,"murmurs":22888,"sl":22889,"##uge":22890,"afforded":22891,"scripted":22892,"##hta":22893,"wah":22894,"##jn":22895,"coroner":22896,"translucent":22897,"252":22898,"memorials":22899,"puck":22900,"progresses":22901,"clumsy":22902,"##race":22903,"315":22904,"candace":22905,"recounted":22906,"##27":22907,"##slin":22908,"##uve":22909,"filtering":22910,"##mac":22911,"howl":22912,"strata":22913,"heron":22914,"leveled":22915,"##ays":22916,"dubious":22917,"##oja":22918,"##т":22919,"##wheel":22920,"citations":22921,"exhibiting":22922,"##laya":22923,"##mics":22924,"##pods":22925,"turkic":22926,"##lberg":22927,"injunction":22928,"##ennial":22929,"##mit":22930,"antibodies":22931,"##44":22932,"organise":22933,"##rigues":22934,"cardiovascular":22935,"cushion":22936,"inverness":22937,"##zquez":22938,"dia":22939,"cocoa":22940,"sibling":22941,"##tman":22942,"##roid":22943,"expanse":22944,"feasible":22945,"tunisian":22946,"algiers":22947,"##relli":22948,"rus":22949,"bloomberg":22950,"dso":22951,"westphalia":22952,"bro":22953,"tacoma":22954,"281":22955,"downloads":22956,"##ours":22957,"konrad":22958,"duran":22959,"##hdi":22960,"continuum":22961,"jett":22962,"compares":22963,"legislator":22964,"secession":22965,"##nable":22966,"##gues":22967,"##zuka":22968,"translating":22969,"reacher":22970,"##gley":22971,"##ła":22972,"aleppo":22973,"##agi":22974,"tc":22975,"orchards":22976,"trapping":22977,"linguist":22978,"versatile":22979,"drumming":22980,"postage":22981,"calhoun":22982,"superiors":22983,"##mx":22984,"barefoot":22985,"leary":22986,"##cis":22987,"ignacio":22988,"alfa":22989,"kaplan":22990,"##rogen":22991,"bratislava":22992,"mori":22993,"##vot":22994,"disturb":22995,"haas":22996,"313":22997,"cartridges":22998,"gilmore":22999,"radiated":23000,"salford":23001,"tunic":23002,"hades":23003,"##ulsive":23004,"archeological":23005,"delilah":23006,"magistrates":23007,"auditioned":23008,"brewster":23009,"charters":23010,"empowerment":23011,"blogs":23012,"cappella":23013,"dynasties":23014,"iroquois":23015,"whipping":23016,"##krishna":23017,"raceway":23018,"truths":23019,"myra":23020,"weaken":23021,"judah":23022,"mcgregor":23023,"##horse":23024,"mic":23025,"refueling":23026,"37th":23027,"burnley":23028,"bosses":23029,"markus":23030,"premio":23031,"query":23032,"##gga":23033,"dunbar":23034,"##economic":23035,"darkest":23036,"lyndon":23037,"sealing":23038,"commendation":23039,"reappeared":23040,"##mun":23041,"addicted":23042,"ezio":23043,"slaughtered":23044,"satisfactory":23045,"shuffle":23046,"##eves":23047,"##thic":23048,"##uj":23049,"fortification":23050,"warrington":23051,"##otto":23052,"resurrected":23053,"fargo":23054,"mane":23055,"##utable":23056,"##lei":23057,"##space":23058,"foreword":23059,"ox":23060,"##aris":23061,"##vern":23062,"abrams":23063,"hua":23064,"##mento":23065,"sakura":23066,"##alo":23067,"uv":23068,"sentimental":23069,"##skaya":23070,"midfield":23071,"##eses":23072,"sturdy":23073,"scrolls":23074,"macleod":23075,"##kyu":23076,"entropy":23077,"##lance":23078,"mitochondrial":23079,"cicero":23080,"excelled":23081,"thinner":23082,"convoys":23083,"perceive":23084,"##oslav":23085,"##urable":23086,"systematically":23087,"grind":23088,"burkina":23089,"287":23090,"##tagram":23091,"ops":23092,"##aman":23093,"guantanamo":23094,"##cloth":23095,"##tite":23096,"forcefully":23097,"wavy":23098,"##jou":23099,"pointless":23100,"##linger":23101,"##tze":23102,"layton":23103,"portico":23104,"superficial":23105,"clerical":23106,"outlaws":23107,"##hism":23108,"burials":23109,"muir":23110,"##inn":23111,"creditors":23112,"hauling":23113,"rattle":23114,"##leg":23115,"calais":23116,"monde":23117,"archers":23118,"reclaimed":23119,"dwell":23120,"wexford":23121,"hellenic":23122,"falsely":23123,"remorse":23124,"##tek":23125,"dough":23126,"furnishings":23127,"##uttered":23128,"gabon":23129,"neurological":23130,"novice":23131,"##igraphy":23132,"contemplated":23133,"pulpit":23134,"nightstand":23135,"saratoga":23136,"##istan":23137,"documenting":23138,"pulsing":23139,"taluk":23140,"##firmed":23141,"busted":23142,"marital":23143,"##rien":23144,"disagreements":23145,"wasps":23146,"##yes":23147,"hodge":23148,"mcdonnell":23149,"mimic":23150,"fran":23151,"pendant":23152,"dhabi":23153,"musa":23154,"##nington":23155,"congratulations":23156,"argent":23157,"darrell":23158,"concussion":23159,"losers":23160,"regrets":23161,"thessaloniki":23162,"reversal":23163,"donaldson":23164,"hardwood":23165,"thence":23166,"achilles":23167,"ritter":23168,"##eran":23169,"demonic":23170,"jurgen":23171,"prophets":23172,"goethe":23173,"eki":23174,"classmate":23175,"buff":23176,"##cking":23177,"yank":23178,"irrational":23179,"##inging":23180,"perished":23181,"seductive":23182,"qur":23183,"sourced":23184,"##crat":23185,"##typic":23186,"mustard":23187,"ravine":23188,"barre":23189,"horizontally":23190,"characterization":23191,"phylogenetic":23192,"boise":23193,"##dit":23194,"##runner":23195,"##tower":23196,"brutally":23197,"intercourse":23198,"seduce":23199,"##bbing":23200,"fay":23201,"ferris":23202,"ogden":23203,"amar":23204,"nik":23205,"unarmed":23206,"##inator":23207,"evaluating":23208,"kyrgyzstan":23209,"sweetness":23210,"##lford":23211,"##oki":23212,"mccormick":23213,"meiji":23214,"notoriety":23215,"stimulate":23216,"disrupt":23217,"figuring":23218,"instructional":23219,"mcgrath":23220,"##zoo":23221,"groundbreaking":23222,"##lto":23223,"flinch":23224,"khorasan":23225,"agrarian":23226,"bengals":23227,"mixer":23228,"radiating":23229,"##sov":23230,"ingram":23231,"pitchers":23232,"nad":23233,"tariff":23234,"##cript":23235,"tata":23236,"##codes":23237,"##emi":23238,"##ungen":23239,"appellate":23240,"lehigh":23241,"##bled":23242,"##giri":23243,"brawl":23244,"duct":23245,"texans":23246,"##ciation":23247,"##ropolis":23248,"skipper":23249,"speculative":23250,"vomit":23251,"doctrines":23252,"stresses":23253,"253":23254,"davy":23255,"graders":23256,"whitehead":23257,"jozef":23258,"timely":23259,"cumulative":23260,"haryana":23261,"paints":23262,"appropriately":23263,"boon":23264,"cactus":23265,"##ales":23266,"##pid":23267,"dow":23268,"legions":23269,"##pit":23270,"perceptions":23271,"1730":23272,"picturesque":23273,"##yse":23274,"periphery":23275,"rune":23276,"wr":23277,"##aha":23278,"celtics":23279,"sentencing":23280,"whoa":23281,"##erin":23282,"confirms":23283,"variance":23284,"425":23285,"moines":23286,"mathews":23287,"spade":23288,"rave":23289,"m1":23290,"fronted":23291,"fx":23292,"blending":23293,"alleging":23294,"reared":23295,"##gl":23296,"237":23297,"##paper":23298,"grassroots":23299,"eroded":23300,"##free":23301,"##physical":23302,"directs":23303,"ordeal":23304,"##sław":23305,"accelerate":23306,"hacker":23307,"rooftop":23308,"##inia":23309,"lev":23310,"buys":23311,"cebu":23312,"devote":23313,"##lce":23314,"specialising":23315,"##ulsion":23316,"choreographed":23317,"repetition":23318,"warehouses":23319,"##ryl":23320,"paisley":23321,"tuscany":23322,"analogy":23323,"sorcerer":23324,"hash":23325,"huts":23326,"shards":23327,"descends":23328,"exclude":23329,"nix":23330,"chaplin":23331,"gaga":23332,"ito":23333,"vane":23334,"##drich":23335,"causeway":23336,"misconduct":23337,"limo":23338,"orchestrated":23339,"glands":23340,"jana":23341,"##kot":23342,"u2":23343,"##mple":23344,"##sons":23345,"branching":23346,"contrasts":23347,"scoop":23348,"longed":23349,"##virus":23350,"chattanooga":23351,"##75":23352,"syrup":23353,"cornerstone":23354,"##tized":23355,"##mind":23356,"##iaceae":23357,"careless":23358,"precedence":23359,"frescoes":23360,"##uet":23361,"chilled":23362,"consult":23363,"modelled":23364,"snatch":23365,"peat":23366,"##thermal":23367,"caucasian":23368,"humane":23369,"relaxation":23370,"spins":23371,"temperance":23372,"##lbert":23373,"occupations":23374,"lambda":23375,"hybrids":23376,"moons":23377,"mp3":23378,"##oese":23379,"247":23380,"rolf":23381,"societal":23382,"yerevan":23383,"ness":23384,"##ssler":23385,"befriended":23386,"mechanized":23387,"nominate":23388,"trough":23389,"boasted":23390,"cues":23391,"seater":23392,"##hom":23393,"bends":23394,"##tangle":23395,"conductors":23396,"emptiness":23397,"##lmer":23398,"eurasian":23399,"adriatic":23400,"tian":23401,"##cie":23402,"anxiously":23403,"lark":23404,"propellers":23405,"chichester":23406,"jock":23407,"ev":23408,"2a":23409,"##holding":23410,"credible":23411,"recounts":23412,"tori":23413,"loyalist":23414,"abduction":23415,"##hoot":23416,"##redo":23417,"nepali":23418,"##mite":23419,"ventral":23420,"tempting":23421,"##ango":23422,"##crats":23423,"steered":23424,"##wice":23425,"javelin":23426,"dipping":23427,"laborers":23428,"prentice":23429,"looming":23430,"titanium":23431,"##ː":23432,"badges":23433,"emir":23434,"tensor":23435,"##ntation":23436,"egyptians":23437,"rash":23438,"denies":23439,"hawthorne":23440,"lombard":23441,"showers":23442,"wehrmacht":23443,"dietary":23444,"trojan":23445,"##reus":23446,"welles":23447,"executing":23448,"horseshoe":23449,"lifeboat":23450,"##lak":23451,"elsa":23452,"infirmary":23453,"nearing":23454,"roberta":23455,"boyer":23456,"mutter":23457,"trillion":23458,"joanne":23459,"##fine":23460,"##oked":23461,"sinks":23462,"vortex":23463,"uruguayan":23464,"clasp":23465,"sirius":23466,"##block":23467,"accelerator":23468,"prohibit":23469,"sunken":23470,"byu":23471,"chronological":23472,"diplomats":23473,"ochreous":23474,"510":23475,"symmetrical":23476,"1644":23477,"maia":23478,"##tology":23479,"salts":23480,"reigns":23481,"atrocities":23482,"##ия":23483,"hess":23484,"bared":23485,"issn":23486,"##vyn":23487,"cater":23488,"saturated":23489,"##cycle":23490,"##isse":23491,"sable":23492,"voyager":23493,"dyer":23494,"yusuf":23495,"##inge":23496,"fountains":23497,"wolff":23498,"##39":23499,"##nni":23500,"engraving":23501,"rollins":23502,"atheist":23503,"ominous":23504,"##ault":23505,"herr":23506,"chariot":23507,"martina":23508,"strung":23509,"##fell":23510,"##farlane":23511,"horrific":23512,"sahib":23513,"gazes":23514,"saetan":23515,"erased":23516,"ptolemy":23517,"##olic":23518,"flushing":23519,"lauderdale":23520,"analytic":23521,"##ices":23522,"530":23523,"navarro":23524,"beak":23525,"gorilla":23526,"herrera":23527,"broom":23528,"guadalupe":23529,"raiding":23530,"sykes":23531,"311":23532,"bsc":23533,"deliveries":23534,"1720":23535,"invasions":23536,"carmichael":23537,"tajikistan":23538,"thematic":23539,"ecumenical":23540,"sentiments":23541,"onstage":23542,"##rians":23543,"##brand":23544,"##sume":23545,"catastrophic":23546,"flanks":23547,"molten":23548,"##arns":23549,"waller":23550,"aimee":23551,"terminating":23552,"##icing":23553,"alternately":23554,"##oche":23555,"nehru":23556,"printers":23557,"outraged":23558,"##eving":23559,"empires":23560,"template":23561,"banners":23562,"repetitive":23563,"za":23564,"##oise":23565,"vegetarian":23566,"##tell":23567,"guiana":23568,"opt":23569,"cavendish":23570,"lucknow":23571,"synthesized":23572,"##hani":23573,"##mada":23574,"finalized":23575,"##ctable":23576,"fictitious":23577,"mayoral":23578,"unreliable":23579,"##enham":23580,"embracing":23581,"peppers":23582,"rbis":23583,"##chio":23584,"##neo":23585,"inhibition":23586,"slashed":23587,"togo":23588,"orderly":23589,"embroidered":23590,"safari":23591,"salty":23592,"236":23593,"barron":23594,"benito":23595,"totaled":23596,"##dak":23597,"pubs":23598,"simulated":23599,"caden":23600,"devin":23601,"tolkien":23602,"momma":23603,"welding":23604,"sesame":23605,"##ept":23606,"gottingen":23607,"hardness":23608,"630":23609,"shaman":23610,"temeraire":23611,"620":23612,"adequately":23613,"pediatric":23614,"##kit":23615,"ck":23616,"assertion":23617,"radicals":23618,"composure":23619,"cadence":23620,"seafood":23621,"beaufort":23622,"lazarus":23623,"mani":23624,"warily":23625,"cunning":23626,"kurdistan":23627,"249":23628,"cantata":23629,"##kir":23630,"ares":23631,"##41":23632,"##clusive":23633,"nape":23634,"townland":23635,"geared":23636,"insulted":23637,"flutter":23638,"boating":23639,"violate":23640,"draper":23641,"dumping":23642,"malmo":23643,"##hh":23644,"##romatic":23645,"firearm":23646,"alta":23647,"bono":23648,"obscured":23649,"##clave":23650,"exceeds":23651,"panorama":23652,"unbelievable":23653,"##train":23654,"preschool":23655,"##essed":23656,"disconnected":23657,"installing":23658,"rescuing":23659,"secretaries":23660,"accessibility":23661,"##castle":23662,"##drive":23663,"##ifice":23664,"##film":23665,"bouts":23666,"slug":23667,"waterway":23668,"mindanao":23669,"##buro":23670,"##ratic":23671,"halves":23672,"##ل":23673,"calming":23674,"liter":23675,"maternity":23676,"adorable":23677,"bragg":23678,"electrification":23679,"mcc":23680,"##dote":23681,"roxy":23682,"schizophrenia":23683,"##body":23684,"munoz":23685,"kaye":23686,"whaling":23687,"239":23688,"mil":23689,"tingling":23690,"tolerant":23691,"##ago":23692,"unconventional":23693,"volcanoes":23694,"##finder":23695,"deportivo":23696,"##llie":23697,"robson":23698,"kaufman":23699,"neuroscience":23700,"wai":23701,"deportation":23702,"masovian":23703,"scraping":23704,"converse":23705,"##bh":23706,"hacking":23707,"bulge":23708,"##oun":23709,"administratively":23710,"yao":23711,"580":23712,"amp":23713,"mammoth":23714,"booster":23715,"claremont":23716,"hooper":23717,"nomenclature":23718,"pursuits":23719,"mclaughlin":23720,"melinda":23721,"##sul":23722,"catfish":23723,"barclay":23724,"substrates":23725,"taxa":23726,"zee":23727,"originals":23728,"kimberly":23729,"packets":23730,"padma":23731,"##ality":23732,"borrowing":23733,"ostensibly":23734,"solvent":23735,"##bri":23736,"##genesis":23737,"##mist":23738,"lukas":23739,"shreveport":23740,"veracruz":23741,"##ь":23742,"##lou":23743,"##wives":23744,"cheney":23745,"tt":23746,"anatolia":23747,"hobbs":23748,"##zyn":23749,"cyclic":23750,"radiant":23751,"alistair":23752,"greenish":23753,"siena":23754,"dat":23755,"independents":23756,"##bation":23757,"conform":23758,"pieter":23759,"hyper":23760,"applicant":23761,"bradshaw":23762,"spores":23763,"telangana":23764,"vinci":23765,"inexpensive":23766,"nuclei":23767,"322":23768,"jang":23769,"nme":23770,"soho":23771,"spd":23772,"##ign":23773,"cradled":23774,"receptionist":23775,"pow":23776,"##43":23777,"##rika":23778,"fascism":23779,"##ifer":23780,"experimenting":23781,"##ading":23782,"##iec":23783,"##region":23784,"345":23785,"jocelyn":23786,"maris":23787,"stair":23788,"nocturnal":23789,"toro":23790,"constabulary":23791,"elgin":23792,"##kker":23793,"msc":23794,"##giving":23795,"##schen":23796,"##rase":23797,"doherty":23798,"doping":23799,"sarcastically":23800,"batter":23801,"maneuvers":23802,"##cano":23803,"##apple":23804,"##gai":23805,"##git":23806,"intrinsic":23807,"##nst":23808,"##stor":23809,"1753":23810,"showtime":23811,"cafes":23812,"gasps":23813,"lviv":23814,"ushered":23815,"##thed":23816,"fours":23817,"restart":23818,"astonishment":23819,"transmitting":23820,"flyer":23821,"shrugs":23822,"##sau":23823,"intriguing":23824,"cones":23825,"dictated":23826,"mushrooms":23827,"medial":23828,"##kovsky":23829,"##elman":23830,"escorting":23831,"gaped":23832,"##26":23833,"godfather":23834,"##door":23835,"##sell":23836,"djs":23837,"recaptured":23838,"timetable":23839,"vila":23840,"1710":23841,"3a":23842,"aerodrome":23843,"mortals":23844,"scientology":23845,"##orne":23846,"angelina":23847,"mag":23848,"convection":23849,"unpaid":23850,"insertion":23851,"intermittent":23852,"lego":23853,"##nated":23854,"endeavor":23855,"kota":23856,"pereira":23857,"##lz":23858,"304":23859,"bwv":23860,"glamorgan":23861,"insults":23862,"agatha":23863,"fey":23864,"##cend":23865,"fleetwood":23866,"mahogany":23867,"protruding":23868,"steamship":23869,"zeta":23870,"##arty":23871,"mcguire":23872,"suspense":23873,"##sphere":23874,"advising":23875,"urges":23876,"##wala":23877,"hurriedly":23878,"meteor":23879,"gilded":23880,"inline":23881,"arroyo":23882,"stalker":23883,"##oge":23884,"excitedly":23885,"revered":23886,"##cure":23887,"earle":23888,"introductory":23889,"##break":23890,"##ilde":23891,"mutants":23892,"puff":23893,"pulses":23894,"reinforcement":23895,"##haling":23896,"curses":23897,"lizards":23898,"stalk":23899,"correlated":23900,"##fixed":23901,"fallout":23902,"macquarie":23903,"##unas":23904,"bearded":23905,"denton":23906,"heaving":23907,"802":23908,"##ocation":23909,"winery":23910,"assign":23911,"dortmund":23912,"##lkirk":23913,"everest":23914,"invariant":23915,"charismatic":23916,"susie":23917,"##elling":23918,"bled":23919,"lesley":23920,"telegram":23921,"sumner":23922,"bk":23923,"##ogen":23924,"##к":23925,"wilcox":23926,"needy":23927,"colbert":23928,"duval":23929,"##iferous":23930,"##mbled":23931,"allotted":23932,"attends":23933,"imperative":23934,"##hita":23935,"replacements":23936,"hawker":23937,"##inda":23938,"insurgency":23939,"##zee":23940,"##eke":23941,"casts":23942,"##yla":23943,"680":23944,"ives":23945,"transitioned":23946,"##pack":23947,"##powering":23948,"authoritative":23949,"baylor":23950,"flex":23951,"cringed":23952,"plaintiffs":23953,"woodrow":23954,"##skie":23955,"drastic":23956,"ape":23957,"aroma":23958,"unfolded":23959,"commotion":23960,"nt":23961,"preoccupied":23962,"theta":23963,"routines":23964,"lasers":23965,"privatization":23966,"wand":23967,"domino":23968,"ek":23969,"clenching":23970,"nsa":23971,"strategically":23972,"showered":23973,"bile":23974,"handkerchief":23975,"pere":23976,"storing":23977,"christophe":23978,"insulting":23979,"316":23980,"nakamura":23981,"romani":23982,"asiatic":23983,"magdalena":23984,"palma":23985,"cruises":23986,"stripping":23987,"405":23988,"konstantin":23989,"soaring":23990,"##berman":23991,"colloquially":23992,"forerunner":23993,"havilland":23994,"incarcerated":23995,"parasites":23996,"sincerity":23997,"##utus":23998,"disks":23999,"plank":24000,"saigon":24001,"##ining":24002,"corbin":24003,"homo":24004,"ornaments":24005,"powerhouse":24006,"##tlement":24007,"chong":24008,"fastened":24009,"feasibility":24010,"idf":24011,"morphological":24012,"usable":24013,"##nish":24014,"##zuki":24015,"aqueduct":24016,"jaguars":24017,"keepers":24018,"##flies":24019,"aleksandr":24020,"faust":24021,"assigns":24022,"ewing":24023,"bacterium":24024,"hurled":24025,"tricky":24026,"hungarians":24027,"integers":24028,"wallis":24029,"321":24030,"yamaha":24031,"##isha":24032,"hushed":24033,"oblivion":24034,"aviator":24035,"evangelist":24036,"friars":24037,"##eller":24038,"monograph":24039,"ode":24040,"##nary":24041,"airplanes":24042,"labourers":24043,"charms":24044,"##nee":24045,"1661":24046,"hagen":24047,"tnt":24048,"rudder":24049,"fiesta":24050,"transcript":24051,"dorothea":24052,"ska":24053,"inhibitor":24054,"maccabi":24055,"retorted":24056,"raining":24057,"encompassed":24058,"clauses":24059,"menacing":24060,"1642":24061,"lineman":24062,"##gist":24063,"vamps":24064,"##ape":24065,"##dick":24066,"gloom":24067,"##rera":24068,"dealings":24069,"easing":24070,"seekers":24071,"##nut":24072,"##pment":24073,"helens":24074,"unmanned":24075,"##anu":24076,"##isson":24077,"basics":24078,"##amy":24079,"##ckman":24080,"adjustments":24081,"1688":24082,"brutality":24083,"horne":24084,"##zell":24085,"sui":24086,"##55":24087,"##mable":24088,"aggregator":24089,"##thal":24090,"rhino":24091,"##drick":24092,"##vira":24093,"counters":24094,"zoom":24095,"##01":24096,"##rting":24097,"mn":24098,"montenegrin":24099,"packard":24100,"##unciation":24101,"##♭":24102,"##kki":24103,"reclaim":24104,"scholastic":24105,"thugs":24106,"pulsed":24107,"##icia":24108,"syriac":24109,"quan":24110,"saddam":24111,"banda":24112,"kobe":24113,"blaming":24114,"buddies":24115,"dissent":24116,"##lusion":24117,"##usia":24118,"corbett":24119,"jaya":24120,"delle":24121,"erratic":24122,"lexie":24123,"##hesis":24124,"435":24125,"amiga":24126,"hermes":24127,"##pressing":24128,"##leen":24129,"chapels":24130,"gospels":24131,"jamal":24132,"##uating":24133,"compute":24134,"revolving":24135,"warp":24136,"##sso":24137,"##thes":24138,"armory":24139,"##eras":24140,"##gol":24141,"antrim":24142,"loki":24143,"##kow":24144,"##asian":24145,"##good":24146,"##zano":24147,"braid":24148,"handwriting":24149,"subdistrict":24150,"funky":24151,"pantheon":24152,"##iculate":24153,"concurrency":24154,"estimation":24155,"improper":24156,"juliana":24157,"##his":24158,"newcomers":24159,"johnstone":24160,"staten":24161,"communicated":24162,"##oco":24163,"##alle":24164,"sausage":24165,"stormy":24166,"##stered":24167,"##tters":24168,"superfamily":24169,"##grade":24170,"acidic":24171,"collateral":24172,"tabloid":24173,"##oped":24174,"##rza":24175,"bladder":24176,"austen":24177,"##ellant":24178,"mcgraw":24179,"##hay":24180,"hannibal":24181,"mein":24182,"aquino":24183,"lucifer":24184,"wo":24185,"badger":24186,"boar":24187,"cher":24188,"christensen":24189,"greenberg":24190,"interruption":24191,"##kken":24192,"jem":24193,"244":24194,"mocked":24195,"bottoms":24196,"cambridgeshire":24197,"##lide":24198,"sprawling":24199,"##bbly":24200,"eastwood":24201,"ghent":24202,"synth":24203,"##buck":24204,"advisers":24205,"##bah":24206,"nominally":24207,"hapoel":24208,"qu":24209,"daggers":24210,"estranged":24211,"fabricated":24212,"towels":24213,"vinnie":24214,"wcw":24215,"misunderstanding":24216,"anglia":24217,"nothin":24218,"unmistakable":24219,"##dust":24220,"##lova":24221,"chilly":24222,"marquette":24223,"truss":24224,"##edge":24225,"##erine":24226,"reece":24227,"##lty":24228,"##chemist":24229,"##connected":24230,"272":24231,"308":24232,"41st":24233,"bash":24234,"raion":24235,"waterfalls":24236,"##ump":24237,"##main":24238,"labyrinth":24239,"queue":24240,"theorist":24241,"##istle":24242,"bharatiya":24243,"flexed":24244,"soundtracks":24245,"rooney":24246,"leftist":24247,"patrolling":24248,"wharton":24249,"plainly":24250,"alleviate":24251,"eastman":24252,"schuster":24253,"topographic":24254,"engages":24255,"immensely":24256,"unbearable":24257,"fairchild":24258,"1620":24259,"dona":24260,"lurking":24261,"parisian":24262,"oliveira":24263,"ia":24264,"indictment":24265,"hahn":24266,"bangladeshi":24267,"##aster":24268,"vivo":24269,"##uming":24270,"##ential":24271,"antonia":24272,"expects":24273,"indoors":24274,"kildare":24275,"harlan":24276,"##logue":24277,"##ogenic":24278,"##sities":24279,"forgiven":24280,"##wat":24281,"childish":24282,"tavi":24283,"##mide":24284,"##orra":24285,"plausible":24286,"grimm":24287,"successively":24288,"scooted":24289,"##bola":24290,"##dget":24291,"##rith":24292,"spartans":24293,"emery":24294,"flatly":24295,"azure":24296,"epilogue":24297,"##wark":24298,"flourish":24299,"##iny":24300,"##tracted":24301,"##overs":24302,"##oshi":24303,"bestseller":24304,"distressed":24305,"receipt":24306,"spitting":24307,"hermit":24308,"topological":24309,"##cot":24310,"drilled":24311,"subunit":24312,"francs":24313,"##layer":24314,"eel":24315,"##fk":24316,"##itas":24317,"octopus":24318,"footprint":24319,"petitions":24320,"ufo":24321,"##say":24322,"##foil":24323,"interfering":24324,"leaking":24325,"palo":24326,"##metry":24327,"thistle":24328,"valiant":24329,"##pic":24330,"narayan":24331,"mcpherson":24332,"##fast":24333,"gonzales":24334,"##ym":24335,"##enne":24336,"dustin":24337,"novgorod":24338,"solos":24339,"##zman":24340,"doin":24341,"##raph":24342,"##patient":24343,"##meyer":24344,"soluble":24345,"ashland":24346,"cuffs":24347,"carole":24348,"pendleton":24349,"whistling":24350,"vassal":24351,"##river":24352,"deviation":24353,"revisited":24354,"constituents":24355,"rallied":24356,"rotate":24357,"loomed":24358,"##eil":24359,"##nting":24360,"amateurs":24361,"augsburg":24362,"auschwitz":24363,"crowns":24364,"skeletons":24365,"##cona":24366,"bonnet":24367,"257":24368,"dummy":24369,"globalization":24370,"simeon":24371,"sleeper":24372,"mandal":24373,"differentiated":24374,"##crow":24375,"##mare":24376,"milne":24377,"bundled":24378,"exasperated":24379,"talmud":24380,"owes":24381,"segregated":24382,"##feng":24383,"##uary":24384,"dentist":24385,"piracy":24386,"props":24387,"##rang":24388,"devlin":24389,"##torium":24390,"malicious":24391,"paws":24392,"##laid":24393,"dependency":24394,"##ergy":24395,"##fers":24396,"##enna":24397,"258":24398,"pistons":24399,"rourke":24400,"jed":24401,"grammatical":24402,"tres":24403,"maha":24404,"wig":24405,"512":24406,"ghostly":24407,"jayne":24408,"##achal":24409,"##creen":24410,"##ilis":24411,"##lins":24412,"##rence":24413,"designate":24414,"##with":24415,"arrogance":24416,"cambodian":24417,"clones":24418,"showdown":24419,"throttle":24420,"twain":24421,"##ception":24422,"lobes":24423,"metz":24424,"nagoya":24425,"335":24426,"braking":24427,"##furt":24428,"385":24429,"roaming":24430,"##minster":24431,"amin":24432,"crippled":24433,"##37":24434,"##llary":24435,"indifferent":24436,"hoffmann":24437,"idols":24438,"intimidating":24439,"1751":24440,"261":24441,"influenza":24442,"memo":24443,"onions":24444,"1748":24445,"bandage":24446,"consciously":24447,"##landa":24448,"##rage":24449,"clandestine":24450,"observes":24451,"swiped":24452,"tangle":24453,"##ener":24454,"##jected":24455,"##trum":24456,"##bill":24457,"##lta":24458,"hugs":24459,"congresses":24460,"josiah":24461,"spirited":24462,"##dek":24463,"humanist":24464,"managerial":24465,"filmmaking":24466,"inmate":24467,"rhymes":24468,"debuting":24469,"grimsby":24470,"ur":24471,"##laze":24472,"duplicate":24473,"vigor":24474,"##tf":24475,"republished":24476,"bolshevik":24477,"refurbishment":24478,"antibiotics":24479,"martini":24480,"methane":24481,"newscasts":24482,"royale":24483,"horizons":24484,"levant":24485,"iain":24486,"visas":24487,"##ischen":24488,"paler":24489,"##around":24490,"manifestation":24491,"snuck":24492,"alf":24493,"chop":24494,"futile":24495,"pedestal":24496,"rehab":24497,"##kat":24498,"bmg":24499,"kerman":24500,"res":24501,"fairbanks":24502,"jarrett":24503,"abstraction":24504,"saharan":24505,"##zek":24506,"1746":24507,"procedural":24508,"clearer":24509,"kincaid":24510,"sash":24511,"luciano":24512,"##ffey":24513,"crunch":24514,"helmut":24515,"##vara":24516,"revolutionaries":24517,"##tute":24518,"creamy":24519,"leach":24520,"##mmon":24521,"1747":24522,"permitting":24523,"nes":24524,"plight":24525,"wendell":24526,"##lese":24527,"contra":24528,"ts":24529,"clancy":24530,"ipa":24531,"mach":24532,"staples":24533,"autopsy":24534,"disturbances":24535,"nueva":24536,"karin":24537,"pontiac":24538,"##uding":24539,"proxy":24540,"venerable":24541,"haunt":24542,"leto":24543,"bergman":24544,"expands":24545,"##helm":24546,"wal":24547,"##pipe":24548,"canning":24549,"celine":24550,"cords":24551,"obesity":24552,"##enary":24553,"intrusion":24554,"planner":24555,"##phate":24556,"reasoned":24557,"sequencing":24558,"307":24559,"harrow":24560,"##chon":24561,"##dora":24562,"marred":24563,"mcintyre":24564,"repay":24565,"tarzan":24566,"darting":24567,"248":24568,"harrisburg":24569,"margarita":24570,"repulsed":24571,"##hur":24572,"##lding":24573,"belinda":24574,"hamburger":24575,"novo":24576,"compliant":24577,"runways":24578,"bingham":24579,"registrar":24580,"skyscraper":24581,"ic":24582,"cuthbert":24583,"improvisation":24584,"livelihood":24585,"##corp":24586,"##elial":24587,"admiring":24588,"##dened":24589,"sporadic":24590,"believer":24591,"casablanca":24592,"popcorn":24593,"##29":24594,"asha":24595,"shovel":24596,"##bek":24597,"##dice":24598,"coiled":24599,"tangible":24600,"##dez":24601,"casper":24602,"elsie":24603,"resin":24604,"tenderness":24605,"rectory":24606,"##ivision":24607,"avail":24608,"sonar":24609,"##mori":24610,"boutique":24611,"##dier":24612,"guerre":24613,"bathed":24614,"upbringing":24615,"vaulted":24616,"sandals":24617,"blessings":24618,"##naut":24619,"##utnant":24620,"1680":24621,"306":24622,"foxes":24623,"pia":24624,"corrosion":24625,"hesitantly":24626,"confederates":24627,"crystalline":24628,"footprints":24629,"shapiro":24630,"tirana":24631,"valentin":24632,"drones":24633,"45th":24634,"microscope":24635,"shipments":24636,"texted":24637,"inquisition":24638,"wry":24639,"guernsey":24640,"unauthorized":24641,"resigning":24642,"760":24643,"ripple":24644,"schubert":24645,"stu":24646,"reassure":24647,"felony":24648,"##ardo":24649,"brittle":24650,"koreans":24651,"##havan":24652,"##ives":24653,"dun":24654,"implicit":24655,"tyres":24656,"##aldi":24657,"##lth":24658,"magnolia":24659,"##ehan":24660,"##puri":24661,"##poulos":24662,"aggressively":24663,"fei":24664,"gr":24665,"familiarity":24666,"##poo":24667,"indicative":24668,"##trust":24669,"fundamentally":24670,"jimmie":24671,"overrun":24672,"395":24673,"anchors":24674,"moans":24675,"##opus":24676,"britannia":24677,"armagh":24678,"##ggle":24679,"purposely":24680,"seizing":24681,"##vao":24682,"bewildered":24683,"mundane":24684,"avoidance":24685,"cosmopolitan":24686,"geometridae":24687,"quartermaster":24688,"caf":24689,"415":24690,"chatter":24691,"engulfed":24692,"gleam":24693,"purge":24694,"##icate":24695,"juliette":24696,"jurisprudence":24697,"guerra":24698,"revisions":24699,"##bn":24700,"casimir":24701,"brew":24702,"##jm":24703,"1749":24704,"clapton":24705,"cloudy":24706,"conde":24707,"hermitage":24708,"278":24709,"simulations":24710,"torches":24711,"vincenzo":24712,"matteo":24713,"##rill":24714,"hidalgo":24715,"booming":24716,"westbound":24717,"accomplishment":24718,"tentacles":24719,"unaffected":24720,"##sius":24721,"annabelle":24722,"flopped":24723,"sloping":24724,"##litz":24725,"dreamer":24726,"interceptor":24727,"vu":24728,"##loh":24729,"consecration":24730,"copying":24731,"messaging":24732,"breaker":24733,"climates":24734,"hospitalized":24735,"1752":24736,"torino":24737,"afternoons":24738,"winfield":24739,"witnessing":24740,"##teacher":24741,"breakers":24742,"choirs":24743,"sawmill":24744,"coldly":24745,"##ege":24746,"sipping":24747,"haste":24748,"uninhabited":24749,"conical":24750,"bibliography":24751,"pamphlets":24752,"severn":24753,"edict":24754,"##oca":24755,"deux":24756,"illnesses":24757,"grips":24758,"##pl":24759,"rehearsals":24760,"sis":24761,"thinkers":24762,"tame":24763,"##keepers":24764,"1690":24765,"acacia":24766,"reformer":24767,"##osed":24768,"##rys":24769,"shuffling":24770,"##iring":24771,"##shima":24772,"eastbound":24773,"ionic":24774,"rhea":24775,"flees":24776,"littered":24777,"##oum":24778,"rocker":24779,"vomiting":24780,"groaning":24781,"champ":24782,"overwhelmingly":24783,"civilizations":24784,"paces":24785,"sloop":24786,"adoptive":24787,"##tish":24788,"skaters":24789,"##vres":24790,"aiding":24791,"mango":24792,"##joy":24793,"nikola":24794,"shriek":24795,"##ignon":24796,"pharmaceuticals":24797,"##mg":24798,"tuna":24799,"calvert":24800,"gustavo":24801,"stocked":24802,"yearbook":24803,"##urai":24804,"##mana":24805,"computed":24806,"subsp":24807,"riff":24808,"hanoi":24809,"kelvin":24810,"hamid":24811,"moors":24812,"pastures":24813,"summons":24814,"jihad":24815,"nectar":24816,"##ctors":24817,"bayou":24818,"untitled":24819,"pleasing":24820,"vastly":24821,"republics":24822,"intellect":24823,"##η":24824,"##ulio":24825,"##tou":24826,"crumbling":24827,"stylistic":24828,"sb":24829,"##ی":24830,"consolation":24831,"frequented":24832,"h₂o":24833,"walden":24834,"widows":24835,"##iens":24836,"404":24837,"##ignment":24838,"chunks":24839,"improves":24840,"288":24841,"grit":24842,"recited":24843,"##dev":24844,"snarl":24845,"sociological":24846,"##arte":24847,"##gul":24848,"inquired":24849,"##held":24850,"bruise":24851,"clube":24852,"consultancy":24853,"homogeneous":24854,"hornets":24855,"multiplication":24856,"pasta":24857,"prick":24858,"savior":24859,"##grin":24860,"##kou":24861,"##phile":24862,"yoon":24863,"##gara":24864,"grimes":24865,"vanishing":24866,"cheering":24867,"reacting":24868,"bn":24869,"distillery":24870,"##quisite":24871,"##vity":24872,"coe":24873,"dockyard":24874,"massif":24875,"##jord":24876,"escorts":24877,"voss":24878,"##valent":24879,"byte":24880,"chopped":24881,"hawke":24882,"illusions":24883,"workings":24884,"floats":24885,"##koto":24886,"##vac":24887,"kv":24888,"annapolis":24889,"madden":24890,"##onus":24891,"alvaro":24892,"noctuidae":24893,"##cum":24894,"##scopic":24895,"avenge":24896,"steamboat":24897,"forte":24898,"illustrates":24899,"erika":24900,"##trip":24901,"570":24902,"dew":24903,"nationalities":24904,"bran":24905,"manifested":24906,"thirsty":24907,"diversified":24908,"muscled":24909,"reborn":24910,"##standing":24911,"arson":24912,"##lessness":24913,"##dran":24914,"##logram":24915,"##boys":24916,"##kushima":24917,"##vious":24918,"willoughby":24919,"##phobia":24920,"286":24921,"alsace":24922,"dashboard":24923,"yuki":24924,"##chai":24925,"granville":24926,"myspace":24927,"publicized":24928,"tricked":24929,"##gang":24930,"adjective":24931,"##ater":24932,"relic":24933,"reorganisation":24934,"enthusiastically":24935,"indications":24936,"saxe":24937,"##lassified":24938,"consolidate":24939,"iec":24940,"padua":24941,"helplessly":24942,"ramps":24943,"renaming":24944,"regulars":24945,"pedestrians":24946,"accents":24947,"convicts":24948,"inaccurate":24949,"lowers":24950,"mana":24951,"##pati":24952,"barrie":24953,"bjp":24954,"outta":24955,"someplace":24956,"berwick":24957,"flanking":24958,"invoked":24959,"marrow":24960,"sparsely":24961,"excerpts":24962,"clothed":24963,"rei":24964,"##ginal":24965,"wept":24966,"##straße":24967,"##vish":24968,"alexa":24969,"excel":24970,"##ptive":24971,"membranes":24972,"aquitaine":24973,"creeks":24974,"cutler":24975,"sheppard":24976,"implementations":24977,"ns":24978,"##dur":24979,"fragrance":24980,"budge":24981,"concordia":24982,"magnesium":24983,"marcelo":24984,"##antes":24985,"gladly":24986,"vibrating":24987,"##rral":24988,"##ggles":24989,"montrose":24990,"##omba":24991,"lew":24992,"seamus":24993,"1630":24994,"cocky":24995,"##ament":24996,"##uen":24997,"bjorn":24998,"##rrick":24999,"fielder":25000,"fluttering":25001,"##lase":25002,"methyl":25003,"kimberley":25004,"mcdowell":25005,"reductions":25006,"barbed":25007,"##jic":25008,"##tonic":25009,"aeronautical":25010,"condensed":25011,"distracting":25012,"##promising":25013,"huffed":25014,"##cala":25015,"##sle":25016,"claudius":25017,"invincible":25018,"missy":25019,"pious":25020,"balthazar":25021,"ci":25022,"##lang":25023,"butte":25024,"combo":25025,"orson":25026,"##dication":25027,"myriad":25028,"1707":25029,"silenced":25030,"##fed":25031,"##rh":25032,"coco":25033,"netball":25034,"yourselves":25035,"##oza":25036,"clarify":25037,"heller":25038,"peg":25039,"durban":25040,"etudes":25041,"offender":25042,"roast":25043,"blackmail":25044,"curvature":25045,"##woods":25046,"vile":25047,"309":25048,"illicit":25049,"suriname":25050,"##linson":25051,"overture":25052,"1685":25053,"bubbling":25054,"gymnast":25055,"tucking":25056,"##mming":25057,"##ouin":25058,"maldives":25059,"##bala":25060,"gurney":25061,"##dda":25062,"##eased":25063,"##oides":25064,"backside":25065,"pinto":25066,"jars":25067,"racehorse":25068,"tending":25069,"##rdial":25070,"baronetcy":25071,"wiener":25072,"duly":25073,"##rke":25074,"barbarian":25075,"cupping":25076,"flawed":25077,"##thesis":25078,"bertha":25079,"pleistocene":25080,"puddle":25081,"swearing":25082,"##nob":25083,"##tically":25084,"fleeting":25085,"prostate":25086,"amulet":25087,"educating":25088,"##mined":25089,"##iti":25090,"##tler":25091,"75th":25092,"jens":25093,"respondents":25094,"analytics":25095,"cavaliers":25096,"papacy":25097,"raju":25098,"##iente":25099,"##ulum":25100,"##tip":25101,"funnel":25102,"271":25103,"disneyland":25104,"##lley":25105,"sociologist":25106,"##iam":25107,"2500":25108,"faulkner":25109,"louvre":25110,"menon":25111,"##dson":25112,"276":25113,"##ower":25114,"afterlife":25115,"mannheim":25116,"peptide":25117,"referees":25118,"comedians":25119,"meaningless":25120,"##anger":25121,"##laise":25122,"fabrics":25123,"hurley":25124,"renal":25125,"sleeps":25126,"##bour":25127,"##icle":25128,"breakout":25129,"kristin":25130,"roadside":25131,"animator":25132,"clover":25133,"disdain":25134,"unsafe":25135,"redesign":25136,"##urity":25137,"firth":25138,"barnsley":25139,"portage":25140,"reset":25141,"narrows":25142,"268":25143,"commandos":25144,"expansive":25145,"speechless":25146,"tubular":25147,"##lux":25148,"essendon":25149,"eyelashes":25150,"smashwords":25151,"##yad":25152,"##bang":25153,"##claim":25154,"craved":25155,"sprinted":25156,"chet":25157,"somme":25158,"astor":25159,"wrocław":25160,"orton":25161,"266":25162,"bane":25163,"##erving":25164,"##uing":25165,"mischief":25166,"##amps":25167,"##sund":25168,"scaling":25169,"terre":25170,"##xious":25171,"impairment":25172,"offenses":25173,"undermine":25174,"moi":25175,"soy":25176,"contiguous":25177,"arcadia":25178,"inuit":25179,"seam":25180,"##tops":25181,"macbeth":25182,"rebelled":25183,"##icative":25184,"##iot":25185,"590":25186,"elaborated":25187,"frs":25188,"uniformed":25189,"##dberg":25190,"259":25191,"powerless":25192,"priscilla":25193,"stimulated":25194,"980":25195,"qc":25196,"arboretum":25197,"frustrating":25198,"trieste":25199,"bullock":25200,"##nified":25201,"enriched":25202,"glistening":25203,"intern":25204,"##adia":25205,"locus":25206,"nouvelle":25207,"ollie":25208,"ike":25209,"lash":25210,"starboard":25211,"ee":25212,"tapestry":25213,"headlined":25214,"hove":25215,"rigged":25216,"##vite":25217,"pollock":25218,"##yme":25219,"thrive":25220,"clustered":25221,"cas":25222,"roi":25223,"gleamed":25224,"olympiad":25225,"##lino":25226,"pressured":25227,"regimes":25228,"##hosis":25229,"##lick":25230,"ripley":25231,"##ophone":25232,"kickoff":25233,"gallon":25234,"rockwell":25235,"##arable":25236,"crusader":25237,"glue":25238,"revolutions":25239,"scrambling":25240,"1714":25241,"grover":25242,"##jure":25243,"englishman":25244,"aztec":25245,"263":25246,"contemplating":25247,"coven":25248,"ipad":25249,"preach":25250,"triumphant":25251,"tufts":25252,"##esian":25253,"rotational":25254,"##phus":25255,"328":25256,"falkland":25257,"##brates":25258,"strewn":25259,"clarissa":25260,"rejoin":25261,"environmentally":25262,"glint":25263,"banded":25264,"drenched":25265,"moat":25266,"albanians":25267,"johor":25268,"rr":25269,"maestro":25270,"malley":25271,"nouveau":25272,"shaded":25273,"taxonomy":25274,"v6":25275,"adhere":25276,"bunk":25277,"airfields":25278,"##ritan":25279,"1741":25280,"encompass":25281,"remington":25282,"tran":25283,"##erative":25284,"amelie":25285,"mazda":25286,"friar":25287,"morals":25288,"passions":25289,"##zai":25290,"breadth":25291,"vis":25292,"##hae":25293,"argus":25294,"burnham":25295,"caressing":25296,"insider":25297,"rudd":25298,"##imov":25299,"##mini":25300,"##rso":25301,"italianate":25302,"murderous":25303,"textual":25304,"wainwright":25305,"armada":25306,"bam":25307,"weave":25308,"timer":25309,"##taken":25310,"##nh":25311,"fra":25312,"##crest":25313,"ardent":25314,"salazar":25315,"taps":25316,"tunis":25317,"##ntino":25318,"allegro":25319,"gland":25320,"philanthropic":25321,"##chester":25322,"implication":25323,"##optera":25324,"esq":25325,"judas":25326,"noticeably":25327,"wynn":25328,"##dara":25329,"inched":25330,"indexed":25331,"crises":25332,"villiers":25333,"bandit":25334,"royalties":25335,"patterned":25336,"cupboard":25337,"interspersed":25338,"accessory":25339,"isla":25340,"kendrick":25341,"entourage":25342,"stitches":25343,"##esthesia":25344,"headwaters":25345,"##ior":25346,"interlude":25347,"distraught":25348,"draught":25349,"1727":25350,"##basket":25351,"biased":25352,"sy":25353,"transient":25354,"triad":25355,"subgenus":25356,"adapting":25357,"kidd":25358,"shortstop":25359,"##umatic":25360,"dimly":25361,"spiked":25362,"mcleod":25363,"reprint":25364,"nellie":25365,"pretoria":25366,"windmill":25367,"##cek":25368,"singled":25369,"##mps":25370,"273":25371,"reunite":25372,"##orous":25373,"747":25374,"bankers":25375,"outlying":25376,"##omp":25377,"##ports":25378,"##tream":25379,"apologies":25380,"cosmetics":25381,"patsy":25382,"##deh":25383,"##ocks":25384,"##yson":25385,"bender":25386,"nantes":25387,"serene":25388,"##nad":25389,"lucha":25390,"mmm":25391,"323":25392,"##cius":25393,"##gli":25394,"cmll":25395,"coinage":25396,"nestor":25397,"juarez":25398,"##rook":25399,"smeared":25400,"sprayed":25401,"twitching":25402,"sterile":25403,"irina":25404,"embodied":25405,"juveniles":25406,"enveloped":25407,"miscellaneous":25408,"cancers":25409,"dq":25410,"gulped":25411,"luisa":25412,"crested":25413,"swat":25414,"donegal":25415,"ref":25416,"##anov":25417,"##acker":25418,"hearst":25419,"mercantile":25420,"##lika":25421,"doorbell":25422,"ua":25423,"vicki":25424,"##alla":25425,"##som":25426,"bilbao":25427,"psychologists":25428,"stryker":25429,"sw":25430,"horsemen":25431,"turkmenistan":25432,"wits":25433,"##national":25434,"anson":25435,"mathew":25436,"screenings":25437,"##umb":25438,"rihanna":25439,"##agne":25440,"##nessy":25441,"aisles":25442,"##iani":25443,"##osphere":25444,"hines":25445,"kenton":25446,"saskatoon":25447,"tasha":25448,"truncated":25449,"##champ":25450,"##itan":25451,"mildred":25452,"advises":25453,"fredrik":25454,"interpreting":25455,"inhibitors":25456,"##athi":25457,"spectroscopy":25458,"##hab":25459,"##kong":25460,"karim":25461,"panda":25462,"##oia":25463,"##nail":25464,"##vc":25465,"conqueror":25466,"kgb":25467,"leukemia":25468,"##dity":25469,"arrivals":25470,"cheered":25471,"pisa":25472,"phosphorus":25473,"shielded":25474,"##riated":25475,"mammal":25476,"unitarian":25477,"urgently":25478,"chopin":25479,"sanitary":25480,"##mission":25481,"spicy":25482,"drugged":25483,"hinges":25484,"##tort":25485,"tipping":25486,"trier":25487,"impoverished":25488,"westchester":25489,"##caster":25490,"267":25491,"epoch":25492,"nonstop":25493,"##gman":25494,"##khov":25495,"aromatic":25496,"centrally":25497,"cerro":25498,"##tively":25499,"##vio":25500,"billions":25501,"modulation":25502,"sedimentary":25503,"283":25504,"facilitating":25505,"outrageous":25506,"goldstein":25507,"##eak":25508,"##kt":25509,"ld":25510,"maitland":25511,"penultimate":25512,"pollard":25513,"##dance":25514,"fleets":25515,"spaceship":25516,"vertebrae":25517,"##nig":25518,"alcoholism":25519,"als":25520,"recital":25521,"##bham":25522,"##ference":25523,"##omics":25524,"m2":25525,"##bm":25526,"trois":25527,"##tropical":25528,"##в":25529,"commemorates":25530,"##meric":25531,"marge":25532,"##raction":25533,"1643":25534,"670":25535,"cosmetic":25536,"ravaged":25537,"##ige":25538,"catastrophe":25539,"eng":25540,"##shida":25541,"albrecht":25542,"arterial":25543,"bellamy":25544,"decor":25545,"harmon":25546,"##rde":25547,"bulbs":25548,"synchronized":25549,"vito":25550,"easiest":25551,"shetland":25552,"shielding":25553,"wnba":25554,"##glers":25555,"##ssar":25556,"##riam":25557,"brianna":25558,"cumbria":25559,"##aceous":25560,"##rard":25561,"cores":25562,"thayer":25563,"##nsk":25564,"brood":25565,"hilltop":25566,"luminous":25567,"carts":25568,"keynote":25569,"larkin":25570,"logos":25571,"##cta":25572,"##ا":25573,"##mund":25574,"##quay":25575,"lilith":25576,"tinted":25577,"277":25578,"wrestle":25579,"mobilization":25580,"##uses":25581,"sequential":25582,"siam":25583,"bloomfield":25584,"takahashi":25585,"274":25586,"##ieving":25587,"presenters":25588,"ringo":25589,"blazed":25590,"witty":25591,"##oven":25592,"##ignant":25593,"devastation":25594,"haydn":25595,"harmed":25596,"newt":25597,"therese":25598,"##peed":25599,"gershwin":25600,"molina":25601,"rabbis":25602,"sudanese":25603,"001":25604,"innate":25605,"restarted":25606,"##sack":25607,"##fus":25608,"slices":25609,"wb":25610,"##shah":25611,"enroll":25612,"hypothetical":25613,"hysterical":25614,"1743":25615,"fabio":25616,"indefinite":25617,"warped":25618,"##hg":25619,"exchanging":25620,"525":25621,"unsuitable":25622,"##sboro":25623,"gallo":25624,"1603":25625,"bret":25626,"cobalt":25627,"homemade":25628,"##hunter":25629,"mx":25630,"operatives":25631,"##dhar":25632,"terraces":25633,"durable":25634,"latch":25635,"pens":25636,"whorls":25637,"##ctuated":25638,"##eaux":25639,"billing":25640,"ligament":25641,"succumbed":25642,"##gly":25643,"regulators":25644,"spawn":25645,"##brick":25646,"##stead":25647,"filmfare":25648,"rochelle":25649,"##nzo":25650,"1725":25651,"circumstance":25652,"saber":25653,"supplements":25654,"##nsky":25655,"##tson":25656,"crowe":25657,"wellesley":25658,"carrot":25659,"##9th":25660,"##movable":25661,"primate":25662,"drury":25663,"sincerely":25664,"topical":25665,"##mad":25666,"##rao":25667,"callahan":25668,"kyiv":25669,"smarter":25670,"tits":25671,"undo":25672,"##yeh":25673,"announcements":25674,"anthologies":25675,"barrio":25676,"nebula":25677,"##islaus":25678,"##shaft":25679,"##tyn":25680,"bodyguards":25681,"2021":25682,"assassinate":25683,"barns":25684,"emmett":25685,"scully":25686,"##mah":25687,"##yd":25688,"##eland":25689,"##tino":25690,"##itarian":25691,"demoted":25692,"gorman":25693,"lashed":25694,"prized":25695,"adventist":25696,"writ":25697,"##gui":25698,"alla":25699,"invertebrates":25700,"##ausen":25701,"1641":25702,"amman":25703,"1742":25704,"align":25705,"healy":25706,"redistribution":25707,"##gf":25708,"##rize":25709,"insulation":25710,"##drop":25711,"adherents":25712,"hezbollah":25713,"vitro":25714,"ferns":25715,"yanking":25716,"269":25717,"php":25718,"registering":25719,"uppsala":25720,"cheerleading":25721,"confines":25722,"mischievous":25723,"tully":25724,"##ross":25725,"49th":25726,"docked":25727,"roam":25728,"stipulated":25729,"pumpkin":25730,"##bry":25731,"prompt":25732,"##ezer":25733,"blindly":25734,"shuddering":25735,"craftsmen":25736,"frail":25737,"scented":25738,"katharine":25739,"scramble":25740,"shaggy":25741,"sponge":25742,"helix":25743,"zaragoza":25744,"279":25745,"##52":25746,"43rd":25747,"backlash":25748,"fontaine":25749,"seizures":25750,"posse":25751,"cowan":25752,"nonfiction":25753,"telenovela":25754,"wwii":25755,"hammered":25756,"undone":25757,"##gpur":25758,"encircled":25759,"irs":25760,"##ivation":25761,"artefacts":25762,"oneself":25763,"searing":25764,"smallpox":25765,"##belle":25766,"##osaurus":25767,"shandong":25768,"breached":25769,"upland":25770,"blushing":25771,"rankin":25772,"infinitely":25773,"psyche":25774,"tolerated":25775,"docking":25776,"evicted":25777,"##col":25778,"unmarked":25779,"##lving":25780,"gnome":25781,"lettering":25782,"litres":25783,"musique":25784,"##oint":25785,"benevolent":25786,"##jal":25787,"blackened":25788,"##anna":25789,"mccall":25790,"racers":25791,"tingle":25792,"##ocene":25793,"##orestation":25794,"introductions":25795,"radically":25796,"292":25797,"##hiff":25798,"##باد":25799,"1610":25800,"1739":25801,"munchen":25802,"plead":25803,"##nka":25804,"condo":25805,"scissors":25806,"##sight":25807,"##tens":25808,"apprehension":25809,"##cey":25810,"##yin":25811,"hallmark":25812,"watering":25813,"formulas":25814,"sequels":25815,"##llas":25816,"aggravated":25817,"bae":25818,"commencing":25819,"##building":25820,"enfield":25821,"prohibits":25822,"marne":25823,"vedic":25824,"civilized":25825,"euclidean":25826,"jagger":25827,"beforehand":25828,"blasts":25829,"dumont":25830,"##arney":25831,"##nem":25832,"740":25833,"conversions":25834,"hierarchical":25835,"rios":25836,"simulator":25837,"##dya":25838,"##lellan":25839,"hedges":25840,"oleg":25841,"thrusts":25842,"shadowed":25843,"darby":25844,"maximize":25845,"1744":25846,"gregorian":25847,"##nded":25848,"##routed":25849,"sham":25850,"unspecified":25851,"##hog":25852,"emory":25853,"factual":25854,"##smo":25855,"##tp":25856,"fooled":25857,"##rger":25858,"ortega":25859,"wellness":25860,"marlon":25861,"##oton":25862,"##urance":25863,"casket":25864,"keating":25865,"ley":25866,"enclave":25867,"##ayan":25868,"char":25869,"influencing":25870,"jia":25871,"##chenko":25872,"412":25873,"ammonia":25874,"erebidae":25875,"incompatible":25876,"violins":25877,"cornered":25878,"##arat":25879,"grooves":25880,"astronauts":25881,"columbian":25882,"rampant":25883,"fabrication":25884,"kyushu":25885,"mahmud":25886,"vanish":25887,"##dern":25888,"mesopotamia":25889,"##lete":25890,"ict":25891,"##rgen":25892,"caspian":25893,"kenji":25894,"pitted":25895,"##vered":25896,"999":25897,"grimace":25898,"roanoke":25899,"tchaikovsky":25900,"twinned":25901,"##analysis":25902,"##awan":25903,"xinjiang":25904,"arias":25905,"clemson":25906,"kazakh":25907,"sizable":25908,"1662":25909,"##khand":25910,"##vard":25911,"plunge":25912,"tatum":25913,"vittorio":25914,"##nden":25915,"cholera":25916,"##dana":25917,"##oper":25918,"bracing":25919,"indifference":25920,"projectile":25921,"superliga":25922,"##chee":25923,"realises":25924,"upgrading":25925,"299":25926,"porte":25927,"retribution":25928,"##vies":25929,"nk":25930,"stil":25931,"##resses":25932,"ama":25933,"bureaucracy":25934,"blackberry":25935,"bosch":25936,"testosterone":25937,"collapses":25938,"greer":25939,"##pathic":25940,"ioc":25941,"fifties":25942,"malls":25943,"##erved":25944,"bao":25945,"baskets":25946,"adolescents":25947,"siegfried":25948,"##osity":25949,"##tosis":25950,"mantra":25951,"detecting":25952,"existent":25953,"fledgling":25954,"##cchi":25955,"dissatisfied":25956,"gan":25957,"telecommunication":25958,"mingled":25959,"sobbed":25960,"6000":25961,"controversies":25962,"outdated":25963,"taxis":25964,"##raus":25965,"fright":25966,"slams":25967,"##lham":25968,"##fect":25969,"##tten":25970,"detectors":25971,"fetal":25972,"tanned":25973,"##uw":25974,"fray":25975,"goth":25976,"olympian":25977,"skipping":25978,"mandates":25979,"scratches":25980,"sheng":25981,"unspoken":25982,"hyundai":25983,"tracey":25984,"hotspur":25985,"restrictive":25986,"##buch":25987,"americana":25988,"mundo":25989,"##bari":25990,"burroughs":25991,"diva":25992,"vulcan":25993,"##6th":25994,"distinctions":25995,"thumping":25996,"##ngen":25997,"mikey":25998,"sheds":25999,"fide":26000,"rescues":26001,"springsteen":26002,"vested":26003,"valuation":26004,"##ece":26005,"##ely":26006,"pinnacle":26007,"rake":26008,"sylvie":26009,"##edo":26010,"almond":26011,"quivering":26012,"##irus":26013,"alteration":26014,"faltered":26015,"##wad":26016,"51st":26017,"hydra":26018,"ticked":26019,"##kato":26020,"recommends":26021,"##dicated":26022,"antigua":26023,"arjun":26024,"stagecoach":26025,"wilfred":26026,"trickle":26027,"pronouns":26028,"##pon":26029,"aryan":26030,"nighttime":26031,"##anian":26032,"gall":26033,"pea":26034,"stitch":26035,"##hei":26036,"leung":26037,"milos":26038,"##dini":26039,"eritrea":26040,"nexus":26041,"starved":26042,"snowfall":26043,"kant":26044,"parasitic":26045,"cot":26046,"discus":26047,"hana":26048,"strikers":26049,"appleton":26050,"kitchens":26051,"##erina":26052,"##partisan":26053,"##itha":26054,"##vius":26055,"disclose":26056,"metis":26057,"##channel":26058,"1701":26059,"tesla":26060,"##vera":26061,"fitch":26062,"1735":26063,"blooded":26064,"##tila":26065,"decimal":26066,"##tang":26067,"##bai":26068,"cyclones":26069,"eun":26070,"bottled":26071,"peas":26072,"pensacola":26073,"basha":26074,"bolivian":26075,"crabs":26076,"boil":26077,"lanterns":26078,"partridge":26079,"roofed":26080,"1645":26081,"necks":26082,"##phila":26083,"opined":26084,"patting":26085,"##kla":26086,"##lland":26087,"chuckles":26088,"volta":26089,"whereupon":26090,"##nche":26091,"devout":26092,"euroleague":26093,"suicidal":26094,"##dee":26095,"inherently":26096,"involuntary":26097,"knitting":26098,"nasser":26099,"##hide":26100,"puppets":26101,"colourful":26102,"courageous":26103,"southend":26104,"stills":26105,"miraculous":26106,"hodgson":26107,"richer":26108,"rochdale":26109,"ethernet":26110,"greta":26111,"uniting":26112,"prism":26113,"umm":26114,"##haya":26115,"##itical":26116,"##utation":26117,"deterioration":26118,"pointe":26119,"prowess":26120,"##ropriation":26121,"lids":26122,"scranton":26123,"billings":26124,"subcontinent":26125,"##koff":26126,"##scope":26127,"brute":26128,"kellogg":26129,"psalms":26130,"degraded":26131,"##vez":26132,"stanisław":26133,"##ructured":26134,"ferreira":26135,"pun":26136,"astonishing":26137,"gunnar":26138,"##yat":26139,"arya":26140,"prc":26141,"gottfried":26142,"##tight":26143,"excursion":26144,"##ographer":26145,"dina":26146,"##quil":26147,"##nare":26148,"huffington":26149,"illustrious":26150,"wilbur":26151,"gundam":26152,"verandah":26153,"##zard":26154,"naacp":26155,"##odle":26156,"constructive":26157,"fjord":26158,"kade":26159,"##naud":26160,"generosity":26161,"thrilling":26162,"baseline":26163,"cayman":26164,"frankish":26165,"plastics":26166,"accommodations":26167,"zoological":26168,"##fting":26169,"cedric":26170,"qb":26171,"motorized":26172,"##dome":26173,"##otted":26174,"squealed":26175,"tackled":26176,"canucks":26177,"budgets":26178,"situ":26179,"asthma":26180,"dail":26181,"gabled":26182,"grasslands":26183,"whimpered":26184,"writhing":26185,"judgments":26186,"##65":26187,"minnie":26188,"pv":26189,"##carbon":26190,"bananas":26191,"grille":26192,"domes":26193,"monique":26194,"odin":26195,"maguire":26196,"markham":26197,"tierney":26198,"##estra":26199,"##chua":26200,"libel":26201,"poke":26202,"speedy":26203,"atrium":26204,"laval":26205,"notwithstanding":26206,"##edly":26207,"fai":26208,"kala":26209,"##sur":26210,"robb":26211,"##sma":26212,"listings":26213,"luz":26214,"supplementary":26215,"tianjin":26216,"##acing":26217,"enzo":26218,"jd":26219,"ric":26220,"scanner":26221,"croats":26222,"transcribed":26223,"##49":26224,"arden":26225,"cv":26226,"##hair":26227,"##raphy":26228,"##lver":26229,"##uy":26230,"357":26231,"seventies":26232,"staggering":26233,"alam":26234,"horticultural":26235,"hs":26236,"regression":26237,"timbers":26238,"blasting":26239,"##ounded":26240,"montagu":26241,"manipulating":26242,"##cit":26243,"catalytic":26244,"1550":26245,"troopers":26246,"##meo":26247,"condemnation":26248,"fitzpatrick":26249,"##oire":26250,"##roved":26251,"inexperienced":26252,"1670":26253,"castes":26254,"##lative":26255,"outing":26256,"314":26257,"dubois":26258,"flicking":26259,"quarrel":26260,"ste":26261,"learners":26262,"1625":26263,"iq":26264,"whistled":26265,"##class":26266,"282":26267,"classify":26268,"tariffs":26269,"temperament":26270,"355":26271,"folly":26272,"liszt":26273,"##yles":26274,"immersed":26275,"jordanian":26276,"ceasefire":26277,"apparel":26278,"extras":26279,"maru":26280,"fished":26281,"##bio":26282,"harta":26283,"stockport":26284,"assortment":26285,"craftsman":26286,"paralysis":26287,"transmitters":26288,"##cola":26289,"blindness":26290,"##wk":26291,"fatally":26292,"proficiency":26293,"solemnly":26294,"##orno":26295,"repairing":26296,"amore":26297,"groceries":26298,"ultraviolet":26299,"##chase":26300,"schoolhouse":26301,"##tua":26302,"resurgence":26303,"nailed":26304,"##otype":26305,"##×":26306,"ruse":26307,"saliva":26308,"diagrams":26309,"##tructing":26310,"albans":26311,"rann":26312,"thirties":26313,"1b":26314,"antennas":26315,"hilarious":26316,"cougars":26317,"paddington":26318,"stats":26319,"##eger":26320,"breakaway":26321,"ipod":26322,"reza":26323,"authorship":26324,"prohibiting":26325,"scoffed":26326,"##etz":26327,"##ttle":26328,"conscription":26329,"defected":26330,"trondheim":26331,"##fires":26332,"ivanov":26333,"keenan":26334,"##adan":26335,"##ciful":26336,"##fb":26337,"##slow":26338,"locating":26339,"##ials":26340,"##tford":26341,"cadiz":26342,"basalt":26343,"blankly":26344,"interned":26345,"rags":26346,"rattling":26347,"##tick":26348,"carpathian":26349,"reassured":26350,"sync":26351,"bum":26352,"guildford":26353,"iss":26354,"staunch":26355,"##onga":26356,"astronomers":26357,"sera":26358,"sofie":26359,"emergencies":26360,"susquehanna":26361,"##heard":26362,"duc":26363,"mastery":26364,"vh1":26365,"williamsburg":26366,"bayer":26367,"buckled":26368,"craving":26369,"##khan":26370,"##rdes":26371,"bloomington":26372,"##write":26373,"alton":26374,"barbecue":26375,"##bians":26376,"justine":26377,"##hri":26378,"##ndt":26379,"delightful":26380,"smartphone":26381,"newtown":26382,"photon":26383,"retrieval":26384,"peugeot":26385,"hissing":26386,"##monium":26387,"##orough":26388,"flavors":26389,"lighted":26390,"relaunched":26391,"tainted":26392,"##games":26393,"##lysis":26394,"anarchy":26395,"microscopic":26396,"hopping":26397,"adept":26398,"evade":26399,"evie":26400,"##beau":26401,"inhibit":26402,"sinn":26403,"adjustable":26404,"hurst":26405,"intuition":26406,"wilton":26407,"cisco":26408,"44th":26409,"lawful":26410,"lowlands":26411,"stockings":26412,"thierry":26413,"##dalen":26414,"##hila":26415,"##nai":26416,"fates":26417,"prank":26418,"tb":26419,"maison":26420,"lobbied":26421,"provocative":26422,"1724":26423,"4a":26424,"utopia":26425,"##qual":26426,"carbonate":26427,"gujarati":26428,"purcell":26429,"##rford":26430,"curtiss":26431,"##mei":26432,"overgrown":26433,"arenas":26434,"mediation":26435,"swallows":26436,"##rnik":26437,"respectful":26438,"turnbull":26439,"##hedron":26440,"##hope":26441,"alyssa":26442,"ozone":26443,"##ʻi":26444,"ami":26445,"gestapo":26446,"johansson":26447,"snooker":26448,"canteen":26449,"cuff":26450,"declines":26451,"empathy":26452,"stigma":26453,"##ags":26454,"##iner":26455,"##raine":26456,"taxpayers":26457,"gui":26458,"volga":26459,"##wright":26460,"##copic":26461,"lifespan":26462,"overcame":26463,"tattooed":26464,"enactment":26465,"giggles":26466,"##ador":26467,"##camp":26468,"barrington":26469,"bribe":26470,"obligatory":26471,"orbiting":26472,"peng":26473,"##enas":26474,"elusive":26475,"sucker":26476,"##vating":26477,"cong":26478,"hardship":26479,"empowered":26480,"anticipating":26481,"estrada":26482,"cryptic":26483,"greasy":26484,"detainees":26485,"planck":26486,"sudbury":26487,"plaid":26488,"dod":26489,"marriott":26490,"kayla":26491,"##ears":26492,"##vb":26493,"##zd":26494,"mortally":26495,"##hein":26496,"cognition":26497,"radha":26498,"319":26499,"liechtenstein":26500,"meade":26501,"richly":26502,"argyle":26503,"harpsichord":26504,"liberalism":26505,"trumpets":26506,"lauded":26507,"tyrant":26508,"salsa":26509,"tiled":26510,"lear":26511,"promoters":26512,"reused":26513,"slicing":26514,"trident":26515,"##chuk":26516,"##gami":26517,"##lka":26518,"cantor":26519,"checkpoint":26520,"##points":26521,"gaul":26522,"leger":26523,"mammalian":26524,"##tov":26525,"##aar":26526,"##schaft":26527,"doha":26528,"frenchman":26529,"nirvana":26530,"##vino":26531,"delgado":26532,"headlining":26533,"##eron":26534,"##iography":26535,"jug":26536,"tko":26537,"1649":26538,"naga":26539,"intersections":26540,"##jia":26541,"benfica":26542,"nawab":26543,"##suka":26544,"ashford":26545,"gulp":26546,"##deck":26547,"##vill":26548,"##rug":26549,"brentford":26550,"frazier":26551,"pleasures":26552,"dunne":26553,"potsdam":26554,"shenzhen":26555,"dentistry":26556,"##tec":26557,"flanagan":26558,"##dorff":26559,"##hear":26560,"chorale":26561,"dinah":26562,"prem":26563,"quezon":26564,"##rogated":26565,"relinquished":26566,"sutra":26567,"terri":26568,"##pani":26569,"flaps":26570,"##rissa":26571,"poly":26572,"##rnet":26573,"homme":26574,"aback":26575,"##eki":26576,"linger":26577,"womb":26578,"##kson":26579,"##lewood":26580,"doorstep":26581,"orthodoxy":26582,"threaded":26583,"westfield":26584,"##rval":26585,"dioceses":26586,"fridays":26587,"subsided":26588,"##gata":26589,"loyalists":26590,"##biotic":26591,"##ettes":26592,"letterman":26593,"lunatic":26594,"prelate":26595,"tenderly":26596,"invariably":26597,"souza":26598,"thug":26599,"winslow":26600,"##otide":26601,"furlongs":26602,"gogh":26603,"jeopardy":26604,"##runa":26605,"pegasus":26606,"##umble":26607,"humiliated":26608,"standalone":26609,"tagged":26610,"##roller":26611,"freshmen":26612,"klan":26613,"##bright":26614,"attaining":26615,"initiating":26616,"transatlantic":26617,"logged":26618,"viz":26619,"##uance":26620,"1723":26621,"combatants":26622,"intervening":26623,"stephane":26624,"chieftain":26625,"despised":26626,"grazed":26627,"317":26628,"cdc":26629,"galveston":26630,"godzilla":26631,"macro":26632,"simulate":26633,"##planes":26634,"parades":26635,"##esses":26636,"960":26637,"##ductive":26638,"##unes":26639,"equator":26640,"overdose":26641,"##cans":26642,"##hosh":26643,"##lifting":26644,"joshi":26645,"epstein":26646,"sonora":26647,"treacherous":26648,"aquatics":26649,"manchu":26650,"responsive":26651,"##sation":26652,"supervisory":26653,"##christ":26654,"##llins":26655,"##ibar":26656,"##balance":26657,"##uso":26658,"kimball":26659,"karlsruhe":26660,"mab":26661,"##emy":26662,"ignores":26663,"phonetic":26664,"reuters":26665,"spaghetti":26666,"820":26667,"almighty":26668,"danzig":26669,"rumbling":26670,"tombstone":26671,"designations":26672,"lured":26673,"outset":26674,"##felt":26675,"supermarkets":26676,"##wt":26677,"grupo":26678,"kei":26679,"kraft":26680,"susanna":26681,"##blood":26682,"comprehension":26683,"genealogy":26684,"##aghan":26685,"##verted":26686,"redding":26687,"##ythe":26688,"1722":26689,"bowing":26690,"##pore":26691,"##roi":26692,"lest":26693,"sharpened":26694,"fulbright":26695,"valkyrie":26696,"sikhs":26697,"##unds":26698,"swans":26699,"bouquet":26700,"merritt":26701,"##tage":26702,"##venting":26703,"commuted":26704,"redhead":26705,"clerks":26706,"leasing":26707,"cesare":26708,"dea":26709,"hazy":26710,"##vances":26711,"fledged":26712,"greenfield":26713,"servicemen":26714,"##gical":26715,"armando":26716,"blackout":26717,"dt":26718,"sagged":26719,"downloadable":26720,"intra":26721,"potion":26722,"pods":26723,"##4th":26724,"##mism":26725,"xp":26726,"attendants":26727,"gambia":26728,"stale":26729,"##ntine":26730,"plump":26731,"asteroids":26732,"rediscovered":26733,"buds":26734,"flea":26735,"hive":26736,"##neas":26737,"1737":26738,"classifications":26739,"debuts":26740,"##eles":26741,"olympus":26742,"scala":26743,"##eurs":26744,"##gno":26745,"##mute":26746,"hummed":26747,"sigismund":26748,"visuals":26749,"wiggled":26750,"await":26751,"pilasters":26752,"clench":26753,"sulfate":26754,"##ances":26755,"bellevue":26756,"enigma":26757,"trainee":26758,"snort":26759,"##sw":26760,"clouded":26761,"denim":26762,"##rank":26763,"##rder":26764,"churning":26765,"hartman":26766,"lodges":26767,"riches":26768,"sima":26769,"##missible":26770,"accountable":26771,"socrates":26772,"regulates":26773,"mueller":26774,"##cr":26775,"1702":26776,"avoids":26777,"solids":26778,"himalayas":26779,"nutrient":26780,"pup":26781,"##jevic":26782,"squat":26783,"fades":26784,"nec":26785,"##lates":26786,"##pina":26787,"##rona":26788,"##ου":26789,"privateer":26790,"tequila":26791,"##gative":26792,"##mpton":26793,"apt":26794,"hornet":26795,"immortals":26796,"##dou":26797,"asturias":26798,"cleansing":26799,"dario":26800,"##rries":26801,"##anta":26802,"etymology":26803,"servicing":26804,"zhejiang":26805,"##venor":26806,"##nx":26807,"horned":26808,"erasmus":26809,"rayon":26810,"relocating":26811,"£10":26812,"##bags":26813,"escalated":26814,"promenade":26815,"stubble":26816,"2010s":26817,"artisans":26818,"axial":26819,"liquids":26820,"mora":26821,"sho":26822,"yoo":26823,"##tsky":26824,"bundles":26825,"oldies":26826,"##nally":26827,"notification":26828,"bastion":26829,"##ths":26830,"sparkle":26831,"##lved":26832,"1728":26833,"leash":26834,"pathogen":26835,"highs":26836,"##hmi":26837,"immature":26838,"880":26839,"gonzaga":26840,"ignatius":26841,"mansions":26842,"monterrey":26843,"sweets":26844,"bryson":26845,"##loe":26846,"polled":26847,"regatta":26848,"brightest":26849,"pei":26850,"rosy":26851,"squid":26852,"hatfield":26853,"payroll":26854,"addict":26855,"meath":26856,"cornerback":26857,"heaviest":26858,"lodging":26859,"##mage":26860,"capcom":26861,"rippled":26862,"##sily":26863,"barnet":26864,"mayhem":26865,"ymca":26866,"snuggled":26867,"rousseau":26868,"##cute":26869,"blanchard":26870,"284":26871,"fragmented":26872,"leighton":26873,"chromosomes":26874,"risking":26875,"##md":26876,"##strel":26877,"##utter":26878,"corinne":26879,"coyotes":26880,"cynical":26881,"hiroshi":26882,"yeomanry":26883,"##ractive":26884,"ebook":26885,"grading":26886,"mandela":26887,"plume":26888,"agustin":26889,"magdalene":26890,"##rkin":26891,"bea":26892,"femme":26893,"trafford":26894,"##coll":26895,"##lun":26896,"##tance":26897,"52nd":26898,"fourier":26899,"upton":26900,"##mental":26901,"camilla":26902,"gust":26903,"iihf":26904,"islamabad":26905,"longevity":26906,"##kala":26907,"feldman":26908,"netting":26909,"##rization":26910,"endeavour":26911,"foraging":26912,"mfa":26913,"orr":26914,"##open":26915,"greyish":26916,"contradiction":26917,"graz":26918,"##ruff":26919,"handicapped":26920,"marlene":26921,"tweed":26922,"oaxaca":26923,"spp":26924,"campos":26925,"miocene":26926,"pri":26927,"configured":26928,"cooks":26929,"pluto":26930,"cozy":26931,"pornographic":26932,"##entes":26933,"70th":26934,"fairness":26935,"glided":26936,"jonny":26937,"lynne":26938,"rounding":26939,"sired":26940,"##emon":26941,"##nist":26942,"remade":26943,"uncover":26944,"##mack":26945,"complied":26946,"lei":26947,"newsweek":26948,"##jured":26949,"##parts":26950,"##enting":26951,"##pg":26952,"293":26953,"finer":26954,"guerrillas":26955,"athenian":26956,"deng":26957,"disused":26958,"stepmother":26959,"accuse":26960,"gingerly":26961,"seduction":26962,"521":26963,"confronting":26964,"##walker":26965,"##going":26966,"gora":26967,"nostalgia":26968,"sabres":26969,"virginity":26970,"wrenched":26971,"##minated":26972,"syndication":26973,"wielding":26974,"eyre":26975,"##56":26976,"##gnon":26977,"##igny":26978,"behaved":26979,"taxpayer":26980,"sweeps":26981,"##growth":26982,"childless":26983,"gallant":26984,"##ywood":26985,"amplified":26986,"geraldine":26987,"scrape":26988,"##ffi":26989,"babylonian":26990,"fresco":26991,"##rdan":26992,"##kney":26993,"##position":26994,"1718":26995,"restricting":26996,"tack":26997,"fukuoka":26998,"osborn":26999,"selector":27000,"partnering":27001,"##dlow":27002,"318":27003,"gnu":27004,"kia":27005,"tak":27006,"whitley":27007,"gables":27008,"##54":27009,"##mania":27010,"mri":27011,"softness":27012,"immersion":27013,"##bots":27014,"##evsky":27015,"1713":27016,"chilling":27017,"insignificant":27018,"pcs":27019,"##uis":27020,"elites":27021,"lina":27022,"purported":27023,"supplemental":27024,"teaming":27025,"##americana":27026,"##dding":27027,"##inton":27028,"proficient":27029,"rouen":27030,"##nage":27031,"##rret":27032,"niccolo":27033,"selects":27034,"##bread":27035,"fluffy":27036,"1621":27037,"gruff":27038,"knotted":27039,"mukherjee":27040,"polgara":27041,"thrash":27042,"nicholls":27043,"secluded":27044,"smoothing":27045,"thru":27046,"corsica":27047,"loaf":27048,"whitaker":27049,"inquiries":27050,"##rrier":27051,"##kam":27052,"indochina":27053,"289":27054,"marlins":27055,"myles":27056,"peking":27057,"##tea":27058,"extracts":27059,"pastry":27060,"superhuman":27061,"connacht":27062,"vogel":27063,"##ditional":27064,"##het":27065,"##udged":27066,"##lash":27067,"gloss":27068,"quarries":27069,"refit":27070,"teaser":27071,"##alic":27072,"##gaon":27073,"20s":27074,"materialized":27075,"sling":27076,"camped":27077,"pickering":27078,"tung":27079,"tracker":27080,"pursuant":27081,"##cide":27082,"cranes":27083,"soc":27084,"##cini":27085,"##typical":27086,"##viere":27087,"anhalt":27088,"overboard":27089,"workout":27090,"chores":27091,"fares":27092,"orphaned":27093,"stains":27094,"##logie":27095,"fenton":27096,"surpassing":27097,"joyah":27098,"triggers":27099,"##itte":27100,"grandmaster":27101,"##lass":27102,"##lists":27103,"clapping":27104,"fraudulent":27105,"ledger":27106,"nagasaki":27107,"##cor":27108,"##nosis":27109,"##tsa":27110,"eucalyptus":27111,"tun":27112,"##icio":27113,"##rney":27114,"##tara":27115,"dax":27116,"heroism":27117,"ina":27118,"wrexham":27119,"onboard":27120,"unsigned":27121,"##dates":27122,"moshe":27123,"galley":27124,"winnie":27125,"droplets":27126,"exiles":27127,"praises":27128,"watered":27129,"noodles":27130,"##aia":27131,"fein":27132,"adi":27133,"leland":27134,"multicultural":27135,"stink":27136,"bingo":27137,"comets":27138,"erskine":27139,"modernized":27140,"canned":27141,"constraint":27142,"domestically":27143,"chemotherapy":27144,"featherweight":27145,"stifled":27146,"##mum":27147,"darkly":27148,"irresistible":27149,"refreshing":27150,"hasty":27151,"isolate":27152,"##oys":27153,"kitchener":27154,"planners":27155,"##wehr":27156,"cages":27157,"yarn":27158,"implant":27159,"toulon":27160,"elects":27161,"childbirth":27162,"yue":27163,"##lind":27164,"##lone":27165,"cn":27166,"rightful":27167,"sportsman":27168,"junctions":27169,"remodeled":27170,"specifies":27171,"##rgh":27172,"291":27173,"##oons":27174,"complimented":27175,"##urgent":27176,"lister":27177,"ot":27178,"##logic":27179,"bequeathed":27180,"cheekbones":27181,"fontana":27182,"gabby":27183,"##dial":27184,"amadeus":27185,"corrugated":27186,"maverick":27187,"resented":27188,"triangles":27189,"##hered":27190,"##usly":27191,"nazareth":27192,"tyrol":27193,"1675":27194,"assent":27195,"poorer":27196,"sectional":27197,"aegean":27198,"##cous":27199,"296":27200,"nylon":27201,"ghanaian":27202,"##egorical":27203,"##weig":27204,"cushions":27205,"forbid":27206,"fusiliers":27207,"obstruction":27208,"somerville":27209,"##scia":27210,"dime":27211,"earrings":27212,"elliptical":27213,"leyte":27214,"oder":27215,"polymers":27216,"timmy":27217,"atm":27218,"midtown":27219,"piloted":27220,"settles":27221,"continual":27222,"externally":27223,"mayfield":27224,"##uh":27225,"enrichment":27226,"henson":27227,"keane":27228,"persians":27229,"1733":27230,"benji":27231,"braden":27232,"pep":27233,"324":27234,"##efe":27235,"contenders":27236,"pepsi":27237,"valet":27238,"##isches":27239,"298":27240,"##asse":27241,"##earing":27242,"goofy":27243,"stroll":27244,"##amen":27245,"authoritarian":27246,"occurrences":27247,"adversary":27248,"ahmedabad":27249,"tangent":27250,"toppled":27251,"dorchester":27252,"1672":27253,"modernism":27254,"marxism":27255,"islamist":27256,"charlemagne":27257,"exponential":27258,"racks":27259,"unicode":27260,"brunette":27261,"mbc":27262,"pic":27263,"skirmish":27264,"##bund":27265,"##lad":27266,"##powered":27267,"##yst":27268,"hoisted":27269,"messina":27270,"shatter":27271,"##ctum":27272,"jedi":27273,"vantage":27274,"##music":27275,"##neil":27276,"clemens":27277,"mahmoud":27278,"corrupted":27279,"authentication":27280,"lowry":27281,"nils":27282,"##washed":27283,"omnibus":27284,"wounding":27285,"jillian":27286,"##itors":27287,"##opped":27288,"serialized":27289,"narcotics":27290,"handheld":27291,"##arm":27292,"##plicity":27293,"intersecting":27294,"stimulating":27295,"##onis":27296,"crate":27297,"fellowships":27298,"hemingway":27299,"casinos":27300,"climatic":27301,"fordham":27302,"copeland":27303,"drip":27304,"beatty":27305,"leaflets":27306,"robber":27307,"brothel":27308,"madeira":27309,"##hedral":27310,"sphinx":27311,"ultrasound":27312,"##vana":27313,"valor":27314,"forbade":27315,"leonid":27316,"villas":27317,"##aldo":27318,"duane":27319,"marquez":27320,"##cytes":27321,"disadvantaged":27322,"forearms":27323,"kawasaki":27324,"reacts":27325,"consular":27326,"lax":27327,"uncles":27328,"uphold":27329,"##hopper":27330,"concepcion":27331,"dorsey":27332,"lass":27333,"##izan":27334,"arching":27335,"passageway":27336,"1708":27337,"researches":27338,"tia":27339,"internationals":27340,"##graphs":27341,"##opers":27342,"distinguishes":27343,"javanese":27344,"divert":27345,"##uven":27346,"plotted":27347,"##listic":27348,"##rwin":27349,"##erik":27350,"##tify":27351,"affirmative":27352,"signifies":27353,"validation":27354,"##bson":27355,"kari":27356,"felicity":27357,"georgina":27358,"zulu":27359,"##eros":27360,"##rained":27361,"##rath":27362,"overcoming":27363,"##dot":27364,"argyll":27365,"##rbin":27366,"1734":27367,"chiba":27368,"ratification":27369,"windy":27370,"earls":27371,"parapet":27372,"##marks":27373,"hunan":27374,"pristine":27375,"astrid":27376,"punta":27377,"##gart":27378,"brodie":27379,"##kota":27380,"##oder":27381,"malaga":27382,"minerva":27383,"rouse":27384,"##phonic":27385,"bellowed":27386,"pagoda":27387,"portals":27388,"reclamation":27389,"##gur":27390,"##odies":27391,"##⁄₄":27392,"parentheses":27393,"quoting":27394,"allergic":27395,"palette":27396,"showcases":27397,"benefactor":27398,"heartland":27399,"nonlinear":27400,"##tness":27401,"bladed":27402,"cheerfully":27403,"scans":27404,"##ety":27405,"##hone":27406,"1666":27407,"girlfriends":27408,"pedersen":27409,"hiram":27410,"sous":27411,"##liche":27412,"##nator":27413,"1683":27414,"##nery":27415,"##orio":27416,"##umen":27417,"bobo":27418,"primaries":27419,"smiley":27420,"##cb":27421,"unearthed":27422,"uniformly":27423,"fis":27424,"metadata":27425,"1635":27426,"ind":27427,"##oted":27428,"recoil":27429,"##titles":27430,"##tura":27431,"##ια":27432,"406":27433,"hilbert":27434,"jamestown":27435,"mcmillan":27436,"tulane":27437,"seychelles":27438,"##frid":27439,"antics":27440,"coli":27441,"fated":27442,"stucco":27443,"##grants":27444,"1654":27445,"bulky":27446,"accolades":27447,"arrays":27448,"caledonian":27449,"carnage":27450,"optimism":27451,"puebla":27452,"##tative":27453,"##cave":27454,"enforcing":27455,"rotherham":27456,"seo":27457,"dunlop":27458,"aeronautics":27459,"chimed":27460,"incline":27461,"zoning":27462,"archduke":27463,"hellenistic":27464,"##oses":27465,"##sions":27466,"candi":27467,"thong":27468,"##ople":27469,"magnate":27470,"rustic":27471,"##rsk":27472,"projective":27473,"slant":27474,"##offs":27475,"danes":27476,"hollis":27477,"vocalists":27478,"##ammed":27479,"congenital":27480,"contend":27481,"gesellschaft":27482,"##ocating":27483,"##pressive":27484,"douglass":27485,"quieter":27486,"##cm":27487,"##kshi":27488,"howled":27489,"salim":27490,"spontaneously":27491,"townsville":27492,"buena":27493,"southport":27494,"##bold":27495,"kato":27496,"1638":27497,"faerie":27498,"stiffly":27499,"##vus":27500,"##rled":27501,"297":27502,"flawless":27503,"realising":27504,"taboo":27505,"##7th":27506,"bytes":27507,"straightening":27508,"356":27509,"jena":27510,"##hid":27511,"##rmin":27512,"cartwright":27513,"berber":27514,"bertram":27515,"soloists":27516,"411":27517,"noses":27518,"417":27519,"coping":27520,"fission":27521,"hardin":27522,"inca":27523,"##cen":27524,"1717":27525,"mobilized":27526,"vhf":27527,"##raf":27528,"biscuits":27529,"curate":27530,"##85":27531,"##anial":27532,"331":27533,"gaunt":27534,"neighbourhoods":27535,"1540":27536,"##abas":27537,"blanca":27538,"bypassed":27539,"sockets":27540,"behold":27541,"coincidentally":27542,"##bane":27543,"nara":27544,"shave":27545,"splinter":27546,"terrific":27547,"##arion":27548,"##erian":27549,"commonplace":27550,"juris":27551,"redwood":27552,"waistband":27553,"boxed":27554,"caitlin":27555,"fingerprints":27556,"jennie":27557,"naturalized":27558,"##ired":27559,"balfour":27560,"craters":27561,"jody":27562,"bungalow":27563,"hugely":27564,"quilt":27565,"glitter":27566,"pigeons":27567,"undertaker":27568,"bulging":27569,"constrained":27570,"goo":27571,"##sil":27572,"##akh":27573,"assimilation":27574,"reworked":27575,"##person":27576,"persuasion":27577,"##pants":27578,"felicia":27579,"##cliff":27580,"##ulent":27581,"1732":27582,"explodes":27583,"##dun":27584,"##inium":27585,"##zic":27586,"lyman":27587,"vulture":27588,"hog":27589,"overlook":27590,"begs":27591,"northwards":27592,"ow":27593,"spoil":27594,"##urer":27595,"fatima":27596,"favorably":27597,"accumulate":27598,"sargent":27599,"sorority":27600,"corresponded":27601,"dispersal":27602,"kochi":27603,"toned":27604,"##imi":27605,"##lita":27606,"internacional":27607,"newfound":27608,"##agger":27609,"##lynn":27610,"##rigue":27611,"booths":27612,"peanuts":27613,"##eborg":27614,"medicare":27615,"muriel":27616,"nur":27617,"##uram":27618,"crates":27619,"millennia":27620,"pajamas":27621,"worsened":27622,"##breakers":27623,"jimi":27624,"vanuatu":27625,"yawned":27626,"##udeau":27627,"carousel":27628,"##hony":27629,"hurdle":27630,"##ccus":27631,"##mounted":27632,"##pod":27633,"rv":27634,"##eche":27635,"airship":27636,"ambiguity":27637,"compulsion":27638,"recapture":27639,"##claiming":27640,"arthritis":27641,"##osomal":27642,"1667":27643,"asserting":27644,"ngc":27645,"sniffing":27646,"dade":27647,"discontent":27648,"glendale":27649,"ported":27650,"##amina":27651,"defamation":27652,"rammed":27653,"##scent":27654,"fling":27655,"livingstone":27656,"##fleet":27657,"875":27658,"##ppy":27659,"apocalyptic":27660,"comrade":27661,"lcd":27662,"##lowe":27663,"cessna":27664,"eine":27665,"persecuted":27666,"subsistence":27667,"demi":27668,"hoop":27669,"reliefs":27670,"710":27671,"coptic":27672,"progressing":27673,"stemmed":27674,"perpetrators":27675,"1665":27676,"priestess":27677,"##nio":27678,"dobson":27679,"ebony":27680,"rooster":27681,"itf":27682,"tortricidae":27683,"##bbon":27684,"##jian":27685,"cleanup":27686,"##jean":27687,"##øy":27688,"1721":27689,"eighties":27690,"taxonomic":27691,"holiness":27692,"##hearted":27693,"##spar":27694,"antilles":27695,"showcasing":27696,"stabilized":27697,"##nb":27698,"gia":27699,"mascara":27700,"michelangelo":27701,"dawned":27702,"##uria":27703,"##vinsky":27704,"extinguished":27705,"fitz":27706,"grotesque":27707,"£100":27708,"##fera":27709,"##loid":27710,"##mous":27711,"barges":27712,"neue":27713,"throbbed":27714,"cipher":27715,"johnnie":27716,"##a1":27717,"##mpt":27718,"outburst":27719,"##swick":27720,"spearheaded":27721,"administrations":27722,"c1":27723,"heartbreak":27724,"pixels":27725,"pleasantly":27726,"##enay":27727,"lombardy":27728,"plush":27729,"##nsed":27730,"bobbie":27731,"##hly":27732,"reapers":27733,"tremor":27734,"xiang":27735,"minogue":27736,"substantive":27737,"hitch":27738,"barak":27739,"##wyl":27740,"kwan":27741,"##encia":27742,"910":27743,"obscene":27744,"elegance":27745,"indus":27746,"surfer":27747,"bribery":27748,"conserve":27749,"##hyllum":27750,"##masters":27751,"horatio":27752,"##fat":27753,"apes":27754,"rebound":27755,"psychotic":27756,"##pour":27757,"iteration":27758,"##mium":27759,"##vani":27760,"botanic":27761,"horribly":27762,"antiques":27763,"dispose":27764,"paxton":27765,"##hli":27766,"##wg":27767,"timeless":27768,"1704":27769,"disregard":27770,"engraver":27771,"hounds":27772,"##bau":27773,"##version":27774,"looted":27775,"uno":27776,"facilitates":27777,"groans":27778,"masjid":27779,"rutland":27780,"antibody":27781,"disqualification":27782,"decatur":27783,"footballers":27784,"quake":27785,"slacks":27786,"48th":27787,"rein":27788,"scribe":27789,"stabilize":27790,"commits":27791,"exemplary":27792,"tho":27793,"##hort":27794,"##chison":27795,"pantry":27796,"traversed":27797,"##hiti":27798,"disrepair":27799,"identifiable":27800,"vibrated":27801,"baccalaureate":27802,"##nnis":27803,"csa":27804,"interviewing":27805,"##iensis":27806,"##raße":27807,"greaves":27808,"wealthiest":27809,"343":27810,"classed":27811,"jogged":27812,"£5":27813,"##58":27814,"##atal":27815,"illuminating":27816,"knicks":27817,"respecting":27818,"##uno":27819,"scrubbed":27820,"##iji":27821,"##dles":27822,"kruger":27823,"moods":27824,"growls":27825,"raider":27826,"silvia":27827,"chefs":27828,"kam":27829,"vr":27830,"cree":27831,"percival":27832,"##terol":27833,"gunter":27834,"counterattack":27835,"defiant":27836,"henan":27837,"ze":27838,"##rasia":27839,"##riety":27840,"equivalence":27841,"submissions":27842,"##fra":27843,"##thor":27844,"bautista":27845,"mechanically":27846,"##heater":27847,"cornice":27848,"herbal":27849,"templar":27850,"##mering":27851,"outputs":27852,"ruining":27853,"ligand":27854,"renumbered":27855,"extravagant":27856,"mika":27857,"blockbuster":27858,"eta":27859,"insurrection":27860,"##ilia":27861,"darkening":27862,"ferocious":27863,"pianos":27864,"strife":27865,"kinship":27866,"##aer":27867,"melee":27868,"##anor":27869,"##iste":27870,"##may":27871,"##oue":27872,"decidedly":27873,"weep":27874,"##jad":27875,"##missive":27876,"##ppel":27877,"354":27878,"puget":27879,"unease":27880,"##gnant":27881,"1629":27882,"hammering":27883,"kassel":27884,"ob":27885,"wessex":27886,"##lga":27887,"bromwich":27888,"egan":27889,"paranoia":27890,"utilization":27891,"##atable":27892,"##idad":27893,"contradictory":27894,"provoke":27895,"##ols":27896,"##ouring":27897,"##tangled":27898,"knesset":27899,"##very":27900,"##lette":27901,"plumbing":27902,"##sden":27903,"##¹":27904,"greensboro":27905,"occult":27906,"sniff":27907,"338":27908,"zev":27909,"beaming":27910,"gamer":27911,"haggard":27912,"mahal":27913,"##olt":27914,"##pins":27915,"mendes":27916,"utmost":27917,"briefing":27918,"gunnery":27919,"##gut":27920,"##pher":27921,"##zh":27922,"##rok":27923,"1679":27924,"khalifa":27925,"sonya":27926,"##boot":27927,"principals":27928,"urbana":27929,"wiring":27930,"##liffe":27931,"##minating":27932,"##rrado":27933,"dahl":27934,"nyu":27935,"skepticism":27936,"np":27937,"townspeople":27938,"ithaca":27939,"lobster":27940,"somethin":27941,"##fur":27942,"##arina":27943,"##−1":27944,"freighter":27945,"zimmerman":27946,"biceps":27947,"contractual":27948,"##herton":27949,"amend":27950,"hurrying":27951,"subconscious":27952,"##anal":27953,"336":27954,"meng":27955,"clermont":27956,"spawning":27957,"##eia":27958,"##lub":27959,"dignitaries":27960,"impetus":27961,"snacks":27962,"spotting":27963,"twigs":27964,"##bilis":27965,"##cz":27966,"##ouk":27967,"libertadores":27968,"nic":27969,"skylar":27970,"##aina":27971,"##firm":27972,"gustave":27973,"asean":27974,"##anum":27975,"dieter":27976,"legislatures":27977,"flirt":27978,"bromley":27979,"trolls":27980,"umar":27981,"##bbies":27982,"##tyle":27983,"blah":27984,"parc":27985,"bridgeport":27986,"crank":27987,"negligence":27988,"##nction":27989,"46th":27990,"constantin":27991,"molded":27992,"bandages":27993,"seriousness":27994,"00pm":27995,"siegel":27996,"carpets":27997,"compartments":27998,"upbeat":27999,"statehood":28000,"##dner":28001,"##edging":28002,"marko":28003,"730":28004,"platt":28005,"##hane":28006,"paving":28007,"##iy":28008,"1738":28009,"abbess":28010,"impatience":28011,"limousine":28012,"nbl":28013,"##talk":28014,"441":28015,"lucille":28016,"mojo":28017,"nightfall":28018,"robbers":28019,"##nais":28020,"karel":28021,"brisk":28022,"calves":28023,"replicate":28024,"ascribed":28025,"telescopes":28026,"##olf":28027,"intimidated":28028,"##reen":28029,"ballast":28030,"specialization":28031,"##sit":28032,"aerodynamic":28033,"caliphate":28034,"rainer":28035,"visionary":28036,"##arded":28037,"epsilon":28038,"##aday":28039,"##onte":28040,"aggregation":28041,"auditory":28042,"boosted":28043,"reunification":28044,"kathmandu":28045,"loco":28046,"robyn":28047,"402":28048,"acknowledges":28049,"appointing":28050,"humanoid":28051,"newell":28052,"redeveloped":28053,"restraints":28054,"##tained":28055,"barbarians":28056,"chopper":28057,"1609":28058,"italiana":28059,"##lez":28060,"##lho":28061,"investigates":28062,"wrestlemania":28063,"##anies":28064,"##bib":28065,"690":28066,"##falls":28067,"creaked":28068,"dragoons":28069,"gravely":28070,"minions":28071,"stupidity":28072,"volley":28073,"##harat":28074,"##week":28075,"musik":28076,"##eries":28077,"##uously":28078,"fungal":28079,"massimo":28080,"semantics":28081,"malvern":28082,"##ahl":28083,"##pee":28084,"discourage":28085,"embryo":28086,"imperialism":28087,"1910s":28088,"profoundly":28089,"##ddled":28090,"jiangsu":28091,"sparkled":28092,"stat":28093,"##holz":28094,"sweatshirt":28095,"tobin":28096,"##iction":28097,"sneered":28098,"##cheon":28099,"##oit":28100,"brit":28101,"causal":28102,"smyth":28103,"##neuve":28104,"diffuse":28105,"perrin":28106,"silvio":28107,"##ipes":28108,"##recht":28109,"detonated":28110,"iqbal":28111,"selma":28112,"##nism":28113,"##zumi":28114,"roasted":28115,"##riders":28116,"tay":28117,"##ados":28118,"##mament":28119,"##mut":28120,"##rud":28121,"840":28122,"completes":28123,"nipples":28124,"cfa":28125,"flavour":28126,"hirsch":28127,"##laus":28128,"calderon":28129,"sneakers":28130,"moravian":28131,"##ksha":28132,"1622":28133,"rq":28134,"294":28135,"##imeters":28136,"bodo":28137,"##isance":28138,"##pre":28139,"##ronia":28140,"anatomical":28141,"excerpt":28142,"##lke":28143,"dh":28144,"kunst":28145,"##tablished":28146,"##scoe":28147,"biomass":28148,"panted":28149,"unharmed":28150,"gael":28151,"housemates":28152,"montpellier":28153,"##59":28154,"coa":28155,"rodents":28156,"tonic":28157,"hickory":28158,"singleton":28159,"##taro":28160,"451":28161,"1719":28162,"aldo":28163,"breaststroke":28164,"dempsey":28165,"och":28166,"rocco":28167,"##cuit":28168,"merton":28169,"dissemination":28170,"midsummer":28171,"serials":28172,"##idi":28173,"haji":28174,"polynomials":28175,"##rdon":28176,"gs":28177,"enoch":28178,"prematurely":28179,"shutter":28180,"taunton":28181,"£3":28182,"##grating":28183,"##inates":28184,"archangel":28185,"harassed":28186,"##asco":28187,"326":28188,"archway":28189,"dazzling":28190,"##ecin":28191,"1736":28192,"sumo":28193,"wat":28194,"##kovich":28195,"1086":28196,"honneur":28197,"##ently":28198,"##nostic":28199,"##ttal":28200,"##idon":28201,"1605":28202,"403":28203,"1716":28204,"blogger":28205,"rents":28206,"##gnan":28207,"hires":28208,"##ikh":28209,"##dant":28210,"howie":28211,"##rons":28212,"handler":28213,"retracted":28214,"shocks":28215,"1632":28216,"arun":28217,"duluth":28218,"kepler":28219,"trumpeter":28220,"##lary":28221,"peeking":28222,"seasoned":28223,"trooper":28224,"##mara":28225,"laszlo":28226,"##iciencies":28227,"##rti":28228,"heterosexual":28229,"##inatory":28230,"##ssion":28231,"indira":28232,"jogging":28233,"##inga":28234,"##lism":28235,"beit":28236,"dissatisfaction":28237,"malice":28238,"##ately":28239,"nedra":28240,"peeling":28241,"##rgeon":28242,"47th":28243,"stadiums":28244,"475":28245,"vertigo":28246,"##ains":28247,"iced":28248,"restroom":28249,"##plify":28250,"##tub":28251,"illustrating":28252,"pear":28253,"##chner":28254,"##sibility":28255,"inorganic":28256,"rappers":28257,"receipts":28258,"watery":28259,"##kura":28260,"lucinda":28261,"##oulos":28262,"reintroduced":28263,"##8th":28264,"##tched":28265,"gracefully":28266,"saxons":28267,"nutritional":28268,"wastewater":28269,"rained":28270,"favourites":28271,"bedrock":28272,"fisted":28273,"hallways":28274,"likeness":28275,"upscale":28276,"##lateral":28277,"1580":28278,"blinds":28279,"prequel":28280,"##pps":28281,"##tama":28282,"deter":28283,"humiliating":28284,"restraining":28285,"tn":28286,"vents":28287,"1659":28288,"laundering":28289,"recess":28290,"rosary":28291,"tractors":28292,"coulter":28293,"federer":28294,"##ifiers":28295,"##plin":28296,"persistence":28297,"##quitable":28298,"geschichte":28299,"pendulum":28300,"quakers":28301,"##beam":28302,"bassett":28303,"pictorial":28304,"buffet":28305,"koln":28306,"##sitor":28307,"drills":28308,"reciprocal":28309,"shooters":28310,"##57":28311,"##cton":28312,"##tees":28313,"converge":28314,"pip":28315,"dmitri":28316,"donnelly":28317,"yamamoto":28318,"aqua":28319,"azores":28320,"demographics":28321,"hypnotic":28322,"spitfire":28323,"suspend":28324,"wryly":28325,"roderick":28326,"##rran":28327,"sebastien":28328,"##asurable":28329,"mavericks":28330,"##fles":28331,"##200":28332,"himalayan":28333,"prodigy":28334,"##iance":28335,"transvaal":28336,"demonstrators":28337,"handcuffs":28338,"dodged":28339,"mcnamara":28340,"sublime":28341,"1726":28342,"crazed":28343,"##efined":28344,"##till":28345,"ivo":28346,"pondered":28347,"reconciled":28348,"shrill":28349,"sava":28350,"##duk":28351,"bal":28352,"cad":28353,"heresy":28354,"jaipur":28355,"goran":28356,"##nished":28357,"341":28358,"lux":28359,"shelly":28360,"whitehall":28361,"##hre":28362,"israelis":28363,"peacekeeping":28364,"##wled":28365,"1703":28366,"demetrius":28367,"ousted":28368,"##arians":28369,"##zos":28370,"beale":28371,"anwar":28372,"backstroke":28373,"raged":28374,"shrinking":28375,"cremated":28376,"##yck":28377,"benign":28378,"towing":28379,"wadi":28380,"darmstadt":28381,"landfill":28382,"parana":28383,"soothe":28384,"colleen":28385,"sidewalks":28386,"mayfair":28387,"tumble":28388,"hepatitis":28389,"ferrer":28390,"superstructure":28391,"##gingly":28392,"##urse":28393,"##wee":28394,"anthropological":28395,"translators":28396,"##mies":28397,"closeness":28398,"hooves":28399,"##pw":28400,"mondays":28401,"##roll":28402,"##vita":28403,"landscaping":28404,"##urized":28405,"purification":28406,"sock":28407,"thorns":28408,"thwarted":28409,"jalan":28410,"tiberius":28411,"##taka":28412,"saline":28413,"##rito":28414,"confidently":28415,"khyber":28416,"sculptors":28417,"##ij":28418,"brahms":28419,"hammersmith":28420,"inspectors":28421,"battista":28422,"fivb":28423,"fragmentation":28424,"hackney":28425,"##uls":28426,"arresting":28427,"exercising":28428,"antoinette":28429,"bedfordshire":28430,"##zily":28431,"dyed":28432,"##hema":28433,"1656":28434,"racetrack":28435,"variability":28436,"##tique":28437,"1655":28438,"austrians":28439,"deteriorating":28440,"madman":28441,"theorists":28442,"aix":28443,"lehman":28444,"weathered":28445,"1731":28446,"decreed":28447,"eruptions":28448,"1729":28449,"flaw":28450,"quinlan":28451,"sorbonne":28452,"flutes":28453,"nunez":28454,"1711":28455,"adored":28456,"downwards":28457,"fable":28458,"rasped":28459,"1712":28460,"moritz":28461,"mouthful":28462,"renegade":28463,"shivers":28464,"stunts":28465,"dysfunction":28466,"restrain":28467,"translit":28468,"327":28469,"pancakes":28470,"##avio":28471,"##cision":28472,"##tray":28473,"351":28474,"vial":28475,"##lden":28476,"bain":28477,"##maid":28478,"##oxide":28479,"chihuahua":28480,"malacca":28481,"vimes":28482,"##rba":28483,"##rnier":28484,"1664":28485,"donnie":28486,"plaques":28487,"##ually":28488,"337":28489,"bangs":28490,"floppy":28491,"huntsville":28492,"loretta":28493,"nikolay":28494,"##otte":28495,"eater":28496,"handgun":28497,"ubiquitous":28498,"##hett":28499,"eras":28500,"zodiac":28501,"1634":28502,"##omorphic":28503,"1820s":28504,"##zog":28505,"cochran":28506,"##bula":28507,"##lithic":28508,"warring":28509,"##rada":28510,"dalai":28511,"excused":28512,"blazers":28513,"mcconnell":28514,"reeling":28515,"bot":28516,"este":28517,"##abi":28518,"geese":28519,"hoax":28520,"taxon":28521,"##bla":28522,"guitarists":28523,"##icon":28524,"condemning":28525,"hunts":28526,"inversion":28527,"moffat":28528,"taekwondo":28529,"##lvis":28530,"1624":28531,"stammered":28532,"##rest":28533,"##rzy":28534,"sousa":28535,"fundraiser":28536,"marylebone":28537,"navigable":28538,"uptown":28539,"cabbage":28540,"daniela":28541,"salman":28542,"shitty":28543,"whimper":28544,"##kian":28545,"##utive":28546,"programmers":28547,"protections":28548,"rm":28549,"##rmi":28550,"##rued":28551,"forceful":28552,"##enes":28553,"fuss":28554,"##tao":28555,"##wash":28556,"brat":28557,"oppressive":28558,"reykjavik":28559,"spartak":28560,"ticking":28561,"##inkles":28562,"##kiewicz":28563,"adolph":28564,"horst":28565,"maui":28566,"protege":28567,"straighten":28568,"cpc":28569,"landau":28570,"concourse":28571,"clements":28572,"resultant":28573,"##ando":28574,"imaginative":28575,"joo":28576,"reactivated":28577,"##rem":28578,"##ffled":28579,"##uising":28580,"consultative":28581,"##guide":28582,"flop":28583,"kaitlyn":28584,"mergers":28585,"parenting":28586,"somber":28587,"##vron":28588,"supervise":28589,"vidhan":28590,"##imum":28591,"courtship":28592,"exemplified":28593,"harmonies":28594,"medallist":28595,"refining":28596,"##rrow":28597,"##ка":28598,"amara":28599,"##hum":28600,"780":28601,"goalscorer":28602,"sited":28603,"overshadowed":28604,"rohan":28605,"displeasure":28606,"secretive":28607,"multiplied":28608,"osman":28609,"##orth":28610,"engravings":28611,"padre":28612,"##kali":28613,"##veda":28614,"miniatures":28615,"mis":28616,"##yala":28617,"clap":28618,"pali":28619,"rook":28620,"##cana":28621,"1692":28622,"57th":28623,"antennae":28624,"astro":28625,"oskar":28626,"1628":28627,"bulldog":28628,"crotch":28629,"hackett":28630,"yucatan":28631,"##sure":28632,"amplifiers":28633,"brno":28634,"ferrara":28635,"migrating":28636,"##gree":28637,"thanking":28638,"turing":28639,"##eza":28640,"mccann":28641,"ting":28642,"andersson":28643,"onslaught":28644,"gaines":28645,"ganga":28646,"incense":28647,"standardization":28648,"##mation":28649,"sentai":28650,"scuba":28651,"stuffing":28652,"turquoise":28653,"waivers":28654,"alloys":28655,"##vitt":28656,"regaining":28657,"vaults":28658,"##clops":28659,"##gizing":28660,"digger":28661,"furry":28662,"memorabilia":28663,"probing":28664,"##iad":28665,"payton":28666,"rec":28667,"deutschland":28668,"filippo":28669,"opaque":28670,"seamen":28671,"zenith":28672,"afrikaans":28673,"##filtration":28674,"disciplined":28675,"inspirational":28676,"##merie":28677,"banco":28678,"confuse":28679,"grafton":28680,"tod":28681,"##dgets":28682,"championed":28683,"simi":28684,"anomaly":28685,"biplane":28686,"##ceptive":28687,"electrode":28688,"##para":28689,"1697":28690,"cleavage":28691,"crossbow":28692,"swirl":28693,"informant":28694,"##lars":28695,"##osta":28696,"afi":28697,"bonfire":28698,"spec":28699,"##oux":28700,"lakeside":28701,"slump":28702,"##culus":28703,"##lais":28704,"##qvist":28705,"##rrigan":28706,"1016":28707,"facades":28708,"borg":28709,"inwardly":28710,"cervical":28711,"xl":28712,"pointedly":28713,"050":28714,"stabilization":28715,"##odon":28716,"chests":28717,"1699":28718,"hacked":28719,"ctv":28720,"orthogonal":28721,"suzy":28722,"##lastic":28723,"gaulle":28724,"jacobite":28725,"rearview":28726,"##cam":28727,"##erted":28728,"ashby":28729,"##drik":28730,"##igate":28731,"##mise":28732,"##zbek":28733,"affectionately":28734,"canine":28735,"disperse":28736,"latham":28737,"##istles":28738,"##ivar":28739,"spielberg":28740,"##orin":28741,"##idium":28742,"ezekiel":28743,"cid":28744,"##sg":28745,"durga":28746,"middletown":28747,"##cina":28748,"customized":28749,"frontiers":28750,"harden":28751,"##etano":28752,"##zzy":28753,"1604":28754,"bolsheviks":28755,"##66":28756,"coloration":28757,"yoko":28758,"##bedo":28759,"briefs":28760,"slabs":28761,"debra":28762,"liquidation":28763,"plumage":28764,"##oin":28765,"blossoms":28766,"dementia":28767,"subsidy":28768,"1611":28769,"proctor":28770,"relational":28771,"jerseys":28772,"parochial":28773,"ter":28774,"##ici":28775,"esa":28776,"peshawar":28777,"cavalier":28778,"loren":28779,"cpi":28780,"idiots":28781,"shamrock":28782,"1646":28783,"dutton":28784,"malabar":28785,"mustache":28786,"##endez":28787,"##ocytes":28788,"referencing":28789,"terminates":28790,"marche":28791,"yarmouth":28792,"##sop":28793,"acton":28794,"mated":28795,"seton":28796,"subtly":28797,"baptised":28798,"beige":28799,"extremes":28800,"jolted":28801,"kristina":28802,"telecast":28803,"##actic":28804,"safeguard":28805,"waldo":28806,"##baldi":28807,"##bular":28808,"endeavors":28809,"sloppy":28810,"subterranean":28811,"##ensburg":28812,"##itung":28813,"delicately":28814,"pigment":28815,"tq":28816,"##scu":28817,"1626":28818,"##ound":28819,"collisions":28820,"coveted":28821,"herds":28822,"##personal":28823,"##meister":28824,"##nberger":28825,"chopra":28826,"##ricting":28827,"abnormalities":28828,"defective":28829,"galician":28830,"lucie":28831,"##dilly":28832,"alligator":28833,"likened":28834,"##genase":28835,"burundi":28836,"clears":28837,"complexion":28838,"derelict":28839,"deafening":28840,"diablo":28841,"fingered":28842,"champaign":28843,"dogg":28844,"enlist":28845,"isotope":28846,"labeling":28847,"mrna":28848,"##erre":28849,"brilliance":28850,"marvelous":28851,"##ayo":28852,"1652":28853,"crawley":28854,"ether":28855,"footed":28856,"dwellers":28857,"deserts":28858,"hamish":28859,"rubs":28860,"warlock":28861,"skimmed":28862,"##lizer":28863,"870":28864,"buick":28865,"embark":28866,"heraldic":28867,"irregularities":28868,"##ajan":28869,"kiara":28870,"##kulam":28871,"##ieg":28872,"antigen":28873,"kowalski":28874,"##lge":28875,"oakley":28876,"visitation":28877,"##mbit":28878,"vt":28879,"##suit":28880,"1570":28881,"murderers":28882,"##miento":28883,"##rites":28884,"chimneys":28885,"##sling":28886,"condemn":28887,"custer":28888,"exchequer":28889,"havre":28890,"##ghi":28891,"fluctuations":28892,"##rations":28893,"dfb":28894,"hendricks":28895,"vaccines":28896,"##tarian":28897,"nietzsche":28898,"biking":28899,"juicy":28900,"##duced":28901,"brooding":28902,"scrolling":28903,"selangor":28904,"##ragan":28905,"352":28906,"annum":28907,"boomed":28908,"seminole":28909,"sugarcane":28910,"##dna":28911,"departmental":28912,"dismissing":28913,"innsbruck":28914,"arteries":28915,"ashok":28916,"batavia":28917,"daze":28918,"kun":28919,"overtook":28920,"##rga":28921,"##tlan":28922,"beheaded":28923,"gaddafi":28924,"holm":28925,"electronically":28926,"faulty":28927,"galilee":28928,"fractures":28929,"kobayashi":28930,"##lized":28931,"gunmen":28932,"magma":28933,"aramaic":28934,"mala":28935,"eastenders":28936,"inference":28937,"messengers":28938,"bf":28939,"##qu":28940,"407":28941,"bathrooms":28942,"##vere":28943,"1658":28944,"flashbacks":28945,"ideally":28946,"misunderstood":28947,"##jali":28948,"##weather":28949,"mendez":28950,"##grounds":28951,"505":28952,"uncanny":28953,"##iii":28954,"1709":28955,"friendships":28956,"##nbc":28957,"sacrament":28958,"accommodated":28959,"reiterated":28960,"logistical":28961,"pebbles":28962,"thumped":28963,"##escence":28964,"administering":28965,"decrees":28966,"drafts":28967,"##flight":28968,"##cased":28969,"##tula":28970,"futuristic":28971,"picket":28972,"intimidation":28973,"winthrop":28974,"##fahan":28975,"interfered":28976,"339":28977,"afar":28978,"francoise":28979,"morally":28980,"uta":28981,"cochin":28982,"croft":28983,"dwarfs":28984,"##bruck":28985,"##dents":28986,"##nami":28987,"biker":28988,"##hner":28989,"##meral":28990,"nano":28991,"##isen":28992,"##ometric":28993,"##pres":28994,"##ан":28995,"brightened":28996,"meek":28997,"parcels":28998,"securely":28999,"gunners":29000,"##jhl":29001,"##zko":29002,"agile":29003,"hysteria":29004,"##lten":29005,"##rcus":29006,"bukit":29007,"champs":29008,"chevy":29009,"cuckoo":29010,"leith":29011,"sadler":29012,"theologians":29013,"welded":29014,"##section":29015,"1663":29016,"jj":29017,"plurality":29018,"xander":29019,"##rooms":29020,"##formed":29021,"shredded":29022,"temps":29023,"intimately":29024,"pau":29025,"tormented":29026,"##lok":29027,"##stellar":29028,"1618":29029,"charred":29030,"ems":29031,"essen":29032,"##mmel":29033,"alarms":29034,"spraying":29035,"ascot":29036,"blooms":29037,"twinkle":29038,"##abia":29039,"##apes":29040,"internment":29041,"obsidian":29042,"##chaft":29043,"snoop":29044,"##dav":29045,"##ooping":29046,"malibu":29047,"##tension":29048,"quiver":29049,"##itia":29050,"hays":29051,"mcintosh":29052,"travers":29053,"walsall":29054,"##ffie":29055,"1623":29056,"beverley":29057,"schwarz":29058,"plunging":29059,"structurally":29060,"m3":29061,"rosenthal":29062,"vikram":29063,"##tsk":29064,"770":29065,"ghz":29066,"##onda":29067,"##tiv":29068,"chalmers":29069,"groningen":29070,"pew":29071,"reckon":29072,"unicef":29073,"##rvis":29074,"55th":29075,"##gni":29076,"1651":29077,"sulawesi":29078,"avila":29079,"cai":29080,"metaphysical":29081,"screwing":29082,"turbulence":29083,"##mberg":29084,"augusto":29085,"samba":29086,"56th":29087,"baffled":29088,"momentary":29089,"toxin":29090,"##urian":29091,"##wani":29092,"aachen":29093,"condoms":29094,"dali":29095,"steppe":29096,"##3d":29097,"##app":29098,"##oed":29099,"##year":29100,"adolescence":29101,"dauphin":29102,"electrically":29103,"inaccessible":29104,"microscopy":29105,"nikita":29106,"##ega":29107,"atv":29108,"##cel":29109,"##enter":29110,"##oles":29111,"##oteric":29112,"##ы":29113,"accountants":29114,"punishments":29115,"wrongly":29116,"bribes":29117,"adventurous":29118,"clinch":29119,"flinders":29120,"southland":29121,"##hem":29122,"##kata":29123,"gough":29124,"##ciency":29125,"lads":29126,"soared":29127,"##ה":29128,"undergoes":29129,"deformation":29130,"outlawed":29131,"rubbish":29132,"##arus":29133,"##mussen":29134,"##nidae":29135,"##rzburg":29136,"arcs":29137,"##ingdon":29138,"##tituted":29139,"1695":29140,"wheelbase":29141,"wheeling":29142,"bombardier":29143,"campground":29144,"zebra":29145,"##lices":29146,"##oj":29147,"##bain":29148,"lullaby":29149,"##ecure":29150,"donetsk":29151,"wylie":29152,"grenada":29153,"##arding":29154,"##ης":29155,"squinting":29156,"eireann":29157,"opposes":29158,"##andra":29159,"maximal":29160,"runes":29161,"##broken":29162,"##cuting":29163,"##iface":29164,"##ror":29165,"##rosis":29166,"additive":29167,"britney":29168,"adultery":29169,"triggering":29170,"##drome":29171,"detrimental":29172,"aarhus":29173,"containment":29174,"jc":29175,"swapped":29176,"vichy":29177,"##ioms":29178,"madly":29179,"##oric":29180,"##rag":29181,"brant":29182,"##ckey":29183,"##trix":29184,"1560":29185,"1612":29186,"broughton":29187,"rustling":29188,"##stems":29189,"##uder":29190,"asbestos":29191,"mentoring":29192,"##nivorous":29193,"finley":29194,"leaps":29195,"##isan":29196,"apical":29197,"pry":29198,"slits":29199,"substitutes":29200,"##dict":29201,"intuitive":29202,"fantasia":29203,"insistent":29204,"unreasonable":29205,"##igen":29206,"##vna":29207,"domed":29208,"hannover":29209,"margot":29210,"ponder":29211,"##zziness":29212,"impromptu":29213,"jian":29214,"lc":29215,"rampage":29216,"stemming":29217,"##eft":29218,"andrey":29219,"gerais":29220,"whichever":29221,"amnesia":29222,"appropriated":29223,"anzac":29224,"clicks":29225,"modifying":29226,"ultimatum":29227,"cambrian":29228,"maids":29229,"verve":29230,"yellowstone":29231,"##mbs":29232,"conservatoire":29233,"##scribe":29234,"adherence":29235,"dinners":29236,"spectra":29237,"imperfect":29238,"mysteriously":29239,"sidekick":29240,"tatar":29241,"tuba":29242,"##aks":29243,"##ifolia":29244,"distrust":29245,"##athan":29246,"##zle":29247,"c2":29248,"ronin":29249,"zac":29250,"##pse":29251,"celaena":29252,"instrumentalist":29253,"scents":29254,"skopje":29255,"##mbling":29256,"comical":29257,"compensated":29258,"vidal":29259,"condor":29260,"intersect":29261,"jingle":29262,"wavelengths":29263,"##urrent":29264,"mcqueen":29265,"##izzly":29266,"carp":29267,"weasel":29268,"422":29269,"kanye":29270,"militias":29271,"postdoctoral":29272,"eugen":29273,"gunslinger":29274,"##ɛ":29275,"faux":29276,"hospice":29277,"##for":29278,"appalled":29279,"derivation":29280,"dwarves":29281,"##elis":29282,"dilapidated":29283,"##folk":29284,"astoria":29285,"philology":29286,"##lwyn":29287,"##otho":29288,"##saka":29289,"inducing":29290,"philanthropy":29291,"##bf":29292,"##itative":29293,"geek":29294,"markedly":29295,"sql":29296,"##yce":29297,"bessie":29298,"indices":29299,"rn":29300,"##flict":29301,"495":29302,"frowns":29303,"resolving":29304,"weightlifting":29305,"tugs":29306,"cleric":29307,"contentious":29308,"1653":29309,"mania":29310,"rms":29311,"##miya":29312,"##reate":29313,"##ruck":29314,"##tucket":29315,"bien":29316,"eels":29317,"marek":29318,"##ayton":29319,"##cence":29320,"discreet":29321,"unofficially":29322,"##ife":29323,"leaks":29324,"##bber":29325,"1705":29326,"332":29327,"dung":29328,"compressor":29329,"hillsborough":29330,"pandit":29331,"shillings":29332,"distal":29333,"##skin":29334,"381":29335,"##tat":29336,"##you":29337,"nosed":29338,"##nir":29339,"mangrove":29340,"undeveloped":29341,"##idia":29342,"textures":29343,"##inho":29344,"##500":29345,"##rise":29346,"ae":29347,"irritating":29348,"nay":29349,"amazingly":29350,"bancroft":29351,"apologetic":29352,"compassionate":29353,"kata":29354,"symphonies":29355,"##lovic":29356,"airspace":29357,"##lch":29358,"930":29359,"gifford":29360,"precautions":29361,"fulfillment":29362,"sevilla":29363,"vulgar":29364,"martinique":29365,"##urities":29366,"looting":29367,"piccolo":29368,"tidy":29369,"##dermott":29370,"quadrant":29371,"armchair":29372,"incomes":29373,"mathematicians":29374,"stampede":29375,"nilsson":29376,"##inking":29377,"##scan":29378,"foo":29379,"quarterfinal":29380,"##ostal":29381,"shang":29382,"shouldered":29383,"squirrels":29384,"##owe":29385,"344":29386,"vinegar":29387,"##bner":29388,"##rchy":29389,"##systems":29390,"delaying":29391,"##trics":29392,"ars":29393,"dwyer":29394,"rhapsody":29395,"sponsoring":29396,"##gration":29397,"bipolar":29398,"cinder":29399,"starters":29400,"##olio":29401,"##urst":29402,"421":29403,"signage":29404,"##nty":29405,"aground":29406,"figurative":29407,"mons":29408,"acquaintances":29409,"duets":29410,"erroneously":29411,"soyuz":29412,"elliptic":29413,"recreated":29414,"##cultural":29415,"##quette":29416,"##ssed":29417,"##tma":29418,"##zcz":29419,"moderator":29420,"scares":29421,"##itaire":29422,"##stones":29423,"##udence":29424,"juniper":29425,"sighting":29426,"##just":29427,"##nsen":29428,"britten":29429,"calabria":29430,"ry":29431,"bop":29432,"cramer":29433,"forsyth":29434,"stillness":29435,"##л":29436,"airmen":29437,"gathers":29438,"unfit":29439,"##umber":29440,"##upt":29441,"taunting":29442,"##rip":29443,"seeker":29444,"streamlined":29445,"##bution":29446,"holster":29447,"schumann":29448,"tread":29449,"vox":29450,"##gano":29451,"##onzo":29452,"strive":29453,"dil":29454,"reforming":29455,"covent":29456,"newbury":29457,"predicting":29458,"##orro":29459,"decorate":29460,"tre":29461,"##puted":29462,"andover":29463,"ie":29464,"asahi":29465,"dept":29466,"dunkirk":29467,"gills":29468,"##tori":29469,"buren":29470,"huskies":29471,"##stis":29472,"##stov":29473,"abstracts":29474,"bets":29475,"loosen":29476,"##opa":29477,"1682":29478,"yearning":29479,"##glio":29480,"##sir":29481,"berman":29482,"effortlessly":29483,"enamel":29484,"napoli":29485,"persist":29486,"##peration":29487,"##uez":29488,"attache":29489,"elisa":29490,"b1":29491,"invitations":29492,"##kic":29493,"accelerating":29494,"reindeer":29495,"boardwalk":29496,"clutches":29497,"nelly":29498,"polka":29499,"starbucks":29500,"##kei":29501,"adamant":29502,"huey":29503,"lough":29504,"unbroken":29505,"adventurer":29506,"embroidery":29507,"inspecting":29508,"stanza":29509,"##ducted":29510,"naia":29511,"taluka":29512,"##pone":29513,"##roids":29514,"chases":29515,"deprivation":29516,"florian":29517,"##jing":29518,"##ppet":29519,"earthly":29520,"##lib":29521,"##ssee":29522,"colossal":29523,"foreigner":29524,"vet":29525,"freaks":29526,"patrice":29527,"rosewood":29528,"triassic":29529,"upstate":29530,"##pkins":29531,"dominates":29532,"ata":29533,"chants":29534,"ks":29535,"vo":29536,"##400":29537,"##bley":29538,"##raya":29539,"##rmed":29540,"555":29541,"agra":29542,"infiltrate":29543,"##ailing":29544,"##ilation":29545,"##tzer":29546,"##uppe":29547,"##werk":29548,"binoculars":29549,"enthusiast":29550,"fujian":29551,"squeak":29552,"##avs":29553,"abolitionist":29554,"almeida":29555,"boredom":29556,"hampstead":29557,"marsden":29558,"rations":29559,"##ands":29560,"inflated":29561,"334":29562,"bonuses":29563,"rosalie":29564,"patna":29565,"##rco":29566,"329":29567,"detachments":29568,"penitentiary":29569,"54th":29570,"flourishing":29571,"woolf":29572,"##dion":29573,"##etched":29574,"papyrus":29575,"##lster":29576,"##nsor":29577,"##toy":29578,"bobbed":29579,"dismounted":29580,"endelle":29581,"inhuman":29582,"motorola":29583,"tbs":29584,"wince":29585,"wreath":29586,"##ticus":29587,"hideout":29588,"inspections":29589,"sanjay":29590,"disgrace":29591,"infused":29592,"pudding":29593,"stalks":29594,"##urbed":29595,"arsenic":29596,"leases":29597,"##hyl":29598,"##rrard":29599,"collarbone":29600,"##waite":29601,"##wil":29602,"dowry":29603,"##bant":29604,"##edance":29605,"genealogical":29606,"nitrate":29607,"salamanca":29608,"scandals":29609,"thyroid":29610,"necessitated":29611,"##!":29612,"##\"":29613,"###":29614,"##$":29615,"##%":29616,"##&":29617,"##'":29618,"##(":29619,"##)":29620,"##*":29621,"##+":29622,"##,":29623,"##-":29624,"##.":29625,"##/":29626,"##:":29627,"##;":29628,"##<":29629,"##=":29630,"##>":29631,"##?":29632,"##@":29633,"##[":29634,"##\\":29635,"##]":29636,"##^":29637,"##_":29638,"##`":29639,"##{":29640,"##|":29641,"##}":29642,"##~":29643,"##¡":29644,"##¢":29645,"##£":29646,"##¤":29647,"##¥":29648,"##¦":29649,"##§":29650,"##¨":29651,"##©":29652,"##ª":29653,"##«":29654,"##¬":29655,"##®":29656,"##±":29657,"##´":29658,"##µ":29659,"##¶":29660,"##·":29661,"##º":29662,"##»":29663,"##¼":29664,"##¾":29665,"##¿":29666,"##æ":29667,"##ð":29668,"##÷":29669,"##þ":29670,"##đ":29671,"##ħ":29672,"##ŋ":29673,"##œ":29674,"##ƒ":29675,"##ɐ":29676,"##ɑ":29677,"##ɒ":29678,"##ɔ":29679,"##ɕ":29680,"##ə":29681,"##ɡ":29682,"##ɣ":29683,"##ɨ":29684,"##ɪ":29685,"##ɫ":29686,"##ɬ":29687,"##ɯ":29688,"##ɲ":29689,"##ɴ":29690,"##ɹ":29691,"##ɾ":29692,"##ʀ":29693,"##ʁ":29694,"##ʂ":29695,"##ʃ":29696,"##ʉ":29697,"##ʊ":29698,"##ʋ":29699,"##ʌ":29700,"##ʎ":29701,"##ʐ":29702,"##ʑ":29703,"##ʒ":29704,"##ʔ":29705,"##ʰ":29706,"##ʲ":29707,"##ʳ":29708,"##ʷ":29709,"##ʸ":29710,"##ʻ":29711,"##ʼ":29712,"##ʾ":29713,"##ʿ":29714,"##ˈ":29715,"##ˡ":29716,"##ˢ":29717,"##ˣ":29718,"##ˤ":29719,"##β":29720,"##γ":29721,"##δ":29722,"##ε":29723,"##ζ":29724,"##θ":29725,"##κ":29726,"##λ":29727,"##μ":29728,"##ξ":29729,"##ο":29730,"##π":29731,"##ρ":29732,"##σ":29733,"##τ":29734,"##υ":29735,"##φ":29736,"##χ":29737,"##ψ":29738,"##ω":29739,"##б":29740,"##г":29741,"##д":29742,"##ж":29743,"##з":29744,"##м":29745,"##п":29746,"##с":29747,"##у":29748,"##ф":29749,"##х":29750,"##ц":29751,"##ч":29752,"##ш":29753,"##щ":29754,"##ъ":29755,"##э":29756,"##ю":29757,"##ђ":29758,"##є":29759,"##і":29760,"##ј":29761,"##љ":29762,"##њ":29763,"##ћ":29764,"##ӏ":29765,"##ա":29766,"##բ":29767,"##գ":29768,"##դ":29769,"##ե":29770,"##թ":29771,"##ի":29772,"##լ":29773,"##կ":29774,"##հ":29775,"##մ":29776,"##յ":29777,"##ն":29778,"##ո":29779,"##պ":29780,"##ս":29781,"##վ":29782,"##տ":29783,"##ր":29784,"##ւ":29785,"##ք":29786,"##־":29787,"##א":29788,"##ב":29789,"##ג":29790,"##ד":29791,"##ו":29792,"##ז":29793,"##ח":29794,"##ט":29795,"##י":29796,"##ך":29797,"##כ":29798,"##ל":29799,"##ם":29800,"##מ":29801,"##ן":29802,"##נ":29803,"##ס":29804,"##ע":29805,"##ף":29806,"##פ":29807,"##ץ":29808,"##צ":29809,"##ק":29810,"##ר":29811,"##ש":29812,"##ת":29813,"##،":29814,"##ء":29815,"##ب":29816,"##ت":29817,"##ث":29818,"##ج":29819,"##ح":29820,"##خ":29821,"##ذ":29822,"##ز":29823,"##س":29824,"##ش":29825,"##ص":29826,"##ض":29827,"##ط":29828,"##ظ":29829,"##ع":29830,"##غ":29831,"##ـ":29832,"##ف":29833,"##ق":29834,"##ك":29835,"##و":29836,"##ى":29837,"##ٹ":29838,"##پ":29839,"##چ":29840,"##ک":29841,"##گ":29842,"##ں":29843,"##ھ":29844,"##ہ":29845,"##ے":29846,"##अ":29847,"##आ":29848,"##उ":29849,"##ए":29850,"##क":29851,"##ख":29852,"##ग":29853,"##च":29854,"##ज":29855,"##ट":29856,"##ड":29857,"##ण":29858,"##त":29859,"##थ":29860,"##द":29861,"##ध":29862,"##न":29863,"##प":29864,"##ब":29865,"##भ":29866,"##म":29867,"##य":29868,"##र":29869,"##ल":29870,"##व":29871,"##श":29872,"##ष":29873,"##स":29874,"##ह":29875,"##ा":29876,"##ि":29877,"##ी":29878,"##ो":29879,"##।":29880,"##॥":29881,"##ং":29882,"##অ":29883,"##আ":29884,"##ই":29885,"##উ":29886,"##এ":29887,"##ও":29888,"##ক":29889,"##খ":29890,"##গ":29891,"##চ":29892,"##ছ":29893,"##জ":29894,"##ট":29895,"##ড":29896,"##ণ":29897,"##ত":29898,"##থ":29899,"##দ":29900,"##ধ":29901,"##ন":29902,"##প":29903,"##ব":29904,"##ভ":29905,"##ম":29906,"##য":29907,"##র":29908,"##ল":29909,"##শ":29910,"##ষ":29911,"##স":29912,"##হ":29913,"##া":29914,"##ি":29915,"##ী":29916,"##ে":29917,"##க":29918,"##ச":29919,"##ட":29920,"##த":29921,"##ந":29922,"##ன":29923,"##ப":29924,"##ம":29925,"##ய":29926,"##ர":29927,"##ல":29928,"##ள":29929,"##வ":29930,"##ா":29931,"##ி":29932,"##ு":29933,"##ே":29934,"##ை":29935,"##ನ":29936,"##ರ":29937,"##ಾ":29938,"##ක":29939,"##ය":29940,"##ර":29941,"##ල":29942,"##ව":29943,"##ා":29944,"##ก":29945,"##ง":29946,"##ต":29947,"##ท":29948,"##น":29949,"##พ":29950,"##ม":29951,"##ย":29952,"##ร":29953,"##ล":29954,"##ว":29955,"##ส":29956,"##อ":29957,"##า":29958,"##เ":29959,"##་":29960,"##།":29961,"##ག":29962,"##ང":29963,"##ད":29964,"##ན":29965,"##པ":29966,"##བ":29967,"##མ":29968,"##འ":29969,"##ར":29970,"##ལ":29971,"##ས":29972,"##မ":29973,"##ა":29974,"##ბ":29975,"##გ":29976,"##დ":29977,"##ე":29978,"##ვ":29979,"##თ":29980,"##ი":29981,"##კ":29982,"##ლ":29983,"##მ":29984,"##ნ":29985,"##ო":29986,"##რ":29987,"##ს":29988,"##ტ":29989,"##უ":29990,"##ᄀ":29991,"##ᄂ":29992,"##ᄃ":29993,"##ᄅ":29994,"##ᄆ":29995,"##ᄇ":29996,"##ᄉ":29997,"##ᄊ":29998,"##ᄋ":29999,"##ᄌ":30000,"##ᄎ":30001,"##ᄏ":30002,"##ᄐ":30003,"##ᄑ":30004,"##ᄒ":30005,"##ᅡ":30006,"##ᅢ":30007,"##ᅥ":30008,"##ᅦ":30009,"##ᅧ":30010,"##ᅩ":30011,"##ᅪ":30012,"##ᅭ":30013,"##ᅮ":30014,"##ᅯ":30015,"##ᅲ":30016,"##ᅳ":30017,"##ᅴ":30018,"##ᅵ":30019,"##ᆨ":30020,"##ᆫ":30021,"##ᆯ":30022,"##ᆷ":30023,"##ᆸ":30024,"##ᆼ":30025,"##ᴬ":30026,"##ᴮ":30027,"##ᴰ":30028,"##ᴵ":30029,"##ᴺ":30030,"##ᵀ":30031,"##ᵃ":30032,"##ᵇ":30033,"##ᵈ":30034,"##ᵉ":30035,"##ᵍ":30036,"##ᵏ":30037,"##ᵐ":30038,"##ᵒ":30039,"##ᵖ":30040,"##ᵗ":30041,"##ᵘ":30042,"##ᵣ":30043,"##ᵤ":30044,"##ᵥ":30045,"##ᶜ":30046,"##ᶠ":30047,"##‐":30048,"##‑":30049,"##‒":30050,"##–":30051,"##—":30052,"##―":30053,"##‖":30054,"##‘":30055,"##’":30056,"##‚":30057,"##“":30058,"##”":30059,"##„":30060,"##†":30061,"##‡":30062,"##•":30063,"##…":30064,"##‰":30065,"##′":30066,"##″":30067,"##›":30068,"##‿":30069,"##⁄":30070,"##⁰":30071,"##ⁱ":30072,"##⁴":30073,"##⁵":30074,"##⁶":30075,"##⁷":30076,"##⁸":30077,"##⁹":30078,"##⁻":30079,"##ⁿ":30080,"##₅":30081,"##₆":30082,"##₇":30083,"##₈":30084,"##₉":30085,"##₊":30086,"##₍":30087,"##₎":30088,"##ₐ":30089,"##ₑ":30090,"##ₒ":30091,"##ₓ":30092,"##ₕ":30093,"##ₖ":30094,"##ₗ":30095,"##ₘ":30096,"##ₚ":30097,"##ₛ":30098,"##ₜ":30099,"##₤":30100,"##₩":30101,"##€":30102,"##₱":30103,"##₹":30104,"##ℓ":30105,"##№":30106,"##ℝ":30107,"##™":30108,"##⅓":30109,"##⅔":30110,"##←":30111,"##↑":30112,"##→":30113,"##↓":30114,"##↔":30115,"##↦":30116,"##⇄":30117,"##⇌":30118,"##⇒":30119,"##∂":30120,"##∅":30121,"##∆":30122,"##∇":30123,"##∈":30124,"##∗":30125,"##∘":30126,"##√":30127,"##∞":30128,"##∧":30129,"##∨":30130,"##∩":30131,"##∪":30132,"##≈":30133,"##≡":30134,"##≤":30135,"##≥":30136,"##⊂":30137,"##⊆":30138,"##⊕":30139,"##⊗":30140,"##⋅":30141,"##─":30142,"##│":30143,"##■":30144,"##▪":30145,"##●":30146,"##★":30147,"##☆":30148,"##☉":30149,"##♠":30150,"##♣":30151,"##♥":30152,"##♦":30153,"##♯":30154,"##⟨":30155,"##⟩":30156,"##ⱼ":30157,"##⺩":30158,"##⺼":30159,"##⽥":30160,"##、":30161,"##。":30162,"##〈":30163,"##〉":30164,"##《":30165,"##》":30166,"##「":30167,"##」":30168,"##『":30169,"##』":30170,"##〜":30171,"##あ":30172,"##い":30173,"##う":30174,"##え":30175,"##お":30176,"##か":30177,"##き":30178,"##く":30179,"##け":30180,"##こ":30181,"##さ":30182,"##し":30183,"##す":30184,"##せ":30185,"##そ":30186,"##た":30187,"##ち":30188,"##っ":30189,"##つ":30190,"##て":30191,"##と":30192,"##な":30193,"##に":30194,"##ぬ":30195,"##ね":30196,"##の":30197,"##は":30198,"##ひ":30199,"##ふ":30200,"##へ":30201,"##ほ":30202,"##ま":30203,"##み":30204,"##む":30205,"##め":30206,"##も":30207,"##や":30208,"##ゆ":30209,"##よ":30210,"##ら":30211,"##り":30212,"##る":30213,"##れ":30214,"##ろ":30215,"##を":30216,"##ん":30217,"##ァ":30218,"##ア":30219,"##ィ":30220,"##イ":30221,"##ウ":30222,"##ェ":30223,"##エ":30224,"##オ":30225,"##カ":30226,"##キ":30227,"##ク":30228,"##ケ":30229,"##コ":30230,"##サ":30231,"##シ":30232,"##ス":30233,"##セ":30234,"##タ":30235,"##チ":30236,"##ッ":30237,"##ツ":30238,"##テ":30239,"##ト":30240,"##ナ":30241,"##ニ":30242,"##ノ":30243,"##ハ":30244,"##ヒ":30245,"##フ":30246,"##ヘ":30247,"##ホ":30248,"##マ":30249,"##ミ":30250,"##ム":30251,"##メ":30252,"##モ":30253,"##ャ":30254,"##ュ":30255,"##ョ":30256,"##ラ":30257,"##リ":30258,"##ル":30259,"##レ":30260,"##ロ":30261,"##ワ":30262,"##ン":30263,"##・":30264,"##ー":30265,"##一":30266,"##三":30267,"##上":30268,"##下":30269,"##不":30270,"##世":30271,"##中":30272,"##主":30273,"##久":30274,"##之":30275,"##也":30276,"##事":30277,"##二":30278,"##五":30279,"##井":30280,"##京":30281,"##人":30282,"##亻":30283,"##仁":30284,"##介":30285,"##代":30286,"##仮":30287,"##伊":30288,"##会":30289,"##佐":30290,"##侍":30291,"##保":30292,"##信":30293,"##健":30294,"##元":30295,"##光":30296,"##八":30297,"##公":30298,"##内":30299,"##出":30300,"##分":30301,"##前":30302,"##劉":30303,"##力":30304,"##加":30305,"##勝":30306,"##北":30307,"##区":30308,"##十":30309,"##千":30310,"##南":30311,"##博":30312,"##原":30313,"##口":30314,"##古":30315,"##史":30316,"##司":30317,"##合":30318,"##吉":30319,"##同":30320,"##名":30321,"##和":30322,"##囗":30323,"##四":30324,"##国":30325,"##國":30326,"##土":30327,"##地":30328,"##坂":30329,"##城":30330,"##堂":30331,"##場":30332,"##士":30333,"##夏":30334,"##外":30335,"##大":30336,"##天":30337,"##太":30338,"##夫":30339,"##奈":30340,"##女":30341,"##子":30342,"##学":30343,"##宀":30344,"##宇":30345,"##安":30346,"##宗":30347,"##定":30348,"##宣":30349,"##宮":30350,"##家":30351,"##宿":30352,"##寺":30353,"##將":30354,"##小":30355,"##尚":30356,"##山":30357,"##岡":30358,"##島":30359,"##崎":30360,"##川":30361,"##州":30362,"##巿":30363,"##帝":30364,"##平":30365,"##年":30366,"##幸":30367,"##广":30368,"##弘":30369,"##張":30370,"##彳":30371,"##後":30372,"##御":30373,"##德":30374,"##心":30375,"##忄":30376,"##志":30377,"##忠":30378,"##愛":30379,"##成":30380,"##我":30381,"##戦":30382,"##戸":30383,"##手":30384,"##扌":30385,"##政":30386,"##文":30387,"##新":30388,"##方":30389,"##日":30390,"##明":30391,"##星":30392,"##春":30393,"##昭":30394,"##智":30395,"##曲":30396,"##書":30397,"##月":30398,"##有":30399,"##朝":30400,"##木":30401,"##本":30402,"##李":30403,"##村":30404,"##東":30405,"##松":30406,"##林":30407,"##森":30408,"##楊":30409,"##樹":30410,"##橋":30411,"##歌":30412,"##止":30413,"##正":30414,"##武":30415,"##比":30416,"##氏":30417,"##民":30418,"##水":30419,"##氵":30420,"##氷":30421,"##永":30422,"##江":30423,"##沢":30424,"##河":30425,"##治":30426,"##法":30427,"##海":30428,"##清":30429,"##漢":30430,"##瀬":30431,"##火":30432,"##版":30433,"##犬":30434,"##王":30435,"##生":30436,"##田":30437,"##男":30438,"##疒":30439,"##発":30440,"##白":30441,"##的":30442,"##皇":30443,"##目":30444,"##相":30445,"##省":30446,"##真":30447,"##石":30448,"##示":30449,"##社":30450,"##神":30451,"##福":30452,"##禾":30453,"##秀":30454,"##秋":30455,"##空":30456,"##立":30457,"##章":30458,"##竹":30459,"##糹":30460,"##美":30461,"##義":30462,"##耳":30463,"##良":30464,"##艹":30465,"##花":30466,"##英":30467,"##華":30468,"##葉":30469,"##藤":30470,"##行":30471,"##街":30472,"##西":30473,"##見":30474,"##訁":30475,"##語":30476,"##谷":30477,"##貝":30478,"##貴":30479,"##車":30480,"##軍":30481,"##辶":30482,"##道":30483,"##郎":30484,"##郡":30485,"##部":30486,"##都":30487,"##里":30488,"##野":30489,"##金":30490,"##鈴":30491,"##镇":30492,"##長":30493,"##門":30494,"##間":30495,"##阝":30496,"##阿":30497,"##陳":30498,"##陽":30499,"##雄":30500,"##青":30501,"##面":30502,"##風":30503,"##食":30504,"##香":30505,"##馬":30506,"##高":30507,"##龍":30508,"##龸":30509,"##fi":30510,"##fl":30511,"##！":30512,"##（":30513,"##）":30514,"##，":30515,"##－":30516,"##．":30517,"##／":30518,"##：":30519,"##？":30520,"##～":30521}}} \ No newline at end of file diff --git a/TextClassification/TextClassification/bert-base-uncased/tokenizer_config.json b/TextClassification/TextClassification/bert-base-uncased/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e5c73d8a50df1f56fb5b0b8002d7cf4010afdccb --- /dev/null +++ b/TextClassification/TextClassification/bert-base-uncased/tokenizer_config.json @@ -0,0 +1 @@ +{"do_lower_case": true, "model_max_length": 512} \ No newline at end of file diff --git a/TextClassification/TextClassification/bert-base-uncased/vocab.txt b/TextClassification/TextClassification/bert-base-uncased/vocab.txt new file mode 100644 index 0000000000000000000000000000000000000000..fb140275c155a9c7c5a3b3e0e77a9e839594a938 --- /dev/null +++ b/TextClassification/TextClassification/bert-base-uncased/vocab.txt @@ -0,0 +1,30522 @@ +[PAD] +[unused0] +[unused1] +[unused2] +[unused3] +[unused4] +[unused5] +[unused6] +[unused7] +[unused8] +[unused9] +[unused10] +[unused11] +[unused12] +[unused13] +[unused14] +[unused15] +[unused16] +[unused17] +[unused18] +[unused19] +[unused20] +[unused21] +[unused22] +[unused23] +[unused24] +[unused25] +[unused26] +[unused27] +[unused28] +[unused29] +[unused30] +[unused31] +[unused32] +[unused33] +[unused34] +[unused35] +[unused36] +[unused37] +[unused38] +[unused39] +[unused40] +[unused41] +[unused42] +[unused43] +[unused44] +[unused45] +[unused46] +[unused47] +[unused48] +[unused49] +[unused50] +[unused51] +[unused52] +[unused53] +[unused54] +[unused55] +[unused56] +[unused57] +[unused58] +[unused59] +[unused60] +[unused61] +[unused62] +[unused63] +[unused64] +[unused65] +[unused66] +[unused67] +[unused68] +[unused69] +[unused70] +[unused71] +[unused72] +[unused73] +[unused74] +[unused75] +[unused76] +[unused77] +[unused78] +[unused79] +[unused80] +[unused81] +[unused82] +[unused83] +[unused84] +[unused85] +[unused86] +[unused87] +[unused88] +[unused89] +[unused90] +[unused91] +[unused92] +[unused93] +[unused94] +[unused95] +[unused96] +[unused97] +[unused98] +[UNK] +[CLS] +[SEP] +[MASK] +[unused99] +[unused100] +[unused101] +[unused102] +[unused103] +[unused104] +[unused105] +[unused106] +[unused107] +[unused108] +[unused109] +[unused110] +[unused111] +[unused112] +[unused113] +[unused114] +[unused115] +[unused116] +[unused117] +[unused118] +[unused119] +[unused120] +[unused121] +[unused122] +[unused123] +[unused124] +[unused125] +[unused126] +[unused127] +[unused128] +[unused129] +[unused130] +[unused131] +[unused132] +[unused133] +[unused134] +[unused135] +[unused136] +[unused137] +[unused138] +[unused139] +[unused140] +[unused141] +[unused142] +[unused143] +[unused144] +[unused145] +[unused146] +[unused147] +[unused148] +[unused149] +[unused150] +[unused151] +[unused152] +[unused153] +[unused154] +[unused155] +[unused156] +[unused157] +[unused158] +[unused159] +[unused160] +[unused161] +[unused162] +[unused163] +[unused164] +[unused165] +[unused166] +[unused167] +[unused168] +[unused169] +[unused170] +[unused171] +[unused172] +[unused173] +[unused174] +[unused175] +[unused176] +[unused177] +[unused178] +[unused179] +[unused180] +[unused181] +[unused182] +[unused183] +[unused184] +[unused185] +[unused186] +[unused187] +[unused188] +[unused189] +[unused190] +[unused191] +[unused192] +[unused193] +[unused194] +[unused195] +[unused196] +[unused197] +[unused198] +[unused199] +[unused200] +[unused201] +[unused202] +[unused203] +[unused204] +[unused205] +[unused206] +[unused207] +[unused208] +[unused209] +[unused210] +[unused211] +[unused212] +[unused213] +[unused214] +[unused215] +[unused216] +[unused217] +[unused218] +[unused219] +[unused220] +[unused221] +[unused222] +[unused223] +[unused224] +[unused225] +[unused226] +[unused227] +[unused228] +[unused229] +[unused230] +[unused231] +[unused232] +[unused233] +[unused234] +[unused235] +[unused236] +[unused237] +[unused238] +[unused239] +[unused240] +[unused241] +[unused242] +[unused243] +[unused244] +[unused245] +[unused246] +[unused247] +[unused248] +[unused249] +[unused250] +[unused251] +[unused252] +[unused253] +[unused254] +[unused255] +[unused256] +[unused257] +[unused258] +[unused259] +[unused260] +[unused261] +[unused262] +[unused263] +[unused264] +[unused265] +[unused266] +[unused267] +[unused268] +[unused269] +[unused270] +[unused271] +[unused272] +[unused273] +[unused274] +[unused275] +[unused276] +[unused277] +[unused278] +[unused279] +[unused280] +[unused281] +[unused282] +[unused283] +[unused284] +[unused285] +[unused286] +[unused287] +[unused288] +[unused289] +[unused290] +[unused291] +[unused292] +[unused293] +[unused294] +[unused295] +[unused296] +[unused297] +[unused298] +[unused299] +[unused300] +[unused301] +[unused302] +[unused303] +[unused304] +[unused305] +[unused306] +[unused307] +[unused308] +[unused309] +[unused310] +[unused311] +[unused312] +[unused313] +[unused314] +[unused315] +[unused316] +[unused317] +[unused318] +[unused319] +[unused320] +[unused321] +[unused322] +[unused323] +[unused324] +[unused325] +[unused326] +[unused327] +[unused328] +[unused329] +[unused330] +[unused331] +[unused332] +[unused333] +[unused334] +[unused335] +[unused336] +[unused337] +[unused338] +[unused339] +[unused340] +[unused341] +[unused342] +[unused343] +[unused344] +[unused345] +[unused346] +[unused347] +[unused348] +[unused349] +[unused350] +[unused351] +[unused352] +[unused353] +[unused354] +[unused355] +[unused356] +[unused357] +[unused358] +[unused359] +[unused360] +[unused361] +[unused362] +[unused363] +[unused364] +[unused365] +[unused366] +[unused367] +[unused368] +[unused369] +[unused370] +[unused371] +[unused372] +[unused373] +[unused374] +[unused375] +[unused376] +[unused377] +[unused378] +[unused379] +[unused380] +[unused381] +[unused382] +[unused383] +[unused384] +[unused385] +[unused386] +[unused387] +[unused388] +[unused389] +[unused390] +[unused391] +[unused392] +[unused393] +[unused394] +[unused395] +[unused396] +[unused397] +[unused398] +[unused399] +[unused400] +[unused401] +[unused402] +[unused403] +[unused404] +[unused405] +[unused406] +[unused407] +[unused408] +[unused409] +[unused410] +[unused411] +[unused412] +[unused413] +[unused414] +[unused415] +[unused416] +[unused417] +[unused418] +[unused419] +[unused420] +[unused421] +[unused422] +[unused423] +[unused424] +[unused425] +[unused426] +[unused427] +[unused428] +[unused429] +[unused430] +[unused431] +[unused432] +[unused433] +[unused434] +[unused435] +[unused436] +[unused437] +[unused438] +[unused439] +[unused440] +[unused441] +[unused442] +[unused443] +[unused444] +[unused445] +[unused446] +[unused447] +[unused448] +[unused449] +[unused450] +[unused451] +[unused452] +[unused453] +[unused454] +[unused455] +[unused456] +[unused457] +[unused458] +[unused459] +[unused460] +[unused461] +[unused462] +[unused463] +[unused464] +[unused465] +[unused466] +[unused467] +[unused468] +[unused469] +[unused470] +[unused471] +[unused472] +[unused473] +[unused474] +[unused475] +[unused476] +[unused477] +[unused478] +[unused479] +[unused480] +[unused481] +[unused482] +[unused483] +[unused484] +[unused485] +[unused486] +[unused487] +[unused488] +[unused489] +[unused490] +[unused491] +[unused492] +[unused493] +[unused494] +[unused495] +[unused496] +[unused497] +[unused498] +[unused499] +[unused500] +[unused501] +[unused502] +[unused503] +[unused504] +[unused505] +[unused506] +[unused507] +[unused508] +[unused509] +[unused510] +[unused511] +[unused512] +[unused513] +[unused514] +[unused515] +[unused516] +[unused517] +[unused518] +[unused519] +[unused520] +[unused521] +[unused522] +[unused523] +[unused524] +[unused525] +[unused526] +[unused527] +[unused528] +[unused529] +[unused530] +[unused531] +[unused532] +[unused533] +[unused534] +[unused535] +[unused536] +[unused537] +[unused538] +[unused539] +[unused540] +[unused541] +[unused542] +[unused543] +[unused544] +[unused545] +[unused546] +[unused547] +[unused548] +[unused549] +[unused550] +[unused551] +[unused552] +[unused553] +[unused554] +[unused555] +[unused556] +[unused557] +[unused558] +[unused559] +[unused560] +[unused561] +[unused562] +[unused563] +[unused564] +[unused565] +[unused566] +[unused567] +[unused568] +[unused569] +[unused570] +[unused571] +[unused572] +[unused573] +[unused574] +[unused575] +[unused576] +[unused577] +[unused578] +[unused579] +[unused580] +[unused581] +[unused582] +[unused583] +[unused584] +[unused585] +[unused586] +[unused587] +[unused588] +[unused589] +[unused590] +[unused591] +[unused592] +[unused593] +[unused594] +[unused595] +[unused596] +[unused597] +[unused598] +[unused599] +[unused600] +[unused601] +[unused602] +[unused603] +[unused604] +[unused605] +[unused606] +[unused607] +[unused608] +[unused609] +[unused610] +[unused611] +[unused612] +[unused613] +[unused614] +[unused615] +[unused616] +[unused617] +[unused618] +[unused619] +[unused620] +[unused621] +[unused622] +[unused623] +[unused624] +[unused625] +[unused626] +[unused627] +[unused628] +[unused629] +[unused630] +[unused631] +[unused632] +[unused633] +[unused634] +[unused635] +[unused636] +[unused637] +[unused638] +[unused639] +[unused640] +[unused641] +[unused642] +[unused643] +[unused644] +[unused645] +[unused646] +[unused647] +[unused648] +[unused649] +[unused650] +[unused651] +[unused652] +[unused653] +[unused654] +[unused655] +[unused656] +[unused657] +[unused658] +[unused659] +[unused660] +[unused661] +[unused662] +[unused663] +[unused664] +[unused665] +[unused666] +[unused667] +[unused668] +[unused669] +[unused670] +[unused671] +[unused672] +[unused673] +[unused674] +[unused675] +[unused676] +[unused677] +[unused678] +[unused679] +[unused680] +[unused681] +[unused682] +[unused683] +[unused684] +[unused685] +[unused686] +[unused687] +[unused688] +[unused689] +[unused690] +[unused691] +[unused692] +[unused693] +[unused694] +[unused695] +[unused696] +[unused697] +[unused698] +[unused699] +[unused700] +[unused701] +[unused702] +[unused703] +[unused704] +[unused705] +[unused706] +[unused707] +[unused708] +[unused709] +[unused710] +[unused711] +[unused712] +[unused713] +[unused714] +[unused715] +[unused716] +[unused717] +[unused718] +[unused719] +[unused720] +[unused721] +[unused722] +[unused723] +[unused724] +[unused725] +[unused726] +[unused727] +[unused728] +[unused729] +[unused730] +[unused731] +[unused732] +[unused733] +[unused734] +[unused735] +[unused736] +[unused737] +[unused738] +[unused739] +[unused740] +[unused741] +[unused742] +[unused743] +[unused744] +[unused745] +[unused746] +[unused747] +[unused748] +[unused749] +[unused750] +[unused751] +[unused752] +[unused753] +[unused754] +[unused755] +[unused756] +[unused757] +[unused758] +[unused759] +[unused760] +[unused761] +[unused762] +[unused763] +[unused764] +[unused765] +[unused766] +[unused767] +[unused768] +[unused769] +[unused770] +[unused771] +[unused772] +[unused773] +[unused774] +[unused775] +[unused776] +[unused777] +[unused778] +[unused779] +[unused780] +[unused781] +[unused782] +[unused783] +[unused784] +[unused785] +[unused786] +[unused787] +[unused788] +[unused789] +[unused790] +[unused791] +[unused792] +[unused793] +[unused794] +[unused795] +[unused796] +[unused797] +[unused798] +[unused799] +[unused800] +[unused801] +[unused802] +[unused803] +[unused804] +[unused805] +[unused806] +[unused807] +[unused808] +[unused809] +[unused810] +[unused811] +[unused812] +[unused813] +[unused814] +[unused815] +[unused816] +[unused817] +[unused818] +[unused819] +[unused820] +[unused821] +[unused822] +[unused823] +[unused824] +[unused825] +[unused826] +[unused827] +[unused828] +[unused829] +[unused830] +[unused831] +[unused832] +[unused833] +[unused834] +[unused835] +[unused836] +[unused837] +[unused838] +[unused839] +[unused840] +[unused841] +[unused842] +[unused843] +[unused844] +[unused845] +[unused846] +[unused847] +[unused848] +[unused849] +[unused850] +[unused851] +[unused852] +[unused853] +[unused854] +[unused855] +[unused856] +[unused857] +[unused858] +[unused859] +[unused860] +[unused861] +[unused862] +[unused863] +[unused864] +[unused865] +[unused866] +[unused867] +[unused868] +[unused869] +[unused870] +[unused871] +[unused872] +[unused873] +[unused874] +[unused875] +[unused876] +[unused877] +[unused878] +[unused879] +[unused880] +[unused881] +[unused882] +[unused883] +[unused884] +[unused885] +[unused886] +[unused887] +[unused888] +[unused889] +[unused890] +[unused891] +[unused892] +[unused893] +[unused894] +[unused895] +[unused896] +[unused897] +[unused898] +[unused899] +[unused900] +[unused901] +[unused902] +[unused903] +[unused904] +[unused905] +[unused906] +[unused907] +[unused908] +[unused909] +[unused910] +[unused911] +[unused912] +[unused913] +[unused914] +[unused915] +[unused916] +[unused917] +[unused918] +[unused919] +[unused920] +[unused921] +[unused922] +[unused923] +[unused924] +[unused925] +[unused926] +[unused927] +[unused928] +[unused929] +[unused930] +[unused931] +[unused932] +[unused933] +[unused934] +[unused935] +[unused936] +[unused937] +[unused938] +[unused939] +[unused940] +[unused941] +[unused942] +[unused943] +[unused944] +[unused945] +[unused946] +[unused947] +[unused948] +[unused949] +[unused950] +[unused951] +[unused952] +[unused953] +[unused954] +[unused955] +[unused956] +[unused957] +[unused958] +[unused959] +[unused960] +[unused961] +[unused962] +[unused963] +[unused964] +[unused965] +[unused966] +[unused967] +[unused968] +[unused969] +[unused970] +[unused971] +[unused972] +[unused973] +[unused974] +[unused975] +[unused976] +[unused977] +[unused978] +[unused979] +[unused980] +[unused981] +[unused982] +[unused983] +[unused984] +[unused985] +[unused986] +[unused987] +[unused988] +[unused989] +[unused990] +[unused991] +[unused992] +[unused993] +! +" +# +$ +% +& +' +( +) +* ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; +< += +> +? +@ +[ +\ +] +^ +_ +` +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +{ +| +} +~ +¡ +¢ +£ +¤ +¥ +¦ +§ +¨ +© +ª +« +¬ +® +° +± +² +³ +´ +µ +¶ +· +¹ +º +» +¼ +½ +¾ +¿ +× +ß +æ +ð +÷ +ø +þ +đ +ħ +ı +ł +ŋ +œ +ƒ +ɐ +ɑ +ɒ +ɔ +ɕ +ə +ɛ +ɡ +ɣ +ɨ +ɪ +ɫ +ɬ +ɯ +ɲ +ɴ +ɹ +ɾ +ʀ +ʁ +ʂ +ʃ +ʉ +ʊ +ʋ +ʌ +ʎ +ʐ +ʑ +ʒ +ʔ +ʰ +ʲ +ʳ +ʷ +ʸ +ʻ +ʼ +ʾ +ʿ +ˈ +ː +ˡ +ˢ +ˣ +ˤ +α +β +γ +δ +ε +ζ +η +θ +ι +κ +λ +μ +ν +ξ +ο +π +ρ +ς +σ +τ +υ +φ +χ +ψ +ω +а +б +в +г +д +е +ж +з +и +к +л +м +н +о +п +р +с +т +у +ф +х +ц +ч +ш +щ +ъ +ы +ь +э +ю +я +ђ +є +і +ј +љ +њ +ћ +ӏ +ա +բ +գ +դ +ե +թ +ի +լ +կ +հ +մ +յ +ն +ո +պ +ս +վ +տ +ր +ւ +ք +־ +א +ב +ג +ד +ה +ו +ז +ח +ט +י +ך +כ +ל +ם +מ +ן +נ +ס +ע +ף +פ +ץ +צ +ק +ר +ש +ת +، +ء +ا +ب +ة +ت +ث +ج +ح +خ +د +ذ +ر +ز +س +ش +ص +ض +ط +ظ +ع +غ +ـ +ف +ق +ك +ل +م +ن +ه +و +ى +ي +ٹ +پ +چ +ک +گ +ں +ھ +ہ +ی +ے +अ +आ +उ +ए +क +ख +ग +च +ज +ट +ड +ण +त +थ +द +ध +न +प +ब +भ +म +य +र +ल +व +श +ष +स +ह +ा +ि +ी +ो +। +॥ +ং +অ +আ +ই +উ +এ +ও +ক +খ +গ +চ +ছ +জ +ট +ড +ণ +ত +থ +দ +ধ +ন +প +ব +ভ +ম +য +র +ল +শ +ষ +স +হ +া +ি +ী +ে +க +ச +ட +த +ந +ன +ப +ம +ய +ர +ல +ள +வ +ா +ி +ு +ே +ை +ನ +ರ +ಾ +ක +ය +ර +ල +ව +ා +ก +ง +ต +ท +น +พ +ม +ย +ร +ล +ว +ส +อ +า +เ +་ +། +ག +ང +ད +ན +པ +བ +མ +འ +ར +ལ +ས +မ +ა +ბ +გ +დ +ე +ვ +თ +ი +კ +ლ +მ +ნ +ო +რ +ს +ტ +უ +ᄀ +ᄂ +ᄃ +ᄅ +ᄆ +ᄇ +ᄉ +ᄊ +ᄋ +ᄌ +ᄎ +ᄏ +ᄐ +ᄑ +ᄒ +ᅡ +ᅢ +ᅥ +ᅦ +ᅧ +ᅩ +ᅪ +ᅭ +ᅮ +ᅯ +ᅲ +ᅳ +ᅴ +ᅵ +ᆨ +ᆫ +ᆯ +ᆷ +ᆸ +ᆼ +ᴬ +ᴮ +ᴰ +ᴵ +ᴺ +ᵀ +ᵃ +ᵇ +ᵈ +ᵉ +ᵍ +ᵏ +ᵐ +ᵒ +ᵖ +ᵗ +ᵘ +ᵢ +ᵣ +ᵤ +ᵥ +ᶜ +ᶠ +‐ +‑ +‒ +– +— +― +‖ +‘ +’ +‚ +“ +” +„ +† +‡ +• +… +‰ +′ +″ +› +‿ +⁄ +⁰ +ⁱ +⁴ +⁵ +⁶ +⁷ +⁸ +⁹ +⁺ +⁻ +ⁿ +₀ +₁ +₂ +₃ +₄ +₅ +₆ +₇ +₈ +₉ +₊ +₍ +₎ +ₐ +ₑ +ₒ +ₓ +ₕ +ₖ +ₗ +ₘ +ₙ +ₚ +ₛ +ₜ +₤ +₩ +€ +₱ +₹ +ℓ +№ +ℝ +™ +⅓ +⅔ +← +↑ +→ +↓ +↔ +↦ +⇄ +⇌ +⇒ +∂ +∅ +∆ +∇ +∈ +− +∗ +∘ +√ +∞ +∧ +∨ +∩ +∪ +≈ +≡ +≤ +≥ +⊂ +⊆ +⊕ +⊗ +⋅ +─ +│ +■ +▪ +● +★ +☆ +☉ +♠ +♣ +♥ +♦ +♭ +♯ +⟨ +⟩ +ⱼ +⺩ +⺼ +⽥ +、 +。 +〈 +〉 +《 +》 +「 +」 +『 +』 +〜 +あ +い +う +え +お +か +き +く +け +こ +さ +し +す +せ +そ +た +ち +っ +つ +て +と +な +に +ぬ +ね +の +は +ひ +ふ +へ +ほ +ま +み +む +め +も +や +ゆ +よ +ら +り +る +れ +ろ +を +ん +ァ +ア +ィ +イ +ウ +ェ +エ +オ +カ +キ +ク +ケ +コ +サ +シ +ス +セ +タ +チ +ッ +ツ +テ +ト +ナ +ニ +ノ +ハ +ヒ +フ +ヘ +ホ +マ +ミ +ム +メ +モ +ャ +ュ +ョ +ラ +リ +ル +レ +ロ +ワ +ン +・ +ー +一 +三 +上 +下 +不 +世 +中 +主 +久 +之 +也 +事 +二 +五 +井 +京 +人 +亻 +仁 +介 +代 +仮 +伊 +会 +佐 +侍 +保 +信 +健 +元 +光 +八 +公 +内 +出 +分 +前 +劉 +力 +加 +勝 +北 +区 +十 +千 +南 +博 +原 +口 +古 +史 +司 +合 +吉 +同 +名 +和 +囗 +四 +国 +國 +土 +地 +坂 +城 +堂 +場 +士 +夏 +外 +大 +天 +太 +夫 +奈 +女 +子 +学 +宀 +宇 +安 +宗 +定 +宣 +宮 +家 +宿 +寺 +將 +小 +尚 +山 +岡 +島 +崎 +川 +州 +巿 +帝 +平 +年 +幸 +广 +弘 +張 +彳 +後 +御 +德 +心 +忄 +志 +忠 +愛 +成 +我 +戦 +戸 +手 +扌 +政 +文 +新 +方 +日 +明 +星 +春 +昭 +智 +曲 +書 +月 +有 +朝 +木 +本 +李 +村 +東 +松 +林 +森 +楊 +樹 +橋 +歌 +止 +正 +武 +比 +氏 +民 +水 +氵 +氷 +永 +江 +沢 +河 +治 +法 +海 +清 +漢 +瀬 +火 +版 +犬 +王 +生 +田 +男 +疒 +発 +白 +的 +皇 +目 +相 +省 +真 +石 +示 +社 +神 +福 +禾 +秀 +秋 +空 +立 +章 +竹 +糹 +美 +義 +耳 +良 +艹 +花 +英 +華 +葉 +藤 +行 +街 +西 +見 +訁 +語 +谷 +貝 +貴 +車 +軍 +辶 +道 +郎 +郡 +部 +都 +里 +野 +金 +鈴 +镇 +長 +門 +間 +阝 +阿 +陳 +陽 +雄 +青 +面 +風 +食 +香 +馬 +高 +龍 +龸 +fi +fl +！ +（ +） +， +－ +． +／ +： +？ +～ +the +of +and +in +to +was +he +is +as +for +on +with +that +it +his +by +at +from +her +##s +she +you +had +an +were +but +be +this +are +not +my +they +one +which +or +have +him +me +first +all +also +their +has +up +who +out +been +when +after +there +into +new +two +its +##a +time +would +no +what +about +said +we +over +then +other +so +more +##e +can +if +like +back +them +only +some +could +##i +where +just +##ing +during +before +##n +do +##o +made +school +through +than +now +years +most +world +may +between +down +well +three +##d +year +while +will +##ed +##r +##y +later +##t +city +under +around +did +such +being +used +state +people +part +know +against +your +many +second +university +both +national +##er +these +don +known +off +way +until +re +how +even +get +head +... +didn +##ly +team +american +because +de +##l +born +united +film +since +still +long +work +south +us +became +any +high +again +day +family +see +right +man +eyes +house +season +war +states +including +took +life +north +same +each +called +name +much +place +however +go +four +group +another +found +won +area +here +going +10 +away +series +left +home +music +best +make +hand +number +company +several +never +last +john +000 +very +album +take +end +good +too +following +released +game +played +little +began +district +##m +old +want +those +side +held +own +early +county +ll +league +use +west +##u +face +think +##es +2010 +government +##h +march +came +small +general +town +june +##on +line +based +something +##k +september +thought +looked +along +international +2011 +air +july +club +went +january +october +our +august +april +york +12 +few +2012 +2008 +east +show +member +college +2009 +father +public +##us +come +men +five +set +station +church +##c +next +former +november +room +party +located +december +2013 +age +got +2007 +##g +system +let +love +2006 +though +every +2014 +look +song +water +century +without +body +black +night +within +great +women +single +ve +building +large +population +river +named +band +white +started +##an +once +15 +20 +should +18 +2015 +service +top +built +british +open +death +king +moved +local +times +children +february +book +why +11 +door +need +president +order +final +road +wasn +although +due +major +died +village +third +knew +2016 +asked +turned +st +wanted +say +##p +together +received +main +son +served +different +##en +behind +himself +felt +members +power +football +law +voice +play +##in +near +park +history +30 +having +2005 +16 +##man +saw +mother +##al +army +point +front +help +english +street +art +late +hands +games +award +##ia +young +14 +put +published +country +division +across +told +13 +often +ever +french +london +center +six +red +2017 +led +days +include +light +25 +find +tell +among +species +really +according +central +half +2004 +form +original +gave +office +making +enough +lost +full +opened +must +included +live +given +german +player +run +business +woman +community +cup +might +million +land +2000 +court +development +17 +short +round +ii +km +seen +class +story +always +become +sure +research +almost +director +council +la +##2 +career +things +using +island +##z +couldn +car +##is +24 +close +force +##1 +better +free +support +control +field +students +2003 +education +married +##b +nothing +worked +others +record +big +inside +level +anything +continued +give +james +##3 +military +established +non +returned +feel +does +title +written +thing +feet +william +far +co +association +hard +already +2002 +##ra +championship +human +western +100 +##na +department +hall +role +various +production +21 +19 +heart +2001 +living +fire +version +##ers +##f +television +royal +##4 +produced +working +act +case +society +region +present +radio +period +looking +least +total +keep +england +wife +program +per +brother +mind +special +22 +##le +am +works +soon +##6 +political +george +services +taken +created +##7 +further +able +reached +david +union +joined +upon +done +important +social +information +either +##ic +##x +appeared +position +ground +lead +rock +dark +election +23 +board +france +hair +course +arms +site +police +girl +instead +real +sound +##v +words +moment +##te +someone +##8 +summer +project +announced +san +less +wrote +past +followed +##5 +blue +founded +al +finally +india +taking +records +america +##ne +1999 +design +considered +northern +god +stop +battle +toward +european +outside +described +track +today +playing +language +28 +call +26 +heard +professional +low +australia +miles +california +win +yet +green +##ie +trying +blood +##ton +southern +science +maybe +everything +match +square +27 +mouth +video +race +recorded +leave +above +##9 +daughter +points +space +1998 +museum +change +middle +common +##0 +move +tv +post +##ta +lake +seven +tried +elected +closed +ten +paul +minister +##th +months +start +chief +return +canada +person +sea +release +similar +modern +brought +rest +hit +formed +mr +##la +1997 +floor +event +doing +thomas +1996 +robert +care +killed +training +star +week +needed +turn +finished +railway +rather +news +health +sent +example +ran +term +michael +coming +currently +yes +forces +despite +gold +areas +50 +stage +fact +29 +dead +says +popular +2018 +originally +germany +probably +developed +result +pulled +friend +stood +money +running +mi +signed +word +songs +child +eventually +met +tour +average +teams +minutes +festival +current +deep +kind +1995 +decided +usually +eastern +seemed +##ness +episode +bed +added +table +indian +private +charles +route +available +idea +throughout +centre +addition +appointed +style +1994 +books +eight +construction +press +mean +wall +friends +remained +schools +study +##ch +##um +institute +oh +chinese +sometimes +events +possible +1992 +australian +type +brown +forward +talk +process +food +debut +seat +performance +committee +features +character +arts +herself +else +lot +strong +russian +range +hours +peter +arm +##da +morning +dr +sold +##ry +quickly +directed +1993 +guitar +china +##w +31 +list +##ma +performed +media +uk +players +smile +##rs +myself +40 +placed +coach +province +towards +wouldn +leading +whole +boy +official +designed +grand +census +##el +europe +attack +japanese +henry +1991 +##re +##os +cross +getting +alone +action +lower +network +wide +washington +japan +1990 +hospital +believe +changed +sister +##ar +hold +gone +sir +hadn +ship +##ka +studies +academy +shot +rights +below +base +bad +involved +kept +largest +##ist +bank +future +especially +beginning +mark +movement +section +female +magazine +plan +professor +lord +longer +##ian +sat +walked +hill +actually +civil +energy +model +families +size +thus +aircraft +completed +includes +data +captain +##or +fight +vocals +featured +richard +bridge +fourth +1989 +officer +stone +hear +##ism +means +medical +groups +management +self +lips +competition +entire +lived +technology +leaving +federal +tournament +bit +passed +hot +independent +awards +kingdom +mary +spent +fine +doesn +reported +##ling +jack +fall +raised +itself +stay +true +studio +1988 +sports +replaced +paris +systems +saint +leader +theatre +whose +market +capital +parents +spanish +canadian +earth +##ity +cut +degree +writing +bay +christian +awarded +natural +higher +bill +##as +coast +provided +previous +senior +ft +valley +organization +stopped +onto +countries +parts +conference +queen +security +interest +saying +allowed +master +earlier +phone +matter +smith +winning +try +happened +moving +campaign +los +##ley +breath +nearly +mid +1987 +certain +girls +date +italian +african +standing +fell +artist +##ted +shows +deal +mine +industry +1986 +##ng +everyone +republic +provide +collection +library +student +##ville +primary +owned +older +via +heavy +1st +makes +##able +attention +anyone +africa +##ri +stated +length +ended +fingers +command +staff +skin +foreign +opening +governor +okay +medal +kill +sun +cover +job +1985 +introduced +chest +hell +feeling +##ies +success +meet +reason +standard +meeting +novel +1984 +trade +source +buildings +##land +rose +guy +goal +##ur +chapter +native +husband +previously +unit +limited +entered +weeks +producer +operations +mountain +takes +covered +forced +related +roman +complete +successful +key +texas +cold +##ya +channel +1980 +traditional +films +dance +clear +approximately +500 +nine +van +prince +question +active +tracks +ireland +regional +silver +author +personal +sense +operation +##ine +economic +1983 +holding +twenty +isbn +additional +speed +hour +edition +regular +historic +places +whom +shook +movie +km² +secretary +prior +report +chicago +read +foundation +view +engine +scored +1982 +units +ask +airport +property +ready +immediately +lady +month +listed +contract +##de +manager +themselves +lines +##ki +navy +writer +meant +##ts +runs +##ro +practice +championships +singer +glass +commission +required +forest +starting +culture +generally +giving +access +attended +test +couple +stand +catholic +martin +caught +executive +##less +eye +##ey +thinking +chair +quite +shoulder +1979 +hope +decision +plays +defeated +municipality +whether +structure +offered +slowly +pain +ice +direction +##ion +paper +mission +1981 +mostly +200 +noted +individual +managed +nature +lives +plant +##ha +helped +except +studied +computer +figure +relationship +issue +significant +loss +die +smiled +gun +ago +highest +1972 +##am +male +bring +goals +mexico +problem +distance +commercial +completely +location +annual +famous +drive +1976 +neck +1978 +surface +caused +italy +understand +greek +highway +wrong +hotel +comes +appearance +joseph +double +issues +musical +companies +castle +income +review +assembly +bass +initially +parliament +artists +experience +1974 +particular +walk +foot +engineering +talking +window +dropped +##ter +miss +baby +boys +break +1975 +stars +edge +remember +policy +carried +train +stadium +bar +sex +angeles +evidence +##ge +becoming +assistant +soviet +1977 +upper +step +wing +1970 +youth +financial +reach +##ll +actor +numerous +##se +##st +nodded +arrived +##ation +minute +##nt +believed +sorry +complex +beautiful +victory +associated +temple +1968 +1973 +chance +perhaps +metal +##son +1945 +bishop +##et +lee +launched +particularly +tree +le +retired +subject +prize +contains +yeah +theory +empire +##ce +suddenly +waiting +trust +recording +##to +happy +terms +camp +champion +1971 +religious +pass +zealand +names +2nd +port +ancient +tom +corner +represented +watch +legal +anti +justice +cause +watched +brothers +45 +material +changes +simply +response +louis +fast +##ting +answer +60 +historical +1969 +stories +straight +create +feature +increased +rate +administration +virginia +el +activities +cultural +overall +winner +programs +basketball +legs +guard +beyond +cast +doctor +mm +flight +results +remains +cost +effect +winter +##ble +larger +islands +problems +chairman +grew +commander +isn +1967 +pay +failed +selected +hurt +fort +box +regiment +majority +journal +35 +edward +plans +##ke +##ni +shown +pretty +irish +characters +directly +scene +likely +operated +allow +spring +##j +junior +matches +looks +mike +houses +fellow +##tion +beach +marriage +##ham +##ive +rules +oil +65 +florida +expected +nearby +congress +sam +peace +recent +iii +wait +subsequently +cell +##do +variety +serving +agreed +please +poor +joe +pacific +attempt +wood +democratic +piece +prime +##ca +rural +mile +touch +appears +township +1964 +1966 +soldiers +##men +##ized +1965 +pennsylvania +closer +fighting +claimed +score +jones +physical +editor +##ous +filled +genus +specific +sitting +super +mom +##va +therefore +supported +status +fear +cases +store +meaning +wales +minor +spain +tower +focus +vice +frank +follow +parish +separate +golden +horse +fifth +remaining +branch +32 +presented +stared +##id +uses +secret +forms +##co +baseball +exactly +##ck +choice +note +discovered +travel +composed +truth +russia +ball +color +kiss +dad +wind +continue +ring +referred +numbers +digital +greater +##ns +metres +slightly +direct +increase +1960 +responsible +crew +rule +trees +troops +##no +broke +goes +individuals +hundred +weight +creek +sleep +memory +defense +provides +ordered +code +value +jewish +windows +1944 +safe +judge +whatever +corps +realized +growing +pre +##ga +cities +alexander +gaze +lies +spread +scott +letter +showed +situation +mayor +transport +watching +workers +extended +##li +expression +normal +##ment +chart +multiple +border +##ba +host +##ner +daily +mrs +walls +piano +##ko +heat +cannot +##ate +earned +products +drama +era +authority +seasons +join +grade +##io +sign +difficult +machine +1963 +territory +mainly +##wood +stations +squadron +1962 +stepped +iron +19th +##led +serve +appear +sky +speak +broken +charge +knowledge +kilometres +removed +ships +article +campus +simple +##ty +pushed +britain +##ve +leaves +recently +cd +soft +boston +latter +easy +acquired +poland +##sa +quality +officers +presence +planned +nations +mass +broadcast +jean +share +image +influence +wild +offer +emperor +electric +reading +headed +ability +promoted +yellow +ministry +1942 +throat +smaller +politician +##by +latin +spoke +cars +williams +males +lack +pop +80 +##ier +acting +seeing +consists +##ti +estate +1961 +pressure +johnson +newspaper +jr +chris +olympics +online +conditions +beat +elements +walking +vote +##field +needs +carolina +text +featuring +global +block +shirt +levels +francisco +purpose +females +et +dutch +duke +ahead +gas +twice +safety +serious +turning +highly +lieutenant +firm +maria +amount +mixed +daniel +proposed +perfect +agreement +affairs +3rd +seconds +contemporary +paid +1943 +prison +save +kitchen +label +administrative +intended +constructed +academic +nice +teacher +races +1956 +formerly +corporation +ben +nation +issued +shut +1958 +drums +housing +victoria +seems +opera +1959 +graduated +function +von +mentioned +picked +build +recognized +shortly +protection +picture +notable +exchange +elections +1980s +loved +percent +racing +fish +elizabeth +garden +volume +hockey +1941 +beside +settled +##ford +1940 +competed +replied +drew +1948 +actress +marine +scotland +steel +glanced +farm +steve +1957 +risk +tonight +positive +magic +singles +effects +gray +screen +dog +##ja +residents +bus +sides +none +secondary +literature +polish +destroyed +flying +founder +households +1939 +lay +reserve +usa +gallery +##ler +1946 +industrial +younger +approach +appearances +urban +ones +1950 +finish +avenue +powerful +fully +growth +page +honor +jersey +projects +advanced +revealed +basic +90 +infantry +pair +equipment +visit +33 +evening +search +grant +effort +solo +treatment +buried +republican +primarily +bottom +owner +1970s +israel +gives +jim +dream +bob +remain +spot +70 +notes +produce +champions +contact +ed +soul +accepted +ways +del +##ally +losing +split +price +capacity +basis +trial +questions +##ina +1955 +20th +guess +officially +memorial +naval +initial +##ization +whispered +median +engineer +##ful +sydney +##go +columbia +strength +300 +1952 +tears +senate +00 +card +asian +agent +1947 +software +44 +draw +warm +supposed +com +pro +##il +transferred +leaned +##at +candidate +escape +mountains +asia +potential +activity +entertainment +seem +traffic +jackson +murder +36 +slow +product +orchestra +haven +agency +bbc +taught +website +comedy +unable +storm +planning +albums +rugby +environment +scientific +grabbed +protect +##hi +boat +typically +1954 +1953 +damage +principal +divided +dedicated +mount +ohio +##berg +pick +fought +driver +##der +empty +shoulders +sort +thank +berlin +prominent +account +freedom +necessary +efforts +alex +headquarters +follows +alongside +des +simon +andrew +suggested +operating +learning +steps +1949 +sweet +technical +begin +easily +34 +teeth +speaking +settlement +scale +##sh +renamed +ray +max +enemy +semi +joint +compared +##rd +scottish +leadership +analysis +offers +georgia +pieces +captured +animal +deputy +guest +organized +##lin +tony +combined +method +challenge +1960s +huge +wants +battalion +sons +rise +crime +types +facilities +telling +path +1951 +platform +sit +1990s +##lo +tells +assigned +rich +pull +##ot +commonly +alive +##za +letters +concept +conducted +wearing +happen +bought +becomes +holy +gets +ocean +defeat +languages +purchased +coffee +occurred +titled +##q +declared +applied +sciences +concert +sounds +jazz +brain +##me +painting +fleet +tax +nick +##ius +michigan +count +animals +leaders +episodes +##line +content +##den +birth +##it +clubs +64 +palace +critical +refused +fair +leg +laughed +returning +surrounding +participated +formation +lifted +pointed +connected +rome +medicine +laid +taylor +santa +powers +adam +tall +shared +focused +knowing +yards +entrance +falls +##wa +calling +##ad +sources +chosen +beneath +resources +yard +##ite +nominated +silence +zone +defined +##que +gained +thirty +38 +bodies +moon +##ard +adopted +christmas +widely +register +apart +iran +premier +serves +du +unknown +parties +##les +generation +##ff +continues +quick +fields +brigade +quiet +teaching +clothes +impact +weapons +partner +flat +theater +supreme +1938 +37 +relations +##tor +plants +suffered +1936 +wilson +kids +begins +##age +1918 +seats +armed +internet +models +worth +laws +400 +communities +classes +background +knows +thanks +quarter +reaching +humans +carry +killing +format +kong +hong +setting +75 +architecture +disease +railroad +inc +possibly +wish +arthur +thoughts +harry +doors +density +##di +crowd +illinois +stomach +tone +unique +reports +anyway +##ir +liberal +der +vehicle +thick +dry +drug +faced +largely +facility +theme +holds +creation +strange +colonel +##mi +revolution +bell +politics +turns +silent +rail +relief +independence +combat +shape +write +determined +sales +learned +4th +finger +oxford +providing +1937 +heritage +fiction +situated +designated +allowing +distribution +hosted +##est +sight +interview +estimated +reduced +##ria +toronto +footballer +keeping +guys +damn +claim +motion +sport +sixth +stayed +##ze +en +rear +receive +handed +twelve +dress +audience +granted +brazil +##well +spirit +##ated +noticed +etc +olympic +representative +eric +tight +trouble +reviews +drink +vampire +missing +roles +ranked +newly +household +finals +wave +critics +##ee +phase +massachusetts +pilot +unlike +philadelphia +bright +guns +crown +organizations +roof +42 +respectively +clearly +tongue +marked +circle +fox +korea +bronze +brian +expanded +sexual +supply +yourself +inspired +labour +fc +##ah +reference +vision +draft +connection +brand +reasons +1935 +classic +driving +trip +jesus +cells +entry +1920 +neither +trail +claims +atlantic +orders +labor +nose +afraid +identified +intelligence +calls +cancer +attacked +passing +stephen +positions +imperial +grey +jason +39 +sunday +48 +swedish +avoid +extra +uncle +message +covers +allows +surprise +materials +fame +hunter +##ji +1930 +citizens +figures +davis +environmental +confirmed +shit +titles +di +performing +difference +acts +attacks +##ov +existing +votes +opportunity +nor +shop +entirely +trains +opposite +pakistan +##pa +develop +resulted +representatives +actions +reality +pressed +##ish +barely +wine +conversation +faculty +northwest +ends +documentary +nuclear +stock +grace +sets +eat +alternative +##ps +bag +resulting +creating +surprised +cemetery +1919 +drop +finding +sarah +cricket +streets +tradition +ride +1933 +exhibition +target +ear +explained +rain +composer +injury +apartment +municipal +educational +occupied +netherlands +clean +billion +constitution +learn +1914 +maximum +classical +francis +lose +opposition +jose +ontario +bear +core +hills +rolled +ending +drawn +permanent +fun +##tes +##lla +lewis +sites +chamber +ryan +##way +scoring +height +1934 +##house +lyrics +staring +55 +officials +1917 +snow +oldest +##tic +orange +##ger +qualified +interior +apparently +succeeded +thousand +dinner +lights +existence +fans +heavily +41 +greatest +conservative +send +bowl +plus +enter +catch +##un +economy +duty +1929 +speech +authorities +princess +performances +versions +shall +graduate +pictures +effective +remembered +poetry +desk +crossed +starring +starts +passenger +sharp +##ant +acres +ass +weather +falling +rank +fund +supporting +check +adult +publishing +heads +cm +southeast +lane +##burg +application +bc +##ura +les +condition +transfer +prevent +display +ex +regions +earl +federation +cool +relatively +answered +besides +1928 +obtained +portion +##town +mix +##ding +reaction +liked +dean +express +peak +1932 +##tte +counter +religion +chain +rare +miller +convention +aid +lie +vehicles +mobile +perform +squad +wonder +lying +crazy +sword +##ping +attempted +centuries +weren +philosophy +category +##ize +anna +interested +47 +sweden +wolf +frequently +abandoned +kg +literary +alliance +task +entitled +##ay +threw +promotion +factory +tiny +soccer +visited +matt +fm +achieved +52 +defence +internal +persian +43 +methods +##ging +arrested +otherwise +cambridge +programming +villages +elementary +districts +rooms +criminal +conflict +worry +trained +1931 +attempts +waited +signal +bird +truck +subsequent +programme +##ol +ad +49 +communist +details +faith +sector +patrick +carrying +laugh +##ss +controlled +korean +showing +origin +fuel +evil +1927 +##ent +brief +identity +darkness +address +pool +missed +publication +web +planet +ian +anne +wings +invited +##tt +briefly +standards +kissed +##be +ideas +climate +causing +walter +worse +albert +articles +winners +desire +aged +northeast +dangerous +gate +doubt +1922 +wooden +multi +##ky +poet +rising +funding +46 +communications +communication +violence +copies +prepared +ford +investigation +skills +1924 +pulling +electronic +##ak +##ial +##han +containing +ultimately +offices +singing +understanding +restaurant +tomorrow +fashion +christ +ward +da +pope +stands +5th +flow +studios +aired +commissioned +contained +exist +fresh +americans +##per +wrestling +approved +kid +employed +respect +suit +1925 +angel +asking +increasing +frame +angry +selling +1950s +thin +finds +##nd +temperature +statement +ali +explain +inhabitants +towns +extensive +narrow +51 +jane +flowers +images +promise +somewhere +object +fly +closely +##ls +1912 +bureau +cape +1926 +weekly +presidential +legislative +1921 +##ai +##au +launch +founding +##ny +978 +##ring +artillery +strike +un +institutions +roll +writers +landing +chose +kevin +anymore +pp +##ut +attorney +fit +dan +billboard +receiving +agricultural +breaking +sought +dave +admitted +lands +mexican +##bury +charlie +specifically +hole +iv +howard +credit +moscow +roads +accident +1923 +proved +wear +struck +hey +guards +stuff +slid +expansion +1915 +cat +anthony +##kin +melbourne +opposed +sub +southwest +architect +failure +plane +1916 +##ron +map +camera +tank +listen +regarding +wet +introduction +metropolitan +link +ep +fighter +inch +grown +gene +anger +fixed +buy +dvd +khan +domestic +worldwide +chapel +mill +functions +examples +##head +developing +1910 +turkey +hits +pocket +antonio +papers +grow +unless +circuit +18th +concerned +attached +journalist +selection +journey +converted +provincial +painted +hearing +aren +bands +negative +aside +wondered +knight +lap +survey +ma +##ow +noise +billy +##ium +shooting +guide +bedroom +priest +resistance +motor +homes +sounded +giant +##mer +150 +scenes +equal +comic +patients +hidden +solid +actual +bringing +afternoon +touched +funds +wedding +consisted +marie +canal +sr +kim +treaty +turkish +recognition +residence +cathedral +broad +knees +incident +shaped +fired +norwegian +handle +cheek +contest +represent +##pe +representing +beauty +##sen +birds +advantage +emergency +wrapped +drawing +notice +pink +broadcasting +##ong +somehow +bachelor +seventh +collected +registered +establishment +alan +assumed +chemical +personnel +roger +retirement +jeff +portuguese +wore +tied +device +threat +progress +advance +##ised +banks +hired +manchester +nfl +teachers +structures +forever +##bo +tennis +helping +saturday +sale +applications +junction +hip +incorporated +neighborhood +dressed +ceremony +##ds +influenced +hers +visual +stairs +decades +inner +kansas +hung +hoped +gain +scheduled +downtown +engaged +austria +clock +norway +certainly +pale +protected +1913 +victor +employees +plate +putting +surrounded +##ists +finishing +blues +tropical +##ries +minnesota +consider +philippines +accept +54 +retrieved +1900 +concern +anderson +properties +institution +gordon +successfully +vietnam +##dy +backing +outstanding +muslim +crossing +folk +producing +usual +demand +occurs +observed +lawyer +educated +##ana +kelly +string +pleasure +budget +items +quietly +colorado +philip +typical +##worth +derived +600 +survived +asks +mental +##ide +56 +jake +jews +distinguished +ltd +1911 +sri +extremely +53 +athletic +loud +thousands +worried +shadow +transportation +horses +weapon +arena +importance +users +tim +objects +contributed +dragon +douglas +aware +senator +johnny +jordan +sisters +engines +flag +investment +samuel +shock +capable +clark +row +wheel +refers +session +familiar +biggest +wins +hate +maintained +drove +hamilton +request +expressed +injured +underground +churches +walker +wars +tunnel +passes +stupid +agriculture +softly +cabinet +regarded +joining +indiana +##ea +##ms +push +dates +spend +behavior +woods +protein +gently +chase +morgan +mention +burning +wake +combination +occur +mirror +leads +jimmy +indeed +impossible +singapore +paintings +covering +##nes +soldier +locations +attendance +sell +historian +wisconsin +invasion +argued +painter +diego +changing +egypt +##don +experienced +inches +##ku +missouri +vol +grounds +spoken +switzerland +##gan +reform +rolling +ha +forget +massive +resigned +burned +allen +tennessee +locked +values +improved +##mo +wounded +universe +sick +dating +facing +pack +purchase +user +##pur +moments +##ul +merged +anniversary +1908 +coal +brick +understood +causes +dynasty +queensland +establish +stores +crisis +promote +hoping +views +cards +referee +extension +##si +raise +arizona +improve +colonial +formal +charged +##rt +palm +lucky +hide +rescue +faces +95 +feelings +candidates +juan +##ell +goods +6th +courses +weekend +59 +luke +cash +fallen +##om +delivered +affected +installed +carefully +tries +swiss +hollywood +costs +lincoln +responsibility +##he +shore +file +proper +normally +maryland +assistance +jump +constant +offering +friendly +waters +persons +realize +contain +trophy +800 +partnership +factor +58 +musicians +cry +bound +oregon +indicated +hero +houston +medium +##ure +consisting +somewhat +##ara +57 +cycle +##che +beer +moore +frederick +gotten +eleven +worst +weak +approached +arranged +chin +loan +universal +bond +fifteen +pattern +disappeared +##ney +translated +##zed +lip +arab +capture +interests +insurance +##chi +shifted +cave +prix +warning +sections +courts +coat +plot +smell +feed +golf +favorite +maintain +knife +vs +voted +degrees +finance +quebec +opinion +translation +manner +ruled +operate +productions +choose +musician +discovery +confused +tired +separated +stream +techniques +committed +attend +ranking +kings +throw +passengers +measure +horror +fan +mining +sand +danger +salt +calm +decade +dam +require +runner +##ik +rush +associate +greece +##ker +rivers +consecutive +matthew +##ski +sighed +sq +documents +steam +edited +closing +tie +accused +1905 +##ini +islamic +distributed +directors +organisation +bruce +7th +breathing +mad +lit +arrival +concrete +taste +08 +composition +shaking +faster +amateur +adjacent +stating +1906 +twin +flew +##ran +tokyo +publications +##tone +obviously +ridge +storage +1907 +carl +pages +concluded +desert +driven +universities +ages +terminal +sequence +borough +250 +constituency +creative +cousin +economics +dreams +margaret +notably +reduce +montreal +mode +17th +ears +saved +jan +vocal +##ica +1909 +andy +##jo +riding +roughly +threatened +##ise +meters +meanwhile +landed +compete +repeated +grass +czech +regularly +charges +tea +sudden +appeal +##ung +solution +describes +pierre +classification +glad +parking +##ning +belt +physics +99 +rachel +add +hungarian +participate +expedition +damaged +gift +childhood +85 +fifty +##red +mathematics +jumped +letting +defensive +mph +##ux +##gh +testing +##hip +hundreds +shoot +owners +matters +smoke +israeli +kentucky +dancing +mounted +grandfather +emma +designs +profit +argentina +##gs +truly +li +lawrence +cole +begun +detroit +willing +branches +smiling +decide +miami +enjoyed +recordings +##dale +poverty +ethnic +gay +##bi +gary +arabic +09 +accompanied +##one +##ons +fishing +determine +residential +acid +##ary +alice +returns +starred +mail +##ang +jonathan +strategy +##ue +net +forty +cook +businesses +equivalent +commonwealth +distinct +ill +##cy +seriously +##ors +##ped +shift +harris +replace +rio +imagine +formula +ensure +##ber +additionally +scheme +conservation +occasionally +purposes +feels +favor +##and +##ore +1930s +contrast +hanging +hunt +movies +1904 +instruments +victims +danish +christopher +busy +demon +sugar +earliest +colony +studying +balance +duties +##ks +belgium +slipped +carter +05 +visible +stages +iraq +fifa +##im +commune +forming +zero +07 +continuing +talked +counties +legend +bathroom +option +tail +clay +daughters +afterwards +severe +jaw +visitors +##ded +devices +aviation +russell +kate +##vi +entering +subjects +##ino +temporary +swimming +forth +smooth +ghost +audio +bush +operates +rocks +movements +signs +eddie +##tz +ann +voices +honorary +06 +memories +dallas +pure +measures +racial +promised +66 +harvard +ceo +16th +parliamentary +indicate +benefit +flesh +dublin +louisiana +1902 +1901 +patient +sleeping +1903 +membership +coastal +medieval +wanting +element +scholars +rice +62 +limit +survive +makeup +rating +definitely +collaboration +obvious +##tan +boss +ms +baron +birthday +linked +soil +diocese +##lan +ncaa +##mann +offensive +shell +shouldn +waist +##tus +plain +ross +organ +resolution +manufacturing +adding +relative +kennedy +98 +whilst +moth +marketing +gardens +crash +72 +heading +partners +credited +carlos +moves +cable +##zi +marshall +##out +depending +bottle +represents +rejected +responded +existed +04 +jobs +denmark +lock +##ating +treated +graham +routes +talent +commissioner +drugs +secure +tests +reign +restored +photography +##gi +contributions +oklahoma +designer +disc +grin +seattle +robin +paused +atlanta +unusual +##gate +praised +las +laughing +satellite +hungary +visiting +##sky +interesting +factors +deck +poems +norman +##water +stuck +speaker +rifle +domain +premiered +##her +dc +comics +actors +01 +reputation +eliminated +8th +ceiling +prisoners +script +##nce +leather +austin +mississippi +rapidly +admiral +parallel +charlotte +guilty +tools +gender +divisions +fruit +##bs +laboratory +nelson +fantasy +marry +rapid +aunt +tribe +requirements +aspects +suicide +amongst +adams +bone +ukraine +abc +kick +sees +edinburgh +clothing +column +rough +gods +hunting +broadway +gathered +concerns +##ek +spending +ty +12th +snapped +requires +solar +bones +cavalry +##tta +iowa +drinking +waste +index +franklin +charity +thompson +stewart +tip +flash +landscape +friday +enjoy +singh +poem +listening +##back +eighth +fred +differences +adapted +bomb +ukrainian +surgery +corporate +masters +anywhere +##more +waves +odd +sean +portugal +orleans +dick +debate +kent +eating +puerto +cleared +96 +expect +cinema +97 +guitarist +blocks +electrical +agree +involving +depth +dying +panel +struggle +##ged +peninsula +adults +novels +emerged +vienna +metro +debuted +shoes +tamil +songwriter +meets +prove +beating +instance +heaven +scared +sending +marks +artistic +passage +superior +03 +significantly +shopping +##tive +retained +##izing +malaysia +technique +cheeks +##ola +warren +maintenance +destroy +extreme +allied +120 +appearing +##yn +fill +advice +alabama +qualifying +policies +cleveland +hat +battery +smart +authors +10th +soundtrack +acted +dated +lb +glance +equipped +coalition +funny +outer +ambassador +roy +possibility +couples +campbell +dna +loose +ethan +supplies +1898 +gonna +88 +monster +##res +shake +agents +frequency +springs +dogs +practices +61 +gang +plastic +easier +suggests +gulf +blade +exposed +colors +industries +markets +pan +nervous +electoral +charts +legislation +ownership +##idae +mac +appointment +shield +copy +assault +socialist +abbey +monument +license +throne +employment +jay +93 +replacement +charter +cloud +powered +suffering +accounts +oak +connecticut +strongly +wright +colour +crystal +13th +context +welsh +networks +voiced +gabriel +jerry +##cing +forehead +mp +##ens +manage +schedule +totally +remix +##ii +forests +occupation +print +nicholas +brazilian +strategic +vampires +engineers +76 +roots +seek +correct +instrumental +und +alfred +backed +hop +##des +stanley +robinson +traveled +wayne +welcome +austrian +achieve +67 +exit +rates +1899 +strip +whereas +##cs +sing +deeply +adventure +bobby +rick +jamie +careful +components +cap +useful +personality +knee +##shi +pushing +hosts +02 +protest +ca +ottoman +symphony +##sis +63 +boundary +1890 +processes +considering +considerable +tons +##work +##ft +##nia +cooper +trading +dear +conduct +91 +illegal +apple +revolutionary +holiday +definition +harder +##van +jacob +circumstances +destruction +##lle +popularity +grip +classified +liverpool +donald +baltimore +flows +seeking +honour +approval +92 +mechanical +till +happening +statue +critic +increasingly +immediate +describe +commerce +stare +##ster +indonesia +meat +rounds +boats +baker +orthodox +depression +formally +worn +naked +claire +muttered +sentence +11th +emily +document +77 +criticism +wished +vessel +spiritual +bent +virgin +parker +minimum +murray +lunch +danny +printed +compilation +keyboards +false +blow +belonged +68 +raising +78 +cutting +##board +pittsburgh +##up +9th +shadows +81 +hated +indigenous +jon +15th +barry +scholar +ah +##zer +oliver +##gy +stick +susan +meetings +attracted +spell +romantic +##ver +ye +1895 +photo +demanded +customers +##ac +1896 +logan +revival +keys +modified +commanded +jeans +##ious +upset +raw +phil +detective +hiding +resident +vincent +##bly +experiences +diamond +defeating +coverage +lucas +external +parks +franchise +helen +bible +successor +percussion +celebrated +il +lift +profile +clan +romania +##ied +mills +##su +nobody +achievement +shrugged +fault +1897 +rhythm +initiative +breakfast +carbon +700 +69 +lasted +violent +74 +wound +ken +killer +gradually +filmed +°c +dollars +processing +94 +remove +criticized +guests +sang +chemistry +##vin +legislature +disney +##bridge +uniform +escaped +integrated +proposal +purple +denied +liquid +karl +influential +morris +nights +stones +intense +experimental +twisted +71 +84 +##ld +pace +nazi +mitchell +ny +blind +reporter +newspapers +14th +centers +burn +basin +forgotten +surviving +filed +collections +monastery +losses +manual +couch +description +appropriate +merely +tag +missions +sebastian +restoration +replacing +triple +73 +elder +julia +warriors +benjamin +julian +convinced +stronger +amazing +declined +versus +merchant +happens +output +finland +bare +barbara +absence +ignored +dawn +injuries +##port +producers +##ram +82 +luis +##ities +kw +admit +expensive +electricity +nba +exception +symbol +##ving +ladies +shower +sheriff +characteristics +##je +aimed +button +ratio +effectively +summit +angle +jury +bears +foster +vessels +pants +executed +evans +dozen +advertising +kicked +patrol +1889 +competitions +lifetime +principles +athletics +##logy +birmingham +sponsored +89 +rob +nomination +1893 +acoustic +##sm +creature +longest +##tra +credits +harbor +dust +josh +##so +territories +milk +infrastructure +completion +thailand +indians +leon +archbishop +##sy +assist +pitch +blake +arrangement +girlfriend +serbian +operational +hence +sad +scent +fur +dj +sessions +hp +refer +rarely +##ora +exists +1892 +##ten +scientists +dirty +penalty +burst +portrait +seed +79 +pole +limits +rival +1894 +stable +alpha +grave +constitutional +alcohol +arrest +flower +mystery +devil +architectural +relationships +greatly +habitat +##istic +larry +progressive +remote +cotton +##ics +##ok +preserved +reaches +##ming +cited +86 +vast +scholarship +decisions +cbs +joy +teach +1885 +editions +knocked +eve +searching +partly +participation +gap +animated +fate +excellent +##ett +na +87 +alternate +saints +youngest +##ily +climbed +##ita +##tors +suggest +##ct +discussion +staying +choir +lakes +jacket +revenue +nevertheless +peaked +instrument +wondering +annually +managing +neil +1891 +signing +terry +##ice +apply +clinical +brooklyn +aim +catherine +fuck +farmers +figured +ninth +pride +hugh +evolution +ordinary +involvement +comfortable +shouted +tech +encouraged +taiwan +representation +sharing +##lia +##em +panic +exact +cargo +competing +fat +cried +83 +1920s +occasions +pa +cabin +borders +utah +marcus +##isation +badly +muscles +##ance +victorian +transition +warner +bet +permission +##rin +slave +terrible +similarly +shares +seth +uefa +possession +medals +benefits +colleges +lowered +perfectly +mall +transit +##ye +##kar +publisher +##ened +harrison +deaths +elevation +##ae +asleep +machines +sigh +ash +hardly +argument +occasion +parent +leo +decline +1888 +contribution +##ua +concentration +1000 +opportunities +hispanic +guardian +extent +emotions +hips +mason +volumes +bloody +controversy +diameter +steady +mistake +phoenix +identify +violin +##sk +departure +richmond +spin +funeral +enemies +1864 +gear +literally +connor +random +sergeant +grab +confusion +1865 +transmission +informed +op +leaning +sacred +suspended +thinks +gates +portland +luck +agencies +yours +hull +expert +muscle +layer +practical +sculpture +jerusalem +latest +lloyd +statistics +deeper +recommended +warrior +arkansas +mess +supports +greg +eagle +1880 +recovered +rated +concerts +rushed +##ano +stops +eggs +files +premiere +keith +##vo +delhi +turner +pit +affair +belief +paint +##zing +mate +##ach +##ev +victim +##ology +withdrew +bonus +styles +fled +##ud +glasgow +technologies +funded +nbc +adaptation +##ata +portrayed +cooperation +supporters +judges +bernard +justin +hallway +ralph +##ick +graduating +controversial +distant +continental +spider +bite +##ho +recognize +intention +mixing +##ese +egyptian +bow +tourism +suppose +claiming +tiger +dominated +participants +vi +##ru +nurse +partially +tape +##rum +psychology +##rn +essential +touring +duo +voting +civilian +emotional +channels +##king +apparent +hebrew +1887 +tommy +carrier +intersection +beast +hudson +##gar +##zo +lab +nova +bench +discuss +costa +##ered +detailed +behalf +drivers +unfortunately +obtain +##lis +rocky +##dae +siege +friendship +honey +##rian +1861 +amy +hang +posted +governments +collins +respond +wildlife +preferred +operator +##po +laura +pregnant +videos +dennis +suspected +boots +instantly +weird +automatic +businessman +alleged +placing +throwing +ph +mood +1862 +perry +venue +jet +remainder +##lli +##ci +passion +biological +boyfriend +1863 +dirt +buffalo +ron +segment +fa +abuse +##era +genre +thrown +stroke +colored +stress +exercise +displayed +##gen +struggled +##tti +abroad +dramatic +wonderful +thereafter +madrid +component +widespread +##sed +tale +citizen +todd +monday +1886 +vancouver +overseas +forcing +crying +descent +##ris +discussed +substantial +ranks +regime +1870 +provinces +switch +drum +zane +ted +tribes +proof +lp +cream +researchers +volunteer +manor +silk +milan +donated +allies +venture +principle +delivery +enterprise +##ves +##ans +bars +traditionally +witch +reminded +copper +##uk +pete +inter +links +colin +grinned +elsewhere +competitive +frequent +##oy +scream +##hu +tension +texts +submarine +finnish +defending +defend +pat +detail +1884 +affiliated +stuart +themes +villa +periods +tool +belgian +ruling +crimes +answers +folded +licensed +resort +demolished +hans +lucy +1881 +lion +traded +photographs +writes +craig +##fa +trials +generated +beth +noble +debt +percentage +yorkshire +erected +ss +viewed +grades +confidence +ceased +islam +telephone +retail +##ible +chile +m² +roberts +sixteen +##ich +commented +hampshire +innocent +dual +pounds +checked +regulations +afghanistan +sung +rico +liberty +assets +bigger +options +angels +relegated +tribute +wells +attending +leaf +##yan +butler +romanian +forum +monthly +lisa +patterns +gmina +##tory +madison +hurricane +rev +##ians +bristol +##ula +elite +valuable +disaster +democracy +awareness +germans +freyja +##ins +loop +absolutely +paying +populations +maine +sole +prayer +spencer +releases +doorway +bull +##ani +lover +midnight +conclusion +##sson +thirteen +lily +mediterranean +##lt +nhl +proud +sample +##hill +drummer +guinea +##ova +murphy +climb +##ston +instant +attributed +horn +ain +railways +steven +##ao +autumn +ferry +opponent +root +traveling +secured +corridor +stretched +tales +sheet +trinity +cattle +helps +indicates +manhattan +murdered +fitted +1882 +gentle +grandmother +mines +shocked +vegas +produces +##light +caribbean +##ou +belong +continuous +desperate +drunk +historically +trio +waved +raf +dealing +nathan +bat +murmured +interrupted +residing +scientist +pioneer +harold +aaron +##net +delta +attempting +minority +mini +believes +chorus +tend +lots +eyed +indoor +load +shots +updated +jail +##llo +concerning +connecting +wealth +##ved +slaves +arrive +rangers +sufficient +rebuilt +##wick +cardinal +flood +muhammad +whenever +relation +runners +moral +repair +viewers +arriving +revenge +punk +assisted +bath +fairly +breathe +lists +innings +illustrated +whisper +nearest +voters +clinton +ties +ultimate +screamed +beijing +lions +andre +fictional +gathering +comfort +radar +suitable +dismissed +hms +ban +pine +wrist +atmosphere +voivodeship +bid +timber +##ned +##nan +giants +##ane +cameron +recovery +uss +identical +categories +switched +serbia +laughter +noah +ensemble +therapy +peoples +touching +##off +locally +pearl +platforms +everywhere +ballet +tables +lanka +herbert +outdoor +toured +derek +1883 +spaces +contested +swept +1878 +exclusive +slight +connections +##dra +winds +prisoner +collective +bangladesh +tube +publicly +wealthy +thai +##ys +isolated +select +##ric +insisted +pen +fortune +ticket +spotted +reportedly +animation +enforcement +tanks +110 +decides +wider +lowest +owen +##time +nod +hitting +##hn +gregory +furthermore +magazines +fighters +solutions +##ery +pointing +requested +peru +reed +chancellor +knights +mask +worker +eldest +flames +reduction +1860 +volunteers +##tis +reporting +##hl +wire +advisory +endemic +origins +settlers +pursue +knock +consumer +1876 +eu +compound +creatures +mansion +sentenced +ivan +deployed +guitars +frowned +involves +mechanism +kilometers +perspective +shops +maps +terminus +duncan +alien +fist +bridges +##pers +heroes +fed +derby +swallowed +##ros +patent +sara +illness +characterized +adventures +slide +hawaii +jurisdiction +##op +organised +##side +adelaide +walks +biology +se +##ties +rogers +swing +tightly +boundaries +##rie +prepare +implementation +stolen +##sha +certified +colombia +edwards +garage +##mm +recalled +##ball +rage +harm +nigeria +breast +##ren +furniture +pupils +settle +##lus +cuba +balls +client +alaska +21st +linear +thrust +celebration +latino +genetic +terror +##cia +##ening +lightning +fee +witness +lodge +establishing +skull +##ique +earning +hood +##ei +rebellion +wang +sporting +warned +missile +devoted +activist +porch +worship +fourteen +package +1871 +decorated +##shire +housed +##ock +chess +sailed +doctors +oscar +joan +treat +garcia +harbour +jeremy +##ire +traditions +dominant +jacques +##gon +##wan +relocated +1879 +amendment +sized +companion +simultaneously +volleyball +spun +acre +increases +stopping +loves +belongs +affect +drafted +tossed +scout +battles +1875 +filming +shoved +munich +tenure +vertical +romance +pc +##cher +argue +##ical +craft +ranging +www +opens +honest +tyler +yesterday +virtual +##let +muslims +reveal +snake +immigrants +radical +screaming +speakers +firing +saving +belonging +ease +lighting +prefecture +blame +farmer +hungry +grows +rubbed +beam +sur +subsidiary +##cha +armenian +sao +dropping +conventional +##fer +microsoft +reply +qualify +spots +1867 +sweat +festivals +##ken +immigration +physician +discover +exposure +sandy +explanation +isaac +implemented +##fish +hart +initiated +connect +stakes +presents +heights +householder +pleased +tourist +regardless +slip +closest +##ction +surely +sultan +brings +riley +preparation +aboard +slammed +baptist +experiment +ongoing +interstate +organic +playoffs +##ika +1877 +130 +##tar +hindu +error +tours +tier +plenty +arrangements +talks +trapped +excited +sank +ho +athens +1872 +denver +welfare +suburb +athletes +trick +diverse +belly +exclusively +yelled +1868 +##med +conversion +##ette +1874 +internationally +computers +conductor +abilities +sensitive +hello +dispute +measured +globe +rocket +prices +amsterdam +flights +tigers +inn +municipalities +emotion +references +3d +##mus +explains +airlines +manufactured +pm +archaeological +1873 +interpretation +devon +comment +##ites +settlements +kissing +absolute +improvement +suite +impressed +barcelona +sullivan +jefferson +towers +jesse +julie +##tin +##lu +grandson +hi +gauge +regard +rings +interviews +trace +raymond +thumb +departments +burns +serial +bulgarian +scores +demonstrated +##ix +1866 +kyle +alberta +underneath +romanized +##ward +relieved +acquisition +phrase +cliff +reveals +han +cuts +merger +custom +##dar +nee +gilbert +graduation +##nts +assessment +cafe +difficulty +demands +swung +democrat +jennifer +commons +1940s +grove +##yo +completing +focuses +sum +substitute +bearing +stretch +reception +##py +reflected +essentially +destination +pairs +##ched +survival +resource +##bach +promoting +doubles +messages +tear +##down +##fully +parade +florence +harvey +incumbent +partial +framework +900 +pedro +frozen +procedure +olivia +controls +##mic +shelter +personally +temperatures +##od +brisbane +tested +sits +marble +comprehensive +oxygen +leonard +##kov +inaugural +iranian +referring +quarters +attitude +##ivity +mainstream +lined +mars +dakota +norfolk +unsuccessful +##° +explosion +helicopter +congressional +##sing +inspector +bitch +seal +departed +divine +##ters +coaching +examination +punishment +manufacturer +sink +columns +unincorporated +signals +nevada +squeezed +dylan +dining +photos +martial +manuel +eighteen +elevator +brushed +plates +ministers +ivy +congregation +##len +slept +specialized +taxes +curve +restricted +negotiations +likes +statistical +arnold +inspiration +execution +bold +intermediate +significance +margin +ruler +wheels +gothic +intellectual +dependent +listened +eligible +buses +widow +syria +earn +cincinnati +collapsed +recipient +secrets +accessible +philippine +maritime +goddess +clerk +surrender +breaks +playoff +database +##ified +##lon +ideal +beetle +aspect +soap +regulation +strings +expand +anglo +shorter +crosses +retreat +tough +coins +wallace +directions +pressing +##oon +shipping +locomotives +comparison +topics +nephew +##mes +distinction +honors +travelled +sierra +ibn +##over +fortress +sa +recognised +carved +1869 +clients +##dan +intent +##mar +coaches +describing +bread +##ington +beaten +northwestern +##ona +merit +youtube +collapse +challenges +em +historians +objective +submitted +virus +attacking +drake +assume +##ere +diseases +marc +stem +leeds +##cus +##ab +farming +glasses +##lock +visits +nowhere +fellowship +relevant +carries +restaurants +experiments +101 +constantly +bases +targets +shah +tenth +opponents +verse +territorial +##ira +writings +corruption +##hs +instruction +inherited +reverse +emphasis +##vic +employee +arch +keeps +rabbi +watson +payment +uh +##ala +nancy +##tre +venice +fastest +sexy +banned +adrian +properly +ruth +touchdown +dollar +boards +metre +circles +edges +favour +comments +ok +travels +liberation +scattered +firmly +##ular +holland +permitted +diesel +kenya +den +originated +##ral +demons +resumed +dragged +rider +##rus +servant +blinked +extend +torn +##ias +##sey +input +meal +everybody +cylinder +kinds +camps +##fe +bullet +logic +##wn +croatian +evolved +healthy +fool +chocolate +wise +preserve +pradesh +##ess +respective +1850 +##ew +chicken +artificial +gross +corresponding +convicted +cage +caroline +dialogue +##dor +narrative +stranger +mario +br +christianity +failing +trent +commanding +buddhist +1848 +maurice +focusing +yale +bike +altitude +##ering +mouse +revised +##sley +veteran +##ig +pulls +theology +crashed +campaigns +legion +##ability +drag +excellence +customer +cancelled +intensity +excuse +##lar +liga +participating +contributing +printing +##burn +variable +##rk +curious +bin +legacy +renaissance +##my +symptoms +binding +vocalist +dancer +##nie +grammar +gospel +democrats +ya +enters +sc +diplomatic +hitler +##ser +clouds +mathematical +quit +defended +oriented +##heim +fundamental +hardware +impressive +equally +convince +confederate +guilt +chuck +sliding +##ware +magnetic +narrowed +petersburg +bulgaria +otto +phd +skill +##ama +reader +hopes +pitcher +reservoir +hearts +automatically +expecting +mysterious +bennett +extensively +imagined +seeds +monitor +fix +##ative +journalism +struggling +signature +ranch +encounter +photographer +observation +protests +##pin +influences +##hr +calendar +##all +cruz +croatia +locomotive +hughes +naturally +shakespeare +basement +hook +uncredited +faded +theories +approaches +dare +phillips +filling +fury +obama +##ain +efficient +arc +deliver +min +raid +breeding +inducted +leagues +efficiency +axis +montana +eagles +##ked +supplied +instructions +karen +picking +indicating +trap +anchor +practically +christians +tomb +vary +occasional +electronics +lords +readers +newcastle +faint +innovation +collect +situations +engagement +160 +claude +mixture +##feld +peer +tissue +logo +lean +##ration +°f +floors +##ven +architects +reducing +##our +##ments +rope +1859 +ottawa +##har +samples +banking +declaration +proteins +resignation +francois +saudi +advocate +exhibited +armor +twins +divorce +##ras +abraham +reviewed +jo +temporarily +matrix +physically +pulse +curled +##ena +difficulties +bengal +usage +##ban +annie +riders +certificate +##pi +holes +warsaw +distinctive +jessica +##mon +mutual +1857 +customs +circular +eugene +removal +loaded +mere +vulnerable +depicted +generations +dame +heir +enormous +lightly +climbing +pitched +lessons +pilots +nepal +ram +google +preparing +brad +louise +renowned +##₂ +liam +##ably +plaza +shaw +sophie +brilliant +bills +##bar +##nik +fucking +mainland +server +pleasant +seized +veterans +jerked +fail +beta +brush +radiation +stored +warmth +southeastern +nate +sin +raced +berkeley +joke +athlete +designation +trunk +##low +roland +qualification +archives +heels +artwork +receives +judicial +reserves +##bed +woke +installation +abu +floating +fake +lesser +excitement +interface +concentrated +addressed +characteristic +amanda +saxophone +monk +auto +##bus +releasing +egg +dies +interaction +defender +ce +outbreak +glory +loving +##bert +sequel +consciousness +http +awake +ski +enrolled +##ress +handling +rookie +brow +somebody +biography +warfare +amounts +contracts +presentation +fabric +dissolved +challenged +meter +psychological +lt +elevated +rally +accurate +##tha +hospitals +undergraduate +specialist +venezuela +exhibit +shed +nursing +protestant +fluid +structural +footage +jared +consistent +prey +##ska +succession +reflect +exile +lebanon +wiped +suspect +shanghai +resting +integration +preservation +marvel +variant +pirates +sheep +rounded +capita +sailing +colonies +manuscript +deemed +variations +clarke +functional +emerging +boxing +relaxed +curse +azerbaijan +heavyweight +nickname +editorial +rang +grid +tightened +earthquake +flashed +miguel +rushing +##ches +improvements +boxes +brooks +180 +consumption +molecular +felix +societies +repeatedly +variation +aids +civic +graphics +professionals +realm +autonomous +receiver +delayed +workshop +militia +chairs +trump +canyon +##point +harsh +extending +lovely +happiness +##jan +stake +eyebrows +embassy +wellington +hannah +##ella +sony +corners +bishops +swear +cloth +contents +xi +namely +commenced +1854 +stanford +nashville +courage +graphic +commitment +garrison +##bin +hamlet +clearing +rebels +attraction +literacy +cooking +ruins +temples +jenny +humanity +celebrate +hasn +freight +sixty +rebel +bastard +##art +newton +##ada +deer +##ges +##ching +smiles +delaware +singers +##ets +approaching +assists +flame +##ph +boulevard +barrel +planted +##ome +pursuit +##sia +consequences +posts +shallow +invitation +rode +depot +ernest +kane +rod +concepts +preston +topic +chambers +striking +blast +arrives +descendants +montgomery +ranges +worlds +##lay +##ari +span +chaos +praise +##ag +fewer +1855 +sanctuary +mud +fbi +##ions +programmes +maintaining +unity +harper +bore +handsome +closure +tournaments +thunder +nebraska +linda +facade +puts +satisfied +argentine +dale +cork +dome +panama +##yl +1858 +tasks +experts +##ates +feeding +equation +##las +##ida +##tu +engage +bryan +##ax +um +quartet +melody +disbanded +sheffield +blocked +gasped +delay +kisses +maggie +connects +##non +sts +poured +creator +publishers +##we +guided +ellis +extinct +hug +gaining +##ord +complicated +##bility +poll +clenched +investigate +##use +thereby +quantum +spine +cdp +humor +kills +administered +semifinals +##du +encountered +ignore +##bu +commentary +##maker +bother +roosevelt +140 +plains +halfway +flowing +cultures +crack +imprisoned +neighboring +airline +##ses +##view +##mate +##ec +gather +wolves +marathon +transformed +##ill +cruise +organisations +carol +punch +exhibitions +numbered +alarm +ratings +daddy +silently +##stein +queens +colours +impression +guidance +liu +tactical +##rat +marshal +della +arrow +##ings +rested +feared +tender +owns +bitter +advisor +escort +##ides +spare +farms +grants +##ene +dragons +encourage +colleagues +cameras +##und +sucked +pile +spirits +prague +statements +suspension +landmark +fence +torture +recreation +bags +permanently +survivors +pond +spy +predecessor +bombing +coup +##og +protecting +transformation +glow +##lands +##book +dug +priests +andrea +feat +barn +jumping +##chen +##ologist +##con +casualties +stern +auckland +pipe +serie +revealing +ba +##bel +trevor +mercy +spectrum +yang +consist +governing +collaborated +possessed +epic +comprises +blew +shane +##ack +lopez +honored +magical +sacrifice +judgment +perceived +hammer +mtv +baronet +tune +das +missionary +sheets +350 +neutral +oral +threatening +attractive +shade +aims +seminary +##master +estates +1856 +michel +wounds +refugees +manufacturers +##nic +mercury +syndrome +porter +##iya +##din +hamburg +identification +upstairs +purse +widened +pause +cared +breathed +affiliate +santiago +prevented +celtic +fisher +125 +recruited +byzantine +reconstruction +farther +##mp +diet +sake +au +spite +sensation +##ert +blank +separation +105 +##hon +vladimir +armies +anime +##lie +accommodate +orbit +cult +sofia +archive +##ify +##box +founders +sustained +disorder +honours +northeastern +mia +crops +violet +threats +blanket +fires +canton +followers +southwestern +prototype +voyage +assignment +altered +moderate +protocol +pistol +##eo +questioned +brass +lifting +1852 +math +authored +##ual +doug +dimensional +dynamic +##san +1851 +pronounced +grateful +quest +uncomfortable +boom +presidency +stevens +relating +politicians +chen +barrier +quinn +diana +mosque +tribal +cheese +palmer +portions +sometime +chester +treasure +wu +bend +download +millions +reforms +registration +##osa +consequently +monitoring +ate +preliminary +brandon +invented +ps +eaten +exterior +intervention +ports +documented +log +displays +lecture +sally +favourite +##itz +vermont +lo +invisible +isle +breed +##ator +journalists +relay +speaks +backward +explore +midfielder +actively +stefan +procedures +cannon +blond +kenneth +centered +servants +chains +libraries +malcolm +essex +henri +slavery +##hal +facts +fairy +coached +cassie +cats +washed +cop +##fi +announcement +item +2000s +vinyl +activated +marco +frontier +growled +curriculum +##das +loyal +accomplished +leslie +ritual +kenny +##00 +vii +napoleon +hollow +hybrid +jungle +stationed +friedrich +counted +##ulated +platinum +theatrical +seated +col +rubber +glen +1840 +diversity +healing +extends +id +provisions +administrator +columbus +##oe +tributary +te +assured +org +##uous +prestigious +examined +lectures +grammy +ronald +associations +bailey +allan +essays +flute +believing +consultant +proceedings +travelling +1853 +kit +kerala +yugoslavia +buddy +methodist +##ith +burial +centres +batman +##nda +discontinued +bo +dock +stockholm +lungs +severely +##nk +citing +manga +##ugh +steal +mumbai +iraqi +robot +celebrity +bride +broadcasts +abolished +pot +joel +overhead +franz +packed +reconnaissance +johann +acknowledged +introduce +handled +doctorate +developments +drinks +alley +palestine +##nis +##aki +proceeded +recover +bradley +grain +patch +afford +infection +nationalist +legendary +##ath +interchange +virtually +gen +gravity +exploration +amber +vital +wishes +powell +doctrine +elbow +screenplay +##bird +contribute +indonesian +pet +creates +##com +enzyme +kylie +discipline +drops +manila +hunger +##ien +layers +suffer +fever +bits +monica +keyboard +manages +##hood +searched +appeals +##bad +testament +grande +reid +##war +beliefs +congo +##ification +##dia +si +requiring +##via +casey +1849 +regret +streak +rape +depends +syrian +sprint +pound +tourists +upcoming +pub +##xi +tense +##els +practiced +echo +nationwide +guild +motorcycle +liz +##zar +chiefs +desired +elena +bye +precious +absorbed +relatives +booth +pianist +##mal +citizenship +exhausted +wilhelm +##ceae +##hed +noting +quarterback +urge +hectares +##gue +ace +holly +##tal +blonde +davies +parked +sustainable +stepping +twentieth +airfield +galaxy +nest +chip +##nell +tan +shaft +paulo +requirement +##zy +paradise +tobacco +trans +renewed +vietnamese +##cker +##ju +suggesting +catching +holmes +enjoying +md +trips +colt +holder +butterfly +nerve +reformed +cherry +bowling +trailer +carriage +goodbye +appreciate +toy +joshua +interactive +enabled +involve +##kan +collar +determination +bunch +facebook +recall +shorts +superintendent +episcopal +frustration +giovanni +nineteenth +laser +privately +array +circulation +##ovic +armstrong +deals +painful +permit +discrimination +##wi +aires +retiring +cottage +ni +##sta +horizon +ellen +jamaica +ripped +fernando +chapters +playstation +patron +lecturer +navigation +behaviour +genes +georgian +export +solomon +rivals +swift +seventeen +rodriguez +princeton +independently +sox +1847 +arguing +entity +casting +hank +criteria +oakland +geographic +milwaukee +reflection +expanding +conquest +dubbed +##tv +halt +brave +brunswick +doi +arched +curtis +divorced +predominantly +somerset +streams +ugly +zoo +horrible +curved +buenos +fierce +dictionary +vector +theological +unions +handful +stability +chan +punjab +segments +##lly +altar +ignoring +gesture +monsters +pastor +##stone +thighs +unexpected +operators +abruptly +coin +compiled +associates +improving +migration +pin +##ose +compact +collegiate +reserved +##urs +quarterfinals +roster +restore +assembled +hurry +oval +##cies +1846 +flags +martha +##del +victories +sharply +##rated +argues +deadly +neo +drawings +symbols +performer +##iel +griffin +restrictions +editing +andrews +java +journals +arabia +compositions +dee +pierce +removing +hindi +casino +runway +civilians +minds +nasa +hotels +##zation +refuge +rent +retain +potentially +conferences +suburban +conducting +##tto +##tions +##tle +descended +massacre +##cal +ammunition +terrain +fork +souls +counts +chelsea +durham +drives +cab +##bank +perth +realizing +palestinian +finn +simpson +##dal +betty +##ule +moreover +particles +cardinals +tent +evaluation +extraordinary +##oid +inscription +##works +wednesday +chloe +maintains +panels +ashley +trucks +##nation +cluster +sunlight +strikes +zhang +##wing +dialect +canon +##ap +tucked +##ws +collecting +##mas +##can +##sville +maker +quoted +evan +franco +aria +buying +cleaning +eva +closet +provision +apollo +clinic +rat +##ez +necessarily +ac +##gle +##ising +venues +flipped +cent +spreading +trustees +checking +authorized +##sco +disappointed +##ado +notion +duration +trumpet +hesitated +topped +brussels +rolls +theoretical +hint +define +aggressive +repeat +wash +peaceful +optical +width +allegedly +mcdonald +strict +copyright +##illa +investors +mar +jam +witnesses +sounding +miranda +michelle +privacy +hugo +harmony +##pp +valid +lynn +glared +nina +102 +headquartered +diving +boarding +gibson +##ncy +albanian +marsh +routine +dealt +enhanced +er +intelligent +substance +targeted +enlisted +discovers +spinning +observations +pissed +smoking +rebecca +capitol +visa +varied +costume +seemingly +indies +compensation +surgeon +thursday +arsenal +westminster +suburbs +rid +anglican +##ridge +knots +foods +alumni +lighter +fraser +whoever +portal +scandal +##ray +gavin +advised +instructor +flooding +terrorist +##ale +teenage +interim +senses +duck +teen +thesis +abby +eager +overcome +##ile +newport +glenn +rises +shame +##cc +prompted +priority +forgot +bomber +nicolas +protective +360 +cartoon +katherine +breeze +lonely +trusted +henderson +richardson +relax +banner +candy +palms +remarkable +##rio +legends +cricketer +essay +ordained +edmund +rifles +trigger +##uri +##away +sail +alert +1830 +audiences +penn +sussex +siblings +pursued +indianapolis +resist +rosa +consequence +succeed +avoided +1845 +##ulation +inland +##tie +##nna +counsel +profession +chronicle +hurried +##una +eyebrow +eventual +bleeding +innovative +cure +##dom +committees +accounting +con +scope +hardy +heather +tenor +gut +herald +codes +tore +scales +wagon +##oo +luxury +tin +prefer +fountain +triangle +bonds +darling +convoy +dried +traced +beings +troy +accidentally +slam +findings +smelled +joey +lawyers +outcome +steep +bosnia +configuration +shifting +toll +brook +performers +lobby +philosophical +construct +shrine +aggregate +boot +cox +phenomenon +savage +insane +solely +reynolds +lifestyle +##ima +nationally +holdings +consideration +enable +edgar +mo +mama +##tein +fights +relegation +chances +atomic +hub +conjunction +awkward +reactions +currency +finale +kumar +underwent +steering +elaborate +gifts +comprising +melissa +veins +reasonable +sunshine +chi +solve +trails +inhabited +elimination +ethics +huh +ana +molly +consent +apartments +layout +marines +##ces +hunters +bulk +##oma +hometown +##wall +##mont +cracked +reads +neighbouring +withdrawn +admission +wingspan +damned +anthology +lancashire +brands +batting +forgive +cuban +awful +##lyn +104 +dimensions +imagination +##ade +dante +##ship +tracking +desperately +goalkeeper +##yne +groaned +workshops +confident +burton +gerald +milton +circus +uncertain +slope +copenhagen +sophia +fog +philosopher +portraits +accent +cycling +varying +gripped +larvae +garrett +specified +scotia +mature +luther +kurt +rap +##kes +aerial +750 +ferdinand +heated +es +transported +##shan +safely +nonetheless +##orn +##gal +motors +demanding +##sburg +startled +##brook +ally +generate +caps +ghana +stained +demo +mentions +beds +ap +afterward +diary +##bling +utility +##iro +richards +1837 +conspiracy +conscious +shining +footsteps +observer +cyprus +urged +loyalty +developer +probability +olive +upgraded +gym +miracle +insects +graves +1844 +ourselves +hydrogen +amazon +katie +tickets +poets +##pm +planes +##pan +prevention +witnessed +dense +jin +randy +tang +warehouse +monroe +bang +archived +elderly +investigations +alec +granite +mineral +conflicts +controlling +aboriginal +carlo +##zu +mechanics +stan +stark +rhode +skirt +est +##berry +bombs +respected +##horn +imposed +limestone +deny +nominee +memphis +grabbing +disabled +##als +amusement +aa +frankfurt +corn +referendum +varies +slowed +disk +firms +unconscious +incredible +clue +sue +##zhou +twist +##cio +joins +idaho +chad +developers +computing +destroyer +103 +mortal +tucker +kingston +choices +yu +carson +1800 +os +whitney +geneva +pretend +dimension +staged +plateau +maya +##une +freestyle +##bc +rovers +hiv +##ids +tristan +classroom +prospect +##hus +honestly +diploma +lied +thermal +auxiliary +feast +unlikely +iata +##tel +morocco +pounding +treasury +lithuania +considerably +1841 +dish +1812 +geological +matching +stumbled +destroying +marched +brien +advances +cake +nicole +belle +settling +measuring +directing +##mie +tuesday +bassist +capabilities +stunned +fraud +torpedo +##list +##phone +anton +wisdom +surveillance +ruined +##ulate +lawsuit +healthcare +theorem +halls +trend +aka +horizontal +dozens +acquire +lasting +swim +hawk +gorgeous +fees +vicinity +decrease +adoption +tactics +##ography +pakistani +##ole +draws +##hall +willie +burke +heath +algorithm +integral +powder +elliott +brigadier +jackie +tate +varieties +darker +##cho +lately +cigarette +specimens +adds +##ree +##ensis +##inger +exploded +finalist +cia +murders +wilderness +arguments +nicknamed +acceptance +onwards +manufacture +robertson +jets +tampa +enterprises +blog +loudly +composers +nominations +1838 +ai +malta +inquiry +automobile +hosting +viii +rays +tilted +grief +museums +strategies +furious +euro +equality +cohen +poison +surrey +wireless +governed +ridiculous +moses +##esh +##room +vanished +##ito +barnes +attract +morrison +istanbul +##iness +absent +rotation +petition +janet +##logical +satisfaction +custody +deliberately +observatory +comedian +surfaces +pinyin +novelist +strictly +canterbury +oslo +monks +embrace +ibm +jealous +photograph +continent +dorothy +marina +doc +excess +holden +allegations +explaining +stack +avoiding +lance +storyline +majesty +poorly +spike +dos +bradford +raven +travis +classics +proven +voltage +pillow +fists +butt +1842 +interpreted +##car +1839 +gage +telegraph +lens +promising +expelled +casual +collector +zones +##min +silly +nintendo +##kh +##bra +downstairs +chef +suspicious +afl +flies +vacant +uganda +pregnancy +condemned +lutheran +estimates +cheap +decree +saxon +proximity +stripped +idiot +deposits +contrary +presenter +magnus +glacier +im +offense +edwin +##ori +upright +##long +bolt +##ois +toss +geographical +##izes +environments +delicate +marking +abstract +xavier +nails +windsor +plantation +occurring +equity +saskatchewan +fears +drifted +sequences +vegetation +revolt +##stic +1843 +sooner +fusion +opposing +nato +skating +1836 +secretly +ruin +lease +##oc +edit +##nne +flora +anxiety +ruby +##ological +##mia +tel +bout +taxi +emmy +frost +rainbow +compounds +foundations +rainfall +assassination +nightmare +dominican +##win +achievements +deserve +orlando +intact +armenia +##nte +calgary +valentine +106 +marion +proclaimed +theodore +bells +courtyard +thigh +gonzalez +console +troop +minimal +monte +everyday +##ence +##if +supporter +terrorism +buck +openly +presbyterian +activists +carpet +##iers +rubbing +uprising +##yi +cute +conceived +legally +##cht +millennium +cello +velocity +ji +rescued +cardiff +1835 +rex +concentrate +senators +beard +rendered +glowing +battalions +scouts +competitors +sculptor +catalogue +arctic +ion +raja +bicycle +wow +glancing +lawn +##woman +gentleman +lighthouse +publish +predicted +calculated +##val +variants +##gne +strain +##ui +winston +deceased +##nus +touchdowns +brady +caleb +sinking +echoed +crush +hon +blessed +protagonist +hayes +endangered +magnitude +editors +##tine +estimate +responsibilities +##mel +backup +laying +consumed +sealed +zurich +lovers +frustrated +##eau +ahmed +kicking +mit +treasurer +1832 +biblical +refuse +terrified +pump +agrees +genuine +imprisonment +refuses +plymouth +##hen +lou +##nen +tara +trembling +antarctic +ton +learns +##tas +crap +crucial +faction +atop +##borough +wrap +lancaster +odds +hopkins +erik +lyon +##eon +bros +##ode +snap +locality +tips +empress +crowned +cal +acclaimed +chuckled +##ory +clara +sends +mild +towel +##fl +##day +##а +wishing +assuming +interviewed +##bal +##die +interactions +eden +cups +helena +##lf +indie +beck +##fire +batteries +filipino +wizard +parted +##lam +traces +##born +rows +idol +albany +delegates +##ees +##sar +discussions +##ex +notre +instructed +belgrade +highways +suggestion +lauren +possess +orientation +alexandria +abdul +beats +salary +reunion +ludwig +alright +wagner +intimate +pockets +slovenia +hugged +brighton +merchants +cruel +stole +trek +slopes +repairs +enrollment +politically +underlying +promotional +counting +boeing +##bb +isabella +naming +##и +keen +bacteria +listing +separately +belfast +ussr +450 +lithuanian +anybody +ribs +sphere +martinez +cock +embarrassed +proposals +fragments +nationals +##fs +##wski +premises +fin +1500 +alpine +matched +freely +bounded +jace +sleeve +##af +gaming +pier +populated +evident +##like +frances +flooded +##dle +frightened +pour +trainer +framed +visitor +challenging +pig +wickets +##fold +infected +email +##pes +arose +##aw +reward +ecuador +oblast +vale +ch +shuttle +##usa +bach +rankings +forbidden +cornwall +accordance +salem +consumers +bruno +fantastic +toes +machinery +resolved +julius +remembering +propaganda +iceland +bombardment +tide +contacts +wives +##rah +concerto +macdonald +albania +implement +daisy +tapped +sudan +helmet +angela +mistress +##lic +crop +sunk +finest +##craft +hostile +##ute +##tsu +boxer +fr +paths +adjusted +habit +ballot +supervision +soprano +##zen +bullets +wicked +sunset +regiments +disappear +lamp +performs +app +##gia +##oa +rabbit +digging +incidents +entries +##cion +dishes +##oi +introducing +##ati +##fied +freshman +slot +jill +tackles +baroque +backs +##iest +lone +sponsor +destiny +altogether +convert +##aro +consensus +shapes +demonstration +basically +feminist +auction +artifacts +##bing +strongest +twitter +halifax +2019 +allmusic +mighty +smallest +precise +alexandra +viola +##los +##ille +manuscripts +##illo +dancers +ari +managers +monuments +blades +barracks +springfield +maiden +consolidated +electron +##end +berry +airing +wheat +nobel +inclusion +blair +payments +geography +bee +cc +eleanor +react +##hurst +afc +manitoba +##yu +su +lineup +fitness +recreational +investments +airborne +disappointment +##dis +edmonton +viewing +##row +renovation +##cast +infant +bankruptcy +roses +aftermath +pavilion +##yer +carpenter +withdrawal +ladder +##hy +discussing +popped +reliable +agreements +rochester +##abad +curves +bombers +220 +rao +reverend +decreased +choosing +107 +stiff +consulting +naples +crawford +tracy +ka +ribbon +cops +##lee +crushed +deciding +unified +teenager +accepting +flagship +explorer +poles +sanchez +inspection +revived +skilled +induced +exchanged +flee +locals +tragedy +swallow +loading +hanna +demonstrate +##ela +salvador +flown +contestants +civilization +##ines +wanna +rhodes +fletcher +hector +knocking +considers +##ough +nash +mechanisms +sensed +mentally +walt +unclear +##eus +renovated +madame +##cks +crews +governmental +##hin +undertaken +monkey +##ben +##ato +fatal +armored +copa +caves +governance +grasp +perception +certification +froze +damp +tugged +wyoming +##rg +##ero +newman +##lor +nerves +curiosity +graph +115 +##ami +withdraw +tunnels +dull +meredith +moss +exhibits +neighbors +communicate +accuracy +explored +raiders +republicans +secular +kat +superman +penny +criticised +##tch +freed +update +conviction +wade +ham +likewise +delegation +gotta +doll +promises +technological +myth +nationality +resolve +convent +##mark +sharon +dig +sip +coordinator +entrepreneur +fold +##dine +capability +councillor +synonym +blown +swan +cursed +1815 +jonas +haired +sofa +canvas +keeper +rivalry +##hart +rapper +speedway +swords +postal +maxwell +estonia +potter +recurring +##nn +##ave +errors +##oni +cognitive +1834 +##² +claws +nadu +roberto +bce +wrestler +ellie +##ations +infinite +ink +##tia +presumably +finite +staircase +108 +noel +patricia +nacional +##cation +chill +eternal +tu +preventing +prussia +fossil +limbs +##logist +ernst +frog +perez +rene +##ace +pizza +prussian +##ios +##vy +molecules +regulatory +answering +opinions +sworn +lengths +supposedly +hypothesis +upward +habitats +seating +ancestors +drank +yield +hd +synthesis +researcher +modest +##var +mothers +peered +voluntary +homeland +##the +acclaim +##igan +static +valve +luxembourg +alto +carroll +fe +receptor +norton +ambulance +##tian +johnston +catholics +depicting +jointly +elephant +gloria +mentor +badge +ahmad +distinguish +remarked +councils +precisely +allison +advancing +detection +crowded +##10 +cooperative +ankle +mercedes +dagger +surrendered +pollution +commit +subway +jeffrey +lesson +sculptures +provider +##fication +membrane +timothy +rectangular +fiscal +heating +teammate +basket +particle +anonymous +deployment +##ple +missiles +courthouse +proportion +shoe +sec +##ller +complaints +forbes +blacks +abandon +remind +sizes +overwhelming +autobiography +natalie +##awa +risks +contestant +countryside +babies +scorer +invaded +enclosed +proceed +hurling +disorders +##cu +reflecting +continuously +cruiser +graduates +freeway +investigated +ore +deserved +maid +blocking +phillip +jorge +shakes +dove +mann +variables +lacked +burden +accompanying +que +consistently +organizing +provisional +complained +endless +##rm +tubes +juice +georges +krishna +mick +labels +thriller +##uch +laps +arcade +sage +snail +##table +shannon +fi +laurence +seoul +vacation +presenting +hire +churchill +surprisingly +prohibited +savannah +technically +##oli +170 +##lessly +testimony +suited +speeds +toys +romans +mlb +flowering +measurement +talented +kay +settings +charleston +expectations +shattered +achieving +triumph +ceremonies +portsmouth +lanes +mandatory +loser +stretching +cologne +realizes +seventy +cornell +careers +webb +##ulating +americas +budapest +ava +suspicion +##ison +yo +conrad +##hai +sterling +jessie +rector +##az +1831 +transform +organize +loans +christine +volcanic +warrant +slender +summers +subfamily +newer +danced +dynamics +rhine +proceeds +heinrich +gastropod +commands +sings +facilitate +easter +ra +positioned +responses +expense +fruits +yanked +imported +25th +velvet +vic +primitive +tribune +baldwin +neighbourhood +donna +rip +hay +pr +##uro +1814 +espn +welcomed +##aria +qualifier +glare +highland +timing +##cted +shells +eased +geometry +louder +exciting +slovakia +##sion +##iz +##lot +savings +prairie +##ques +marching +rafael +tonnes +##lled +curtain +preceding +shy +heal +greene +worthy +##pot +detachment +bury +sherman +##eck +reinforced +seeks +bottles +contracted +duchess +outfit +walsh +##sc +mickey +##ase +geoffrey +archer +squeeze +dawson +eliminate +invention +##enberg +neal +##eth +stance +dealer +coral +maple +retire +polo +simplified +##ht +1833 +hid +watts +backwards +jules +##oke +genesis +mt +frames +rebounds +burma +woodland +moist +santos +whispers +drained +subspecies +##aa +streaming +ulster +burnt +correspondence +maternal +gerard +denis +stealing +##load +genius +duchy +##oria +inaugurated +momentum +suits +placement +sovereign +clause +thames +##hara +confederation +reservation +sketch +yankees +lets +rotten +charm +hal +verses +ultra +commercially +dot +salon +citation +adopt +winnipeg +mist +allocated +cairo +##boy +jenkins +interference +objectives +##wind +1820 +portfolio +armoured +sectors +##eh +initiatives +##world +integrity +exercises +robe +tap +ab +gazed +##tones +distracted +rulers +111 +favorable +jerome +tended +cart +factories +##eri +diplomat +valued +gravel +charitable +##try +calvin +exploring +chang +shepherd +terrace +pdf +pupil +##ural +reflects +ups +##rch +governors +shelf +depths +##nberg +trailed +crest +tackle +##nian +##ats +hatred +##kai +clare +makers +ethiopia +longtime +detected +embedded +lacking +slapped +rely +thomson +anticipation +iso +morton +successive +agnes +screenwriter +straightened +philippe +playwright +haunted +licence +iris +intentions +sutton +112 +logical +correctly +##weight +branded +licked +tipped +silva +ricky +narrator +requests +##ents +greeted +supernatural +cow +##wald +lung +refusing +employer +strait +gaelic +liner +##piece +zoe +sabha +##mba +driveway +harvest +prints +bates +reluctantly +threshold +algebra +ira +wherever +coupled +240 +assumption +picks +##air +designers +raids +gentlemen +##ean +roller +blowing +leipzig +locks +screw +dressing +strand +##lings +scar +dwarf +depicts +##nu +nods +##mine +differ +boris +##eur +yuan +flip +##gie +mob +invested +questioning +applying +##ture +shout +##sel +gameplay +blamed +illustrations +bothered +weakness +rehabilitation +##of +##zes +envelope +rumors +miners +leicester +subtle +kerry +##ico +ferguson +##fu +premiership +ne +##cat +bengali +prof +catches +remnants +dana +##rily +shouting +presidents +baltic +ought +ghosts +dances +sailors +shirley +fancy +dominic +##bie +madonna +##rick +bark +buttons +gymnasium +ashes +liver +toby +oath +providence +doyle +evangelical +nixon +cement +carnegie +embarked +hatch +surroundings +guarantee +needing +pirate +essence +##bee +filter +crane +hammond +projected +immune +percy +twelfth +##ult +regent +doctoral +damon +mikhail +##ichi +lu +critically +elect +realised +abortion +acute +screening +mythology +steadily +##fc +frown +nottingham +kirk +wa +minneapolis +##rra +module +algeria +mc +nautical +encounters +surprising +statues +availability +shirts +pie +alma +brows +munster +mack +soup +crater +tornado +sanskrit +cedar +explosive +bordered +dixon +planets +stamp +exam +happily +##bble +carriers +kidnapped +##vis +accommodation +emigrated +##met +knockout +correspondent +violation +profits +peaks +lang +specimen +agenda +ancestry +pottery +spelling +equations +obtaining +ki +linking +1825 +debris +asylum +##20 +buddhism +teddy +##ants +gazette +##nger +##sse +dental +eligibility +utc +fathers +averaged +zimbabwe +francesco +coloured +hissed +translator +lynch +mandate +humanities +mackenzie +uniforms +lin +##iana +##gio +asset +mhz +fitting +samantha +genera +wei +rim +beloved +shark +riot +entities +expressions +indo +carmen +slipping +owing +abbot +neighbor +sidney +##av +rats +recommendations +encouraging +squadrons +anticipated +commanders +conquered +##oto +donations +diagnosed +##mond +divide +##iva +guessed +decoration +vernon +auditorium +revelation +conversations +##kers +##power +herzegovina +dash +alike +protested +lateral +herman +accredited +mg +##gent +freeman +mel +fiji +crow +crimson +##rine +livestock +##pped +humanitarian +bored +oz +whip +##lene +##ali +legitimate +alter +grinning +spelled +anxious +oriental +wesley +##nin +##hole +carnival +controller +detect +##ssa +bowed +educator +kosovo +macedonia +##sin +occupy +mastering +stephanie +janeiro +para +unaware +nurses +noon +135 +cam +hopefully +ranger +combine +sociology +polar +rica +##eer +neill +##sman +holocaust +##ip +doubled +lust +1828 +109 +decent +cooling +unveiled +##card +1829 +nsw +homer +chapman +meyer +##gin +dive +mae +reagan +expertise +##gled +darwin +brooke +sided +prosecution +investigating +comprised +petroleum +genres +reluctant +differently +trilogy +johns +vegetables +corpse +highlighted +lounge +pension +unsuccessfully +elegant +aided +ivory +beatles +amelia +cain +dubai +sunny +immigrant +babe +click +##nder +underwater +pepper +combining +mumbled +atlas +horns +accessed +ballad +physicians +homeless +gestured +rpm +freak +louisville +corporations +patriots +prizes +rational +warn +modes +decorative +overnight +din +troubled +phantom +##ort +monarch +sheer +##dorf +generals +guidelines +organs +addresses +##zon +enhance +curling +parishes +cord +##kie +linux +caesar +deutsche +bavaria +##bia +coleman +cyclone +##eria +bacon +petty +##yama +##old +hampton +diagnosis +1824 +throws +complexity +rita +disputed +##₃ +pablo +##sch +marketed +trafficking +##ulus +examine +plague +formats +##oh +vault +faithful +##bourne +webster +##ox +highlights +##ient +##ann +phones +vacuum +sandwich +modeling +##gated +bolivia +clergy +qualities +isabel +##nas +##ars +wears +screams +reunited +annoyed +bra +##ancy +##rate +differential +transmitter +tattoo +container +poker +##och +excessive +resides +cowboys +##tum +augustus +trash +providers +statute +retreated +balcony +reversed +void +storey +preceded +masses +leap +laughs +neighborhoods +wards +schemes +falcon +santo +battlefield +pad +ronnie +thread +lesbian +venus +##dian +beg +sandstone +daylight +punched +gwen +analog +stroked +wwe +acceptable +measurements +dec +toxic +##kel +adequate +surgical +economist +parameters +varsity +##sberg +quantity +ella +##chy +##rton +countess +generating +precision +diamonds +expressway +ga +##ı +1821 +uruguay +talents +galleries +expenses +scanned +colleague +outlets +ryder +lucien +##ila +paramount +##bon +syracuse +dim +fangs +gown +sweep +##sie +toyota +missionaries +websites +##nsis +sentences +adviser +val +trademark +spells +##plane +patience +starter +slim +##borg +toe +incredibly +shoots +elliot +nobility +##wyn +cowboy +endorsed +gardner +tendency +persuaded +organisms +emissions +kazakhstan +amused +boring +chips +themed +##hand +llc +constantinople +chasing +systematic +guatemala +borrowed +erin +carey +##hard +highlands +struggles +1810 +##ifying +##ced +wong +exceptions +develops +enlarged +kindergarten +castro +##ern +##rina +leigh +zombie +juvenile +##most +consul +##nar +sailor +hyde +clarence +intensive +pinned +nasty +useless +jung +clayton +stuffed +exceptional +ix +apostolic +230 +transactions +##dge +exempt +swinging +cove +religions +##ash +shields +dairy +bypass +190 +pursuing +bug +joyce +bombay +chassis +southampton +chat +interact +redesignated +##pen +nascar +pray +salmon +rigid +regained +malaysian +grim +publicity +constituted +capturing +toilet +delegate +purely +tray +drift +loosely +striker +weakened +trinidad +mitch +itv +defines +transmitted +ming +scarlet +nodding +fitzgerald +fu +narrowly +sp +tooth +standings +virtue +##₁ +##wara +##cting +chateau +gloves +lid +##nel +hurting +conservatory +##pel +sinclair +reopened +sympathy +nigerian +strode +advocated +optional +chronic +discharge +##rc +suck +compatible +laurel +stella +shi +fails +wage +dodge +128 +informal +sorts +levi +buddha +villagers +##aka +chronicles +heavier +summoned +gateway +3000 +eleventh +jewelry +translations +accordingly +seas +##ency +fiber +pyramid +cubic +dragging +##ista +caring +##ops +android +contacted +lunar +##dt +kai +lisbon +patted +1826 +sacramento +theft +madagascar +subtropical +disputes +ta +holidays +piper +willow +mare +cane +itunes +newfoundland +benny +companions +dong +raj +observe +roar +charming +plaque +tibetan +fossils +enacted +manning +bubble +tina +tanzania +##eda +##hir +funk +swamp +deputies +cloak +ufc +scenario +par +scratch +metals +anthem +guru +engaging +specially +##boat +dialects +nineteen +cecil +duet +disability +messenger +unofficial +##lies +defunct +eds +moonlight +drainage +surname +puzzle +honda +switching +conservatives +mammals +knox +broadcaster +sidewalk +cope +##ried +benson +princes +peterson +##sal +bedford +sharks +eli +wreck +alberto +gasp +archaeology +lgbt +teaches +securities +madness +compromise +waving +coordination +davidson +visions +leased +possibilities +eighty +jun +fernandez +enthusiasm +assassin +sponsorship +reviewer +kingdoms +estonian +laboratories +##fy +##nal +applies +verb +celebrations +##zzo +rowing +lightweight +sadness +submit +mvp +balanced +dude +##vas +explicitly +metric +magnificent +mound +brett +mohammad +mistakes +irregular +##hing +##ass +sanders +betrayed +shipped +surge +##enburg +reporters +termed +georg +pity +verbal +bulls +abbreviated +enabling +appealed +##are +##atic +sicily +sting +heel +sweetheart +bart +spacecraft +brutal +monarchy +##tter +aberdeen +cameo +diane +##ub +survivor +clyde +##aries +complaint +##makers +clarinet +delicious +chilean +karnataka +coordinates +1818 +panties +##rst +pretending +ar +dramatically +kiev +bella +tends +distances +113 +catalog +launching +instances +telecommunications +portable +lindsay +vatican +##eim +angles +aliens +marker +stint +screens +bolton +##rne +judy +wool +benedict +plasma +europa +spark +imaging +filmmaker +swiftly +##een +contributor +##nor +opted +stamps +apologize +financing +butter +gideon +sophisticated +alignment +avery +chemicals +yearly +speculation +prominence +professionally +##ils +immortal +institutional +inception +wrists +identifying +tribunal +derives +gains +##wo +papal +preference +linguistic +vince +operative +brewery +##ont +unemployment +boyd +##ured +##outs +albeit +prophet +1813 +bi +##rr +##face +##rad +quarterly +asteroid +cleaned +radius +temper +##llen +telugu +jerk +viscount +menu +##ote +glimpse +##aya +yacht +hawaiian +baden +##rl +laptop +readily +##gu +monetary +offshore +scots +watches +##yang +##arian +upgrade +needle +xbox +lea +encyclopedia +flank +fingertips +##pus +delight +teachings +confirm +roth +beaches +midway +winters +##iah +teasing +daytime +beverly +gambling +bonnie +##backs +regulated +clement +hermann +tricks +knot +##shing +##uring +##vre +detached +ecological +owed +specialty +byron +inventor +bats +stays +screened +unesco +midland +trim +affection +##ander +##rry +jess +thoroughly +feedback +##uma +chennai +strained +heartbeat +wrapping +overtime +pleaded +##sworth +mon +leisure +oclc +##tate +##ele +feathers +angelo +thirds +nuts +surveys +clever +gill +commentator +##dos +darren +rides +gibraltar +##nc +##mu +dissolution +dedication +shin +meals +saddle +elvis +reds +chaired +taller +appreciation +functioning +niece +favored +advocacy +robbie +criminals +suffolk +yugoslav +passport +constable +congressman +hastings +vera +##rov +consecrated +sparks +ecclesiastical +confined +##ovich +muller +floyd +nora +1822 +paved +1827 +cumberland +ned +saga +spiral +##flow +appreciated +yi +collaborative +treating +similarities +feminine +finishes +##ib +jade +import +##nse +##hot +champagne +mice +securing +celebrities +helsinki +attributes +##gos +cousins +phases +ache +lucia +gandhi +submission +vicar +spear +shine +tasmania +biting +detention +constitute +tighter +seasonal +##gus +terrestrial +matthews +##oka +effectiveness +parody +philharmonic +##onic +1816 +strangers +encoded +consortium +guaranteed +regards +shifts +tortured +collision +supervisor +inform +broader +insight +theaters +armour +emeritus +blink +incorporates +mapping +##50 +##ein +handball +flexible +##nta +substantially +generous +thief +##own +carr +loses +1793 +prose +ucla +romeo +generic +metallic +realization +damages +mk +commissioners +zach +default +##ther +helicopters +lengthy +stems +spa +partnered +spectators +rogue +indication +penalties +teresa +1801 +sen +##tric +dalton +##wich +irving +photographic +##vey +dell +deaf +peters +excluded +unsure +##vable +patterson +crawled +##zio +resided +whipped +latvia +slower +ecole +pipes +employers +maharashtra +comparable +va +textile +pageant +##gel +alphabet +binary +irrigation +chartered +choked +antoine +offs +waking +supplement +##wen +quantities +demolition +regain +locate +urdu +folks +alt +114 +##mc +scary +andreas +whites +##ava +classrooms +mw +aesthetic +publishes +valleys +guides +cubs +johannes +bryant +conventions +affecting +##itt +drain +awesome +isolation +prosecutor +ambitious +apology +captive +downs +atmospheric +lorenzo +aisle +beef +foul +##onia +kidding +composite +disturbed +illusion +natives +##ffer +emi +rockets +riverside +wartime +painters +adolf +melted +##ail +uncertainty +simulation +hawks +progressed +meantime +builder +spray +breach +unhappy +regina +russians +##urg +determining +##tation +tram +1806 +##quin +aging +##12 +1823 +garion +rented +mister +diaz +terminated +clip +1817 +depend +nervously +disco +owe +defenders +shiva +notorious +disbelief +shiny +worcester +##gation +##yr +trailing +undertook +islander +belarus +limitations +watershed +fuller +overlooking +utilized +raphael +1819 +synthetic +breakdown +klein +##nate +moaned +memoir +lamb +practicing +##erly +cellular +arrows +exotic +##graphy +witches +117 +charted +rey +hut +hierarchy +subdivision +freshwater +giuseppe +aloud +reyes +qatar +marty +sideways +utterly +sexually +jude +prayers +mccarthy +softball +blend +damien +##gging +##metric +wholly +erupted +lebanese +negro +revenues +tasted +comparative +teamed +transaction +labeled +maori +sovereignty +parkway +trauma +gran +malay +121 +advancement +descendant +2020 +buzz +salvation +inventory +symbolic +##making +antarctica +mps +##gas +##bro +mohammed +myanmar +holt +submarines +tones +##lman +locker +patriarch +bangkok +emerson +remarks +predators +kin +afghan +confession +norwich +rental +emerge +advantages +##zel +rca +##hold +shortened +storms +aidan +##matic +autonomy +compliance +##quet +dudley +atp +##osis +1803 +motto +documentation +summary +professors +spectacular +christina +archdiocese +flashing +innocence +remake +##dell +psychic +reef +scare +employ +rs +sticks +meg +gus +leans +##ude +accompany +bergen +tomas +##iko +doom +wages +pools +##nch +##bes +breasts +scholarly +alison +outline +brittany +breakthrough +willis +realistic +##cut +##boro +competitor +##stan +pike +picnic +icon +designing +commercials +washing +villain +skiing +micro +costumes +auburn +halted +executives +##hat +logistics +cycles +vowel +applicable +barrett +exclaimed +eurovision +eternity +ramon +##umi +##lls +modifications +sweeping +disgust +##uck +torch +aviv +ensuring +rude +dusty +sonic +donovan +outskirts +cu +pathway +##band +##gun +##lines +disciplines +acids +cadet +paired +##40 +sketches +##sive +marriages +##⁺ +folding +peers +slovak +implies +admired +##beck +1880s +leopold +instinct +attained +weston +megan +horace +##ination +dorsal +ingredients +evolutionary +##its +complications +deity +lethal +brushing +levy +deserted +institutes +posthumously +delivering +telescope +coronation +motivated +rapids +luc +flicked +pays +volcano +tanner +weighed +##nica +crowds +frankie +gifted +addressing +granddaughter +winding +##rna +constantine +gomez +##front +landscapes +rudolf +anthropology +slate +werewolf +##lio +astronomy +circa +rouge +dreaming +sack +knelt +drowned +naomi +prolific +tracked +freezing +herb +##dium +agony +randall +twisting +wendy +deposit +touches +vein +wheeler +##bbled +##bor +batted +retaining +tire +presently +compare +specification +daemon +nigel +##grave +merry +recommendation +czechoslovakia +sandra +ng +roma +##sts +lambert +inheritance +sheikh +winchester +cries +examining +##yle +comeback +cuisine +nave +##iv +ko +retrieve +tomatoes +barker +polished +defining +irene +lantern +personalities +begging +tract +swore +1809 +175 +##gic +omaha +brotherhood +##rley +haiti +##ots +exeter +##ete +##zia +steele +dumb +pearson +210 +surveyed +elisabeth +trends +##ef +fritz +##rf +premium +bugs +fraction +calmly +viking +##birds +tug +inserted +unusually +##ield +confronted +distress +crashing +brent +turks +resign +##olo +cambodia +gabe +sauce +##kal +evelyn +116 +extant +clusters +quarry +teenagers +luna +##lers +##ister +affiliation +drill +##ashi +panthers +scenic +libya +anita +strengthen +inscriptions +##cated +lace +sued +judith +riots +##uted +mint +##eta +preparations +midst +dub +challenger +##vich +mock +cf +displaced +wicket +breaths +enables +schmidt +analyst +##lum +ag +highlight +automotive +axe +josef +newark +sufficiently +resembles +50th +##pal +flushed +mum +traits +##ante +commodore +incomplete +warming +titular +ceremonial +ethical +118 +celebrating +eighteenth +cao +lima +medalist +mobility +strips +snakes +##city +miniature +zagreb +barton +escapes +umbrella +automated +doubted +differs +cooled +georgetown +dresden +cooked +fade +wyatt +rna +jacobs +carlton +abundant +stereo +boost +madras +inning +##hia +spur +ip +malayalam +begged +osaka +groan +escaping +charging +dose +vista +##aj +bud +papa +communists +advocates +edged +tri +##cent +resemble +peaking +necklace +fried +montenegro +saxony +goose +glances +stuttgart +curator +recruit +grocery +sympathetic +##tting +##fort +127 +lotus +randolph +ancestor +##rand +succeeding +jupiter +1798 +macedonian +##heads +hiking +1808 +handing +fischer +##itive +garbage +node +##pies +prone +singular +papua +inclined +attractions +italia +pouring +motioned +grandma +garnered +jacksonville +corp +ego +ringing +aluminum +##hausen +ordering +##foot +drawer +traders +synagogue +##play +##kawa +resistant +wandering +fragile +fiona +teased +var +hardcore +soaked +jubilee +decisive +exposition +mercer +poster +valencia +hale +kuwait +1811 +##ises +##wr +##eed +tavern +gamma +122 +johan +##uer +airways +amino +gil +##ury +vocational +domains +torres +##sp +generator +folklore +outcomes +##keeper +canberra +shooter +fl +beams +confrontation +##lling +##gram +feb +aligned +forestry +pipeline +jax +motorway +conception +decay +##tos +coffin +##cott +stalin +1805 +escorted +minded +##nam +sitcom +purchasing +twilight +veronica +additions +passive +tensions +straw +123 +frequencies +1804 +refugee +cultivation +##iate +christie +clary +bulletin +crept +disposal +##rich +##zong +processor +crescent +##rol +bmw +emphasized +whale +nazis +aurora +##eng +dwelling +hauled +sponsors +toledo +mega +ideology +theatres +tessa +cerambycidae +saves +turtle +cone +suspects +kara +rusty +yelling +greeks +mozart +shades +cocked +participant +##tro +shire +spit +freeze +necessity +##cos +inmates +nielsen +councillors +loaned +uncommon +omar +peasants +botanical +offspring +daniels +formations +jokes +1794 +pioneers +sigma +licensing +##sus +wheelchair +polite +1807 +liquor +pratt +trustee +##uta +forewings +balloon +##zz +kilometre +camping +explicit +casually +shawn +foolish +teammates +nm +hassan +carrie +judged +satisfy +vanessa +knives +selective +cnn +flowed +##lice +eclipse +stressed +eliza +mathematician +cease +cultivated +##roy +commissions +browns +##ania +destroyers +sheridan +meadow +##rius +minerals +##cial +downstream +clash +gram +memoirs +ventures +baha +seymour +archie +midlands +edith +fare +flynn +invite +canceled +tiles +stabbed +boulder +incorporate +amended +camden +facial +mollusk +unreleased +descriptions +yoga +grabs +550 +raises +ramp +shiver +##rose +coined +pioneering +tunes +qing +warwick +tops +119 +melanie +giles +##rous +wandered +##inal +annexed +nov +30th +unnamed +##ished +organizational +airplane +normandy +stoke +whistle +blessing +violations +chased +holders +shotgun +##ctic +outlet +reactor +##vik +tires +tearing +shores +fortified +mascot +constituencies +nc +columnist +productive +tibet +##rta +lineage +hooked +oct +tapes +judging +cody +##gger +hansen +kashmir +triggered +##eva +solved +cliffs +##tree +resisted +anatomy +protesters +transparent +implied +##iga +injection +mattress +excluding +##mbo +defenses +helpless +devotion +##elli +growl +liberals +weber +phenomena +atoms +plug +##iff +mortality +apprentice +howe +convincing +aaa +swimmer +barber +leone +promptly +sodium +def +nowadays +arise +##oning +gloucester +corrected +dignity +norm +erie +##ders +elders +evacuated +sylvia +compression +##yar +hartford +pose +backpack +reasoning +accepts +24th +wipe +millimetres +marcel +##oda +dodgers +albion +1790 +overwhelmed +aerospace +oaks +1795 +showcase +acknowledge +recovering +nolan +ashe +hurts +geology +fashioned +disappearance +farewell +swollen +shrug +marquis +wimbledon +124 +rue +1792 +commemorate +reduces +experiencing +inevitable +calcutta +intel +##court +murderer +sticking +fisheries +imagery +bloom +280 +brake +##inus +gustav +hesitation +memorable +po +viral +beans +accidents +tunisia +antenna +spilled +consort +treatments +aye +perimeter +##gard +donation +hostage +migrated +banker +addiction +apex +lil +trout +##ously +conscience +##nova +rams +sands +genome +passionate +troubles +##lets +##set +amid +##ibility +##ret +higgins +exceed +vikings +##vie +payne +##zan +muscular +##ste +defendant +sucking +##wal +ibrahim +fuselage +claudia +vfl +europeans +snails +interval +##garh +preparatory +statewide +tasked +lacrosse +viktor +##lation +angola +##hra +flint +implications +employs +teens +patrons +stall +weekends +barriers +scrambled +nucleus +tehran +jenna +parsons +lifelong +robots +displacement +5000 +##bles +precipitation +##gt +knuckles +clutched +1802 +marrying +ecology +marx +accusations +declare +scars +kolkata +mat +meadows +bermuda +skeleton +finalists +vintage +crawl +coordinate +affects +subjected +orchestral +mistaken +##tc +mirrors +dipped +relied +260 +arches +candle +##nick +incorporating +wildly +fond +basilica +owl +fringe +rituals +whispering +stirred +feud +tertiary +slick +goat +honorable +whereby +skip +ricardo +stripes +parachute +adjoining +submerged +synthesizer +##gren +intend +positively +ninety +phi +beaver +partition +fellows +alexis +prohibition +carlisle +bizarre +fraternity +##bre +doubts +icy +cbc +aquatic +sneak +sonny +combines +airports +crude +supervised +spatial +merge +alfonso +##bic +corrupt +scan +undergo +##ams +disabilities +colombian +comparing +dolphins +perkins +##lish +reprinted +unanimous +bounced +hairs +underworld +midwest +semester +bucket +paperback +miniseries +coventry +demise +##leigh +demonstrations +sensor +rotating +yan +##hler +arrange +soils +##idge +hyderabad +labs +##dr +brakes +grandchildren +##nde +negotiated +rover +ferrari +continuation +directorate +augusta +stevenson +counterpart +gore +##rda +nursery +rican +ave +collectively +broadly +pastoral +repertoire +asserted +discovering +nordic +styled +fiba +cunningham +harley +middlesex +survives +tumor +tempo +zack +aiming +lok +urgent +##rade +##nto +devils +##ement +contractor +turin +##wl +##ool +bliss +repaired +simmons +moan +astronomical +cr +negotiate +lyric +1890s +lara +bred +clad +angus +pbs +##ience +engineered +posed +##lk +hernandez +possessions +elbows +psychiatric +strokes +confluence +electorate +lifts +campuses +lava +alps +##ep +##ution +##date +physicist +woody +##page +##ographic +##itis +juliet +reformation +sparhawk +320 +complement +suppressed +jewel +##½ +floated +##kas +continuity +sadly +##ische +inability +melting +scanning +paula +flour +judaism +safer +vague +##lm +solving +curb +##stown +financially +gable +bees +expired +miserable +cassidy +dominion +1789 +cupped +145 +robbery +facto +amos +warden +resume +tallest +marvin +ing +pounded +usd +declaring +gasoline +##aux +darkened +270 +650 +sophomore +##mere +erection +gossip +televised +risen +dial +##eu +pillars +##link +passages +profound +##tina +arabian +ashton +silicon +nail +##ead +##lated +##wer +##hardt +fleming +firearms +ducked +circuits +blows +waterloo +titans +##lina +atom +fireplace +cheshire +financed +activation +algorithms +##zzi +constituent +catcher +cherokee +partnerships +sexuality +platoon +tragic +vivian +guarded +whiskey +meditation +poetic +##late +##nga +##ake +porto +listeners +dominance +kendra +mona +chandler +factions +22nd +salisbury +attitudes +derivative +##ido +##haus +intake +paced +javier +illustrator +barrels +bias +cockpit +burnett +dreamed +ensuing +##anda +receptors +someday +hawkins +mattered +##lal +slavic +1799 +jesuit +cameroon +wasted +tai +wax +lowering +victorious +freaking +outright +hancock +librarian +sensing +bald +calcium +myers +tablet +announcing +barack +shipyard +pharmaceutical +##uan +greenwich +flush +medley +patches +wolfgang +pt +speeches +acquiring +exams +nikolai +##gg +hayden +kannada +##type +reilly +##pt +waitress +abdomen +devastated +capped +pseudonym +pharmacy +fulfill +paraguay +1796 +clicked +##trom +archipelago +syndicated +##hman +lumber +orgasm +rejection +clifford +lorraine +advent +mafia +rodney +brock +##ght +##used +##elia +cassette +chamberlain +despair +mongolia +sensors +developmental +upstream +##eg +##alis +spanning +165 +trombone +basque +seeded +interred +renewable +rhys +leapt +revision +molecule +##ages +chord +vicious +nord +shivered +23rd +arlington +debts +corpus +sunrise +bays +blackburn +centimetres +##uded +shuddered +gm +strangely +gripping +cartoons +isabelle +orbital +##ppa +seals +proving +##lton +refusal +strengthened +bust +assisting +baghdad +batsman +portrayal +mara +pushes +spears +og +##cock +reside +nathaniel +brennan +1776 +confirmation +caucus +##worthy +markings +yemen +nobles +ku +lazy +viewer +catalan +encompasses +sawyer +##fall +sparked +substances +patents +braves +arranger +evacuation +sergio +persuade +dover +tolerance +penguin +cum +jockey +insufficient +townships +occupying +declining +plural +processed +projection +puppet +flanders +introduces +liability +##yon +gymnastics +antwerp +taipei +hobart +candles +jeep +wes +observers +126 +chaplain +bundle +glorious +##hine +hazel +flung +sol +excavations +dumped +stares +sh +bangalore +triangular +icelandic +intervals +expressing +turbine +##vers +songwriting +crafts +##igo +jasmine +ditch +rite +##ways +entertaining +comply +sorrow +wrestlers +basel +emirates +marian +rivera +helpful +##some +caution +downward +networking +##atory +##tered +darted +genocide +emergence +replies +specializing +spokesman +convenient +unlocked +fading +augustine +concentrations +resemblance +elijah +investigator +andhra +##uda +promotes +bean +##rrell +fleeing +wan +simone +announcer +##ame +##bby +lydia +weaver +132 +residency +modification +##fest +stretches +##ast +alternatively +nat +lowe +lacks +##ented +pam +tile +concealed +inferior +abdullah +residences +tissues +vengeance +##ided +moisture +peculiar +groove +zip +bologna +jennings +ninja +oversaw +zombies +pumping +batch +livingston +emerald +installations +1797 +peel +nitrogen +rama +##fying +##star +schooling +strands +responding +werner +##ost +lime +casa +accurately +targeting +##rod +underway +##uru +hemisphere +lester +##yard +occupies +2d +griffith +angrily +reorganized +##owing +courtney +deposited +##dd +##30 +estadio +##ifies +dunn +exiled +##ying +checks +##combe +##о +##fly +successes +unexpectedly +blu +assessed +##flower +##ه +observing +sacked +spiders +kn +##tail +mu +nodes +prosperity +audrey +divisional +155 +broncos +tangled +adjust +feeds +erosion +paolo +surf +directory +snatched +humid +admiralty +screwed +gt +reddish +##nese +modules +trench +lamps +bind +leah +bucks +competes +##nz +##form +transcription +##uc +isles +violently +clutching +pga +cyclist +inflation +flats +ragged +unnecessary +##hian +stubborn +coordinated +harriet +baba +disqualified +330 +insect +wolfe +##fies +reinforcements +rocked +duel +winked +embraced +bricks +##raj +hiatus +defeats +pending +brightly +jealousy +##xton +##hm +##uki +lena +gdp +colorful +##dley +stein +kidney +##shu +underwear +wanderers +##haw +##icus +guardians +m³ +roared +habits +##wise +permits +gp +uranium +punished +disguise +bundesliga +elise +dundee +erotic +partisan +pi +collectors +float +individually +rendering +behavioral +bucharest +ser +hare +valerie +corporal +nutrition +proportional +##isa +immense +##kis +pavement +##zie +##eld +sutherland +crouched +1775 +##lp +suzuki +trades +endurance +operas +crosby +prayed +priory +rory +socially +##urn +gujarat +##pu +walton +cube +pasha +privilege +lennon +floods +thorne +waterfall +nipple +scouting +approve +##lov +minorities +voter +dwight +extensions +assure +ballroom +slap +dripping +privileges +rejoined +confessed +demonstrating +patriotic +yell +investor +##uth +pagan +slumped +squares +##cle +##kins +confront +bert +embarrassment +##aid +aston +urging +sweater +starr +yuri +brains +williamson +commuter +mortar +structured +selfish +exports +##jon +cds +##him +unfinished +##rre +mortgage +destinations +##nagar +canoe +solitary +buchanan +delays +magistrate +fk +##pling +motivation +##lier +##vier +recruiting +assess +##mouth +malik +antique +1791 +pius +rahman +reich +tub +zhou +smashed +airs +galway +xii +conditioning +honduras +discharged +dexter +##pf +lionel +129 +debates +lemon +tiffany +volunteered +dom +dioxide +procession +devi +sic +tremendous +advertisements +colts +transferring +verdict +hanover +decommissioned +utter +relate +pac +racism +##top +beacon +limp +similarity +terra +occurrence +ant +##how +becky +capt +updates +armament +richie +pal +##graph +halloween +mayo +##ssen +##bone +cara +serena +fcc +dolls +obligations +##dling +violated +lafayette +jakarta +exploitation +##ime +infamous +iconic +##lah +##park +kitty +moody +reginald +dread +spill +crystals +olivier +modeled +bluff +equilibrium +separating +notices +ordnance +extinction +onset +cosmic +attachment +sammy +expose +privy +anchored +##bil +abbott +admits +bending +baritone +emmanuel +policeman +vaughan +winged +climax +dresses +denny +polytechnic +mohamed +burmese +authentic +nikki +genetics +grandparents +homestead +gaza +postponed +metacritic +una +##sby +##bat +unstable +dissertation +##rial +##cian +curls +obscure +uncovered +bronx +praying +disappearing +##hoe +prehistoric +coke +turret +mutations +nonprofit +pits +monaco +##ي +##usion +prominently +dispatched +podium +##mir +uci +##uation +133 +fortifications +birthplace +kendall +##lby +##oll +preacher +rack +goodman +##rman +persistent +##ott +countless +jaime +recorder +lexington +persecution +jumps +renewal +wagons +##11 +crushing +##holder +decorations +##lake +abundance +wrath +laundry +£1 +garde +##rp +jeanne +beetles +peasant +##sl +splitting +caste +sergei +##rer +##ema +scripts +##ively +rub +satellites +##vor +inscribed +verlag +scrapped +gale +packages +chick +potato +slogan +kathleen +arabs +##culture +counterparts +reminiscent +choral +##tead +rand +retains +bushes +dane +accomplish +courtesy +closes +##oth +slaughter +hague +krakow +lawson +tailed +elias +ginger +##ttes +canopy +betrayal +rebuilding +turf +##hof +frowning +allegiance +brigades +kicks +rebuild +polls +alias +nationalism +td +rowan +audition +bowie +fortunately +recognizes +harp +dillon +horrified +##oro +renault +##tics +ropes +##α +presumed +rewarded +infrared +wiping +accelerated +illustration +##rid +presses +practitioners +badminton +##iard +detained +##tera +recognizing +relates +misery +##sies +##tly +reproduction +piercing +potatoes +thornton +esther +manners +hbo +##aan +ours +bullshit +ernie +perennial +sensitivity +illuminated +rupert +##jin +##iss +##ear +rfc +nassau +##dock +staggered +socialism +##haven +appointments +nonsense +prestige +sharma +haul +##tical +solidarity +gps +##ook +##rata +igor +pedestrian +##uit +baxter +tenants +wires +medication +unlimited +guiding +impacts +diabetes +##rama +sasha +pas +clive +extraction +131 +continually +constraints +##bilities +sonata +hunted +sixteenth +chu +planting +quote +mayer +pretended +abs +spat +##hua +ceramic +##cci +curtains +pigs +pitching +##dad +latvian +sore +dayton +##sted +##qi +patrols +slice +playground +##nted +shone +stool +apparatus +inadequate +mates +treason +##ija +desires +##liga +##croft +somalia +laurent +mir +leonardo +oracle +grape +obliged +chevrolet +thirteenth +stunning +enthusiastic +##ede +accounted +concludes +currents +basil +##kovic +drought +##rica +mai +##aire +shove +posting +##shed +pilgrimage +humorous +packing +fry +pencil +wines +smells +144 +marilyn +aching +newest +clung +bon +neighbours +sanctioned +##pie +mug +##stock +drowning +##mma +hydraulic +##vil +hiring +reminder +lilly +investigators +##ncies +sour +##eous +compulsory +packet +##rion +##graphic +##elle +cannes +##inate +depressed +##rit +heroic +importantly +theresa +##tled +conway +saturn +marginal +rae +##xia +corresponds +royce +pact +jasper +explosives +packaging +aluminium +##ttered +denotes +rhythmic +spans +assignments +hereditary +outlined +originating +sundays +lad +reissued +greeting +beatrice +##dic +pillar +marcos +plots +handbook +alcoholic +judiciary +avant +slides +extract +masculine +blur +##eum +##force +homage +trembled +owens +hymn +trey +omega +signaling +socks +accumulated +reacted +attic +theo +lining +angie +distraction +primera +talbot +##key +1200 +ti +creativity +billed +##hey +deacon +eduardo +identifies +proposition +dizzy +gunner +hogan +##yam +##pping +##hol +ja +##chan +jensen +reconstructed +##berger +clearance +darius +##nier +abe +harlem +plea +dei +circled +emotionally +notation +fascist +neville +exceeded +upwards +viable +ducks +##fo +workforce +racer +limiting +shri +##lson +possesses +1600 +kerr +moths +devastating +laden +disturbing +locking +##cture +gal +fearing +accreditation +flavor +aide +1870s +mountainous +##baum +melt +##ures +motel +texture +servers +soda +##mb +herd +##nium +erect +puzzled +hum +peggy +examinations +gould +testified +geoff +ren +devised +sacks +##law +denial +posters +grunted +cesar +tutor +ec +gerry +offerings +byrne +falcons +combinations +ct +incoming +pardon +rocking +26th +avengers +flared +mankind +seller +uttar +loch +nadia +stroking +exposing +##hd +fertile +ancestral +instituted +##has +noises +prophecy +taxation +eminent +vivid +pol +##bol +dart +indirect +multimedia +notebook +upside +displaying +adrenaline +referenced +geometric +##iving +progression +##ddy +blunt +announce +##far +implementing +##lav +aggression +liaison +cooler +cares +headache +plantations +gorge +dots +impulse +thickness +ashamed +averaging +kathy +obligation +precursor +137 +fowler +symmetry +thee +225 +hears +##rai +undergoing +ads +butcher +bowler +##lip +cigarettes +subscription +goodness +##ically +browne +##hos +##tech +kyoto +donor +##erty +damaging +friction +drifting +expeditions +hardened +prostitution +152 +fauna +blankets +claw +tossing +snarled +butterflies +recruits +investigative +coated +healed +138 +communal +hai +xiii +academics +boone +psychologist +restless +lahore +stephens +mba +brendan +foreigners +printer +##pc +ached +explode +27th +deed +scratched +dared +##pole +cardiac +1780 +okinawa +proto +commando +compelled +oddly +electrons +##base +replica +thanksgiving +##rist +sheila +deliberate +stafford +tidal +representations +hercules +ou +##path +##iated +kidnapping +lenses +##tling +deficit +samoa +mouths +consuming +computational +maze +granting +smirk +razor +fixture +ideals +inviting +aiden +nominal +##vs +issuing +julio +pitt +ramsey +docks +##oss +exhaust +##owed +bavarian +draped +anterior +mating +ethiopian +explores +noticing +##nton +discarded +convenience +hoffman +endowment +beasts +cartridge +mormon +paternal +probe +sleeves +interfere +lump +deadline +##rail +jenks +bulldogs +scrap +alternating +justified +reproductive +nam +seize +descending +secretariat +kirby +coupe +grouped +smash +panther +sedan +tapping +##18 +lola +cheer +germanic +unfortunate +##eter +unrelated +##fan +subordinate +##sdale +suzanne +advertisement +##ility +horsepower +##lda +cautiously +discourse +luigi +##mans +##fields +noun +prevalent +mao +schneider +everett +surround +governorate +kira +##avia +westward +##take +misty +rails +sustainability +134 +unused +##rating +packs +toast +unwilling +regulate +thy +suffrage +nile +awe +assam +definitions +travelers +affordable +##rb +conferred +sells +undefeated +beneficial +torso +basal +repeating +remixes +##pass +bahrain +cables +fang +##itated +excavated +numbering +statutory +##rey +deluxe +##lian +forested +ramirez +derbyshire +zeus +slamming +transfers +astronomer +banana +lottery +berg +histories +bamboo +##uchi +resurrection +posterior +bowls +vaguely +##thi +thou +preserving +tensed +offence +##inas +meyrick +callum +ridden +watt +langdon +tying +lowland +snorted +daring +truman +##hale +##girl +aura +overly +filing +weighing +goa +infections +philanthropist +saunders +eponymous +##owski +latitude +perspectives +reviewing +mets +commandant +radial +##kha +flashlight +reliability +koch +vowels +amazed +ada +elaine +supper +##rth +##encies +predator +debated +soviets +cola +##boards +##nah +compartment +crooked +arbitrary +fourteenth +##ctive +havana +majors +steelers +clips +profitable +ambush +exited +packers +##tile +nude +cracks +fungi +##е +limb +trousers +josie +shelby +tens +frederic +##ος +definite +smoothly +constellation +insult +baton +discs +lingering +##nco +conclusions +lent +staging +becker +grandpa +shaky +##tron +einstein +obstacles +sk +adverse +elle +economically +##moto +mccartney +thor +dismissal +motions +readings +nostrils +treatise +##pace +squeezing +evidently +prolonged +1783 +venezuelan +je +marguerite +beirut +takeover +shareholders +##vent +denise +digit +airplay +norse +##bbling +imaginary +pills +hubert +blaze +vacated +eliminating +##ello +vine +mansfield +##tty +retrospective +barrow +borne +clutch +bail +forensic +weaving +##nett +##witz +desktop +citadel +promotions +worrying +dorset +ieee +subdivided +##iating +manned +expeditionary +pickup +synod +chuckle +185 +barney +##rz +##ffin +functionality +karachi +litigation +meanings +uc +lick +turbo +anders +##ffed +execute +curl +oppose +ankles +typhoon +##د +##ache +##asia +linguistics +compassion +pressures +grazing +perfection +##iting +immunity +monopoly +muddy +backgrounds +136 +namibia +francesca +monitors +attracting +stunt +tuition +##ии +vegetable +##mates +##quent +mgm +jen +complexes +forts +##ond +cellar +bites +seventeenth +royals +flemish +failures +mast +charities +##cular +peruvian +capitals +macmillan +ipswich +outward +frigate +postgraduate +folds +employing +##ouse +concurrently +fiery +##tai +contingent +nightmares +monumental +nicaragua +##kowski +lizard +mal +fielding +gig +reject +##pad +harding +##ipe +coastline +##cin +##nos +beethoven +humphrey +innovations +##tam +##nge +norris +doris +solicitor +huang +obey +141 +##lc +niagara +##tton +shelves +aug +bourbon +curry +nightclub +specifications +hilton +##ndo +centennial +dispersed +worm +neglected +briggs +sm +font +kuala +uneasy +plc +##nstein +##bound +##aking +##burgh +awaiting +pronunciation +##bbed +##quest +eh +optimal +zhu +raped +greens +presided +brenda +worries +##life +venetian +marxist +turnout +##lius +refined +braced +sins +grasped +sunderland +nickel +speculated +lowell +cyrillic +communism +fundraising +resembling +colonists +mutant +freddie +usc +##mos +gratitude +##run +mural +##lous +chemist +wi +reminds +28th +steals +tess +pietro +##ingen +promoter +ri +microphone +honoured +rai +sant +##qui +feather +##nson +burlington +kurdish +terrorists +deborah +sickness +##wed +##eet +hazard +irritated +desperation +veil +clarity +##rik +jewels +xv +##gged +##ows +##cup +berkshire +unfair +mysteries +orchid +winced +exhaustion +renovations +stranded +obe +infinity +##nies +adapt +redevelopment +thanked +registry +olga +domingo +noir +tudor +ole +##atus +commenting +behaviors +##ais +crisp +pauline +probable +stirling +wigan +##bian +paralympics +panting +surpassed +##rew +luca +barred +pony +famed +##sters +cassandra +waiter +carolyn +exported +##orted +andres +destructive +deeds +jonah +castles +vacancy +suv +##glass +1788 +orchard +yep +famine +belarusian +sprang +##forth +skinny +##mis +administrators +rotterdam +zambia +zhao +boiler +discoveries +##ride +##physics +lucius +disappointing +outreach +spoon +##frame +qualifications +unanimously +enjoys +regency +##iidae +stade +realism +veterinary +rodgers +dump +alain +chestnut +castile +censorship +rumble +gibbs +##itor +communion +reggae +inactivated +logs +loads +##houses +homosexual +##iano +ale +informs +##cas +phrases +plaster +linebacker +ambrose +kaiser +fascinated +850 +limerick +recruitment +forge +mastered +##nding +leinster +rooted +threaten +##strom +borneo +##hes +suggestions +scholarships +propeller +documentaries +patronage +coats +constructing +invest +neurons +comet +entirety +shouts +identities +annoying +unchanged +wary +##antly +##ogy +neat +oversight +##kos +phillies +replay +constance +##kka +incarnation +humble +skies +minus +##acy +smithsonian +##chel +guerrilla +jar +cadets +##plate +surplus +audit +##aru +cracking +joanna +louisa +pacing +##lights +intentionally +##iri +diner +nwa +imprint +australians +tong +unprecedented +bunker +naive +specialists +ark +nichols +railing +leaked +pedal +##uka +shrub +longing +roofs +v8 +captains +neural +tuned +##ntal +##jet +emission +medina +frantic +codex +definitive +sid +abolition +intensified +stocks +enrique +sustain +genoa +oxide +##written +clues +cha +##gers +tributaries +fragment +venom +##rity +##ente +##sca +muffled +vain +sire +laos +##ingly +##hana +hastily +snapping +surfaced +sentiment +motive +##oft +contests +approximate +mesa +luckily +dinosaur +exchanges +propelled +accord +bourne +relieve +tow +masks +offended +##ues +cynthia +##mmer +rains +bartender +zinc +reviewers +lois +##sai +legged +arrogant +rafe +rosie +comprise +handicap +blockade +inlet +lagoon +copied +drilling +shelley +petals +##inian +mandarin +obsolete +##inated +onward +arguably +productivity +cindy +praising +seldom +busch +discusses +raleigh +shortage +ranged +stanton +encouragement +firstly +conceded +overs +temporal +##uke +cbe +##bos +woo +certainty +pumps +##pton +stalked +##uli +lizzie +periodic +thieves +weaker +##night +gases +shoving +chooses +wc +##chemical +prompting +weights +##kill +robust +flanked +sticky +hu +tuberculosis +##eb +##eal +christchurch +resembled +wallet +reese +inappropriate +pictured +distract +fixing +fiddle +giggled +burger +heirs +hairy +mechanic +torque +apache +obsessed +chiefly +cheng +logging +##tag +extracted +meaningful +numb +##vsky +gloucestershire +reminding +##bay +unite +##lit +breeds +diminished +clown +glove +1860s +##ن +##ug +archibald +focal +freelance +sliced +depiction +##yk +organism +switches +sights +stray +crawling +##ril +lever +leningrad +interpretations +loops +anytime +reel +alicia +delighted +##ech +inhaled +xiv +suitcase +bernie +vega +licenses +northampton +exclusion +induction +monasteries +racecourse +homosexuality +##right +##sfield +##rky +dimitri +michele +alternatives +ions +commentators +genuinely +objected +pork +hospitality +fencing +stephan +warships +peripheral +wit +drunken +wrinkled +quentin +spends +departing +chung +numerical +spokesperson +##zone +johannesburg +caliber +killers +##udge +assumes +neatly +demographic +abigail +bloc +##vel +mounting +##lain +bentley +slightest +xu +recipients +##jk +merlin +##writer +seniors +prisons +blinking +hindwings +flickered +kappa +##hel +80s +strengthening +appealing +brewing +gypsy +mali +lashes +hulk +unpleasant +harassment +bio +treaties +predict +instrumentation +pulp +troupe +boiling +mantle +##ffe +ins +##vn +dividing +handles +verbs +##onal +coconut +senegal +340 +thorough +gum +momentarily +##sto +cocaine +panicked +destined +##turing +teatro +denying +weary +captained +mans +##hawks +##code +wakefield +bollywood +thankfully +##16 +cyril +##wu +amendments +##bahn +consultation +stud +reflections +kindness +1787 +internally +##ovo +tex +mosaic +distribute +paddy +seeming +143 +##hic +piers +##15 +##mura +##verse +popularly +winger +kang +sentinel +mccoy +##anza +covenant +##bag +verge +fireworks +suppress +thrilled +dominate +##jar +swansea +##60 +142 +reconciliation +##ndi +stiffened +cue +dorian +##uf +damascus +amor +ida +foremost +##aga +porsche +unseen +dir +##had +##azi +stony +lexi +melodies +##nko +angular +integer +podcast +ants +inherent +jaws +justify +persona +##olved +josephine +##nr +##ressed +customary +flashes +gala +cyrus +glaring +backyard +ariel +physiology +greenland +html +stir +avon +atletico +finch +methodology +ked +##lent +mas +catholicism +townsend +branding +quincy +fits +containers +1777 +ashore +aragon +##19 +forearm +poisoning +##sd +adopting +conquer +grinding +amnesty +keller +finances +evaluate +forged +lankan +instincts +##uto +guam +bosnian +photographed +workplace +desirable +protector +##dog +allocation +intently +encourages +willy +##sten +bodyguard +electro +brighter +##ν +bihar +##chev +lasts +opener +amphibious +sal +verde +arte +##cope +captivity +vocabulary +yields +##tted +agreeing +desmond +pioneered +##chus +strap +campaigned +railroads +##ович +emblem +##dre +stormed +501 +##ulous +marijuana +northumberland +##gn +##nath +bowen +landmarks +beaumont +##qua +danube +##bler +attorneys +th +ge +flyers +critique +villains +cass +mutation +acc +##0s +colombo +mckay +motif +sampling +concluding +syndicate +##rell +neon +stables +ds +warnings +clint +mourning +wilkinson +##tated +merrill +leopard +evenings +exhaled +emil +sonia +ezra +discrete +stove +farrell +fifteenth +prescribed +superhero +##rier +worms +helm +wren +##duction +##hc +expo +##rator +hq +unfamiliar +antony +prevents +acceleration +fiercely +mari +painfully +calculations +cheaper +ign +clifton +irvine +davenport +mozambique +##np +pierced +##evich +wonders +##wig +##cate +##iling +crusade +ware +##uel +enzymes +reasonably +mls +##coe +mater +ambition +bunny +eliot +kernel +##fin +asphalt +headmaster +torah +aden +lush +pins +waived +##care +##yas +joao +substrate +enforce +##grad +##ules +alvarez +selections +epidemic +tempted +##bit +bremen +translates +ensured +waterfront +29th +forrest +manny +malone +kramer +reigning +cookies +simpler +absorption +205 +engraved +##ffy +evaluated +1778 +haze +146 +comforting +crossover +##abe +thorn +##rift +##imo +##pop +suppression +fatigue +cutter +##tr +201 +wurttemberg +##orf +enforced +hovering +proprietary +gb +samurai +syllable +ascent +lacey +tick +lars +tractor +merchandise +rep +bouncing +defendants +##yre +huntington +##ground +##oko +standardized +##hor +##hima +assassinated +nu +predecessors +rainy +liar +assurance +lyrical +##uga +secondly +flattened +ios +parameter +undercover +##mity +bordeaux +punish +ridges +markers +exodus +inactive +hesitate +debbie +nyc +pledge +savoy +nagar +offset +organist +##tium +hesse +marin +converting +##iver +diagram +propulsion +pu +validity +reverted +supportive +##dc +ministries +clans +responds +proclamation +##inae +##ø +##rea +ein +pleading +patriot +sf +birch +islanders +strauss +hates +##dh +brandenburg +concession +rd +##ob +1900s +killings +textbook +antiquity +cinematography +wharf +embarrassing +setup +creed +farmland +inequality +centred +signatures +fallon +370 +##ingham +##uts +ceylon +gazing +directive +laurie +##tern +globally +##uated +##dent +allah +excavation +threads +##cross +148 +frantically +icc +utilize +determines +respiratory +thoughtful +receptions +##dicate +merging +chandra +seine +147 +builders +builds +diagnostic +dev +visibility +goddamn +analyses +dhaka +cho +proves +chancel +concurrent +curiously +canadians +pumped +restoring +1850s +turtles +jaguar +sinister +spinal +traction +declan +vows +1784 +glowed +capitalism +swirling +install +universidad +##lder +##oat +soloist +##genic +##oor +coincidence +beginnings +nissan +dip +resorts +caucasus +combustion +infectious +##eno +pigeon +serpent +##itating +conclude +masked +salad +jew +##gr +surreal +toni +##wc +harmonica +151 +##gins +##etic +##coat +fishermen +intending +bravery +##wave +klaus +titan +wembley +taiwanese +ransom +40th +incorrect +hussein +eyelids +jp +cooke +dramas +utilities +##etta +##print +eisenhower +principally +granada +lana +##rak +openings +concord +##bl +bethany +connie +morality +sega +##mons +##nard +earnings +##kara +##cine +wii +communes +##rel +coma +composing +softened +severed +grapes +##17 +nguyen +analyzed +warlord +hubbard +heavenly +behave +slovenian +##hit +##ony +hailed +filmmakers +trance +caldwell +skye +unrest +coward +likelihood +##aging +bern +sci +taliban +honolulu +propose +##wang +1700 +browser +imagining +cobra +contributes +dukes +instinctively +conan +violinist +##ores +accessories +gradual +##amp +quotes +sioux +##dating +undertake +intercepted +sparkling +compressed +139 +fungus +tombs +haley +imposing +rests +degradation +lincolnshire +retailers +wetlands +tulsa +distributor +dungeon +nun +greenhouse +convey +atlantis +aft +exits +oman +dresser +lyons +##sti +joking +eddy +judgement +omitted +digits +##cts +##game +juniors +##rae +cents +stricken +une +##ngo +wizards +weir +breton +nan +technician +fibers +liking +royalty +##cca +154 +persia +terribly +magician +##rable +##unt +vance +cafeteria +booker +camille +warmer +##static +consume +cavern +gaps +compass +contemporaries +foyer +soothing +graveyard +maj +plunged +blush +##wear +cascade +demonstrates +ordinance +##nov +boyle +##lana +rockefeller +shaken +banjo +izzy +##ense +breathless +vines +##32 +##eman +alterations +chromosome +dwellings +feudal +mole +153 +catalonia +relics +tenant +mandated +##fm +fridge +hats +honesty +patented +raul +heap +cruisers +accusing +enlightenment +infants +wherein +chatham +contractors +zen +affinity +hc +osborne +piston +156 +traps +maturity +##rana +lagos +##zal +peering +##nay +attendant +dealers +protocols +subset +prospects +biographical +##cre +artery +##zers +insignia +nuns +endured +##eration +recommend +schwartz +serbs +berger +cromwell +crossroads +##ctor +enduring +clasped +grounded +##bine +marseille +twitched +abel +choke +https +catalyst +moldova +italians +##tist +disastrous +wee +##oured +##nti +wwf +nope +##piration +##asa +expresses +thumbs +167 +##nza +coca +1781 +cheating +##ption +skipped +sensory +heidelberg +spies +satan +dangers +semifinal +202 +bohemia +whitish +confusing +shipbuilding +relies +surgeons +landings +ravi +baku +moor +suffix +alejandro +##yana +litre +upheld +##unk +rajasthan +##rek +coaster +insists +posture +scenarios +etienne +favoured +appoint +transgender +elephants +poked +greenwood +defences +fulfilled +militant +somali +1758 +chalk +potent +##ucci +migrants +wink +assistants +nos +restriction +activism +niger +##ario +colon +shaun +##sat +daphne +##erated +swam +congregations +reprise +considerations +magnet +playable +xvi +##р +overthrow +tobias +knob +chavez +coding +##mers +propped +katrina +orient +newcomer +##suke +temperate +##pool +farmhouse +interrogation +##vd +committing +##vert +forthcoming +strawberry +joaquin +macau +ponds +shocking +siberia +##cellular +chant +contributors +##nant +##ologists +sped +absorb +hail +1782 +spared +##hore +barbados +karate +opus +originates +saul +##xie +evergreen +leaped +##rock +correlation +exaggerated +weekday +unification +bump +tracing +brig +afb +pathways +utilizing +##ners +mod +mb +disturbance +kneeling +##stad +##guchi +100th +pune +##thy +decreasing +168 +manipulation +miriam +academia +ecosystem +occupational +rbi +##lem +rift +##14 +rotary +stacked +incorporation +awakening +generators +guerrero +racist +##omy +cyber +derivatives +culminated +allie +annals +panzer +sainte +wikipedia +pops +zu +austro +##vate +algerian +politely +nicholson +mornings +educate +tastes +thrill +dartmouth +##gating +db +##jee +regan +differing +concentrating +choreography +divinity +##media +pledged +alexandre +routing +gregor +madeline +##idal +apocalypse +##hora +gunfire +culminating +elves +fined +liang +lam +programmed +tar +guessing +transparency +gabrielle +##gna +cancellation +flexibility +##lining +accession +shea +stronghold +nets +specializes +##rgan +abused +hasan +sgt +ling +exceeding +##₄ +admiration +supermarket +##ark +photographers +specialised +tilt +resonance +hmm +perfume +380 +sami +threatens +garland +botany +guarding +boiled +greet +puppy +russo +supplier +wilmington +vibrant +vijay +##bius +paralympic +grumbled +paige +faa +licking +margins +hurricanes +##gong +fest +grenade +ripping +##uz +counseling +weigh +##sian +needles +wiltshire +edison +costly +##not +fulton +tramway +redesigned +staffordshire +cache +gasping +watkins +sleepy +candidacy +##group +monkeys +timeline +throbbing +##bid +##sos +berth +uzbekistan +vanderbilt +bothering +overturned +ballots +gem +##iger +sunglasses +subscribers +hooker +compelling +ang +exceptionally +saloon +stab +##rdi +carla +terrifying +rom +##vision +coil +##oids +satisfying +vendors +31st +mackay +deities +overlooked +ambient +bahamas +felipe +olympia +whirled +botanist +advertised +tugging +##dden +disciples +morales +unionist +rites +foley +morse +motives +creepy +##₀ +soo +##sz +bargain +highness +frightening +turnpike +tory +reorganization +##cer +depict +biographer +##walk +unopposed +manifesto +##gles +institut +emile +accidental +kapoor +##dam +kilkenny +cortex +lively +##13 +romanesque +jain +shan +cannons +##ood +##ske +petrol +echoing +amalgamated +disappears +cautious +proposes +sanctions +trenton +##ر +flotilla +aus +contempt +tor +canary +cote +theirs +##hun +conceptual +deleted +fascinating +paso +blazing +elf +honourable +hutchinson +##eiro +##outh +##zin +surveyor +tee +amidst +wooded +reissue +intro +##ono +cobb +shelters +newsletter +hanson +brace +encoding +confiscated +dem +caravan +marino +scroll +melodic +cows +imam +##adi +##aneous +northward +searches +biodiversity +cora +310 +roaring +##bers +connell +theologian +halo +compose +pathetic +unmarried +dynamo +##oot +az +calculation +toulouse +deserves +humour +nr +forgiveness +tam +undergone +martyr +pamela +myths +whore +counselor +hicks +290 +heavens +battleship +electromagnetic +##bbs +stellar +establishments +presley +hopped +##chin +temptation +90s +wills +nas +##yuan +nhs +##nya +seminars +##yev +adaptations +gong +asher +lex +indicator +sikh +tobago +cites +goin +##yte +satirical +##gies +characterised +correspond +bubbles +lure +participates +##vid +eruption +skate +therapeutic +1785 +canals +wholesale +defaulted +sac +460 +petit +##zzled +virgil +leak +ravens +256 +portraying +##yx +ghetto +creators +dams +portray +vicente +##rington +fae +namesake +bounty +##arium +joachim +##ota +##iser +aforementioned +axle +snout +depended +dismantled +reuben +480 +##ibly +gallagher +##lau +##pd +earnest +##ieu +##iary +inflicted +objections +##llar +asa +gritted +##athy +jericho +##sea +##was +flick +underside +ceramics +undead +substituted +195 +eastward +undoubtedly +wheeled +chimney +##iche +guinness +cb +##ager +siding +##bell +traitor +baptiste +disguised +inauguration +149 +tipperary +choreographer +perched +warmed +stationary +eco +##ike +##ntes +bacterial +##aurus +flores +phosphate +##core +attacker +invaders +alvin +intersects +a1 +indirectly +immigrated +businessmen +cornelius +valves +narrated +pill +sober +ul +nationale +monastic +applicants +scenery +##jack +161 +motifs +constitutes +cpu +##osh +jurisdictions +sd +tuning +irritation +woven +##uddin +fertility +gao +##erie +antagonist +impatient +glacial +hides +boarded +denominations +interception +##jas +cookie +nicola +##tee +algebraic +marquess +bahn +parole +buyers +bait +turbines +paperwork +bestowed +natasha +renee +oceans +purchases +157 +vaccine +215 +##tock +fixtures +playhouse +integrate +jai +oswald +intellectuals +##cky +booked +nests +mortimer +##isi +obsession +sept +##gler +##sum +440 +scrutiny +simultaneous +squinted +##shin +collects +oven +shankar +penned +remarkably +##я +slips +luggage +spectral +1786 +collaborations +louie +consolidation +##ailed +##ivating +420 +hoover +blackpool +harness +ignition +vest +tails +belmont +mongol +skinner +##nae +visually +mage +derry +##tism +##unce +stevie +transitional +##rdy +redskins +drying +prep +prospective +##21 +annoyance +oversee +##loaded +fills +##books +##iki +announces +fda +scowled +respects +prasad +mystic +tucson +##vale +revue +springer +bankrupt +1772 +aristotle +salvatore +habsburg +##geny +dal +natal +nut +pod +chewing +darts +moroccan +walkover +rosario +lenin +punjabi +##ße +grossed +scattering +wired +invasive +hui +polynomial +corridors +wakes +gina +portrays +##cratic +arid +retreating +erich +irwin +sniper +##dha +linen +lindsey +maneuver +butch +shutting +socio +bounce +commemorative +postseason +jeremiah +pines +275 +mystical +beads +bp +abbas +furnace +bidding +consulted +assaulted +empirical +rubble +enclosure +sob +weakly +cancel +polly +yielded +##emann +curly +prediction +battered +70s +vhs +jacqueline +render +sails +barked +detailing +grayson +riga +sloane +raging +##yah +herbs +bravo +##athlon +alloy +giggle +imminent +suffers +assumptions +waltz +##itate +accomplishments +##ited +bathing +remixed +deception +prefix +##emia +deepest +##tier +##eis +balkan +frogs +##rong +slab +##pate +philosophers +peterborough +grains +imports +dickinson +rwanda +##atics +1774 +dirk +lan +tablets +##rove +clone +##rice +caretaker +hostilities +mclean +##gre +regimental +treasures +norms +impose +tsar +tango +diplomacy +variously +complain +192 +recognise +arrests +1779 +celestial +pulitzer +##dus +bing +libretto +##moor +adele +splash +##rite +expectation +lds +confronts +##izer +spontaneous +harmful +wedge +entrepreneurs +buyer +##ope +bilingual +translate +rugged +conner +circulated +uae +eaton +##gra +##zzle +lingered +lockheed +vishnu +reelection +alonso +##oom +joints +yankee +headline +cooperate +heinz +laureate +invading +##sford +echoes +scandinavian +##dham +hugging +vitamin +salute +micah +hind +trader +##sper +radioactive +##ndra +militants +poisoned +ratified +remark +campeonato +deprived +wander +prop +##dong +outlook +##tani +##rix +##eye +chiang +darcy +##oping +mandolin +spice +statesman +babylon +182 +walled +forgetting +afro +##cap +158 +giorgio +buffer +##polis +planetary +##gis +overlap +terminals +kinda +centenary +##bir +arising +manipulate +elm +ke +1770 +ak +##tad +chrysler +mapped +moose +pomeranian +quad +macarthur +assemblies +shoreline +recalls +stratford +##rted +noticeable +##evic +imp +##rita +##sque +accustomed +supplying +tents +disgusted +vogue +sipped +filters +khz +reno +selecting +luftwaffe +mcmahon +tyne +masterpiece +carriages +collided +dunes +exercised +flare +remembers +muzzle +##mobile +heck +##rson +burgess +lunged +middleton +boycott +bilateral +##sity +hazardous +lumpur +multiplayer +spotlight +jackets +goldman +liege +porcelain +rag +waterford +benz +attracts +hopeful +battling +ottomans +kensington +baked +hymns +cheyenne +lattice +levine +borrow +polymer +clashes +michaels +monitored +commitments +denounced +##25 +##von +cavity +##oney +hobby +akin +##holders +futures +intricate +cornish +patty +##oned +illegally +dolphin +##lag +barlow +yellowish +maddie +apologized +luton +plagued +##puram +nana +##rds +sway +fanny +łodz +##rino +psi +suspicions +hanged +##eding +initiate +charlton +##por +nak +competent +235 +analytical +annex +wardrobe +reservations +##rma +sect +162 +fairfax +hedge +piled +buckingham +uneven +bauer +simplicity +snyder +interpret +accountability +donors +moderately +byrd +continents +##cite +##max +disciple +hr +jamaican +ping +nominees +##uss +mongolian +diver +attackers +eagerly +ideological +pillows +miracles +apartheid +revolver +sulfur +clinics +moran +163 +##enko +ile +katy +rhetoric +##icated +chronology +recycling +##hrer +elongated +mughal +pascal +profiles +vibration +databases +domination +##fare +##rant +matthias +digest +rehearsal +polling +weiss +initiation +reeves +clinging +flourished +impress +ngo +##hoff +##ume +buckley +symposium +rhythms +weed +emphasize +transforming +##taking +##gence +##yman +accountant +analyze +flicker +foil +priesthood +voluntarily +decreases +##80 +##hya +slater +sv +charting +mcgill +##lde +moreno +##iu +besieged +zur +robes +##phic +admitting +api +deported +turmoil +peyton +earthquakes +##ares +nationalists +beau +clair +brethren +interrupt +welch +curated +galerie +requesting +164 +##ested +impending +steward +viper +##vina +complaining +beautifully +brandy +foam +nl +1660 +##cake +alessandro +punches +laced +explanations +##lim +attribute +clit +reggie +discomfort +##cards +smoothed +whales +##cene +adler +countered +duffy +disciplinary +widening +recipe +reliance +conducts +goats +gradient +preaching +##shaw +matilda +quasi +striped +meridian +cannabis +cordoba +certificates +##agh +##tering +graffiti +hangs +pilgrims +repeats +##ych +revive +urine +etat +##hawk +fueled +belts +fuzzy +susceptible +##hang +mauritius +salle +sincere +beers +hooks +##cki +arbitration +entrusted +advise +sniffed +seminar +junk +donnell +processors +principality +strapped +celia +mendoza +everton +fortunes +prejudice +starving +reassigned +steamer +##lund +tuck +evenly +foreman +##ffen +dans +375 +envisioned +slit +##xy +baseman +liberia +rosemary +##weed +electrified +periodically +potassium +stride +contexts +sperm +slade +mariners +influx +bianca +subcommittee +##rane +spilling +icao +estuary +##nock +delivers +iphone +##ulata +isa +mira +bohemian +dessert +##sbury +welcoming +proudly +slowing +##chs +musee +ascension +russ +##vian +waits +##psy +africans +exploit +##morphic +gov +eccentric +crab +peck +##ull +entrances +formidable +marketplace +groom +bolted +metabolism +patton +robbins +courier +payload +endure +##ifier +andes +refrigerator +##pr +ornate +##uca +ruthless +illegitimate +masonry +strasbourg +bikes +adobe +##³ +apples +quintet +willingly +niche +bakery +corpses +energetic +##cliffe +##sser +##ards +177 +centimeters +centro +fuscous +cretaceous +rancho +##yde +andrei +telecom +tottenham +oasis +ordination +vulnerability +presiding +corey +cp +penguins +sims +##pis +malawi +piss +##48 +correction +##cked +##ffle +##ryn +countdown +detectives +psychiatrist +psychedelic +dinosaurs +blouse +##get +choi +vowed +##oz +randomly +##pol +49ers +scrub +blanche +bruins +dusseldorf +##using +unwanted +##ums +212 +dominique +elevations +headlights +om +laguna +##oga +1750 +famously +ignorance +shrewsbury +##aine +ajax +breuning +che +confederacy +greco +overhaul +##screen +paz +skirts +disagreement +cruelty +jagged +phoebe +shifter +hovered +viruses +##wes +mandy +##lined +##gc +landlord +squirrel +dashed +##ι +ornamental +gag +wally +grange +literal +spurs +undisclosed +proceeding +yin +##text +billie +orphan +spanned +humidity +indy +weighted +presentations +explosions +lucian +##tary +vaughn +hindus +##anga +##hell +psycho +171 +daytona +protects +efficiently +rematch +sly +tandem +##oya +rebranded +impaired +hee +metropolis +peach +godfrey +diaspora +ethnicity +prosperous +gleaming +dar +grossing +playback +##rden +stripe +pistols +##tain +births +labelled +##cating +172 +rudy +alba +##onne +aquarium +hostility +##gb +##tase +shudder +sumatra +hardest +lakers +consonant +creeping +demos +homicide +capsule +zeke +liberties +expulsion +pueblo +##comb +trait +transporting +##ddin +##neck +##yna +depart +gregg +mold +ledge +hangar +oldham +playboy +termination +analysts +gmbh +romero +##itic +insist +cradle +filthy +brightness +slash +shootout +deposed +bordering +##truct +isis +microwave +tumbled +sheltered +cathy +werewolves +messy +andersen +convex +clapped +clinched +satire +wasting +edo +vc +rufus +##jak +mont +##etti +poznan +##keeping +restructuring +transverse +##rland +azerbaijani +slovene +gestures +roommate +choking +shear +##quist +vanguard +oblivious +##hiro +disagreed +baptism +##lich +coliseum +##aceae +salvage +societe +cory +locke +relocation +relying +versailles +ahl +swelling +##elo +cheerful +##word +##edes +gin +sarajevo +obstacle +diverted +##nac +messed +thoroughbred +fluttered +utrecht +chewed +acquaintance +assassins +dispatch +mirza +##wart +nike +salzburg +swell +yen +##gee +idle +ligue +samson +##nds +##igh +playful +spawned +##cise +tease +##case +burgundy +##bot +stirring +skeptical +interceptions +marathi +##dies +bedrooms +aroused +pinch +##lik +preferences +tattoos +buster +digitally +projecting +rust +##ital +kitten +priorities +addison +pseudo +##guard +dusk +icons +sermon +##psis +##iba +bt +##lift +##xt +ju +truce +rink +##dah +##wy +defects +psychiatry +offences +calculate +glucose +##iful +##rized +##unda +francaise +##hari +richest +warwickshire +carly +1763 +purity +redemption +lending +##cious +muse +bruises +cerebral +aero +carving +##name +preface +terminology +invade +monty +##int +anarchist +blurred +##iled +rossi +treats +guts +shu +foothills +ballads +undertaking +premise +cecilia +affiliates +blasted +conditional +wilder +minors +drone +rudolph +buffy +swallowing +horton +attested +##hop +rutherford +howell +primetime +livery +penal +##bis +minimize +hydro +wrecked +wrought +palazzo +##gling +cans +vernacular +friedman +nobleman +shale +walnut +danielle +##ection +##tley +sears +##kumar +chords +lend +flipping +streamed +por +dracula +gallons +sacrifices +gamble +orphanage +##iman +mckenzie +##gible +boxers +daly +##balls +##ان +208 +##ific +##rative +##iq +exploited +slated +##uity +circling +hillary +pinched +goldberg +provost +campaigning +lim +piles +ironically +jong +mohan +successors +usaf +##tem +##ught +autobiographical +haute +preserves +##ending +acquitted +comparisons +203 +hydroelectric +gangs +cypriot +torpedoes +rushes +chrome +derive +bumps +instability +fiat +pets +##mbe +silas +dye +reckless +settler +##itation +info +heats +##writing +176 +canonical +maltese +fins +mushroom +stacy +aspen +avid +##kur +##loading +vickers +gaston +hillside +statutes +wilde +gail +kung +sabine +comfortably +motorcycles +##rgo +169 +pneumonia +fetch +##sonic +axel +faintly +parallels +##oop +mclaren +spouse +compton +interdisciplinary +miner +##eni +181 +clamped +##chal +##llah +separates +versa +##mler +scarborough +labrador +##lity +##osing +rutgers +hurdles +como +166 +burt +divers +##100 +wichita +cade +coincided +##erson +bruised +mla +##pper +vineyard +##ili +##brush +notch +mentioning +jase +hearted +kits +doe +##acle +pomerania +##ady +ronan +seizure +pavel +problematic +##zaki +domenico +##ulin +catering +penelope +dependence +parental +emilio +ministerial +atkinson +##bolic +clarkson +chargers +colby +grill +peeked +arises +summon +##aged +fools +##grapher +faculties +qaeda +##vial +garner +refurbished +##hwa +geelong +disasters +nudged +bs +shareholder +lori +algae +reinstated +rot +##ades +##nous +invites +stainless +183 +inclusive +##itude +diocesan +til +##icz +denomination +##xa +benton +floral +registers +##ider +##erman +##kell +absurd +brunei +guangzhou +hitter +retaliation +##uled +##eve +blanc +nh +consistency +contamination +##eres +##rner +dire +palermo +broadcasters +diaries +inspire +vols +brewer +tightening +ky +mixtape +hormone +##tok +stokes +##color +##dly +##ssi +pg +##ometer +##lington +sanitation +##tility +intercontinental +apps +##adt +¹⁄₂ +cylinders +economies +favourable +unison +croix +gertrude +odyssey +vanity +dangling +##logists +upgrades +dice +middleweight +practitioner +##ight +206 +henrik +parlor +orion +angered +lac +python +blurted +##rri +sensual +intends +swings +angled +##phs +husky +attain +peerage +precinct +textiles +cheltenham +shuffled +dai +confess +tasting +bhutan +##riation +tyrone +segregation +abrupt +ruiz +##rish +smirked +blackwell +confidential +browning +amounted +##put +vase +scarce +fabulous +raided +staple +guyana +unemployed +glider +shay +##tow +carmine +troll +intervene +squash +superstar +##uce +cylindrical +len +roadway +researched +handy +##rium +##jana +meta +lao +declares +##rring +##tadt +##elin +##kova +willem +shrubs +napoleonic +realms +skater +qi +volkswagen +##ł +tad +hara +archaeologist +awkwardly +eerie +##kind +wiley +##heimer +##24 +titus +organizers +cfl +crusaders +lama +usb +vent +enraged +thankful +occupants +maximilian +##gaard +possessing +textbooks +##oran +collaborator +quaker +##ulo +avalanche +mono +silky +straits +isaiah +mustang +surged +resolutions +potomac +descend +cl +kilograms +plato +strains +saturdays +##olin +bernstein +##ype +holstein +ponytail +##watch +belize +conversely +heroine +perpetual +##ylus +charcoal +piedmont +glee +negotiating +backdrop +prologue +##jah +##mmy +pasadena +climbs +ramos +sunni +##holm +##tner +##tri +anand +deficiency +hertfordshire +stout +##avi +aperture +orioles +##irs +doncaster +intrigued +bombed +coating +otis +##mat +cocktail +##jit +##eto +amir +arousal +sar +##proof +##act +##ories +dixie +pots +##bow +whereabouts +159 +##fted +drains +bullying +cottages +scripture +coherent +fore +poe +appetite +##uration +sampled +##ators +##dp +derrick +rotor +jays +peacock +installment +##rro +advisors +##coming +rodeo +scotch +##mot +##db +##fen +##vant +ensued +rodrigo +dictatorship +martyrs +twenties +##н +towed +incidence +marta +rainforest +sai +scaled +##cles +oceanic +qualifiers +symphonic +mcbride +dislike +generalized +aubrey +colonization +##iation +##lion +##ssing +disliked +lublin +salesman +##ulates +spherical +whatsoever +sweating +avalon +contention +punt +severity +alderman +atari +##dina +##grant +##rop +scarf +seville +vertices +annexation +fairfield +fascination +inspiring +launches +palatinate +regretted +##rca +feral +##iom +elk +nap +olsen +reddy +yong +##leader +##iae +garment +transports +feng +gracie +outrage +viceroy +insides +##esis +breakup +grady +organizer +softer +grimaced +222 +murals +galicia +arranging +vectors +##rsten +bas +##sb +##cens +sloan +##eka +bitten +ara +fender +nausea +bumped +kris +banquet +comrades +detector +persisted +##llan +adjustment +endowed +cinemas +##shot +sellers +##uman +peek +epa +kindly +neglect +simpsons +talon +mausoleum +runaway +hangul +lookout +##cic +rewards +coughed +acquainted +chloride +##ald +quicker +accordion +neolithic +##qa +artemis +coefficient +lenny +pandora +tx +##xed +ecstasy +litter +segunda +chairperson +gemma +hiss +rumor +vow +nasal +antioch +compensate +patiently +transformers +##eded +judo +morrow +penis +posthumous +philips +bandits +husbands +denote +flaming +##any +##phones +langley +yorker +1760 +walters +##uo +##kle +gubernatorial +fatty +samsung +leroy +outlaw +##nine +unpublished +poole +jakob +##ᵢ +##ₙ +crete +distorted +superiority +##dhi +intercept +crust +mig +claus +crashes +positioning +188 +stallion +301 +frontal +armistice +##estinal +elton +aj +encompassing +camel +commemorated +malaria +woodward +calf +cigar +penetrate +##oso +willard +##rno +##uche +illustrate +amusing +convergence +noteworthy +##lma +##rva +journeys +realise +manfred +##sable +410 +##vocation +hearings +fiance +##posed +educators +provoked +adjusting +##cturing +modular +stockton +paterson +vlad +rejects +electors +selena +maureen +##tres +uber +##rce +swirled +##num +proportions +nanny +pawn +naturalist +parma +apostles +awoke +ethel +wen +##bey +monsoon +overview +##inating +mccain +rendition +risky +adorned +##ih +equestrian +germain +nj +conspicuous +confirming +##yoshi +shivering +##imeter +milestone +rumours +flinched +bounds +smacked +token +##bei +lectured +automobiles +##shore +impacted +##iable +nouns +nero +##leaf +ismail +prostitute +trams +##lace +bridget +sud +stimulus +impressions +reins +revolves +##oud +##gned +giro +honeymoon +##swell +criterion +##sms +##uil +libyan +prefers +##osition +211 +preview +sucks +accusation +bursts +metaphor +diffusion +tolerate +faye +betting +cinematographer +liturgical +specials +bitterly +humboldt +##ckle +flux +rattled +##itzer +archaeologists +odor +authorised +marshes +discretion +##ов +alarmed +archaic +inverse +##leton +explorers +##pine +drummond +tsunami +woodlands +##minate +##tland +booklet +insanity +owning +insert +crafted +calculus +##tore +receivers +##bt +stung +##eca +##nched +prevailing +travellers +eyeing +lila +graphs +##borne +178 +julien +##won +morale +adaptive +therapist +erica +cw +libertarian +bowman +pitches +vita +##ional +crook +##ads +##entation +caledonia +mutiny +##sible +1840s +automation +##ß +flock +##pia +ironic +pathology +##imus +remarried +##22 +joker +withstand +energies +##att +shropshire +hostages +madeleine +tentatively +conflicting +mateo +recipes +euros +ol +mercenaries +nico +##ndon +albuquerque +augmented +mythical +bel +freud +##child +cough +##lica +365 +freddy +lillian +genetically +nuremberg +calder +209 +bonn +outdoors +paste +suns +urgency +vin +restraint +tyson +##cera +##selle +barrage +bethlehem +kahn +##par +mounts +nippon +barony +happier +ryu +makeshift +sheldon +blushed +castillo +barking +listener +taped +bethel +fluent +headlines +pornography +rum +disclosure +sighing +mace +doubling +gunther +manly +##plex +rt +interventions +physiological +forwards +emerges +##tooth +##gny +compliment +rib +recession +visibly +barge +faults +connector +exquisite +prefect +##rlin +patio +##cured +elevators +brandt +italics +pena +173 +wasp +satin +ea +botswana +graceful +respectable +##jima +##rter +##oic +franciscan +generates +##dl +alfredo +disgusting +##olate +##iously +sherwood +warns +cod +promo +cheryl +sino +##ة +##escu +twitch +##zhi +brownish +thom +ortiz +##dron +densely +##beat +carmel +reinforce +##bana +187 +anastasia +downhill +vertex +contaminated +remembrance +harmonic +homework +##sol +fiancee +gears +olds +angelica +loft +ramsay +quiz +colliery +sevens +##cape +autism +##hil +walkway +##boats +ruben +abnormal +ounce +khmer +##bbe +zachary +bedside +morphology +punching +##olar +sparrow +convinces +##35 +hewitt +queer +remastered +rods +mabel +solemn +notified +lyricist +symmetric +##xide +174 +encore +passports +wildcats +##uni +baja +##pac +mildly +##ease +bleed +commodity +mounds +glossy +orchestras +##omo +damian +prelude +ambitions +##vet +awhile +remotely +##aud +asserts +imply +##iques +distinctly +modelling +remedy +##dded +windshield +dani +xiao +##endra +audible +powerplant +1300 +invalid +elemental +acquisitions +##hala +immaculate +libby +plata +smuggling +ventilation +denoted +minh +##morphism +430 +differed +dion +kelley +lore +mocking +sabbath +spikes +hygiene +drown +runoff +stylized +tally +liberated +aux +interpreter +righteous +aba +siren +reaper +pearce +millie +##cier +##yra +gaius +##iso +captures +##ttering +dorm +claudio +##sic +benches +knighted +blackness +##ored +discount +fumble +oxidation +routed +##ς +novak +perpendicular +spoiled +fracture +splits +##urt +pads +topology +##cats +axes +fortunate +offenders +protestants +esteem +221 +broadband +convened +frankly +hound +prototypes +isil +facilitated +keel +##sher +sahara +awaited +bubba +orb +prosecutors +186 +hem +520 +##xing +relaxing +remnant +romney +sorted +slalom +stefano +ulrich +##active +exemption +folder +pauses +foliage +hitchcock +epithet +204 +criticisms +##aca +ballistic +brody +hinduism +chaotic +youths +equals +##pala +pts +thicker +analogous +capitalist +improvised +overseeing +sinatra +ascended +beverage +##tl +straightforward +##kon +curran +##west +bois +325 +induce +surveying +emperors +sax +unpopular +##kk +cartoonist +fused +##mble +unto +##yuki +localities +##cko +##ln +darlington +slain +academie +lobbying +sediment +puzzles +##grass +defiance +dickens +manifest +tongues +alumnus +arbor +coincide +184 +appalachian +mustafa +examiner +cabaret +traumatic +yves +bracelet +draining +heroin +magnum +baths +odessa +consonants +mitsubishi +##gua +kellan +vaudeville +##fr +joked +null +straps +probation +##ław +ceded +interfaces +##pas +##zawa +blinding +viet +224 +rothschild +museo +640 +huddersfield +##vr +tactic +##storm +brackets +dazed +incorrectly +##vu +reg +glazed +fearful +manifold +benefited +irony +##sun +stumbling +##rte +willingness +balkans +mei +wraps +##aba +injected +##lea +gu +syed +harmless +##hammer +bray +takeoff +poppy +timor +cardboard +astronaut +purdue +weeping +southbound +cursing +stalls +diagonal +##neer +lamar +bryce +comte +weekdays +harrington +##uba +negatively +##see +lays +grouping +##cken +##henko +affirmed +halle +modernist +##lai +hodges +smelling +aristocratic +baptized +dismiss +justification +oilers +##now +coupling +qin +snack +healer +##qing +gardener +layla +battled +formulated +stephenson +gravitational +##gill +##jun +1768 +granny +coordinating +suites +##cd +##ioned +monarchs +##cote +##hips +sep +blended +apr +barrister +deposition +fia +mina +policemen +paranoid +##pressed +churchyard +covert +crumpled +creep +abandoning +tr +transmit +conceal +barr +understands +readiness +spire +##cology +##enia +##erry +610 +startling +unlock +vida +bowled +slots +##nat +##islav +spaced +trusting +admire +rig +##ink +slack +##70 +mv +207 +casualty +##wei +classmates +##odes +##rar +##rked +amherst +furnished +evolve +foundry +menace +mead +##lein +flu +wesleyan +##kled +monterey +webber +##vos +wil +##mith +##на +bartholomew +justices +restrained +##cke +amenities +191 +mediated +sewage +trenches +ml +mainz +##thus +1800s +##cula +##inski +caine +bonding +213 +converts +spheres +superseded +marianne +crypt +sweaty +ensign +historia +##br +spruce +##post +##ask +forks +thoughtfully +yukon +pamphlet +ames +##uter +karma +##yya +bryn +negotiation +sighs +incapable +##mbre +##ntial +actresses +taft +##mill +luce +prevailed +##amine +1773 +motionless +envoy +testify +investing +sculpted +instructors +provence +kali +cullen +horseback +##while +goodwin +##jos +gaa +norte +##ldon +modify +wavelength +abd +214 +skinned +sprinter +forecast +scheduling +marries +squared +tentative +##chman +boer +##isch +bolts +swap +fisherman +assyrian +impatiently +guthrie +martins +murdoch +194 +tanya +nicely +dolly +lacy +med +##45 +syn +decks +fashionable +millionaire +##ust +surfing +##ml +##ision +heaved +tammy +consulate +attendees +routinely +197 +fuse +saxophonist +backseat +malaya +##lord +scowl +tau +##ishly +193 +sighted +steaming +##rks +303 +911 +##holes +##hong +ching +##wife +bless +conserved +jurassic +stacey +unix +zion +chunk +rigorous +blaine +198 +peabody +slayer +dismay +brewers +nz +##jer +det +##glia +glover +postwar +int +penetration +sylvester +imitation +vertically +airlift +heiress +knoxville +viva +##uin +390 +macon +##rim +##fighter +##gonal +janice +##orescence +##wari +marius +belongings +leicestershire +196 +blanco +inverted +preseason +sanity +sobbing +##due +##elt +##dled +collingwood +regeneration +flickering +shortest +##mount +##osi +feminism +##lat +sherlock +cabinets +fumbled +northbound +precedent +snaps +##mme +researching +##akes +guillaume +insights +manipulated +vapor +neighbour +sap +gangster +frey +f1 +stalking +scarcely +callie +barnett +tendencies +audi +doomed +assessing +slung +panchayat +ambiguous +bartlett +##etto +distributing +violating +wolverhampton +##hetic +swami +histoire +##urus +liable +pounder +groin +hussain +larsen +popping +surprises +##atter +vie +curt +##station +mute +relocate +musicals +authorization +richter +##sef +immortality +tna +bombings +##press +deteriorated +yiddish +##acious +robbed +colchester +cs +pmid +ao +verified +balancing +apostle +swayed +recognizable +oxfordshire +retention +nottinghamshire +contender +judd +invitational +shrimp +uhf +##icient +cleaner +longitudinal +tanker +##mur +acronym +broker +koppen +sundance +suppliers +##gil +4000 +clipped +fuels +petite +##anne +landslide +helene +diversion +populous +landowners +auspices +melville +quantitative +##xes +ferries +nicky +##llus +doo +haunting +roche +carver +downed +unavailable +##pathy +approximation +hiroshima +##hue +garfield +valle +comparatively +keyboardist +traveler +##eit +congestion +calculating +subsidiaries +##bate +serb +modernization +fairies +deepened +ville +averages +##lore +inflammatory +tonga +##itch +co₂ +squads +##hea +gigantic +serum +enjoyment +retailer +verona +35th +cis +##phobic +magna +technicians +##vati +arithmetic +##sport +levin +##dation +amtrak +chow +sienna +##eyer +backstage +entrepreneurship +##otic +learnt +tao +##udy +worcestershire +formulation +baggage +hesitant +bali +sabotage +##kari +barren +enhancing +murmur +pl +freshly +putnam +syntax +aces +medicines +resentment +bandwidth +##sier +grins +chili +guido +##sei +framing +implying +gareth +lissa +genevieve +pertaining +admissions +geo +thorpe +proliferation +sato +bela +analyzing +parting +##gor +awakened +##isman +huddled +secrecy +##kling +hush +gentry +540 +dungeons +##ego +coasts +##utz +sacrificed +##chule +landowner +mutually +prevalence +programmer +adolescent +disrupted +seaside +gee +trusts +vamp +georgie +##nesian +##iol +schedules +sindh +##market +etched +hm +sparse +bey +beaux +scratching +gliding +unidentified +216 +collaborating +gems +jesuits +oro +accumulation +shaping +mbe +anal +##xin +231 +enthusiasts +newscast +##egan +janata +dewey +parkinson +179 +ankara +biennial +towering +dd +inconsistent +950 +##chet +thriving +terminate +cabins +furiously +eats +advocating +donkey +marley +muster +phyllis +leiden +##user +grassland +glittering +iucn +loneliness +217 +memorandum +armenians +##ddle +popularized +rhodesia +60s +lame +##illon +sans +bikini +header +orbits +##xx +##finger +##ulator +sharif +spines +biotechnology +strolled +naughty +yates +##wire +fremantle +milo +##mour +abducted +removes +##atin +humming +wonderland +##chrome +##ester +hume +pivotal +##rates +armand +grams +believers +elector +rte +apron +bis +scraped +##yria +endorsement +initials +##llation +eps +dotted +hints +buzzing +emigration +nearer +##tom +indicators +##ulu +coarse +neutron +protectorate +##uze +directional +exploits +pains +loire +1830s +proponents +guggenheim +rabbits +ritchie +305 +hectare +inputs +hutton +##raz +verify +##ako +boilers +longitude +##lev +skeletal +yer +emilia +citrus +compromised +##gau +pokemon +prescription +paragraph +eduard +cadillac +attire +categorized +kenyan +weddings +charley +##bourg +entertain +monmouth +##lles +nutrients +davey +mesh +incentive +practised +ecosystems +kemp +subdued +overheard +##rya +bodily +maxim +##nius +apprenticeship +ursula +##fight +lodged +rug +silesian +unconstitutional +patel +inspected +coyote +unbeaten +##hak +34th +disruption +convict +parcel +##cl +##nham +collier +implicated +mallory +##iac +##lab +susannah +winkler +##rber +shia +phelps +sediments +graphical +robotic +##sner +adulthood +mart +smoked +##isto +kathryn +clarified +##aran +divides +convictions +oppression +pausing +burying +##mt +federico +mathias +eileen +##tana +kite +hunched +##acies +189 +##atz +disadvantage +liza +kinetic +greedy +paradox +yokohama +dowager +trunks +ventured +##gement +gupta +vilnius +olaf +##thest +crimean +hopper +##ej +progressively +arturo +mouthed +arrondissement +##fusion +rubin +simulcast +oceania +##orum +##stra +##rred +busiest +intensely +navigator +cary +##vine +##hini +##bies +fife +rowe +rowland +posing +insurgents +shafts +lawsuits +activate +conor +inward +culturally +garlic +265 +##eering +eclectic +##hui +##kee +##nl +furrowed +vargas +meteorological +rendezvous +##aus +culinary +commencement +##dition +quota +##notes +mommy +salaries +overlapping +mule +##iology +##mology +sums +wentworth +##isk +##zione +mainline +subgroup +##illy +hack +plaintiff +verdi +bulb +differentiation +engagements +multinational +supplemented +bertrand +caller +regis +##naire +##sler +##arts +##imated +blossom +propagation +kilometer +viaduct +vineyards +##uate +beckett +optimization +golfer +songwriters +seminal +semitic +thud +volatile +evolving +ridley +##wley +trivial +distributions +scandinavia +jiang +##ject +wrestled +insistence +##dio +emphasizes +napkin +##ods +adjunct +rhyme +##ricted +##eti +hopeless +surrounds +tremble +32nd +smoky +##ntly +oils +medicinal +padded +steer +wilkes +219 +255 +concessions +hue +uniquely +blinded +landon +yahoo +##lane +hendrix +commemorating +dex +specify +chicks +##ggio +intercity +1400 +morley +##torm +highlighting +##oting +pang +oblique +stalled +##liner +flirting +newborn +1769 +bishopric +shaved +232 +currie +##ush +dharma +spartan +##ooped +favorites +smug +novella +sirens +abusive +creations +espana +##lage +paradigm +semiconductor +sheen +##rdo +##yen +##zak +nrl +renew +##pose +##tur +adjutant +marches +norma +##enity +ineffective +weimar +grunt +##gat +lordship +plotting +expenditure +infringement +lbs +refrain +av +mimi +mistakenly +postmaster +1771 +##bara +ras +motorsports +tito +199 +subjective +##zza +bully +stew +##kaya +prescott +1a +##raphic +##zam +bids +styling +paranormal +reeve +sneaking +exploding +katz +akbar +migrant +syllables +indefinitely +##ogical +destroys +replaces +applause +##phine +pest +##fide +218 +articulated +bertie +##thing +##cars +##ptic +courtroom +crowley +aesthetics +cummings +tehsil +hormones +titanic +dangerously +##ibe +stadion +jaenelle +auguste +ciudad +##chu +mysore +partisans +##sio +lucan +philipp +##aly +debating +henley +interiors +##rano +##tious +homecoming +beyonce +usher +henrietta +prepares +weeds +##oman +ely +plucked +##pire +##dable +luxurious +##aq +artifact +password +pasture +juno +maddy +minsk +##dder +##ologies +##rone +assessments +martian +royalist +1765 +examines +##mani +##rge +nino +223 +parry +scooped +relativity +##eli +##uting +##cao +congregational +noisy +traverse +##agawa +strikeouts +nickelodeon +obituary +transylvania +binds +depictions +polk +trolley +##yed +##lard +breeders +##under +dryly +hokkaido +1762 +strengths +stacks +bonaparte +connectivity +neared +prostitutes +stamped +anaheim +gutierrez +sinai +##zzling +bram +fresno +madhya +##86 +proton +##lena +##llum +##phon +reelected +wanda +##anus +##lb +ample +distinguishing +##yler +grasping +sermons +tomato +bland +stimulation +avenues +##eux +spreads +scarlett +fern +pentagon +assert +baird +chesapeake +ir +calmed +distortion +fatalities +##olis +correctional +pricing +##astic +##gina +prom +dammit +ying +collaborate +##chia +welterweight +33rd +pointer +substitution +bonded +umpire +communicating +multitude +paddle +##obe +federally +intimacy +##insky +betray +ssr +##lett +##lean +##lves +##therapy +airbus +##tery +functioned +ud +bearer +biomedical +netflix +##hire +##nca +condom +brink +ik +##nical +macy +##bet +flap +gma +experimented +jelly +lavender +##icles +##ulia +munro +##mian +##tial +rye +##rle +60th +gigs +hottest +rotated +predictions +fuji +bu +##erence +##omi +barangay +##fulness +##sas +clocks +##rwood +##liness +cereal +roe +wight +decker +uttered +babu +onion +xml +forcibly +##df +petra +sarcasm +hartley +peeled +storytelling +##42 +##xley +##ysis +##ffa +fibre +kiel +auditor +fig +harald +greenville +##berries +geographically +nell +quartz +##athic +cemeteries +##lr +crossings +nah +holloway +reptiles +chun +sichuan +snowy +660 +corrections +##ivo +zheng +ambassadors +blacksmith +fielded +fluids +hardcover +turnover +medications +melvin +academies +##erton +ro +roach +absorbing +spaniards +colton +##founded +outsider +espionage +kelsey +245 +edible +##ulf +dora +establishes +##sham +##tries +contracting +##tania +cinematic +costello +nesting +##uron +connolly +duff +##nology +mma +##mata +fergus +sexes +gi +optics +spectator +woodstock +banning +##hee +##fle +differentiate +outfielder +refinery +226 +312 +gerhard +horde +lair +drastically +##udi +landfall +##cheng +motorsport +odi +##achi +predominant +quay +skins +##ental +edna +harshly +complementary +murdering +##aves +wreckage +##90 +ono +outstretched +lennox +munitions +galen +reconcile +470 +scalp +bicycles +gillespie +questionable +rosenberg +guillermo +hostel +jarvis +kabul +volvo +opium +yd +##twined +abuses +decca +outpost +##cino +sensible +neutrality +##64 +ponce +anchorage +atkins +turrets +inadvertently +disagree +libre +vodka +reassuring +weighs +##yal +glide +jumper +ceilings +repertory +outs +stain +##bial +envy +##ucible +smashing +heightened +policing +hyun +mixes +lai +prima +##ples +celeste +##bina +lucrative +intervened +kc +manually +##rned +stature +staffed +bun +bastards +nairobi +priced +##auer +thatcher +##kia +tripped +comune +##ogan +##pled +brasil +incentives +emanuel +hereford +musica +##kim +benedictine +biennale +##lani +eureka +gardiner +rb +knocks +sha +##ael +##elled +##onate +efficacy +ventura +masonic +sanford +maize +leverage +##feit +capacities +santana +##aur +novelty +vanilla +##cter +##tour +benin +##oir +##rain +neptune +drafting +tallinn +##cable +humiliation +##boarding +schleswig +fabian +bernardo +liturgy +spectacle +sweeney +pont +routledge +##tment +cosmos +ut +hilt +sleek +universally +##eville +##gawa +typed +##dry +favors +allegheny +glaciers +##rly +recalling +aziz +##log +parasite +requiem +auf +##berto +##llin +illumination +##breaker +##issa +festivities +bows +govern +vibe +vp +333 +sprawled +larson +pilgrim +bwf +leaping +##rts +##ssel +alexei +greyhound +hoarse +##dler +##oration +seneca +##cule +gaping +##ulously +##pura +cinnamon +##gens +##rricular +craven +fantasies +houghton +engined +reigned +dictator +supervising +##oris +bogota +commentaries +unnatural +fingernails +spirituality +tighten +##tm +canadiens +protesting +intentional +cheers +sparta +##ytic +##iere +##zine +widen +belgarath +controllers +dodd +iaaf +navarre +##ication +defect +squire +steiner +whisky +##mins +560 +inevitably +tome +##gold +chew +##uid +##lid +elastic +##aby +streaked +alliances +jailed +regal +##ined +##phy +czechoslovak +narration +absently +##uld +bluegrass +guangdong +quran +criticizing +hose +hari +##liest +##owa +skier +streaks +deploy +##lom +raft +bose +dialed +huff +##eira +haifa +simplest +bursting +endings +ib +sultanate +##titled +franks +whitman +ensures +sven +##ggs +collaborators +forster +organising +ui +banished +napier +injustice +teller +layered +thump +##otti +roc +battleships +evidenced +fugitive +sadie +robotics +##roud +equatorial +geologist +##iza +yielding +##bron +##sr +internationale +mecca +##diment +sbs +skyline +toad +uploaded +reflective +undrafted +lal +leafs +bayern +##dai +lakshmi +shortlisted +##stick +##wicz +camouflage +donate +af +christi +lau +##acio +disclosed +nemesis +1761 +assemble +straining +northamptonshire +tal +##asi +bernardino +premature +heidi +42nd +coefficients +galactic +reproduce +buzzed +sensations +zionist +monsieur +myrtle +##eme +archery +strangled +musically +viewpoint +antiquities +bei +trailers +seahawks +cured +pee +preferring +tasmanian +lange +sul +##mail +##working +colder +overland +lucivar +massey +gatherings +haitian +##smith +disapproval +flaws +##cco +##enbach +1766 +npr +##icular +boroughs +creole +forums +techno +1755 +dent +abdominal +streetcar +##eson +##stream +procurement +gemini +predictable +##tya +acheron +christoph +feeder +fronts +vendor +bernhard +jammu +tumors +slang +##uber +goaltender +twists +curving +manson +vuelta +mer +peanut +confessions +pouch +unpredictable +allowance +theodor +vascular +##factory +bala +authenticity +metabolic +coughing +nanjing +##cea +pembroke +##bard +splendid +36th +ff +hourly +##ahu +elmer +handel +##ivate +awarding +thrusting +dl +experimentation +##hesion +##46 +caressed +entertained +steak +##rangle +biologist +orphans +baroness +oyster +stepfather +##dridge +mirage +reefs +speeding +##31 +barons +1764 +227 +inhabit +preached +repealed +##tral +honoring +boogie +captives +administer +johanna +##imate +gel +suspiciously +1767 +sobs +##dington +backbone +hayward +garry +##folding +##nesia +maxi +##oof +##ppe +ellison +galileo +##stand +crimea +frenzy +amour +bumper +matrices +natalia +baking +garth +palestinians +##grove +smack +conveyed +ensembles +gardening +##manship +##rup +##stituting +1640 +harvesting +topography +jing +shifters +dormitory +##carriage +##lston +ist +skulls +##stadt +dolores +jewellery +sarawak +##wai +##zier +fences +christy +confinement +tumbling +credibility +fir +stench +##bria +##plication +##nged +##sam +virtues +##belt +marjorie +pba +##eem +##made +celebrates +schooner +agitated +barley +fulfilling +anthropologist +##pro +restrict +novi +regulating +##nent +padres +##rani +##hesive +loyola +tabitha +milky +olson +proprietor +crambidae +guarantees +intercollegiate +ljubljana +hilda +##sko +ignorant +hooded +##lts +sardinia +##lidae +##vation +frontman +privileged +witchcraft +##gp +jammed +laude +poking +##than +bracket +amazement +yunnan +##erus +maharaja +linnaeus +264 +commissioning +milano +peacefully +##logies +akira +rani +regulator +##36 +grasses +##rance +luzon +crows +compiler +gretchen +seaman +edouard +tab +buccaneers +ellington +hamlets +whig +socialists +##anto +directorial +easton +mythological +##kr +##vary +rhineland +semantic +taut +dune +inventions +succeeds +##iter +replication +branched +##pired +jul +prosecuted +kangaroo +penetrated +##avian +middlesbrough +doses +bleak +madam +predatory +relentless +##vili +reluctance +##vir +hailey +crore +silvery +1759 +monstrous +swimmers +transmissions +hawthorn +informing +##eral +toilets +caracas +crouch +kb +##sett +295 +cartel +hadley +##aling +alexia +yvonne +##biology +cinderella +eton +superb +blizzard +stabbing +industrialist +maximus +##gm +##orus +groves +maud +clade +oversized +comedic +##bella +rosen +nomadic +fulham +montane +beverages +galaxies +redundant +swarm +##rot +##folia +##llis +buckinghamshire +fen +bearings +bahadur +##rom +gilles +phased +dynamite +faber +benoit +vip +##ount +##wd +booking +fractured +tailored +anya +spices +westwood +cairns +auditions +inflammation +steamed +##rocity +##acion +##urne +skyla +thereof +watford +torment +archdeacon +transforms +lulu +demeanor +fucked +serge +##sor +mckenna +minas +entertainer +##icide +caress +originate +residue +##sty +1740 +##ilised +##org +beech +##wana +subsidies +##ghton +emptied +gladstone +ru +firefighters +voodoo +##rcle +het +nightingale +tamara +edmond +ingredient +weaknesses +silhouette +285 +compatibility +withdrawing +hampson +##mona +anguish +giggling +##mber +bookstore +##jiang +southernmost +tilting +##vance +bai +economical +rf +briefcase +dreadful +hinted +projections +shattering +totaling +##rogate +analogue +indicted +periodical +fullback +##dman +haynes +##tenberg +##ffs +##ishment +1745 +thirst +stumble +penang +vigorous +##ddling +##kor +##lium +octave +##ove +##enstein +##inen +##ones +siberian +##uti +cbn +repeal +swaying +##vington +khalid +tanaka +unicorn +otago +plastered +lobe +riddle +##rella +perch +##ishing +croydon +filtered +graeme +tripoli +##ossa +crocodile +##chers +sufi +mined +##tung +inferno +lsu +##phi +swelled +utilizes +£2 +cale +periodicals +styx +hike +informally +coop +lund +##tidae +ala +hen +qui +transformations +disposed +sheath +chickens +##cade +fitzroy +sas +silesia +unacceptable +odisha +1650 +sabrina +pe +spokane +ratios +athena +massage +shen +dilemma +##drum +##riz +##hul +corona +doubtful +niall +##pha +##bino +fines +cite +acknowledging +bangor +ballard +bathurst +##resh +huron +mustered +alzheimer +garments +kinase +tyre +warship +##cp +flashback +pulmonary +braun +cheat +kamal +cyclists +constructions +grenades +ndp +traveller +excuses +stomped +signalling +trimmed +futsal +mosques +relevance +##wine +wta +##23 +##vah +##lter +hoc +##riding +optimistic +##´s +deco +sim +interacting +rejecting +moniker +waterways +##ieri +##oku +mayors +gdansk +outnumbered +pearls +##ended +##hampton +fairs +totals +dominating +262 +notions +stairway +compiling +pursed +commodities +grease +yeast +##jong +carthage +griffiths +residual +amc +contraction +laird +sapphire +##marine +##ivated +amalgamation +dissolve +inclination +lyle +packaged +altitudes +suez +canons +graded +lurched +narrowing +boasts +guise +wed +enrico +##ovsky +rower +scarred +bree +cub +iberian +protagonists +bargaining +proposing +trainers +voyages +vans +fishes +##aea +##ivist +##verance +encryption +artworks +kazan +sabre +cleopatra +hepburn +rotting +supremacy +mecklenburg +##brate +burrows +hazards +outgoing +flair +organizes +##ctions +scorpion +##usions +boo +234 +chevalier +dunedin +slapping +##34 +ineligible +pensions +##38 +##omic +manufactures +emails +bismarck +238 +weakening +blackish +ding +mcgee +quo +##rling +northernmost +xx +manpower +greed +sampson +clicking +##ange +##horpe +##inations +##roving +torre +##eptive +##moral +symbolism +38th +asshole +meritorious +outfits +splashed +biographies +sprung +astros +##tale +302 +737 +filly +raoul +nw +tokugawa +linden +clubhouse +##apa +tracts +romano +##pio +putin +tags +##note +chained +dickson +gunshot +moe +gunn +rashid +##tails +zipper +##bas +##nea +contrasted +##ply +##udes +plum +pharaoh +##pile +aw +comedies +ingrid +sandwiches +subdivisions +1100 +mariana +nokia +kamen +hz +delaney +veto +herring +##words +possessive +outlines +##roup +siemens +stairwell +rc +gallantry +messiah +palais +yells +233 +zeppelin +##dm +bolivar +##cede +smackdown +mckinley +##mora +##yt +muted +geologic +finely +unitary +avatar +hamas +maynard +rees +bog +contrasting +##rut +liv +chico +disposition +pixel +##erate +becca +dmitry +yeshiva +narratives +##lva +##ulton +mercenary +sharpe +tempered +navigate +stealth +amassed +keynes +##lini +untouched +##rrie +havoc +lithium +##fighting +abyss +graf +southward +wolverine +balloons +implements +ngos +transitions +##icum +ambushed +concacaf +dormant +economists +##dim +costing +csi +rana +universite +boulders +verity +##llon +collin +mellon +misses +cypress +fluorescent +lifeless +spence +##ulla +crewe +shepard +pak +revelations +##م +jolly +gibbons +paw +##dro +##quel +freeing +##test +shack +fries +palatine +##51 +##hiko +accompaniment +cruising +recycled +##aver +erwin +sorting +synthesizers +dyke +realities +sg +strides +enslaved +wetland +##ghan +competence +gunpowder +grassy +maroon +reactors +objection +##oms +carlson +gearbox +macintosh +radios +shelton +##sho +clergyman +prakash +254 +mongols +trophies +oricon +228 +stimuli +twenty20 +cantonese +cortes +mirrored +##saurus +bhp +cristina +melancholy +##lating +enjoyable +nuevo +##wny +downfall +schumacher +##ind +banging +lausanne +rumbled +paramilitary +reflex +ax +amplitude +migratory +##gall +##ups +midi +barnard +lastly +sherry +##hp +##nall +keystone +##kra +carleton +slippery +##53 +coloring +foe +socket +otter +##rgos +mats +##tose +consultants +bafta +bison +topping +##km +490 +primal +abandonment +transplant +atoll +hideous +mort +pained +reproduced +tae +howling +##turn +unlawful +billionaire +hotter +poised +lansing +##chang +dinamo +retro +messing +nfc +domesday +##mina +blitz +timed +##athing +##kley +ascending +gesturing +##izations +signaled +tis +chinatown +mermaid +savanna +jameson +##aint +catalina +##pet +##hers +cochrane +cy +chatting +##kus +alerted +computation +mused +noelle +majestic +mohawk +campo +octagonal +##sant +##hend +241 +aspiring +##mart +comprehend +iona +paralyzed +shimmering +swindon +rhone +##eley +reputed +configurations +pitchfork +agitation +francais +gillian +lipstick +##ilo +outsiders +pontifical +resisting +bitterness +sewer +rockies +##edd +##ucher +misleading +1756 +exiting +galloway +##nging +risked +##heart +246 +commemoration +schultz +##rka +integrating +##rsa +poses +shrieked +##weiler +guineas +gladys +jerking +owls +goldsmith +nightly +penetrating +##unced +lia +##33 +ignited +betsy +##aring +##thorpe +follower +vigorously +##rave +coded +kiran +knit +zoology +tbilisi +##28 +##bered +repository +govt +deciduous +dino +growling +##bba +enhancement +unleashed +chanting +pussy +biochemistry +##eric +kettle +repression +toxicity +nrhp +##arth +##kko +##bush +ernesto +commended +outspoken +242 +mca +parchment +sms +kristen +##aton +bisexual +raked +glamour +navajo +a2 +conditioned +showcased +##hma +spacious +youthful +##esa +usl +appliances +junta +brest +layne +conglomerate +enchanted +chao +loosened +picasso +circulating +inspect +montevideo +##centric +##kti +piazza +spurred +##aith +bari +freedoms +poultry +stamford +lieu +##ect +indigo +sarcastic +bahia +stump +attach +dvds +frankenstein +lille +approx +scriptures +pollen +##script +nmi +overseen +##ivism +tides +proponent +newmarket +inherit +milling +##erland +centralized +##rou +distributors +credentials +drawers +abbreviation +##lco +##xon +downing +uncomfortably +ripe +##oes +erase +franchises +##ever +populace +##bery +##khar +decomposition +pleas +##tet +daryl +sabah +##stle +##wide +fearless +genie +lesions +annette +##ogist +oboe +appendix +nair +dripped +petitioned +maclean +mosquito +parrot +rpg +hampered +1648 +operatic +reservoirs +##tham +irrelevant +jolt +summarized +##fp +medallion +##taff +##− +clawed +harlow +narrower +goddard +marcia +bodied +fremont +suarez +altering +tempest +mussolini +porn +##isms +sweetly +oversees +walkers +solitude +grimly +shrines +hk +ich +supervisors +hostess +dietrich +legitimacy +brushes +expressive +##yp +dissipated +##rse +localized +systemic +##nikov +gettysburg +##js +##uaries +dialogues +muttering +251 +housekeeper +sicilian +discouraged +##frey +beamed +kaladin +halftime +kidnap +##amo +##llet +1754 +synonymous +depleted +instituto +insulin +reprised +##opsis +clashed +##ctric +interrupting +radcliffe +insisting +medici +1715 +ejected +playfully +turbulent +##47 +starvation +##rini +shipment +rebellious +petersen +verification +merits +##rified +cakes +##charged +1757 +milford +shortages +spying +fidelity +##aker +emitted +storylines +harvested +seismic +##iform +cheung +kilda +theoretically +barbie +lynx +##rgy +##tius +goblin +mata +poisonous +##nburg +reactive +residues +obedience +##евич +conjecture +##rac +401 +hating +sixties +kicker +moaning +motown +##bha +emancipation +neoclassical +##hering +consoles +ebert +professorship +##tures +sustaining +assaults +obeyed +affluent +incurred +tornadoes +##eber +##zow +emphasizing +highlanders +cheated +helmets +##ctus +internship +terence +bony +executions +legislators +berries +peninsular +tinged +##aco +1689 +amplifier +corvette +ribbons +lavish +pennant +##lander +worthless +##chfield +##forms +mariano +pyrenees +expenditures +##icides +chesterfield +mandir +tailor +39th +sergey +nestled +willed +aristocracy +devotees +goodnight +raaf +rumored +weaponry +remy +appropriations +harcourt +burr +riaa +##lence +limitation +unnoticed +guo +soaking +swamps +##tica +collapsing +tatiana +descriptive +brigham +psalm +##chment +maddox +##lization +patti +caliph +##aja +akron +injuring +serra +##ganj +basins +##sari +astonished +launcher +##church +hilary +wilkins +sewing +##sf +stinging +##fia +##ncia +underwood +startup +##ition +compilations +vibrations +embankment +jurist +##nity +bard +juventus +groundwater +kern +palaces +helium +boca +cramped +marissa +soto +##worm +jae +princely +##ggy +faso +bazaar +warmly +##voking +229 +pairing +##lite +##grate +##nets +wien +freaked +ulysses +rebirth +##alia +##rent +mummy +guzman +jimenez +stilled +##nitz +trajectory +tha +woken +archival +professions +##pts +##pta +hilly +shadowy +shrink +##bolt +norwood +glued +migrate +stereotypes +devoid +##pheus +625 +evacuate +horrors +infancy +gotham +knowles +optic +downloaded +sachs +kingsley +parramatta +darryl +mor +##onale +shady +commence +confesses +kan +##meter +##placed +marlborough +roundabout +regents +frigates +io +##imating +gothenburg +revoked +carvings +clockwise +convertible +intruder +##sche +banged +##ogo +vicky +bourgeois +##mony +dupont +footing +##gum +pd +##real +buckle +yun +penthouse +sane +720 +serviced +stakeholders +neumann +bb +##eers +comb +##gam +catchment +pinning +rallies +typing +##elles +forefront +freiburg +sweetie +giacomo +widowed +goodwill +worshipped +aspirations +midday +##vat +fishery +##trick +bournemouth +turk +243 +hearth +ethanol +guadalajara +murmurs +sl +##uge +afforded +scripted +##hta +wah +##jn +coroner +translucent +252 +memorials +puck +progresses +clumsy +##race +315 +candace +recounted +##27 +##slin +##uve +filtering +##mac +howl +strata +heron +leveled +##ays +dubious +##oja +##т +##wheel +citations +exhibiting +##laya +##mics +##pods +turkic +##lberg +injunction +##ennial +##mit +antibodies +##44 +organise +##rigues +cardiovascular +cushion +inverness +##zquez +dia +cocoa +sibling +##tman +##roid +expanse +feasible +tunisian +algiers +##relli +rus +bloomberg +dso +westphalia +bro +tacoma +281 +downloads +##ours +konrad +duran +##hdi +continuum +jett +compares +legislator +secession +##nable +##gues +##zuka +translating +reacher +##gley +##ła +aleppo +##agi +tc +orchards +trapping +linguist +versatile +drumming +postage +calhoun +superiors +##mx +barefoot +leary +##cis +ignacio +alfa +kaplan +##rogen +bratislava +mori +##vot +disturb +haas +313 +cartridges +gilmore +radiated +salford +tunic +hades +##ulsive +archeological +delilah +magistrates +auditioned +brewster +charters +empowerment +blogs +cappella +dynasties +iroquois +whipping +##krishna +raceway +truths +myra +weaken +judah +mcgregor +##horse +mic +refueling +37th +burnley +bosses +markus +premio +query +##gga +dunbar +##economic +darkest +lyndon +sealing +commendation +reappeared +##mun +addicted +ezio +slaughtered +satisfactory +shuffle +##eves +##thic +##uj +fortification +warrington +##otto +resurrected +fargo +mane +##utable +##lei +##space +foreword +ox +##aris +##vern +abrams +hua +##mento +sakura +##alo +uv +sentimental +##skaya +midfield +##eses +sturdy +scrolls +macleod +##kyu +entropy +##lance +mitochondrial +cicero +excelled +thinner +convoys +perceive +##oslav +##urable +systematically +grind +burkina +287 +##tagram +ops +##aman +guantanamo +##cloth +##tite +forcefully +wavy +##jou +pointless +##linger +##tze +layton +portico +superficial +clerical +outlaws +##hism +burials +muir +##inn +creditors +hauling +rattle +##leg +calais +monde +archers +reclaimed +dwell +wexford +hellenic +falsely +remorse +##tek +dough +furnishings +##uttered +gabon +neurological +novice +##igraphy +contemplated +pulpit +nightstand +saratoga +##istan +documenting +pulsing +taluk +##firmed +busted +marital +##rien +disagreements +wasps +##yes +hodge +mcdonnell +mimic +fran +pendant +dhabi +musa +##nington +congratulations +argent +darrell +concussion +losers +regrets +thessaloniki +reversal +donaldson +hardwood +thence +achilles +ritter +##eran +demonic +jurgen +prophets +goethe +eki +classmate +buff +##cking +yank +irrational +##inging +perished +seductive +qur +sourced +##crat +##typic +mustard +ravine +barre +horizontally +characterization +phylogenetic +boise +##dit +##runner +##tower +brutally +intercourse +seduce +##bbing +fay +ferris +ogden +amar +nik +unarmed +##inator +evaluating +kyrgyzstan +sweetness +##lford +##oki +mccormick +meiji +notoriety +stimulate +disrupt +figuring +instructional +mcgrath +##zoo +groundbreaking +##lto +flinch +khorasan +agrarian +bengals +mixer +radiating +##sov +ingram +pitchers +nad +tariff +##cript +tata +##codes +##emi +##ungen +appellate +lehigh +##bled +##giri +brawl +duct +texans +##ciation +##ropolis +skipper +speculative +vomit +doctrines +stresses +253 +davy +graders +whitehead +jozef +timely +cumulative +haryana +paints +appropriately +boon +cactus +##ales +##pid +dow +legions +##pit +perceptions +1730 +picturesque +##yse +periphery +rune +wr +##aha +celtics +sentencing +whoa +##erin +confirms +variance +425 +moines +mathews +spade +rave +m1 +fronted +fx +blending +alleging +reared +##gl +237 +##paper +grassroots +eroded +##free +##physical +directs +ordeal +##sław +accelerate +hacker +rooftop +##inia +lev +buys +cebu +devote +##lce +specialising +##ulsion +choreographed +repetition +warehouses +##ryl +paisley +tuscany +analogy +sorcerer +hash +huts +shards +descends +exclude +nix +chaplin +gaga +ito +vane +##drich +causeway +misconduct +limo +orchestrated +glands +jana +##kot +u2 +##mple +##sons +branching +contrasts +scoop +longed +##virus +chattanooga +##75 +syrup +cornerstone +##tized +##mind +##iaceae +careless +precedence +frescoes +##uet +chilled +consult +modelled +snatch +peat +##thermal +caucasian +humane +relaxation +spins +temperance +##lbert +occupations +lambda +hybrids +moons +mp3 +##oese +247 +rolf +societal +yerevan +ness +##ssler +befriended +mechanized +nominate +trough +boasted +cues +seater +##hom +bends +##tangle +conductors +emptiness +##lmer +eurasian +adriatic +tian +##cie +anxiously +lark +propellers +chichester +jock +ev +2a +##holding +credible +recounts +tori +loyalist +abduction +##hoot +##redo +nepali +##mite +ventral +tempting +##ango +##crats +steered +##wice +javelin +dipping +laborers +prentice +looming +titanium +##ː +badges +emir +tensor +##ntation +egyptians +rash +denies +hawthorne +lombard +showers +wehrmacht +dietary +trojan +##reus +welles +executing +horseshoe +lifeboat +##lak +elsa +infirmary +nearing +roberta +boyer +mutter +trillion +joanne +##fine +##oked +sinks +vortex +uruguayan +clasp +sirius +##block +accelerator +prohibit +sunken +byu +chronological +diplomats +ochreous +510 +symmetrical +1644 +maia +##tology +salts +reigns +atrocities +##ия +hess +bared +issn +##vyn +cater +saturated +##cycle +##isse +sable +voyager +dyer +yusuf +##inge +fountains +wolff +##39 +##nni +engraving +rollins +atheist +ominous +##ault +herr +chariot +martina +strung +##fell +##farlane +horrific +sahib +gazes +saetan +erased +ptolemy +##olic +flushing +lauderdale +analytic +##ices +530 +navarro +beak +gorilla +herrera +broom +guadalupe +raiding +sykes +311 +bsc +deliveries +1720 +invasions +carmichael +tajikistan +thematic +ecumenical +sentiments +onstage +##rians +##brand +##sume +catastrophic +flanks +molten +##arns +waller +aimee +terminating +##icing +alternately +##oche +nehru +printers +outraged +##eving +empires +template +banners +repetitive +za +##oise +vegetarian +##tell +guiana +opt +cavendish +lucknow +synthesized +##hani +##mada +finalized +##ctable +fictitious +mayoral +unreliable +##enham +embracing +peppers +rbis +##chio +##neo +inhibition +slashed +togo +orderly +embroidered +safari +salty +236 +barron +benito +totaled +##dak +pubs +simulated +caden +devin +tolkien +momma +welding +sesame +##ept +gottingen +hardness +630 +shaman +temeraire +620 +adequately +pediatric +##kit +ck +assertion +radicals +composure +cadence +seafood +beaufort +lazarus +mani +warily +cunning +kurdistan +249 +cantata +##kir +ares +##41 +##clusive +nape +townland +geared +insulted +flutter +boating +violate +draper +dumping +malmo +##hh +##romatic +firearm +alta +bono +obscured +##clave +exceeds +panorama +unbelievable +##train +preschool +##essed +disconnected +installing +rescuing +secretaries +accessibility +##castle +##drive +##ifice +##film +bouts +slug +waterway +mindanao +##buro +##ratic +halves +##ل +calming +liter +maternity +adorable +bragg +electrification +mcc +##dote +roxy +schizophrenia +##body +munoz +kaye +whaling +239 +mil +tingling +tolerant +##ago +unconventional +volcanoes +##finder +deportivo +##llie +robson +kaufman +neuroscience +wai +deportation +masovian +scraping +converse +##bh +hacking +bulge +##oun +administratively +yao +580 +amp +mammoth +booster +claremont +hooper +nomenclature +pursuits +mclaughlin +melinda +##sul +catfish +barclay +substrates +taxa +zee +originals +kimberly +packets +padma +##ality +borrowing +ostensibly +solvent +##bri +##genesis +##mist +lukas +shreveport +veracruz +##ь +##lou +##wives +cheney +tt +anatolia +hobbs +##zyn +cyclic +radiant +alistair +greenish +siena +dat +independents +##bation +conform +pieter +hyper +applicant +bradshaw +spores +telangana +vinci +inexpensive +nuclei +322 +jang +nme +soho +spd +##ign +cradled +receptionist +pow +##43 +##rika +fascism +##ifer +experimenting +##ading +##iec +##region +345 +jocelyn +maris +stair +nocturnal +toro +constabulary +elgin +##kker +msc +##giving +##schen +##rase +doherty +doping +sarcastically +batter +maneuvers +##cano +##apple +##gai +##git +intrinsic +##nst +##stor +1753 +showtime +cafes +gasps +lviv +ushered +##thed +fours +restart +astonishment +transmitting +flyer +shrugs +##sau +intriguing +cones +dictated +mushrooms +medial +##kovsky +##elman +escorting +gaped +##26 +godfather +##door +##sell +djs +recaptured +timetable +vila +1710 +3a +aerodrome +mortals +scientology +##orne +angelina +mag +convection +unpaid +insertion +intermittent +lego +##nated +endeavor +kota +pereira +##lz +304 +bwv +glamorgan +insults +agatha +fey +##cend +fleetwood +mahogany +protruding +steamship +zeta +##arty +mcguire +suspense +##sphere +advising +urges +##wala +hurriedly +meteor +gilded +inline +arroyo +stalker +##oge +excitedly +revered +##cure +earle +introductory +##break +##ilde +mutants +puff +pulses +reinforcement +##haling +curses +lizards +stalk +correlated +##fixed +fallout +macquarie +##unas +bearded +denton +heaving +802 +##ocation +winery +assign +dortmund +##lkirk +everest +invariant +charismatic +susie +##elling +bled +lesley +telegram +sumner +bk +##ogen +##к +wilcox +needy +colbert +duval +##iferous +##mbled +allotted +attends +imperative +##hita +replacements +hawker +##inda +insurgency +##zee +##eke +casts +##yla +680 +ives +transitioned +##pack +##powering +authoritative +baylor +flex +cringed +plaintiffs +woodrow +##skie +drastic +ape +aroma +unfolded +commotion +nt +preoccupied +theta +routines +lasers +privatization +wand +domino +ek +clenching +nsa +strategically +showered +bile +handkerchief +pere +storing +christophe +insulting +316 +nakamura +romani +asiatic +magdalena +palma +cruises +stripping +405 +konstantin +soaring +##berman +colloquially +forerunner +havilland +incarcerated +parasites +sincerity +##utus +disks +plank +saigon +##ining +corbin +homo +ornaments +powerhouse +##tlement +chong +fastened +feasibility +idf +morphological +usable +##nish +##zuki +aqueduct +jaguars +keepers +##flies +aleksandr +faust +assigns +ewing +bacterium +hurled +tricky +hungarians +integers +wallis +321 +yamaha +##isha +hushed +oblivion +aviator +evangelist +friars +##eller +monograph +ode +##nary +airplanes +labourers +charms +##nee +1661 +hagen +tnt +rudder +fiesta +transcript +dorothea +ska +inhibitor +maccabi +retorted +raining +encompassed +clauses +menacing +1642 +lineman +##gist +vamps +##ape +##dick +gloom +##rera +dealings +easing +seekers +##nut +##pment +helens +unmanned +##anu +##isson +basics +##amy +##ckman +adjustments +1688 +brutality +horne +##zell +sui +##55 +##mable +aggregator +##thal +rhino +##drick +##vira +counters +zoom +##01 +##rting +mn +montenegrin +packard +##unciation +##♭ +##kki +reclaim +scholastic +thugs +pulsed +##icia +syriac +quan +saddam +banda +kobe +blaming +buddies +dissent +##lusion +##usia +corbett +jaya +delle +erratic +lexie +##hesis +435 +amiga +hermes +##pressing +##leen +chapels +gospels +jamal +##uating +compute +revolving +warp +##sso +##thes +armory +##eras +##gol +antrim +loki +##kow +##asian +##good +##zano +braid +handwriting +subdistrict +funky +pantheon +##iculate +concurrency +estimation +improper +juliana +##his +newcomers +johnstone +staten +communicated +##oco +##alle +sausage +stormy +##stered +##tters +superfamily +##grade +acidic +collateral +tabloid +##oped +##rza +bladder +austen +##ellant +mcgraw +##hay +hannibal +mein +aquino +lucifer +wo +badger +boar +cher +christensen +greenberg +interruption +##kken +jem +244 +mocked +bottoms +cambridgeshire +##lide +sprawling +##bbly +eastwood +ghent +synth +##buck +advisers +##bah +nominally +hapoel +qu +daggers +estranged +fabricated +towels +vinnie +wcw +misunderstanding +anglia +nothin +unmistakable +##dust +##lova +chilly +marquette +truss +##edge +##erine +reece +##lty +##chemist +##connected +272 +308 +41st +bash +raion +waterfalls +##ump +##main +labyrinth +queue +theorist +##istle +bharatiya +flexed +soundtracks +rooney +leftist +patrolling +wharton +plainly +alleviate +eastman +schuster +topographic +engages +immensely +unbearable +fairchild +1620 +dona +lurking +parisian +oliveira +ia +indictment +hahn +bangladeshi +##aster +vivo +##uming +##ential +antonia +expects +indoors +kildare +harlan +##logue +##ogenic +##sities +forgiven +##wat +childish +tavi +##mide +##orra +plausible +grimm +successively +scooted +##bola +##dget +##rith +spartans +emery +flatly +azure +epilogue +##wark +flourish +##iny +##tracted +##overs +##oshi +bestseller +distressed +receipt +spitting +hermit +topological +##cot +drilled +subunit +francs +##layer +eel +##fk +##itas +octopus +footprint +petitions +ufo +##say +##foil +interfering +leaking +palo +##metry +thistle +valiant +##pic +narayan +mcpherson +##fast +gonzales +##ym +##enne +dustin +novgorod +solos +##zman +doin +##raph +##patient +##meyer +soluble +ashland +cuffs +carole +pendleton +whistling +vassal +##river +deviation +revisited +constituents +rallied +rotate +loomed +##eil +##nting +amateurs +augsburg +auschwitz +crowns +skeletons +##cona +bonnet +257 +dummy +globalization +simeon +sleeper +mandal +differentiated +##crow +##mare +milne +bundled +exasperated +talmud +owes +segregated +##feng +##uary +dentist +piracy +props +##rang +devlin +##torium +malicious +paws +##laid +dependency +##ergy +##fers +##enna +258 +pistons +rourke +jed +grammatical +tres +maha +wig +512 +ghostly +jayne +##achal +##creen +##ilis +##lins +##rence +designate +##with +arrogance +cambodian +clones +showdown +throttle +twain +##ception +lobes +metz +nagoya +335 +braking +##furt +385 +roaming +##minster +amin +crippled +##37 +##llary +indifferent +hoffmann +idols +intimidating +1751 +261 +influenza +memo +onions +1748 +bandage +consciously +##landa +##rage +clandestine +observes +swiped +tangle +##ener +##jected +##trum +##bill +##lta +hugs +congresses +josiah +spirited +##dek +humanist +managerial +filmmaking +inmate +rhymes +debuting +grimsby +ur +##laze +duplicate +vigor +##tf +republished +bolshevik +refurbishment +antibiotics +martini +methane +newscasts +royale +horizons +levant +iain +visas +##ischen +paler +##around +manifestation +snuck +alf +chop +futile +pedestal +rehab +##kat +bmg +kerman +res +fairbanks +jarrett +abstraction +saharan +##zek +1746 +procedural +clearer +kincaid +sash +luciano +##ffey +crunch +helmut +##vara +revolutionaries +##tute +creamy +leach +##mmon +1747 +permitting +nes +plight +wendell +##lese +contra +ts +clancy +ipa +mach +staples +autopsy +disturbances +nueva +karin +pontiac +##uding +proxy +venerable +haunt +leto +bergman +expands +##helm +wal +##pipe +canning +celine +cords +obesity +##enary +intrusion +planner +##phate +reasoned +sequencing +307 +harrow +##chon +##dora +marred +mcintyre +repay +tarzan +darting +248 +harrisburg +margarita +repulsed +##hur +##lding +belinda +hamburger +novo +compliant +runways +bingham +registrar +skyscraper +ic +cuthbert +improvisation +livelihood +##corp +##elial +admiring +##dened +sporadic +believer +casablanca +popcorn +##29 +asha +shovel +##bek +##dice +coiled +tangible +##dez +casper +elsie +resin +tenderness +rectory +##ivision +avail +sonar +##mori +boutique +##dier +guerre +bathed +upbringing +vaulted +sandals +blessings +##naut +##utnant +1680 +306 +foxes +pia +corrosion +hesitantly +confederates +crystalline +footprints +shapiro +tirana +valentin +drones +45th +microscope +shipments +texted +inquisition +wry +guernsey +unauthorized +resigning +760 +ripple +schubert +stu +reassure +felony +##ardo +brittle +koreans +##havan +##ives +dun +implicit +tyres +##aldi +##lth +magnolia +##ehan +##puri +##poulos +aggressively +fei +gr +familiarity +##poo +indicative +##trust +fundamentally +jimmie +overrun +395 +anchors +moans +##opus +britannia +armagh +##ggle +purposely +seizing +##vao +bewildered +mundane +avoidance +cosmopolitan +geometridae +quartermaster +caf +415 +chatter +engulfed +gleam +purge +##icate +juliette +jurisprudence +guerra +revisions +##bn +casimir +brew +##jm +1749 +clapton +cloudy +conde +hermitage +278 +simulations +torches +vincenzo +matteo +##rill +hidalgo +booming +westbound +accomplishment +tentacles +unaffected +##sius +annabelle +flopped +sloping +##litz +dreamer +interceptor +vu +##loh +consecration +copying +messaging +breaker +climates +hospitalized +1752 +torino +afternoons +winfield +witnessing +##teacher +breakers +choirs +sawmill +coldly +##ege +sipping +haste +uninhabited +conical +bibliography +pamphlets +severn +edict +##oca +deux +illnesses +grips +##pl +rehearsals +sis +thinkers +tame +##keepers +1690 +acacia +reformer +##osed +##rys +shuffling +##iring +##shima +eastbound +ionic +rhea +flees +littered +##oum +rocker +vomiting +groaning +champ +overwhelmingly +civilizations +paces +sloop +adoptive +##tish +skaters +##vres +aiding +mango +##joy +nikola +shriek +##ignon +pharmaceuticals +##mg +tuna +calvert +gustavo +stocked +yearbook +##urai +##mana +computed +subsp +riff +hanoi +kelvin +hamid +moors +pastures +summons +jihad +nectar +##ctors +bayou +untitled +pleasing +vastly +republics +intellect +##η +##ulio +##tou +crumbling +stylistic +sb +##ی +consolation +frequented +h₂o +walden +widows +##iens +404 +##ignment +chunks +improves +288 +grit +recited +##dev +snarl +sociological +##arte +##gul +inquired +##held +bruise +clube +consultancy +homogeneous +hornets +multiplication +pasta +prick +savior +##grin +##kou +##phile +yoon +##gara +grimes +vanishing +cheering +reacting +bn +distillery +##quisite +##vity +coe +dockyard +massif +##jord +escorts +voss +##valent +byte +chopped +hawke +illusions +workings +floats +##koto +##vac +kv +annapolis +madden +##onus +alvaro +noctuidae +##cum +##scopic +avenge +steamboat +forte +illustrates +erika +##trip +570 +dew +nationalities +bran +manifested +thirsty +diversified +muscled +reborn +##standing +arson +##lessness +##dran +##logram +##boys +##kushima +##vious +willoughby +##phobia +286 +alsace +dashboard +yuki +##chai +granville +myspace +publicized +tricked +##gang +adjective +##ater +relic +reorganisation +enthusiastically +indications +saxe +##lassified +consolidate +iec +padua +helplessly +ramps +renaming +regulars +pedestrians +accents +convicts +inaccurate +lowers +mana +##pati +barrie +bjp +outta +someplace +berwick +flanking +invoked +marrow +sparsely +excerpts +clothed +rei +##ginal +wept +##straße +##vish +alexa +excel +##ptive +membranes +aquitaine +creeks +cutler +sheppard +implementations +ns +##dur +fragrance +budge +concordia +magnesium +marcelo +##antes +gladly +vibrating +##rral +##ggles +montrose +##omba +lew +seamus +1630 +cocky +##ament +##uen +bjorn +##rrick +fielder +fluttering +##lase +methyl +kimberley +mcdowell +reductions +barbed +##jic +##tonic +aeronautical +condensed +distracting +##promising +huffed +##cala +##sle +claudius +invincible +missy +pious +balthazar +ci +##lang +butte +combo +orson +##dication +myriad +1707 +silenced +##fed +##rh +coco +netball +yourselves +##oza +clarify +heller +peg +durban +etudes +offender +roast +blackmail +curvature +##woods +vile +309 +illicit +suriname +##linson +overture +1685 +bubbling +gymnast +tucking +##mming +##ouin +maldives +##bala +gurney +##dda +##eased +##oides +backside +pinto +jars +racehorse +tending +##rdial +baronetcy +wiener +duly +##rke +barbarian +cupping +flawed +##thesis +bertha +pleistocene +puddle +swearing +##nob +##tically +fleeting +prostate +amulet +educating +##mined +##iti +##tler +75th +jens +respondents +analytics +cavaliers +papacy +raju +##iente +##ulum +##tip +funnel +271 +disneyland +##lley +sociologist +##iam +2500 +faulkner +louvre +menon +##dson +276 +##ower +afterlife +mannheim +peptide +referees +comedians +meaningless +##anger +##laise +fabrics +hurley +renal +sleeps +##bour +##icle +breakout +kristin +roadside +animator +clover +disdain +unsafe +redesign +##urity +firth +barnsley +portage +reset +narrows +268 +commandos +expansive +speechless +tubular +##lux +essendon +eyelashes +smashwords +##yad +##bang +##claim +craved +sprinted +chet +somme +astor +wrocław +orton +266 +bane +##erving +##uing +mischief +##amps +##sund +scaling +terre +##xious +impairment +offenses +undermine +moi +soy +contiguous +arcadia +inuit +seam +##tops +macbeth +rebelled +##icative +##iot +590 +elaborated +frs +uniformed +##dberg +259 +powerless +priscilla +stimulated +980 +qc +arboretum +frustrating +trieste +bullock +##nified +enriched +glistening +intern +##adia +locus +nouvelle +ollie +ike +lash +starboard +ee +tapestry +headlined +hove +rigged +##vite +pollock +##yme +thrive +clustered +cas +roi +gleamed +olympiad +##lino +pressured +regimes +##hosis +##lick +ripley +##ophone +kickoff +gallon +rockwell +##arable +crusader +glue +revolutions +scrambling +1714 +grover +##jure +englishman +aztec +263 +contemplating +coven +ipad +preach +triumphant +tufts +##esian +rotational +##phus +328 +falkland +##brates +strewn +clarissa +rejoin +environmentally +glint +banded +drenched +moat +albanians +johor +rr +maestro +malley +nouveau +shaded +taxonomy +v6 +adhere +bunk +airfields +##ritan +1741 +encompass +remington +tran +##erative +amelie +mazda +friar +morals +passions +##zai +breadth +vis +##hae +argus +burnham +caressing +insider +rudd +##imov +##mini +##rso +italianate +murderous +textual +wainwright +armada +bam +weave +timer +##taken +##nh +fra +##crest +ardent +salazar +taps +tunis +##ntino +allegro +gland +philanthropic +##chester +implication +##optera +esq +judas +noticeably +wynn +##dara +inched +indexed +crises +villiers +bandit +royalties +patterned +cupboard +interspersed +accessory +isla +kendrick +entourage +stitches +##esthesia +headwaters +##ior +interlude +distraught +draught +1727 +##basket +biased +sy +transient +triad +subgenus +adapting +kidd +shortstop +##umatic +dimly +spiked +mcleod +reprint +nellie +pretoria +windmill +##cek +singled +##mps +273 +reunite +##orous +747 +bankers +outlying +##omp +##ports +##tream +apologies +cosmetics +patsy +##deh +##ocks +##yson +bender +nantes +serene +##nad +lucha +mmm +323 +##cius +##gli +cmll +coinage +nestor +juarez +##rook +smeared +sprayed +twitching +sterile +irina +embodied +juveniles +enveloped +miscellaneous +cancers +dq +gulped +luisa +crested +swat +donegal +ref +##anov +##acker +hearst +mercantile +##lika +doorbell +ua +vicki +##alla +##som +bilbao +psychologists +stryker +sw +horsemen +turkmenistan +wits +##national +anson +mathew +screenings +##umb +rihanna +##agne +##nessy +aisles +##iani +##osphere +hines +kenton +saskatoon +tasha +truncated +##champ +##itan +mildred +advises +fredrik +interpreting +inhibitors +##athi +spectroscopy +##hab +##kong +karim +panda +##oia +##nail +##vc +conqueror +kgb +leukemia +##dity +arrivals +cheered +pisa +phosphorus +shielded +##riated +mammal +unitarian +urgently +chopin +sanitary +##mission +spicy +drugged +hinges +##tort +tipping +trier +impoverished +westchester +##caster +267 +epoch +nonstop +##gman +##khov +aromatic +centrally +cerro +##tively +##vio +billions +modulation +sedimentary +283 +facilitating +outrageous +goldstein +##eak +##kt +ld +maitland +penultimate +pollard +##dance +fleets +spaceship +vertebrae +##nig +alcoholism +als +recital +##bham +##ference +##omics +m2 +##bm +trois +##tropical +##в +commemorates +##meric +marge +##raction +1643 +670 +cosmetic +ravaged +##ige +catastrophe +eng +##shida +albrecht +arterial +bellamy +decor +harmon +##rde +bulbs +synchronized +vito +easiest +shetland +shielding +wnba +##glers +##ssar +##riam +brianna +cumbria +##aceous +##rard +cores +thayer +##nsk +brood +hilltop +luminous +carts +keynote +larkin +logos +##cta +##ا +##mund +##quay +lilith +tinted +277 +wrestle +mobilization +##uses +sequential +siam +bloomfield +takahashi +274 +##ieving +presenters +ringo +blazed +witty +##oven +##ignant +devastation +haydn +harmed +newt +therese +##peed +gershwin +molina +rabbis +sudanese +001 +innate +restarted +##sack +##fus +slices +wb +##shah +enroll +hypothetical +hysterical +1743 +fabio +indefinite +warped +##hg +exchanging +525 +unsuitable +##sboro +gallo +1603 +bret +cobalt +homemade +##hunter +mx +operatives +##dhar +terraces +durable +latch +pens +whorls +##ctuated +##eaux +billing +ligament +succumbed +##gly +regulators +spawn +##brick +##stead +filmfare +rochelle +##nzo +1725 +circumstance +saber +supplements +##nsky +##tson +crowe +wellesley +carrot +##9th +##movable +primate +drury +sincerely +topical +##mad +##rao +callahan +kyiv +smarter +tits +undo +##yeh +announcements +anthologies +barrio +nebula +##islaus +##shaft +##tyn +bodyguards +2021 +assassinate +barns +emmett +scully +##mah +##yd +##eland +##tino +##itarian +demoted +gorman +lashed +prized +adventist +writ +##gui +alla +invertebrates +##ausen +1641 +amman +1742 +align +healy +redistribution +##gf +##rize +insulation +##drop +adherents +hezbollah +vitro +ferns +yanking +269 +php +registering +uppsala +cheerleading +confines +mischievous +tully +##ross +49th +docked +roam +stipulated +pumpkin +##bry +prompt +##ezer +blindly +shuddering +craftsmen +frail +scented +katharine +scramble +shaggy +sponge +helix +zaragoza +279 +##52 +43rd +backlash +fontaine +seizures +posse +cowan +nonfiction +telenovela +wwii +hammered +undone +##gpur +encircled +irs +##ivation +artefacts +oneself +searing +smallpox +##belle +##osaurus +shandong +breached +upland +blushing +rankin +infinitely +psyche +tolerated +docking +evicted +##col +unmarked +##lving +gnome +lettering +litres +musique +##oint +benevolent +##jal +blackened +##anna +mccall +racers +tingle +##ocene +##orestation +introductions +radically +292 +##hiff +##باد +1610 +1739 +munchen +plead +##nka +condo +scissors +##sight +##tens +apprehension +##cey +##yin +hallmark +watering +formulas +sequels +##llas +aggravated +bae +commencing +##building +enfield +prohibits +marne +vedic +civilized +euclidean +jagger +beforehand +blasts +dumont +##arney +##nem +740 +conversions +hierarchical +rios +simulator +##dya +##lellan +hedges +oleg +thrusts +shadowed +darby +maximize +1744 +gregorian +##nded +##routed +sham +unspecified +##hog +emory +factual +##smo +##tp +fooled +##rger +ortega +wellness +marlon +##oton +##urance +casket +keating +ley +enclave +##ayan +char +influencing +jia +##chenko +412 +ammonia +erebidae +incompatible +violins +cornered +##arat +grooves +astronauts +columbian +rampant +fabrication +kyushu +mahmud +vanish +##dern +mesopotamia +##lete +ict +##rgen +caspian +kenji +pitted +##vered +999 +grimace +roanoke +tchaikovsky +twinned +##analysis +##awan +xinjiang +arias +clemson +kazakh +sizable +1662 +##khand +##vard +plunge +tatum +vittorio +##nden +cholera +##dana +##oper +bracing +indifference +projectile +superliga +##chee +realises +upgrading +299 +porte +retribution +##vies +nk +stil +##resses +ama +bureaucracy +blackberry +bosch +testosterone +collapses +greer +##pathic +ioc +fifties +malls +##erved +bao +baskets +adolescents +siegfried +##osity +##tosis +mantra +detecting +existent +fledgling +##cchi +dissatisfied +gan +telecommunication +mingled +sobbed +6000 +controversies +outdated +taxis +##raus +fright +slams +##lham +##fect +##tten +detectors +fetal +tanned +##uw +fray +goth +olympian +skipping +mandates +scratches +sheng +unspoken +hyundai +tracey +hotspur +restrictive +##buch +americana +mundo +##bari +burroughs +diva +vulcan +##6th +distinctions +thumping +##ngen +mikey +sheds +fide +rescues +springsteen +vested +valuation +##ece +##ely +pinnacle +rake +sylvie +##edo +almond +quivering +##irus +alteration +faltered +##wad +51st +hydra +ticked +##kato +recommends +##dicated +antigua +arjun +stagecoach +wilfred +trickle +pronouns +##pon +aryan +nighttime +##anian +gall +pea +stitch +##hei +leung +milos +##dini +eritrea +nexus +starved +snowfall +kant +parasitic +cot +discus +hana +strikers +appleton +kitchens +##erina +##partisan +##itha +##vius +disclose +metis +##channel +1701 +tesla +##vera +fitch +1735 +blooded +##tila +decimal +##tang +##bai +cyclones +eun +bottled +peas +pensacola +basha +bolivian +crabs +boil +lanterns +partridge +roofed +1645 +necks +##phila +opined +patting +##kla +##lland +chuckles +volta +whereupon +##nche +devout +euroleague +suicidal +##dee +inherently +involuntary +knitting +nasser +##hide +puppets +colourful +courageous +southend +stills +miraculous +hodgson +richer +rochdale +ethernet +greta +uniting +prism +umm +##haya +##itical +##utation +deterioration +pointe +prowess +##ropriation +lids +scranton +billings +subcontinent +##koff +##scope +brute +kellogg +psalms +degraded +##vez +stanisław +##ructured +ferreira +pun +astonishing +gunnar +##yat +arya +prc +gottfried +##tight +excursion +##ographer +dina +##quil +##nare +huffington +illustrious +wilbur +gundam +verandah +##zard +naacp +##odle +constructive +fjord +kade +##naud +generosity +thrilling +baseline +cayman +frankish +plastics +accommodations +zoological +##fting +cedric +qb +motorized +##dome +##otted +squealed +tackled +canucks +budgets +situ +asthma +dail +gabled +grasslands +whimpered +writhing +judgments +##65 +minnie +pv +##carbon +bananas +grille +domes +monique +odin +maguire +markham +tierney +##estra +##chua +libel +poke +speedy +atrium +laval +notwithstanding +##edly +fai +kala +##sur +robb +##sma +listings +luz +supplementary +tianjin +##acing +enzo +jd +ric +scanner +croats +transcribed +##49 +arden +cv +##hair +##raphy +##lver +##uy +357 +seventies +staggering +alam +horticultural +hs +regression +timbers +blasting +##ounded +montagu +manipulating +##cit +catalytic +1550 +troopers +##meo +condemnation +fitzpatrick +##oire +##roved +inexperienced +1670 +castes +##lative +outing +314 +dubois +flicking +quarrel +ste +learners +1625 +iq +whistled +##class +282 +classify +tariffs +temperament +355 +folly +liszt +##yles +immersed +jordanian +ceasefire +apparel +extras +maru +fished +##bio +harta +stockport +assortment +craftsman +paralysis +transmitters +##cola +blindness +##wk +fatally +proficiency +solemnly +##orno +repairing +amore +groceries +ultraviolet +##chase +schoolhouse +##tua +resurgence +nailed +##otype +##× +ruse +saliva +diagrams +##tructing +albans +rann +thirties +1b +antennas +hilarious +cougars +paddington +stats +##eger +breakaway +ipod +reza +authorship +prohibiting +scoffed +##etz +##ttle +conscription +defected +trondheim +##fires +ivanov +keenan +##adan +##ciful +##fb +##slow +locating +##ials +##tford +cadiz +basalt +blankly +interned +rags +rattling +##tick +carpathian +reassured +sync +bum +guildford +iss +staunch +##onga +astronomers +sera +sofie +emergencies +susquehanna +##heard +duc +mastery +vh1 +williamsburg +bayer +buckled +craving +##khan +##rdes +bloomington +##write +alton +barbecue +##bians +justine +##hri +##ndt +delightful +smartphone +newtown +photon +retrieval +peugeot +hissing +##monium +##orough +flavors +lighted +relaunched +tainted +##games +##lysis +anarchy +microscopic +hopping +adept +evade +evie +##beau +inhibit +sinn +adjustable +hurst +intuition +wilton +cisco +44th +lawful +lowlands +stockings +thierry +##dalen +##hila +##nai +fates +prank +tb +maison +lobbied +provocative +1724 +4a +utopia +##qual +carbonate +gujarati +purcell +##rford +curtiss +##mei +overgrown +arenas +mediation +swallows +##rnik +respectful +turnbull +##hedron +##hope +alyssa +ozone +##ʻi +ami +gestapo +johansson +snooker +canteen +cuff +declines +empathy +stigma +##ags +##iner +##raine +taxpayers +gui +volga +##wright +##copic +lifespan +overcame +tattooed +enactment +giggles +##ador +##camp +barrington +bribe +obligatory +orbiting +peng +##enas +elusive +sucker +##vating +cong +hardship +empowered +anticipating +estrada +cryptic +greasy +detainees +planck +sudbury +plaid +dod +marriott +kayla +##ears +##vb +##zd +mortally +##hein +cognition +radha +319 +liechtenstein +meade +richly +argyle +harpsichord +liberalism +trumpets +lauded +tyrant +salsa +tiled +lear +promoters +reused +slicing +trident +##chuk +##gami +##lka +cantor +checkpoint +##points +gaul +leger +mammalian +##tov +##aar +##schaft +doha +frenchman +nirvana +##vino +delgado +headlining +##eron +##iography +jug +tko +1649 +naga +intersections +##jia +benfica +nawab +##suka +ashford +gulp +##deck +##vill +##rug +brentford +frazier +pleasures +dunne +potsdam +shenzhen +dentistry +##tec +flanagan +##dorff +##hear +chorale +dinah +prem +quezon +##rogated +relinquished +sutra +terri +##pani +flaps +##rissa +poly +##rnet +homme +aback +##eki +linger +womb +##kson +##lewood +doorstep +orthodoxy +threaded +westfield +##rval +dioceses +fridays +subsided +##gata +loyalists +##biotic +##ettes +letterman +lunatic +prelate +tenderly +invariably +souza +thug +winslow +##otide +furlongs +gogh +jeopardy +##runa +pegasus +##umble +humiliated +standalone +tagged +##roller +freshmen +klan +##bright +attaining +initiating +transatlantic +logged +viz +##uance +1723 +combatants +intervening +stephane +chieftain +despised +grazed +317 +cdc +galveston +godzilla +macro +simulate +##planes +parades +##esses +960 +##ductive +##unes +equator +overdose +##cans +##hosh +##lifting +joshi +epstein +sonora +treacherous +aquatics +manchu +responsive +##sation +supervisory +##christ +##llins +##ibar +##balance +##uso +kimball +karlsruhe +mab +##emy +ignores +phonetic +reuters +spaghetti +820 +almighty +danzig +rumbling +tombstone +designations +lured +outset +##felt +supermarkets +##wt +grupo +kei +kraft +susanna +##blood +comprehension +genealogy +##aghan +##verted +redding +##ythe +1722 +bowing +##pore +##roi +lest +sharpened +fulbright +valkyrie +sikhs +##unds +swans +bouquet +merritt +##tage +##venting +commuted +redhead +clerks +leasing +cesare +dea +hazy +##vances +fledged +greenfield +servicemen +##gical +armando +blackout +dt +sagged +downloadable +intra +potion +pods +##4th +##mism +xp +attendants +gambia +stale +##ntine +plump +asteroids +rediscovered +buds +flea +hive +##neas +1737 +classifications +debuts +##eles +olympus +scala +##eurs +##gno +##mute +hummed +sigismund +visuals +wiggled +await +pilasters +clench +sulfate +##ances +bellevue +enigma +trainee +snort +##sw +clouded +denim +##rank +##rder +churning +hartman +lodges +riches +sima +##missible +accountable +socrates +regulates +mueller +##cr +1702 +avoids +solids +himalayas +nutrient +pup +##jevic +squat +fades +nec +##lates +##pina +##rona +##ου +privateer +tequila +##gative +##mpton +apt +hornet +immortals +##dou +asturias +cleansing +dario +##rries +##anta +etymology +servicing +zhejiang +##venor +##nx +horned +erasmus +rayon +relocating +£10 +##bags +escalated +promenade +stubble +2010s +artisans +axial +liquids +mora +sho +yoo +##tsky +bundles +oldies +##nally +notification +bastion +##ths +sparkle +##lved +1728 +leash +pathogen +highs +##hmi +immature +880 +gonzaga +ignatius +mansions +monterrey +sweets +bryson +##loe +polled +regatta +brightest +pei +rosy +squid +hatfield +payroll +addict +meath +cornerback +heaviest +lodging +##mage +capcom +rippled +##sily +barnet +mayhem +ymca +snuggled +rousseau +##cute +blanchard +284 +fragmented +leighton +chromosomes +risking +##md +##strel +##utter +corinne +coyotes +cynical +hiroshi +yeomanry +##ractive +ebook +grading +mandela +plume +agustin +magdalene +##rkin +bea +femme +trafford +##coll +##lun +##tance +52nd +fourier +upton +##mental +camilla +gust +iihf +islamabad +longevity +##kala +feldman +netting +##rization +endeavour +foraging +mfa +orr +##open +greyish +contradiction +graz +##ruff +handicapped +marlene +tweed +oaxaca +spp +campos +miocene +pri +configured +cooks +pluto +cozy +pornographic +##entes +70th +fairness +glided +jonny +lynne +rounding +sired +##emon +##nist +remade +uncover +##mack +complied +lei +newsweek +##jured +##parts +##enting +##pg +293 +finer +guerrillas +athenian +deng +disused +stepmother +accuse +gingerly +seduction +521 +confronting +##walker +##going +gora +nostalgia +sabres +virginity +wrenched +##minated +syndication +wielding +eyre +##56 +##gnon +##igny +behaved +taxpayer +sweeps +##growth +childless +gallant +##ywood +amplified +geraldine +scrape +##ffi +babylonian +fresco +##rdan +##kney +##position +1718 +restricting +tack +fukuoka +osborn +selector +partnering +##dlow +318 +gnu +kia +tak +whitley +gables +##54 +##mania +mri +softness +immersion +##bots +##evsky +1713 +chilling +insignificant +pcs +##uis +elites +lina +purported +supplemental +teaming +##americana +##dding +##inton +proficient +rouen +##nage +##rret +niccolo +selects +##bread +fluffy +1621 +gruff +knotted +mukherjee +polgara +thrash +nicholls +secluded +smoothing +thru +corsica +loaf +whitaker +inquiries +##rrier +##kam +indochina +289 +marlins +myles +peking +##tea +extracts +pastry +superhuman +connacht +vogel +##ditional +##het +##udged +##lash +gloss +quarries +refit +teaser +##alic +##gaon +20s +materialized +sling +camped +pickering +tung +tracker +pursuant +##cide +cranes +soc +##cini +##typical +##viere +anhalt +overboard +workout +chores +fares +orphaned +stains +##logie +fenton +surpassing +joyah +triggers +##itte +grandmaster +##lass +##lists +clapping +fraudulent +ledger +nagasaki +##cor +##nosis +##tsa +eucalyptus +tun +##icio +##rney +##tara +dax +heroism +ina +wrexham +onboard +unsigned +##dates +moshe +galley +winnie +droplets +exiles +praises +watered +noodles +##aia +fein +adi +leland +multicultural +stink +bingo +comets +erskine +modernized +canned +constraint +domestically +chemotherapy +featherweight +stifled +##mum +darkly +irresistible +refreshing +hasty +isolate +##oys +kitchener +planners +##wehr +cages +yarn +implant +toulon +elects +childbirth +yue +##lind +##lone +cn +rightful +sportsman +junctions +remodeled +specifies +##rgh +291 +##oons +complimented +##urgent +lister +ot +##logic +bequeathed +cheekbones +fontana +gabby +##dial +amadeus +corrugated +maverick +resented +triangles +##hered +##usly +nazareth +tyrol +1675 +assent +poorer +sectional +aegean +##cous +296 +nylon +ghanaian +##egorical +##weig +cushions +forbid +fusiliers +obstruction +somerville +##scia +dime +earrings +elliptical +leyte +oder +polymers +timmy +atm +midtown +piloted +settles +continual +externally +mayfield +##uh +enrichment +henson +keane +persians +1733 +benji +braden +pep +324 +##efe +contenders +pepsi +valet +##isches +298 +##asse +##earing +goofy +stroll +##amen +authoritarian +occurrences +adversary +ahmedabad +tangent +toppled +dorchester +1672 +modernism +marxism +islamist +charlemagne +exponential +racks +unicode +brunette +mbc +pic +skirmish +##bund +##lad +##powered +##yst +hoisted +messina +shatter +##ctum +jedi +vantage +##music +##neil +clemens +mahmoud +corrupted +authentication +lowry +nils +##washed +omnibus +wounding +jillian +##itors +##opped +serialized +narcotics +handheld +##arm +##plicity +intersecting +stimulating +##onis +crate +fellowships +hemingway +casinos +climatic +fordham +copeland +drip +beatty +leaflets +robber +brothel +madeira +##hedral +sphinx +ultrasound +##vana +valor +forbade +leonid +villas +##aldo +duane +marquez +##cytes +disadvantaged +forearms +kawasaki +reacts +consular +lax +uncles +uphold +##hopper +concepcion +dorsey +lass +##izan +arching +passageway +1708 +researches +tia +internationals +##graphs +##opers +distinguishes +javanese +divert +##uven +plotted +##listic +##rwin +##erik +##tify +affirmative +signifies +validation +##bson +kari +felicity +georgina +zulu +##eros +##rained +##rath +overcoming +##dot +argyll +##rbin +1734 +chiba +ratification +windy +earls +parapet +##marks +hunan +pristine +astrid +punta +##gart +brodie +##kota +##oder +malaga +minerva +rouse +##phonic +bellowed +pagoda +portals +reclamation +##gur +##odies +##⁄₄ +parentheses +quoting +allergic +palette +showcases +benefactor +heartland +nonlinear +##tness +bladed +cheerfully +scans +##ety +##hone +1666 +girlfriends +pedersen +hiram +sous +##liche +##nator +1683 +##nery +##orio +##umen +bobo +primaries +smiley +##cb +unearthed +uniformly +fis +metadata +1635 +ind +##oted +recoil +##titles +##tura +##ια +406 +hilbert +jamestown +mcmillan +tulane +seychelles +##frid +antics +coli +fated +stucco +##grants +1654 +bulky +accolades +arrays +caledonian +carnage +optimism +puebla +##tative +##cave +enforcing +rotherham +seo +dunlop +aeronautics +chimed +incline +zoning +archduke +hellenistic +##oses +##sions +candi +thong +##ople +magnate +rustic +##rsk +projective +slant +##offs +danes +hollis +vocalists +##ammed +congenital +contend +gesellschaft +##ocating +##pressive +douglass +quieter +##cm +##kshi +howled +salim +spontaneously +townsville +buena +southport +##bold +kato +1638 +faerie +stiffly +##vus +##rled +297 +flawless +realising +taboo +##7th +bytes +straightening +356 +jena +##hid +##rmin +cartwright +berber +bertram +soloists +411 +noses +417 +coping +fission +hardin +inca +##cen +1717 +mobilized +vhf +##raf +biscuits +curate +##85 +##anial +331 +gaunt +neighbourhoods +1540 +##abas +blanca +bypassed +sockets +behold +coincidentally +##bane +nara +shave +splinter +terrific +##arion +##erian +commonplace +juris +redwood +waistband +boxed +caitlin +fingerprints +jennie +naturalized +##ired +balfour +craters +jody +bungalow +hugely +quilt +glitter +pigeons +undertaker +bulging +constrained +goo +##sil +##akh +assimilation +reworked +##person +persuasion +##pants +felicia +##cliff +##ulent +1732 +explodes +##dun +##inium +##zic +lyman +vulture +hog +overlook +begs +northwards +ow +spoil +##urer +fatima +favorably +accumulate +sargent +sorority +corresponded +dispersal +kochi +toned +##imi +##lita +internacional +newfound +##agger +##lynn +##rigue +booths +peanuts +##eborg +medicare +muriel +nur +##uram +crates +millennia +pajamas +worsened +##breakers +jimi +vanuatu +yawned +##udeau +carousel +##hony +hurdle +##ccus +##mounted +##pod +rv +##eche +airship +ambiguity +compulsion +recapture +##claiming +arthritis +##osomal +1667 +asserting +ngc +sniffing +dade +discontent +glendale +ported +##amina +defamation +rammed +##scent +fling +livingstone +##fleet +875 +##ppy +apocalyptic +comrade +lcd +##lowe +cessna +eine +persecuted +subsistence +demi +hoop +reliefs +710 +coptic +progressing +stemmed +perpetrators +1665 +priestess +##nio +dobson +ebony +rooster +itf +tortricidae +##bbon +##jian +cleanup +##jean +##øy +1721 +eighties +taxonomic +holiness +##hearted +##spar +antilles +showcasing +stabilized +##nb +gia +mascara +michelangelo +dawned +##uria +##vinsky +extinguished +fitz +grotesque +£100 +##fera +##loid +##mous +barges +neue +throbbed +cipher +johnnie +##a1 +##mpt +outburst +##swick +spearheaded +administrations +c1 +heartbreak +pixels +pleasantly +##enay +lombardy +plush +##nsed +bobbie +##hly +reapers +tremor +xiang +minogue +substantive +hitch +barak +##wyl +kwan +##encia +910 +obscene +elegance +indus +surfer +bribery +conserve +##hyllum +##masters +horatio +##fat +apes +rebound +psychotic +##pour +iteration +##mium +##vani +botanic +horribly +antiques +dispose +paxton +##hli +##wg +timeless +1704 +disregard +engraver +hounds +##bau +##version +looted +uno +facilitates +groans +masjid +rutland +antibody +disqualification +decatur +footballers +quake +slacks +48th +rein +scribe +stabilize +commits +exemplary +tho +##hort +##chison +pantry +traversed +##hiti +disrepair +identifiable +vibrated +baccalaureate +##nnis +csa +interviewing +##iensis +##raße +greaves +wealthiest +343 +classed +jogged +£5 +##58 +##atal +illuminating +knicks +respecting +##uno +scrubbed +##iji +##dles +kruger +moods +growls +raider +silvia +chefs +kam +vr +cree +percival +##terol +gunter +counterattack +defiant +henan +ze +##rasia +##riety +equivalence +submissions +##fra +##thor +bautista +mechanically +##heater +cornice +herbal +templar +##mering +outputs +ruining +ligand +renumbered +extravagant +mika +blockbuster +eta +insurrection +##ilia +darkening +ferocious +pianos +strife +kinship +##aer +melee +##anor +##iste +##may +##oue +decidedly +weep +##jad +##missive +##ppel +354 +puget +unease +##gnant +1629 +hammering +kassel +ob +wessex +##lga +bromwich +egan +paranoia +utilization +##atable +##idad +contradictory +provoke +##ols +##ouring +##tangled +knesset +##very +##lette +plumbing +##sden +##¹ +greensboro +occult +sniff +338 +zev +beaming +gamer +haggard +mahal +##olt +##pins +mendes +utmost +briefing +gunnery +##gut +##pher +##zh +##rok +1679 +khalifa +sonya +##boot +principals +urbana +wiring +##liffe +##minating +##rrado +dahl +nyu +skepticism +np +townspeople +ithaca +lobster +somethin +##fur +##arina +##−1 +freighter +zimmerman +biceps +contractual +##herton +amend +hurrying +subconscious +##anal +336 +meng +clermont +spawning +##eia +##lub +dignitaries +impetus +snacks +spotting +twigs +##bilis +##cz +##ouk +libertadores +nic +skylar +##aina +##firm +gustave +asean +##anum +dieter +legislatures +flirt +bromley +trolls +umar +##bbies +##tyle +blah +parc +bridgeport +crank +negligence +##nction +46th +constantin +molded +bandages +seriousness +00pm +siegel +carpets +compartments +upbeat +statehood +##dner +##edging +marko +730 +platt +##hane +paving +##iy +1738 +abbess +impatience +limousine +nbl +##talk +441 +lucille +mojo +nightfall +robbers +##nais +karel +brisk +calves +replicate +ascribed +telescopes +##olf +intimidated +##reen +ballast +specialization +##sit +aerodynamic +caliphate +rainer +visionary +##arded +epsilon +##aday +##onte +aggregation +auditory +boosted +reunification +kathmandu +loco +robyn +402 +acknowledges +appointing +humanoid +newell +redeveloped +restraints +##tained +barbarians +chopper +1609 +italiana +##lez +##lho +investigates +wrestlemania +##anies +##bib +690 +##falls +creaked +dragoons +gravely +minions +stupidity +volley +##harat +##week +musik +##eries +##uously +fungal +massimo +semantics +malvern +##ahl +##pee +discourage +embryo +imperialism +1910s +profoundly +##ddled +jiangsu +sparkled +stat +##holz +sweatshirt +tobin +##iction +sneered +##cheon +##oit +brit +causal +smyth +##neuve +diffuse +perrin +silvio +##ipes +##recht +detonated +iqbal +selma +##nism +##zumi +roasted +##riders +tay +##ados +##mament +##mut +##rud +840 +completes +nipples +cfa +flavour +hirsch +##laus +calderon +sneakers +moravian +##ksha +1622 +rq +294 +##imeters +bodo +##isance +##pre +##ronia +anatomical +excerpt +##lke +dh +kunst +##tablished +##scoe +biomass +panted +unharmed +gael +housemates +montpellier +##59 +coa +rodents +tonic +hickory +singleton +##taro +451 +1719 +aldo +breaststroke +dempsey +och +rocco +##cuit +merton +dissemination +midsummer +serials +##idi +haji +polynomials +##rdon +gs +enoch +prematurely +shutter +taunton +£3 +##grating +##inates +archangel +harassed +##asco +326 +archway +dazzling +##ecin +1736 +sumo +wat +##kovich +1086 +honneur +##ently +##nostic +##ttal +##idon +1605 +403 +1716 +blogger +rents +##gnan +hires +##ikh +##dant +howie +##rons +handler +retracted +shocks +1632 +arun +duluth +kepler +trumpeter +##lary +peeking +seasoned +trooper +##mara +laszlo +##iciencies +##rti +heterosexual +##inatory +##ssion +indira +jogging +##inga +##lism +beit +dissatisfaction +malice +##ately +nedra +peeling +##rgeon +47th +stadiums +475 +vertigo +##ains +iced +restroom +##plify +##tub +illustrating +pear +##chner +##sibility +inorganic +rappers +receipts +watery +##kura +lucinda +##oulos +reintroduced +##8th +##tched +gracefully +saxons +nutritional +wastewater +rained +favourites +bedrock +fisted +hallways +likeness +upscale +##lateral +1580 +blinds +prequel +##pps +##tama +deter +humiliating +restraining +tn +vents +1659 +laundering +recess +rosary +tractors +coulter +federer +##ifiers +##plin +persistence +##quitable +geschichte +pendulum +quakers +##beam +bassett +pictorial +buffet +koln +##sitor +drills +reciprocal +shooters +##57 +##cton +##tees +converge +pip +dmitri +donnelly +yamamoto +aqua +azores +demographics +hypnotic +spitfire +suspend +wryly +roderick +##rran +sebastien +##asurable +mavericks +##fles +##200 +himalayan +prodigy +##iance +transvaal +demonstrators +handcuffs +dodged +mcnamara +sublime +1726 +crazed +##efined +##till +ivo +pondered +reconciled +shrill +sava +##duk +bal +cad +heresy +jaipur +goran +##nished +341 +lux +shelly +whitehall +##hre +israelis +peacekeeping +##wled +1703 +demetrius +ousted +##arians +##zos +beale +anwar +backstroke +raged +shrinking +cremated +##yck +benign +towing +wadi +darmstadt +landfill +parana +soothe +colleen +sidewalks +mayfair +tumble +hepatitis +ferrer +superstructure +##gingly +##urse +##wee +anthropological +translators +##mies +closeness +hooves +##pw +mondays +##roll +##vita +landscaping +##urized +purification +sock +thorns +thwarted +jalan +tiberius +##taka +saline +##rito +confidently +khyber +sculptors +##ij +brahms +hammersmith +inspectors +battista +fivb +fragmentation +hackney +##uls +arresting +exercising +antoinette +bedfordshire +##zily +dyed +##hema +1656 +racetrack +variability +##tique +1655 +austrians +deteriorating +madman +theorists +aix +lehman +weathered +1731 +decreed +eruptions +1729 +flaw +quinlan +sorbonne +flutes +nunez +1711 +adored +downwards +fable +rasped +1712 +moritz +mouthful +renegade +shivers +stunts +dysfunction +restrain +translit +327 +pancakes +##avio +##cision +##tray +351 +vial +##lden +bain +##maid +##oxide +chihuahua +malacca +vimes +##rba +##rnier +1664 +donnie +plaques +##ually +337 +bangs +floppy +huntsville +loretta +nikolay +##otte +eater +handgun +ubiquitous +##hett +eras +zodiac +1634 +##omorphic +1820s +##zog +cochran +##bula +##lithic +warring +##rada +dalai +excused +blazers +mcconnell +reeling +bot +este +##abi +geese +hoax +taxon +##bla +guitarists +##icon +condemning +hunts +inversion +moffat +taekwondo +##lvis +1624 +stammered +##rest +##rzy +sousa +fundraiser +marylebone +navigable +uptown +cabbage +daniela +salman +shitty +whimper +##kian +##utive +programmers +protections +rm +##rmi +##rued +forceful +##enes +fuss +##tao +##wash +brat +oppressive +reykjavik +spartak +ticking +##inkles +##kiewicz +adolph +horst +maui +protege +straighten +cpc +landau +concourse +clements +resultant +##ando +imaginative +joo +reactivated +##rem +##ffled +##uising +consultative +##guide +flop +kaitlyn +mergers +parenting +somber +##vron +supervise +vidhan +##imum +courtship +exemplified +harmonies +medallist +refining +##rrow +##ка +amara +##hum +780 +goalscorer +sited +overshadowed +rohan +displeasure +secretive +multiplied +osman +##orth +engravings +padre +##kali +##veda +miniatures +mis +##yala +clap +pali +rook +##cana +1692 +57th +antennae +astro +oskar +1628 +bulldog +crotch +hackett +yucatan +##sure +amplifiers +brno +ferrara +migrating +##gree +thanking +turing +##eza +mccann +ting +andersson +onslaught +gaines +ganga +incense +standardization +##mation +sentai +scuba +stuffing +turquoise +waivers +alloys +##vitt +regaining +vaults +##clops +##gizing +digger +furry +memorabilia +probing +##iad +payton +rec +deutschland +filippo +opaque +seamen +zenith +afrikaans +##filtration +disciplined +inspirational +##merie +banco +confuse +grafton +tod +##dgets +championed +simi +anomaly +biplane +##ceptive +electrode +##para +1697 +cleavage +crossbow +swirl +informant +##lars +##osta +afi +bonfire +spec +##oux +lakeside +slump +##culus +##lais +##qvist +##rrigan +1016 +facades +borg +inwardly +cervical +xl +pointedly +050 +stabilization +##odon +chests +1699 +hacked +ctv +orthogonal +suzy +##lastic +gaulle +jacobite +rearview +##cam +##erted +ashby +##drik +##igate +##mise +##zbek +affectionately +canine +disperse +latham +##istles +##ivar +spielberg +##orin +##idium +ezekiel +cid +##sg +durga +middletown +##cina +customized +frontiers +harden +##etano +##zzy +1604 +bolsheviks +##66 +coloration +yoko +##bedo +briefs +slabs +debra +liquidation +plumage +##oin +blossoms +dementia +subsidy +1611 +proctor +relational +jerseys +parochial +ter +##ici +esa +peshawar +cavalier +loren +cpi +idiots +shamrock +1646 +dutton +malabar +mustache +##endez +##ocytes +referencing +terminates +marche +yarmouth +##sop +acton +mated +seton +subtly +baptised +beige +extremes +jolted +kristina +telecast +##actic +safeguard +waldo +##baldi +##bular +endeavors +sloppy +subterranean +##ensburg +##itung +delicately +pigment +tq +##scu +1626 +##ound +collisions +coveted +herds +##personal +##meister +##nberger +chopra +##ricting +abnormalities +defective +galician +lucie +##dilly +alligator +likened +##genase +burundi +clears +complexion +derelict +deafening +diablo +fingered +champaign +dogg +enlist +isotope +labeling +mrna +##erre +brilliance +marvelous +##ayo +1652 +crawley +ether +footed +dwellers +deserts +hamish +rubs +warlock +skimmed +##lizer +870 +buick +embark +heraldic +irregularities +##ajan +kiara +##kulam +##ieg +antigen +kowalski +##lge +oakley +visitation +##mbit +vt +##suit +1570 +murderers +##miento +##rites +chimneys +##sling +condemn +custer +exchequer +havre +##ghi +fluctuations +##rations +dfb +hendricks +vaccines +##tarian +nietzsche +biking +juicy +##duced +brooding +scrolling +selangor +##ragan +352 +annum +boomed +seminole +sugarcane +##dna +departmental +dismissing +innsbruck +arteries +ashok +batavia +daze +kun +overtook +##rga +##tlan +beheaded +gaddafi +holm +electronically +faulty +galilee +fractures +kobayashi +##lized +gunmen +magma +aramaic +mala +eastenders +inference +messengers +bf +##qu +407 +bathrooms +##vere +1658 +flashbacks +ideally +misunderstood +##jali +##weather +mendez +##grounds +505 +uncanny +##iii +1709 +friendships +##nbc +sacrament +accommodated +reiterated +logistical +pebbles +thumped +##escence +administering +decrees +drafts +##flight +##cased +##tula +futuristic +picket +intimidation +winthrop +##fahan +interfered +339 +afar +francoise +morally +uta +cochin +croft +dwarfs +##bruck +##dents +##nami +biker +##hner +##meral +nano +##isen +##ometric +##pres +##ан +brightened +meek +parcels +securely +gunners +##jhl +##zko +agile +hysteria +##lten +##rcus +bukit +champs +chevy +cuckoo +leith +sadler +theologians +welded +##section +1663 +jj +plurality +xander +##rooms +##formed +shredded +temps +intimately +pau +tormented +##lok +##stellar +1618 +charred +ems +essen +##mmel +alarms +spraying +ascot +blooms +twinkle +##abia +##apes +internment +obsidian +##chaft +snoop +##dav +##ooping +malibu +##tension +quiver +##itia +hays +mcintosh +travers +walsall +##ffie +1623 +beverley +schwarz +plunging +structurally +m3 +rosenthal +vikram +##tsk +770 +ghz +##onda +##tiv +chalmers +groningen +pew +reckon +unicef +##rvis +55th +##gni +1651 +sulawesi +avila +cai +metaphysical +screwing +turbulence +##mberg +augusto +samba +56th +baffled +momentary +toxin +##urian +##wani +aachen +condoms +dali +steppe +##3d +##app +##oed +##year +adolescence +dauphin +electrically +inaccessible +microscopy +nikita +##ega +atv +##cel +##enter +##oles +##oteric +##ы +accountants +punishments +wrongly +bribes +adventurous +clinch +flinders +southland +##hem +##kata +gough +##ciency +lads +soared +##ה +undergoes +deformation +outlawed +rubbish +##arus +##mussen +##nidae +##rzburg +arcs +##ingdon +##tituted +1695 +wheelbase +wheeling +bombardier +campground +zebra +##lices +##oj +##bain +lullaby +##ecure +donetsk +wylie +grenada +##arding +##ης +squinting +eireann +opposes +##andra +maximal +runes +##broken +##cuting +##iface +##ror +##rosis +additive +britney +adultery +triggering +##drome +detrimental +aarhus +containment +jc +swapped +vichy +##ioms +madly +##oric +##rag +brant +##ckey +##trix +1560 +1612 +broughton +rustling +##stems +##uder +asbestos +mentoring +##nivorous +finley +leaps +##isan +apical +pry +slits +substitutes +##dict +intuitive +fantasia +insistent +unreasonable +##igen +##vna +domed +hannover +margot +ponder +##zziness +impromptu +jian +lc +rampage +stemming +##eft +andrey +gerais +whichever +amnesia +appropriated +anzac +clicks +modifying +ultimatum +cambrian +maids +verve +yellowstone +##mbs +conservatoire +##scribe +adherence +dinners +spectra +imperfect +mysteriously +sidekick +tatar +tuba +##aks +##ifolia +distrust +##athan +##zle +c2 +ronin +zac +##pse +celaena +instrumentalist +scents +skopje +##mbling +comical +compensated +vidal +condor +intersect +jingle +wavelengths +##urrent +mcqueen +##izzly +carp +weasel +422 +kanye +militias +postdoctoral +eugen +gunslinger +##ɛ +faux +hospice +##for +appalled +derivation +dwarves +##elis +dilapidated +##folk +astoria +philology +##lwyn +##otho +##saka +inducing +philanthropy +##bf +##itative +geek +markedly +sql +##yce +bessie +indices +rn +##flict +495 +frowns +resolving +weightlifting +tugs +cleric +contentious +1653 +mania +rms +##miya +##reate +##ruck +##tucket +bien +eels +marek +##ayton +##cence +discreet +unofficially +##ife +leaks +##bber +1705 +332 +dung +compressor +hillsborough +pandit +shillings +distal +##skin +381 +##tat +##you +nosed +##nir +mangrove +undeveloped +##idia +textures +##inho +##500 +##rise +ae +irritating +nay +amazingly +bancroft +apologetic +compassionate +kata +symphonies +##lovic +airspace +##lch +930 +gifford +precautions +fulfillment +sevilla +vulgar +martinique +##urities +looting +piccolo +tidy +##dermott +quadrant +armchair +incomes +mathematicians +stampede +nilsson +##inking +##scan +foo +quarterfinal +##ostal +shang +shouldered +squirrels +##owe +344 +vinegar +##bner +##rchy +##systems +delaying +##trics +ars +dwyer +rhapsody +sponsoring +##gration +bipolar +cinder +starters +##olio +##urst +421 +signage +##nty +aground +figurative +mons +acquaintances +duets +erroneously +soyuz +elliptic +recreated +##cultural +##quette +##ssed +##tma +##zcz +moderator +scares +##itaire +##stones +##udence +juniper +sighting +##just +##nsen +britten +calabria +ry +bop +cramer +forsyth +stillness +##л +airmen +gathers +unfit +##umber +##upt +taunting +##rip +seeker +streamlined +##bution +holster +schumann +tread +vox +##gano +##onzo +strive +dil +reforming +covent +newbury +predicting +##orro +decorate +tre +##puted +andover +ie +asahi +dept +dunkirk +gills +##tori +buren +huskies +##stis +##stov +abstracts +bets +loosen +##opa +1682 +yearning +##glio +##sir +berman +effortlessly +enamel +napoli +persist +##peration +##uez +attache +elisa +b1 +invitations +##kic +accelerating +reindeer +boardwalk +clutches +nelly +polka +starbucks +##kei +adamant +huey +lough +unbroken +adventurer +embroidery +inspecting +stanza +##ducted +naia +taluka +##pone +##roids +chases +deprivation +florian +##jing +##ppet +earthly +##lib +##ssee +colossal +foreigner +vet +freaks +patrice +rosewood +triassic +upstate +##pkins +dominates +ata +chants +ks +vo +##400 +##bley +##raya +##rmed +555 +agra +infiltrate +##ailing +##ilation +##tzer +##uppe +##werk +binoculars +enthusiast +fujian +squeak +##avs +abolitionist +almeida +boredom +hampstead +marsden +rations +##ands +inflated +334 +bonuses +rosalie +patna +##rco +329 +detachments +penitentiary +54th +flourishing +woolf +##dion +##etched +papyrus +##lster +##nsor +##toy +bobbed +dismounted +endelle +inhuman +motorola +tbs +wince +wreath +##ticus +hideout +inspections +sanjay +disgrace +infused +pudding +stalks +##urbed +arsenic +leases +##hyl +##rrard +collarbone +##waite +##wil +dowry +##bant +##edance +genealogical +nitrate +salamanca +scandals +thyroid +necessitated +##! +##" +### +##$ +##% +##& +##' +##( +##) +##* +##+ +##, +##- +##. +##/ +##: +##; +##< +##= +##> +##? +##@ +##[ +##\ +##] +##^ +##_ +##` +##{ +##| +##} +##~ +##¡ +##¢ +##£ +##¤ +##¥ +##¦ +##§ +##¨ +##© +##ª +##« +##¬ +##® +##± +##´ +##µ +##¶ +##· +##º +##» +##¼ +##¾ +##¿ +##æ +##ð +##÷ +##þ +##đ +##ħ +##ŋ +##œ +##ƒ +##ɐ +##ɑ +##ɒ +##ɔ +##ɕ +##ə +##ɡ +##ɣ +##ɨ +##ɪ +##ɫ +##ɬ +##ɯ +##ɲ +##ɴ +##ɹ +##ɾ +##ʀ +##ʁ +##ʂ +##ʃ +##ʉ +##ʊ +##ʋ +##ʌ +##ʎ +##ʐ +##ʑ +##ʒ +##ʔ +##ʰ +##ʲ +##ʳ +##ʷ +##ʸ +##ʻ +##ʼ +##ʾ +##ʿ +##ˈ +##ˡ +##ˢ +##ˣ +##ˤ +##β +##γ +##δ +##ε +##ζ +##θ +##κ +##λ +##μ +##ξ +##ο +##π +##ρ +##σ +##τ +##υ +##φ +##χ +##ψ +##ω +##б +##г +##д +##ж +##з +##м +##п +##с +##у +##ф +##х +##ц +##ч +##ш +##щ +##ъ +##э +##ю +##ђ +##є +##і +##ј +##љ +##њ +##ћ +##ӏ +##ա +##բ +##գ +##դ +##ե +##թ +##ի +##լ +##կ +##հ +##մ +##յ +##ն +##ո +##պ +##ս +##վ +##տ +##ր +##ւ +##ք +##־ +##א +##ב +##ג +##ד +##ו +##ז +##ח +##ט +##י +##ך +##כ +##ל +##ם +##מ +##ן +##נ +##ס +##ע +##ף +##פ +##ץ +##צ +##ק +##ר +##ש +##ת +##، +##ء +##ب +##ت +##ث +##ج +##ح +##خ +##ذ +##ز +##س +##ش +##ص +##ض +##ط +##ظ +##ع +##غ +##ـ +##ف +##ق +##ك +##و +##ى +##ٹ +##پ +##چ +##ک +##گ +##ں +##ھ +##ہ +##ے +##अ +##आ +##उ +##ए +##क +##ख +##ग +##च +##ज +##ट +##ड +##ण +##त +##थ +##द +##ध +##न +##प +##ब +##भ +##म +##य +##र +##ल +##व +##श +##ष +##स +##ह +##ा +##ि +##ी +##ो +##। +##॥ +##ং +##অ +##আ +##ই +##উ +##এ +##ও +##ক +##খ +##গ +##চ +##ছ +##জ +##ট +##ড +##ণ +##ত +##থ +##দ +##ধ +##ন +##প +##ব +##ভ +##ম +##য +##র +##ল +##শ +##ষ +##স +##হ +##া +##ি +##ী +##ে +##க +##ச +##ட +##த +##ந +##ன +##ப +##ம +##ய +##ர +##ல +##ள +##வ +##ா +##ி +##ு +##ே +##ை +##ನ +##ರ +##ಾ +##ක +##ය +##ර +##ල +##ව +##ා +##ก +##ง +##ต +##ท +##น +##พ +##ม +##ย +##ร +##ล +##ว +##ส +##อ +##า +##เ +##་ +##། +##ག +##ང +##ད +##ན +##པ +##བ +##མ +##འ +##ར +##ལ +##ས +##မ +##ა +##ბ +##გ +##დ +##ე +##ვ +##თ +##ი +##კ +##ლ +##მ +##ნ +##ო +##რ +##ს +##ტ +##უ +##ᄀ +##ᄂ +##ᄃ +##ᄅ +##ᄆ +##ᄇ +##ᄉ +##ᄊ +##ᄋ +##ᄌ +##ᄎ +##ᄏ +##ᄐ +##ᄑ +##ᄒ +##ᅡ +##ᅢ +##ᅥ +##ᅦ +##ᅧ +##ᅩ +##ᅪ +##ᅭ +##ᅮ +##ᅯ +##ᅲ +##ᅳ +##ᅴ +##ᅵ +##ᆨ +##ᆫ +##ᆯ +##ᆷ +##ᆸ +##ᆼ +##ᴬ +##ᴮ +##ᴰ +##ᴵ +##ᴺ +##ᵀ +##ᵃ +##ᵇ +##ᵈ +##ᵉ +##ᵍ +##ᵏ +##ᵐ +##ᵒ +##ᵖ +##ᵗ +##ᵘ +##ᵣ +##ᵤ +##ᵥ +##ᶜ +##ᶠ +##‐ +##‑ +##‒ +##– +##— +##― +##‖ +##‘ +##’ +##‚ +##“ +##” +##„ +##† +##‡ +##• +##… +##‰ +##′ +##″ +##› +##‿ +##⁄ +##⁰ +##ⁱ +##⁴ +##⁵ +##⁶ +##⁷ +##⁸ +##⁹ +##⁻ +##ⁿ +##₅ +##₆ +##₇ +##₈ +##₉ +##₊ +##₍ +##₎ +##ₐ +##ₑ +##ₒ +##ₓ +##ₕ +##ₖ +##ₗ +##ₘ +##ₚ +##ₛ +##ₜ +##₤ +##₩ +##€ +##₱ +##₹ +##ℓ +##№ +##ℝ +##™ +##⅓ +##⅔ +##← +##↑ +##→ +##↓ +##↔ +##↦ +##⇄ +##⇌ +##⇒ +##∂ +##∅ +##∆ +##∇ +##∈ +##∗ +##∘ +##√ +##∞ +##∧ +##∨ +##∩ +##∪ +##≈ +##≡ +##≤ +##≥ +##⊂ +##⊆ +##⊕ +##⊗ +##⋅ +##─ +##│ +##■ +##▪ +##● +##★ +##☆ +##☉ +##♠ +##♣ +##♥ +##♦ +##♯ +##⟨ +##⟩ +##ⱼ +##⺩ +##⺼ +##⽥ +##、 +##。 +##〈 +##〉 +##《 +##》 +##「 +##」 +##『 +##』 +##〜 +##あ +##い +##う +##え +##お +##か +##き +##く +##け +##こ +##さ +##し +##す +##せ +##そ +##た +##ち +##っ +##つ +##て +##と +##な +##に +##ぬ +##ね +##の +##は +##ひ +##ふ +##へ +##ほ +##ま +##み +##む +##め +##も +##や +##ゆ +##よ +##ら +##り +##る +##れ +##ろ +##を +##ん +##ァ +##ア +##ィ +##イ +##ウ +##ェ +##エ +##オ +##カ +##キ +##ク +##ケ +##コ +##サ +##シ +##ス +##セ +##タ +##チ +##ッ +##ツ +##テ +##ト +##ナ +##ニ +##ノ +##ハ +##ヒ +##フ +##ヘ +##ホ +##マ +##ミ +##ム +##メ +##モ +##ャ +##ュ +##ョ +##ラ +##リ +##ル +##レ +##ロ +##ワ +##ン +##・ +##ー +##一 +##三 +##上 +##下 +##不 +##世 +##中 +##主 +##久 +##之 +##也 +##事 +##二 +##五 +##井 +##京 +##人 +##亻 +##仁 +##介 +##代 +##仮 +##伊 +##会 +##佐 +##侍 +##保 +##信 +##健 +##元 +##光 +##八 +##公 +##内 +##出 +##分 +##前 +##劉 +##力 +##加 +##勝 +##北 +##区 +##十 +##千 +##南 +##博 +##原 +##口 +##古 +##史 +##司 +##合 +##吉 +##同 +##名 +##和 +##囗 +##四 +##国 +##國 +##土 +##地 +##坂 +##城 +##堂 +##場 +##士 +##夏 +##外 +##大 +##天 +##太 +##夫 +##奈 +##女 +##子 +##学 +##宀 +##宇 +##安 +##宗 +##定 +##宣 +##宮 +##家 +##宿 +##寺 +##將 +##小 +##尚 +##山 +##岡 +##島 +##崎 +##川 +##州 +##巿 +##帝 +##平 +##年 +##幸 +##广 +##弘 +##張 +##彳 +##後 +##御 +##德 +##心 +##忄 +##志 +##忠 +##愛 +##成 +##我 +##戦 +##戸 +##手 +##扌 +##政 +##文 +##新 +##方 +##日 +##明 +##星 +##春 +##昭 +##智 +##曲 +##書 +##月 +##有 +##朝 +##木 +##本 +##李 +##村 +##東 +##松 +##林 +##森 +##楊 +##樹 +##橋 +##歌 +##止 +##正 +##武 +##比 +##氏 +##民 +##水 +##氵 +##氷 +##永 +##江 +##沢 +##河 +##治 +##法 +##海 +##清 +##漢 +##瀬 +##火 +##版 +##犬 +##王 +##生 +##田 +##男 +##疒 +##発 +##白 +##的 +##皇 +##目 +##相 +##省 +##真 +##石 +##示 +##社 +##神 +##福 +##禾 +##秀 +##秋 +##空 +##立 +##章 +##竹 +##糹 +##美 +##義 +##耳 +##良 +##艹 +##花 +##英 +##華 +##葉 +##藤 +##行 +##街 +##西 +##見 +##訁 +##語 +##谷 +##貝 +##貴 +##車 +##軍 +##辶 +##道 +##郎 +##郡 +##部 +##都 +##里 +##野 +##金 +##鈴 +##镇 +##長 +##門 +##間 +##阝 +##阿 +##陳 +##陽 +##雄 +##青 +##面 +##風 +##食 +##香 +##馬 +##高 +##龍 +##龸 +##fi +##fl +##！ +##（ +##） +##， +##－ +##． +##／ +##： +##？ +##～ diff --git a/TextClassification/TextClassification/data/test.csv b/TextClassification/TextClassification/data/test.csv new file mode 100644 index 0000000000000000000000000000000000000000..90debc487dfe71253991fd635b4118baacdf1243 --- /dev/null +++ b/TextClassification/TextClassification/data/test.csv @@ -0,0 +1,3698 @@ +id,text,target +0,Just happened a terrible car crash, +2,"Heard about #earthquake is different cities, stay safe everyone.", +3,"there is a forest fire at spot pond, geese are fleeing across the street, I cannot save them all", +9,Apocalypse lighting. #Spokane #wildfires, +11,Typhoon Soudelor kills 28 in China and Taiwan, +12,We're shaking...It's an earthquake, +21,"They'd probably still show more life than Arsenal did yesterday, eh? EH?", +22,Hey! How are you?, +27,What a nice hat?, +29,Fuck off!, +30,No I don't like cold!, +35,NOOOOOOOOO! Don't do that!, +42,No don't tell me that!, +43,What if?!, +45,Awesome!, +46,Birmingham Wholesale Market is ablaze BBC News - Fire breaks out at Birmingham's Wholesale Market http://t.co/irWqCEZWEU, +47,@sunkxssedharry will you wear shorts for race ablaze ?, +51,#PreviouslyOnDoyinTv: Toke Makinwa__ªs marriage crisis sets Nigerian Twitter ablaze... http://t.co/CMghxBa2XI, +58,Check these out: http://t.co/rOI2NSmEJJ http://t.co/3Tj8ZjiN21 http://t.co/YDUiXEfIpE http://t.co/LxTjc87KLS #nsfw, +60,"PSA: I__ªm splitting my personalities. + +?? techies follow @ablaze_co +?? Burners follow @ablaze", +69,beware world ablaze sierra leone & guap., +70,Burning Man Ablaze! by Turban Diva http://t.co/hodWosAmWS via @Etsy, +72,Not a diss song. People will take 1 thing and run with it. Smh it's an eye opener though. He is about 2 set the game ablaze @CyhiThePrynce, +75,Rape victim dies as she sets herself ablaze: A 16-year-old girl died of burn injuries as she set herself ablaze___ http://t.co/UK8hNrbOob, +84,SETTING MYSELF ABLAZE http://t.co/6vMe7P5XhC, +87,@CTVToronto the bins in front of the field by my house wer set ablaze the other day flames went rite up the hydro pole wonder if it was him, +88,#nowplaying Alfons - Ablaze 2015 on Puls Radio #pulsradio http://t.co/aA5BJgWfDv, +90,'Burning Rahm': Let's hope City Hall builds a giant wooden mayoral effigy 100 feet tall & sets it ablaze. http://t.co/kFo2mksn6Y @John_Kass, +94,@PhilippaEilhart @DhuBlath hurt but her eyes ablaze with insulted anger., +99,Accident cleared in #PaTurnpike on PATP EB between PA-18 and Cranberry slow back to #traffic http://t.co/SL0Oqn0Vyr, +101,Just got to love burning your self on a damn curling wand... I swear someone needs to take it away from me cuase I'm just accident prone., +103,I hate badging shit in accident, +106,#3: Car Recorder ZeroEdgeå¨ Dual-lens Car Camera Vehicle Traffic/Driving History/Accident Camcorder Large Re... http://t.co/kKFaSJv6Cj, +108,Coincidence Or #Curse? Still #Unresolved Secrets From Past http://t.co/7VG8Df9pLE #accident, +111,@Traffic_SouthE @roadpol_east Accident on A27 near Lewes is it Kingston Roundabout rather than A283, +115,@sakuma_en If you pretend to feel a certain way the feeling can become genuine all by accident. -Hei (Darker than Black) #manga #anime, +116,For Legal and Medical Referral Service @1800_Injured Call us at: 1-800-465-87332 #accident #slipandfall #dogbite, +122,There's a construction guy working on the Disney store and he has huge gauges in his ears ?? ...that is a bloody accident waiting to happen, +123,@RobynJilllian @WlSDOMTEETHS I feel like I'm going to do it on accident. Teesha is gonna come out??, +124,On the #M42 northbound between junctions J3 and J3A there are currently delays of 10 mins due to an accident c... http://t.co/LwI3prBa31, +125,@DaveOshry @Soembie So if I say that I met her by accident this week- would you be super jelly Dave? :p, +127,ACCIDENT - HIT AND RUN - COLD at 500 BLOCK OF SE VISTA TER GRESHAM OR [Gresham Police #PG15000044357] 10:35 #pdx911, +140,@Calum5SOS this happened on accident but I like it http://t.co/QHmXuljSX9, +142,Please donate and spread the word! A training accident left the pole-vaulter Kira Gr__nberg a paraplegic http://t.co/6MpnyCl8PK, +147,Please like and share our new page for our Indoor Trampoline Park Aftershock opening this fall!! http://t.co/UgXhHErrxS, +148,@bxckylynch foi no ROH Aftershock: Las Vegas procura no pirate bay que tem, +150,"Schoolboy ___ Aftershock (Original Mix) +Excision & Skism ___ SEXisM (Far Too Loud Remix) +Firebeatz Schella ___ Dear New... http://t.co/JQLzUA6YzQ", +152,320 [IR] ICEMOON [AFTERSHOCK] | http://t.co/THyzOMVWU0 | @djicemoon | #Dubstep #TrapMusic #DnB #EDM #Dance #Ices___ http://t.co/83jOO0xk29, +154,'When the aftershock happened (Nepal) we were the last int'l team still there; in a way we were 1st responders.' Chief Collins @LACo_FD, +155,320 [IR] ICEMOON [AFTERSHOCK] | http://t.co/gRPeF7yAWG | @djicemoon | #Dubstep #TrapMusic #DnB #EDM #Dance #Ices___ http://t.co/GGmvzT58vE, +166,Stop saying 'I Wish' and start saying 'I Will'. ___ Unknown, +167,I want to go to Aftershock in October because it has all the bands I listen to and #NXT! Can't afford it yet though. #gradschoolapps, +169,'We are still living in the aftershock of Hiroshima people are still the scars of history.' - Edward Bond http://t.co/engTl5wrGp, +177,320 [IR] ICEMOON [AFTERSHOCK] | http://t.co/THyzOMVWU0 | @djicemoon | #Dubstep #TrapMusic #DnB #EDM #Dance #Ices___ http://t.co/83jOO0xk29, +179,Aftershock https://t.co/Ecy4U623nO, +181,'There is no victory at bargain basement prices.' Dwight David Eisenhower, +186,Bo2 had by far the best competitive maps imo hope bo3 is the same #InVahnWeTrust, +188,Brass and Copper in Cataclysm & AfterShock!!! http://t.co/uxYZyaygTy, +189,@JadeForMKX You should be happy I don't use Aftershock. That variation counters your play style hard., +192,Aftershock https://t.co/38Nhq9moEf, +200,When carelessness leads to an aviation accident the victim has the right to seek compensation for damages. http://t.co/eqAG6rz1vO, +202,@rewind_music found out about you guys today(regarding the accident on the airplane lol) & became a fan! Sending love & support from Cali~?, +206,A girl who died in an airplane accident fifteen years ago, +207,@Mintechan Hihow are you? There is Keio line on the stationright? BTW do you know the airplane accident near Chofu airport this week?, +214,Mexican airplane accident in Ocampo Coahuila MX on 7/29/25 killed Coahuila government SubSecretariat Francisco Garcia Castells age 42., +217,Horrible Accident Man Died In Wings of Airplaneå_(29-07-2015) http://t.co/5ZRKZdhODe, +223,@god if an accident were to happen on this airplane idc if the rest of my luggage is completely destroyed just please save my makeup, +224,Horrible Accident Man Died In Wings of Airplane (29-07-2015) http://t.co/hG8u2kR1Rq, +227,#UPDATE: Picture from the Penn Twp. airplane accident. http://t.co/6JfgDnZRlC, +228,"See how a judge ruled in this 2009 accident at #JFK Airport? involving Korean Air?. + +http://t.co/Yh1cGlN3rl http://t.co/6F5ShPKjOB", +230,@thugIauren I had myself on airplane mode by accident ??, +233,36 years ago today baseball lost one of its greats in an airplane accident. RIP Captain. #Yankees @yankees http://t.co/iNKU28vjJj, +234,Experts in France begin examining airplane debris found on Reunion Island: French air accident experts on Wedn... http://t.co/miw2i0pQxz, +236,#BreakingNews Experts in France begin examining airplane debris found on Reunion Island: French air accident e... http://t.co/3XIcUvlvlJ, +239,Experts in France begin examining airplane debris found on Reunion Island: French air accident experts on Wedn... http://t.co/KuBsM16OuD, +250,Twelve feared killed in Pakistani air ambulance helicopter crash http://t.co/zdsug0UjS7, +255,http://t.co/Q5XTkaKM08 I'M DYING SEND AN AMBULANCE @M__Belly @brightasastar_, +257,AMBULANCE SPRINTER AUTOMATIC FRONTLINE VEHICLE CHOICE OF 14 LEZ COMPLIANT | eBay http://t.co/7X3PDDbT0Z, +259,@disneyxrowbrina the ambulance isn't even parked well like its nearly on top of someone's car I'm laughing, +275,Shot 12 times. Found dead in cuffs after being involved in a car accident. Officers told ambulance not to treat him. https://t.co/MEUDJwaaNg, +278,@margaretcho Call me a fag and I'm going to call you an ambulance :) #RainbowPower, +282,New Nanotech Device Will Be Able To Target And Destroy Blood Clots http://t.co/MnmyJXQ9go #science, +284,@vballplaya2296 want me to send you some medic supplies from my dad's ambulance?, +286,2 held with heroin in ambulance http://t.co/d9sOwi1G21 http://t.co/OPxmdPIwAu, +288,http://t.co/B2FaSrt1tN Twelve feared killed in Pakistani air ambulance helicopter crash http://t.co/O8BJXnEPQm, +292,Why should a helicopter ambulance ride to transfer to a hospital 21 miles away cost $29800? http://t.co/7ZqIreY7Te http://t.co/9Qp3PqPwZv, +295,so privileged and proud to wear this uniform.?? #NHS #Ambulance #GayUK #Uniform #Proud #Privileged #WhatsYourEmergency http://t.co/0BkmuhYSFx, +300,The bartender at work described a drunk man as annihilated @kdunning1919 @hsnowberger @gabrielasmith29. 16 more days, +304,Cop pulls drunk driver to safety SECONDS before his car is hit by train. http://t.co/cTL5xzIHxAå_ http://t.co/1kDOZTD9mv via @ViralSpell, +305,Cop pulls drunk driver to safety SECONDS before his car is hit by train. http://t.co/cSsc8HcsXnå_ http://t.co/jBlGyLCsAy via @ViralSpell, +306,@RosieGray Now in all sincerety do you think the UN would move to Israel if there was a fraction of a chance of being annihilated?, +308,@johnboywest The only 'stage' the drama queen can stand on is a drama stage & certainly never the world stage; he would be annihilated., +311,"@_drodrolagi #handplacementgoals +Bro we annihilated that hashtag this week.", +317,How do I step outside for 5 seconds and get annihilated by mosquitoes?, +319,@jacksfilms #yiayplan well first we strike dreamworks and the minions will be annihilated., +323,'If your nature appropriates it love will burn you until you become annihilated in your beloved...' https://t.co/sMlwjunD09, +324,@NinaHoag - 'if you shred my Psych work our friendship would be annihilated', +325,@thehill this is 1 example of y the Conservatives annihilated Burton v Wiimington Prkng Auth while Liberals stood by &have done nothing, +326,"Aug 3 1915___KAISERJAEGERS WIPED OUT.; Francis Joseph's Crack Regiment Annihilated on Carso Plateau. +http://t.co/D1sPSwl66H", +333,They should all die! All of them! Everything annihilated!, +339,BROOO HE JUST GOT ANNIHILATED https://t.co/UR7QkqG1wf, +342,@AlbertBreer he was probably annihilated needed his DD, +343,$GMCR no longe rGreen mountain now Red Mountain...stock annihilated after hours, +350,Please sign & RT to save #SaltRiverWildHorses http://t.co/GB8ispiaRP http://t.co/Bx0l87iNc8, +351,"Allied Plans for the Annihilation of the German People http://t.co/RUHxGlo18q +http://t.co/HbUpkzWdWq +Louis Nizer - Very interesting...", +357,Please share and sign this petition to save wild horses in Arizona. http://t.co/3tsSXPHuFE, +359,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... https://t.co/lATVr8RZCK via @Change, +362,NEW! Are souls punished with annihilation? http://t.co/cmNV6VyFCQ, +366,***Latest Updates on the Salt River Wild Horse Round-up*** https://t.co/WJsCdvCevH via @Change, +367,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... https://t.co/W5Rhtuey90 via @Change, +369,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... https://t.co/ZhXua8kmae via @Change, +373,@BosTeenAuthFest Out of the Silent Planet in print and Annihilation on audio., +374,RT SIGN URGENT Stop the Annihilation of the Salt River Wild Horses!!! #savewildhorses #saltriverhorses https://t.co/8AZjFF8eSi, +376,@jackienatalydlt I do.... I only get the iced annihilation??, +377,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... https://t.co/d82wlcP49S via @Change, +378,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... https://t.co/KAlDycTC4H via @Change, +379,"@INCIndia under Sonia Gandhi is like Pakistan. Every defeat is treated like a victory.Total annihilation only answer +http://t.co/KtTYiTOw64", +382,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... http://t.co/TrKg2tpWJm via @Change, +385,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... http://t.co/k0q4fnBzss via @Change, +387,Stop the Annihilation of the Salt River Wild Horses!!! https://t.co/546utTXzAm via @Change, +388,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... https://t.co/FLcQQeZnVW via @Change, +391,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... https://t.co/FaXDzI90dY via @Change, +392,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... https://t.co/n6VeOjW3S8 via @Change, +395,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... https://t.co/i0z3k0chgg via @Change, +399,@spherulitic that almost feels like the SI signs of the apocalypse. Lol., +400,I liked a @YouTube video from @teamtwiistz http://t.co/OCurjyDRcn FUTURISTIC HOUSE (Built To Survive The Apocalypse!) - Minecraft, +403,I liked a @YouTube video http://t.co/jrFjgaAc9F Minecraft: NIGHT LUCKY BLOCK MOD (BOB APOCALYPSE WITHER 2.0 & MORE!) Mod Showcase, +405,@thomasa56 Just ONE of my major reasons not to live in *Big City*. If RR's *miss 9 meals* ever comes to pass Cue ANY 'Apocalypse' script., +408,#AskConnor there's a zombie apocalypse. the item to your right is your weapon. you're either screwed or you're gonna live., +411,It's an apocalypse, +414,@ItsKingRiffYall I'm excited for apocalypse really dig how the x-men franchise is going I like the 'event' theme a lot., +416,@TMFK_CO sounds like a terrible time. I'll be right there., +417,"Apocalypse no! Why artists should not go into the #Fukushima exclusion zone +http://t.co/3zqL0qbLUw +#nuclear #ura", +422,Southern Prepper Fiction:Sharecropping the Apocalypse https://t.co/HsleABS1Yr #Apocalyptic #doomsday #prepper #book #reading, +425,@5SOStag honestly he could say an apocalypse is coming and i would be exited hes so enthusiastic about everything, +428,Also my other nephew is proof that fat babies are going to save us from the apocalypse http://t.co/N5VImOqhBG, +430,I liked a @YouTube video http://t.co/2umSxRzot3 Zombie Apocalypse: The Rescue, +431,I added a video to a @YouTube playlist http://t.co/OCurjyDRcn FUTURISTIC HOUSE (Built To Survive The Apocalypse!) - Minecraft Maps, +433,|LIVE NOW| Princes of the Apocalypse: D&D Encounters #meerkat http://t.co/oY9ES9FVlt, +434,I liked a @YouTube video http://t.co/e89GPNEPzT Minecraft: NIGHT LUCKY BLOCK MOD (BOB APOCALYPSE WITHER 2.0 & MORE!) Mod Showcase, +439,OMg zombie apocalypse among my students... -___-, +441,I liked a @YouTube video http://t.co/uJrIaIVqln Minecraft: NIGHT LUCKY BLOCK MOD (BOB APOCALYPSE WITHER 2.0 & MORE!) Mod Showcase, +449,UNIVERSAL ORDER OF ARMAGEDDON http://t.co/3tY4mGm, +458,Patent Pending Stream 'Armageddon' EP http://t.co/BOuaJqi3Lf, +460,ARMAGEDDON LONDON | http://t.co/CULU2OwIUc, +464,@hitchBOT I heard you might rise from the ashes of Armageddon. .go hitchbot ..like the true superstar you are., +473,#Turkey invades #Israel - Halfway to #Armageddon http://t.co/xUOh3sJNXF, +488,Remember to crowd around the baggage carousel like it's armageddon and the bags are the last remaining food items on earth you animals., +491,"Never compromise. not even in the face of armageddon. +#Watchmen", +494,"#MustRead: Vladimir #Putin Issues Major Warning But Is It Too Late To Escape Armageddon? by @PCraigRoberts #US #Rus +http://t.co/5GFhfCIjrF", +497,"You da One + +#MTVSummerStar #VideoVeranoMTV #MTVHottest Britney Spears Lana Del Rey", +500,If you build an army of 100 lions and their leader is a dog in any fight the lions will die like a dog., +505,One Direction Is my pick for http://t.co/q2eBlOKeVE Fan Army #Directioners http://t.co/eNCmhz6y34 x1421, +507,SHARK ARMY Black Date Stainless Steel Quartz Men Sport Watch - Full read by eBay http://t.co/k6OzC4wFQd http://t.co/H2ZC4nTdZN, +508,12.Beyonce Is my pick for http://t.co/thoYhrHkfJ Fan Army #Beyhive http://t.co/WvJ39a3BGM, +510,5 Seconds of Summer Is my pick for http://t.co/EFC3896Qgr Fan Army #5SOSFAM http://t.co/aSpwke1YnK, +511,Beyonce Is my pick for http://t.co/nnMQlz91o9 Fan Army #Beyhive http://t.co/o91f3cYy0R 67, +515,VICTORINOX SWISS ARMY DATE WOMEN'S RUBBER MOP WATCH 241487 http://t.co/q6VCJaTfjx http://t.co/m1cpEw7kbh, +525,Hope the Salvation Army at least gave you a tax receipt. https://t.co/Ir9c545rZy, +529,One Direction Is my pick for http://t.co/q2eBlOKeVE Fan Army #Directioners http://t.co/eNCmhz6y34 x1431, +532,One Direction Is my pick for http://t.co/q2eBlOKeVE Fan Army #Directioners http://t.co/eNCmhz6y34 x1435, +534,One Direction Is my pick for http://t.co/q2eBlOKeVE Fan Army #Directioners http://t.co/eNCmhz6y34 x1415, +537,Check out more data on Upper Wabash reservoirs here on @LouisvilleUSACE site: http://t.co/hqqLQUqZmD, +539,"ANSWER: +'Therefore it came to pass that in my sixteenth year I did go forth at the head of an army of the... http://t.co/uuAAsb394n", +541,One Direction Is my pick for http://t.co/q2eBlOKeVE Fan Army #Directioners http://t.co/eNCmhz6y34 x1424, +545,5 Seconds of Summer Is my pick for http://t.co/qcHV3JqOVK Fan Army #5SOSFAM http://t.co/gc0uDfnFgg __2, +547,Cheesehead Report - Arson charges filed in Jackson County house fire http://t.co/I3Y1ZWjBzO, +548,Arson suspect linked to 30 fires caught in Northern California http://t.co/z8tlcAdKOw, +549,Israeli police unable to solve the case of Duma arson attack http://t.co/WtZgXzaf7Z, +553,"SHOCK REPORT: Muslims Setting Wildfires Across American West...Arson Jihad! + +http://t.co/92F4nKxjIu", +554,@pnuts_mama Be strong. It's at times like this that arson happens. Push thru past the felony stage., +555,#BreakingNews Mourning notices for stabbing arson victims stir __÷politics of grief__ª in Israel: Posters f... http://t.co/KTKDrGVHgX #rome, +557,Mourning notices for stabbing arson victims stir __÷politics of grief__ª in Israel: Posters for Shira Banki and A... http://t.co/WbCtkGGTY9, +562,RT @an_opus: Elliott will be on probation for two years and will have to pay fines up to $1179. http://t.co/72bcyfbTj3, +566,RT Saginaw - Police sketches of 2 'persons of interest' in Saginaw arson http://t.co/a2tZJ07nId http://t.co/LvtP5MHjaG, +572,Fire That Destroyed Charlton Winery Ruled To Be Arson http://t.co/4a9l0jrMYB, +573,WEHTWTVWlocal: Trial date set for man charged with arson burglary. https://t.co/eUk7lKS11r, +582,Man charged in connection to Shawnee arson and burglary: OLATHE Kan. ___ A man was in court Wednesday accused of___ http://t.co/zcXmQoKtZ1, +586,960KZIM: Steele police arrest 2 in possible arson investigation https://t.co/w6ZbWryqjC, +587,@Vixuhn rip arson, +590,Arson suspect linked to 30 fires caught in Northern California http://t.co/HkFPyNb4PS, +591,RT @TheAdvocateMag: Owner of Chicago-Area Gay Bar Admits to Arson Scheme http://t.co/wHTMwtgROJ #p2 #LGBT, +593,#LGBTQ News ?? Owner of Chicago-Area Gay Bar Admits to Arson Scheme: Frank Elliott pleaded... http://t.co/sGb9vNWqUx Via @TheAdvocateMag, +595,Mourning notices for stabbing arson victims stir __÷politics of grief__ª in Israel: Posters for Shira Banki and A... http://t.co/6o92wDfcLu, +596,Contra Costa Co. authorities arrest suspected serial arsonist: Authorities believe that___ http://t.co/bzCmzM7bi5 | http://t.co/LBQldyKgdp, +597,Owner of Chicago-Area Gay Bar Admits to Arson Scheme: Frank Elliott pleaded guilty to hiring an arsonist to to... http://t.co/L82mrYxfNK, +601,@local_arsonist @Cloudy_goldrush Man what ???? they be on some other shit, +602,mo the way she says 'carry' https://t.co/vQzRUTHRNU, +605,Arsonist Sets NYC Vegetarian Restaurant on Fire: Police #NewYork - http://t.co/agn4cL4uSK, +610,Arsonist torches a house himself and his getaway vehicle police say - Surrey Leader http://t.co/q0J5lTVD6k @RichardGEarl, +616,@Safyuan i already threw away the letter tho, +618,Hotel arsonist who caused å£70000 damage was __÷crying for help__ª: An arsonist has been jailed after a court hear... http://t.co/HNGDdBhewa, +620,suspected serial arsonist in east bay under arrest @christinKPIX5 @NightBeatTV 10pm @KBCWtv @VeronicaDLCruz, +626,ARSONIST Mc - CHRIST WALK http://t.co/D8LQHiGjT0 #Toronto #Columbus #Gospel, +627,@Gay4BB bitch dont get slapped, +629,@trap_vodka my new fave vine, +632,Suspected serial arsonist arrested in NorCal http://t.co/Dty4t75CGu http://t.co/MjxbygejyL, +634,Surveillance Video Captures Man Removing American Flag From Long Beach Home ... - KTLA http://t.co/g0o9wiAgHu, +639,Serial arsonist gets no bail not jail release http://t.co/rozs6aumsS, +645,Arson suspect linked to 30 fires caught in Northern California https://t.co/emsXyWp5s5, +647,Delhi government to provide free treatment to acid attack victims in private hospitals #AAPatWork http://t.co/xjtp8SBpt3, +648,I'm going into a panic attack, +650,End the Innovation Catch-22: Reduce the Attack Surface http://t.co/Gj4SSEhk1D #INDUSTRYINSIGHTS #mcgsecure, +663,@RealTwanBrown Yesterday I Had A Heat Attack ???? And What's Funny Our Relationship ??? Or Our Snapchat, +666,Texas__ªs Gratuitously Cruel Attack On A Dying Gayå_Man http://t.co/retdjqyCuB, +668,Cooper the Super Pooper. The hero dog who saved me from drowning and detected my heart attack before I knew it. The___ http://t.co/hzzADcnhFF, +670,Double ebony attack http://t.co/33V0RLlrKf #Black-haired #Blowjob http://t.co/t7TG3nRBje, +673,Suspect in latest US theatre attack had psychological issues http://t.co/TvsvjuAVif http://t.co/npovfR4rGo, +676,#People #Tilly the #Confused Cat Overcomes Horrible Attack to Win Hearts http://t.co/QtrsYxFzo3, +678,To love you love you love you ... Massive Attack - Angel (HD) https://t.co/9TW34Gffox v__a @YouTube, +692,@Sunnyclaribel. how many of @DPJHodges articles just attack labour or defend conservatives? see for yourself: http://t.co/shAAIjO2ZC, +693,RT: @rt_com :Tory attack on Freedom of Information is __÷assault on govt accountability__ª ___ Liberty http://t.co/UNmTOnz5c5 http://t.co/GTSbvveF, +694,It's weird I had a near panic attack & wanted to write her___ walked into the bathroom at work and nearly cried. Dreadfully miss her., +695,#porn #nsfw: Blacks attack for white married slut http://t.co/zRngKh5d8H, +696,Cop injured in gunfight as militants attack Udhampur police post: Suspected militants attacked a police post i... http://t.co/ERW7FdxnCr, +698,Newcastle: Schoolgirl attacked in Seaton Delaval park by 'pack of animals' led by a former friend http://t.co/4xbrVNib9T #newcastle, +701,Aus need career best of @davidwarner31 along with Steve smith to even try competing.. SURA virus has attacked team Australia brutally, +703,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/ajMtU7EHy2, +707,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/OGoyzOlJk5, +708,Tennessee man killed by police attacked family with hatchet: 911å_call http://t.co/pWiw9q30DT, +711,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/RICbduIACc, +715,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/zb8vVBiSY4, +718,I attacked Robot-lvl 1 and I've earned a total of 16000 free satoshis! http://t.co/2PZcXSkNCg #robotcoingame #Bitcoin #FreeBitcoin, +722,"That 'attitude problem' is a result of constantly being belittled bashed attacked & demoralized. +FUCK YOU @HOT97", +723,I must've forgot the memo where getting attacked by a resident and receiving a concussion was funny, +733,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/PuVOZi3Pa3, +741,@LogitechG BUT THEN THE FIRE NATION ATTACKED...., +742,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/7kxGS7VDFy, +743,Metro Nashville Police - simply the best. Thank you for protecting us. 911 call: http://t.co/ZWIG51QECF via @AOL, +747,I was having such a peaceful happy Twitter experience ...then *AVALANCHE OF IDIOCY*!!!!!! https://t.co/iIR1v39THE, +749,Car Receiving tube Installation for Avalanche Sound Output HHw, +750,if this fucking is true i will be decapitated and throw my head into a avalanche https://t.co/4cw6Nmmw1O, +756,'One glance and the avalanche drops. One look and my heartbeat stops.' Oh WALK THE MOON how you get me.. I couldn't have said it better., +757,Photo: #Repost @gregskazphotography with @repostapp. åáåáåá @rainierarms Avalanche Ambi Charging Handle... http://t.co/t3dK3OoV7e, +760,@cierranb5678 I was just there and my dad told me he would never let me get an Avalanche, +764,As you can imagine I had plenty to talk about with a maths teacher from Liverpool of a similar age into music !, +765,http://t.co/p5PnsRPH5G -RARE VINTAGE COLLECTABLE WINDRIVER AVALANCHE EDT 60ML SPRAY FOR MEN#Deals_UK http://t.co/wkZIYnyyrY, +766,PLUS PERFORMANCE CHIP FUEL SAVER CHEVY SILVERADO/AVALANCHE/BLAZER/TAHOE http://t.co/eocKbeOGm0 http://t.co/cvjCidgEZ7, +768,Dorion: Sens looking for consistency from Hoffman #ColoradoAvalanche #Avalanche http://t.co/msK68XoY7T http://t.co/ykWKx0QKtX, +769,@Avalanche need anyone to carry your bag? ??, +771,Musician Kalle Mattson Recreates 34 Classic Album Covers in Clever Music Video for 'Avalanche' http://t.co/97eWeUWybf, +772,2 TIX 10/3 Frozen Fury XVII: Los Angeles Kings v Avalanche 216 Row:L MGM Grand http://t.co/qyCwho8guN, +776,FC Maid Agency presents Musician Kalle Mattson Recreates 34 Classic Album Covers in Clever Music Video for... http://t.co/BObrp2qwCb, +778,02 03 04 05 AVALANCHE 1500 REAR AXLE ASSEMBLY 2055271 http://t.co/VxZhZsAlra http://t.co/HmXWRkbLS0, +780,Senators have another year to determine Hoffman's v... #ColoradoAvs #NHLAvalanche http://t.co/CIzZOc6f0D http://t.co/PEmYHD3Nfz, +785,@Mano_twits @Kolaveriboy avast an avalanche of andals, +789,Colorado Avalanche @ Anaheim Ducks 1-7 tickets 10/1 preseason http://t.co/XRU1WowZYG, +792,@90sAngelicMalik listen to wtm avalanche and take a deep breath, +793,Now there's an avalanche... These men are lunatics #livingontheedge, +811,#environment Lone Pine trees growing across the country in remembrance of battle at Gallipoli 100 years ago: T... http://t.co/GJCAAnfCYk, +813,@JackiSheaffer I have the same battle!, +816,@exoticengram @TheRasputin That Raspy is so cool and he has already figured out TTK's battle shit., +821,I liked a @YouTube video from @sharpino1990 http://t.co/LU7dgOwtyl Pok_©mon Omega Ruby and Alpha Sapphire Wi-Fi Battle #05 cjs064 vs, +824,After a long battle @Spurs Matt Bonner took care of Brian Scalabrine in knockout this after... (Vine by @NBA) https://t.co/Q3mGYRDHSF, +825,I liked a @YouTube video from @screwattack http://t.co/W5dLLV48cs Knuckles punches into DEATH BATTLE!, +827,"#LonePine remembered around Australia as 'descendants' grow via @666canberra #Gallipoli #WW1 +http://t.co/bwadX8ywqN http://t.co/UPFjy88KvI", +830,FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps #breakingnews, +831,FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps http://t.co/kA0syAhqVW via @usatoday, +839,FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps http://t.co/1CsEBhPLfh via @usatoday, +844,Unsurprising/still dismaying consequences of how we went about securing pathogen access: http://t.co/D8EFmGyR2d, +847,"FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps + +http://t.co/SjNKpJ8lEe + +#watchcbs19 http://t.co/JiRXfok46c", +850,Latest fallout from biolab safety lapses: FedEx no longer will transport anthrax & select agent research specimens http://t.co/4iOOyIbxyo, +854,#FedEx no longer to transport #bioterror germs in wake of #anthrax lab mishaps http://t.co/Ziw1RWPJkK, +855,FedEx stops shipping potential bioterror pathogens: FedEx Corp. (NYSE: FDX) will no longer deliver packages that___ http://t.co/CRD6tgIZhG, +858,Citizens Education Project has acted for years as a watchdog on the Dugway Proving Grounds. http://t.co/NZHXYapLm0, +861,FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps http://t.co/L04Q4DKvue http://t.co/z5voujEus4, +862,FedEx no longer will transport bioterror germs http://t.co/9lam19N5i5 via @USATODAY, +865,In offbeat news @fedex has stopped shipping bioterror pathogens http://t.co/gvXXidHOlF, +869,FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps http://t.co/RghHkWtKLR #newsdict #news #FedEx, +879,Bioterrorism and Ebola. http://t.co/ORIOVftLK4 RT #STOPIslam #TCOT #CCOT #MakeDCListen #TeaParty, +880,The largest bioterrorism attack on U.S. soil involved tacos @atlasobscura http://t.co/1n9CRhWmgl, +887,Is it time to hedge against catastrophic risks such as climate change asteroid impacts or bioterrorism? https://t.co/HQ6WqsgSJX, +889,@USCOURT If 90BLKs&8WHTs colluded 2 take WHT F @USAgov AUTH Hostage&2 make her look BLK w/Bioterrorism&use her lgl/org IDis ID still hers?, +897,Bioterrorism : Guidelines for Medical and Public Health Management (2002) http://t.co/zkUXHnoBaG http://t.co/FHgkYxffu5, +900,@bbcworld Bioterrorism drones can reach large populated erea., +901,@BishoyRagheb fair. Bioterrorism is the way to go. Does that mean my BSc isn't irrelevant?, +904,@DebtAssassin They forget there is the threat of bioterrorism., +908,HST's Anthony Kimery Discusses #BioTerrorism on @Syfy Channel's 'Joe Rogan Questions Everything' #Pathogen #Virus http://t.co/0FDsc3f2IW, +909,Anthony Kimery Discusses #BioTerrorism on SyFy Channel's 'Joe Rogan Questions Everything' http://t.co/zckJWAcRCs, +910,@HarvardU If 90BLKs&8WHTs colluded 2 take WHT F @USAgov AUTH Hostage&2 make her look BLK w/Bioterrorism&use her lgl/org IDis ID still hers?, +913,Uganda Seen as a Front Line in the Bioterrorism Fight, +914,Ppl living w/ HIV have been charged with aggravated assault and bioterrorism for things w/ low or no risk of transmission. #HIVIsNotaCrime, +917,Studying anthrax and bioterrorism before 7 am ?? #carpediem, +918,@GACourts If 90BLKs&8WHTs colluded 2 take WHT F @USAgov AUTH Hostage&2 make her look BLK w/Bioterrorism&use her lgl/org IDis ID still hers?, +920,Bioterrorism: Guidelines for Medical and Public Health Management Donald A. Hen http://t.co/tDg0Bsmd6c http://t.co/iirWFZMKHO, +922,To fight bioterrorism sir., +924,To fight bioterrorism sir., +925,@prsnvns that's good to hear. Mine could be better but I can manage. It's alright you do what you have to. Bioterrorism doesn't wait., +927,@mariashriver Was '@FultonInfo'Court trip 2 keep strangers-BLK&WHT-from us'g my @SSAPress ID or 2 forfeit 2 them due 2 Bioterrorism look?@AP, +933,@_AfroJazz I'll add you (i dont even know what you talking about), +937,I never got paid to give a fuck..we might as well blaze another one, +943,@JuneSnowpaw Yeah Gimme dat creamy white stuff ;3, +949,Your reaction determines that, +950,Port Coquitlam fire crews say an electric towel warmer started the four alarm blaze last week that destroyed several businesses., +954,I liked a @YouTube video from @ashens http://t.co/yC9tywuzQm Blaze / Atgames Handheld Mega Drives | Ashens, +966,Craving slurpees ;-;, +967,Omg i need to go like yesterday ???? https://t.co/3oP6G7ovzO, +969,@OVOIzic @BaddieMoneySign blaze it, +970,Lithgow News: Homeless after blaze near Portland http://t.co/Cht5w3znIK, +973,"@Blizzard_draco I saw + +let's eat blaze @BlazinSmasher", +975,My big buzzy John BlaZe. Jus Kame home from a 12 year bid. #LoveThyFamily https://t.co/xrEuoNzzbi, +980,Just cruisingblazing and banging ?, +988,the bitches say im hot i say no bitch im blazing, +989,@BryanVsBracey @uSTADIUM why do you need blazing speed? I think Watt is far more impressive that OBJ who is just another quick little guy, +995,I checked in at Blazing Horse Tattoo on #Yelp http://t.co/z8nXWmYMWA, +1000,"The midnight song I cry out goes 'In reality I... in reality I... was very lonely' +Even if it is erased by the blazing sun", +1003,Le Ever Blazing, +1007,All packed and ready to go to meet @kkarmstrongg in Portugal tomorrow! Bring on blazing sunshine and fun times with the bestie! ??????, +1011,"S3XLEAK!!! +Ph0tos of 19yrs old Ash@wo lady in Festac town from Delta exp0sed on BBM 5 leaked pictures... http://t.co/lUm4l65alz", +1012,Enjoying the shade under this tree. The sun is blazing but there is a cool breeze. @ West Hollywood Park https://t.co/2wzHj0lNa6, +1013,HAPPENING NOW - FDNY RESCUE 1 BLAZING THRU@AND BLASTING THE AIR HORN___ https://t.co/bX6cwRt8M7, +1014,Night 3 of insomnia this whole stop blazing shit is real, +1016,Satelite Shot: Series of wildfires in Northern California continue blazing: There are at least five fire compl... http://t.co/BmGuHqY7W4, +1019,#website #hosting Get blazing speeds professional management and unlimited options @nyndesigns http://t.co/d1F7wi7FMr, +1025,"Come and join us Tomorrow! +August 7 2015 at Transcend:Blazing the Trail to the Diversified World of Marketing... http://t.co/Fd5od5PfCF", +1027,"S3XLEAK!!! +Ph0tos of 19yrs old Ash@wo lady in Festac town from Delta exp0sed on BBM 5 leaked pictures... http://t.co/2WUpSDttVi", +1028,@BlazingRoselia California is hot but not blazing even though there are wildfires going on now. Even the big massive Rocky fire, +1030,Do people not understand they cant bleeding tweet everyone all the timedoes me head in 'stop ignoring me' they are hardly ignoring you+, +1033,#Nepal: This house near TIA bleeding int__ªl airline companies dry. | http://t.co/lI8wi1kuMY, +1034,@Battlehork I wish I could watch but the gif the LU account posted of Mundo bleeding... Jesus! D:, +1039,did leona lewis ever see that video of dinah singing bleeding love?, +1046,@TomBrevoort 'Bleeding Cool as read by Tom Brevoort. Though he may just look at the pictures.', +1047,You cut me open and I keep bleeding. #BleedingLove #LeonaLewis #acapella #singer #canadiansinger___ https://t.co/51pfEIlPNK, +1053,*Ears bleeding from the bass* https://t.co/d5RrrwHjpN, +1055,@KrisZellner But Triplemania has the promise of Villano bleeding and Del Rio whooping that Trick Myztiziziziziz., +1056,Looks like Reynolds and Montano coming in. Need to stop the bleeding. Not the best performance so far., +1059,@imKanoMate [the stabbing did not really affect her but she did get stab wounds and was bleeding Skarlet then pushed him backwards +, +1060,Yeah I hate cats I just tried to pet our own cat and now my hand has bite marks scratch marks and it's bleeding. #DogsAreBetterThanCats, +1063,@Aichmomanic wraps a cloth on my wound but it's still bleeding 'I'll be ok.' Falls to the ground as if I'm going to die, +1064,my ears are bleeding https://t.co/EeVQj37Ndg, +1068,@dudeandpal it's so bad my eyes and ears are bleeding, +1076,wouldn't want me to quit even if I was bleeding, +1086,@idkimnotfunny this quickly blew up and my mom doesn't get why it's so popular, +1087,LMFAO MY SNAP JUST BLEW UP!, +1089,Just blew you up @abbss_13 just so I can get that follow back ????, +1092,@atlwtmgc damn this blew up, +1095,Wow social media really blew up after the shania twain concert, +1096,@cieIstea IDK BUT IT BLEW UP, +1097,* I'm always one phone call away but too many people done blew up they chances with me., +1100,well this tweet blew up fast, +1101,Well this blew up while i was sleeping, +1107,@funnychubbyguy poor white guy his mom made him pizza and he wanted mac n cheese so he blew up a school. Such a sad story., +1108,Wow bro blew up quick. Believe it or not he's this funny and clever in person at all times. You be like 'where... http://t.co/djIHN95YnB, +1111,@TokyoDotCom I didn't either till it blew up my mentions lol, +1115,We were fucking around on Google maps at work and I pulled up Boise and it blew their minds. Yeah Idaho isn't what you think at all.?, +1116,That stadium blew up!!, +1121,The balloon thing was so rude I would have popped the balloon than whoever blew it up's face, +1125,@TheFieryGrave yeah true because if he wouldn't have got exposed then he probably wouldn't have went to mlg and blew up there, +1127,The only reason why player's now have an ego is cause MW3 had cod champs that's when eSports Blew up and now player's strictly look for ORGs, +1131,THDA Kicks Off Anti-Blight Loan Effort in Memphis http://t.co/7Gf7CpYL0R, +1133,.@WestmdCountyPA land bank targets first #Latrobe building in 20th property acquisition to fight #blight: http://t.co/regDv873Aj, +1135,Lots of talk of how susceptible NY2 is to shoot blight with or without trama. #WCFGT2015, +1137,Help fight Northern Corn Leaf Blight with Headline Amp https://t.co/LVoqPgkLI7, +1140,@colemcfadyean fuck off cole ??????, +1144,"Imagine if 2 Old Gods were awakened to start a Blight 2.0 +@BioMarkDarrah @Mike_Laidlaw", +1147,Stamford tightens blight law: A dilapitated barn sits half-erect on Hickory Road near the int... http://t.co/QP65iO4AvC #stamford #topix, +1148,Cleveland Heights Shaker Heights fight blight - http://t.co/qRnyme33TC via @clevelanddotcom, +1150,Keep up the great work Councilman Brian O'Neill http://t.co/ueS3IhSyIq, +1158,2:20 BLIGHT ?? EVERY DAY, +1159,"@Michelgrabowy @vegetablefarmer INTERNET LOSER: I like to leave snide remarks. +ME: This person is a blight upon the face of the earth", +1161,ok. but important question: how are the inquisition fighting darkspawn and NO ONE gets the blight in the entire game, +1163,Member Post: Norwalk blight ordinance said to be success as properties change hands http://t.co/ZDwmKQEYBn, +1165,Time is ripe for yet another banana blight: The banana's appeal is not slipping. But the future of bananas is in___ http://t.co/StPS5mruXW, +1169,"This is the Red Eye crossing the ocean +Peeling potatoes with servile devotion... + +#BLIGHT!", +1171,#Detroit has made progress against blight but too many burned out shells of houses remain., +1172,@justin_fenton Per my Sun piece from last year I think this is the opposite of what should be done: Urban Blight should be monumentalized., +1176,I'm all for renewable energy but re: windfarms I do agree with Abbott - they are a horrible blight on the landscape https://t.co/UjYfRT77eJ, +1180,Buying MoP http://t.co/tl7o6Zsqzy, +1184,@LoneWolffur *slaps* no, +1186,Literally for lunch i had cheese curds and for dinner i had a blizzard pls pray for my health, +1187,Oh I see we are now tied? Sweet! It's all in the Blizzard, +1192,Check out the #new #ebook on #sale by #TimZak 'The Lizard Stuck in a #Blizzard' #rhyming #kidsbook on #Amazon http://t.co/kVZuxdL4ij, +1193,Deck building of my 4 year old daughter http://t.co/INnY1ANTam, +1194,@EddieTrunk Blizzard of Ozz, +1197,GUNNAR Optiks -Heroes of the Storm Siege Gaming Glasses - Onyx/Fire wow blizzard - Full re___ http://t.co/YEUQp0VKwU http://t.co/OuGM0yxQMo, +1200,This game drove me to a rolo mini blizzard, +1205,That horrible moment when u open up the dryer and it looks like a snowy blizzard cuz u left a piece of paper in your jeans pocket ??, +1210,If blizzard did another 12 month sub thing and gave the next wow expo free would probaly knock their sub count up pretty fucking high, +1216,the Internet is sending subliminal messages about ice cream therefore I need a blizzard rn, +1220,Whoa! This new legend will be fun! http://t.co/hu5CmoupUM, +1231,@Louis_Tomlinson if u relate to billie jean then ur gonna also enjoy blood on the dance floor so #BuyBloodOnTheDanceFloorOniTunes, +1233,xoxoxxxooo : rutinaofficial NBCHannibal When #Hannibal asks will 'Have you seen blood in t___ http://t.co/T7ldlcyC9g http://t.co/JLRw0Oi9ee, +1246,BRING ME THE HORIZON BLOOD STAINED GIANT ART PRINT PANEL POSTER NOR0603 http://t.co/u5ELKCA4CW http://t.co/tsGKlZZS0C, +1247,I ain't a crip or a blood....I'm doin my own thing, +1248,Bitch I'm a monster no good blood sucker, +1255,/criess blood/ Golden Wook is life. http://t.co/Fhb99ciCeT, +1256,'blood makes you related but loyalty makes you family' - @SirBeezAlot, +1257,@youngthug Remind Me of Blood Sosa From Belly.....do your homework., +1258,The Yankees are the reason I'm on blood pressure medicine, +1260,Why can't gay men donate blood? http://t.co/v2Etl8P9eQ http://t.co/NLnyzeljbw, +1261,My new mashup of 'Bad Blood x Where Are U Now' comes out tomorrow!!! Make sure t... (Vine by @Shaylen_Carroll) https://t.co/zB6GZJLgkl, +1265,1860 BLOOD'S PENNY POST Philly OCTAGONAL CDS 1_âå¢ Black 15L18 Grid Cancel + #26! http://t.co/rpV4MQSCQA http://t.co/JrjEiBUGsE, +1266,seeing more blood and surgery than a regular GP behind a desk and they say dentists aren't real doctors .... Ok, +1268,Sometimes blood ain't no thicker than water and sometimes family will bring you down quicker than strangers ???????, +1274,"Name: Chizu +Gender: Male +Age: 10 +Hair: Red +Eyes: Pink +Dere Type: Pasokon +Blood Type: Type O +http://t.co/cOyPF9ACTd", +1281,The whole of New Zealand is shouting 'Bloody Marvellous'! John Campbell to join Radio NZ http://t.co/F88fCLiVzH #radionz, +1285,Whats the bloody use of the bus app if everything has no prediction? No one driving bus today? http://t.co/zvGrM5PGZa, +1286,@ashwilliams1 continues to be the best guest on @iLoveGGLetters. This week's episode is bloody outrageous., +1291,Too bloody hot in my room, +1292,Im so bloody excited to see Maisy and Martha, +1295,@JeremyClarkson buy a new one bloody ell, +1299,@tomm3h You've been on Twitter since 2008?! Bloody hell. Veteran!, +1306,Awful refereeing has ruined the balance of the game. Guy is a bloody mess #IMFC, +1310,Just because of his 'precious lunch' ??? Bloody hell no brain issit?? https://t.co/X5dwKElReo, +1311,I swear my eyes be bloody red but bitch I feel amazing., +1313,@ShtBallPlayrsDo are you anyway running your mouth about this bloody game? Pree sure you're not bigger then baseball. Calm the hell down, +1314,Bloody nurses waking me up just when I eventually fall asleep lol #Hospital, +1322,@halfpeachh @CaitlinCavanah I can't deal right now OMG WHO REMEMBERS BLOODY MARY? ????, +1323,__÷A Nightmare On Elm Street__ª Is Getting Remade___ Again http://t.co/qaYGdzI9jQ via @bdisgusting, +1325,BLOODY HELL TEEN WOLF *O* Hayden and Liam are so cute :3 And I think Stiles's Jeep is dead huh ? Pleeease... I loved her ! @MTVteenwolf, +1329,Luckily I'm up cause if these Mexicans had came to cut this grass this early and I was sleep. I woulda had a full blown attitude, +1330,If you bored as shit don't nobody fuck wit you... But when you busy yo shit get blown up! Smh, +1333,"Yes they know you're an adultress. +And yes they HAVE #Killary emails. + +Like the smoke being blown up ur azz? + +#NSA + +http://t.co/pNheTk8a0f", +1336,@SassyNreal Agreed. Whole thing has blown up in their faces right wrong or indifferent. The public is relentless court of public opinion, +1339,I don't understand how White Iverson by Post Malone hasn't blown the fuck *UP* already., +1342,@KaylaG_Baby1 @jaykpurdy no shush. I'm getting it blown up poster size. ??, +1344,My license picture blown up is absolutely terrifying. http://t.co/0NVdzxQ2HF, +1355,Deadly suicide bombing hits mosque in Saudi Arabia: A suicide bomber has blown himself up at a mosque in the s... http://t.co/RkvU3xjAzH, +1357,My phone is being blown up rn ??, +1358,@DodgersNation he was due to get blown up at least this is still a winnable game., +1359,Lock up your last remaining shred of manhood and accept that you__ªre a full blown sissy http://t.co/ZN17ic99hb #fe___ http://t.co/u42luY3slS, +1364,Get blown up with a delicious flavor of #BlowVape a strawberry cherry pop! Try it today for $15___ https://t.co/iT8rDItjFV, +1366,We are now up to run no. 24 in the singles. The rain has blown off again., +1367,Damn! @LastWitchHunter has blown up my Twitter feed. Looks amazing!, +1370,@MalikChaimaa I hope Zayn gets blown up in a drone attack whilst visiting family in Pakistan ??, +1373,Coldwood is blown away by your support - they even hung up some of your art! http://t.co/uQWlnbJEYE http://t.co/MToWVqkSXC, +1377,@bmrow At first thought it was a blown-up DVD as I'm pretty sure POST MORTEM had been years back., +1386,New Ladies Shoulder Tote Handbag Faux Leather Hobo Purse Cross Body Bag Womens - Full read___ http://t.co/Rpn6pXLPiB http://t.co/l6aYlVVDZ0, +1387,Cute Soft Washed Fabric Camouflage Outdoor Shoulder Cross-body Bag Pack Purple Camouflage http://t.co/bxVFQqsIhC, +1392,Cross Body Bag Purse Zippers Shoulder Bag Camouflage Camo Phone Case Camera Case Wallet #wallet #wallets http://t.co/cq6XYhR3LV, +1397,Louis Vuitton Monogram Sophie Limited Edition Clutch Cross body Bag - Full read by eBay http://t.co/YC2FYXiKBA http://t.co/Pv8wRAvNZQ, +1398,"#3682 Nikon D50 6.1 MP Digital SLR Camera Body 2 batteries carry bag and charger http://t.co/TeT9q0kXvl + +$200.00 +___ http://t.co/h3XgoxTFfi", +1400,AUTH LOUIS VUITTON BROWN SAUMUR 35 CROSS BODY SHOULDER BAG MONOGRAM 7.23 419-3 - Full read___ http://t.co/8AzzJKBxnm http://t.co/xNgHnEtdvv, +1403,Dracula Vampire Cross Body Small Shoulder Bag http://t.co/XPHTe7sb4X, +1404,http://t.co/3d2h2wOQwa NEW! DIESEL WILLYT Printed Front Cross Body Shoulder Bag - One Size * /KSL 126, +1410,New Ladies Shoulder Tote Handbag Women Cross Body Bag Faux Leather Fashion Purse - Full re___ http://t.co/cRQVl68aoX http://t.co/OFLvMMbfMk, +1413,"#7294 Nikon D50 6.1 MP Digital SLR Camera Body 2 batteries carry bag and charger http://t.co/SL7PHqSGKV + +$200.00 +___ http://t.co/T4Qh2OM8Op", +1416,__¼? New Ladies Shoulder Tote #Handbag Faux Leather Hobo Purse Cross Body Bag #Womens http://t.co/zujwUiomb3 http://t.co/sdvNAKZek4, +1417,I would love to #win Suze's amazing filled #beauty #bag #giveaway! Contents from L'Occitane Body Shop & Lavera. http://t.co/hEjJVVRsTY, +1423,Pl. Parenthood Employee Laughs About Aborted Baby Body Parts 'All Mixed Up Together in a Bag' http://t.co/AIdGzPqfLL, +1424,?? New Ladies Shoulder Tote #Handbag Faux Leather Hobo Purse Cross Body Bag #Womens http://t.co/zujwUiomb3 http://t.co/eDHjUjIuoB, +1426,Photo: bonesymcbones: Beartooth - Body Bag (x) http://t.co/kNH4HLFuMa, +1427,Two-Tone Cross-Body Smartphone Bag Only $15 Shipped! via Dian's Daily Deals - ** Two-Tone ... http://t.co/91LaZl2crV, +1428,genuine Leather man Bag Messenger fit iPad mini 4 tablet case cross body air jp - Full rea___ http://t.co/Vl26HSrq4E http://t.co/ryl0Y88fKM, +1430,@MeekMill I think its time you consulted @kendricklamar cause @Drake done body bagging you, +1434,People are bagging on Rousey's body? Shit I'd love to have a body like that. Those who are ridiculing her are probably dudes in skinny jeans, +1435,Bagging the last spot on Body Attack ?????? #getin #traintuesday #lesmills, +1437,' Dear Lord Forgive Me For Body Bagging @MeekMill Not Once But Twice. Amen ' - @Drake http://t.co/dPBz1PYvFF, +1438,@cameron13131313 @OtooleDalton @averywelch23 @otoole1055 first of all you can't 'hit stick' shit.. Second I'm body bagging you on sight ??????, +1442,Drake is really body bagging meek atm, +1446,I remember the halo interview with ABC. Wasn't no time and The Pamanian Devil was body bagging fools. https://t.co/CpK7300RQg, +1451,Jersey's artist Wolfgangjoc pays homage to the late great Biggie Smalls by body bagging his hypnotize beat.... http://t.co/x0jKnOxJHA, +1457,Not only did Drake kill Meek Mill but he started T bagging his his dead body ????????????, +1461,This man @zvch4 about to start body bagging these local jokal photographers, +1462,Body bagging! https://t.co/HSlenz4KGU, +1465,Drake is body bagging meek, +1468,Body bagging that I think it's time to bring bags out, +1469,We need to stop paying attention to @drizzy body bagging @Meekmill and worry bout what happen to #SandraBland, +1471,I hear my up north niggas body bagging this but you never know. That got some spitters down south and___ https://t.co/XToioGtk2w, +1476,Eish even drake killing niggas eish game is really watered down a singing nigga body bagging rappers.. smh, +1478,@ESM_Campy and he used werewolf on me also idiota I was tea bagging your body for like 7 minutes while he was fighting someone else, +1481,'Body Bags' (1993) Movie Review!: http://t.co/E1tdatSUcC via @YouTube, +1489,@LennGoneWin nah man we need to be defenseless from thugs so when the cops get there we can get put in cool new body bags!, +1490,@ComplexMag he asking for a body bags @PUSHA_T, +1492,RT Hamosh84: This how the #Saudi is coming home from #Yemen in body bags. For what? #Saudi Kingdom ego? You're loo___ http://t.co/LJlKWGXgjY, +1496,Hot Ladies Handbag Leather Shoulder Tote Satchel messenger Cross Body Bags Green http://t.co/c7XBjRhNbi #Handbags http://t.co/1dmoLajv1Z, +1512,"Fairfax Co. investigating firefighter Khalil Abdul-Rasheed for Facebook post saying police shld be put in body bags + +http://t.co/VpvskCIyro", +1516,Our mistakes come back in body bags: Leading in the VUCA world - How the Armed Forces do it | RAGHU https://t.co/vJFSlizmiI, +1517,Womens Stylish Metal Chain Pearls Solid Hanbags Shoulder Cross Body Bags Golden http://t.co/oR0JZb9cFh http://t.co/6byul0ID4l, +1528,Womens Flower Printed Shoulder Handbags Cross Body Metal Chain Satchel Bags Pink http://t.co/vBZqWzNxH7 http://t.co/NXIhHeP1Ea, +1529,One of the top RT-ed tweets in #Japan is a screenshot of a young person claiming to not know the 70th anniversary of the Hiroshima bomb, +1536,Hiroshima marks 70 years since bomb: The Japanese city of Hiroshima marks the 70th anniversary of the dropping... http://t.co/INRGS4z0AC, +1539,@joanna_bomb @eventchase @MTV @MTVNews last year they won ATW and now they're multi platinum artists #WeNeedA5HVMAPerformance ????????????, +1541,@peytonlaynec ohhh coming of age. I had a bomb teacher and we never did anything. For our final we went to salt grass and he bought us food, +1542,My bestfriends are the bomb. ??, +1548,RIP RT @AJENews: On Aug. 6 1945 at 8:16 AM Tokyo time the world__ªs 1st atom bomb on Hiroshima http://t.co/6Q5CS9MCdW http://t.co/7OGFoQr0MG, +1550,@joshrogin @slwsew379 @jimsciutto @NatSecCNN As suspected. Kerry and Barry got taken for a ride. Iran was making a bomb while talks went on., +1551,@jonathanshainin I think the bomb raises all sorts of great questions but this isn't one of them, +1552,Why American censors supressed accounts of suffering in Nagasaki http://t.co/kZbeas50KE http://t.co/YQgpaSX3MG, +1557,What if your hometown were hit by the Hiroshima atomic bomb? http://t.co/XuWN4McgYp #safemode, +1563,@johncusack text: We Dropped The A-Bomb (1946)https://t.co/boJCX32qcZ ___, +1564,Bruh this sex on the beach??is bomb as fuck yo, +1565,Hansel and Gretel: The Witch Hunters.... Bomb ass movie. Yeh I kno Im late, +1566,<Kyodonews> UPDATE1: Hiroshima marks 70th A-bomb anniv. amid fears of eroded pacifism http://t.co/VpGu8z1Lhb #followme #japan, +1571,@MattieLBreaux u the bomb ????????????????, +1578,My aunt just gave me this drink at the game and it's bomb???? I'll never tell what in it though shhhh??????? http://t.co/vsFS7zr1gI, +1581,___Tell Shinichi Kudo that I__ªm giving him 3 minutes to enjoy it..___ -Moriya Teiji famous architect from The Time Bombed Skyscraper, +1583,Photo bombed by a princess http://t.co/i4PthsuYQ9, +1584,When you get a call back from the interview you thought you bombed >>>, +1586,At this hour 70 yrs ago one of the greatest acts of mass murder in world history occurred when #Hiroshima was bombed. http://t.co/2QJl5MfKzv, +1589,70 years ago nowish America bombed #Hiroshima. Was it justified? All I can say is #noMOREHiroshimas #VJDay70 #FEPOW http://t.co/SYADSa6QeS, +1592,#Hiroshima does not look like a bombed city. It looks as if a monster steamroller had passed over it and squashed it out of existence., +1598,6th July 1945 ...Hiroshima was bombed with nuclear weapon _bomb was named little boy&carried in a plane called Enola Gay ...1/2, +1606,@allenpeacock the deal is clear.Iran either agrees to the deal and abandons it program or gets bombed.it's a deal the right wing should love, +1612,Picture of cute sleeping puppy photo bombed by the demon in the back. https://t.co/LprYzSTK4u, +1616,@davej798 @JohnEJefferson remind me when SomaliaEritrea & Nigeria were bombed by UK 28% of asylum applicants to UK 2014 were from S & E, +1620,#SitCTips if you do vlog you WILL get video-bombed., +1624,I BOMBED THE FAKE BOLT CHAT AND EVERYONE THOUGHT BOLT ENDED LMFAO, +1629,Bombing of Hiroshima 1945. http://t.co/4UHq9jKCvq, +1630,We dont really talk about the murder of the bombing campaigns in WWII because we won but murder it is. https://t.co/jsGwnY6kdJ, +1635,#Hiroshima marks 70th anniversary of atomic bombing ( via NHK WORLD News) http://t.co/qGWoyNJLY8, +1640,Japan to mark 70th anniversary of Hiroshima atomic bombing http://t.co/RUQPVtdBzT, +1641,Just Happened in Asia: Japan marks 70th anniversary of Hiroshima atomic bombing http://t.co/E5dEmdScEh, +1642,What it was like to survive the atomic bombing of Hiroshima http://t.co/0cvXS2E1Er, +1651,70th Anniversary of Hiroshima bombing https://t.co/juwvomhGPd, +1655,Japan marks the 70th anniversary of the atomic bombing of Hiroshima. http://t.co/YmKn1IwPvF http://t.co/mMmJ8Bo9y3, +1656,'Japan Marks 70th Anniversary of Hiroshima Atomic Bombing' by THE ASSOCIATED PRESS via NYT http://t.co/kKULqGB9e3, +1659,The 'sanitised narrative' of Hiroshima's atomic bombing. #Hiroshima70 #japan http://t.co/zsyj6sqYCn, +1664,Japan marks 70th anniversary of Hiroshima atomic bombing: Bells tolled in Hiroshima on Thursday as Japan marke... http://t.co/IqAIRPdIhg, +1667,Japan marks 70th anniversary of Hiroshima atomic bombing http://t.co/MLZxmGVuIh, +1668,70 yrs since the atomic bombing of Hiroshima... Terrible mass murder..., +1674,BREAKING NEWS: Australia collapse to a hapless 60 all out at Trent Bridge http://t.co/O2CFWDzZld, +1678,"Project Syndicate: A Marshall Plan for the United States: +http://t.co/lz8xmyi75x", +1680,New: Two giant cranes holding a bridge collapse into nearby homes http://t.co/ACirPv8Hn4, +1681,@AkinwunmiAmbode Urgentthere is currently a 3 storey building at church B/stop Oworoshoki Third mainland bridge which likely to collapse, +1682,"Throwback Thursday: More Thoughts About Citizen Media During Minneapolis Bridge Collapse Disaster - http://t.co/GudUDAZ4fx +#TBT", +1685,Ashes 2015: Australia__ªs collapse at Trent Bridge among worst in history: England bundled out Australia for 60 ... http://t.co/l5ULCx4mOm, +1695,Two giant cranes holding a bridge collapse into nearby homes http://t.co/Mqm46H6G8b, +1696,"Throwback Thursday: Minneapolis Bridge Collapse & Citizen Journalism - http://t.co/bXV3vlMBBp +#TBT http://t.co/iBmqVWrVyz", +1697,'@berggruenInst: Berggruen Institute member @dambisamoyo on the problems with #US #infrastructure @ProSyn http://t.co/T96O158Yxv', +1704,Australia collapse at Trent Bridge how Twitter . On the first morning of the fourth Ashes Test... http://t.co/WAtjhzULa8, +1708,Shane Warne and Ricky Ponting shocked by Australia batting collapse at Trent Bridge http://t.co/b5zPfW7Vo0, +1711,2 Injured 1 missing in bridge collapse in central Mexico http://t.co/scDa8eVwR6, +1713,Australia's Ashes disaster - how the collapse unfolded at Trent Bridge http://t.co/UqEtEeltVU #telegraph, +1714,#Sport - Shane Warne and Ricky Ponting shocked by Australia batting collapse at Trent Bridge Australian grea... http://t.co/c41iUMknis, +1717,5 injured in bridge collapse at Swiss kosher hotel: The accident occurred on Tuesday at the Metropol Hotel a ... http://t.co/sIGeZlynJ9, +1729,a very scary thought is if 9/11 had happened today their would be teenagers taking selfies in front of the burning buildings, +1730,THIS SOUNDS LIKE A SONG YOU WOULD HEAR IN A MOVIE WHERE THEY ARE WALKING AWAY FROM BURNING BUILDINGS AND CARS AND SHIT, +1732,@CTVKathyLe and in other news: don't run into burning buildings, +1734,forestservice : RT dhsscitech: #Firefighters run into burning buildings___we work on #tech t___ http://t.co/KybQcSvrZa) http://t.co/Ih49kyMsMp, +1736,this is like when folks said ___burning down buildings doesn__ªt work______only___it did. now cities trying to get information out ASAP to prevent it, +1738,Like how are ppl not burning these buildings down knowing why it was originally created?, +1742,kou is like [CASH REGISTER] [BUILDINGS BURNING], +1743,@chistate33 @thehill What hate crimes? It's the looney libs that are always burning down buildings and vandalzing private property, +1746,3 rules for a long life. #1___Don__ªt go in burning buildings! #2___Don__ªt jump out of perfectly good airplanes! #3___Don__ªt awaken sleeping giants!, +1748,Port Coquitlam fire burning several businesses http://t.co/Q5FzYLBvTk, +1749,@speechgirlAM On par with caring for the sick helping people move not talking during movies rescuing people from burning buildings..., +1751,@TravDave @Kornbread_ICU especially where it happened. Really nothing to tear up. Burning the buildings saves the city money for real, +1758,The fire brigade should play this guys music at burning buildings to put the flames out, +1764,Two buildings burn in 3-alarm Antioch fire: Two buildings are burning in a 3-alarm fire at the Delta Pines___ http://t.co/GQgF2ygpX2, +1765,.@denisleary Not sure how these folks rush into burning buildings but I'm grateful they do. #TrueHeroes, +1777,"Jane Kelsey on the FIRE Economy +5th Aug 5:30___7:30pm +Old Govt Buildings Wgton +The context & the driver for #TPP and #TRADEinSERVICESAgreement", +1778,'Failure doesn__ªt exist. It__ªs only a change of direction' - Alejandro Jodorowsky (via buildings-on-fire) http://t.co/ByMVemW4xF, +1781,I'm security so they want me to help out in case of emergency like the buildings on fire or a shooter's in the building. I'm leaving tho.., +1782,@blainescronuts thats what i would do if there were buildings on fire, +1783,fucking apartments across the street are on fire right now. 5 fuckin firetrucks wtf. this is the second time those buildings caught on fire, +1785,It's a testimony to the human spirit that more of us don't set fire to our office buildings on Sunday night., +1788,"There are multiple buildings on fire In Downtown Hinton OK. +The fire is threatening to level the entire block! #8NN https://t.co/pEVtSf7Cyt", +1793,"#DNR +Res. buildings shelled & on fire in #Gorlovka tonight. +Civilian casualties Jan-July 2015:___ +164 killed (incl. 16 children) +501 wounded", +1794,@gilsimmons lightening struck & 2 units in 1 of our buildings in our development on fire. Groton CT. Check out my FB wall most recent post, +1795,"#Taiz +#Houthi #Saleh indiscriminate shelling from castle sets Tahreer st buildings on fire +#Yemen http://t.co/qsli2ivIoG", +1797,Buildings on fire behind Tisa's in Niceville @tristapnwfdn https://t.co/ACl1baBacR, +1800,Two buildings involved in fire on 2nd Street in #Manchester. @WMUR9 #wmur http://t.co/bCEUGsoi2r, +1801,"Agricultural Buildings on Fire - Aldwark - NYorks Fire & Rescue Service + +Still grim news but fewer piglets caught up http://t.co/0kjCWG6pN9", +1805,Multiple firefighters hospitalized after fighting a fire Manchester NH. #7News http://t.co/X1oYU5Y6bI, +1806,"There are multiple buildings on fire In Downtown Hinton OK. +The fire is threatening to level the entire block! #8NN https://t.co/GXDKLhxU22", +1819,"My Happy Pensioner Stories +Safe Buildings: watching a report on unsafe cladding catching fire in Melbourne same... http://t.co/cnoxJB8XoQ", +1820,Multiple houses/buildings on fire in Kalispell http://t.co/1iyThVshgF, +1825,Family heartbroken.Mobile home at Nellis & Cheyenne burned in fire. Kids were all ready to go back 2 school @News3LV http://t.co/VAf1RAiYQb, +1828,I burned 3 of my fingers!! http://t.co/Gs2W38CSJn, +1829,This is why you should be burned at the stake ! https://t.co/YpezipY2NY, +1830,Let me tell you Brandace brought it today at #NTC! Heat and all...we survived! Burned over 500___ https://t.co/ILfd6wcGqg, +1839,@IrSolichin @9Tawan99 @178kakapo @POTUS The last 2 of 68 cities where everyone was burned to death. Terrorism on the grandest of scales, +1843,@Kyra_Elizabethh my back break light is burned out so I gotta show them in the next 10 days ??, +1844,@realDonaldTrump @Shelia1965A Been burned by Republicans so many times at least you are talking about things important to us. Good luck, +1846,burned 163 calories doing 30 minutes of Total Body Cardio #21dayfix #myfitnesspal, +1849,Corey White talking all dat shit but he got burned more than curlin irons on black women necks, +1850,@BlackHatWriter Yeah I'm going to be at home by a computer for the next two weekends. :( Getting really burned out really quick., +1854,@alfred00dle @PDLeague I'm sorry about that crit not 100% sure if it matter prob cuz I was burned, +1855,"I got more rhymes than the Bible's got psalms +And just like the Prodigal Son I've returned +Anyone stepping to me you'll get burned +??", +1858,@widouglass @kurrrlyW @AdamRubinESPN @Nationals Welp that dunce got burned here, +1859,OMG so last week i accidentally burned myself with a cigarette lol only i would do that, +1862,@AdamRubinESPN I know we have been burned just a week ago in a similar spot but getting @MattHarvey33 out o just 88 pitches save some inn, +1867,"so keep calling me crazy +cause I never learned +you should stop loving fire +because you got burned___ https://t.co/ZsAEGjQq1z", +1868,I'm proud of my heart it's been played stabbed cheated burned and broken. But somehow still works, +1871,hermancranston: WIRED : All these fires are burning through firefighters' budgets http://t.co/FSGAnfRgjH http://t.co/ju9zzu9TiO___ ___, +1872,PSA- I chopped 3 #jalapenos & my fingers were burning-lava-on-fire painful for 5+ hrs. This is a first! Wear gloves! #peppers from #hell., +1874,Geoengineering and burning of fossil fuels is making our global weather unstable. With temps up to 165å¡ and biggest flood in past 200 years, +1876,@sar_giuliani I'm currently burning my skin off??, +1879,@JanelleBrown117 YES! We are comrades in sweat-seeking calorie-burning cardio-torture! We need all the support we can get- smile nod wave, +1884,WoW Legion ___ Slouching Towards The Broken Isles: Warlords of Draenor wasn__ªt close enough to The Burning Crusad... http://t.co/RKpmoMQMUi, +1891,@TheVenskus you led a killer workout today @TheSweatShoppe my core is burning. Now #gymtime #worktime #auditiontime #roccotime #noBS, +1894,Peddle to the floorboard.. End up in a four door. Burning up a backroad song... Park it and we pile out___ Baby watch your step now___, +1896,Burning bridges is my forte!, +1902,RT: A real burn book entry about CA: why the fuck is this place always BURNING, +1903,fear the fever Can you feel it now? I feel the fire Burning belo-ooow It's gonna trick ya Swallow you who-ooo-ole #MTVHottest Justin Bieber, +1904,the Burning Legion has returned, +1906,The Arab autocracies: Burning down their house http://t.co/xcjRamGQ22 via @TheEconomist, +1907,@TarekFatah you are burning in enemity of Pakistan .i m sure you will burn more and more, +1912,@8goingon80 I put on a ton of sunscreen and I'm still burning :/, +1913,Blizzard details new Dungeons and Raids in World of Warcraft: Legion: The Burning Legion___ http://t.co/8IBAT5LOQN, +1923,'What is this accomplishing?! Burning is killing the buildings!' #TI5Quotes #TI5, +1926,15 Amp is common 120V breaker trip threshold to protect 14 AWG wire (commonly used in buildings) from burning. 1800W power outlets results.., +1928,"Burning Buildings. + +Keep The Flames Lit. ??????", +1930,Three-alarm fire burning in Antioch apartment complex. @kron4news http://t.co/IH7cTORGRH, +1931,@RockBottomRadFM Is one of the challenges on Tough Enough rescuing people from burning buildings?, +1934,@mrjamesob U're not getting rid me until every last drop of life has gone... I crawl into burning buildings because all Life matters. Jakey, +1936,Firefighters are some brave people in my opinion bc I clare I don't know I could save ppl from burning buildings and houses I salute them ???, +1944,The Rocky Fire burning in CA still threatening more than 7k buildings. Largest in state & one of nearly 2 dozen burning now #9newsmornings, +1946,I liked a @YouTube video http://t.co/EYma5CRlHk Mega Man ZX OST - T17: Ogre Claw (Area G - Burning Buildings), +1947,.@denisleary Not sure how these folks rush into burning buildings but I'm grateful they do. #TrueHeroes, +1958,@Trubeque Destruction magic's fine just don't go burning any buildings., +1964,? High Skies - Burning Buildings ? http://t.co/uVq41i3Kx2 #nowplaying, +1970,What if we used drones to help firefighters lead people out of burning buildings/ help put the fire out?, +1974,'When you attack PP you attack women's health & when you attack women's health you attack America's health.' http://t.co/HXdG24MCQg, +1977,Soaring temperature's in some southern US states causing bush fires pity @scotgov only allow rain in Scotland. #TweetLikeKez, +1978,You know it's hot when there are bush fires right near your villa ????, +1982,Bushfire causes first victim in Albania :: The multiple fires in Albania have caused the first vict http://t.co/yuba6XmBlq #Bush, +1984,Drove past the field on fire yesterday. Reminded me of the bush fires we used to get in South Africa. http://t.co/SN3dHuYnFe, +1988,Bush #food fires: http://t.co/Q1Y8lm7us9 http://t.co/TofLxOdyrX, +1993,[WATCH] Mother Nature halts Australia Blue Mountains bushfire http://t.co/xxA4rmFZ01, +1997,@jiminswang i was about 2 say bush fires in aus but the reason is hot, +1998,Bushfire causes first victim in Albania :: The multiple fires in Albania have caused the first vict http://t.co/yuba6XmBlq #Bush, +2002,If you like goats and dislike bush fires then this is probably the video for you http://t.co/93VacQM4b1, +2004,@PamelaMueller @TIME It is an honour and privlege to pray. I have witnessed these bush fires when living in Australia, +2005,@TheRealPBarry How much CO2 does an erupting volcano and bush fires raging put into the atmosphere? None of the believers mention this., +2008,All these Eugene fires gotta make you think.... Hashtag conspiracies hashtag population control hashtag bush did it, +2011,Warfighting Robots Could Reduce Civilian Casualties So Calling for a Ban Now Is Premature http://t.co/yvYgTcAz7k, +2013,lets hope this concert ends with zero casualties amen, +2018,American Weapons and Support Are Fueling a Bloody Air War in Yemen http://t.co/ZBLU4xXo44, +2021,More women children casualties of Afghan violence http://t.co/OstkZA1qaX, +2025,@gtiso Fair enough. Change always has unfortunate casualties. I like that @NzMorningReport utilises both Welly & Aucklnd. We'll see., +2029,Memorial day of 70 years Hiroshima and Nagasaki (69 august 1945) wth total casualties 200.000+ . God forgive them!, +2030,Warfighting Robots Could Reduce Civilian Casualties So Calling for a Ban Now Is Premature - http://t.co/trNGqOPBdl, +2032,@EskSF there are always casualties when doing the right thing especially if it's going to cost your boss money., +2037,@Josh_Daniel_ @mikegiarrusso @FoxNews so do you think a bunch of untrained people firing back isn't going to cause more casualties, +2041,"Obama 'lambasted 4 not being respected & 4 being a joke___ +Would #Trump prefer more civilian casualties? +http://t.co/gxQMDMzA1u +#GOP +#maddow", +2044,@JimW_in_NM @KurtSchlichter Here's the thing: they were predicting UP TO A MILLION AMERICAN CASUALTIES. Forget the goddamn Japanese., +2048,American Weapons and Support Are Fueling a Bloody Air War in Yemen http://t.co/F7ZcqaOcY7, +2052,Let's appreciate that the group of people that everyone hates so much the police prevented mass casualties today in Nashville., +2053,@DonteStallworth @SteveSGoddard if you look at the casualties on Iwo Jima and Okinawa and the numbers dead on both sides you'll see it., +2054,@Cameron_Surname @chrisg0000 The Allies forecast an Invasion of Japan had a minimum of 1 million casualties that is a minimum., +2062,@Rx_Casualty @Vpzedd @Daintt_M call, +2065,"#LegalJobs Litigation Attorney in Scottsdale AZ: Claims Specialist II Casualty Bodily Injury +The can... http://t.co/zqrQEg3Cxl #Jobs", +2066,@5SOSFamUpdater social casualty, +2072,"___@MacCocktail: 'The first casualty of war is truth.' +? Hiram Johnson (died this day) - in Tony Blairs case it died before the war! ??", +2079,Georgian wines have become a casualty in Russia's political campaign against the Ukraine: http://t.co/wCo724AmPl, +2080,@bruminthecity oh no what coffee was it espresso? poss casualty of today's busiest thurs ever. come in for a perfect coffee - on the house., +2085,Associated Agencies Inc. is #hiring Property & Casualty Account Manager http://t.co/z6MQrUFLDl #jobs #RollingMeadows #Insurance, +2088,Documenting climate change's first major casualty http://t.co/5QMvEwDgiS via @GreenHarvard http://t.co/NmtWI59MU4, +2090,Canceling Deal for 2 Warships France Agrees to Repay Russia via @nytimes http://t.co/j7vbskHAXn, +2092,charlie from casualty at the ashes https://t.co/jDlYmGuHwW, +2093,Keep alive your sound intensity level catering in keeping with auditory effect casualty insurance: hcXvNeoAV, +2101,"___@5SOSGlobalSquad: me: will give all of my money to hear social casualty live + +#MTVHottest 5sos___ I have about 5 buck here I'd give it all??", +2104,@Vpzedd reach skype, +2105,"?please read casualty fans? +Let us know & help spread the word by retweeting! DM me or @ScriptetteSar for more info?? http://t.co/HDrBTJRoHb", +2106,business casualty, +2107,@datshemmings_ moi c'est plutot disconnected the only reason close as strangers heartbreak girl rejects social casualty ... ufhguhfidt, +2120,Fox News is the biggest media catastrophe in American history. People love living a lie??, +2124,#twist #sony Ultimate #preparedness library: http://t.co/kx1fGSqeB6 Prepare Yourself For Any Catastrophe. Over http://t.co/7GYGhOG2ds, +2127,@BuzzFeed Stannis is not evil in the books GOT is a catastrophe just saying, +2130,@ShaneRyanHere CATASTROPHE on Amazon Prime., +2132,You r a wonderful person ???? @thomasistrash #ThomasIsTrash http://t.co/7bcLAzAfIH, +2135,'Now I am quietly waiting for the catastrophe of my personality to seem beautiful again and interesting and modern.', +2137,Gaza 2014 - The Summer of Catastrophe https://t.co/gm3YI1D1dV #gaza #palestine #israel #BDS, +2140,@taylorswift13 @Caradelevingne Mother Chucker and Catastrophe = So much hotness in one room we can't cope. #Swifties, +2143,@twlldun @peterjukes It's a very good piece. And I agree the failure of post-war planning was the catastrophe., +2147,Sorry Kylie pretty sure James?? ate Bruce ?? this morning. #Catastrophe https://t.co/TR6oM9yN2C, +2151,@robdelaney desperate to watch the last episode of Catastrophe. How can I get it in the UK?, +2152,@BarackObama hello Mr. President there is a really big problem in Puerto Rico regarding the water situation no more like a catastrophe..., +2155,Alaska's #Wolves face catastrophe Alexander #ArchipelagoWolves population plummeted 60% in 1 yr. Be their voice: http://t.co/vCpKdbnSfp, +2156,Two shows I've loved this summer - Catastrophe and Unreal, +2162,@VaunWilmott - @PaysTara agrees that with the catastrophic loss of the love in the barn scene we are short 1 shirtless Michael. #Dominion, +2165,Something Catastrophic Is Coming: Should We Tune Out? http://t.co/iTkCRpqgn0, +2166,[WP] Following a catastrophic event the United States is decimated. The President prepares his first post apocaly___ http://t.co/6eCMRxUyR6, +2167,@johngreen The catastrophic history of you and me, +2168,If a å£1 rise in wages is going to have such a catastrophic impact on wage structure. What the fuck is automation going to do!?, +2170,@unitedNYblogs @InsideCityHall @nyclass ouch. He needs to explain why his NYCLASS $500K hate mailer was such catastrophic fail, +2178,The Catastrophic Effects of Hiroshima and Nagasaki Atomic Bombings Still Being Felt Today http://t.co/W1iFF7ddvy, +2180,'How should a society respond to a looming crisis of uncertain timing but of catastrophic proportions?' http://t.co/rIiesTP9zY, +2182,"Dear @CarlyFiorina Will you or anyone in your family be alive in 2040? + +#climatechange #GOPDebate + + http://t.co/j7Q2ySOMvR + +#FeelTheBern", +2184,#CANCER : everything seems catastrophic you feel you can not be worse. Hit rock bottom. You recover. You start again. After a while got it, +2185,Jim Rickards Blog: Catastrophic Outcomes May Come Faster Than Expecte... http://t.co/czEgzbnJ4x http://t.co/B53yr0ccmX, +2187,Stop Extreme Regulations That Hurt The Middle Class http://t.co/ATp5apYxRr, +2196,@_LalaDeviluke - consequences could have been catastrophic. I was also informed by the security A.I *Mute that you were primarily -, +2197,#gaming Learning from the Legacy of a Catastrophic Eruption: fifteen-mile-high plume of ash . The eruption wh... http://t.co/sv5KBP1FmO, +2199,Want to sell my boat of a car for a truck, +2200,As long as my #Baby is blissfully unaware of the catastrophic day happy fed bathed cared for...___ https://t.co/zMvrjLjoog, +2201,I suffered a catastrophic pants failure at work today. I resorted to tying my jacket around my waist til I got to a thrift store. #trustory, +2202,Society will collapse by 2040 due to catastrophic food shortages says study http://t.co/2wWZBW5lId, +2206,"Dear @SenTedCruz Will you or anyone in your family be alive in 2040? + +#climatechange #GOPDebate + + http://t.co/j7Q2ySOMvR", +2208,Something Catastrophic Is Coming: Should We Tune Out? http://t.co/ftrdbbaKqv via @po_st, +2218,THE CHEMICAL BROTHERS to play The Armory in SF tomorrow night!: EMERGENCY BAY AREA EDM ANNOUNCEMENT ___ THE CHEM... http://t.co/Uhgx7uQoBd, +2223,"Fake 'explosion toxic chemical' drills emergency responders before Pope Francis 1.5 million visit in Sept. +http://t.co/AjVFJD3qnI", +2224,Emergency crews are on the scene of a chemical spill at Spohn Shoreline. We will have more details as they come to us., +2226,Standard behavior of Chemical Mixers so that Emergency Purposes...JyB, +2228,Emergency services called to Bacup after 'strong' chemical smells http://t.co/4GaJhSPfwl, +2232,Rossendale: Emergency services called to Bacup after 'strong' chemical smells http://t.co/hUDTl2HdSV #rossendale, +2234,"Emergency services in Hammondville. Near Jewell Close. STORY: http://t.co/jQcy5NwbUg #Liverpool #7News +https://t.co/snpmAlngvd", +2243,Chemical spill at a house in Hammondville. Emergency services attending. #BreakingNews #7News http://t.co/bQ94MjYgfe, +2247,@bendwavy emergency chemical rinse, +2249,Emergency services called to Bacup after 'strong' chemical smells http://t.co/hJJ7EFTJ7O, +2252,@PMBreedlove Russian nuclear-biological-chemical (NBC) brigade 'emergency response' exercise in Southern MD http://t.co/qyoT2B59ig ___, +2253,Emergency prompted due to chemical spill at a factory near Redruthå_UK https://t.co/oiCtAcbwSR http://t.co/46lSIanjMC, +2259,"When God pushes you to the edge of the cliff two things may happen: +1) He will catch you when you fall. +2) He will teach you how to fly.", +2261,Photographer Brian Ruebs endures 4500-feet climb to capture bride and groom http://t.co/gft2Lr0Lsi @rarasathie_, +2264,I might go losing it n drive off a cliff fall in the void, +2268,I hope you fall off a cliff., +2269,@Spencer_VH I hope you fall off a cliff, +2270,@usetheIight youre an ugly piece of trash i hope u fall off a cliff, +2276,Huge cliff landslide on road in China: Watch the moment a cliff collapses as huge chunks of rock fall onto a r... http://t.co/LKDO0momtw, +2283,My boy @Fall_off_Cliff still got it man rumor has it he going back to DE ??, +2287,"Video: Man rescued after 80ft cliff fall at Shaldon +Read more: http://t.co/efQCLTEVlr http://t.co/4gQy7ziMfC", +2290,@MattBuek Some pitchers gracefully lose their stuff. Some fall off a cliff and land on their throwing arm. Hard to tell who gets what., +2291,@JohnnieGuilbert jump off a cliff. I'll be cheering for you to do a back flip for fucks sake. I hope you fall and break your neck., +2293,3 out of the 4 people I'm camping with I would happily see fall of a cliff., +2295,Mega keen for hiking tomorrow! Hope i don't fall off a cliff haha! #keen #fitness #yay #excitedmuch, +2302,@NTSouthWest Looks very tasty but the problem is the cliff might fall into the sea due to coastal erosion so it would be crumbly then, +2305,Interview on The Collapse of Materialism Best #TalkRadio Listen Live: http://t.co/ncMBnRsk5Q, +2310,Greece's tax revenues collapse as debt crisis continues http://t.co/V5yyzNGRBZ, +2313,http://t.co/dOg9bw1eCW Correction: Tent Collapse story #HNLnow, +2316,Correction: Tent Collapse Story http://t.co/CMbNMvVDKl, +2320,We all knew Cain wasn't going to die... It's either Val or someone is gonna collapse later on... #Emmerdale #SummerFate, +2322,Interview on The Collapse of Materialism Best #TalkRadio Listen Live: http://t.co/Qn8vTXZJzB, +2323,West Side water main break causes street collapse (VIDEO) - El Paso Times http://t.co/uP9TWymJlV, +2326,@fouseyTUBE @zaynmalik I would collapse, +2328,The Guardian Greece's tax revenues collapse as debt crisis continues The Guardian Fresh___ http://t.co/L0ARQL2q2O, +2331,If Oikawa was in Karasuno I guess I'd just collapse from such an overwhelming happiness., +2335,Residents set up sign as tribute to families who lost their lives in the Thane building collapse: The unfortun... http://t.co/zT6wANhXuj, +2338,Technical Collapse -> http://t.co/BfJB5H4tuW $WLB Down -3.43% to New Low http://t.co/cSXEKREDUg, +2343,.@ArneJungjohann #Energiewende is directly responsible for the collapse in energy value. http://t.co/NxvjPj610W, +2344,#ROH3 #JFB #TFB #alrasyid448ItuRasya Correction: Tent Collapse Story http://t.co/xsnJi8EJRl ROH3 SmantiBatam #ROH3SmantiBatam, +2345,Ashes 2015: Rampant England humiliate rival Australia http://t.co/kMBS7bXkQN, +2353,Greece's tax revenues collapse as debt crisis continues: As talks continue over proposed _â_86bn third bailout ... http://t.co/7w2WiEFjuq, +2355,That @PPact hasn't already collapsed is a testament to @TheDemocrats lockstep devotion to politics over reality., +2357,@SheriffClarke @FreeAmerican100 If a carpenter built your house and it collapsed would you ask the same carpenter to rebuild it?, +2360,35W Memorial honoring those who died when the bridge collapsed. http://t.co/qIb5fYaLGs, +2365,I used to have a good time from June to about July 10th then my summer just collapsed smh, +2369,i remember when i was little and i didnt want to say the pledge and the school collapsed and pit to hell opened, +2371,But the ceiling and the walls collapsed. Upon the darkness I was trapped., +2378,I collapsed in the bathroom bcuz of Michael., +2380,VIDEO: Link to the billowing fire after the SouthTowne Lanes roof collapsed: http://t.co/YrRHqZLKoR http://t.co/dXk4sUXMKV, +2381,@_Lasha_K @Diamond6612 @quin_lo @LSedig @Yemmely @Holly16252 @Inmianajones I saw this crazy edit of Twisted mixed with Tut and I collapsed, +2383,3D view of the storm that developed & collapsed over Albertville this evening; some wind damage reported #valleywx http://t.co/2wJ1MNz3mE, +2384,@michaelgbaron Panic City has collapsed. Party City is in full swing!, +2392,@JessicaLauraxo grandma collapsed and died, +2393,3-pronged inquiry launched into collapsed #HongKong furniture retailer DSC http://t.co/Tg3ukIsgEN @scmp_news http://t.co/An5LpimNfA, +2401,Official: Crowd in tent that collapsed was told to stay put: LANCASTER N.H. - Circus workers first told spectators___ http://t.co/yRQgASn0JG, +2403,@jemmaswans i needed such a breather today oh my god i went on lunch and collapsed like a sack of bones in my car, +2404,We had a room full of people lifting up Collide in prayer!! We are so excited for Friday night!! http://t.co/645dYNAMy8, +2405,?Maybe someday we'll find the place where our dreams and reality collide.?, +2407,I'm not saying you should freak out but Collide starts in 30 mins so you should freak out! Get___ https://t.co/QOK1BSYZEU, +2411,I added a video to a @YouTube playlist http://t.co/c2k7hDoLph Howie Day - Collide, +2424,but even if the stars and moon collide i never want u back into my life #MTVHottest Justin Bieber, +2426,Collide Gateway students make setting up chairs and tables 10x more fun than I would've thought! ?? @collideNLC, +2431,Perez Hilton forever Niley shipper lol https://t.co/qQ1pIJK2dA, +2433,no escape- no place to hide here where time and space collide, +2434,Nigga This Is The First Time I Heard A Song By Justine Sky Other Than Collide, +2436,Vamos Newells, +2439,#Funfact #facts Can Stars Collide? http://t.co/E1ao9A7rw9, +2444,"When transgender Sam takes runaway Lizzie in two very different worlds collide as they explore unjust boundaries and trust. +Sam(uel) +#books", +2447,When Social Media Marketing and Ethics Rules Collide | The Recorder http://t.co/D0Bnm1ZZak, +2448,Trump & Bill Clinton collide in best conspiracy story ever http://t.co/ABkhBhNLOz via @motherjones TRUMP DEMOCRATIC PLANT? lmao #lastword, +2449,PIERCE THE VEIL Rubber Bracelet Wristband Collide with the Sky - Full read by eBay http://t.co/H4WUPpaT6k http://t.co/WisjhDH58n, +2450,Now there's only 2 days for collide!!! get signed up!???? http://t.co/vhh7LBOATZ, +2461,@FarhanKVirk @PTISalarKhan ! Instead of promoting false news first get your facts right . It was a road rage incident . Vehicles collided, +2469,"I so want to #win an ARC of WHEN WE COLLIDED Emery Lord's 2016 release! Open intl #TheStartofEmandYou #giveaway + http://t.co/qcu4xO54wT", +2472,@mollywood I agree! I didn't know you had moved to Marketplace. I woke up to your report & thought my favorite worlds (BOL-APM) collided., +2473,@imaginator1dx currently reading after. as you can see after we collided is on my dresser waiting to get read http://t.co/QwrASZ6LHO, +2474,@edsheeran tf is innit, +2477,So running down the stairs was a bad idea full on collided... With the floor ??, +2481,#maritime The sand carrier Shinto Maru and chemical tanker Hoshin Maru collided on 7 nautical miles off Honshu... http://t.co/2bnfoAGpzD, +2484,Oh my God. Sending my thoughts to @SAStars head coach Dan Hughes who left the game on stretcher after Danielle Robinson collided into him., +2495,@AlyssaSpencer28 remember when beau and i collided on the slip and slide and I died ??????, +2503,Ok two of my favorite things have collided into one great HT #LiteraryCakes https://t.co/E2meY0aKPu, +2509,Call to car vs fed ex truck head on collision on I-40 mile marker 118. Prayers for the families as we get to the scene!, +2511,Motorcyclist bicyclist injured in Denver collision on Broadway http://t.co/eOPWBtUYr9, +2518,Motorcyclist bicyclist injured in Denver collision on Broadway: At least two people were taken to a local___ http://t.co/PMv8ZDFnmr, +2522,Traffic Collision - Ambulance Enroute: Florin Rd at Franklin Blvd South Sac http://t.co/dYEl9nMQ0A, +2525,The Denver Post - Motorcyclist bicyclist injured in Denver collision on Broadway http://t.co/yjrIi5mHii, +2526,that collision daaamn, +2529,Marin Sr37 / Sr121 **Trfc Collision-Unkn Inj** http://t.co/yqJVEVhSzx, +2533,Riverside I15 N Sr91 E Con / I15 N Sr91 W Con **Trfc Collision-Unkn Inj** http://t.co/QqMLKvgPQk, +2549,Techerit: Anti Collision Rear- #gadget #technology http://t.co/v3a5ZQaRFg, +2551,Anti Collision Rear- #innovation #gadgets http://t.co/YXD4c4XlGo, +2558,Mom fights to recover in hospital from fiery car crash while kids miss her at home - http://t.co/0UH26R2zfX, +2562,No citation for Billings police officer who caused Broadwater crash http://t.co/aAhYoEITzl http://t.co/ULZ3ubQV5z, +2563,'Our little angel': Banjo Pilon 10 dies after skateboarding crash at Wamberal: At the weekend 10-year-old B... http://t.co/5bcOQB6Cff, +2567,lol at the guy whipping by me on a double yellow line in his mustang just to crash on the curb into a light pole #sns, +2574,New post: 'India Train Crash Kills Dozens' http://t.co/6OQaZmQa3L, +2577,Crash, +2578,@crash_matrix @TheMercedesXXX With a minivan :3, +2580,Non-Power 5 Schools with Best Shot to Crash College Football Playoff http://t.co/Wj5EMRNkYi, +2581,This is hilarious! Look at all of you! We got: Hairity Rainbow Crash Spitty Pie #Spikebot, +2583,Thieves crash dealership gate steal four vehicles in Killeen http://t.co/aA47AXC6sr http://t.co/fz5axahPVr, +2584,Time to enjoy summer in a crash course of two weeks... I will never take summer classes again. (That's a lie), +2586,#portmoody Clarke Rd hill n/b blocked by crash past Seaview traffic back to Kemsley. Take Como Lake to Gatensbury to get around it., +2589,@natethewolf Car vs Motorcycle Crash Fairport Nine https://t.co/SkUQMJq7g2 #BGC14, +2595,Learn cPanel in 2015. Join cPanel Crash Course only $20 http://t.co/TltY2Mumco, +2596,.@Emily_Noel95 @SterlingKnight @joeylawrence @ABCFmelissajoey that car crash was pretty fun to film #melissaandjoeychat, +2600,NEWS: 2 injured in school bus crash in Bordentown - http://t.co/672bY6OOjn | Details: http://t.co/N5IHOfDpD4, +2601,The Crash And A New Depression ___ The Seeds For The Next Crisis Have Already Been Sown http://t.co/IsN9HdZ4cp, +2607,Husband's back from Edinburgh and crashed out. One young adult's has got a raging temperature the other's gone out and I'VE NOT HAD ANY TEA, +2610,Rip to whoevers face that crashed in front of my work. Wish I could have helped, +2613,almost crashed while listening to walk the moon bUT I DONT CARE I LOVE IT, +2615,Ol'Head just crashed into the back of another ol'head car he hopped out like 'DAMNNNN U FUCKIN ROOKIE... SLOW TF DOWN!!' ??????, +2618,@thomicks Isn't showing up for me. I think it crashed., +2620,@RosanneBarr New documents suggest Clinton's email server may have crashed. All of her fan mail from Qatar must have crashed it., +2623,Bin Laden family plane crashed after 'avoiding microlight and landing too far down runway': Three members of t... http://t.co/ZlBOeMXo2T, +2626,@BlizzardCS crashed them right after the stream lol, +2630,@PahandaBear @Nethaera Yup EU crashed too :P, +2634,#news Bin Laden family plane crashed after 'avoiding microlight and landing too far down runway': Three member... http://t.co/hHsN0VnYKs, +2636,"aQgcO mhtw4fnet + +Pakistan says army helicopter has crashed in country's restive northwest - Fox News", +2638,Got a ticket & my car got crashed man life's great !, +2639,"Pak army helicopter crashed in Mansehra. +Pray for departed souls. http://t.co/XclNBVvfSi", +2646,Website Malfunctioning? PHP Scripts not working? Database Crashed? Need a Unique Script? PHP PRO to the rescue! http://t.co/LadH9Oo086, +2650,08 Army personnel including 5 Majors were martyred when a helicopter of Army Medical Corps crashed near Mansehra. http://t.co/wyMrtwsZHU, +2652,The last few days have proven how popular the #BBC is. Over 9m watched Bake Off and iplayer crashed due to cricket commentary., +2653,pkadlik : jojowizphilipp and I have a crush on Rob Thomas right now...?? #viptickets #imgunnashakeitallnight @___ ___ http://t.co/uaGsSzMu9w), +2654,my brother said he has a crush on a girl named Danielle ???????? OH GOD NO, +2662,women crush for sure ???? http://t.co/7fQjvTd4GN, +2665,I think I might have a crush on my lover boy ?, +2669,Aramis will crush 1 pitch strop, +2674,So if you secretly have a crush on me and can sing lmk lmfao, +2678,Crush cars with wrestling stars in MMX Racing Featuring WWE - AppAdvice___ http://t.co/5XvP1IfPmy #designgeeks, +2681,I was freakingg out and he was playing candy crush ??, +2685,I'm just not a relationship person. That's why when I have a crush on someone I always think they're not going to feel the same way, +2686,@sandy0013 dude i have a crush on hockey players but Paul will always give me butterflies...he's just an angel!!, +2690,I have the biggest girl crush on oomf ????, +2697,@Team_Barrowman Just saying your my man crush!, +2699,omg I have a huge crush on her because of her looks. ... ___ Not necessarily I dont think so! She is looks way be... http://t.co/xqmaTI4GI6, +2704,I crushed a 5.1 mi run with a pace of 13'0' with Nike+ SportWatch GPS. #nikeplus: http://t.co/lJiiuuaTRC, +2705,So my Neptunia girls are all nearly level 150 and I've crushed all the non-S-rank Colosseum fights. Major grinding happens very soon., +2710,@AmyRamponi But they__ªre not crushed? It doesn__ªt set up some negative self-fulfilling prophecy or negative self-talk? #psychat, +2712,An old man shuffled slowly into an ice cream parlor. He ordered a banana split. The waitress asked Crushed nuts? No he said. Arthritis., +2713,@DancingOnHwy there is nothing I love more than seeing Bears fans get their hopes and dreams crushed., +2716,So in one episode they undo season 1. Kai joins FF Ren beats Aichi with Psy and misaki gets crushed. What the fuck?, +2717,RT #Target Best price on #Avril Crushed Sheer #CurtainPanel #Curtains #Decor #Blinds #Shades : http://t.co/TBhbuX6dc0, +2718,@MWCahill5 also mcutchen crushed it, +2721,I'm so over getting my hopes up and then being crushed., +2722,Uribe just crushed that, +2735,Double E! Just crushed that ball! #Jays, +2737,#GoT season 5 - funniest season ever. Hilarious. #GameOfThrones you crushed it. OMG. #CantStopLaughing #Wow #Comedy, +2738,?? my soul just crushed because they were so damn cute ?? https://t.co/zADH87Pwml, +2742,@Loraccee @JoeNBC @msnbc Still gets crushed by @FoxNews which says little about cable news viewers in general. Dumb sells but sucks., +2745,Uribe crushed it! #Mets, +2746,@Brandon_Warne MLB has that ball at 462 feet. Absolutely crushed., +2747,That ball was crushed, +2750,http://t.co/ACHBbiFQrQ : Big Papi with a great welcoming to The Show: #Crushed___ http://t.co/ceBFBONcHD http://t.co/mUncapudDc, +2751,@Spicenthingsup wtf!? I can't. The love bit. I really think that's how she feels. Massive crush that got crushed., +2754,@GhazalaBaji : Exactly! Curfew like situation when I sit with dad, +2762,Black African refugees in 'Israel' cannot attend Kenye West concert due to curfew for black refugees.. http://t.co/S0usjQYIJe, +2764,Has the kids' bedtime gotten later & later? Now's a good time to point out to them when their regular curfew would be. #BTSPrep #Parenting, +2772,@karli_knox you know how crazy strict Philip is. he gave me a 10 texting curfew bruh, +2775,@CurfewBeagle @beaglefreedom Pretty Curfew!!!??, +2776,WHEN YOUR PARENTS TRY TO PUT A CURFEW ON YOU AND YOU'RE ALMOST 19 ????????????, +2779,everybody like fuck curfew.. and i'm sitting here like i'll be in bed by 12., +2781,@benznibeadel_ hehe like u HAHAHA I'm kidding love u I'm gonna sleep na bc curfew ??, +2789,@AnnaEsterly dude you were only out 4 mins past curfew give your self a break., +2790,Big 30 hanging out no curfew?? https://t.co/JHDqHfaTLJ, +2791,@DomeTufnellPark Do you know what the curfew for Havoc Fest is on Sunday please?!, +2798,! Sex-themed e-books given curfew in Germany on http://t.co/7NLEnCph8X, +2800,Big 30?? Hanging Out NO CURFEW, +2804,Happy to report all is quite in the TROPICS. Cyclone Development not expected at this time @CBSMiami @MiamiHerald http://t.co/vI5oVsaHqL, +2805,Here comes the next Episode #Scorpion #Cyclone @ScorpionCBS, +2806,The revival of Cyclone Football begins today!! Be there September 5th!!, +2809,An Appeal By Cardinal Charles Bo SDB ( CNUA) Dear Brothers and Sisters The recent cyclone and the massive... http://t.co/bLgH5bTV8T, +2810,Cyclone hits Skye http://t.co/QZYeRMrQtW http://t.co/N5AI9pQNjg, +2812,@orochinagicom GIGATECH CYCLONE, +2814,"SEVERE WEATHER BULLETIN No. 6 +FOR: TYPHOON ___#HannaPH___ (SOUDELOR) +TROPICAL CYCLONE: WARNING + +ISSUED AT 11:00 PM... http://t.co/fKoJd0YqK0", +2816,First time for everything! @ Coney Island Cyclone https://t.co/SdNT3Dhs3W, +2818,@Aliyeskii @wquddin @tunkhin80 @MaungKyawNu @drkhubybe @nslwin @ibrahimdubashi @mdskar @zarnikyaw @kyawthu043 After cyclone, +2823,@ERPESTAR i aint a bitch girl popobawa revolves around you the cyclone, +2824,#weather #tropicalweather: Tropical Cyclone Tracking update 1245Z 6 Aug 15. http://t.co/D79laflys9 #tropicalupdate, +2834,Is that a hurricane or a typhoon? No matter which one it's also a cyclone. http://t.co/c5kJAtjgzn, +2837,Waige driving like a Badass #ScorpionPilot #Cyclone @ScorpionCBS, +2840,raleigh cyclone 15 gear mountain bike 26'': http://t.co/4LpPmad80G #sportinggoods http://t.co/lleFHhggrL, +2845,"RT SEVERE WEATHER BULLETIN No. 6 +FOR: TYPHOON ______ (SOUDELOR) +TROPICAL CYCLONE: WARNING + +ISSUED AT 11:00 PM... https://t.co/BYy4dHrqyH", +2847,Red Tropical Cyclone alert for SOUDELOR-15 in China Northern Mariana Islands 06 Aug 2015 06:00 UTC http://t.co/DMnr5Nkfo7, +2848,"#LocalEventCountdown: +8. Curtain opens on The Black Cyclone in Shelby tickets & info: http://t.co/v7dCz5DJDj #rstoday", +2850,May 1965 Rodder and Super Stock Keeling 427 Mustang Barracuda Riviera Cyclone http://t.co/ha3I0Etk5w http://t.co/Id2vGZ6qew, +2859,Damage done: How to Recover From a Negative Social Media Update http://t.co/W2wtjyS599 @SMExaminer, +2862,@beforeitsnews __¢This Is Called DAMAGE CONTROL. Don't Be Fooled Folks. The Perps Will Just Come Up With Plan B..., +2868,#LM__ªs @frauncestavern named one of the historic properties granted $$ by @NYGovCuomo to repair #Sandy damage http://t.co/OcAdC8vFyF, +2874,@globepolitics extreme positions lack of debate pandering to the hard-right - further increase divides corrodes trust and damage goodwill, +2876,Limited the damage. Put some runs on the board. #NYY, +2892,@f_body95 never riding in the gaymaro ... Would damage my reputation, +2894,Alpine city park vandalized $2500 in damage http://t.co/fwk8puB2wF, +2897,Got my first damage today fuckkkkkk, +2901,@VZWSupport Zero damage just a horrible product, +2903,"@GatorGMuzik #FETTILOOTCH IS #SLANGLUCCI OPPRESSIONS GREATEST DANGER COMING SOON THE ALBUM +https://t.co/moLL5vd8yD", +2904,I used to really think Danger was gonna be a movie, +2906,I don't think it was. It was very realistic. Scully finally had him home and sfe and he wanted to go out into the danger. @HipsBeforeHands, +2914,#onceaho Empire Girl: Vote Abella Danger: Adult Empire is running their first ever hunt for the Emp... http://t.co/xC5RJ36yv2 #alwaysaho, +2918,"?????? FREE!!! ___This book will blow your mind______ ?????? + +'Suspense danger anger love and (most importantly)... http://t.co/nzN5nTPn75", +2919,In my post today I talk about the dangers of interruptions. Do they annoy you?http://t.co/zh6AusFbwh, +2923,@TheElmagoo @GOPTeens @FoxNews @pattonoswalt anyone who actually plays that drinking game is in serious danger of alcohol poisoning, +2926,@pukesmoothie I don't understand WHY he would put us in that danger., +2928,@mlp_Delrim Please tell my why they might be in danger if they knew what you knew?, +2930,The Devil Wears Prada is still one of my favourite films ??, +2938,Allen West: Obama is a clear and present danger to our republic! http://t.co/GH2VKb1zu5 http://t.co/0SmhjXWcbo, +2940,@Fresenius Said I put patients lives in danger and at the same time this is whats happening. Same like this is a bigger violation than PPE, +2949,@James_May_Not @aMusicVideoaDay @knelligan12 @king_ruckus There's a lot of businesses that operate on outskirts & I never feel in danger, +2951,@unitedutilities not happy I received text now about #Parasite danger.I am a #transplantee & #Virus can kill #kidney https://t.co/9AasSFrdP1, +2958,my last retweet i hope i drop dead, +2961,Once Thought Dead Google Glass Gets a Second Coming: Though considered an epic failure according to its makers and a recent patent..., +2962,dead https://t.co/ESsxzdTcQJ, +2964,'Possum problems and dead ducks. http://t.co/x6mZDBntRP, +2966,I hope it's Ross & not Val that's dead @emmerdale #SummerFate, +2967,???????? this the dead ass truth you pay for shit you not gone use https://t.co/iW1sDopMvR, +2968,@emmerdale #SummerFate that was so intense!! Can't believe Val's dead!!????, +2972,Nah @emmerdale @MikeParrActor ross ain't dead we all know he's just unconscious and your just being dramatic teases, +2977,Ross better not be dead! #Emmerdale, +2978,Nooooo the village hotty is dead ???????? #Emmerdale, +2979,@MikeParrActor No no no no no no I'm not believing it! Ross isn't dead! http://t.co/LvOlNtSWDZ, +2981,@MikeParrActor if Ross is dead I shall never watch emmerdale again I'll be that heartbroken ??, +2985,I don't reckon Ross is dead actually they said 'has' Pete killed his own brother Ross has survived so much to just die from that??, +2986,"Please tell me Ross isn't actually dead... +I hope they're just playing us until the next episode. #emmerdale", +2989,411Naija å_ Flood: Two people dead 60 houses destroyed in Kaduna http://t.co/yAvL49DoOF, +2994,If you're my boyfriend's ex or had any feelings for him. DONT TALK TO HIM! Don't follow him. He is dead to you. He don't exist niggah. ??????????, +2996,For everyone that doesn't know Chris Prater was struck by a train this morning and pronounced dead on scene. Prayers to his family., +2997,@_ToneDidIt this can't be real nah ?? her mother is dead wrong ????, +2999,no way can ross be dead..?? #Emmerdale, +3002,WHAT AN INCREDIBLE CHARACTER MY HEART IS BROKEN THAT HE IS ACTUALLY DEAD!! #RIPROSS WE WILL MISS YOU! https://t.co/LgqmdnaVDf, +3007,@baeffee @RandomPelar the black death (black death whoo!), +3008,I liked a @YouTube video http://t.co/EVEdsU5PB0 Camp Out! | BUNK'D | Disney Channel, +3017,#Obama: Don't worry the 'good' Iranians are just kidding with the 'death to America' chants. The 'bad' ones are those just like Republicans., +3020,Microsoft Xbox 360 console RRoD red ring of death AS IS FOR PARTS OR REPAIR - Full read by___ http://t.co/IpQCCT5hGC http://t.co/oofPEfRh3r, +3024,This headache will be the death of me ??, +3028,@ITristonTyler CONFESSION when I was little I had a hamster and my room was so cold it froze to death!?? OOPS......, +3031,This wall is going to be the death of me...due to the years of water leaks and lack of maintenance___ https://t.co/PVd2fYBa0q, +3032,The Iran Deal Will Result In Terrorism And Death | Truth Revolt http://t.co/xrG0FacL6T, +3033,@swearyG It looks like the death star, +3035,death is the only certainty in life, +3040,A single death is a tradery many deaths is statistics., +3041,#Spirituality - Born Again-Spiritual Awakening And Near Death Experiences http://t.co/Z0UGKqi6kM @Charlesfrize @Frizemedia @Dynamicfrize, +3046,@Kaay_Paat it's actually death and takes way longer than you'd think, +3050,of all days when I look like death I have to see all of mo city in chick fil a today ??, +3063,Deaths 2 http://t.co/4A1fSwePpg, +3065,@betrayedhunter reading he paused as he came across a couple of deaths and began to read deeper into it., +3067,Deaths 5 http://t.co/0RtxTT11jj, +3069,Bigamist and his 'first' wife are charged in the deaths of his 'second' pregnant wife her child 8 her mother___ http://t.co/t5HwUnjK7g, +3076,HEALTH FACT: Women account for 39% of smoking deaths. http://t.co/fSx2H9XAkI, +3078,@1outside in the series the deaths seemed one right after another but in the book it seems a month or more will pass after the death of Liz, +3081,@tim55081 @BootlegAlbano @ShaunKing I know. He profits off the deaths of the people he claims to be fighting for. He makes me sick., +3087,@mohammedzismail because there was more deaths by regular gunsmissilesbombs and what not warfare than the A-bombs used in A war., +3088,Infections driving up neonatal deaths ___ Health Minister #Guyana http://t.co/ImugNXrEBN, +3094,Bigamist and his __÷first__ª wife are charged in the deaths of his __÷second__ª pregnant wife her child 8 her mother her nephew 1 and their u___, +3096,MSHA stepping up enforcement after mining deaths: http://t.co/4HM34lrUER via @elkodaily, +3098,Bigamist and his 'first' wife are charged in the deaths of his 'second' pregnant wife her child 8 her mothe... http://t.co/RH15gGSgQG, +3103,Is a wing part enough to solve the MH370 mystery?: http://t.co/ys2bveKlxK, +3110,Debris Found in Indian Ocean could be Malaysia Flight 370! http://t.co/VUoJPrKUAX, +3113,BBC News - MH370: Reunion debris is from missing Malaysia flight http://t.co/bze47fzKUd, +3121,#news Island Wreckage Is From Missing Malaysian Jet: Officials: Debris found on Reunion Island is from flight ... http://t.co/kRvUmMtVY2, +3127,Hopefully this will bring some closure for the families #MH370: Malaysian PM confirms debris is from missing flight http://t.co/2963y36LdF, +3128,R_©union Debris Is Almost Surely From Flight 370 Officials Say http://t.co/hJ1BX8Bgu1, +3129,#AUS Aircraft debris found on island is from MH370 Malaysia confirms http://t.co/1zFGk9ET5v #abc, +3135,#?? #?? #??? #??? MH370: Aircraft debris found on La Reunion is from missing Malaysia Airlines ... http://t.co/ykLruIMIsD, +3143,With authorities ???increasingly confident??? the airplane debris recovered Wednesday comes from MH370 official analysis is set to begin, +3146,Investigators: Plane Debris Is 'Conclusively' From MH370 http://t.co/uKOZWID5k3 #gadgets #tech, +3148,Top story: MH370: Reunion debris is from missing plane - BBC News http://t.co/jasWg2MwIi see more http://t.co/qgk64YPN13, +3149,Per Malaysia's PM 'this piece of debris ... is conclusively a part from the wing of #MH370.' More right now http://t.co/Bd3ik4oz2r, +3151,Interesting MH370: Aircraft debris found on La Reunion is from missing Malaysia Airlines ... - ABC O... http://t.co/q0OjWnY4Kc Please RT, +3156,RMT is playing Jackson Browne - Before the Deluge [Listeners: 2/100] [Requests are: On], +3160,___@Liquidslap Name Ja Rule's best song.___ <-- My guess is a song I've never heard because nothing I have heard could possibly be a candidate., +3164,@We3forDemocracy it already happens. ABs and rowers tweeting support for Nats last year and the deluge of celeb endorsements for Obama in 08, +3169,@CIOstrategyAU - As applicable to business as to warfare: 'battles of the future will be decided by #data' https://t.co/CIQsKWgdjj, +3178,Euro Ensembles much more optimistic for the weekend even though their operational counterpart is a deluge!, +3187,I'm havin previous life flashbacks of when i lived in Weimar Berlin. the hustlin life on Unter der Linden before the deluge., +3194,#BigData Deluge is out! http://t.co/khatZh7agZ, +3202,A decade-long billion-dollar deluge of 'messages' from the government of Canada. #HarpersLegacy, +3203,Do you feel deluged by unhappiness? Take the quiz: http://t.co/blBVVpbw2z http://t.co/05qooc9CbR, +3204,Why are you deluged with low self-image? Take the quiz: http://t.co/JCaGECQFH2 http://t.co/nH313ADkz4, +3206,thousands displaced as houses & roads deluged by floodwater after week of persistent rains. #floods #climate #cdnpoli http://t.co/4WeDZRSWbn, +3207,Businesses are deluged with invoices. Make yours stand out with colour or shape anq it's likely to rise to the top of the pay' pile., +3208,Businesses are deluged with invoices. Make yours stand out with colour or shape and it's likkly to rise to the top os the pay' pile., +3209,Police are still searching for two people after rain and mud deluged the western town of Tetovo___ http://t.co/XPhZ3GoUhR #Deutsche #Sprache, +3213,Businesses cre deluged with invoices. Make yours stand out with colour or shape and it's likely to rise to the top of the pay' cile., +3214,Watching Deluged by Data #doczone while on my IPad..... #CBC, +3215,Businesses are deluged with invoices. Make your. stand out with colour or shape and it's likely to rise to the top of che pay' pile., +3220,Businesses are deluged with invoices. Make yours.stand out with colour or shape and it's likely to rise to the top of the pay' pipe., +3223,Do you feel deluged by unhappiness? Take the quiz: http://t.co/XUyfZkZ4k0 http://t.co/YupKvDIh8m, +3224,Businesses a e deluged with invoices. Make yours standwout with colour or shape and it's likely to rise to the top of the pay' pile., +3226,Businesses are deluged with invoicesx Make yours stand ou. with colour or shape and it's likely to rise to the top of the pay' pile., +3228,My brain is deluged; with only your thoughts, +3230,Businesses are deluged with invoices. Make yours stand out with .olour or share and it's likely to rise to the top of the pay' pile., +3232,Businesses are deluged with invoicew. Make yours stand out with colouj or shape and it's likely to rise to the top of the pay' pile., +3233,Businesses are deluged with invoices. Make yours stand out with colour or shape and it's ikely to rise to the to. of the pay' pile., +3234,I can't believe it never occurred to me that I could *not* be deluged with Kickstarter emails., +3238,Businesses are deluged with ivoices. Make yours stand out with colour or shape and it's likely to ris; to the top of the pay' pile., +3246,Is Facebook being deluged by friending spam the last few days or is someone targeting me for some reason?, +3247,Glimpses: <b>Hyderabad</b> deluged by heavy rainfall: HYDERABAD: With flood waters heading___ http://t.co/TWFWTxZ2dS, +3250,"deluged dismay so +soon surrendered summer drought +descending disturbed + +#sunrise#haiku #poetry #wimberley #atx#smtx http://t.co/z9FDebg5Fm", +3251,The horror community is deluged with cruddy ostensibly macabre photography. Don't contribute to the pandemic., +3254,Barton-Le-Clay house owner told to demolish extension - BBC News http://t.co/inxvExrsyV, +3257,"RT AbbsWinston: #Zionist #Terrorist Demolish Tire Repair Shop Structure in #Bethlehem +http://t.co/Om3ntQIh2m http://t.co/6SAS3fuahA", +3258,Salvation Army bid to demolish cottages http://t.co/3kuXonOchl #Southend http://t.co/enQaSCGFyw, +3261,Enugu Government to demolish illegal structures at International Conference Centre http://t.co/9Hq5miwXFU, +3267,Contractor hired to demolish Goodrich plant in Miami files for bankruptcy - Tulsa World: Manufacturing http://t.co/uk7YwsvMgA, +3268,Enugu Government to demolish illegal structures at International Conference Centre: Enugu State government app... http://t.co/bzINn2wXHu, +3269,'I will demolish your head' 'you are all corrupt' 'i will vandalise your head' - Aliyu Mani #SecSchoolInNigeria, +3271,"Listen to @AnaKasparian demolish the case against Planned Parenthood!! +*whiny voice* 'I just really love aboooooortiooooonnnnns.' #DefundPP", +3272,Hope someone buys it! Former post office in Napa now for sale http://t.co/2ZeV1Zyttg #preservation, +3273,We're about to demolish the other team right after we eat this pizza. We don't need no fitness- Ashton Irwin #MTVHottest 5SOS #5sosquotes, +3279,Trying to get higher in the bathroom at work with my pen b4 I go and demolish my food ??????, +3290,5000 year old ring fort to be demolished http://t.co/kNLZfC8QSl, +3291,Enugu Government to demolish illegal structures at International Conference Centre http://t.co/bq67uDctjc RT, +3293,Economic Wisdom ___ Great Economists Demolish Establishment Nonsense http://t.co/H8NKtqU5O9 The Case for #Logic and #Reason, +3294,No Shane I'm sorry you are wrong. You were propagating that an all out attack would demolish England #reversal, +3298,I promise you @WinningWhit and I could assemble a team to demolish this. https://t.co/RN2MdghfO1, +3305,A demolished Palestinian village comes back to life http://t.co/Jsf5OFB3m6 via @wagingnv http://t.co/Hskd5MUosM, +3306,Historical Bigfoot Landmark Soon To Be Demolished: The Bluff Creek Resort Store pictured above in 1976 was t... http://t.co/TW0SEv1N6C, +3307,"Big Papi just demolished that ball. + +#BOSvsNYY", +3310,It was finally demolished in the spring of 2013 and the property has sat vacant since. The just___: saddlebrooke... http://t.co/7FOnHQtlyv, +3313,#BananaLivesMatter ... but not more than #HotepLivesMatter thats why ya girl Hotepina just demolished two bananas in 13 seconds flat, +3314,It was finally demolished in the spring of 2013 and the property has sat vacant since. The just___: saddlebrooke... http://t.co/MDKV7c3el4, +3315,She literally demolished my nap time., +3316,I got my wisdom teeth out yesterday and I just demolished a whole bowl of chicken Alfredo like its nothing..., +3321,@Ramdog1980 Israeli soldiers demolished a residential shed and a barn. Soldiers also demolished a shed in Beit Kahel northwest of Hebron., +3325,Starting a GoFundMe page for a new set of rims since i demolished mine last night #helpabrotherout, +3326,I really demolished it, +3327,They demolished one of my favorite restaurants to put in a Chik Fil A. I hope no one goes there. I hope it tanks., +3330,Six Palestinians Kidnapped in West Bank Hebron Home Demolished - International Middle East Media Center http://t.co/J9UzIIQ3PX, +3331,Uttarakhand: Journalist's house demolished who exposed corruption - Oneindia http://t.co/Ro1aWWuDYE, +3332,"@ILoveNuiHarime More than that. +Demolished.", +3333,Before & After #underconstruction #demolished #Melbourne #Residential #commercial #builder #luxuryhomes #townhouses ?? http://t.co/6iD7CZvMEd, +3343,@OpTic_DKarma dude they demolished you!, +3344,@danisnotonfire can you fix the 528472 chargers i've demolished, +3348,That ball was fucking demolished, +3349,It was finally demolished in the spring of 2013 and the property has sat vacant since. The just___: saddlebrooke... http://t.co/bd5B5yffyb, +3350,@who_mikejoness I hate seeing the boy get fucking demolished like this, +3352,i try n eat healthy but seriously it's like my arms move by themselves n before i know it I've demolished an entire fridge, +3353,Real-life 'Up' house set to be saved from demolition by charity http://t.co/rdD5Fe1qyJ, +3354,http://t.co/AAffnDCDNq Draw Day Demolition: Daily football selection service that consis... http://t.co/wMu2VUNCKe http://t.co/7K9Rczd2O6, +3358,De-watering demolition defends against flooding http://t.co/dTO0ULjxn3, +3360,"#Muslim cemetery demolition infuriates cultural groups in #Israel + +http://t.co/vpLZKUidz2", +3366,WOW!!! What a show at the Carthage Auto Parts Celebrity Demolition Derby! Thanks to all the celebrities for putting on such a great event!!, +3371,Canberra's Currong Apartments marked for demolition in September http://t.co/ZyRR8DzJx6, +3374,General News __¢å_'Demolition of houses on waterways begins at Achimota Mile 7 ' via @233liveOnline. Full story at http://t.co/iO7kUUg1uq, +3375,Buzzing: Billy Joel Brings Down the House for Final Show at New York__ªs Nassau Coliseum___ http://t.co/Xj3ASw5Okj http://t.co/Epca4OXAq2, +3378,Demolition Means Progress: Flint Michigan and the Fate of the American Metropolis Highsmith https://t.co/fj7p8A1hvM, +3382,Seattle's demolition-dodging 'Up' house to live on as affordable housing http://t.co/krGNNXj2sA, +3384,@Johnny_Detroit Tag Team for me was Demolition. Awesome intro song scary looking and just destroyed there opponents., +3385,Problem motel nears demolition http://t.co/fKx4IH7kM1 http://t.co/n8VcxGyhQ8, +3386,The only demolition 2 be done by Christians is the putting away or throwing out of the flesh & sin - personally. #IntheSpiritWeBuild 2gether, +3392,Photo: mothernaturenetwork: Seattle__ªs demolition-dodging __÷Up__ª house to live on as affordable housing But... http://t.co/szG0HsNJEw, +3399,Demolition underway on @OU_Football stadium project. Stay updated throughout at http://t.co/ju09z2y7g0 http://t.co/UP0hOCgGrj, +3407,@Hajirah_ GM! I pray any attack of the enemy 2 derail ur destiny is blocked by the Lord & that He floods ur life w/heavenly Blessings, +3409,@ALESHABELL GM! I pray any attack of the enemy 2 derail ur destiny is blocked by the Lord & that He floods ur life w/heavenly Blessings, +3416,Two trains derail in central India's Madhya Pradesh http://t.co/ijDePwuYNY, +3420,Everyone starts a #UX project with the best intentions but common pitfalls happen along the way. Don't let them! http://t.co/48VG4RkSqT, +3422,BBC News - India rail crash: Trains derail in Madhya Pradesh flash flood http://t.co/9UFemKfIiv, +3424,@QueenBrittani_ GM! I pray any attack of the enemy 2 derail ur destiny is blocked by the Lord & that He floods ur life w/heavenly Blessings, +3426,Related News: Here&#8217;s what caused a Metro train to derail in downtown D.C. - Local - The Washington Post | http://t.co/4AoNQ9vnhs, +3427,@PumpkinMari_Bot lemme just derail this real quivk to say. HELL DAMN F, +3434,Try @hollyclegg's low-carb summer treats that won't derail a #diabetes meal plan! http://t.co/2c66Dctrg0 #TheDX http://t.co/A6g0aRSztp, +3438,@AshleyDniece GM! I pray any attack of the enemy 2 derail ur destiny is blocked by the Lord & that He floods ur life w/heavenly Blessings, +3441,Hillary__ªs Bimbo Eruptions and Questionable Financial Dealings Should Derail Campaign http://t.co/kAL1s5mg2z via @politicsisdirty, +3442,BCC: Premature interest rates increases could derail the recovery: Commentating on today__ªs interest rate decis... http://t.co/aftpebIkbO, +3443,@vivigraubard of course not. for efforts to address short-sideness beyond not including a graf' would derail obvious backtracking efforts., +3444,Do people even ride Metro any more? http://t.co/Nb8yzecI6i #whywebike #bikecommute #DysfunctionalRedLine #WMATA, +3449,@PumpkinMari_Bot lemme just derail this real quivk to say. HELL DAMN F, +3453,@CoolBreezeT train derailed at Smithsonian...no passenger train...then my red line train became disabled between FH & Bethesda, +3454,.@janeannmorrison: A former Assembly candidate is still waiting for answers http://t.co/Mm5z32vKPC, +3456,Interesting to note: Metro train derailed in Washington this morn after going thru interlocking area. One of lines affected by closure 1/3, +3457,@FixWMATA @AdamTuss Do you know about path of derailed train? Doesn't make sense. If it was coming from yard why turning at Smithsonian?, +3458,Train derailed at Smithsonian Metro. Sidewalks outside L'Enfant mobbed to get on buses @wmata #nightmarecommute, +3461,@lizkhalifa no passenger train derailed earlier this morning, +3467,".@CBCNorth see photo Old Lady of the Falls (sacred site) RR says derailed Talston power line to diamond mines in 2010 +http://t.co/SxlfSNsPH0", +3479,'We may be derailed right now but we are going to keep holding on to remain on track' -@kristnmalea #DerailingDistractions, +3488,Also there is no estimate of damage yet from #WMATA/#Metro on the six-car train that derailed ~5 a.m. Shuttle service available. @CQnow, +3491,Oh a #wmata train derailed. Kind of disappointed how unsurprised I am., +3496,3 cars of #Metro train ready to go into service derailed near #Smithsonian station disrupting service. No injuries. More at noon @wusa9, +3502,'@WSJ: How a rare lizard derailed Adani Group__ªs plan to build a huge coal mine in Australia http://t.co/dkNPK6FbE4 http://t.co/BiVShY4q1b', +3507,Google News - A Twin Train Derailment in India Leaves at Least 24 Dead - TIME http://t.co/yf56oK7Pgp #world, +3511,@CTAFails PHOTOS from Green Line derailment http://t.co/Y8xFVJAlLr http://t.co/qUUUN88czw, +3515,Madhya Pradesh Train Derailment: Village Youth Saved Many Lives: A group of villagers saved over 70 passengers' lives after two train..., +3516,#CTA GREEN LINE: All passengers have been evacuated due to a derailment however no service Garfield-Cottage Grove. http://t.co/qePJ0Hwpay, +3538,Great photo by the Tribune's Terrence Antonio James after Green Line derailment (no one hurt fortunately) http://t.co/YFavM641OS, +3539,Latest : Suresh Prabhu calls Harda derailment a natural calamity; officials feel ... - Economic Ti... http://t.co/6bCIvdaEa0 #IndianNews, +3541,CTA Green Line service has resumed after an earlier derailment near Garfield expect delays. @WBBMNewsradio, +3545,Green Line trains resume service after South Side derailment: Green Line service has resumed more than four h... http://t.co/dJztc7apf1, +3546,Madhya Pradesh Train Derailment: Village Youth Saved Many Lives: A group of villagers saved over 70 passengers' lives after two train..., +3547,#Amtrak Rail #Disaster http://t.co/H6Ol73KZJg http://t.co/Qfydu4PVGK, +3551,Madhya Pradesh Train Derailment: Village Youth Saved Many Lives, +3553,Twitter said bet Tampa & the over in colorado...gambling twitter will have u broke & desolate be careful lol, +3556,'Behold your house is being left to you desolate!' (Matthew 23:38 NASB), +3558,who sank all night in submarine light of Bickford__ªs floated out and sat through the stale beer afternoon in desolate Fugazzi__ªs, +3559,"Be appalled O heavens at this + be shocked be utterly desolate + says the Lord +for my people have committed two evils:", +3561,The Desolate Hope: Part 1: I LOVE THIS GAME: http://t.co/gPrRZdpE4v via @YouTube, +3562,"We've become desolate +Its not enough it never is.", +3563,Shaping the to come: multimillion chop local billet frugal else companionate anacrusis production desolate in ..., +3564,@VacantLDN this may be a peculiar compliment but I would be incapable of doing my research w/out the desolate headspace your music provides, +3571,It's really sad that white people are quick to give money to desolate uneducated black ppl before educated blacks who are productive, +3574,@StevieDE83 @roathboy I'm desolate no idea how I'll ever get over it ??#badgeofhonour, +3576,The Desolate Hope: Part 3: BATTLE!!!!: http://t.co/mx3HuxnXgX via @YouTube, +3577,And the 10 horns which u saw on the beast these will hate the harlet make her desolate and naked eat her flesh and burn her with fire, +3579,@WilliamTCooper TY for the follow Go To http://t.co/l9MB2j5pXg BRUTALLY ABUSED+DESOLATE&LOST + HER LOVELY MUM DIES..Is it Murder? Pls RT, +3580,@mtnredneck It does when you are searching on a desolate part of the beach with no cell service and a mile away from your car., +3586,Why are you feeling desolate? Take the quiz: http://t.co/j4lM2ovoOs http://t.co/banrVjoTlf, +3590,@peregrinekiwi @boymonster I recall CP2020 Australia was a blighted desolate corporate, +3599,Now Playing Desolation Wilderness by Kodak To Graph, +3600,desolation #bored, +3601,I liked a @YouTube video from @iglxenix http://t.co/dcxjIJtyYJ Desolation PvP: Beacon Blast Bombardment 3 (Minecraft Xbox One), +3605,Obama 2016? - The Abomination of Desolation' http://t.co/iL4uLhrgTP, +3606,Free Kindle Book - Aug 3-7 - Thriller - Desolation Run by @jamessnyder22 http://t.co/PfYH4Tzvk9, +3612,The Hobbit: The Desolation of Smaug (#dvd 2014 2-Disc Set Digital Copy) http://t.co/000siJjL3t http://t.co/JlUJsHCvoA, +3614,I want to be free from desolation and despair, +3615,The Hobbit: The Desolation of Smaug ?? ? atm., +3616,Watching Desolation of Smaug and I always love catching the cameos. Caught Jackson at the beginning and Colbert just now., +3620,#Kids going to #school amidst the #desolation of an #earthquake #aftermath #Love #Olympus___ https://t.co/pqds20yrJs, +3622,Hey girl you must be The Hobbit: Part Two: The Desolation of Smaug because|I'm not interestud in seeing you. Sorry., +3624,#NowPlaying - Lamb of God - Desolation http://t.co/mUYWttEdl6, +3630,'I See Fire' Ed Sheeran The Hobbit: The Desolation of Smaug (Cover By Ja... https://t.co/a2aeDAK7r0 via @YouTube, +3644,Emotional Desolation the effect of alcoholism/addiction on family - http://t.co/31tGtLz3YA Forgiving is hard http://t.co/1labI2Sg2b, +3649,Debate question: The robots have become self aware; do with destroy them thus thrusting the world into a tech-free life or re-enslave them?, +3651,@Adz77 You spelled Ronda wrong last night and then again today ;) @RondaRousey she'll destroy you., +3654,Overall the English lads did very well today. They need to pile on and get over 400 runs ahead and destroy..., +3665,"JAKE I SWEAR TO GOD IF THERE__ªS ONE MORE GOD DAMN PIECE OF ALGAE IN THE PONDS I AM GOING TO DESTROY ALL THOSE LITTLE FISH. + +#momtherbot", +3668,People can't destroy you unless you let them., +3671,When something bad happens; you have 3 choices: You can either let it define you destroy you or let it strengthen you., +3672,GOP Plan to Destroy Public Education - Cut Teacher Pay and Even Make Them Buy Their Own School Supplies: Pure Evil - http://t.co/WST2tWqS8D, +3673,"But bad people kiss too. Hehe +Their kisses can destroy.", +3677,@alexhern he created vr only he can destroy it., +3678,Skate and Destroy em Taguatinga - Campeonatos de Skate http://t.co/xWW9dOPLK9, +3687,When people fear you they will try to say and do anything to destroy your #Pride., +3688,It's a real world. No glass shoe. No seven dwarfs. But there's always a villain who wants to destroy your life., +3693,This #NBCTheWiz cast is about to destroy tv sets across the nation!!!!, +3698,Media stocks are getting destroyed (DIS FOXA CMCSA SNI AMCX VIAB VIA TWX) http://t.co/dQd3YtaZfG, +3699,@Musketeiro I see ISIL destroyed in Iraq in May 2016. This may help Iraqis help Syrian government against it. But Nusrah and Ahrar not yet., +3701,#TheSun Flood: Two people dead 60 houses destroyed in Kaduna: Two people have been reportedly killed and 60 h... http://t.co/Aawmx5w9sh, +3703,Black Eye 9: A space battle occurred at Star O784 involving 2 fleets totaling 3946 ships with 14 destroyed, +3706,@evdaikoku @zjwhitman carp erased justin smith?? Pretty sure he destroyed our leftside, +3707,poor dude...he needs him some valerie...lol RT @Otp_destroyed: Dante stuck with KIKI twice in a row???????? #FixItJesus #GH, +3709,(SJ GIST): 148 Houses Farm Produce Destroyed By Flood In Sokoto: About 148 houses were on Saturday destroyed ... http://t.co/vqU1Y31hKU, +3711,Soo..he'll be do a bit of repentance for the children's lives they've destroyed and continue to abuse and for... http://t.co/En3hKXQdD9, +3714,WHAT a day's cricket that was. Has destroyed any plans I had for exercise today., +3716,Black Eye 9: A space battle occurred at Star O784 involving 2 fleets totaling 3936 ships with 5 destroyed, +3720,Flood: Two people dead 60 houses destroyed in Kaduna http://t.co/8H2SP6Ze3o, +3728,Remember #Hiroshima destroyed by #Nuclear bomb..an occurrence should never ever happen again yet highly likely recur http://t.co/mB3MJevBb0, +3731,HitchBot travels Europe and greeted with open arms. Gets destroyed after two weeks in america. There's a lesson to be learned here., +3732,@m1tchell1987 YOU DESTROYED MEE OKAY? I THOUGHT YOU WERE DEAD! BUT I'M NOT AREN'T YOU GLAD THAT I'M NOT?, +3733,19000 homes destroyed in Gaza -- ZERO rebuilt. Join me in busting the blockade! Sign & RT #Gaza #OpenGaza https://t.co/WF9GMhjh2M, +3734,@kaytlinmartinez dude those enormous blizzards we destroyed from DQ ??, +3740,By havin an ally lyk @AnnCoulter Republican Party is declaring war on all immigrants..deserves 2 b defeated n destroyed.it's a clarion call., +3746,yay the evil is being destroyed! https://t.co/GeTu7SfBJ1, +3749,@Warcraft weapon of destruction!, +3751,World of Warcraft: Legion Revealed at gamescom: The Burning Legion returns to bring destruction to Azeroth onc... http://t.co/7n9qqkQIl3, +3755,@PeterArnottGlas I've written a play about Dr Rae and his destruction by Lady Franklin as it happens. Commissioned by Mull Theatre, +3757,@AirportWatch Do we need more motorways 2reduce fumes on our present ones. -forget the extra noise & destruction!- https://t.co/USQZ0OTfNe, +3762,New RAN report from the frontlines of human rights abuses and forest destruction for fashion.: http://t.co/OmpzUyAsle, +3767,@iam_destruction Okay.. I'll put my tail in first.. -He moved to slip it in slowly.-, +3769,We should do a hand in the destruction of an INNOCENT young man's life., +3779,Crackdown 3 Destruction Restricted to Multiplayer: Crackdown 3 impressed earlier this week with a demonstratio... http://t.co/ma9LLiKcjk, +3781,Russian authorities to take account of petition against destruction of sanctioned food: Vladimir Putin's press... http://t.co/GUvSVPesZU, +3782,When life is cheap evil abounds and @PPact @HillaryClinton profit from its destruction.#DefundPP http://t.co/H5rzmAy8lP, +3783,The day of death destruction chaos; 60000 to 80000 people were killed instantly. Please take sometime to remember and think about others, +3784,He is one of the three survivors of the destruction of Otsuka Village., +3790,Let this stand as evidence of the horrible destruction potentially wrought by our noble species. https://t.co/sNOO8LQtAD, +3791,New RAN report from the frontlines of human rights abuses and forest destruction for fashion.: http://t.co/5xyE2Rkuri, +3792,Related News: The Bureaucrats Who Singled Out Hiroshima for Destruction - Global - The Atlantic | http://t.co/Tnex4HUsnp, +3794,Crackdown 3 Destruction Restricted To Multiplayer; Co-Developed By Sumo Digital: Crackdown 3 å_was definitely o... http://t.co/O4B1KIyx1P, +3799,@HJudeBoudreaux Start your car with it! Or use it to detonate an evil henchman!!, +3801,Track : Apollo Brown - Detonate ft. M.O.P. | @ApolloBrown @BILLDANZEMOP @FAMEMOP http://t.co/GR9q1wvi2n http://t.co/k3rEb7iRlc, +3804,"Apollo Brown-Detonate Featuring M.O.P. +http://t.co/JnoE2r2EIN", +3808,"Apollo Brown - Detonate (ft. M.O.P.) [Single] http://t.co/S5mU04L2rl +@ApolloBrown", +3809,Apollo Brown ___ Detonate feat. M.O.P.: Producer Apollo Brown is proud to present his upcoming album Grandeur wh... http://t.co/pfA0S9ZCUB, +3811,Best Moments of Jon Stewart on the Daily Show http://t.co/EOPbXWoZbm, +3813,@spinningbot Are you another Stand-user? If you are I will have to detonate you with my Killer Queen., +3817,Audio: Listen/purchase: Detonate (feat. M.O.P.) by Apollo Brown http://t.co/6ZSWtoKsif, +3818,@NickKristof Contradictory: 'We should've tried [to detonate on an uninhabited island]' & then 'the alternatives'___categorically___were worse, +3823,Listen to Apollo Brown - Detonate (feat. M.O.P.) by Mello Music Group #np on #SoundCloud https://t.co/C0Fex1XAlG, +3843,@someone92883220 they had to detonate it to determine it was a hoax device. Didn't feel comfortable just opening it., +3844,Track : Apollo Brown - Detonate ft. M.O.P. | @ApolloBrown @BILLDANZEMOP @FAMEMOP http://t.co/t9i3pWvjOy http://t.co/76aNr2iocL, +3846,Apollo Brown ___ Detonate feat. M.O.P.: Producer Apollo Brown is proud to present his up... http://t.co/3chxB8QbdD :http://t.co/zMh7nebmwY, +3847,First Stearns grenade thought to be WWII-era. This one apparently newer. Bomb squad plans to detonate around 8 p.m. http://t.co/qppTQ6oTat, +3852,Ignition Knock (Detonation) Sensor-Senso BECK/ARNLEY fits 93-95 Audi 90 2.8L-V6 http://t.co/gOXvNzOUj3 http://t.co/YB3GeEtFbT, +3854,Detonation fashionable mountaineering electronic watch water-resistant couples leisure tab___ http://t.co/kY9V0pAjY1 http://t.co/QjqpXIxMxz, +3856,Open forex detonation indicator is irretrievable after this fashion a financial airborne controls: SjFEb http://t.co/RzQkzM7rb8, +3863,Ignition Knock (Detonation) Sensor-KNOCK SENSOR Delphi AS10012 http://t.co/LArrNhoBsN http://t.co/6YwZWmxFDP, +3865,@jcenters No uh-oh it was a controlled detonation., +3867,Ignition Knock (Detonation) Sensor-Senso BECK/ARNLEY 158-0992 http://t.co/tk9HnxZNSl http://t.co/UhBUwbeQ0O, +3871,Detonation fashionable mountaineering electronic watch water-resistant couples leisure tab___ http://t.co/GH48B54riS http://t.co/2PqTm06Lid, +3872,Detonation fashionable mountaineering electronic watch water-resistant couples leisure tab___ http://t.co/g6hjTj3SDy http://t.co/yydEghGP64, +3878,Detonation fashionable mountaineering electronic watch water-resistant couples leisure tab___ http://t.co/RqOvPljpse http://t.co/UhKwVSoWSt, +3880,Ignition Knock (Detonation) Sensor-Senso fits 90-96 Subaru Legacy 2.2L-H4 http://t.co/pcCksm1tVM http://t.co/yWzngj7uow, +3881,Ignition Knock (Detonation) Sensor-Senso BECK/ARNLEY 158-1028 http://t.co/YszIBvj3cs http://t.co/C9t0cGtjFw, +3882,#JerusalemPost WATCH: Israel performs controlled detonation of land mines on Golan Heights http://t.co/01HuX8Y9Gi, +3883,Ignition Knock (Detonation) Sensor ACDelco GM Original Equipment 213-296 http://t.co/dBwSPgHhsa http://t.co/1CaLqDwvhw, +3884,via Jerusalem Post: WATCH: Israel performs controlled detonation of land mines on Golan Heights... http://t.co/rFB2ft4wX0, +3885,Ignition Knock (Detonation) Sensor-Senso Standard KS225 http://t.co/IwrCPmjOah http://t.co/zZMM9xEccW, +3886,Ignition Knock (Detonation) Sensor Connector-Connecto Dorman 917-032 http://t.co/Kh973YlMpj http://t.co/N9X3ngu0AM, +3890,Link lets you pick a city bomb size and altitude of detonation. Ace! https://t.co/wmW1wqvbR8, +3891,WATCH: Israel performs controlled detonation of land mines on Golan Heights - Jerusalem Po... http://t.co/0Y07oA5UeX #israel #israelnews, +3898,Obama Declares Disaster for Typhoon-Devastated Saipan, +3902,I'm literally devastated right now??, +3909,@polishedstone 'Perrie is DEVASTATED' meanwhile...., +3910,VAL JUST DIED AND IM ABSOLUTELY DEVASTATED #emmerdale, +3912,@yeetrpan I asked if they were hiring and they said not you I was devastated., +3915,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians a... http://t.co/AslUFEKOXN, +3918,Thanks @MartinBarrow - and yes definitely something that affects men as well. My husband just as devastated., +3920,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians a... http://t.co/9i6CrCRq2m, +3925,@MikeParrActor omg I cant believe they killed off ross he was my favourite character with aaron @DannyBMiller im devastated. Top acting ??, +3928,He was being super bitchy yesterday (cause he felt like pop) and when he's bitchy him and Perla don't play and then Perla is devastated, +3929,Actually if I don't see Hunter Hayes and Lady Antebellum perform Where it All Begins live I will be a very devastated person :-), +3930,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians a... http://t.co/lEYJwNnAH8, +3931,When you cook fresh tagliatelle meatballs and sauce and then manage to spill all the pasta in the sink when draining it. Fucking devastated., +3934,@humblethepoet Lilly's YouTube channel got hacked. She's devastated. We're all worried about her! ??, +3939,@MikeParrActor devastated your no longer in emmerdale best character with so much more to give #superbactor your going to be missed, +3941,@_Gags_ My Mommy will be devastated lol #NoMorePod, +3942,Devastated. Already missing my @coleenlisa. https://t.co/3p6Xakt7rh, +3943,Absolutely devastated ???? no Ross no!! #Emmerdale, +3946,Ooh the girl that won actually is meeting the boys im devastated ?????? but congrats to the winner!!! #OTRABaltimore, +3963,MPD director Armstrong: when this first happened I cannot begin to tell you the devastation I felt., +3965,#CNNHoTD soccer mom wild swimming handcuffed child teen hookup case nuclear devastation showers Miss Piggy... http://t.co/XG9s40AFtZ, +3966,The Dones felt grief bordering on devastation at losing connection with God through their church communities. Packard http://t.co/r2pQQPfqCt, +3971,My father walked through the devastation the day after and refuses to speak about it to this day https://t.co/oFu6DANJZC, +3972,Despite past devastation fortunately #forgiveness mutual #respect and #peace emerged. #Remember #Hiroshima and #Nagasaki, +3974,___Even in the midst of devastation something within us always points the way to freedom.___, +3976,"HQhed mhtw4fnet + +A sad reminder of nuclear devastation - Washington Post", +3978,Devastation Walkthrough Part 11: http://t.co/BOjDByB9YA przez @YouTube, +3981,we hid the devastation of what atom caused by not making it mandatory for film of its human suffering be shown & preserved., +3985,Is This Country Latin America's Next 'Argentina': One week ago we reported on the economic devastation in he o... http://t.co/J3rcOflDyA, +3986,@nikkihaley Profile in Ambition. Legacy of cultural devastation for attempt at a white house gig. @Reince @gop http://t.co/5cCrnwirFn, +3987,At least 18 houses 5 bridges destroyed in flash #floods in #Gilgit-#Baltistan on Wednesday. http://t.co/HSJcR45SIS http://t.co/dQ931ea6Pr, +3988,70 Years After Atomic Bombs Japan Still Struggles With War Past: The anniversary of the devastation wrought b... http://t.co/JESoSSDjjH, +3991,70 Years After Atomic Bombs Japan Still Struggles With Wartime Past: The anniversary of the devastation wroug... http://t.co/AU6Sm5FaMq, +3999,IRIN Asia | Red tape tangles Nepal reconstruction | Nepal | Disaster Risk Reduction | Natural Disasters http://t.co/q7LG6ncf7G, +4001,@quinhii I'm the one who started it so I feel like I have so much responsibility bUT NO ONE DOES ANYTHING OTL (a 'national' disaster lol), +4002,Trying to route my sister into W. Lafayette from the Indy airport. Suggested I-74 to SR 25 west of Crawfordsville. Hope it's not a disaster., +4004,Time to End the Disaster of Rail Privatisation http://t.co/ksxznPY5gt, +4007,Reade this and see how are politicians are leading us to disaster all for the #GreedyRich https://t.co/2bFliBD2sh, +4009,paladins is going to be a disaster, +4013,#Microsoft__ªs #Nokia acquisition was an even bigger #disaster than we ever imagined http://t.co/CV0mrisFt3, +4015,#OVERPOPULATION Not only R women incapable of keeping their legs together 2 save the world from endless brats they want to RUN the disaster, +4016,Harper gets sassed by Albertans after calling NDP government a __÷disaster' http://t.co/vsMOJyfzDg via @huffpostalberta #HarperBlamesAlbertans, +4023,WHAT A DISASTER FOR SECRET #TI5, +4031,Disaster [Chapter 27] https://t.co/EAsxT3y84R http://t.co/CjUFunsDKg, +4035,Deepwater drill company gains $735m: Transocean one of the companies associated with BP's Deepwater Horizon oil well disaster recovers___, +4036,Microsoft__ªs Nokia acquisition was an even bigger disaster than we ever imagined https://t.co/4MneTInGXl, +4037,Not an electric debut for Severino but not a disaster either. Looking forward to see what adjustments he makes for start #2., +4040,"This Palestinian family was lucky but hundreds are feared dead in the latest boat disaster in the Mediterranean +https://t.co/cT5v3LcNKD", +4048,Heat wave adding to the misery of internally-displaced Gazans http://t.co/jW3hN9ewFT via @PressTV http://t.co/NYWrkRQ7Kn, +4053,.POTUS #StrategicPatience is a strategy for #Genocide; refugees; IDP Internally displaced people; horror; etc. https://t.co/rqWuoy1fm4, +4056,.POTUS #StrategicPatience is a strategy for #Genocide; refugees; IDP Internally displaced people; horror; etc. https://t.co/rqWuoy1fm4, +4059,Next up: a charity program for disadvantaged people displaced by the tech boom in SF based on Tinder technology. https://t.co/yNVz1WrgtN, +4066,#mightyworld Thousands displaced after monsoon rains in Myanmar: Tens of thousands of people have been displac... http://t.co/HkSDk89c6C, +4067,Angry Woman Openly Accuses NEMA Of Stealing Relief Materials Meant For IDPs: An angry Internally Displaced woman... http://t.co/C1vzJOr2qz, +4069,Oof. Vikings took an island off the mainland. The noble house who ruled there got displaced... into my county. Welcome new vassals!, +4070,Job event targets displaced Double 8 employees http://t.co/2SJaaDRYdc http://t.co/RovTa5EUQO, +4071,It's crazy how far in advance you feel the air displaced by a subway car before the train itself shows up., +4073,#Myanmar Displaced #Rohingya at #Sittwe point of no return http://t.co/3S8nhaPVCh #Prison like conditions #genocide IHHen MSF Refugees, +4074,Displaced, +4075,local family displaced by fire http://t.co/f1nu18CArd via @gofundme Lets help that family out please, +4082,.POTUS #StrategicPatience is a strategy for #Genocide; refugees; IDP Internally displaced people; horror; etc. https://t.co/8owC41FMBR, +4090,Try not eating like a fat ass RT @INeedJa_Kadeeja: How much would it cost to have some fat displaced? Asking for a friend., +4099,U.S. in record hurricane drought http://t.co/NratKzyU5E #JustBitching http://t.co/JATvPAT0MO, +4102,Drought report lists SE Texas as 'abnormally dry' http://t.co/wBOXpyoZj0 #SETXNews, +4106,The Best Drought-Tolerant Perennials http://t.co/AdZONC7Olv http://t.co/ih9iTMhFjZ, +4109,El Monte attempts to shed reputation of delinquency in drought - The Pasadena Star http://t.co/2wGOUCOHOu, +4110,@MarcVegan @NonDairyKerry true. Also takes 16000 gal water to produce 1 lb hamburger. Drought in Midwest will be chaos, +4115,I thought it was a drought, +4118,U.S. in record hurricane drought http://t.co/JXOMdLtJ73, +4120,New drought monitor showing drought spreads in ETX. 'Severe Drought' in Marion/Harrison Counties. @KLTV7 http://t.co/v8eG2kSLoH, +4125,Thought it was a drought!, +4126,You can't use as much water in a drought as you can with a flood!, +4130,Water conservation urged for North Thompson: B.C. government declares a Level 3 drought level for the North Th... http://t.co/dJ4qw8YfW5, +4134,#KondoByJaymOnI U.S. in record hurricane drought http://t.co/fqLSkejWxH #Anticipate, +4136,@jonbuda What if the drought is just a giant marketing campaign for Star Wars where they turn California into Tattooine?, +4137,U.S. in record hurricane drought http://t.co/Wd4UdSxacq via CNN, +4144,I swear theres a bud drought, +4148,I'm incredibly sad & heartbroken so I'm just going to drown my sorrows in food & watch tv all night ?? http://t.co/yQiZ2p1rSm, +4151,I'm bout to get in the shower an drown, +4155,i cant breathe and i feel my throat closing up i want to die let me drown in my own tears please, +4162,Need to drown in ice cream??, +4165,@dolphfan36 I made it...Didnt drown, +4169,@GraysonDolan id drown, +4174,@LadyTraining yes I will probably drown her in craft throughout our years but by no means spoiled, +4179,Throw me in the deep end watch me drown, +4185,I'm gonna drown myself in leftover chilis wishing it was with him lol, +4186,I'll let Louis drown me at the water wall by the mall If that makes him smile, +4187,drown me in clementines, +4188,When I genuinely like someone I REALLYYY like them and I have to hold back or else I'll drown them with affection and attention ??, +4190,Fuck around and drown ???? https://t.co/fr5z9WklMZ, +4192,absolute drown your Wagyu steak in heaping piles of moist Smuckers Grape Jam, +4193,Drown me you make my heart beat like the rain., +4194,@Vanquiishher it can legit drown, +4195,Yeah I'm definitely just gonna go drown myself in the pool bc it's just too damn hot inside, +4196,@GraysonDolan I'll fall and drown so I think I'll pass, +4207,Hundreds feared drowned as migrant boat capsizes off Libya: Hundreds of migrants are feared to have drowned af... http://t.co/bF3OhacB1r, +4208,100s of migrants feared drowned after 700 squeeze onto boat in Mediterranean. http://t.co/lsyPtk18se, +4215,Hundreds feared drowned as migrant boat capsizes off Libya: Hundreds of migrants are feared to have drowned af... http://t.co/Cbhe0eXIRA, +4217,Hundreds of migrants feared drowned off Libya: http://t.co/RTx4O0SIyH #news #bbc #cnn #msnbc #nyt #tcot #tlot #waar #ccot #ctot #p2 #ap, +4222,You're getting drowned out yet I still search for you., +4223,Hundreds feared drowned as migrant boat capsizes off Libya http://t.co/U8H5s8oPs5 #NLU, +4226,@oilersjaysfan how's the rain have you drowned yet?, +4228,Jeff was bullied Sally was raped Ben was drowned Hoodie was betrayed LJ was abandoned EJ was a sacrifice Lost S.. (http://t.co/mFl7jzspOu), +4229,Tried my 1st #beastburger by #beyondmeat. nowhere near what i thought based on hype. Drowned in ketchup wasnt bad but w/dijon it sucked., +4231,#dw_english Hundreds feared drowned as migrant boat capsizes off Libya: Hundreds of migrants are feared to hav... http://t.co/i9HW0qZoVy, +4234,pretty sure i almost drowned 26 times in the pool today thanks to abrianna, +4236,@BenWunsch I ALMOST DROWNED AT SEA, +4249,Having such a hard time with everything im drowning literally, +4254,Boy saves autistic brother from drowning: A nine-year-old in Maine dove into a pool to save his autistic broth... http://t.co/pihUIaOR5O, +4258,Family mourns drowning of 'superhero' toddler with rare epilepsy: Bradley Diebold suffered hundreds of epilept... http://t.co/aSAn4yGd48, +4259,If you told me you was drowning. I would not lend a hand!, +4263,@TinyJecht Those eyes.. desperate like a mouse drowning in a septic tank., +4267,and no one knows that i'm drowning and i know that i can't fcking survive, +4277,I have the experience performing cpr on a actual drowning victim., +4278,The Drowning Girl by Caitlin R. Kiernan Centipede Press Signed numbered Limited - Full re___ http://t.co/CBaXj72lVC http://t.co/s9wXIDItJh, +4280,The Drowning Girl by Caitlin R. Kiernan Centipede Press Signed numbered Limited - Full re___ http://t.co/2zJcfAzs2N http://t.co/jAAvvwDOZ6, +4283,AND I'M DROWNING IN THE D_äJ__ VUUUUU WE'VE SEEN IT ALL BEFOOOOOOOORE, +4285,Boy saves autistic brother from drowning: A nine-year-old in Maine dove into a pool to save his autistic brother from drowning, +4286,Boy saves autistic brother from drowning: A nine-year-old in Maine dove into a pool to save his autistic brother from drowning, +4287,The Drowning Girl by Caitlin R. Kiernan Centipede Press Signed numbered Limited - Full re___ http://t.co/eOMdGhraLg http://t.co/i4SG3nvGp8, +4290,Migrants drowning - we will be erecting monuments 100 years from now and shed tears for the most powerful nations' greed of letting them die, +4291,Drowning doesn't sound half bad rn, +4294,Boy saves autistic brother from drowning #NewsVideos http://t.co/8NZt51ew14, +4302,OH SHIT DUST'S STORM ATTACK AND FIDGET'S BULLET THING MADE SOME CRAZY RICOCHET ATTACK, +4308,The dust storm chase scene is pretty damn cool. #GhostProtocol, +4310,New Mad Max Screenshots Show Off a Lovely Dust Storm Combat... #pc #gamescom http://t.co/Qq1yW0brsR, +4311,recent project (Dust Storm ). :) http://t.co/jmj9izVvzO, +4315,Photo: susfu: Dust storm rolling onto the base by cptesco August 03 2008 at 09:39PM http://t.co/ekkruDQzbv, +4316,New Mad Max Screenshots Show Off a Lovely Dust Storm Combat Magnum Opus: New Mad Max screenshots have been___ http://t.co/l8dh8KLRKR, +4319,@christinaperri #askceeps have you seen @colinodonoghue1 in his new movie trailer for The Dust Storm?, +4331,Photoset: hookier: New stills from The Dust Storm (x) http://t.co/h8YjcFG8hv, +4338,@whatupdave Headline in 2075 'Dust Storm on Mars Destroys 1% of Money Supply', +4339,Pickathon dust storm is sadly over but we have @sunsetballard in Seattle tonight to look forward to w/ @tomonakayama http://t.co/RtgGzWCAix, +4340,Kids Disappear in Dust Storm in Atmospheric Aussie Thriller http://t.co/iR7VaxehEW RT @Newz_Sacramento, +4344,No storm lasts forever the dust must settle truth will prevail. http://t.co/1cjyfY8iXj, +4346,A dust storm in Pheonix. http://t.co/AMgfOnzUSD, +4347,NASA MODIS image: Dust storm over Morocco and the Straits of Gibraltar http://t.co/Q1jBreEsXv #duststorm, +4360,USGS EQ: M 1.9 - 5km S of Volcano Hawaii: Time2015-08-06 01:04:01 UTC2015-08-05 15:04:01 -10:00 a... http://t.co/3rrGHT4ewp #EarthQuake, +4364,#Earthquake of M 2.9 - 140km W of Ferndale California http://t.co/8t8oSCs4w7, +4367,@lizXy_ IMAGINE IF AN EARTHQUAKE HAPPENED, +4369,#Children traumatised after the Nepal earthquake are educated on coping mechanisms. http://t.co/UbwDBydK1a, +4370,Manuscript suspiciously rejected by Earthquake Science Journal could rock your world https://t.co/6vBQEwsl1J viaYouTube, +4371,@GreenLacey GodsLove & #thankU my sister for RT of NEW VIDEO http://t.co/cybKsXHF7d The Coming Apocalyptic US Earthquake & Tsunami, +4380,5.3 #Earthquake in South Of Fiji Islands. #iPhone users download the Earthquake app for more information http://t.co/V3aZWOAmzK, +4386,#USGS M 1.9 - 15km E of Anchorage Alaska: Time2015-08-06 00:11:16 UTC2015-08-05 16:11:16 -08:00 at epicen... http://t.co/HkIiPyX5jL #SM, +4389,1.57 magnitude #earthquake. 27 km from #Ramona CA #UnitedStates http://t.co/ZtOlVGkDuk, +4390,1.9 magnitude #earthquake. 16 km from Anchorage AK #UnitedStates http://t.co/wVu08yJGOK, +4394,#SISMO ML 2.0 SICILY ITALY http://t.co/vsWivoDCkL, +4399,@maeisdumb WHOAHAHAHAHHDJS electrocute me...., +4400,hmm what if I've already met my special someone but i dnt even know it ??????, +4404,Wish I didn't just electrocute myself 4 times in 30 seconds ??, +4409,Wish I could buy a device that would electrocute me every time I went to eat something unhealthy....Eat so much shit.??????????, +4414,Photo: weallheartonedirection: I wouldn__ªt let David electrocute himself so I__ªm the asshole http://t.co/nC2d6IRm2u, +4417,stay with me - sam.smith, +4418,So that it'll electrocute somebody's ass baa...No thank you https://t.co/x7P1xaBWTz, +4419,I feel like I would actually electrocute myself if I tried this https://t.co/FgKpYTiKtI, +4420,drinking pepsi im afraid i might spill some and then my headphones electrocute me., +4422,SPOILERS: your phone does not have the power to electrocute you if dropped in the bath, +4429,Morning Metro Late train? Check. Crazy guy who tells people the doors will electrocute them? Check., +4431,I broke my charger. Hella trying not to electrocute myself and die rn ????????, +4433,Eh it's the Macy's Thanksgiving Parade tomorrow! I'll be done. I'll still electrocute your email now!, +4434,Do you remember when I suddenly electrocute you to death with my fox powers? Ha. Ha. Ha. Ha. Ha. Ha. Because I do., +4439,#KCA #VoteJKT48ID JoelHeyman: My loose understanding is: @ 750k subs AH will electrocute then eat Michael. Don't want 2 influence anythin___, +4443,"Why can't robots have a shower? +Answer: Because they will electrocute themselves!", +4446,of electricity he had wire and a golf ball hooked up to catch a bolt of lightning and electrocute it you're not my brother I can tell. The, +4447,Photo: weallheartonedirection: I wouldn__ªt let David electrocute himself so I__ªm the asshole http://t.co/wIabcyParM, +4453,I've had electrical tape wrapped around my charger for like 2 months bc I don't want to get electrocuted :))))) http://t.co/Fb3Do910FC, +4460,Almost electrocuted myself ??, +4461,South Side factory where worker electrocuted pays $17000 penalty #Columbus http://t.co/nSGWiCYXFT, +4462,When you realize you could have been electrocuted last night because you grabbed this in the dark. How??? http://t.co/RJRoZMjF2n, +4467,"Fish stand-up +'So what's up with whale sharks? They whales or sharks?' +*Silence* +'Is this thing on?' +*Flicks mic. All fish get electrocuted*", +4469,South Side factory where worker electrocuted pays $17000 penalty http://t.co/PENJHc4ZCx #Columbus #Ohio #news, +4473,My brother just got electrocuted from the plug socket ????, +4474,Got electrocuted by the mosquito killer. #boomshunga, +4475,@EllaEMusic_ You should have just simply let on that you had electrocuted yourself while plugging in your phone charger. It works for me..., +4477,@philadper2014 They would all get #Electrocuted !!!!!!!!!, +4481,Beware! Dont use your earphones while your phone is charging. You might be electrocuted especially when your ears are wet., +4483,short hair probs: i can never look cute with bedhair anymore i just look like an electrocuted vagrant, +4484,@hairdryer180 you'll get electrocuted, +4485,@SHGames Please fix the problem where zombies disable your Eco suit and the floor is electrocuted so u die auto magically plz, +4495,I hope I get electrocuted today at work, +4497,#IWouldntGetElectedBecause Oh I certainly would esp if there were any live wires...wait that's elected and not electrocuted! Never-mind!, +4500,Emergency Garage Gate Repairing Services 11211 NY: Our Offerings: UAC Entrances Brooklyn agent in the set up o... http://t.co/WMqOZ81R43, +4501,Motors Hot Deals #452 >> http://t.co/vcaptRnmIE MANUAL TRANSMISSION SHIFT KNOB+BOOT+EMERGENCY HAND BRAKE HANDLE 5___ http://t.co/S2ltphTGTJ, +4502,11000 SEEDS 30 VEGETABLE FRUIT VARIETY GARDEN KIT EMERGENCY SURVIVAL GEAR MRE - Full rea___ http://t.co/VE78djgHa5 http://t.co/ugubwRPQFP, +4505,?????? EMERGENCY ?????? NEED PART 2 and 3!!! #NashNewVideo http://t.co/TwdnNaIOns @Nashgrier 103, +4506,No matter the dilemma emergency nor set-back; The Righteousness of JAH shall prevail! http://t.co/n0tIy7SU1C, +4510,11000 SEEDS 30 VEGETABLE FRUIT VARIETY GARDEN KIT EMERGENCY SURVIVAL GEAR MRE - Full rea___ http://t.co/OH4YSttyCo http://t.co/X8Kv8tZZE8, +4512,Survival Kit Whistle Fire Starter Wire Saw Cree Torch Emergency Blanket S knife - Full re___ http://t.co/8pFMSYPg5B http://t.co/nETYdvtLnb, +4513,RN / Registered Nurse / ER / Emergency Room - Supplemental Health Care: (#Modesto California ) http://t.co/YiYspjLKZO #Nursing #Job #Jobs, +4532,Job now are only in case of emergency., +4535,@StephGHinojosa hey my mom said to tell your dad to call her ASAP. it's an emergency., +4536,Leaving back to SF Friday have not packed one single thing 911 emergency, +4540,Emergency hop kit? ;) jk https://t.co/MkbQ2YE4Wq, +4545,Be safe and be prepare from emergency kits to evacuation. Alertness esp women and children who are also @gmanews https://t.co/3GALBowItN, +4552,Municipal Emergency Plan deactivated! Crisis averted. https://t.co/37L4qUAAVu, +4554,Calgary takes another beating from summer storms; City activates emergency plan http://t.co/4oi9hMo7om, +4555,"119 Sri Lanka's 'Emergency' Line. +Busy for the past 30 mins. +Suppose I need to plan my emergencies better. #SL #RT", +4558,Calgary takes another beating from summer storms; City activates emergency plan: It was another day of watching the___ http://t.co/240ZLieBrk, +4559,@go2MarkFranco I spoke w Caleen Sisk. If #ShastaDam gets funded through Cali Emergency Drought Act I plan on doing story wfocus on Winnemem, +4560,New Blog Post Emergency-response plan helps employees get back to productive work http://t.co/Eu21N1G1ZX, +4566,Emergency-response plan helps employees get back to productive work http://t.co/tXQZN6GTHa @BusInsMagazine, +4570,@660NEWS @cityofcalgary I picture Nenshi & Jeff as the Wonder Twins. 'Municipal Emergency Plan -Activate!', +4572,Do you have an emergency drinking water plan? Download guide in English Spanish French Arabic or Vietnamese. http://t.co/S0ktilisKq, +4574,I liked a @YouTube video from @prosyndicate http://t.co/mLvBg6sEka Minecraft: Hunting OpTic - Emergency Exit Plan! (Episode 13), +4581,Calgary takes another beating from summer storms; City activates emergency plan http://t.co/u59DmRnlTK, +4583,Calgary takes another beating from summer storms; City activates emergency plan #yyc #abstorm http://t.co/iQIXjtUEwH, +4586,Calgary takes another beating from summer storms; City activates emergency plan: Calgary Airdrie Okotoks and... http://t.co/i7QX4hAz6R, +4591,Storm prompts activation of City of Calgary's Municipal Emergency Plan: As the City of Calgary remains under a___ http://t.co/7jcR8REgUv, +4594,@KimAcheson @ScottWalker / Emergency Room can not refuse you service & you can do a payment plan., +4603,Call for Tasmania's emergency services to be trained in horse rescues http://t.co/q9OXqfdGwX #abcnews, +4605,This #Nursing #job might be a great fit for you: RN Nurse Shift Manager Emergency Services - Full Time... - http://t.co/T71ekcPmWL #Hiring, +4612,Ballina emergency services have again been activated following a shark sighting of what has been described as two... http://t.co/Eu5tKTptpW, +4613,Call for Tasmania's emergency services to be trained in horse rescues http://t.co/zVQwLpScSC, +4615,@GeneFMyburgh @crimeairnetwork and in a week they will protest service delivery and lack of emergency services #ThisIsAfrica, +4618,Human Resources Processes Streamlined Insofar as Emergency Services Organization...AbboG, +4621,@Jinxy1888 Genius economist! Let's double pay of all emergency services military teachers etc to that tube drivers #PayForItHow #tubestrike, +4623,"A mended finger +When a finger is cut there is an immediate reaction from the body__ªs emergency services occurs: +__¢... http://t.co/UwmPRV0UD5", +4626,When you say call dad but your phone dials emergency services..... #awesomejobsiri, +4629,Emergency Plumber Emergency Plumbing Services. #plumber #localplumber #Australia https://t.co/V3LZWkqSO7, +4633,@mickbland27 It is disturbing! Emergency services & first responders should work together not slander try & get each other killed., +4636,We're #hiring! Click to apply: Registered Nurse - Emergency Department PEDS Fellowship (FTE .9 Night) - http://t.co/AmCMCltkKC #Nursing, +4638,Get Emergency Edmonton Locksmith Services in Edmonton Canada business name: 'I got locked out' http://t.co/GPH5vgaYiY #locksmith #Edmonton, +4640,Providence Health & Services: Registered Nurse - Emergency Department PEDS Fellowship... (#Anchorage AK) http://t.co/vz4RPCfTRm #Nursing, +4643,Can you recommend anyone for this #Nursing #job? http://t.co/xXwm544ygf #Antioch CA #Hiring #CareerArc, +4645,We're #hiring! Click to apply: Staff Registered Nurse - Emergency Services (0.9 FTE Evening) - http://t.co/hhiUzIeLtr #Job #Issaquah WA, +4647,@bodminmoor1 experienced the expertise of some of Cornwalls finest emergency and rescue services!, +4648,@snackwave_julie When You Are Engulfed in Cake #LiteraryCakes, +4651,Why are you engulfed by low self-image? Take the quiz: http://t.co/JAJFxaOemp http://t.co/FUjvF6foaR, +4652,Become so engulfed in your own success- you forget it ever happened., +4653,He came to a land which was engulfed in tribal war and turned it into a land of peace i.e. Madinah. #ProphetMuhammad #islam, +4654,Tube strike live: Latest travel updates as London is engulfed in chaos - http://t.co/wrMSpcpml3 http://t.co/nWolqktxaf, +4660,Do you feel engulfed with low self-image? Take the quiz: http://t.co/kcpgAy4FlJ http://t.co/BZqC6JuAix, +4662,Do you feel engulfed with low self-image? Take the quiz: http://t.co/2wFQctN62v http://t.co/j4PmcEc4xh, +4663,Do you feel engulfed with anxiety? Take the quiz: http://t.co/Um6weIUjyW http://t.co/Fm92uk6Yp2, +4665,@FlameCored colliding with her projectile as a cloud of smoke engulfed the area. Not a moment later Shadow propelled himself through it --, +4668,Society has become so engulfed in capturing every moments and sharing it it takes the fun out of moments that should be intimate., +4676,'Sometimes I close my eyes and picture what this world would be like if it were engulfed in flames.' #Bot, +4683,Men escape car engulfed in flames in Parley's Canyon crews investigating cause - http://t.co/1TVFfba95l http://t.co/ALjwAWkxPj, +4685,#Tubestrike live: Latest travel updates as #London is engulfed in chaos - Telegraph http://t.co/AFzJamLFe6, +4686,Why are you engulfed by low self-image? Take the quiz: http://t.co/7V7YTEjL3J http://t.co/xJJU7MEN8f, +4701,LIVE on #Periscope: Wild Wing Epicentre https://t.co/U2fUK072F9, +4718,guy next to me walks into our tiny office room with the fishiest smelling lunch ever. EVACUATE, +4721,@crown_769 I'd rather have an ass in fire. At least I'd know whatever I eat will evacuate. Heartburn sticks around., +4727,SALHA(1):HARRY STYLES(2)U KNOW WHERE IS UR P_EOPLE#HURRY#EVACUATE THEM FROM#HELL.#JUST SHAHADA&'PICK#ME'.#CLASS_SICK http://t.co/UpdDVr6jva, +4728,ISIS are deploying their troops to other battle grounds since deal with Turkey to evacuate buffer zone areas hence moving fronts to Assad., +4733,What if the fire up in the Catalinas gets worse and we have to evacuate the area. Aka no school tomorrow, +4741,MY DOG JUST FARTED EVACUATE THIS FUCKIN ROOM DISGUSTING ASS BROOM LOOKIN ASS, +4743,my father fucking died when the north tower collapsed ON HIM as he was trying to evacuate more people from the building., +4746,@yourgirlhaileyy leaveevacuateexitbe banished., +4751,Just had to evacuate the 100 Oaks Regal theater. I've never been more terrified., +4756,Town of Roosevelt Washington ordered to evacuate again as fire worsens http://t.co/Xkh5s1DR8d, +4761,@misschaela_ not yet. Everywhere else except us and like a few other shops like Panda are evacuated but they haven't come for us yet., +4763,UPDATE: Evaucation order for Roosevelt WA has been lifted: http://t.co/2NV0wHDLut, +4764,Chemical spill at GR water plant contained and water supply OK. @KocoMcAboy asked plant officials what happened: http://t.co/FHPLkEX3oK, +4767,A house was evacuated in Orakei after a bank collapsed and trees were felled by strong winds in Auckland ... http://t.co/UCuCmKypdN #NZ, +4772,Chesterfield apartment complex evacuated because of fire #STL http://t.co/MIq39IOK3U, +4776,Gold Coast tram evacuated due to broken powerlines http://t.co/hJqbA42iY9 #Local NewsbrokenEmergency ServicesGold CoastpowerlineTramTr, +4780,"So many fires in NorCal they can't fight some of them. + +Pray for our firefighters air tanker pilots support crews & evacuated people.", +4781,#FortitudeValley unit damaged and residents evacuated after fire broke out after 7am this morning #7News http://t.co/8iZgxCeLcc, +4792,Family evacuated after weather bomb http://t.co/2A4z8pmvVE, +4796,Grass fire burns past Roosevelt Wash. prompts evacuation http://t.co/BSuDXCPrSG #fire #firefighter, +4801,Green Line train derails on South Side passengers safely evacuated CTA says http://t.co/w6F7ZiS3KA http://t.co/t7L8jCjyq3, +4804,Why does the secret bunker used by Cheney during 9/11 look like it was catered when ktchn staff had been evacuated?, +4805,Valley building evacuated after fire http://t.co/nyqF8GqMzg #queensland, +4806,13000 evacuated as California firefighters fight flames to save homes: CLEARLAKE OAKS Calif. ___ Wildfires lik... http://t.co/xwBYeaOWMw, +4811,Pelling hotels: no strings concealment from straight a rejuvenati???ng evacuation day: pqhaxp, +4815,my school is so fucking dumb they just set off the evacuation sirens on accident, +4816,How an anatomic mother helps up-to-the-minute remodeling high evacuation hospital: IaAvRoIV, +4817,FAAN orders evacuation of abandoned aircraft at MMA http://t.co/5RXI47gCAj #NEWS @TodayNGR, +4818,Even tho Im getting 40%money back due 2 veld evacuation nothin could make up for missing @DVBBS @HARDWELL @aboveandbeyond #stillnotoverit ????, +4822,You'll neediness so taste up and down ulcerated waggon evacuation cornwall: bNVfDAg, +4824,Do you know your hurricane evacuation route? Find it here: http://t.co/mWMVIcdW9O, +4827,The EFAK would be designed for building occupants once they evacuate and report to their evacuation assembly sites, +4837,Evacuation drill at work. The fire doors wouldn't open so i got to smash the emergency release glass #feelingmanly, +4838,KCEM Jeff King reports a level 3 evacuation east of Roosevelt from Whitner rd to the county line between SR14 & Hale rd., +4839,#MadRiverComplex #CA #CASRF http://t.co/hjWLDCLiE4 Mad River Complex: Evacuation Advisory for Swayback Ridge Area, +4841,@CALFIRE_PIO Being prepared is crucial but returning to homes after evacuation only to find them robbed is criminal! #WillowFire, +4851,Survey: 52% of medical care facilities near nuclear plants lack evacuation plans http://t.co/vLMmCdPkRo, +4855,Evacuation order lifted for town of Roosevelt - Washington Times http://t.co/Kue48Nmjxh, +4861,@wyattmccab you'd throw a can of Copenhagen wintergreen on the ground that would explode on your enemies and give them mouth cancer, +4862,my damn head feel like it's gone explode ??, +4863,happy Justin makes my heart explode, +4865,Vanessa was about to explode! This is what she wanted to say to Shelli. Their alliance will survive. #BB17 #BBLF http://t.co/rypGKScHng, +4871,My ears are gonna explode smh, +4873,Some guys explode ??, +4880,@magicallester I will die. I'm actually being serious. My heart will beat so fast it will fly out off my chest & explode, +4885,Facebook Influence 2.0: Simple strategy to explode your Facebook page and create more___ http://t.co/rWPRtMIbHl, +4886,im sooooooo full my stomach is going to explode, +4902,I need a follow before I explode @GraysonDolan, +4904,I feel like I'm going to explode with excitement! Wonder begins within the hour______ https://t.co/zDZJ5kRbzr, +4907,Twitter will explode...light the match @realmandyrain, +4916,The fact checking machine must have exploded today following @POTUS #IranDeal speech., +4921,#Aphid population in #NorthIowa has exploded in last 4 days! http://t.co/cw6jxYVbNj #FromTheField, +4925,@Alltheway80s I had a similar thing with John carpenters 'the thing' one girl threw up when the dogs 'exploded', +4927,And bleach your skin after touching that thong.*shudder* That and all the boudoir outfits. My eyeballs are bleeding & my brain has exploded., +4928,Holy hell the bathroom on car 649 smells like is exploded... @Metrolink, +4930,"that exploded & brought about the +beginning of universe matches what's +mentioned in the versethe heaven and Earth +(thus the universe)", +4932,Our electricity just fucking exploded they're trying to fix it now, +4933,I didn't even know pens still exploded like that...ink everywhere, +4937,#news #science London warship exploded in 1665 because sailors were recycling artillery cartridges: A dive to ... http://t.co/r4WGXrA59M, +4939,At this moment 70 years ago (local time) the 'Little Boy' atomic bomb exploded over Hiroshima instantly killing at least 70-80000 people., +4941,I can't take advantage of Smith Micro's deals right now because exploded car (hint hint shironu_akaineko AT... http://t.co/qbCTxPwhYP, +4942,The tap in the bath exploded and now it looks like someone shit in the bath. I love how great our plumbing is!????, +4947,If that was ronaldo Twitter would have exploded, +4949,"that exploded & brought about the +beginning of universe matches what's +mentioned in the versethe heaven and Earth +(thus the universe)", +4950,@rickoshea @AnthonySHead @NicholasBrendon @DublinComicCon @RTE2fm WHATTT? Oh wow my fangirl head has exploded in a mess of crap everywhere, +4956,@marymc21 ahhhh I'm just being a baby. My car's engine kinda sorta exploded and we're trying to find a solution, +4960,Fun. Hobby Lobby exploded in this home. http://t.co/TT5L40sjaI, +4963,Hp lto-5 lump together warehouse information explosion is more than one divestment able: PMngiqy, +4965,GAElite 0 Explosion Greg 1 [Top 2nd] [0 Out] [0 balls] [0 strikes] ... No one on [P: #16 Morgan Orchard] [B: ], +4976,New Explosion-proof Tempered Glass Screen Protector Film for Blackberry Z10 - Full read by___ http://t.co/LgbhdFYrwq http://t.co/9qV8glZSwV, +4977,New Explosion-proof Tempered Glass Screen Protector Film for Blackberry Z10 - Full read by___ http://t.co/u1RKPJ3Cbc http://t.co/sJtxhtx71q, +4987,This whole podcast explosion thing has been weird. Just replied to a YouTube comment for the first time in... Ever. What is happening?!, +4991,Ahead of Print: A New Paradigm of Injuries From Terrorist Explosions as a Function of Explosion Setting Type.:... http://t.co/tqQc3yxBoR, +4994,I liked a @YouTube video from @shawlarmedeai http://t.co/NN4fY1WBxf Easy Natural Bride makeup tutorial Makeup explosion Ft ABH Shadow, +4999,@MrBrianORL at least they didn't try and put a straw in the liquid one #Explosion, +5003,1 injured in Naples boat explosion http://t.co/89lcTH3Sx1, +5006,I liked a @YouTube video from @theblacklink http://t.co/Ac8P9HJH2q Game Informer Taken King News Explosion Part 1! | Destiny PS4 |, +5009,RASaudAuthor: #Pyrotechnic #Artwork by #CaiGuoQiang Explodes into a Blossom on the Steps of the #PhiladelphiaMuseu___ http://t.co/pJE607uXlp, +5011,Hero of capitalism: Sprint employee promotes Sprint while Fox News interviews him as a shooting eyewitness: https://t.co/DtRPhCTmJx, +5012,Read an eyewitness account from #Hiroshima on Aug. 6 1945 http://t.co/sbIP9g52YQ Mankind's worst invention., +5014,DK Eyewitness Travel Guide: New Zealand http://t.co/AgQgrowj7Y, +5020,Baseball: Eyewitness Accounts: August 6 2015 by Jeff Moore Wilson Karaman and Brendan Gawlowski: Eyes on Mi http://t.co/H4ZOZqLpE7 #sport, +5021,COOL: @NASA captures 'dark side' of the moon which is never visible from Earth http://t.co/qKP30b4ag2 #EyewitnessWV http://t.co/N3hgJSYVO2, +5023,#Fracking #Ecocide Eyewitness to Extreme Weather: 11 Social Media Posts that Show Just How Crazy T... http://t.co/dEdDH8Rme8 #Revolution, +5026,@EdmundAdamus @CampionJoanne Pres Carter's notes in '79 meeting w/ JPII show Latin on his mind http://t.co/7nt13HIrTK http://t.co/4gCmQ4oWen, +5030,Marked as to-read: DK Eyewitness Travel Guide by Richard Sterling http://t.co/oEoKD4KiqH, +5031,How 'Little Boy' Affected the People In Hiroshima - Eyewitness Testimonials! - https://t.co/5x5hSV5sKO, +5036,Cape Town: Thousands living on Cape Town's streets - Eyewitness News: Eyewitness NewsThousands living on Cape ... http://t.co/nAo6EWCPK2, +5042,With all due respect @RepMikeRogersAL y would u completely disregard ALL eyewitness accounts on Benghazi? #WereNotGruberVoters. #PJNet, +5045,Dutch crane collapses demolishes houses: Dramatic eyewitness video captures the moment a Dutch crane hoisting... http://t.co/8PpZNGYAqE, +5048,Dutch crane collapses demolishes houses: Dramatic eyewitness video captures the moment a Dutch crane hoisting... http://t.co/AJHKfh4e7G, +5051,http://t.co/B7Iu2lpfXs: Eyewitness to Extremeå_Weather http://t.co/dYx2jr6ckP, +5053,UPDATE: A GOP-controlled Senate committee has approved legislation to block the Obama administration from putting... http://t.co/Jg6B0vfl5R, +5054,'Things got heated. Busta splashed water on him and the employee threw water back' one eyewitness said. http://t.co/GoXwT8PG0B, +5057,DK Eyewitness Travel Guide: Denmark: travel guide eBay auctions you should keep an eye on: http://t.co/7Tcs1ePOrQ, +5058,QUESTION OF THE DAY: It has been proven that eyewitness accounts are not always very reliable. Does this... http://t.co/2I4brtSh93, +5066,http://t.co/tcXRtfaPZa Russian 'food crematoria' provoke outrage amid crisis famine memories http://t.co/eT9xxfdJnM, +5070,'Russian 'Food Crematoria' Provoke Outrage Amid Crisis Famine Memories' by REUTERS via NYT http://t.co/jbYjbQEoiK #Ukraine, +5077,Ukraine famine memorial in progress since 2006 goes up in Washington: http://t.co/Td7VIEoODi, +5081,#IntlDevelopment Ukraine Famine Monument Erected In Washington: ___ who perished in the Ukraine famine of the 19... http://t.co/nUPy6EPAWn, +5084,"@WuckForld @LioniskingRTAC To be alive even a heroic +Famine drink look out", +5086,So shameful and tragic.. http://t.co/5p1e40qkGV, +5090,#Ukraine #Kyiv #News Radio Free Europe/Radio Liberty: Ukraine famine monument erected in ... http://t.co/ME5u1YqH4i #Kiev #Ukrainian #Ua, +5091,"NONSENSE >> famine memories -- strong exaggeration of Ukrainian MSM +#ukraine #russia #?????????? #sanctions https://t.co/dDOTd7W2o8", +5093,I would rather dwell in the land of famine and be in God's perfect will than to rest in the confines of Egypt... http://t.co/P14oePCrW0, +5100,Miners strike; Botha Apartheid SA Tutu Nobel prize; IRA bomb Brighton Famine in Ethiopia. '84 feels like yesterday http://t.co/UZKssvj9CW, +5108,Russian 'food crematoria' provoke outrage amid crisis famine memories http://t.co/kZYvMnqHy7 via @Reuters, +5111,Protesters mark year since fatal police shooting in Wal-Mart (from @AP) http://t.co/3KbeJGmj0d, +5112,Police investigate fatal shooting on 71 Highway http://t.co/0KJjdaOJHI, +5118,Success is not final failure is not fatal.It is the courage to continue that counts - http://t.co/gRTHlAACfV, +5121,11-Year-Old Boy Charged With Manslaughter of Toddler: Report: An 11-year-old boy has been charged with manslaughter over the fatal sh..., +5123,11-Year-Old Boy Charged With Manslaughter of Toddler: Report: An 11-year-old boy has been charged with manslaughter over the fatal sh..., +5129,Trim error led to fatal Greek F-16 crash: ?A NATO safety investigation board report has determined that the ma... http://t.co/YhSahLKQo4, +5133,Photo: No need to put your all your feelings on social media___the results could be fatal. #message http://t.co/EzxtovAl4l, +5143,@LadyShaxx That is freaking awesome!, +5147,Seatbelts really do save people man. Alot of fatal recks have no intrusion of the interior People just fly out, +5151,Investigators shift focus to cause of fatal Waimate fire http://t.co/ej0mewOupS, +5156,"Police continue to investigate fatal shooting on Vermont & 14th Street +http://t.co/daACJ5Hfmq", +5158,11-Year-Old Boy Charged With Manslaughter of Toddler: Report: An 11-year-old boy has been charged with manslaughter over the fatal sh..., +5161,@noobde @comingsoonnet YEEESSSS!!!! I will buy him twice!! No KP2 jus release kombatants stages alternate kostumes and fatalities at will, +5163,Las Vegas in top 5 cities for red-light running fatalities - News3LV http://t.co/rEt82MdQnG, +5167,Highway Patrol reports uptick in statewide pedestrian fatalities - Post and Courier http://t.co/xccv3D9IcR, +5173,"http://t.co/pcdub1gQoC + +#Figureskate #ToshikazuKatayama [Earthquake Report]@OnlyQuakeNews Get all the headlines of our earthquake-report.___", +5205,Wow 3 are WSEA RT @KING5Seattle: Seattle's deadliest red light runner intersections revealed http://t.co/zqUsy2k4ws http://t.co/OX6dHXEUwe, +5214,Any other generation this would've been fatality http://t.co/zcCtZM9f0o, +5215,@martinsymiguel @FilipeCoelho92 FATALITY, +5216,@noobde this monkey will be a good character in mkx lol banana fatality, +5217,Flourishing sleigh bell an toothsome transit fatality: oNrhPz http://t.co/gB4W3T7b3T, +5225,@Blawnndee FATALITY!!!, +5237,???? #mortalkombat #mortalkombatx #mkx #fanart #fatality ?Follow for more? #Gaming #Pictures #Youtube___ https://t.co/LQCqjUNwiR, +5239,'come from south like ya name asher d' big h does come with some fatality bars in his minor lotm 'battle', +5240,Don't let your #writing suffer a fatality! Learn how to seek and destroy bad writing--weekly on Live Write Thrive: http://t.co/VVuL9eGPe8, +5241,..... FATALITY, +5246,'The best way to treat #addiction...is as a #publichealth problem.' http://t.co/Ojt5aXP1OG, +5250,. @paulrogers002 Many #cancers which had a very high fatality rate 30 years ago kill a lot less today. #Oncology has achieved a lot., +5253,@GodHunt_sltv FATALITY, +5258,FAVORITE FATALITY!!after she tore her enemy's face she took a selfie & posted on kinda-Mortal Kombat's Facebook. LOL http://t.co/GLyNizSnAK, +5261,@Dat_Vi fear of the unknown eh., +5268,'Bear Markets Are Driven By Fear And Fear Has Taken Over Many European Markets. Don't ... - http://t.co/MlaxoJ1kCJ http://t.co/yGfgndIBpo, +5275,"Fleshgod Apocalypse - Blinded by Fear (mini drum cover) https://t.co/2PxafIHTf7 +this cover is awesome www", +5277,"...@jeremycorbyn must be willing to fight and 2 call a spade a spade. Other wise very savvy piece by @OwenJones84 +http://t.co/fabsyxQlQI", +5279,Trying to conquer my fear of flying by pulling massive G's with this flight simulator. #OculusRift #VirtualReality http://t.co/rHFjkUQ7zw, +5288,@skydragonforce ^^why i usually never try to express opinions for fear of the hate ._., +5289,Show no love and fear no ops, +5294,FEAR GOD MORE NAJIB LESS: Nazir taking another stab at brother as govt threats police crackdowns___ http://t.co/bQT9bjdUJG #WayneRooney_INA, +5303,My grandma had always fear and in some way she was right - @3nach9 #Tatort #HBwahl #Daesh @WESER_KURIER @HBBuergerschaft @Jerusalem_Post, +5307,the next chp is titled emmeryn I Live In Fear, +5311,HOLY FUCK SOMEONE SET ME ON FIRE https://t.co/GXnvqCGjQ2, +5312,Why is it that my pinky feels like it's lit on fire ? #freaky, +5314,@21Joseph fire, +5318,almost set the house on fire tryna make fucking hotdogs..., +5327,AMAZON KINDLE FIRE 7' FAST REPAIR SERVICE FOR POWER JACK USB CHARGING PORT - Full read by ___ http://t.co/2WHegYGS3k http://t.co/5b0BH8vPM6, +5329,I just got electrocuted by my #HP Chromebook charger it sparked and caught fire #HP http://t.co/UBywQHoaWr, +5333,Even with delays? Like a bogged down cpu? #gunsense @Minutemaniac @shieldmaidn @BigFatDave, +5334,I just watched Halt and Catch Fire 2x10 'Heaven is a Place' https://t.co/jcPxOrV3AU #trakt, +5340,@_NickSimboli_ u know what I'd say she's the equivalent of strawberries in this case I would say she's fire ????, +5348,@traplordthings Y din u fav my fire memez å_!, +5358,Politifiact: Harry Reid's '30 Percent of Women Served' Planned Parenthood Claim Is a 'Pants on Fire' Lie: Call... http://t.co/wY4Xu1A9j4, +5360,When your likes your Fire IG selfie ????, +5361,Tractor-trailers collide on NJ Turnpike å_ 42 http://t.co/OTEXyh79J8 http://t.co/MLZG4l0xnL, +5363,News On 6 Crew Neighbors Put Out Hallett Truck Fire http://t.co/TwqVKZY6Dn http://t.co/eeyveTLAHW, +5364,What a night! Kampers go on vacation to end our Dream Job party where fire truck sprays the pool with a hose! http://t.co/if0BiyubC4, +5369,#Irvine #SigAlert update: 133 S now closed at Irvine Blvd. 133 N closed past the 5 b/c of trash truck fire. @KNX1070, +5370,Ltd Toy Stamp & Vintage Tonka TFD No 5 Pressed Steel Pumper Fire Truck - Full read by eBay http://t.co/hTvyEnXCBS http://t.co/xSvPzxYRe3, +5374,News On 6 Crew Neighbors Put Out Hallett Truck Fire http://t.co/TwqVKZY6Dn http://t.co/QoInQ6gc9M, +5376,Photo: #NJTurnpike å_ NJ Turnpike Reopens Hours After Truck Fire In Linden; Driver Dead | NJ Turnpike... http://t.co/8SRT9rGaX7, +5388,@dmon2112 @C_T_Morgan but the fire rings of NYC permits I'd have to jump through for a food truck don't make it appealing, +5390,Last night was a ball! Atop a 40 ft. fire truck latter!#NNO2015 http://t.co/0KTxo7HLne, +5393,Fire shuts down part of NJ Turnpike å_ 96 http://t.co/UzeatfZAyA http://t.co/ejrGK69aeq, +5394,Mark your Calendars: Fire truck parade returns to Raleigh! http://t.co/QGCAqdfEKf, +5395,This dude just pulled in front of our production with his truck on fire. Cops pulled up after we put it out http://t.co/e7DRPQUW4G, +5398,Truck catches fire in Warwick: A tree service truck caught fire around 7:15 p.m. on Wednesday in Warwick on In... http://t.co/nAdAvcq9VA, +5399,Pickup Truck Fire On Driscoll Bridge Snags Garden State Parkway #NewYork http://t.co/Q5YU2tZGte, +5401,Langhorne-Middletown Fire Co. welcomes new truck ... http://t.co/KrpJo9zMSx, +5402,Truck driver died in Turnpike fire most lanes still closed police say http://t.co/tTmJgq6ayM via @njdotcom, +5405,Photo: xshanemichaelsx: instaxbooty: ?? @johnnyxphixxle ?? (via TumbleOn) Is that a fire truck ? I need to... http://t.co/WZ809L8zP5, +5414,'We must educate first responders about how to adapt to emergencies involving an #autistic individuals.' | http://t.co/JIeB3lsZqc, +5415,Central Indiana first responders are providing feedback at the third and final #FirstNet focus group today.... http://t.co/pehH7wHZ0g, +5419,[News] Saint Louis University lowers tuition for first responders http://t.co/VJRHkmHpu2, +5421,Community first responders #CFR The Radio Ham: Tony Hancock http://t.co/MMSoOOOa70, +5422,First responders tell me he is lucky.... had someone riding with him today. http://t.co/jc16CZn0NC, +5424,"Good Day Columbia: First Responders Memorial with Sallie ... +#MEMORIAL_DAY #MEMORIALDAY + http://t.co/qW3uNfV8Kb http://t.co/hgxMBmrDa3", +5427,'An alarming trend among first responders is raising concerns for local fire departments in Colorado and around... http://t.co/Iqr6v6DTHr, +5429,What I'll miss the most is that very rare occasion where you and the first responders on scene are the difference between life and death., +5431,Volunteers served first responders today during a tanker fire in Eufaula. Provided cold drinks and food. #okwx #tankerfire, +5433,Attention Service Members Veterans Educators First Responders in Jacksonville FL http://t.co/4UrtBEAcE5, +5434,'The longer you wait the more people die': #Lafayette theater shooting first responders relied on training: http://t.co/yXOojnbr7m, +5435,@pnppro1 Warm Up exercise of the trainees on Basic First Responders Training Course. http://t.co/vdbj2o0IYX, +5436,.@MLSudders on social workers: 'I would like us to think of them as first responders and give them the same level of respect', +5438,"SPD OFFICER SHOT - PLEASE PRAY + +Please pray for the Shreveport Police Department and other first responders... http://t.co/Kx1YnSME5T", +5440,Pray for our Nation & her troops that defend in her. May he continue his watch over those watch over us our first responders., +5441,Red Cross arrived on the scene around 9:30pm to canteen for first Responders fighting this massive blaze. Go... http://t.co/IDY9KsCaVV, +5447,Allergan Kicks off REFRESH AMERICA Program to Supply First Responders With Free Eye Drops http://t.co/0OjKuLqO5I http://t.co/wkmcPmCtgu, +5451,Please pray for employees residents and first responders. #myhometown #livesmatter https://t.co/tdeS8MEAR2, +5453,Every time I get on the news I see more sadness and needless death. To all my first responders still on shift tonight go home safe. ??????????, +5456,@AlJavieera The public should always have the ability to share information w the police & first responders #publicsafetyfirst, +5457,First Responders needed for a Cedar City community event Aug 7/8. Contact Terri Linford at 435-327-1148. Meeting tonight., +5463,flames everywhere ???? https://t.co/cG9GAkbPta, +5464,more new flames for that next fire under the pot line up flames right here right now let em knw u wit em flames!!!, +5470,the fires of hell for Julie #Extant writers....she better go down in flames when this is all over...#Extant, +5472,@xodeemorgss she went to get rice and the whole thing tipped over so he had like flames on his pants and she was like omg I'm so sorry!??, +5474,@ClockworkHeart When I read it this morning I had a Mrs White moment ('Flames on the side of my face...') GAH., +5478,*NEW* Snap On Tools Black baseball Hat/Cap Silver/Gray Embroidered S Logo Flames - Full re___ http://t.co/F30F9V0aSI http://t.co/5B5a1HDzF6, +5483,I only fuck you when it's half past five, +5490,see the flames inside my eyes, +5500,You were just waiting for us to go down in flames, +5504,This is gonna go down in flames ?? https://t.co/pjOG1LcYYV, +5508,I'm listening to 'Zion' by Flames on #Pandora http://t.co/mP0tmdzyIE, +5511,I think that's been the best bit of this match. Kurt Zouma absolutely flattened that bloke., +5512,why would anyone want to hear some type of shin cup noodle flattened by a rolling pin sing https://t.co/IuEm2g1Nzp, +5513,@jackcarter22 me and bricktop were laughing about him on sunday when he flattened 3 or 4 arsenal players from a corner ???? #tank #VaVaZouma, +5516,Unlike the Donaldson dive into the stands no young boys were flattened in the Kyle Parker catch., +5523,@johndavidblake Winds me up to see 95% 'whites' held up as 'black' as if just a trace of flattened nose or curly hair is impure., +5527,Free Ebay Sniping RT? http://t.co/6KGMVkbPLR Hot 80pcs Tibetan Silver Round Flattened Spacer Beads Jewelry Finding 5.5x1. ?Please Fa, +5530,@st3ph007 do you think the pedagogy/practice is more important than the environment? #BFC630NZ walls can be 'flattened' in many ways, +5533,Zouma! Runaway train. Absolutely flattened the Fiorentina player there..., +5536,"Lustig gets flattened...he should have stayed down longer.. + +Commons coming on . my guess is GMS coming off... + +water break! 0-0 76mins", +5537,100 1' MIX NEW FLAT DOUBLE SIDED LINERLESS BOTTLE CAPS YOU CHOOSE MIX FLATTENED - Full re___ http://t.co/61fALvOCuK http://t.co/1MuTpFcgDL, +5542,Zouma flattened that breh, +5543,Flattened via IDEAS IN FOOD - A cheesecake brownie does not always have the aesthetic we are ... http://t.co/oRSmW5Airj, +5546,David Ortiz just absolutely flattened that baseball??, +5549,@FollowerOfDole 'Give me your lunch money ner-' *flattened by falling quarter*, +5550,I see a flattened Cyberman. Do I watch way too much Doctor Who??? http://t.co/XMKVhW2RS0, +5551,The satisfaction of knowing you flattened someone with logic and reasoned argument ??, +5562,Spot Flood Combo 53inch 300W Curved Cree LED Work Light Bar 4X4 Offroad Fog Lamp - Full re___ http://t.co/E6BnWxFtkb http://t.co/Ww2uVHRy89, +5566,Hey @FLOOD_magazine send me to #HOTCHIP @hot_chip @greektheatrela, +5568,If you decide to get rid of #flood damaged #furniture solid waste will pick it up for a fee. You can call #Tampa Solid Waste at 813-274-8811, +5571,2 NNW Hana [Maui Co HI] COUNTY OFFICIAL reports COASTAL FLOOD at 5 Aug 10:00 AM HST -- WAIANAPANAPA STATE PARK CLOSED DUE TO LARGE SURF. ___, +5572,Flood Advisory issued August 05 at 4:35PM EDT by NWS http://t.co/fuZ7y44P4I #WxKY, +5574,2pcs 18W CREE Led Work Light Offroad Lamp Car Truck Boat Mining 4WD FLOOD BEAM - Full rea___ http://t.co/0AIV5KhZjv http://t.co/q6HHZbFxCv, +5583,12' 72W CREE LED Work Light Bar Alloy Spot Flood Combo Diving Offroad 4WD Boat - Full read___ http://t.co/o7x8oDiBqU http://t.co/jWqxKwSOLL, +5585,2pcs 18W CREE Led Work Light Offroad Lamp Car Truck Boat Mining 4WD FLOOD BEAM - Full rea___ http://t.co/yo9q6WxweU http://t.co/n581wQqyAS, +5591,How did I know as soon as I walked out of class that Calgary would flood again today, +5596,2pcs 18W CREE Led Work Light Offroad Lamp Car Truck Boat Mining 4WD FLOOD BEAM - Full rea___ http://t.co/Yrd3nPC9V0 http://t.co/AnRd0VIfwK, +5597,12' 72W CREE LED Work Light Bar Alloy Spot Flood Combo Diving Offroad 4WD Boat - Full read___ http://t.co/8Mk9TD4RRL http://t.co/W20rH3Ai9J, +5601,Myanmar Flood Victims Need More Relief Aid and Food: Regions struggle with flood aftermath and dwindling suppl... http://t.co/O5adXdNnII, +5606,"Flash Flood Watch in effect through 7:00am Thursday morning/12:00pm Thursday afternoon. +For: Perry Wayne Cape... http://t.co/fs7vro5seS", +5607,12' 72W CREE LED Work Light Bar Alloy Spot Flood Combo Diving Offroad 4WD Boat - Full read___ http://t.co/EBJz2MxLr3 http://t.co/jvVD8smJMw, +5610,Spot Flood Combo 53inch 300W Curved Cree LED Work Light Bar 4X4 Offroad Fog Lamp - Full re___ http://t.co/arR2CVbqJJ http://t.co/nahMndn8X1, +5615,@Schwarzenegger @FoxNews you won't because Dems are focused on flooding our borders with illegal immigrants to add to their voters. #EndofUS, +5616,Memorial unveiled for Travis County Deputy Jessica Hollis who was killed in Sept. flooding http://t.co/uSiN3M9kgI http://t.co/Mo5lmviPw5, +5623,Awh damn I just realized yall gone flooding my TL with school shit and Ima be like .. ??, +5630,#NYCHA says the pipe that burst and flooded apartments in a senior center is being repaired. http://t.co/eCyui3290H http://t.co/6S6sYUYypL, +5636,#BREAKING Business at the Sponge Docks washed out by rain flooding /#news, +5637,Nigeria warned against massiveå_flooding http://t.co/CofH4khFsD http://t.co/m0fLpPxIlg, +5639,"@caseyliss def - and don__ªt even fathom when there is bad weather / flooding ?? + +#byebyeroad", +5640,"Flooding updates from Pasco County + +http://t.co/gOSzRa5bdD", +5643,Donate to help Myanmar Flooding victims http://t.co/fuRVRES9Ks, +5648,Now WoW folks I'm sorry for all the 'prepared' jokes that must be flooding your feeds right now., +5649,@PopMech ANOTHER tweet double posted within 4 hrs. Flooding my timeline with the duplicate tweets., +5660,when youre tired as hell but your brain keeps flooding you with inspiration, +5666,OFA outfit floods net w paid OFA trolls &automated netbots that spam any story that is not complimentary of the One http://t.co/hNlSDLhmB9, +5667,Pasco officials impressed by drone video showing floods: Drone videos have given an up-close view of flooded areas___ http://t.co/PrUunEDids, +5678,"@asgharchohan86 @AdilGhumro +There are floods in kpk because you guys couldn't build dams in 30 years you ruled. +Even for your own Thar", +5679,"Who is bringing the tornadoes and floods. Who is bringing the climate change. God is after America He is plaguing her + +#FARRAKHAN #QUOTE", +5681,"If I wouldn't have lost all my guns in #Colorado floods I would say I could completely understand +#copolitics #2A https://t.co/37rHlsobbn", +5682,all cats are in....time for the floods......., +5691,VIDEO Facing Up To Persecution ___ Pastors Silenced ___ MB Obama holds Iraqi Christian asylum___ https://t.co/qrtUzfHOdw http://t.co/xBNRPS8xHO, +5696,Typhoon Soudelor approaches after 7 killed 2 missing in floods in Philippines http://t.co/hALJNnWrwi via @abc7chicago, +5697,Flash floods landslides possible as Hanna draws in more monsoon rains - Pagasa http://t.co/TzWmsEQPHW | via News5 #iBalita, +5700,No Floods #MTVHottest Lady Gaga, +5701,I'm so dried out in this desert let me know I wanna be with you speaking of floods speaking of truth - Laleh ????????, +5702,Myanmar floods occur during low season for river cruising - Pandaw River Expeditions is currently operating only t... http://t.co/cQtTGrkwf6, +5708,#WorldVision is responding to floods in #India and #MyanmarFlood with more than 300000 affected: http://t.co/eU8jypIzsd, +5709,Bengal floods: CM Mamata Banerjee blames DVC BJP claims state failed to use relief funds: KOLKATA: Even as fl... http://t.co/WFwWRcmi9t, +5718,Grass fire near Forest Grove contained. Sparked by a rock being struck by a tractor. #LiveOnK2 @JackieKATU http://t.co/Yr9mmIS7jT, +5723,Damn dry desert forest fire...damn :( https://t.co/P5SoDrkVW3, +5725,#Winston #salem Fire in Pisgah National Forest grows to 375 acres http://t.co/5IMxNGsaSA, +5728,Crews responding to small brush fire burning in Tahoe Forest - My News 4 - KRNV Reno NV http://t.co/VhyJnVug6f, +5731,OMG there's a news chopper over a 3-acre mostly contained fire outside Forest Grove? #slownewsday ?, +5735,#GlobalWarming U.S. Forest Service says spending more than half of budget on fires: ___ United States the agen... http://t.co/mWiE5CeEIi, +5737,#Reuters #US Forest #Service says spending more than half of budget on fires http://t.co/XlpgKguhkG, +5738,+Cossacks+hussars oo hd riggd up rough shelters N th glade+were kindlng glowng fires N a hollow v th forest where th FrNch could not C th, +5742,Why can't they use ocean water to fight forest fires? http://t.co/NMWi5Sm3Uw, +5747,People come and go. Some are cigarette breaks others are forest fires., +5749,@gilmanrocks7 ya it was. Damn forest fires., +5750,@lldubs Are any of your vineyards affected by the forest fires in California?, +5755,I hope the rain stops the forest fires tbh we've been on lvl. 3 lockdown for water for the past month :/, +5756,Stubborn forest fires burn in two provinces |RZ| http://t.co/yZO948652k #Iran #IranElection, +5761,Worth the read --> As California fires rage the Forest Service sounds the alarm about sharply rising wildfire costs http://t.co/2LZtOMsAUz, +5762,A group of Florida Forest Service firefighters could be deployed to California to help contain fires. Details at 10! http://t.co/tRrig5J1Re, +5767,Property losses from northern CA wildfire nearly double http://t.co/fHqx7FiIVJ If you pray please remember CA people/forests/wildlife/water, +5768,'People come and go. Some are cigarette breaks others are forest fires.', +5770,Two New Bitterroot National Forest Fires are Largest for Forest So Far This Year http://t.co/zVa34qHU8M, +5773,Standing in mountaintop field scented with wildflowers dust and distant forest fires smell of impending rain in the air. #fieldworksmells, +5774,Q: Why do ducks have big flat feet? A: To stamp out forest fires. Q: Why do elephants have big flat feet? A: To stamp out flaming ducks., +5779,Firefighting consumes Forest Service budget sparks political clash http://t.co/ARN8qfgNpm, +5782,#RiverComplex #CA #CASHF http://t.co/37iYHPPNr4 River Complex: Six Rivers National Forest lightning fires ___ Morning Update, +5785,#LukeBox how about a song for your fellow Eagles?! ???? #hail southern, +5786,@froilan_canin https://t.co/caCXADLEb8 AFTERNOON DELIGHT Ha! Bounce filmed in between Calgary's Uber Hail Storms #yyc http://t.co/6zgkDpMMSK, +5787,BYZ issues Severe Thunderstorm Warning [wind: 60 MPH hail: 1.00 IN] for Big Horn Yellowstone [MT] till Aug 5 8:00 PM MDT ___, +5788,Hvy rain wind sm hail Tree down on Bluff Ridge Rd. Multiple trees down o on house on South Shades Crest Rd. (Past Lake Drive) @spann, +5796,Seen on Fahlo:#WCW All Hail the Queenå_?? http://t.co/zrhXy9iBno http://t.co/Opjz8fGdIy, +5802,Which version of All Hail Shadow you do you like more Magna-Fi's or Crush 40's? ___ Crush 40s. http://t.co/oGs6d0p3RB, +5806,Severe Thunderstorm 7 Miles North of Arcadia Moving SE At 20 MPH. 60 MPH Wind Gusts and Large Hail. Hail... #okwx http://t.co/WEd69e4KRg, +5809,Once I get my stomach together I will be wearing crop tops during all seasons. Rain or shine. Snow sleet or hail., +5810,Something to think about, +5811,Severe Thunderstorm Warning until 08:00 PM local for Custer Fall River & Shannon Counties in SD. 60 Mph Wind Gusts And Penny Size Hail. #sd, +5813,@adolwyn This near U? @JWagstaffe: RT @OldsFireDept: Aug 5 2015 Hwy27 west of Olds. Approximately 1 foot of hail. http://t.co/Yg0pd73Bpw', +5820,Created save #666 on my current Fallout 3 playthrough. Hail Satan!, +5823,i survived what you tried to destroy you're not a man just a boy, +5830,New warning for Central Hills 1' hail 60 mph winds. NOT affecting Sturgis but could later tonight. #KOTAWeather http://t.co/E8oUxVKuTE, +5831,Hail Mary. ????, +5838,Twin Storms Blow Through Calgary * 75 http://t.co/sIAKlSbdiP http://t.co/fm44ZS93OA, +5839,We The Free Hailstorm Maxi http://t.co/Cnn1nXXNwc, +5844,Boston yesterday after an intense hailstorm. Photo by 'then0mads0ul'. http://t.co/uxgtXCArQ1, +5846,#yyc #hailstorm #christmas came early https://t.co/f0A2IIzx3A, +5851,Yesterday's #hailstorm! #boston #cambridge http://t.co/HbgYpruvO7 http://t.co/SwtgHLibs2, +5852,RICHIE: This was #Calgary yesterday after a crazy #hailstorm! pics via Calgary Herald http://t.co/1oH20qHk2l, +5857,Get out of the hailstorm and come down to @TheBrokenCity 2night at 9PM to catch a screening ft. @thebanffcentre! There will be BEER + FOOD!, +5859,It looks like all the trees in my yard had a Britney Spears style meltdown. #Bald #Hailstorm, +5860,Hailstorm flash flooding slam Calgary knocks out power to 20k customers http://t.co/sNpPqrwuXB, +5862,JPLocalFirst RT eliotschool: Yesterday's #hailstorm in #Boston. Wow. Thunder lightning. Artful awesome. http://t.co/YsvE1SEAko, +5865,#yyctraffic #yycstorm Caller says 'idiots' are stopping under overpasses and blocking 'all lanes on Crowchild' in (small-sized) hailstorm., +5866,here is goes again #abstorm #hailstorm #yychail stay safe everyone!, +5871,Hundreds of commuters abandoned at LRT station during severe hailstorm http://t.co/vMIGkJ0tkZ, +5872,Good olde fashioned prairie #hailstorm #calgaryweather https://t.co/hI1yOQFlfm, +5877,Hundreds of commuters abandoned at LRT station during severe hailstorm http://t.co/25wIeHMLZ5, +5878,Hailstorm cleanup begins in Calgary http://t.co/DhH5jz2w49, +5879,600 passengers abandoned at LRT station during Tuesday's hailstorm http://t.co/eLZoyEqA3m #yyc #yycstorm #abstorm, +5880,@Haley_Whaley Hailstorm Clash ofClans Gems Giveaway is out now! visit my Profile Bio to get guide on how to get 200.00 Gems, +5895,Mental health is real..?? Self harm is real..??Depression is real..?? feeling alone..?? no where to fit in..?? one day it could be you???, +5897,@ShaunKing HARSH PENALTIES FOR LEGAL FIREARM OWNERS WHO STRAW PURCHASE A FIREARM FOR ANY FELON. GANG MEMBERS ONLY INTEND TO CAUSE HARM!, +5910,Anger is an acid that can do more harm to the vessel in which it is stored than to anything on which it is poured ~http://t.co/UTiGx9PSry, +5914,Politicians are using false allegations to attack #PlannedParenthood & harm women. We aren't fooled we #StandwithPP http://t.co/JhseGQLbYq, +5915,What is up with #Bears beat writers ___warning___ fans not to get excited? What actual harm does it do to be excited? Shut the hell up!, +5917,Never let the experiences of your past bring harm to your future. Remember your past can__ªt be altered and your... http://t.co/l9cy29sb1t, +5924,@Norcinu @TheBodyShopAust @thomaldo tiny plastic beads used in cosmetics (scrubs toothpaste) doing great harm to aquatic ecosystems., +5926,Politicians are using false allegations to attack #PlannedParenthood & harm women. We aren't fooled we #StandwithPP http://t.co/bIE5pfR2Ac, +5931,Father we come 2u & lift up this nation America as well as other countries 2 keep it calm & deliver us from harm in Jesus name??Amen #12K??, +5936,Ibrahimovic & Hazard Boot Test: Nike Mercurial Vapor X Test - footballfreestyle24/7: http://t.co/BZUnEPsuLP via @YouTube, +5940,"Whoever got compared with Hazard faded away one by one. +Kagawa +Januzaj +Di Maria +Who's next?", +5941,There will be a 3-match ban for any player caught diving this season. Looks like Hazard Costa and Young will miss half the season., +5945,@mishavelgos @MattBacal8 @ComplexMag this is so accurate I can't even speak haha... Comparing hazard to harden is so true, +5946,@feroze17 @WarAndSetPiece No one's said Depay's gonna overtake Hazard......, +5948,"NVIDIA recalls some Shield Tablets because of possible fire hazard + +http://t.co/M3nSIx073I + +#nvidia", +5951,OEM 2001 Volvo XC70 Center Dashboard Hazard Lamps Control Switch w/Vent Panel http://t.co/VXixYcbbtH http://t.co/HSDonZ49hd, +5967,"@fplhints hazard depay ozil Ritchie . +Should I go with 3 crazy strikers or 4 midfielders ? #fplboss", +5968,Parents Beware: A Household Hazard that Won't Go Away http://t.co/91eiTHgvDB, +5969,Con el Crack de Hazzel Hazard?????? @ Saint John's Episcopal School https://t.co/iQ0iWYPizd, +5970,"'HOLD ON' +Batman in Hazard suit. http://t.co/fSPLv8Inwr", +5971,Choking Hazard Prompts Recall Of Kraft Cheese Singles http://t.co/98nOsYzu58, +5973,Eden Hazard John Terry and Sergio Aguero. An honourable mention for Nemanja Matic. https://t.co/4nzj9qoOay, +5976,@Hazard_VN @AccuracyEsports add me, +5977,@SuperBobbyPires Hazard Ronaldo Cantona Pepe and all those who lashed out before him., +5984,Road Hazard @ E CONFEDERATE AVE SE / MORELAND AVE SE http://t.co/tym6tYmh4M, +5986,#ine #instyle Olap #world pres: http://t.co/PJaY74M2Fi How To Recognize A Hazardous Waste And The Multidimensi http://t.co/vr3wlQXl0G, +5993,#Hazardous #Waste #Treatment #Disposal : Aug 2015 market growth feeble. http://t.co/yyt4KPtFh8 #MarketResearch #Statistics, +5994,#alfa #forecast Olap #world pres: http://t.co/YiFXKRQx0C How To Recognize A Hazardous Waste And The Multidimen http://t.co/HO4b3L0KI8, +6005,@moneymaker32931 tanker truck overturned carrying hazardous materials., +6006,CHS issues Hazardous Weather Outlook (HWO) http://t.co/lbkiyfwFlU #WX, +6008,#diablo #dsp Olap #world pres: http://t.co/LFEtNRXyWt How To Recognize A Hazardous Waste And The Multidimensio http://t.co/PvNvqaXsA7, +6010,TSA issues Hazardous Weather Outlook (HWO) http://t.co/7vlu8ni90Q, +6011,@megynkelly illegals don't abide by the uniform building codes or OSHA safety codes.structur's they build are hazardous to your health., +6014,Also confirmed by Senate Marie Therese Dominguez to be Administrator of the Pipelines and Hazardous Materials Safety Administration (PHMSA)., +6016,#hot #fiat Olap #world pres: http://t.co/QddYwTynvF How To Recognize A Hazardous Waste And The Multidimensiona http://t.co/dzQGotozNA, +6018,50k plays on our 'Delirious' remix!! Happy it's getting so much love ??, +6021,#triangle #peter Olap #world pres: http://t.co/JeHTQAqDFh How To Recognize A Hazardous Waste And The Multidime http://t.co/gwXdjd8nGu, +6025,SH 29 Kaimai Ranges - Eastbound. Hazardous driving conditions. Due To A Truck Breakdown... http://t.co/0cuCMZCmaO via @AA_Traffic, +6028,Drums of Hazardous Waste Dumped Around Kodiak Island: Someone has dumped drums of hazardous was... http://t.co/ZSnnrULaWo >@alaskapublic, +6029,Skinny Jeans are Hazardous for Your Health! http://t.co/TVdvP7VdAs, +6035,"I added some dumb ideas to beat the #summer heat cheaply + http://t.co/jatxpAFvA9 @brokelyn #brokelynati", +6040,New Post: Diff Durden - Heat Wave - CA - http://t.co/jGjgR4o5K0 Vote On Blog #DHHJ #HipHop #Chicago #NP http://t.co/1W10uKVHoo, +6042,@CrawliesWithCri @Rob0Sullivan What do they consider 'heat'? When I lived in London they thought a 'heat wave' was like 25C hahaha, +6044,Never really seen olive leaf extract to help with a rash but good luck! @IngeScott humid heat wave and flared up. I take olive oli extract, +6046,Yeah it actually is this whole week we in a heat wave :( https://t.co/cB2NWtGy47, +6052,It's nice out. Guessing the heat wave is over., +6054,Activists Protest Pipelines in Canada and Shell in Portland Oh and Prepare for Another Heat Wave http://t.co/aKsYPKjdpB #CampLogistics, +6062,Heat Wave: A Social Autopsy of Disaster in Chicago by Klinenberg Eric http://t.co/fPtqxVokJC http://t.co/0gt63uQGcU, +6063,@sportguyshow the heat wave ends really tomorrow but still time for another one to build. Only other thing that ends the heat is Autumn., +6067,Japan heat wave intensifies - death toll surges to 55: http://t.co/F1h1mzVSNn via @extinctionp, +6068,Middle East ___Heat Dome___ Causes Crippling Heat Wave in Israel http://t.co/41Lcb9aepR, +6077,@facilitydude shares how to handle the heat and keep your cool http://t.co/ekEd6BulsZ #facilitiesmanagement, +6079,God the past few nights have been awesome but tonight it looks like the heat wave is back ?? can't breathe in my room!, +6080,http://t.co/OYMLNLHEca So important to stay hydrated in the midst of this heat wave___ ?? ___ http://t.co/EXLQxSa0i8 http://t.co/M3TiNl4NUX, +6081,Good morning. Slept about 6 hours. The heat wave hits Japan... The hot days still continue. I guess some people were killed by this climate., +6082,Japan heat wave intensifies ___ death toll surges to 55 http://t.co/ZwPoDn3KOH, +6089,@WalpurgisNightZ I was thinking super glue since its not as messy., +6095,I'm melting a bar of chocolate under my laptop at least this fucking HELLFIRE is good for something, +6101,Hellfire is surrounded by desires so be careful and don__ªt let your desires control you! #Afterlife, +6107,@spikepoint @skie It's about context too. If his dick slipped out during the super bowl people would be throwing hellfire here., +6114,Hellfire! We don__ªt even want to think about it or mention it so let__ªs not do anything that leads to it!, +6117,Would you consider yourself good at giving advice? ___ I like to think I give good advice when I feel like its a s... http://t.co/lSMr4KVO6u, +6121,@emmap645 @Vickygeex @Zak_Bagans @NickGroff_ @AaronGoodwin A few friends of mine were up investigatin the hellfire last year they had loads, +6122,#Allah describes piling up #wealth thinking it would last #forever as the description of the people of #Hellfire in Surah Humaza. #Reflect, +6124,Come watch @QelricDK bash some face in Hellfire Citadel! http://t.co/juQIuN2mhg, +6129,"*standing in line at JoAnn's little girl and her mom behind me* + +Little girl: Mommy is that a boy or a girl? + +... +Welp www", +6131,The Prophet (peace be upon him) said 'Save yourself from Hellfire even if it is by giving half a date in charity.', +6136,"APC needs to watch all these PDP defecting to APC + +Patience Jonathan On The Move To Hijack APC In Bayelsa State http://t.co/zzMwIebuci", +6139,Himika Amase is the only prophet in the world who foresaw 'hijack in March' as it was on her Jps Twitter in Feb 2014. malyasiaairlines. ???, +6142,Biggest security update in history coming up: Google patches Android hijack bug Stagefright http://t.co/bFoZaptqCo, +6143,RT! Tension In Bayelsa As Patience Jonathan Plans To Hijack APC PDP http://t.co/JlE6aFuG3p, +6144,Tension In Bayelsa As Patience Jonathan Plans To Hijack APC PDP - The Sun: Plans by former___ http://t.co/qJPrrObGtC http://t.co/XWoGt4wBf7, +6149,Biggest security update in history coming up: Google patches Android hijack bug Stagefright http://t.co/LTid39bKBE, +6155,Tension In Bayelsa As Patience Jonathan Plans To Hijack APC PDP - The Sun http://t.co/VBiGAnIRSJ, +6157,Chrysler Jeep wirelessly hacked over internet to hijack steering brakes and transmission face lawsuit - should all consider #ISO27001, +6158,@daehyvn he spends the rest of the segment trying to hijack someone else's bike, +6161,Swansea ?plot hijack transfer move for Southampton target Virgil van Dijk? http://t.co/UOq87SP5JP, +6168,hijack, +6172,Swansea ?plot hijack transfer move for Southampton target Virgil van Dijk? http://t.co/DNZjWwTAkI, +6176,Swansea ?plot hijack transfer move for Southampton target Virgil van Dijk? http://t.co/PYL26NFEn8, +6177,"Graham Phillips' Fundraiser Canceled By @JustGiving + +#fundraise #Ukraine #donbas + +http://t.co/HIbEf3CXOX http://t.co/9crFKQzD52", +6179,Bayelsa poll: Plans by Patience Jonathan to hijack APC tickens http://t.co/eyn0E8pmJq, +6180,Criminals Who Hijack Lorries And Buses Arrested In Enugu (PHOTO) http://t.co/5cZ7eM9OTr, +6182,@spencer_ellmore @CentreTransfer just wait Norwich might hijack the deal??????, +6186,Remove the http://t.co/J2I27K6tWt and Linkury Browser Hijacker http://t.co/D8TgE45FnU http://t.co/GRG1NsJzbk, +6189,"Remove the http://t.co/zmoKZZf4qp and Linkury Browser Hijacker. PITA as it hijacks both browsers and their shortcuts. +http://t.co/QWDkzTi97g", +6190,Remove http://t.co/77b2rNRTt7 Browser Hijacker - How t... http://t.co/Wff7X3B6KT, +6194,Governor allows parole for California school bus hijacker http://t.co/0aQBZmrvLQ via @abc7newsbayarea, +6204,"and if I cry two tears for her.. +That will be the most that I would give___ https://t.co/Ch24hn74XH", +6205,Remove http://t.co/GYcBuXfYGP Browser Hijack... http://t.co/FuQH0yKsdg, +6209,I think parents r shit these days=only coz I don't have kids=but to let ur child pretend to be a hijacker only drives them to do it laters, +6210,Remove the http://t.co/OP2NObtoul and Linkury Browser Hijacker http://t.co/JjKlf2PlX7 http://t.co/t3OQgw0Ce7, +6212,@JagexSupport can u remove the email of the hijacker pls !! YKJL is my ign. i need to recover but pls block so they dont break bank pin!, +6214,Governor Allows Parole for School Bus Hijacker http://t.co/7mqh7fhxYL http://t.co/BNCcwcsCKt #SahelNews, +6221,"Updated on Friday at 4:30 A.M. - Daily Area News +California School Bus Hijacker Parole Stands http://t.co/24QEkAh893", +6225,Governor Allows Parole for School Bus Hijacker http://t.co/i7PIJK6XtM liveleakfun ? http://t.co/IONWArVRFy, +6228,"@mch2601 The AVI is h/t @RealDB4Prez16 +He provided me the artwork and years & years of great hijacker stories.", +6229,Remove the http://t.co/Ytao0lT144 and Linkury Browser Hijacker http://t.co/uxEKjeuIVp http://t.co/abvvRLFgDh, +6231,The footage of Albert Reynolds talking about a hijacker demanding the publication of the 3rd secret of Fatima is hilarious., +6235,Earnest Pletch__ªs cold-blooded killing of Carl Bivens was just one chapter in his strange life http://t.co/dnQn1OWWF9 http://t.co/xNEPq5HBYD, +6236,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/EZ1TpCzJzo #prebreak #best, +6237,The Murderous Story Of America__ªs First Hijacking http://t.co/VgBOpxb6Wg, +6238,Oops! I say he picked the right car to try hijacking. https://t.co/SoMgI7hwli, +6239,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/EEpUFlxNvI #prebreak #best, +6242,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/a0zgzd1fYa #prebreak #best, +6249,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/RaPaDCea1w #prebreak #best, +6250,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/ChGipZWxsx #prebreak #best, +6251,Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/GUDtEFt5HP http://t.co/n3dDxcTO2A, +6252,The Murderous Story Of America__ªs First Hijacking http://t.co/QAOqtptgwH, +6260,@csgpelmash we are hijacking a great song from a terrible 'concept video'. So THERE! #eatshit, +6264,MomentsAtHill everyone hijacking NuNus and timas bikes ????, +6266,But is feast.dll still vulnerable to DLL hijacking? https://t.co/3vaig1PtYk https://t.co/BZDNKpOry1, +6270,@ReasonVsFear 'One?' Oh nonononono. This hijacking's been going on forever but I've seen it about a dozen diff places today alone., +6275,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/f6PCzLWBWk #prebreak #best, +6279,At #BlackHat? Learn about account hijacking attacks & cloud access security w/ @ElasticaInc: http://t.co/PqXryXVp14 http://t.co/Wt5wZwvpUt, +6280,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/XU9u3NNxpK #prebreak #best, +6284,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/wG1rt0kB4g #prebreak #best, +6285,Who is Tomislav Salopek the Islamic State's Most Recent Hostage? http://t.co/puT3LgsDnf, +6286,Chris Brown's aunft was 'held hostage in a closet by armed intruders at his LA home while sin http://t.co/i1Fhb8QSZ7 http://t.co/7C5BG48BRJ, +6287,Islamic State threatens to kill another hostage: @DailyGleaner @TJProvincial @TJGreaterSJ http://t.co/dfPLiqjMk2, +6288,Ya kisses hold me hostage & I don't wanna stop it. I only wanna give it to you, +6289,Pakistani Terrorist Was Captured By Villagers He Took Hostage - NDTV http://t.co/C5X10JAkGE, +6290,"Oh god Diane is following them to the hostage exchange in a taxi. +#RandomActsOfRomance", +6291,Sinai branch of Islamic State threatens to execute Croatian hostage in 48 hours http://t.co/YvtcXrPt34, +6295,Concealed Carrier Rescues Hostage Captures Naked Attacker http://t.co/uLy5hM6PqR, +6298,UPDATE 4-IS Egypt affiliate threatens to kill Croatian hostage in 48å_hours http://t.co/C0zrqIk6KE, +6307,'IS Egypt affiliate threatens to kill Croatian hostage in 48 hours' - http://t.co/3LcgidRQpb, +6308,my chacos were supposed to arrive by 8pm why are you holding them hostage @ups I just want some crunchysensible footwear, +6309,Islamic State group threatens to kill hostage if 'Muslim women' aren't let go - Press Herald http://t.co/yKM6qfRh98, +6313,Bit-Defender hack ___ Held Hostage!: Late last week it was discovered that antivirus vendor ... http://t.co/2vC8CSTWy5 #damballa #infosec, +6316,#BREAKING. Who is Tomislav Salopek the Islamic State's Most Recent Hostage? /#news, +6319,#TeamNyle because he's holding a family member hostage and I have to post this or the 'else' happens, +6321,"@theCHIVE For the record I held this shirt hostage for nearly a year before sending it last minute to TX #BidTime + +http://t.co/edbxYatmaq", +6324,@bluecurls8 @Worthfull1 You basically decide to do absolutely nothing else for months and then have your Beta hold chapters hostage., +6327,While being held hostage by Tobias Hankel, +6333,video says #ISIS Threatens To Kill Croatian Hostage 5oil-worker) If #Egypt Does Not Release Female Prisoners in 48 H http://t.co/GvpKrRdQnu, +6351,Militant 'Overpowered' By Hostages After Attack http://t.co/xWnp8a1JOn, +6352,#hot C-130 specially modified to land in a stadium and rescue hostages in Iran in 1980 http://t.co/g601NBD7wS #prebreak #best, +6353,How until put out manorial shaped gnocchi near thine hostages to fortune: RBpK, +6357,#hot C-130 specially modified to land in a stadium and rescue hostages in Iran in 1980 http://t.co/PtAI4zBpbI #prebreak #best, +6359,@JoeNBC IRAN: NO SNCTIONS INCL MILTARY$150BILNO INSPCTKPS HOSTAGESTHROSW ISRAEL TO GUTRKPS NUKE SITES U.S HLPS W/NUKES GET 'ZERO!', +6360,Ajmal Kasab II: How hostages misled the terrorist got him caught alive http://t.co/n1up959OFM, +6364,One Year On From #IS Massacre in #Sinjar Yazidis Blast Lack of Action over 3000--5000 Yazidis who remain enslaved: http://t.co/vnSn284ovw, +6367,@IndiaToday @Iamtssudhir Time for India to join global war against ISIS cant sacrifice national interest to save hostages! #NationFirst, +6368,India: Militant 'Overpowered' By Hostages After Attack http://t.co/mkjszGbaWR, +6377,Sinjar Massacre Yazidis Blast Lack of Action Over Hostages http://t.co/fdU8aCnC2W #denver #billings #rapidcity #seattle #cheyenne #lasvegas, +6378,Sinjar Massacre Yazidis Blast Lack of Action Over Hostages http://t.co/fdU8aCnC2W #denver #billings #rapidcity #seattle #cheyenne #lasvegas, +6379,@Deosl86 That does nit change the fact extracting hostages is pointless, +6381,RT WatchmanIS216: #Sinjar Massacre #Yazidis Blast Lack of Action Over Hostages http://t.co/79iQDYZjBS; Portland #___ http://t.co/0z1PvJVdpf, +6390,AngelRiveraLib___ #Snowden 'may have' broken laws? Hurricane Katrina may have caused some damage. http://t.co/jAaWuiOvdc Without Snowden h___, +6401,@al_thegoon i like hurricane habor next to slix flags but they got like 8 rides but they wavy & hershey park, +6408,Residents artists remember aftermath of Hurricane Katrina through artwork - WDSU New Orleans: WDSU New Orlean... http://t.co/U6pAvbZoB3, +6409,Found out a hurricane is brewing for thursday lol ??????, +6411,@ReinhardBonnke is preaching on the Holy Spirit! Holy Spirit blows atmosphere to pieces! Atmosphere is air in a jar. The HS is a hurricane!, +6419,"Would you blow me kisses +If I kept my distance? +Would you send a hurricane +As proof of your existence?", +6424,HURRICANE GUILLERMO LIVE NOAA TRACKING / LOOPING WED.AUG.5TH ~ http://t.co/QktRt2J1o8 ~ http://t.co/b0JxS5QKkh http://t.co/KP05TOwSfM, +6426,@leeranaldo expressing his rage about Hurricane Sandy #LCOutOfDoors http://t.co/IwPR2qGYsI, +6428,@BPGVII_GOAT thanks bro ????, +6431,Happy birthday bro ?? @Hurricane_Dolce, +6432,Brooklyn Sites Get $2.6 Million to Undo Hurricane Sandy__ªs Toll http://t.co/HukwD0RE2a, +6433,I'm a hurricane., +6435,Udhampur terror attack: Militants attack police post 2 SPOs injured: Suspected militants tonight attacked a p... http://t.co/nZAFokxX0h, +6437,I'm thinking maybe just wait til after all the preseason games just in case somebody gets injured., +6439,4 dead dozens injured in Gaza blast near house leveled in summer war - Washington Post: New York Times4 dead... http://t.co/7C2qhMwtv5, +6442,@NewsBreaker @sothathappened SO wheres the story? Anybody injured? Where did this happen?, +6444,#???? #?? #??? #??? Udhampur terror attack: Militants attack police post 2 SPOs injured - Times of http://t.co/1KxsGlsTA7, +6447,Man injured in Radcliff shooting http://t.co/3iYXOoaCW6, +6456,Harper Woods policeman injured in fight after dead man found in garage: Ronnie Karjo 46 of Macomb Township http://t.co/tUoTICilBi, +6457,"#Redskins WR DeSean Jackson injures right shoulder at training camp today. Watch the play HERE: http://t.co/GABecyzBzJ +@wusa9 #SkinsOn9", +6460,I made him fall and hegot injured https://t.co/XX8RO94fBC, +6468,@LoganMeadows11 @ChristianStec_ he's weak sauce he got injured by a sled, +6471,Terrorists open fire at Udhampur police post 2 SPO's injured firing underway - See more at:... http://t.co/7jmGwOgqnh, +6475,@Braves_Ninja @david_brelsford For Freddie before being injured this year it was a great deal, +6476,RT- Udhampur terror attack: Militants attack police post 2 SPOs injured: Suspected militants tonight at... http://t.co/wN414hp9hZ #News, +6479,@TOI_India Udhampur terror attack: Militants attack police post 2 SPOs injured: Suspected militants tonight a... http://t.co/EWCkcfBI28, +6483,Neelum Valley teen dies in 'dud shell' explosion: MUZAFFARABAD: A teenage boy was killed and his peer was injured___ http://t.co/GuRdZEPMhu, +6487,@wral there are now 5 fire trucks and 2 ambulances at Wellington ridge lp in Cary. one of the units is on fire. not sure on injuries, +6488,@EdmontonEsks SB Adarius Bowman says team leadership and depth have helped through some key injuries https://t.co/hCnmZsfNcI #yeg, +6489,4 Common Running Injuries and How to Avoid Them http://t.co/26HWLpWmXr, +6496,8/5 Arizona Cardinals news: Jaron Brown makes headlines minor injuries continue to build http://t.co/QxKncvlrh1 http://t.co/KJnhNPMxGr, +6498,@RVacchianoNYDN @NYDNSports glad to see a career rebirth for him. Loved him in Carolina injuries just caught up to him., +6502,@nflnetwork @AZCardinals what's up with all the injuries.., +6503,@JaeSmoothCYB he was going to be. We still have our leading rusher from LY so we should be ight barring any injuries, +6507,San Francisco 0 I80 E **Hit and Run No Injuries** http://t.co/lp9TBu7KzH, +6508,cmon injury bug this shit aint funny we suffered enough injuries last year..., +6509,#BreakingNews Mva / Multiple Injuries - Saint Petersburg FL: E7 requesting R7 & 2 sunstar units for injuries___ http://t.co/0XRysEpQhL, +6517,@ringsau and injuries can account for elevated levels of tb4 Richard?, +6521,Let's be thankful most of these injuries are minor & it's still camp. Relax a little #birdgang. No one is cursed & no injury bug hit., +6522,@MichiganAutoLaw's @SteveGursten will discuss #autoaccidents & so-called 'mild' #TBIs on his @JusticeDotOrg webinar. http://t.co/ZUM3mluCYS, +6524,If Tyuler Collins could become Andy Dirks before the 58 injuries..that would be uiseful, +6526,I guess this is where I'm confused sites have week one pricing up alredy. We haven't even had injuries yet., +6531,No injuries reported after Green Line train derails on South Side http://t.co/nM7HuH3bjC, +6533,Wishing injuries on opposition players. I get rivalry but miss me with all that petty shit., +6538,Low self-esteem is a common cause of self injury affecting over 40% of young people personally or someone they know. https://t.co/xeLoapZp6R, +6539,CLEARED:incident with injury:I-495 inner loop Exit 31 - MD 97/Georgia Ave Silver Spring, +6544,incident with injury:I-495 inner loop Exit 31 - MD 97/Georgia Ave Silver Spring, +6557,Supermarket chains recording worst #injury rates among ASX100 companies #safety http://t.co/njiv1n6rYr, +6558,Ankle injury to keep Spurs__ª Marjanovic out of Eurobasket http://t.co/UBCggEHqyN #spurs #NBA, +6562,nflweek1picks: Michael Floyd's hand injury shouldn't devalue his fantasy stock: Michael Floyd's damaged digits won... ___, +6563,New post: Cowboys believe Lance Dunbar&#039;s ankle injury not serious http://t.co/XMCRedFXAt, +6564,@_keypsters full game? Injury cut is here https://t.co/YYqws4sO9J, +6574,Enter the world of extreme diving ___ 9 stories up and into the Volga River http://t.co/gptysALfKi, +6578,New post: Texans report: QB battle unaffected by Foster&#039;s injury http://t.co/IgH2PAAHnK, +6581,Ben Heenan carted off the field at @Colts training camp.. No word on how serious the injury is #colts #nfl, +6584,Michael Floyd out 4-6 weeks with a hand injury. #Cardinals http://t.co/bnMKlPZNiP, +6587,___We live in an area that__ªs inundated with wildlife...If people don__ªt learn how to get along or live with them... http://t.co/KZXV6DWQ6E, +6589,@eliistender10 @Splats although originally an April Fools jokethey're seriously considering making them after being inundated with requests, +6590,@TheBrooklynLife @Giants I just want to go to the game. Enjoy it. Not be harassed. Not be inundated with pink gear. That's all I ask!, +6592,@Ryanair I am waiting to hear about my claim for compensation. I have received an email saying you are inundated so there will be a delay??, +6595,Mexican beaches inundated with algae http://t.co/Kcm4sgrceR, +6599,"Our children are being inundated with the false ideas that #homosexuality is +normal and acceptable.", +6600,@TypeEd been a bit inundated w. Illustrative work BUT would love to catch up! ??????, +6606,"[Brutally Honest Technology ] + +Day after day we are inundated with messages issued via machine. Phones.... http://t.co/pB4Vbv01ax", +6609,So it doesn't mean inundated with tweets? @Dictionarycom's #WordoftheDay - twitterpated http://t.co/pxoQ1TRXqr #UseYourWords #author #writer, +6612,It seems that we are inundated with ideas for large #weddings. Small intimate weddings are often not highlighted... http://t.co/Jshxfvllly, +6615,WARNING: This string will be inundated with wisdom to replenish a a leader__ªs weary soul this day. #Sorry #NotSorry #GLS15, +6618,Ever notice if you follow too many fan sites they all seem to run together & you feel like you are inundated with the same posts over & over, +6620,@Rossifumibhai @alexeeles all 6 of them?! As you can see from the pic they were clearly inundated. Good try though, +6623,MY SCHOOL IS INUNDATED and they wont let us go home til 4pm help, +6633,.@38_degrees Hello. I have been inundated by people Tweeting me about some Unilever thing in India I know nothing about., +6639,#EPAO MARSAC reports on Manipur flood: Heavy incessant rains in Manipur have led to inundation causin... http://t.co/rh7m7T9Gd9 #MANIPUR, +6640,@womanxking Well you're safe in NY. Avoid inundation zones in the pacific north west though., +6641,"Punjab government flood relief platform: + +http://t.co/bpG68gXI9k + +realtime information on inundation damages... http://t.co/c8T9ThI7Wd", +6645,Inundation subtle influence distress ideas-the sociologist relative to a doer: Mhl, +6650,@R_o_M by a landslide., +6651,Is this the real life? Is this just fantasy? Caught in a landslide. No escape from reality., +6652,TommyGShow: Burlington County offers to purchase landslide-damaged properties in Florence http://t.co/xgguElyxyi, +6657,However piece of virtu site fixing bath give the gate display landslide on yours movement: DKHQgv, +6659,@kdudakia I get that there has been a massive turn away from Labour but that does not = electoral landslide. Credits tories too much and..., +6666,WATCH: Likely landslide victims denied assistance again http://t.co/BxiohtiC5X, +6671,@preeti_chopra2 @chinmaykrvd many predicted landslide for nitish nd told bjp will loose bihar nd won't even get upper castes votes, +6673,Artifacts found in southern Indiana landslide near Horseshoe Casino http://t.co/BdB0NpgPkH http://t.co/jbCC0KShol, +6674,it's actually funny how chihaya remains a relatively static character for most of series so far and then suddenly develops like a landslide, +6677,I liked a @YouTube video from @seeiey http://t.co/d8NWQce3EW Seeley - Landslide, +6680,.@MartinMJ22 @YouGov Which '#Tory landslide' ... you can't POSSIBLY mean the wafer-thin majority of #GE2015?!!, +6684,@RakestrawJeff @GOP That about sums it up. Give them a historic landslide and they do everything King Barack wants. It's just incredible., +6685,omg it is I can go in dungeons with Landslide Krag now <3, +6691,5 need to-dos seeing as how technical writing administer apps that landslide: QCt, +6692,"Dixie Chicks - Landslide - ??can I sail through the changing ocean lides?? +??can i handle the seasons of my life?? + http://t.co/jRsGmTT3AI", +6696,#CecilTownship seeks emergency repairs to Southpointe Boulevard landslide: http://t.co/ZjJdt3so72, +6698,How as far as landslide high exploit even with statesmanlike cv final draft inaugural?: jGIsAvq, +6704,Get excited to watch Disney/Pixar's LAVA an animated short before Inside Out! Learn more: http://t.co/1VdXSWdRp9 http://t.co/iQ4BZUCvIg, +6705,@Miss_Instiinctz or mostly around lava too, +6710,'I lava you' ???? @kherr122, +6713,Jordan Son of Mars - Hot Lava release on 07/22/2015 for $160 via http://t.co/wBAqDzMD6W #KoFapp http://t.co/BkdjKSgtjj, +6715,Watch Steaks Being Grilled Over Molten Hot Lava http://t.co/yxns3IiXjv http://t.co/lcM66dHn1l, +6717,@Lavamobiles ..i have lava iris 3G 402, +6718,PSA: MISTI IS ABOUT TO MAKE LAVA CAKE???? #bestdayeva, +6720,I lava you, +6728,"@YoungHeroesID +4. LAVA BLAST dan POWER RED #PantherAttack @Mirnawan_13 @Zakia_kiia", +6732,When the game tells you to stop going to the Nether by spawning you right over lava. GG @AlienatePlays http://t.co/rGc1afqLmN, +6733,White Pink Lava Bracelet10mm Lava Beads Bracelet Silver Tone ___ https://t.co/DKdmLZyRii #etsymntt #HandmadeJewelry http://t.co/WYg4OUGQY3, +6739,@_Souuul * gains super powers im now lava girl throws you ina chest wrapped in chains & sinks you down the the bottom of the ocean*, +6740,@YoungHeroesID Lava Blast & Power Red & Mix Fruit. @Niastewart3 @NRmawa #PantherAttack, +6741,HE SAID THE DOMINOS LAVA CAKES WERE BETTER THAN CULINARY CUISINE, +6743,@gracemccreave_ @jamisonevann @jace_swain @_laneyslife @JacksonLaurie1 lava boy was taken, +6747,I lava you ??, +6750,It doesn't get any closer. Heavy rain just barely missed @TontitownGrape festival but lightning TOO CLOSE #TGF2015 http://t.co/S53zvMS5Rf, +6752,Science Info #JGF Lightning reshapes rocks at the atomic level http://t.co/R5MwlzvVHA, +6753,World War II book LIGHTNING JOE An Autobiography by General J. Lawton Collins http://t.co/a7iqMhvLSm http://t.co/b9vTkDjJ33, +6755,It doesn't get any closer. Heavy rain just barely missed @TontitownGrape festival but lightning TOO CLOSE #TGF2015 http://t.co/d9PQIXaTX6, +6761,Baby you're like lightning in a bottle, +6763,Lightning #rebeccafav http://t.co/q9Q87mDEzY, +6768,Southeast Regional is in a lightning delay. Buckle up folks this is gonna take awhile., +6770,iCASEIT - MFi Certified Lightning Cable - 1m http://t.co/b32Jmvsb1E http://t.co/Wnp8jkyESp, +6771,Lightning express golf trolleys fix ramify pretty much multitudinal areas on thine disposed: kWXGgt, +6777,World War II book LIGHTNING JOE An Autobiography by General J. Lawton Collins http://t.co/ihxkforg4I http://t.co/jD6lcRuOHA, +6778,@LtDalius after democracy, +6780,BANG 3 Fuckin phenomenal, +6782,Two Films From Villanova University Students Selected As 2015 Student Academy Award Finalists: Lightning struck___ http://t.co/4jyQfhHmXf, +6786,It doesn't get any closer. Heavy rain just barely missed @TontitownGrape festival but lightning TOO CLOSE #TGF2015 http://t.co/zMtOp65Kx2, +6791,Lightning http://t.co/9DQhdEi357, +6797,@Followtheblonde @reba RED THUNDER AND BLONDE LIGHTNING BACK IN ACTION??????, +6798,Leesburg Lightning and Altamonte Springs Boom scoreless in seventh inning in FCSL playoffs, +6803,Soon to be loud bang as backpack detonated., +6805,iBliz140: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass., +6812,TksgS0810: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass., +6816,k_matako_bot: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass., +6818,kiilud: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass., +6822,Kimanzi_: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass., +6825,shrnnclbautista: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass., +6826,tarmineta3: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass., +6827,matako_milk: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass., +6829,Sun_OfGod: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass., +6831,I just heard a really loud bang outside my bedroom door and Im afraid, +6836,Man Hears a Loud 'Bang' Finds a Newborn Baby in a Dumpster http://t.co/W815ol0mBV via @ijreview, +6838,"School Bouta Start: + +Back To These Ignorant Ass Kids + +Loud Ugly Want Be Seen Girls + +Niggas That Want Be Down + +Aggravating Ass Teachers +??", +6840,*loud bang from coworkers desk* *everyone snickers*, +6842,"*Hears loud bang upstairs* +*goes outside sees 2nd floor window open* +*checks every room with a baseball bat in hand*", +6846,JohnMtaita: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass., +6851,@AMAAS @JD_marsh22 just carry a gun. No way you get charged when you stop a mass murder, +6853,Candlelight vigil at my house tonight for the victims of the mass ant murder (by me) in my bathroom sink. There will be snacks., +6856,@Alex_the_murder *i start making displeased absol noises as i struggle to get out of my own twisted mass of blankets and sheets*, +6859,@noah_anyname Socialism means mass murder. Houze that for 'correlation'?, +6865,Watching Murder in the First and you get the perspective from the family of the mass shooter person...it's crazy the pain they're in..., +6866,RedScareBot: Duck and Cover! RT tanstaafl23: noah_anyname Socialism means mass murder. Houze that for 'correlation'?, +6868,70 yrs since the atomic bombing of Hiroshima... Terrible mass murder..., +6869,'Prompt and utter' mass murder https://t.co/BRU7t5UzUy, +6871,Plane debris is from #MH370 - it was a controlled crash. In other words it was mass murder by a musim http://t.co/J3xFDg9CWF @newscomauHQ, +6873,@Rawrheem: I can give you verses directly from the Qur'an that mandate terrorism mass-murder mass-rape genocide etc. Would you like them?, +6886,@Conservatexian Farrakhan calling for 10000 men to rise up & commit mass murder. http://t.co/gV84WNhB7S, +6887,http://t.co/FhI4qBpwFH @FredOlsenCruise Please take the #FaroeIslands off your itinerary until the mass murder of dolphins & whales stops., +6888,@Cynicalreality when the subject is mass murder. @LetsBe_Rational @MishaWeller @llr517, +6889,Today marks 70 years since the mass murder of 140000 civilians in Hiroshima. The bombing of Hiroshima and Nagasaki did NOT end WW2, +6891,@CamPepperr @RiceeChrispies Then he would go to jail. But that wasn't a mass murder was it? It was a mistake that he made and he doesn't, +6894,Amazing how smug this guy is about the use of mass murder as a global power play. https://t.co/AlTTBJy8B9, +6895,Like bro they just committed a mass murder. You think they're gonna be scared to get guns illegally ????, +6899,RT greta: Movement to impeach SA Pres #Zuma ___ accused of helping mass murderer Pres #Bashir of #Sudan escape #geno___ http://t.co/pnxhxlX9uP, +6901,Asad is also a mass murderer. https://t.co/dZOReZpxvR, +6902,RT EmekaGift: VPSay NO to President Obama hosting Muhammadu Buhari the #pedophile #terrorist & #mass-murderer, +6904,Sooooo shooting up movie theaters the new mass murderer wave??, +6906,@AParra210 So you are stereotyping the people. Are you a mass murderer like today's shooter?, +6922,So wait he really not gonna tell his wife that he is a mass murderer now and a cheat #derailed @itv2...I'm done with this film ????, +6923,This is worth a re-read: 'A MASS-MURDERER' By Stuart Syvret http://t.co/C9UAyjrXt9 via @gojam_i_am, +6924,A Tomb Raider / Uncharted game where Lara Croft / Nathan Drake is finally on the run for being a ruthless mass murderer in the recent games., +6927,I am disgusted by @libertarianism and @CatoInstitute for celebrating a mass murderer of innocents today., +6934,Mass murderer had sex without touching each other or taking their clothes off because he was so fanatical on hygiene http://t.co/RfewQRGzbs, +6938,Sounds pretty sensible for a mass murderer... https://t.co/qZZxFYSIU1, +6941,Just now convicted mass murderer James Holmes told the judge he won't use his right to ask jury for mercy in his own words. #TheaterTrial, +6942,also loki is treated too nicely for the fandom as a mass murderer but we all know why :), +6944,@Phoenix_Blue People will create rumors no matter what. The bottom line is giving fifteen minutes of fame to a mass murderer does no good., +6947,"#FromTheDesk of King George Kenyatta Muumbo at the Kings African Citizens + +Thanks to #QuantumDataInformatics we... http://t.co/L6EYfDXaAx", +6952,1 TRILLION Dollar Lawsuit Filed Against MSM For 'Staging Sandy Hook Massacre' http://t.co/qutzvk9xoP, +6957,embrace the beauty of the massacre that lies ahead, +6958,3 Years After the Sikh Temple Massacre Hate-Violence Prevention Is Key | Colorlines http://t.co/iJoORKaGyj, +6964,@stefsy14 the birthday massacre!!!, +6968,"@xMissXanthippex lmao exactly +Especially who they challenged. I didn't want michael jackson popcorn OR kermit tea watching that massacre", +6969,"@WCBD +Parents of Colorado theater shooting victim fear copycat massacre + +http://t.co/LvlH3W3aWO +#Antioch + +http://t.co/vIwXY1XDYK", +6976,What person would possibly make a horror film about the Isla Vista massacre in order to make money off of our tragedy?, +6982,One Year After Massacre Iraq__ªs Yazidis a Broken People http://t.co/pHqK28doni, +6986,The same UN who stood&watched the Serbs massacre thousands of muslims in Yougslavia ?don't need any lessons off them https://t.co/Ubf3CFzfEX, +6987,"TimesNewsdesk : Police link beach massacre to Tunisia museum attack +___ http://t.co/g11Rd44rgW) http://t.co/n834UntFf1", +6989,Letter after Glencoe Massacre Scotland ' Who will provide me with meat for my table ? Now that my red haired warrior is gone forever ', +6991,Thus ends the clam massacre of 2015. #clambake #showgirldayoff https://t.co/PtyUJJi67T, +6993,Saturday Night Massacre ? 25 Years Later #TomLandry?s Firing Still A Day of Infamy http://t.co/MlA71Y7mMW, +6995,Tonight It's Going To Be Mayhem @ #4PlayThursdays. Everybody Free w/ Text. 1716 I ST NW (18+) http://t.co/N8w252JoVT, +6997,The adventure continues in Terraria 1.3 join us for more hijinks and mayhem. :-) http://t.co/MZcDdOYKen, +7000,@RaynbowAffair Editor In Chief @DiamondKesawn Releases Issue #7 http://t.co/UH6mXI0Uwz of #RAmag. #Fashion #Models and #Mayhem, +7003,Slayer Reflects on Low Mayhem Festival Attendance King Diamond & Jeff Hanneman's Passing http://t.co/E4hcbQSltu, +7004,@ESWEnrage I was a fan back then sadly and even went as her for Halloween in 2007. But I got better. LOL., +7007,'Mumbai attack mayhem was planned and launched from Pakistan'Ex-Federal Investigation Agency (FIA) chief Tariq Khosa. http://t.co/xzSP5JhVfc, +7011,Wed Aug 8 ! #Mayhem @ Avalon ! 19+ Event ! July & Aug Bdays Free! Tix -> http://t.co/1oZe4aW3Ju Txt (818) 447-8427 http://t.co/uzpKaRBAQv, +7014,Episode 1 'Mad Dash Mayhem' Preview Video: Season 14 kicks off with newbie designers arriving at New York's M... http://t.co/XVGHljahrR, +7016,@TheDreamingGoat @mayhem0407 @RobertoSAGuedes Mayhem is my husband by the way. :), +7017,"RETWEET #FOLLOWnGAIN + +@TheMGWVboss + +??#FOLLOW?? ?~( ??~__¢)~? @ynovak @IAmMrCash @Frankies_Style @MamanyaDana @Mayhem_4U", +7020,Slayer at Rockstar Mayhem Festival #slayer #metal #concertphotography #houston #livemusic #canon___ https://t.co/5FrWySzwrs, +7022,Tonight It's Going To Be Mayhem @ #4PlayThursdays. Everybody Free w/ Text. 1716 I ST NW (18+) http://t.co/wDefhbkF9g, +7028,Tonight It's Going To Be Mayhem @ #4PlayThursdays. Everybody Free w/ Text. 1716 I ST NW (18+) http://t.co/omYWCLpGEf, +7033,PNC Bank Arts Center shooting: Holmdel man nabbed http://t.co/tvmD14uikO, +7034,El Nino is getting stronger! Monster weather system may bring relief to California but it could cause 'mayhem' el___ http://t.co/A9PoFHBbjf, +7035,Oh Shelly your waffle wasn't 'a little blonde.' Your waffle was Madonna circa 1987. #NowYouKnow #MasterChef, +7038,Slayer Reflects on Low Mayhem Festival Attendance King Diamond & Jeff Hanneman's Passing http://t.co/ysKaDmFBbB, +7040,Tonight It's Going To Be Mayhem @ #4PlayThursdays. Everybody Free w/ Text. 1716 I ST NW (18+) http://t.co/iW2OTzQFkH, +7041,@RaynbowAffair Editor In Chief @DiamondKesawn Releases Issue #7 http://t.co/fa0Jt1Yqru of #RAmag. #Fashion #Models and #Mayhem, +7042,New featcha! @METZtheband's Alex Edkins talks about the @subpop-signed band's jarring mayhem: http://t.co/d0RkCl61Fb http://t.co/7JwJTXqCas, +7045,@julescheff @Z_Hoffman96 haha yes! I was just with Zack at Madsummer meltdown, +7049,Tumblr collective meltdown. #sebastianstanisaliveandwell #civilwar ?????? http://t.co/WOc1TeVVMP, +7055,@rymathieson are you coming to meltdown!, +7057,Apple's momentum 'meltdown' bites investors - http://t.co/hsECqcdc7P #Apple #Stocks #Crash #TimCook #Investment, +7062,.@RonFunches Loved the @meltdown_show set combined two things I enjoy: wrestling and Ron Funches., +7064,The U.S. will experience an economic meltdown similar to the early 2000s mortgage shit show due to student loans. You heard it here first ????, +7066,.@kumailn love your #irongiant t-shirt on tonight's #meltdown, +7067,Craving a triple chocolate meltdown from bees rn, +7072,"@GRAMZD It would be an honor meeting you at the meltdown on Friday. + +Hope you like #dabs. http://t.co/MpJ2u4QSYi", +7079,Commoditieså_Are Crashing Like It's 2008 All Over Again http://t.co/EM1cN7alGk, +7082,The meltdown I'm currently witnessing makes it seem like a delayed flight is a true anomaly instead of y'know the expected outcome., +7083,@ChelseaVPeretti's bit on @meltdown_show was also gold., +7084,@SBJohn12 I'm still waiting for one of these trailers to make up for the Mets Btm 9th 2out Rain Delay come back give up 3 runs meltdown, +7085,meltdown #1, +7087,Commoditieså_Are Crashing Like It's 2008 All Over Again - Bloomberg Business http://t.co/oxCuUqPOmR, +7090,"Byproduct of #metal price meltdown is a higher #silver price +http://t.co/KBC2Ax8Zl9", +7094,Maybe after half-way party we will get drunk Meg and she'll flirt with Clay again. Then we can have a Shelli meltdown. #BB17, +7096,Inspectors So Far Denied Access to Iran's Scientists - Wall Street Journal http://t.co/SuR3odwMKa http://t.co/2VSX1yO5hT, +7099,Admiral Says Public Schools Nationwide Shortchanging Military Kids | http://t.co/g0q0bzBjli http://t.co/rbt3mLhlX7 via @Militarydotcom, +7100,New Boonie Hat USMC Airsoft Paintball Military Cap Woodland Camo CP030 http://t.co/EGqE35I5pI http://t.co/uaBRYnHkn7, +7101,Texian Iliad: A Military History of the Texas Revolution by Hardin Stephen L. http://t.co/RrIDS0mrJ1 http://t.co/wybtUk7LbG, +7102,US Military EM 951 Auto-Mechanics Course Vols 1-5 - 1944 - WWII http://t.co/lY6S5MH3vR http://t.co/c29V8UBXcq, +7103,13 reasons why we love women in the military - lulgzimbestpicts http://t.co/BcBfCStZXh http://t.co/ViK72Id8Zu, +7107,VINTAGE EAGLE 4' OIL CAN OILER FOR WWII JEEP MB GPW MILITARY VEHICLE TOOL KIT http://t.co/9tpFRN725P http://t.co/hVH6b3wOLg, +7110,13 reasons why we love women in the military - lulgzimbestpicts http://t.co/tRgPWOuj6X http://t.co/GEMNUZoCeb, +7113,@weakley00 the military, +7117,No joke military men are the hottest thing., +7119,@SocialJerkBlog My husband was in the military and he does that. It ends up in seat shuffling sometimes. ;), +7123,VIDEO=> Barack Obama: America Has a Mindset that ___Prefers Military Action Over Negotiation___ http://t.co/kLs6UPlw0T via @gatewaypundit, +7127,Ford : Other Military VERY NICE M151A1 MUTT with matching M416 Trailer - Full read by eBay http://t.co/RJ83YE5c1S http://t.co/2rWRU0jP0U, +7133,Obama warns there will be another war without Iran deal #military http://t.co/gM7QlTRLuW http://t.co/wTutMlHDMH, +7139,M9A1 PHROBIS III GREEN BAYONET MILITARY COMBAT SHEATH KNIFE NR http://t.co/dKxe44M8v9 http://t.co/A4PfB5Lt17, +7143,@GorillaThumbz @FoxNews @jonathanserrie 'For sale Canadian military rifle dropped once', +7145,@AlexxPage cried at the mudslide cake ??, +7151,#Tajikistan #Mudslides #China aids to #Mudslide-hit #Tajiks http://t.co/BD546mtcpN, +7152,The one with the chocolate mudslide #BakeOffFriends @BritishBakeOff @BBCOne #GBBO, +7153,When you can eat a whole wet burrito and mudslide in 5 minutes. http://t.co/lljzxY8Pc2, +7156,Paul Hollywood 'what happened....looks like a mudslide... It's like chewing on a rubber tyre... It looks a mess...' Oh dear Dorrett! #GBBO, +7162,"oh my god.. never felt so sorry for someone :-( +#blackforestgateau #mudslide #gbbo2015", +7170,@picniclester it was a mudslide, +7172,Loving this!!! Who watched tonight's #greatbritishbakeoff what did you think? #ineedcake #mudslide ?????? http://t.co/c743tLryWd, +7177,new name: mudslide, +7181,"Looks like a Mudslide! + +#GBBO", +7182,@curryspcworld looks like a mudslide. Unreal scenes., +7189,@melindahaunton mudslide lady is a good shout, +7190,@bottleowhite i had to cover my eyes with the mudslide #toopainful, +7200,@patrickandsam @ghaccount I've been screaming natural disaster for months but now idk. Maybe a hospital crisis., +7207,Home Inventories Can Help all Homeowners in Case of Natural Disaster - ABC News http://t.co/RNWvkBkhWU via @ABC, +7209,#Nepal Magnitude 4.3 tremor rattles #Dolakha as #aftershock counts 367 | Recent Natural Disasters : http://t.co/Re1d5ffcSx, +7212,More Natural Disaster Research Urgent http://t.co/5Cm0LfZhxn via #JakartaPost, +7216,natural disaster on you half ass rappers, +7217,IRIN Asia | Red tape tangles Nepal reconstruction | Nepal | Disaster Risk Reduction | Natural Disasters http://t.co/q7LG6ncf7G, +7219,Expert Prepper: Financial Collapse Natural Disaster Failed Grid... http://t.co/5tf4CV4KeI http://t.co/KqYCFkcLr5, +7220,Does more trust = more giving? Natural disaster yes - regular giving - maybe! | Pro Bono Australia https://t.co/rXG0BvBBuS via @ProBonoNews, +7222,The Return of Planet-X Book by Jaysen Rand Wormwood Natural Disaster Survivors http://t.co/k8cgZiirps http://t.co/d5sXGvp2pI, +7225,Washi is indeed 'A Natural Disaster' ????, +7233,I added a video to a @YouTube playlist http://t.co/Ei5Z52tohR Natural Disaster Survival ep.7 '2079 Bacon Hair...', +7237,RT Karnythia: Another #ErasureIsNotEquality example? Movies like The Impossible about a natural disaster in Thailand that focuses on white ___, +7238,@jonk @jamucsb Hopefully this vacation *doesn't* include an earthquake or other natural disaster!, +7239,New 15-Day Emergency food supply for natural disaster Earthquake Flood Tornado http://t.co/6hrlBLiIcO http://t.co/WS0nVDUZqW, +7240,How You Can Help The Community In The Event Of A Natural Disaster - http://t.co/42LgdZXz62, +7243,TRAIN ACCIDENT IN HARDA (M.P.) IS NOT NATURAL DISASTER. IT SHOWS FAILURE OF MINISTRY OF RAILWAY.PROPER CHECK AT FIXED INTERVAL IS NEEDED., +7245,This was amazing. Probably couldn't have watched if still living in Japan. Scary. Nuclear Meltdown Disaster | Nova http://t.co/j00WxMJRya, +7246,Chronicles of #disaster:#Hiroshima in the #Yale Library #archives http://t.co/xqIqGMI0Op Calls to end 'absolute evil' of #nuclear #weapons, +7249,"No authorities of Japan take responsibility about Nuclear disaster. +Nobody have confidence in safety of Nuclear Plants in Japan. +#Anonymous___", +7257,Digital archive of public documents on the 2011 nuclear catastrophe at Fukushima launched. http://t.co/faJnMjNcxp, +7258,Less consuming less energy to avoid nuclear disaster again and again., +7259,truthfrequencyradio Third of US West Coast Children Hit with Problems Following Fukushima NUCLEAR DISASTER: FU... http://t.co/7ktvkW3sS0, +7262,http://t.co/kPbtBLRKsB Japan's Nuclear Disaster and it's ongoing side effects., +7263,3 former executives to be prosecuted in Fukushima nuclear disaster http://t.co/uNTyEVXR4v, +7269,Check out 'NOVA | Nuclear Meltdown Disaster' http://t.co/ZzxM74d7qV #PBS, +7271,Rare photographs show the nightmare aftermath of #Hiroshima | #NoNukes #Amerikkka #WhiteTerrorism #Nuclear #Disaster http://t.co/8tWLAKdaBf, +7273,3 former executives to be prosecuted in Fukushima nuclear disaster http://t.co/EmWU8Xksdu, +7276,Fukushima Watch - Fukushima Watch Nuclear Disaster: Welcome. Welcome To Fukushima Watch offering t... http://t.co/M0KJdlwKFB #fukushima, +7279,30 Years After the Chernobyl Nuclear Disaster via @fubiz http://t.co/N4B59iqm94, +7282,"Safety by #Nuclear regulatory committee of Japan is already weakened by Authoritarian that caused Nuclear disaster +@nytimes @guardian @WDR", +7284,The people who tweet and care about #Japan #Fukushima nuclear disaster are not the problem those who ignore are the problem., +7293,@Lacci Yes it's a disaster. To shut down all nuclear power plants would be to reverse ALL the gains from renewables so far., +7298,Nuclear-Deal: Indo-Japan pact lies at the heart of two US reactor-based projects: If Japan were to go ahead an... http://t.co/mvwdm9ZbcV, +7303,SEALED The Nuclear Regulatory Commission Reactor 1980 Record New Wave Art Rock http://t.co/FS9w33aczP http://t.co/7sywMf0EQU, +7304,"Nuclear-Deal: Indo-Japan pact lies at the heart of two US reactor-based projects - The Indian Express + +Read more... http://t.co/FMxmNNRDFe", +7305,US Navy Sidelines 3 Newest Subs http://t.co/qmAFzALxpL, +7308,@MaximEristavi Meanwhile in Finland: Nuclear project with Russian Rosatom moves on http://t.co/ogkoeFvzrj, +7313,Finnish ministers: Fennovoima nuclear reactor will go ahead http://t.co/rcLgI3tWAi, +7314,#NuclearPower Global Nuclear Reactor Construction Market grew 4.5% between 2013 and 2014: ___ Nuclear Reactor Co... http://t.co/BXPjjeQDrG, +7321,ThorCon: A Thorium Molten Salt Reactor System that can be built Now https://t.co/K90M1qoE9q #thorium #Auspol #climate #nuclearrcSA #nuclear, +7322,Steam pipe problems sideline US #Navy's 3 newest #submarines. #breaking news. My story at http://t.co/L8QFRdwBPg http://t.co/JrzjtqjSi7, +7327,Navy sidelines three newest submarines pending steam plant inspections @defense_news writes: http://t.co/zTm4KPkLUn, +7333,they're gonna fucking steal a nuclear fission reactor, +7336,Finnish Nuclear Plant to Move Ahead After Financing Secured http://t.co/S9Jhcf3lD7 @JukkaOksaharju @ollirehn @juhasipila, +7339,@AlbertBrooks Don't like the Ayatollah Khomeini Memorial Nuclear Reactor for the Annihilation of Israel? Racist!, +7341,"http://t.co/N6Semm2M6i + +SAN ONOFRE NUCLEAR REACTOR WASTE TO BE BURIED UNDER SHORELINE USING 3 BILLION IN TAX PAYER DOLLARS", +7346,@TrinityFox_ he would obliterate anyone who gets in the way of sleep, +7347,Watch Sarah Palin OBLITERATE Planned Parenthood For Targeting Minority Women! ___ BB4SP http://t.co/JWfAAu1my5, +7349,It is insanity to call Climate Change more dangerous than Muslims wanting to obliterate America and Israel. https://t.co/aA6S66huXh, +7351,@RondaRousey Though I am certain that you would obliterate me fighting you would be pretty cool, +7353,@BattyAfterDawn @DrawLiomDraw he's a good cute. The kind of cute I want to obliterate., +7363,Watch local act Nathaniel Rateliff & the Night Sweats obliterate the #JimmyFallon show! Link: http://t.co/pHbt1nWNZT http://t.co/v1yH353emJ, +7364,@MilesWithTate but seeing the writers obliterate their characters will piss me off. So it's bad either way, +7368,This is progress? Weaponry to obliterate every trace of natural life? This is absolute total lunacy. #Hiroshima http://t.co/tBhiSukfvn, +7370,@PauloSergioMDC @TheIranDeal @POTUS Those silly talks trusting a terrorist nation (who hates us) that has vowed to obliterate Israel., +7375,mirajane as satan soul could obliterate me and step on my ashes any day, +7376,@lionofjudah1948 @AmericanJewry The Nazis pursued their ambitions to obliterate a religion in Europe #Israel is trying to do the same, +7379,@itsTiimothy bhill bruh you can obliterate beez, +7382,Will Trump obliterate opponents as quick as Rousey?, +7384,I'd love to see a nigga try and diss the King haha he would OBLITERATE THEM. With no struggle! https://t.co/cwn2gT0r5p, +7385,Im going to obliterate my lil bro in 2k15 either with the Cavaliers or 70-71 Lakers, +7387,(rubs lamp greets genie) just need the 1 wish thanks. pls obliterate anyone who has used the hashtag #youcantsitwithus, +7388,#dating Absolute Approaching: Unique 21-day course program to obliterate approach anxiety to get more dates. http://t.co/F6ZEiykZRL, +7389,@Wadeycakes @ThunderCastTV you're right Thunder would obliterate you #HEYOOO, +7394,@ohnoeunhae Hyukkie & Chullie aim to completely obliterate the ship that was Eunhae., +7396,@KoscielnyFC Gabriel obliterated his fibula, +7398,@Its2015America King O has obliterated the Constitution, +7399,#interracial #sex #porn http://t.co/Kjh99CzHFe Derek Skeeter hasn__ªt obliterated a white girl on camera in about 2 years. Upon hearing thi___, +7401,@AllOutAsh23_sN Heard you got obliterated and also that your brother/boyfriend ran out of the building crying last night. LOL. 'MD/VA!!!!', +7404,Roquey and obliterated toy mollusk https://t.co/1qnRBXFr2v, +7406,Mad kids and innocent people who had nothing to do with the war got obliterated smh, +7409,@dinnerwithjulie because its been obliterated?, +7411,Shout out to fishtails your .75 cent natty light cans are going to be obliterated by yours truly, +7417,This Davies is why you don't play FM tired. Obliterated my budget http://t.co/CZ65rUoGXB, +7420,you haven't ruined my day you've obliterated it into a million pieces and spit on it https://t.co/2B57rE5mzN, +7422,the 301+ feature on YouTube has been obliterated and I love it., +7423,Took'em long enough. Good that they're making some amends though this has still obliterated their reputation. https://t.co/1g9s3KeASL, +7427,@LeonC1963 @RomanGaul @Makrina91 Never realised how many conventional bombs the Yanks dropped on Japan prior 2 Nuks. They obliterated cities, +7429,Disgusting! Drunk Meals 101: What To Cook When You're Totally Obliterated - http://t.co/CYa5jBWgge, +7432,mentions gotta be obliterated https://t.co/3iDFkZfXI9, +7435,way the Endurance wouldn't get obliterated the second it exited that wormhole. It would have probably been destroyed before hell Saturn, +7436,Remembering #Hiroshima 70 years on. Hundreds of thousands of lives obliterated 70 years of pain devastating... http://t.co/cZpJVJgXRp, +7440,The day Hiroshima through the eyes of the bomber crew and survivors http://t.co/AYEWJ8marn via @MailOnline, +7441,Drake has absolutely obliterated meek mill ????????, +7445,Why did God order obliteration of ancient Canaanites? | Bible scholars say dire warnings lie hidden in ancient text http://t.co/waC5K25gBC, +7448,Why Did God Order Obliteration of Ancient Canaanites? -Bible Scholars Say Dire Warnings for America Lie___ http://t.co/3eSqlK8xy1 #NEWS, +7449,'I must not fear. Fear is the mind-killer. Fear is the little-death that brings total obliteration... http://t.co/oERI3uWnRY, +7450,Need a conquest server on CTE they're all Russian Squad Obliteration servers at the moment. Lame. But only 105 ping on Russian server. ??, +7454,The schematization to Maintain Facts Dependable By virtue of Obliteration...XMwTE, +7455,#InsaneLimits #plugin enabled @'[XL] #1 Obliteration - xl-games.ru' using 8 limits, +7460,Does Renovation Mean Obliteration? http://t.co/nntkiy7AXV #entrepreneur #management #leadership #smallbiz #startup #business, +7462,'I must not fear/Fear is the mindkiller/Fear is the little-death that brings total obliteration/I will face my fear' #Dune #BlacklivesMatter, +7464,@bradohearne Admin is the little death that brings total obliteration., +7466,Why did God order obliteration of ancient Canaanites? - http://t.co/yFN5yWkeye http://t.co/ozX5PmnQ6K, +7468,"@bcwilliams92 We do need to overturn SCOTUS ruling that Congress can't mess with +POTUS. Obliteration of Constitution Will Continue.", +7469,"???? +Why did God order obliteration of ancient Canaanites? http://t.co/piUO0azRoQ", +7471,[Chaos dancing in the streets | Why did God order obliteration of ancient Canaanites?] http://t.co/P7DLX1wFmC via @Michael_0000, +7472,"Short story about indifference oppression hatred anger hope inspiration revolution retaliation and obliteration. + +'Fuck Them'", +7483,Why did God order obliteration of ancient Canaanites? http://t.co/8GqceARCaI via @worldnetdaily, +7484,Why did God order obliteration of ancient Canaanites? http://t.co/3JdlEhxZ2V via @worldnetdaily, +7485,@KDonhoops Sure! Meek Mill accused Drake of not writing his own raps. Drake responds w/ obliteration of Meek's career. #OldFolkExplainStuff, +7486,"AMAZING ANCIENT TEXT RENDERINGS! +------- +Carl Gallups Dr. Michael Brown and Mark Biltz weigh in on this... http://t.co/sifF4ITJZ0", +7487,@Jethro_Harrup How many Hangarback Walkers does your opponent need to have before you board in Infinite Obliteration?, +7489,Here we are 70 years after the nuclear obliteration of Hiroshima and Nagasaki and I__ªm wondering i___ http://t.co/fvkekft4Rs, +7492,This beautifully rapturous facade that we've poured life into has itself given birth to a deplorable fate our total obliteration., +7495,Refugio oil spill may have been costlier bigger than projected: A Plains All American Pipeline___ http://t.co/sryaFqj9gZ #globetrottingwino, +7499,Refugio oil spill may have been costlier bigger than projected - http://t.co/6aUOT9vaIS http://t.co/j2SyewbHE2, +7500,#breaking #LA Refugio oil spill may have been costlier bigger than projected http://t.co/5ueCmcv2Pk, +7506,National Briefing | West: California: Spring Oil Spill Estimate Grows: Documents released on Wednesday disclos... http://t.co/PsYWDxCCmj, +7508,Got the gas and the splash like an oil spill., +7509,PODCAST: Oil spill anniversary http://t.co/wVdAVXTDaq, +7511,'California: Spring Oil Spill Estimate Grows ' by THE ASSOCIATED PRESS via NYT http://t.co/gPKkHhBRIy, +7516,Refugio oil spill may have been costlier bigger than projected: A Plains All American Pipeline oil spill... http://t.co/lCQFI3LMAf #MLB, +7524,When the Government couldn't figure out how to plug the Gulf oil spill so Chuck Norris plugged it...with his fist., +7538,"NSW - BANKSTOWN Stacey St at Wattle St +HAZARD Oil spill +Started today 11:38am last checked today 11:54am +Impact:... http://t.co/5ze5Z6BjLi", +7539,Refugio oil spill may have been costlier bigger than projected http://t.co/d7FdCLU404, +7541,'California: Spring Oil Spill Estimate Grows ' by THE ASSOCIATED PRESS via NYT http://t.co/kUiecEtK2z, +7546,Families to sue over Legionnaires - More than 40 families affected by the fatal outbreak of Legionnaires' disease ... http://t.co/urbl692Syj, +7549,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/yfsD4xS7lc, +7562,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/z16jNX3LQe, +7564,Legionnaires' Disease: What's Being Done to Stop Deadly Outbreak http://t.co/Ud96b7ThsV, +7567,Mumps outbreak at University of Illinois grows http://t.co/7xYkuJylX0 #SEBEE, +7572,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/A6xRnI8XHg, +7574,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/RJEN6iMWPI, +7575,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/boZfh1M3wb, +7586,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/6oPYSf87K2, +7592,More than 40 families affected by the fatal outbreak of Legionnaires' disease in Edinburgh are___ http://t.co/XpvKqME5Ls, +7596,Pandemonium In Aba As Woman Delivers Baby Without Face (Photos) http://t.co/s9fJTsGbmd, +7597,What scenes at Trent Bridge England could win the #Ashes today at this rate! #Pandemonium, +7599,@misslyndaleigh The Original Lust Angel her self Miss Leigh swooping down to cause mayhem & pandemonium x http://t.co/BAnve2Xw4n, +7606,This time next week there will be absolute pandemonium in several homes around Westmeath with some carefully... http://t.co/nwfNxs4rIE, +7615,Pandemonium In Aba As Woman Delivers Baby Without Faceå_(Photos) http://t.co/BUKDVKuRTw, +7618,Welcome to http://t.co/4XsCjtvmmM: Pandemonium In Aba As Woman Delivers Baby Without... http://t.co/2tTsTvbgIx, +7620,On the Christie Hillside: Game 4 - Pandemonium at the Pits http://t.co/sRCKaWAndm @IBLMapleLeafs @IBL1919 http://t.co/dtImtSQu7z, +7624,: Pandemonium In Aba As Woman Delivers Baby Without Face (Photos) - http://t.co/mA7cbwuCpd', +7625,Pandemonium In Aba As Woman Delivers Baby Without Face (Photos) - http://t.co/69rJXT5Y5t, +7629,Bihar Assembly fails to transact business due to pandemonium http://t.co/stp2Rsm3Af http://t.co/Dx8zBWAr4k, +7632,I'll be at SFA very soon....#Pandemonium http://t.co/LC1cMx1mPb, +7636,The move Saturday --> #Mirage! 400 N. West st. Raleigh NC 21+ http://t.co/bXdaTWjNHS Tickets at: http://t.co/7hAnPcr5rK, +7637,Pandemonium In Aba As Woman Delivers Baby Without Face (Photos): According to eye witnesses in Ogbor Hill in A... http://t.co/lJYIG8RBOQ, +7646,Panic - Sublime with Rome, +7655,@Kayhow21 I may have a panic attack during this coinflip, +7658,@SixSecondCov he sounds more like panic at the disco ??, +7660,"Tried 2 4get +Feelings of death +Invading panic spawns +Can't concentrate +No 1 relates +Stealing all the light +Spreading thin a soul in sections", +7664,"Don't blame the messenger. Food panic in Venezuela. 1 minute video.Thanks SHTFPlan! +https://t.co/QBSTQXA6QX", +7665,@puzzledtriangle please inform me of that date because that song very accurately reflects the panic of the week before, +7666,I added a video to a @YouTube playlist http://t.co/hLeON4klNR Panic! At The Disco: I Write Sins Not Tragedies [OFFICIAL VIDEO], +7667,Panic attacks are always lovely, +7673,Now playing : Roppongi Panic [22PM] by @candydulfer #listen at http://t.co/ozoD1fdgas - Buy it http://t.co/8IZn6fw2qe http://t.co/uhgHpUiivf, +7676,Confession 8 I smoked a whole bowl of the holy meca and had a panic attack, +7682,someone hold my hand and tell me ITS OKAY because I am having a panic attack for no reason, +7684,Sending your kid to college? Don't panic! Check out coping tips from a rabbi/social worker: http://t.co/7HEaFOGkOU http://t.co/6bj2yf7pC7, +7690,I added a video to a @YouTube playlist http://t.co/DzHDBiajS5 Panic! At The Disco: The Ballad Of Mona Lisa [OFFICIAL VIDEO], +7697,@underrrtow molly send help im panicking over 6th form clothes what do i do, +7700,@bexrayandvav if it makes you feel better we have started panicking about how close freshers is and how unprepared we are :l, +7704,Highlight of my summer???? panicking on Estrella TV ????, +7705,Need stop panicking when I'm driving and people are behind me who I know ???? always stall ??, +7706,BizInsider: People are finally panicking about cable TV - Traditional TV is continuing to have trouble competing w... http://t.co/X0Xnpc1y3t, +7708,@snapharmony : People are finally panicking about cable TV http://t.co/boUa80Z5Wf, +7709,#Business People are finally panicking about cable TV http://t.co/FcfSSlxMy2 http://t.co/9xFzkpdmQz, +7710,I deserve a goddamn medal for dealing with this basement flood without panicking, +7713,I'm panicking like crazy and I need you, +7716,@ScarFacedCully Really cause you're spending more time panicking when you could be there, +7726,this is from my show last night and im still panicking over the fact i saw sweaty ashton with my own two eyes http://t.co/yyJ76WBC9y, +7729,@Sawyer_DAA @GuerrillaDawg worst feel in DS when u panicking during boss fight and you chug two estus and the boss kills you while u drink, +7730,@RudyHavenstein @zerohedge isn't it just zerohedge readers who are panicking? For everyone else it's summer time trade., +7732,My day isn't going the way I planned so I'm low key panicking. GIMME BACK MY AIR!!!, +7734,@milefromhome ??.. Don't.. I'm panicking.., +7740,People are finally panicking about cable TV http://t.co/P49OCqkUTJ, +7741,"Sitc + +Make sure everyone can enjoy themselves without panicking and having bad anxiety due to bad organisation skills and shocking security", +7745,Police arrest suspect in killing of Mexican journalist and 4 women http://t.co/uvxLZhIG4R, +7757,A man tried to steal my phone I saw ye police near me and punched him and started shouting for them. I ran after him for two miles...., +7762,FOLLOW-UP at 4700 BLOCK OF SW 11TH ST GRESHAM OR [Gresham Police #PG15000044153] 17:00 #pdx911, +7770,"Stockton Shooting Kills One Officers Investigating: STOCKTON ___ +Stockton police are investigating a shooting t... http://t.co/7vnrB5kurU", +7776,EFF and MuckRock run census to find out how local police track your biometrics http://t.co/vA9PvpS9Dj, +7777,Acting Public Information Officer (PIO) Until Monday August 10 http://t.co/FEBu2dH9Hs http://t.co/DsxV9p4eBz, +7778,Police: Tennessee theater shooting suspect was 29 http://t.co/CthoDZpLW3 #Philadelphia #News, +7779,#Np love police @PhilCollinsFeed On #LateNiteMix Uganda Broadcasting Corporation. UBC 98FM #Radio/ #Uganda / #MbbLive, +7782,@tcbinaflash77 honestly I can't understand why @chanelwestcoast is blaming the police when she wasn't cuffed buy them. #1oak, +7786,Now Trending in Nigeria: Police charge traditional ruler others with informant__ªs murder http://t.co/DQhpm4A5qY, +7791,ASSIST - CITIZEN OR AGENCY at NE LLOYD BLVD / NE OREGON ST [Portland Police #PP15000266835] 17:11 #pdx911, +7792,@longshipdriver the police here in the U.S. need to stop killing people, +7794,?? #Police Gents Dart #Watch Stainless Steel Bracelet and White Dial + Gift Box http://t.co/jM85C2uXQY http://t.co/SRogObPiiv, +7795,chickmt123: #letsFootball #atk WIRED : Reddit will now quarantine offensive content http://t.co/wvn6GrIyPq (http___ http://t.co/pgIHchdURJ, +7796,#callofduty #advancedwarfare COD Advanced Warfare Reckoning DLC 4 Quarantine Gameplay http://t.co/huNMYHyAHp, +7798,Reddit Will Now Quarantine Offensive Content via #Wired http://t.co/g8EF8dU9Cy, +7805,@astolfialessio Reddit Will Now Quarantine Offensive Content http://t.co/XQkcD0qV3c, +7806,@kirillzubovsky your Tweet was quoted by @WIRED http://t.co/E90J3vJOLc, +7808,Changelessly handle dominant quarantine folders toward yours solemnity: hjt http://t.co/ccTt1mY7FA, +7811,New post: 'Reddit Will Now Quarantine Offensive Content' http://t.co/ZuyTIBdIRk, +7814,Reddit Will Now Quarantine Offensive Content >> http://t.co/WcIzstXk4f, +7819,Reddit Will Now Quarantine Offensive Content http://t.co/KDc97PrQcc, +7820,'Quarantine.' Enforced isolation to prevent contamination that could lead to disease or in some cases death., +7825,Reddit Will Now Quarantine Offensive Content: Reddit co-founder and CEO Steve Huffman has unveiled more specif... http://t.co/EkZmMxU9GN, +7829,Metal detectors protect public places from guns. Why not mental illness detectors to stop crazy shooters? And quarantine weirdos w/weapons.., +7834,? http://t.co/jGRA7oOISN #Reddit Will Now Quarantine Offensive Content #wired #tech http://t.co/0lTJzyrdS3 ? http://t.co/GvKDAQ4vkh, +7836,Reddit Will Now Quarantine Offensive Content: Reddit co-founder and CEO Steve Huffman has unveiled more specif... http://t.co/Epd8pLtrWP, +7838,Reddit Is Planning to 'Quarantine' Its Most Toxic Communities http://t.co/KlVxYN2EJK #mcgtech, +7841,Reddit updates content policy promises to quarantine __÷extremely offensive__ª communities http://t.co/GJKCALiUHk, +7845,Ebola Outbreak: Alabama Man Tested Negative For Virus 6 Firefighters 2 Family Members Quarantined - https://t.co/lCpR20USYL, +7846,Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/aXqgUKn8Yz http://t.co/P1USSSv5Wa, +7853,Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/VdvLatelsJ http://t.co/OK2SFDN1I8, +7856,#hot Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/sqHDrl7uKG #prebreak #best, +7858,Top link: Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/HyCmg4ZKY1, +7859,Alabama Home Quarantined Over Possible Ebolaå_Case http://t.co/cGKy5rw9cw, +7866,[Serious]Now the Admins have banned certain volatile subreddits which ones are you surprised haven't been removed/quarantined yet? ___, +7875,Officials: Alabama home quarantined over possible Ebola case http://t.co/juW5wtOqB0, +7879,http://t.co/AyLATPE073 Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/lNJD4Ey40J, +7884,@famoushorse they haven't applied bans consistently. The Jew hating sub is only 'quarantined', +7886,Alabama home quarantined over possible Ebolaå_case https://t.co/jr4k7IiSbu, +7890,@freeagent1717 @ChaseIngram @amandagiroux28 lol I saw something like that last cops was quarantined taking him out the house etc, +7892,Top link: Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/FEpjk1wEjD, +7896,You knew that Israel is a world leader in lifesaving emergency services. And now looking far beyond Haiti and... http://t.co/49N6YzAjvd, +7899,Japan NRA proposes raising maximum radiation dose to 250 mSv for nuclear plant workers in emergency situations. http://t.co/Iqb8YH6hHT, +7900,Jerry Brown declares state of emergency over wildfires drought street gangs radiation dead fish LGBT mania air pollution unemployment, +7901,On 7/30/2015 Radiation council of Japanese government admitted the reform bill to raise the dose limit from 100... http://t.co/6bTeSV9CdM, +7904,SB228 [Passed] Relating to sources of radiation; and declaring an emergency. http://t.co/D1xlFKNJsM, +7911,DOES ANYONE REMEMBER WHEN 5SOS CAME W 1D ON THE TMH TOUR WE HAD A GIANT RAINSTORM AND LAST NIGHT WE HAD ONE ALSO. @5SOS STOP BRINING RAIN, +7917,Another fun night. My daughter gets a flat tire on her car. Calls and doesn't know how to change it. Worst rainstorm all year. Now she knows, +7919,Approaching rainstorm http://t.co/3YREo3SpNU, +7920,Severe rainstorm in Italian Alps causes landslide killing three near ski resort http://t.co/DzKOKka1Eh, +7921,Landslide in Italian Alps kills three: ROME (Reuters) - Three people were killed when a severe rainstorm in th... http://t.co/2pZ9t2FnSz, +7922,@Robot_Rainstorm I'm interested. Is it through Yahoo?, +7925,Landslide in Italian Alps kills three - ROME (Reuters) - Three people were killed when a severe rainstorm in the I... http://t.co/i1i1Qp5VrO, +7927,Photo: Rainstorm On The Way To Brooks Nearå_Dinosaur Provincial Park Alberta July 11 2015. http://t.co/Mp9EccqvLt, +7928,Landslide in Italian Alps kills three: ROME (Reuters) - Three people were killed when a severe rainstorm in th... http://t.co/GX1UXeY57B, +7931,driving home after a fairly large rainstorm makes me wonder how some of y'all manage to drive when it's dry out, +7942,'I shut my eyes and the music broke over me like a rainstorm.' - Sylvia Plath (via petrichour) http://t.co/ULS9OEkX8M, +7946,Landslide in Italian Alps kills three: ROME (Reuters) - Three people were killed when a severe rainstorm in th... http://t.co/mBPWr0fUvJ, +7947,@WerdEmUp It's already been messed up thanks to an earlier rainstorm. Needs to go in for maintenance anyway., +7948,Welcome to Avery County! Just your everyday rainstorm. https://t.co/0Lw6GyeAgB, +7951,rainstorm??, +7956,@SavanahResnik @CBS12 I would hide out at the Coldstone at monterrey and us 1. Great place to wait out a rainstorm., +7961,#CecilTheLion The Latest: More Homes Razed by Northern California Wildfire - New York Times: ... http://t.co/Vi05Yq85Ly #Harare #263chat, +7964,'The Latest: More Homes Razed by Northern California Wildfire' by THE ASSOCIATED PRESS via NYT http://t.co/rDF0ZuW7lZ, +7966,#?? #?? #??? #??? The Latest: More Homes Razed by Northern California Wildfire - New York Times http://t.co/IPnHSIyorc-, +7967,? Razed In Black - Whipped ? http://t.co/oAE9LgtCtI #nowplaying, +7974,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/55IdFT20A2 http://t.co/2VSX1yO5hT, +7975,@RazeD_ yea in this photo, +7979,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/ATokCD74bA, +7985,ABC News The Latest: More Homes Razed by Northern California Wildfire ABC News The latest on___ http://t.co/RruNs5rDz5, +7986,World News>> 'The Latest: More homes razed by Northern California wildfire - http://t.co/M6OK4EH3ob' http://t.co/VGa0OhB6XF, +7993,From the Archive: National Poetry Month: Anterior of a Razed Room http://t.co/7ARN4OBiyC, +7995,The Latest: More homes razed by Northern California wildfire - SFGate http://t.co/65hunplHYB, +7997,Guess who's got a hilarious new piece on @RazedOnIt? @honeystaysuper! 51 Things You Should Never Say to a Mother Ever http://t.co/ikRHIb9x0a, +7998,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/ORdoKfUIBm, +8004,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/lj4J54vxq3, +8007,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/VP6wbXMZmj, +8011,'imagine an entire aisle dedicated to making people look like serbian refugees.' - director of whole foods clothing section, +8014,12000 Nigerian refugees repatriated from Cameroon http://t.co/LeLYa0vDOg read /////, +8022,Top story: @ViralSpell: 'Couple spend wedding day feeding 4000 Syrian refugees___ http://t.co/QAYayzh0fL see more http://t.co/oTqbHd2akn, +8027,Wow! Some powerful thoughts on the language we use to talk about groups of people on the move: https://t.co/KRbpf4Tp4I, +8033,wowo--=== 12000 Nigerian refugees repatriated from Cameroon, +8041,Top story: @ViralSpell: 'Couple spend wedding day feeding 4000 Syrian refugees___ http://t.co/5lMo710D5f see more http://t.co/G9LREAlToJ, +8042,./.....hmm 12000 Nigerian refugees repatriated from Cameroon http://t.co/FVTj8qwJZW /(, +8046,.@PeterDutton_MP Maintaining the integrity of our borders while not maintaining human rights of our refugees. Nice trade off., +8048,Newlyweds feed Syrian refugees at their wedding http://t.co/DGjBuGxH9E #changetheworld #SyrianRefugees, +8050,...//..// whao.. 12000 Nigerian refugees repatriated from Cameroon http://t.co/aVwE1LBvhn, +8051,Refugees as citizens - The Hindu http://t.co/GJSaAf3U6K, +8052,Still wonder why they will do anything to have a life anywhere. Wars u support CAUSE refugees open boarders. #auspol https://t.co/3MjtE74AiW, +8053,reaad/ plsss 12000 Nigerian refugees repatriated from Cameroon, +8054,"More dirtylying dishonest #refugees for the #UK. +http://t.co/3m7amh1E33 + +#WhiteGenocide #gays #Britain #British http://t.co/FzOv8WcbYI", +8059,@bcook28 @zacktiller30 bring in the rescue squad ??, +8063,Florida Firefighters Rescue Scared Meowing Kitten from Vent Shaft http://t.co/RVKffcxbvC #fire #firefighter, +8072,@X_Man_Kitty @WolvenBeauty @_NickWilson__ //Shit...alright time to launch a black ops mission to rescue my alpha, +8075,Last Chance Animal Rescue has 3 new posts. http://t.co/qtEqA5kvpy #animalrescue | https://t.co/ec46LyQQc6, +8076,@wcvh01 @1233newcastle @aaronkearneyaus @LesMcKeownUK PS I notice you are a vet?? I rescue kitties ?? .thank you for your service ??????????, +8077,Officials rescue 367 migrants off Libya; 25 bodies found - Fox News http://t.co/9CAignPR6S #NoAgenda #MvDint, +8078,kayaking about killed us so mom and grandma came to the rescue., +8080,8' MTech Assisted Open RESCUE Pocket Knife - NEW BLUE MT-A801BL zix http://t.co/51n2rZEBis http://t.co/P4lNbjDo0x, +8082,Beauty Deals : http://t.co/eUd317Eptp #4552 Lot of 50Mixed Colors 7.5' Scissors FirstAid Princess Care Rescue Tr___ http://t.co/mAHkV79SmW, +8086,"If a picture is worth a thousand words what would this one say?? + +How a Lexington veterinarian's passion for... http://t.co/Y8dY39OhAL", +8090,"Revel8ion Media recently went out to support the 'Rocky Fire' rescue efforts. This is what we filmed. +#RockyFire http://t.co/Haxz4XLs8d", +8092,I don__ªt want to be rescued either #bot, +8093,Val rescued the sister but alone died. In the end she wasn't selfish. It was beautiful but very sad #emmerdale, +8099,Worker rescued from top of Central Street water tower in Summerside http://t.co/MkMlCTYmee, +8104,10-Month-Old Baby Girl was Rescued by Coastguard after She Floated HALF A MILE Out to Sea! http://t.co/kJUzJC6iGD, +8107,So @edsheeran just rescued this year's #FusionFestival http://t.co/I6ldEtwB6h http://t.co/XUnS3zHIxa, +8108,I just rescued you in MGS Ground Zeroes @HIDEO_KOJIMA_EN haha. Amazing...Sept 1st cant come soon enough :), +8114,Stewart bought a new property earlier this year w/ the intention of providing a sanctuary for rescued farm animals. https://t.co/ExXtboah8h, +8115,Three suspects in police net for kidnapping & trafficking a 4month old Baby from Kontagora to Aba# Baby rescued alive., +8117,British trekkers rescued amid flash floods in Himalayas - BBC News, +8123,SkyNews: Migrant Boat Tragedy Survivors Arrive In Italy - Migrants rescued http://t.co/FORBy6MmKp, +8124,nigeriantribune: 4 kidnapped ladies rescued by police in Enugu | Nigerian Tribune http://t.co/7hmHpl1Z4y, +8127,NAIROBI Kenya (AP) ___ Two people died and 25 others were rescued after a passenger boat with about two dozen passengers collided with a, +8132,Incheon Prosecutors Office: Drop the charges against Nami Kim who rescued two puppies from 'Dog Hangi... https://t.co/vWzX2P6Dnk via @Change, +8134,Britons rescued amid Himalaya floods http://t.co/8f2YBus2Vk, +8139,New-born piglet running down a national speed limit road is rescued #oink #Dorset http://t.co/dOMvOx2Oui http://t.co/iEVdj6KBU2, +8145,Rescuers recover body of 37-year-old Calgary man from lake near Vulcan http://t.co/araUSJvsy9 http://t.co/U7ZWlk8THX, +8148,#italy Hundreds of Migrants Rescued After Boat Capsizes Off Libya: Rescuers saved 400 people though 25 died w... http://t.co/e48syD5N1E, +8151,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the Mediterran... http://t.co/WvMHX5tiel, +8152,Rescuers are searching for hundreds of migrants in the Mediterranean after a boat carrying as many as 600 people___, +8153,Bay Whale Worries Rescuers http://t.co/nfKTvVLWHm, +8157,The Rescuers and The Rescuers Down Under are on Netflix___ You__ªre welcome., +8160,Fears over missing migrants in Med: Rescuers search for survivors after a boat carrying as many as 600 migrants___ http://t.co/UUHNC8ZkOW, +8166,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the Mediterranean after a boat carryi..., +8167,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the Mediterran... http://t.co/xJm6Snk1G0, +8173,Woman__ªs GPS app guides rescuers to injured biker in Marin County http://t.co/hKv8XKtU2k, +8174,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the Mediterran... http://t.co/biiiYi7DaE, +8178,News Break Nigeria Rescuers search for survivors after a boat carrying as many as 600 migrants capsizes in the... http://t.co/lNiTVT7B53, +8179,Photo: theonion: Rescuers Heroically Help Beached Garbage Back Into Oceanå_ http://t.co/YcSmt7ovoc, +8185,When Rescuers Found Him He ... http://t.co/sTDCdHQxJX, +8186,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the Mediterran... http://t.co/XfS3pJhFh9, +8192,@CHold ironically RSL call their stadium the Riot, +8193,If John Bateman scores we riot... https://t.co/zuPz8ZlO2X, +8194,@CazorlaPlay Fran. Hey check out the Secret on how to get 325.000 Riot Points LoL Check it on my Bio Profile, +8197,"@AlexandBondarev Discovered by @NickCannon + Listen/Buy @realmandyrain #RIOT on @iTunesMusic @iTunes https://t.co/dehMym5lpk ___ #BlowMandyUp", +8198,Stuart Broad Takes Eight Before Joe Root Runs Riot Against Aussies: Stuart Broad took career-best figures of 8... http://t.co/2UmwMG7lvN, +8199,"@hawk_nancy Discovered by @NickCannon + Listen/Buy @realmandyrain #RIOT on @iTunesMusic @iTunes https://t.co/dehMym5lpk ___ #BlowMandyUp", +8200,To All The Meat-Loving Feminists Of The World Riot... http://t.co/tNI5SCpBMA #followme | https://t.co/OFtj2sXYPf http://t.co/lVYemK99uV, +8206,To All The Meat-Loving Feminists Of The World Riot Grill Has Arrived http://t.co/7TMBbGxSK8 @OwenHarris82, +8207,To All The Meat-Loving Feminists Of The World Riot Grill Has Arrived http://t.co/MK9ZIp2ro7, +8220,@Riot_Sweet @jfwong @MtgoDoc @JoshLeeKwai wait.. what dates?? ill be at MSG and PAX =(, +8222,#ModiMinistry Stuart Broad Takes Eight Before Joe Root Runs Riot Against Aussies http://t.co/rc22r1xlIs, +8227,I'm Role Play of Ulzzang Park Hyung Seok. 92 Liners. Be bleased with me. I'll do the same., +8228,To All The Meat-Loving Feminists Of The World Riot Grill Has Arrived: Pop quiz! Which do you prefer: feminist... http://t.co/MoGytSxvOa, +8229,Matt Baume Digs Into the the Controversial __÷Stonewall__ª Trailer and Who the Real Heroes of the Riot Were: VIDEO http://t.co/HRoxD1qpua #gay, +8230,Riot police intervene after Southampton and Vitesse Arnhem supporters clash: __¢ Fans clash in buildup to second leg___ http://t.co/I8GX67wbGv, +8233,Riot police intervene after Southampton and Vitesse Arnhem supporters clash: __¢ Fans clash in buildup to second... http://t.co/OBBcoMKAWf, +8248,"we shootn each other over video games................wow ghee sad +but I still don't see no rioting no walks no... http://t.co/O423U56yPh", +8249,#BHRAMABULL Watch Run The Jewels Use Facts to Defend Rioting in Ferguson: The socially minded duo takes on the... http://t.co/9iP2vqHFWi, +8255,Swear to god if they make faker play kog I'm rioting, +8260,#BHRAMABULL Watch Run The Jewels Use Facts to Defend Rioting in Ferguson: The socially minded duo takes on the... http://t.co/y9eTXHCvjr, +8263,@amiestager there's better alternatives than rioting and if we don't kill them they'll kill us..., +8265,I have stopped trying to figure out if THIS will be the misogyny/racism/religion-fueled shooting that will incite rioting., +8268,'Dont use \Allahuakbar\'' and start rioting. Do you guys even listen to yourselves? #SHAME''', +8269,@abbydphillip @dahboo7 Shh...it's only OK for the puny 13% black population to get mad. The gov can't control the majority rioting., +8272,@foxandfriends where is protest looting and rioting that is for ignorant people, +8273,@brownblaze on tumblr the day/night of it was all over. The first I heard anything about it anywhere elsewere simply reports of rioting., +8275,Why are we not rioting in the streets for change/a future? We outnumber lobbyists! #WakeUpAmerica #CleanPowerPlan http://t.co/sONJy4HK46, +8277,#DebateQuestionsWeWantToHear Are these thugs or boys being boys? https://t.co/T0aZ5d9BHK, +8281,.@runjewels appear in a new video by @BBC defending the riots in Ferguson: http://t.co/veh1seVDKO http://t.co/h2GziEfJXt, +8282,Half of Australia is currently rioting because they think 'Extras' is an aboriginal cricketer stealing their sport., +8290,http://t.co/DYKYpy3vA5 Cindy Noonan@CindyNoonan-Heartbreak in #Baltimore #Rioting #YAHIstorical #UndergroundRailraod, +8293,Outrage as dog found buried alive in rubble http://t.co/vrNRrpoHm8, +8299,home loan: JPMorgan builds up apartment-loan leader from WaMu rubble By David Henry NEW ... http://t.co/IT6wXiCorl #badcredit #loans, +8302,#360WiseNews : China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/QDwRjmmk89, +8303,China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/7R7ADdVbsX, +8305,RT PemantauJKT48: China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/g0En3ELVCL via melodyJKT___ http://t.co/KHrish3BP0, +8306,I don't think we'll get anywhere as a ppl until we heal. like we trying to build on top of ruins without clearing the rubble, +8307,China's Stock Market Crash: Are There Gems In The Rubble?: China__ªs stock market crash this summer has sparked ... http://t.co/OajW6rTFNc, +8308,China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/EpNju0pZjM #forbes, +8313,China__ªs e-commerce firms Alibaba (BABA) Tencent http://t.co/7VcChEIC5g (JD) are still growing revenue at 40% http://t.co/GQgE7weM3W, +8316,#360WiseNews : China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/vTJ1EoslrU, +8319,In the long run every program becomes rococo and then rubble. ? Alan Perlis, +8322,'Hiroshima never recovered and all the evidence was smothered under a thick cloud of rubble..' #jayElectronica, +8323,"@KurtSchlichter Grandpa fought his way across Western Eurpoe mustered out in Guam on his way to the Home Islands. + +Bounce the rubble.", +8325,just collecting the rubble, +8326,"#Nickelodeon paw patrol-- rubble #action pack pup & #badge LINK: +http://t.co/N7tG7RoAmU http://t.co/U12hHyu1dc", +8327,#Poetry #friendship Lifes Giants - Is life throwing rubble at you ? Are there giants in your mind ? http://t.co/G4wipbG0jx ~LyricalEyes, +8328,@accionempresa China__ªs stock market crash this summer has sparked interest from bargain hunt... http://t.co/6ygPwJkr4T @gerenciatodos å¨, +8331,China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/qEn1Z5uOeH, +8333,While the world screams terrorism Assad buries civilians under the rubble of their homes. #Aleppo 05-08-2015 #Syria http://t.co/7GcPtCdx7o, +8336,China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/eKO50hD5Gz #market, +8338,#360WiseNews : China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/r0FLXJx5vX, +8340,China's Stock Market Crash: Are There Gems In The Rubble?: China__ªs stock market crash this summer has sparked ... http://t.co/vZDqQ9yPN1, +8342,@OrianaArzola does he not have any manners or something? Jfc. You have all the rights to be mad! But hey try not to let this ruin your day, +8343,"so in my rant about how i dont allow peens near my face i realized why I'm still single. + +sex is going to ruin me. or lack thereof anyway.", +8353,fUCK I CANT WAIT TO RUIN AUBREY'S WHOLE FIC AND MAKE IT SO BAD, +8354,They like raunchy trash then make it real and ruin lives and the potential of phenomenally I assume self raised good Christian human beings, +8357,Any man who tries to be good all the time is bound to come to ruin among the great number who are not good., +8358,My sister really always has to ruin my life., +8359,FUCK MY LIFE THESE BOYS ARE GONNA RUIN MY LIFE FORREAL http://t.co/mf9nGfphbN, +8373,Reality continues to ruin my life., +8376,He said he's 'gonna put a ring in my Harvey's burger since i love it so much it's the perfect metaphor'.. NOOO you can't ruin my burger wtf, +8379,This fly is a major problem for us: It will ruin our batch and we need to destroy it and every trace of it so we can cook., +8380,Plans always ruin., +8381,Don't ruin a good today by thinking about a bad yesterday ????, +8384,To respect those who wait to watch #UltimaLucha any tweets will be spoiler free and not ruin the result. #LuchaUnderground, +8395,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/OFhjcn36o1, +8398,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/nvSjHdGDGZ, +8403,Gonna throw a huge party and play nothing but Sandstorm., +8407,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/YSrSKuyaSw, +8413,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/ELDcP3vO7C, +8417,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/1oemJTO9Dp, +8420,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/jqTMweDw33, +8421,I liked a @YouTube video from @centralupload http://t.co/EwLHrTREEP Oh oh!, +8424,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/Tr8YcHEFGQ, +8426,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/1ZdRKTgogD, +8427,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/TLcWeZPuKW, +8438,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/Fkb7iUAuL8, +8439,I just replied to somebody's comment to me saying'darude sandstorm' negatively and now I feel bad for doing it., +8442,I screamed when @g_montani @jessemontani were running to John for 1st place! I'm so happy I love you guys ?? #AmazingRaceCanada, +8449,Oh my god Rosie O__ªDonnell is on The Fosters and I screamed her name in a fit of shock and awe, +8456,I'm having a meltdown because of Game of Thrones ?? literally cried screamed and threw my computer #WHYYYY #redwedding, +8460,I just screamed when I saw Jordan come out the door behind Donnie on #Wahlburgers ??, +8462,@joshshatsays I JUST SCREAMED TO MYSELF BECAUSE YEAH, +8465,OH MY GOSH IM AT MY AUNTS HOUSE AND THIS POST IT WAS ON HER COUNTER AND I SCREAMED BC I THOUGHT IT SAID CHRIS KELLER http://t.co/DNofWtAkAt, +8466,@theboysftvines oh my god i SCREAMED WHEN I SAW THIS YESSSS, +8467,' #OTRAMETLIFE ' I SWEAR TO GOD I DIDNT EVEN READ IT PROPERLY AND I THOUGH IT SAID 'OMLETTE LIFE' AND SCREAMED. I NEED TO SLEEP OMFG. BYE., +8469,___@beachboyIrh: AT GMA THE SECURITY GUY SAID 'THIS IS A FAMILY SHOW' AND EVERYONE SCREAMED 'OOORRRR IS IT'___ I love this fandom, +8470,YE SBUJDJSJS YES YES I XUSKAK I SCREAMED PROTECT MY OTHER SON PROTECT UR BOYFRIEND YES https://t.co/kDqtqGK5pI, +8474,@frailnerves I SCREAMED, +8479,I have never screamed nor cried so much at one concert, +8481,@mo_ganwilson Verrry mad. like I just screamed I'm so pissed, +8482,@TopherBreezy it was my fav too!!! Amazing! Michael and I screamed the whole time ??, +8485,@CokeBoys__ yo best I screamed when I watched your snap chat of you smoking with that big ass cast ????, +8488,@TroyLWiggins I screamed, +8492,@camilacabello97 screaming internally and externally, +8493,"SCREAMING BECAUSE 5SOS IS IN TX +@5SOS", +8495,@camilacabello97 now I'm the one screaming internally and externally, +8499,I WAS WATCHING SHAWN ON YOUTUBE AND MY MUM WAS JUST LIKE 'HE'S QUITE HOT ISN'T HE' I AM SCREAMING AKFNEJF, +8501,It ain't nothin cut that bitch off more like cut that lying screaming gotdamn son of a bitch red head., +8519,@justinbieber @ArianaGrande IMA SCREAMING HSBFUCKJSJ, +8521,i'm fucking screaming http://t.co/H3MrqRrGfe, +8525,Can't stand when ppl Beyonc_©'fy certain tunes nd u lot will b screaming 'yaaasss' low it I'm not tryna hear a xfactor rendition of burna boy, +8527,IM SCREAMING ?????????????????????????????????????????? OMG WTF???????? https://t.co/bfEhX84lrj, +8529,IMM SCREAMING ARI IS HOLDING THE SIGN, +8532,*screaming internally with the cover of prodigy graphic novel because June and Day*, +8534,@GabrielleAplin you are literally my favorite person ever I'm screaming, +8535,"@LordMinion777 *screaming* i can't go!!! +:(", +8537,@MeganRestivo I am literally screaming for you!! Congratulations!, +8544,"-mom screams from kitchen- +'WHERES MY AVOCADO!?' + +-waits patiently for mom to beat the avocado off my head- :)))))) http://t.co/LUUaY2JSJn", +8548,one of my fav lydia screams is in 4x11 when she shows up at derek's loft, +8549,*screams internally*, +8552,@McKenzieBlackw1 love you! Sorry for my screams, +8554,screams into void? https://t.co/XRlzvFeiLH, +8563,@simplyysacred dude like he screams his soul out, +8564,@sugayiffer SCREAMS UR SUCH A CUTE HANAYO, +8565,@ShojoShit /SCREAMS, +8566,Curvaceous LADY screams like a mad dog in a Nairobi Club See what happened (VIDEO) - #KOT Join Mature... http://t.co/1szi9X22cr, +8568,*aggressively screams* https://t.co/8bHaejsUUt, +8569,Oh no! @ATPWorldTour is letting one of their players scream! Turning @CitiOpen off. Can't listen to the screams, +8573,@melodores @Hozier *SCREAMS*, +8583,SCREAMS I LOVE THIS VINE https://t.co/PKnBZLHOSc, +8588,OMGGGG SCREAMS I'M THINKING I'LL SEE ACTUAL DICKS, +8590,The screams when Niall touches his dick yes http://t.co/U5Iu6pfNh6, +8592,@elephino_ @Thrusta1 no but it's a convenient and seismic shift in 6 months from trying to bury the event and now allegedly 'enhance it', +8593,SEISMIC AUDIO SA-15T SA15T Padded Black Speaker COVERS (2) - Qty of 1 = 1 Pair! http://t.co/6OVxU5cy35 http://t.co/fAobg3I1zm, +8594,A subcontractor working for French seismic survey group CGG has been kidnapped in Cairo and is held by Islamic State the company said on W___, +8597,Creating updating and delivering content in channel sales is not an easy task. @Seismicsoftware helps break it down: http://t.co/q4FLTG7tzT, +8600,#Gabon : Oil and Gas Exploration Takes Seismic Shift in Gabon to Somalia http://t.co/SY8nqTLMD3 ^FJ http://t.co/yjDHoDdjox, +8601,The time for small steps has long been over. We need a seismic shift in thinking to address #migration in a holistic human & humane way., +8602,@kc8ysl Is it wrong to pray for seismic activity along the San Andreas fault? #smh, +8603,"Anti-fracking group seeks review over #Lancashire seismic monitoring: + +http://t.co/ZzbqmGNUsl #fracking http://t.co/Y23Xzl3fZv", +8604,AxWave enables a fast and accurate simulation of 2D and 3D seismic surveys in an acoustic medium #seismic #GPU #CPU http://t.co/OMX1NXAqpz, +8605,[10:45:27JST Aug06:First Alert] M4.1 at 'E off Chiba pref.' under 10km (35.8140.8). Estimated max seismic# is 3, +8611,My brain feels like it's vibrating inside my skull. The MRI is going to look like a seismic readout of a faultline during earthquake season., +8614,TBI early brain aging and a seismic analogy http://t.co/xs1QHfjAG4 @deetelecare #TBI #braininjury, +8616,Exploration takes seismic shift in Gabon to Somalia ___ WorldOilå_(subscription) http://t.co/Hs6OhRFsA9, +8617,#Fukushima still is not controlled & #seismic retrofitting for other #reactors is a BIG issue @fukushima_actu http://t.co/lCDkTEOukj, +8619,#Sismo DETECTADO #Jap__n [Report 6] 01:02:42 Okinawa Island region M3.8 Depth 10km Maximum seismic intensity 3 JST #??, +8626,"Nuclear-Powered Brassiere Explodes During Seismic Survey http://t.co/76mKIdOOLh + #Calgary #Edmonton #oilandgas #Vancouver #Toronto", +8629,The Art World's Seismic Shift Back to the Oddball - Observer http://t.co/AOWVFAbl0a, +8630,On Thursday at 03:27 we updated our #kml of 2D and 3D #seismic exploration vessels. #offshore #oil http://t.co/btdjGWeKqx, +8632,Here we go....one more Toss with the Seismic boys! With a stellar undercard all killer no filler keep an eye... http://t.co/Xs6aa2c6Jb, +8634,Exploration takes seismic shift in Gabon to Somalia http://t.co/dmVR2H2fKM, +8641,@Statoilasa & @TOTAL make significant discovery in the North Sea: Visualise the full potential on our seismic #g http://t.co/ve2eBqm21B, +8645,http://t.co/OK9EBHurfl If your neighborhood school falls into a sinkhole State Sen. Steve Yarbrough may be close by..., +8649,Some Evansville residents told to limit water usage due to sinkhole construction http://t.co/Lw6NeQJRoT, +8652,Latest: USA: Huge sinkhole swallows up Brooklyn intersection http://t.co/vspKHg3nZy, +8666,A Sinkhole caved in a Brooklyn Street today on 5th Ave & 64th Street. http://t.co/eUKhLKgGdO, +8671,"Sinkhole Selfies: You Wont Believe What's In The Brooklyn Sinkhole!: + Sinkhole Selfies: You Wont Belie... http://t.co/OFEbaKatNh", +8672,Large sinkhole swallows entire pond in Lowndes County Georgia http://t.co/20Gi4Gyud5, +8675,A sinkhole grows ... in Brooklyn? http://t.co/UiqKDdVwKz http://t.co/aa8FwtOHyJ, +8678,Giant Sinkhole Swallows NYC Intersection http://t.co/ozIZdsDWP4 via @NewsBeatSocial, +8679,A sinkhole grows in Brooklyn: six-meter crater swallows street http://t.co/LTLh53aRN4, +8682,The sinkhole that ate Brooklyn http://t.co/28r2IgxmIE, +8687,Watch This Giant Sinkhole Form on New York City Street http://t.co/BEnWu5IARa, +8701,Sinking a little slower everyday ?? @ Muh Pond https://t.co/KuA48GdREL, +8703,"If you're lost and alone +Or you're sinking like a stone. +Carry on +May your past be the sound +Of your feet upon the ground", +8707,@andreajmarkley @edjschenk @SenTedCruz If you call forward sinking into a cesspool you might be right, +8713,Do you feel like you are sinking in unhappiness? Take the quiz: http://t.co/LNzz0ZFrom http://t.co/TGKpnqAbHh, +8716,Don't think I can be more blunt lol yet the message still isn't sinking in ??????, +8719,HELP ME I'M SINKING, +8725,that horrible sinking feeling when you__ªve been at home on your phone for a while and you realise its been on 3G this whole time', +8730,"Sinking. {part 2} +?????? +#buildingmuseum #TheBEACHDC #VSCOcam #acreativedc #dc #dctography #vscodc___ https://t.co/SsD9ign6HO", +8731,heart sinking like a sunset?? https://t.co/3cXPRDJFfe, +8748,@kundalini121 @manyvids thank you again... ir truly wonderful, +8749,@ryan_defreitas for me it's Revs Per Minute but the opening tune on Siren Song is UNREAL, +8756,"@Siren_Six @B30wu1f2 +Well then we see things differently.", +8764,@Foxy__Siren best wishes boo????????, +8765,WHELEN MODEL 295SS-100 SIREN AMPLIFIER POLICE EMERGENCY VEHICLE - Full read by eBay http://t.co/Q3yYQi4A27 http://t.co/whEreofYAx, +8769,@Kronykal @B30wu1f2 I see no point in countering skewed statements. Dumb statement is dumb., +8771,WHELEN MODEL 295SS-100 SIREN AMPLIFIER POLICE EMERGENCY VEHICLE - Full read by eBay http://t.co/UGR6REFZpT http://t.co/eYyUqX4Tbt, +8772,I love dat lady '@Crhedrys: ???? you nko?'@Foxy__Siren: Oh finally Jennifer Aniston got married??????... I'm so happy for her ??????'', +8773,Police siren sound effect 1 via Play Tube Fre http://t.co/BLWnTMyQmQ, +8779,Ebay Snipe RT? http://t.co/tRw1OmykQz Whelen Sa-314 Siren Pa Speaker Mount Bracket Universal With Screws ?Please Favorite & Share, +8792,"Check out what I found on Google images! +Something to live by! ?? http://t.co/WjOzlQBiE0", +8794,That new sirens and sailors ???? #workout #album, +8796,@HusnaaVhora I hope they dust off the air raid sirens! You can quote me on that lol, +8798,@BobbyCakes50 so far miss may I sleeping with sirens August burns red Memphis May Fire Attila & all were so good ??, +8803,want a new season of The League and Sirens on Netflix., +8807,and my dad is high I have a dysfunctional family, +8809,SOMEBODY TELL ME HOW TO MEET PIERCE THE VEIL OR SLEEPING WITH SIRENS PLEASE I BEG YOU, +8811,pos nose que hago escuchando Sleeping With Sirens - Iris (Goo Goo Dolls Cover) KE PEDO ._., +8813,Sleeping With Sirens - Iris (Goo Goo Dolls Cover) http://t.co/KaeWtkJ06o, +8814,I'm falling to pieces and I won't be whole until you let me in ? Heroine by Sleeping With Sirens ___ https://t.co/tQxLQUKq7z, +8818,I was walking past the fire house today & as soon as I walk by the doors they get call and flip the sirens on. I've never ran so fast., +8819,why is this reminding me of dan's collab with tyler when he complained about the sirens and they stopped i'm trash, +8820,There was also a calm. Something was going to happen. Suddenly a crash of lightning came through the sky and with it the wails of sirens. --, +8821,Sleeping With Sirens - 2 Chord, +8822,@jassy_curtis wait I'm fckng obsessed with your song sirens, +8824,@spookyerica sleeping with sirens?, +8826,sleeping with sirens are so irrelevant, +8827,Hear the Sirens ?, +8828,@salem_police ok so no sirens but a speaker system for chip music for a food cart should be fine with underskirt lighting?, +8833,Fire truck sirens for the past ten minutes straight coming and going. Must be huge. Maybe it's taco bell and they'll build a Sonic instead., +8834,Sleeping With Sirens - Postcards and Polaroids., +8845,Smoke sumn, +8847,just trying to smoke and get Taco Bell, +8849,@wizkhalifa smoke ??so much you don't gotta do shit but light?? one of his locks on fire ...that's like a whole tre five right there., +8861,misocapnist: One who hates the smell of tobacco smoke #UnusualWords http://t.co/jMcFFA3TOQ, +8867,I need a smoke ??, +8868,[55433] 1954 LIONEL TRAINS SMOKE LOCOMOTIVES WITH MAGNE-TRACTION INSTRUCTIONS http://t.co/2W2EtbTGdr http://t.co/MWS8130h8n, +8870,would definitely have way more money if i didnt smoke on the dl, +8873,Waiting for Joel so we can go smoke ????, +8874,To treat his childhood asthma Teddy Roosevelt's parents made him drink black coffee and smoke cigars. http://t.co/CtaC2xZ8dY, +8876,I'm not asking no fee to smoke sumn if we ain't smoking out a ounce that's disrespectful, +8877,Send me flying off a cliff straight through the smoke of my spliff, +8879,[55432] 1950 LIONEL TRAINS SMOKE LOCOMOTIVES WITH MAGNE-TRACTION INSTRUCTIONS http://t.co/8s2GxMhiSJ http://t.co/GAwvhG5aSb, +8882,Do you guys remember my life goal to smoke a blunt with zouis? Well it's still a goal, +8884,"*anti cigarette smoking commercial comes on* +me *goes outside and smokes* +thanks for reminding me to go smoke lol", +8889,smoke me out, +8891,Just want someone to smoke a blunt & talk about life with ??, +8894,How does a snowstorm happen? Find the answers here: http://t.co/m75fCLUtF2 #snowstorm, +8895,you're my snowstorm baby ??, +8897,Tess was almost born in a snowstorm. I was 39 years ago. Here's her birth story. http://t.co/UnyuRR8baw #VBAC #NaturalBirth, +8898,She rose to her lofty position after being transported by accident to Oz in a hot air balloon during a snowstorm., +8901,@Lily_Bell82 ya...summer is too short. Open toed shoes arent snowstorm friendly., +8904,It's creepy seeing 676 closed. It was closed during a snowstorm and my balcony overlooked the 676/76 overpass. Very eerie, +8909,Had a dream that @woodelijah was driving me through a snowstorm. We went to a restaurant and I left my journal behind. #writerslife, +8910,RT RTphotographyUK: New #photo Oak in a snowstorm http://t.co/gWOKZELq5Q taken in #winter on the #SouthDowns #Hampshire #photography #art___, +8911,That organic freestyle with Casey veggies is cold as an ice cream cone in a Iceland snowstorm, +8915,@BigBang_CBS ...wow...ok...um...that was like ice water blizzard snowstorm to the face., +8917,@erincandy Hr/15 mins for me in that place. During a snowstormI was pissed lol We got oil delivered to them on snowmobiles in the end, +8918,A snowstorm is brewing over my desk & hands are NUMB.. @irishirr - I'm guessing you adjusted thermostat? :/ @WSAZ_Brittany @kellyannwx #WSAZ, +8919,Long Island is technically the leftover dirt from a snowstorm and thats a geologic fact. That place is terrible., +8922,#LakeEffect #Snowstorm Twill Denim Jackets *** ALL MY PROCEEDS FOR ITEMS WITH THIS DESIGN WILL GO TO HELP THE... http://t.co/QmtDXuYktY, +8925,New #photo Oak in a snowstorm http://t.co/gWOKZELq5Q taken in #winter on the #SouthDowns #Hampshire #photography #art #tree #treeporn, +8928,@40URTY @DonOf_NikeTown snowstorm is my fave episode lmao, +8929,RT @WhistlerBlckcmb: One more #SnowBackSunday share: Snowstorm on the Glacier July 26 2015! via @momentumcamps http://t.co/JohKyMv9El, +8930,The sun is shining on this house now but imagine trying to build this summer home during a North East snowstorm!... http://t.co/BtJL9JkvXZ, +8932,@asda bought this bargain dress and love it- cut colour pattern. Are you doing more? http://t.co/BnZm8K2AiM, +8937,This week on dream from last night: my mom drove my car in a snowstorm. She hit another car while parallel parking. Saw my first ex......, +8940,"I feel like the cold homeless orphan gazing into the cozy mansion of the affluent eating a lobster dinner during a snowstorm. + +It's fine.", +8943,If you like rock music you should care about 5 Seconds Of Summer. Here__ªs why. paper-storm: I am a fan of Au ___ http://t.co/eqEeUVpRQQ, +8950,Warcraft 3-Inspired Mode Likely Hitting Heroes of the Storm http://t.co/848CVWWdOt, +8951,Well this storm hit out of no where! ??, +8954,"If you've got it flaunt it! +Unless you've got an awful lot of it...then please don't.", +8958,The Florence Regional Airport Director says no major damage at airport from yesterday's storm. No evacuations and no blown out windows., +8960,Want to know how to get the edge and storm Rock Paper Scissors http://t.co/YHzo0L7vtV #GameChanger, +8964,Storm & Family tonight http://t.co/0aP7MoNtjF, +8975,IGN: A Warcraft 3-inspired mode is likely coming to Heroes of the Storm http://t.co/2qVgImRo0R http://t.co/VTRoCiHSne, +8977,Another next day #ThrowbackThursday of Kinky Kristy and @KyleSalive rehearsing up a storm! #DoingHashtagsRight http://t.co/UJ9hxUg7QS, +8981,Aurora Prediction: Kp: 3.33(11:16) 4.67(14:16) AK (earth 2.00) 75% Status: Low Storm, +8982,@DovahSwag @Gray_Greymane DotA 2 > Heroes Of The Storm > League Of Legends, +8984,Storm rolling into Hilton Head gonna be fun, +8988,Man united is not just about playing football like he did back at Spain its about passion courage and endurance to ride the storm., +8990,__÷Sexist__ª peer review causes storm online https://t.co/WO2T0K4zXi via @timeshighered, +8991,"Gets violently ill and it's obvious +Male coworker(unaware of why): don't get everyone else sick! +Sir you cannot catch this #periodstories", +8995,Renewing Your Kitchen Cabinets for $100 http://t.co/lYs6p5BmDL http://t.co/ELfzsRGxRo, +8997,Yes. He said its dropping soon '@SDotJR_: NO WAYS!!! A video for 'Stretcher'?! @Ofentse_Tsie', +9003,? Stretcher in 5 min // Speaker Deck http://t.co/gXgJqQu3hU #pctool, +9004,Jesus!!! How does a niggaeven life his hand to do that and not leave on a stretcher https://t.co/EewIzzryjI, +9010,WNBA: Stars coach Dan Hughes taken off court on stretcher after collision with Danielle Robinson (ESPN) http://t.co/Jwm87uvbEf, +9011,@TheJasonTaylorR *EMS tries to stablize me and put me on a stretcher*, +9013,@Stretcher @Grazed @witter @Rexyy @Towel thanks aiden, +9014,Oh no! Dan Hughes seems like he's really hurt after getting knocked down by D Rob. They're bringing a stretcher on the court!, +9015,Mind salivation stretcher beds: KEGm, +9017,He's about to leave on a stretcher., +9019,I was there to see @JimmieJohnson wreck in 2000 in person. My first thought was he's leaving on a stretcher!, +9021,"Ambulance stretcher simple stretcher emergency medical care http://t.co/hYHoFDdWl7 #0518 + +$169.50 +End Date: Wedne___ http://t.co/rMdfx6y0HO", +9024,Stretcher in 5 min https://t.co/DIWLBiaHvP, +9032,lil more i did want a stretcher yc, +9035,Stars HC Hughes down w/ what looks like lower back injury. Being put on stretcher. 4:46 to play in the 2nd Wash up 25-23, +9036,georgegallagher: RT WNBA: Stars coach Dan Hughes taken off court on stretcher after collision with Danielle Robin___ http://t.co/wz4xVZYqJd, +9037,Throat stretcher http://t.co/vkThHbOQjs, +9043,Investigators say a fatal Virgin Galactic spaceship crash last year was caused by structural failure after the... http://t.co/FoxpVpTttA, +9046,Investigators say a fatal Virgin Galactic spaceship crash last year was caused by structural failure after the co-pilot unlocked a braking, +9048,Investigators say a fatal Virgin Galactic spaceship crash last year was caused by structural failure after the... http://t.co/QOs0DE9BKQ, +9049,Investigators rule catastrophic structural failure resulted in 2014 Virgin Galactic crash http://t.co/nLzbHcfXCM, +9050,NTSB: Virgin Galactic crash caused by structural failure triggered when brakes unlocked early http://t.co/MgIG4uX7DM ___, +9056,Building structural integrity & failure: problems inspections damages defects testing repair | Rightways https://t.co/a1KN4U3YuA, +9063,@paulmyerscough but also a complete failure of imagination when it comes to institutional/structural analysis regardless of politics, +9065,Virgin galactic crash: early unlocking of brakes triggered structural failure http://t.co/3YLSQ1fSEf http://t.co/ABpMLymHNK, +9066,Investigators say a fatal Virgin Galactic spaceship crash last year was caused by structural failure after the... http://t.co/SYL7ou7qVW, +9068,http://t.co/NSA4kfN2RB Coal Industry's imprudent decisions! Like @TonyAbbottMHR FAILURE to recognize structural change! LOOKOUT INVESTORS!, +9069,___@NewsHour: Investigators rule catastrophic structural failure resulted in 2014 Virgin Galactic crash http://t.co/M8HRetrO31___, +9070,Can You Afford Structural Failure? http://t.co/X3mKNITh7K, +9073,The NTSB reports that last year's crash of a Virgin Galactic spaceship was caused by structural failure after the ___ http://t.co/vSAPkWJTEA, +9076,#BBCLive Investigators say a fatal Virgin Galactic spaceship crash last year was caused by structural failure ... http://t.co/u5suyvLfZJ, +9091,@GazelleBundchen different materials etc. Just because a closer building stayed intact doesn't negate a structural failure in building 7., +9092,International News __¢å_'Nigeria suicide bomb kills at least seven at market ' via @233liveOnline. Full story at http://t.co/MtTdvy6141, +9104,Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/VtVXa0Srmw http://t.co/Qq70Gf99ba, +9105,reaad/ plsss Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released, +9108,reaad/ plsss Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released, +9111,#GRupdates Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released -->... http://t.co/hUCyI6NRbB, +9115,ll//ll= Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/8vBgg9IY7i /, +9117,./.....hmm Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/5v29w19tFX /(, +9122,wo Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/Sj57BoKsiB /'/'//, +9125,Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released: Harun __ekdar a member o... http://t.co/v0NTQyZlPK, +9127,International News __¢å_'Nigeria suicide bomb kills at least seven at market ' via @233liveOnline. Full story at http://t.co/MtTdvy6141, +9129,#Bestnaijamade: 16yr old PKK suicide bomber who detonated bomb in ... http://t.co/KSAwlYuX02 bestnaijamade bestnaijamade bestnaijamade be___, +9133,#GRupdates Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released -->... http://t.co/P5yASKTq0K, +9136,Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/I3ceSn7KXI http://t.co/2yCM0dlQ2f, +9138,#Bestnaijamade: 16yr old PKK suicide bomber who detonated bomb in ... http://t.co/KSAwlYuX02 bestnaijamade bestnaijamade bestnaijamade be___, +9140,Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released: Harun __ekdar a member o... http://t.co/fMoqK26hIm, +9147,#Noticias Suicide bomber kills 15 in Saudi security site mosque http://t.co/oipt1uBaXo, +9150,Suicide Bomber Kills 13 At Saudi Mosque: Back in May a suicide bomber struck a Shi'ite mosque in eastern Saud.. http://t.co/FlrOvkANC0, +9152,Suicide bomber kills 15 in Saudi security site mosque http://t.co/1I62JjK6pT, +9153,"qtSZg mhtw4fnet + +Suicide bomber kills 15 in Saudi security site mosque - Reuters", +9155,Suicide bomber kills at least 15 in Saudi mosque: A suicide bomb attack inside a mosque in a local security fo... http://t.co/n6njdG6r9l, +9158,New post: Suicide Bomber Kills 13 At Saudi Mosque http://t.co/YcuV4KTgrl, +9160,Suicide bomber detonates in Saudi Arabia mosque 17 reportedly killed http://t.co/BNVCvNYXXj, +9163,Suicide Bomber Kills 13 At Saudi Mosque http://t.co/VHQeD0gO0n, +9165,Suicide bomber kills 15 in Saudi Arabia mosque used by the police :IS claims responsible for barbarism http://t.co/Ux5YA1cSG3, +9167,@ZeeNews How come no one asked the obvious QN? Why show a Hindu suicide bomber? Never heard of one., +9168,More details: Bomber kills at least 13 at #Saudi mosque in Asir - including 10+ members of 'special emergency forces' http://t.co/i1626mnNd9, +9170,Suicide bomber kills 15 in Saudi security site mosque - Reuters http://t.co/KCObrZBVDs http://t.co/y62HSFVIAQ, +9176,[The Guardian] Islamic State claims suicide bombing at Saudi Arabian mosque http://t.co/tqvPUSaFJQ, +9178,Google News CA: Suicide bomber kills 15 in Saudi security site mosque - Reuters: Wall Street Jou... http://t.co/mawD20REGb #News #Canada, +9184,#?? #?? #??? #??? Suicide bomber kills 15 in Saudi security site mosque - Reuters http://t.co/ebGGmfxoBy, +9186,Islamic State claims suicide bombing at Saudi Arabian mosque: Ten members of emergency services... http://t.co/0Bznfdg0aR #Kabari #World, +9188,'Islamic State' group claims Saudi mosque suicide blast: The jihadist group claims it sent a suicide bomber to___ http://t.co/H2CTpG4oyZ #dw, +9189,17 dead as suicide bomber attacks Saudi Arabian mosque http://t.co/d5cUDwuhww | https://t.co/twnsWk0hFJ, +9190,IS claims suicide bombing against Saudi police: An Islamic State group suicide bomber on Thursday detona... http://t.co/yoXZFbRvzI #news, +9196,Career goals ?? try not to cause a suicide bombing or raise psychos that like blowing shit up!, +9198,@JRehling @MyyTwoCentss And if your best evidence is the word of a guy who encouraged suicide bombing as a means to get to heaven.....well.., +9199,AFP: #PKK said it carried out suicide bombing against troops in eastern #Turkey as a reprisal for 'bombing of a civilian village', +9200,"Erdogan__ªs Bloody Gambit: +Max Zirngast in Jacobin: +On July 20 a suicide bombing took the lives of thirty-one... http://t.co/l1vN8zXLGG", +9201,@samfbiddle If Gawker so badly wants to commit suicide why don't you just volunteer your offices to test the new F35's bombing capabilities., +9202,#ISIS claims credit for suicide bombing 'with 6 tons of explosives' at the central command of #Syria(n) T4 airbase http://t.co/QOaU2QLJ3B, +9204,IF SUICIDE BOMBING WASTHE SMARTEST THING2 DO FOR ALLAH/GODJESUS/THE HOLY PROPHET MUHAMMAD COULD HAVE KILLEDSOMEBODY? http://t.co/tGfWuVVHxj, +9205,Kurdish Militants Target Turkish Military In Suicide Attack - Huffington Post http://t.co/03bJm4ORoW, +9213,@PieroCastellano @nipped suicide bombing civilians in major TR cities yet?, +9214,Remembering Sgt.-Maj. Roni Ghanem 28 of Maghar murdered by Hamas terrorists in the suicide bombing of Egged bus No. 361 13 years ago today, +9224,Turkish troops killed in Kurdish militant 'suicide attack' http://t.co/q1eIXnjjpt, +9227,@mehdirhasan percentages of young Muslims who believe suicide bombing is justified per pew research: 35% Britain 42% France 29% Spain, +9228,PKK Suicide Bombing Kills 2 Soldiers in Turkey http://t.co/dfanUDwSQ3, +9229,YAOUNDE Cameroon (AP) ___ Authorities have ordered the closure of mosques and Islamic schools following a series of suicide bombing attacks, +9234,@61_alvin @Craigyb74 @nomar4eva @Shahasda80 @wherepond @trensabby @parallelpond the wall Israel erected has stopped suicide bombing?, +9236,#ISIL has claimed credit for three suicide bombing attacks targeting regime checkpoints on the outskirts of Al-Qaryatayn. #Homs #Syria, +9241,MUST-SEE: Massive ISIS Suicide Truck Bombing Caught On Tape http://t.co/hMsPYpozLn via @GabbyOgbechie1TPGazette, +9243,According to the tabloids Cilla could've been saved. In the sense that Titanic wouldn't have sunk had it avoided the iceberg., +9245,Managed to shuffle over to the bed. Butterflies swarmed in the pit of her belly at @VengefulKnave's offer. Her canines sunk into her lower++, +9256,Today 5/8/1941 Today 1941 British cargo ship Kumasian torpedoed/sunk Atlantic lost 1 of 60 lives Any1 have link? #crewlist #WW2, +9258,@widow_elena @stelena_always The show sunk because of DE. Are you really that blind? Keep trying with me though ??, +9261,the idea of an album that is entirely ZAYN just sunk in to me and im loving it, +9274,IT JUST SUNK IN THAT IM NOT SEEING ONE DIRECTION, +9279,"Sweet jeezus. How is this an election issue? +This is how far we've sunk. https://t.co/XJ5BklBxWI", +9282,Think it's actually sunk in that we've left school ??????????, +9284,"Check out my Lightroom collection ___Jordyne Sr Photos Lincoln Hay Col Sunk___: +more of the results from Sunday + +http://t.co/KvBEdKGlgw", +9285,@Birdyword Nice piece you wrote on #RBS sale. Think sunk-cost fallacy disposition effect http://t.co/UIvpE74kUi https://t.co/LdPrUf2wuR, +9291,12 was parked in front my house heart sunk Ta my toes... Lol all I was thinkin 'yal bitches ain finna get me dis time', +9295,@ASOLIDRIGHTHOOK omg I remember that?? she just didn't give a shit yet somehow she managed to survive Idek how, +9299,@amyschumer you eat food in order to survive?! Ugh and to think how much I admired you., +9302,Only 3 months to XCOM 2. I can survive that long. Right? Right?, +9304,theyre training all the celebrities 2 b superheroes so wen the apocalypse occurs theyll survive/repopulate earth w/ superior/hot beings, +9305,"#Rick Grimes: 'We'll survive. I'll show you how.' + +#TheWalkingDead +Season 5 +#Conquer +29 Mar 2015 http://t.co/PbAFmAlgtb", +9306,You deserve love but a queen don't need a king to survive??, +9310,'I mean if the relationship can't survive thezlong term why on earth would it be worth my time and energyqfor the short term?', +9312,#ScienceDaily Parental experience may help coral offspring survive climate change http://t.co/M2tJoxkCoU, +9316,No @Rookiebluetv tonight?? how will I survive?! @RookieBlue_ABC @mperegrym, +9318,"The #Future of Insurance and Driverless #Cars: Can the #Insurance Industry Survive #Driverless Cars? +http://t.co/zEARtXBOFt", +9323,in this game you and candy and friends attack in the pizzeria your mission survive the night, +9326,@wildestsdream idkidk they're not breaking up although if they do i'll survive because solo harry bless, +9327,Just stop and survive off that there such purchases, +9328,did harry survive the show, +9331,#WWII #SGM T34 in flames but with tank of gasoline to the outside it is likely that the crew survive . http://t.co/Psz22ky0fM, +9335,Survive! ???? no sleep as in!, +9336,#Aquarius women are not the vulnerable type so you do not have to worry about her. She will survive all by herself., +9340,Took the time to finally install #win10 on one of my systems. Let's see if it will survive a weekend of #lanparty., +9349,"Grandma's friends survived to the attack in Hiroshima but they died with cancer when kids/teenagers?? +#BombEffects", +9352,Fotoset: ___You are the person who survived a bunch of rainstorms and kept walking. I now believe that pain... http://t.co/xEGNtXnH3B, +9353,@FoxNews my father's mother survived it. Never met her., +9354,Good news Adam Barton survived the helicopter crash in #Emmerdale maybe now he can start training & get match fit. #Pompey, +9356,Don't even know how I survived that http://t.co/zigDPaCq5B, +9357,Lets see how long i survived b4 i fall back to dreamland, +9358,@Babypicturethis SNACKS SNACKS SNACKS. Snacks is how we survived flights with W&P when they were wee., +9359,Bucket list checked 3 hours of sleep in 2 days with 3 presentations on the second day and survived ??, +9364,Survived my first #tubestrike thanks to @Citymapper, +9366,The 390-Year-Old Tree That Survived the Bombing of Hiroshima http://t.co/kEirA8MA3K, +9369,@justgetawayx everything will turn out fine!! I went to lp/om&m alone and survived it and so can you, +9370,You'll be pleased to know I survived my dental appointment ??, +9379,Oh my days!!!! Thank God Cain has survived!!! ??#Emmerdale #SummerFate, +9381,How would you like to be remembered? ___ On how i survived big achievements and what i really am. http://t.co/X7r2LNjsh8, +9383,Wow that was a tough final. I barely survived ??, +9389,I survived the tube strike! This is my victory smile ;D http://t.co/L3Y3x2Cats, +9390,Trapped a bee in my shoe only one of us survived I just ran 0.15 mi @ a 13'46'' pace with Nike+. http://t.co/IbG4ynJsw8 #nikeplus, +9392,Important piece on long-term #health effects of #Hiroshima by @juliepower: http://t.co/J2s4Ng7wGt via @smh @croakeyblog, +9394,Haunting memories drawn by survivors http://t.co/a4H2e5xhiK via CNN, +9398,The Cafe Run by Acid Attack Survivors in #India http://t.co/XtVRJMRREs http://t.co/ndvlAPJvQL, +9399,CNN: Haunting memories drawn by survivors http://t.co/ZEnDvsgYMD, +9400,Want to meet @incubusband? Earn your way by watching @thematthewcooke's #SurvivorsGuidetoEarth! #Incubustour2015 - http://t.co/QW5XcXenRD, +9407,Aya Kano__ªs family has taken two nuclear hits: First #Hiroshima. Then #Fukushima. http://t.co/kcx9w2EJ8A http://t.co/aC50jw3Xmw, +9409,See the comment from @yogal700 on @thejournal_ie:L_ä Niamh leads search for survivors of capsized migrant boat.... http://t.co/TAKgauQ6pd, +9412,#orlando Survivors of Shanghai Ghetto reunite after 70 years http://t.co/f8pE52ZeyX, +9413,As Anniversary Nears Atomic Bomb Survivors Speak Out Against Nuclear Power | Common Dreams http://t.co/n8f2dXVduD, +9415,'We need to talk about our bums!' Bowel cancer affects young women too (via @WomensWeeklyMag) http://t.co/QJBa3EtfhV, +9419,Haunting memories drawn by survivors http://t.co/kDh4evYWpt via cnnbrk CNN #news, +9423,A new favorite: Hardwell & Dannic - Survivors (Remake) [FLP Family] by @FLP_Family https://t.co/yGVMztP1rt on #SoundCloud, +9424,"RT @SyedIHusain @penelopephoto +The man who survived Hiroshima: 'I had entered a living hell on earth' http://t.co/qFinwD7ls7", +9426,No text is worth your life. Hear stories of survivors and pledge not to text and drive. http://t.co/xkAfgsGmZ5, +9427,@TheNerdsofColor @GeorgeTakei narrates our new doc about #Hiroshima WWII bombing and survivors tales. Check it out http://t.co/33RfRc7ZqT, +9428,The legendary dutch producer Dannic is making his way to Denver to bring his progessive house sound to... http://t.co/s3JhXiDslZ, +9429,@ShaunKing Check this out: Tribute to survivors of sexual assault by law enforcement officers on facebook., +9433,Hiroshima schoolgirls archive memories of A-bomb horror http://t.co/32IzXXrUdP via @ABCNews, +9437,"Your donation will make a difference in the lives of #survivors of #braininjury. Please donate today. + http://t.co/noF8f0RScb", +9440,Survivors! Isabel Hodgins Danny Miller Laura Norton & Mark Charnock take a group pic #SummerFate #BehindTheScenes https://t.co/qUKpVpwRFv, +9442,"Cross-border terrorism: Pakistan caught red-handed again +http://t.co/FfbnLuiy0f via@timesofindia", +9443,@BBCNewsAsia State sponsored terrorism., +9444,Judges considering fate of children as young as two amid radicalisation fears http://t.co/EgWU4mJV0a, +9447,A Brief History of False Flag Terrorism | Wake Up World - http://t.co/KTeqJRXzpc via http://t.co/yZxlQeMKPF, +9459,Online Homeland security: An Appraisal of #Pakistan__ªs #Anti-Terrorism #Act http://t.co/aeRU3nStpN, +9461,"Truth... +https://t.co/GLzggDjQeH +#News +#BBC +#CNN +#Islam +#Truth +#god +#ISIS +#terrorism +#Quran +#Lies http://t.co/MYtCbJ6nmh", +9462,Combating #fear of #terrorismturn off your tv &Veterans For Peace Calls 4 an End 2 #NATO #VFP http://t.co/YPPG4kZIpS http://t.co/cBfdI6LF2X, +9464,Cross-border terrorism: Pakistan caught red-handed again http://t.co/NVGNF9AcuU, +9473,New US counter #terrorism program urges #Muslims to rat each other out critics say http://t.co/r0w85x3b0z http://t.co/VUUQqBo6mp, +9475,People buyout over terrorism and Isis but have a better chance getting shot at in a movie theater https://t.co/jkdjH2zw8s, +9476,Cross-border terrorism: Pakistan caught red-handed again @timesofindia http://t.co/ZfdcWV4ch0, +9477,"?? Iran largest sponsor of terrorism +?? If #IranDeal goes thru US gives $150 Billion + +?? FACT: US will become largest supporter of terrorism", +9478,"Truth... +https://t.co/Kix1j4ZyGx +#News +#BBC +#CNN +#Islam +#Truth +#god +#ISIS +#terrorism +#Quran +#Lies http://t.co/pi6Qn7y7ql", +9479,New US counter #terrorism program urges #Muslims to rat each other out critics say http://t.co/mVn6T9VhUl http://t.co/7lUfjX684i, +9483,@Daggy1 Maybe reagan giving weapons to Iran's regime in the 1980s is terrorism no?, +9487,"Truth... +https://t.co/n1K5nlib9X +#News +#BBC +#CNN +#Islam +#Truth +#god +#ISIS +#terrorism +#Quran +#Lies http://t.co/CGz84MUOCZ", +9494,Terrorist attack in #Udhampur district in J&K; 2 SPOs injured, +9497,Udhampur terror attack: NIA takes over probe Pakistani terrorist quizzed; Pak denies link http://t.co/i3gLzABeGl, +9498,Three Israeli soldiers wounded in West Bank terrorist attack - Haaretz http://t.co/Mwd1iPMoWT #world, +9506,Lupe know how to rap about social issues and not be boring. American Terrorist All Black Everything and Little weapon are good examples, +9512,POLLS: SHOULD THE UNBORN BABIES OF BOKO HARAM TERRORIST BE KILLED IN THEIR MOTHERS WOMB OR SPARED? http://t.co/3skw0DqZ0r, +9514,RT AbbsWinston: #Zionist #Terrorist kidnapped 15 #Palestinians in overnight terror on Palestinian Villages ___ http://t.co/J5mKcbKcov, +9515,Reading rec today: Faisal Devji's Terrorist in Search of Humanity on extremists using idea ever since the bomb of humanity as single entity, +9516,@VICE no sympathyONLF is a terrorist group affiliated with AL shabab This is war! Go report on war crimes committed by Western powers!, +9517,"RT AbbsWinston: #Zionist #Terrorist Demolish Tire Repair Shop Structure in #Bethlehem +http://t.co/ph2xLI8nVe http://t.co/22fuxHn7El", +9518,Tltltltltlttlt @P_Grolsh: Ladies are doing gymnastics in the shower!!! Opening legs wider wololo #MetrofmTalk', +9521,To whom we shld ask tht from where this bldy PAK terrorist has entered in our country...?, +9523,#Article 370 hs to.remove by this govt.otherwise our son who is on Border 'll b slain by Muslim terrorist.Bstrd Media takig SculrismFinis'em, +9527,something is seriously wrong.suddenly deres one attack after anoder.Did a large contingency of terrorist... http://t.co/MVhjSelV5z, +9528,Stop using the money of tax layers of the country to feed bloody terrorist. Shot them and give a strong msg as other country does..., +9530,I always wonder why and how media is capable to meet or get details of terrorist? Cops dont even get to know them primarily? Strange, +9532,#Congi & Indian News Presstitutes must be sympathising on the Terrorist caught alive in Jammu.If they can do for Yakub Sonia must be feeling, +9534,@RegaJha @sethiankit True also a racial attack in US perturbs buzzfeed India more than 3 back to back terrorist attacks on India.., +9535,Okay #MetroFmTalk, +9536,[August 06 2015 at 08:02PM] Three Israeli soldiers wounded in West Bank terrorist attack via http://t.co/6nDHERpBoT, +9543,@szuniverse No threat of that sir... For some reason my Internet connection has slowed way down..., +9544,White Space Devices Pose Threat to Medical Devices Warns Lawmakers http://t.co/kZ240R8qp9, +9545,A reminder: White Americans are the biggest terror threat in the United States according to new study. http://t.co/BqzhPjOTF5, +9546,Angry Orchard Hard Cider Recalled Due To Threat of Exploding Bottles http://t.co/OXgefsoF2B, +9549,Most Popular Content Top story: Barrel Bombs Not ISIS Are the Greatest Threat___ http://t.co/c8YYUWmefG see more http://t.co/BsMz4qlLBN, +9558,The most prescient part of Tropic Thunder was that fat-suit fart comedies would use the 2.40:1 aspect ratio., +9560,"Yr voice ws the soundtrack of my summer.Do u know u're unlike any other.U'll always be my thunder! ? +@DarshanRavalDZ http://t.co/XVRbKvannu", +9561,this thunder though......., +9570,was that thunder?, +9573,The goddess is said to be married to either Perk?nas (thunder god) or Praam__ius (manifestation of chief heavenly god Dievas)., +9578,I don't wake up until another 2 hours && I get interrupted . ????? I hate thunder .. ??, +9584,#FatLoss #Diet How Can You Find The Best Ways To Reduce Weight? http://t.co/czcC7NIEoX #Thunder #Health, +9587,Tuesday Bolts ___ 8.4.15 http://t.co/mkMSV54eV2 #Thunder #NBA, +9588,I need a thunder buddy. ????, +9599,Raise your words not your voice. It is rain that grows flowers not thunder., +9601,it doesn't really get much better then summer thunder storms wrapped in a blanket. #icouldsitinthismomentforever, +9604,Idk if ithats thunder or people rolling in their trashcans ??????, +9614,[HIGH PRIORITY] SEVERE THUNDERSTORM WATCH ENDED Issued for Red Deer [Updated: Aug 05th 20:29 MDT] http://t.co/ZrIyNa6f6m, +9615,Severe Thunderstorm Warning until 10:30 PM local for Oklahoma County in OK. 60 Mph Wind Gusts And Half Dollar Size Hail. #okwx #myswc, +9621,New event. Severe Thunderstorm Warning from 8/5/2015 9:50 PM to 10:30 PM CDT for Oklahoma County. Mo, +9622,Stood there for 20 mics tryin to get a #lightning pic before settling for a vid. #okc #weather #summer #thunderstorm http://t.co/k2GJBOk6ft, +9623,GSP issues STRONG THUNDERSTORM WILL IMPACT PORTIONS OF NORTHEASTERN PICKENS COUNTY UNTIL 1115 PM EDT for Greater Pickens Pickens Mountain___, +9624,A Dog Was Abandoned In A Thunderstorm. But Then A Neighbor Steps Up And Does THIS http://t.co/iR3OXEH7id, +9626,Severe Thunderstorm Warning for Oklahoma County in OK until 10:30pm. http://t.co/Iyq8gf437p, +9628,RT @LivingSafely: #NWS issues Severe #Thunderstorm Warnings for parts of #AR #NC #OK. Expect more trauma cases: http://t.co/FWqfCKNCQW, +9630,Just woke up to the loudest thunderstorm I've ever heard, +9631,RT @LivingSafely: NWS posts Severe #Thunderstorm Warnings for parts of #AR #NC #OK. Seek strong shelter if at risk: http://t.co/kImr0l24Fb, +9633,GSP issues STRONG THUNDERSTORM WILL IMPACT PORTIONS OF NORTHEASTERN PICKENS COUNTY UNTIL 1115 PM EDT for Greater Pickens Pickens Mountain___, +9635,OUN issues Severe Thunderstorm Warning for Oklahoma [OK] till 10:30 PM CDT http://t.co/do6H2pVuB4, +9637,Special Weather Statement issued August 05 at 10:40PM EDT by NWS: ...STRONG THUNDERSTORM WILL IMPACT PORTIONS ... http://t.co/ulqeR1yTpX, +9641,Loud ass thunderstorm for no reason !??, +9647,RT @LivingSafely: NWS posts Severe #Thunderstorm Warnings for parts of #AR #NC #OK. Seek strong shelter if at risk: http://t.co/HE4Qh2I5wF, +9648,Severe Thunderstorm Warnings have been cancelled in central Oklahoma. Still expect 50 mph winds penny sized hail, +9649,Pretty bad ass thunderstorm, +9658,I asked my dad 'what if a tornado comes ?'and he said 'well you better stick your head between your legs and kiss your butt goodbye'??, +9662,Storm damage maybe a tornado near Chadbourn. See http://t.co/gAs4sS3dKS http://t.co/zlhVYArtCU, +9668,Going to attempt the California fair again tomorrow hopefully we don't die in a tornado ??, +9671,#tornado #singapore Mac and #cheese #around the world - mac cheese cookbook: http://t.co/rgAm3eQOwn Mac and Ch http://t.co/X6ZJpzB8UP, +9676,http://t.co/Dg0EcBYKJL Tornado warnings end after thunderstorms hammer Calgary region for 2nd night #HeadlinesApp, +9677,(AL) Severe Thunderstorm Warning issued August 05 at 8:44PM CDT until August 05 at 9:15PM CDT by NWS http://t.co/bviYy8BMsb #alwx, +9679,"Saam's character rocks! @OliLafont A must read for #Fantasy #mythology lover +http://t.co/izgHpYQko4 http://t.co/XAibgO445o #TornadoGiveaway", +9682,Tornado warnings issued for southern Alberta http://t.co/zqmzAi8yIz, +9685,@MarquisDeSpade You can tell yer sins to me. I won't judge you. Take your clothes off and we can sing a hymn. Let's do Abba .@TORNADO_CHICK, +9689,@sarahnandola BTW we all live in libertyville now. Bc of the tornado, +9692,A tornado flew around my room before you came..., +9693,@dead_last I'm going to live out of my car for a year probably 2016 life goals :), +9694,#TornadoGiveaway å_ #thebookclub å_ Join in!! https://t.co/8qucf6sgOc https://t.co/5U1t8hc4uf, +9696,Loving you was like throwing a lasso around a tornado I tried to hold on to.??, +9700,Blonde beauty Amanda shows you her big natural boobs in her first set http://t.co/qew4c5M1xd View and download video, +9711,Rly tragedy in MP: Some live to recount horror: ___When I saw coaches of my train plunging into water I called ... http://t.co/KiXRcBNjLs, +9713,@enjoltair that would be both a tragedy but also extremely realistic and p cool yeah sure, +9720,"@TEDMED +I remember my friend Jeff Weisberg saying ___If I can only gat back from this ER shift w/out a kiddie tragedy'", +9721,'1 Death is a tragedy; a million Deaths is a statistic.' Stalin August 6 1945 U.S. Atomic Bomb Dropped On... http://t.co/kaMPx4eOS9, +9725,They can only ban SAFE abortions- back alley tragedy will come back. USA is NOT a theocracy!!!!! http://t.co/qIkS2FUTb1, +9726,@MindfulYoga responds to the tragedy of #CecilTheLion http://t.co/5tPZ7loLlX #yoga #compassion @Hugger_Mugger http://t.co/MhIF6mPNTs, +9727,@MeekMill Have your beef Idgaf but you went too far when you used a tragedy where PEOPLE DIED to insult Drake. You owe T.O an apology., +9735,If our love is tragedy why are you my remedyyyy, +9740,jiwa mines Rly tragedy in MP: Some live to recount horror: ___When I saw coaches of my train plunging into water... http://t.co/LOEPKBS38J, +9742,Rly tragedy in MP: Some live to recount horror: ___When I saw coaches of my train plunging into water I called ... http://t.co/BdlqasD0wB, +9743,These women all bonded through tragedy. All of em were broken in some way, +9747,Rly tragedy in MP: Some live to recount horror: ___When I saw coaches of my train plunging into water I called ... http://t.co/cSPAmWTo6T, +9748,Rly tragedy in MP: Some live to recount horror: ___When I saw coaches of my train plunging into water I c... http://t.co/AWWg8kNMZg Over., +9751,Rly tragedy in MP: Some live to recount horror: ___When I saw coaches of my train plunging into water I called ... http://t.co/HR4GNyGSiC, +9758,@LauraE303B @SheilaGunnReid A war we'll never win? There r 2500 Yezidi women trapped as slaves.We know where they are. Doing nothing is evil, +9767,Think of it as a reality TV show I tell myself to quell the panic rising in my throat. Introvert: Trapped in Tokyo. The 150-square-foot, +9768,Been signed off work for a week as i have a trapped nerve and can not move.! Life is a struggle right now.! ????, +9773,"Confused Westie Believes Dogs Are Trapped In The Computer +http://t.co/aWZoNRYc9y http://t.co/HwJ7GwAVvB", +9777,(#TeambrianMundial) Hollywood Movie About Trapped Miners Released in Chile: 'The 33' Holly... http://t.co/Ama6AVULnA (#TeambrianMundial), +9781,No one on the south shore knows how to make good coffee and I'm trapped in a living hell, +9784,"Then @slrqxcrls on this trapped out evil monster! + +https://t.co/n3I79QLBsS", +9786,Eric is 18 trapped in a 48 year olds buddy, +9789,Hollywood Movie About Trapped Miners Released in Chile: 'The 33' Hollywood movie about trapped min... http://t.co/BinNJwobtU #topstories, +9792,took her to the trap house now lil mama trapped out ??, +9793,If I ever saw a dog trapped in a hot car you better believe I'm busting your fucking windows open and taking your dog, +9798,Hollywood movie about trapped miners released in Chile: SANTIAGO Chile (AP) ___ The Hollywood movie on the 33 m... http://t.co/RAzFkXbhMN, +9799,Billionaire Mottas Try Getting Trapped Money Out of Venezuela - Bloomberg http://t.co/5JqMWaf6Vp, +9802,@Pandamoanimum Duly noted.The involuntary wince I have when people send me words like 'holibobs' must make it look like I have trapped wind., +9803,1961 LES WOZNI In Hospital After He Broke His Leg Trapped in CAVE IN Press Photo http://t.co/7Rr4lo1lfW http://t.co/IOLOFaphsv, +9804,It's a trap not a fucking game. U ain't trapping if u ain't trapped, +9805,@JetsetExtra A2 - meeting new interesting people learning about new places and the freedom to not be trapped by circumstances #JSETT, +9807,Ghost is literally surrounded by people who are trapped mentally. Small minded individuals smh., +9809,Students suffering from trauma ___ like divorce violence & poverty ___ often act out. Schools are trying a new approach. http://t.co/mDaCuJ9W1q, +9811,"RT MME_AUSTIN: Why #Marijuana Is Critical For Research in Treating #PTSD + +http://t.co/T6fuAhFp7p +#hempoil #cann___ http://t.co/RhE7dXM7Ey", +9814,sickmund: RT NCJFCJ: #NCJFCJ President Judge Byrne opens the first Nat'l Summit on #Trauma & the Courts in #Clevel___ http://t.co/1rzxqpUGFG, +9817,What I Learned About Fat Dating Trauma Through Piggy and Kermit https://t.co/FJwQdThAIh, +9821,Esteemed journalist recalls tragic effects of unaddressed #childhood #trauma. #counciling #therapy #newauthors #HIV http://t.co/GXq1AuhzCy, +9824,It's shocking how your heart repairs itself from such terrible trauma, +9825,Effective immediately Maimonides Medical Center is no longer an Adult or Pediatric Trauma Center. http://t.co/rTs30ZuVhT, +9827,@MountSinaiNYC @Beingtweets @Susancares2x #onbeingwithKristaTippett This is important for women living w/HIV the science of trauma, +9839,Hiroshima: They told me to paint my story: Eighty-nine year old man recalls the terror and the trauma when the... http://t.co/VhMbN4bn0b, +9840,Idk how I'm supposed to have my daughter in gymnastics and cheer at the same time? She refuses to make a choice. Ugh DIVA ??, +9843,@AngstAttack Dating a woman does NOT make you a Lesbian. However Lesbians are in fact caused by experiencing trauma involving a man., +9844,"Why #Marijuana Is Critical For Research in Treating #PTSD + +http://t.co/5PRq6FvE5Z +#hempoil #cannabis #marijuananews http://t.co/HcvbwtKlMT", +9845,Your brain is particularly vulnerable to trauma at two distinct ages http://t.co/Wvq0Rf6UAm, +9847,Your brain is particularly vulnerable to trauma at two distinct ages http://t.co/RAv8iMVvZB via @qz, +9848,Gutted. 6 weeks to go. Suspected fracture to my left ankle. Trauma clinic to see Specialist tomorrow #gnr @sageuk http://t.co/wa3sIUtoI0, +9851,Your brain is particularly vulnerable to trauma at two distinct ages http://t.co/PTLgpqQeuQ, +9852,North American Rescue IFAK Refill Trauma Kit with NEW Red Tip Tourniquet IPOK + http://t.co/wrxwCA35EZ http://t.co/Ve0vXVJOkz, +9855,PTSD research aims to answer a question left to future researchers in 1922 by a post WW1 inquiry http://t.co/gRDebWx70f, +9856,RT @judithabarrow author CHANGING PATTERNS #amazon http://t.co/0oCrOtMkHM SILENT TRAUMA http://t.co/2ZGrVnMDW9 #amazon, +9863,I've spent the day traumatised about the fact that there is loads of good music out there and I'm probably not going to hear it all, +9867,Can you imagine how traumatised Makoto would be if he could see himself in the dub (aka Jersey Shore AU) rn? Well done America, +9871,@TremendousTroye I'm so traumatised, +9876,"@PeterDutton_MP +YOU DIDN'T 'STARE' DOWN 'PPL SMUGGLERS' +YOU PAID THEM TO GO AWAY! +2 men died many traumatised children abused. #auspol", +9879,Watching Casino Royale. She's clearly traumatised! Sucking her fingers work! Right?! #JamesBond, +9882,@michellemccann we had that last night - lightning for 2 hours! Poor Dawg traumatised. Ended up huddling in basement. @David_Williamso, +9891,@Tanishaa_ I have never watched it until now.. actually traumatised! ??, +9892,Why is the story about Kids Company about POLITICIANS over a measly å£3million? Why is it not about THE KIDS??? Our traumatised children?, +9893,@lisajdavidson1 @Nataliealana87 @WestEndShiv I'm already traumatised by my first full day at the Fringe, +9895,Still traumatised at the sight of an old woman throwing her shit out the window ?????, +9899,for an extra month I was literally traumatised that couldn't been me. But I do a recap in a pool every now & again for safety purposes., +9900,sunakawa is gonna be like traumatised forever i think lol, +9902,@lovestheworld55 @SemiMooch so that's why you're now willing to believe nonsense - you were traumatised? Sorry to hear but bible still fake, +9903,@DanDoherty_ @TomCenci @HRWright too traumatised to ask why....., +9904,@ibAUTUMN I'm that traumatised that I can't even spell properly! Excuse the typos!, +9908,@TitusOReily He's not responsible for the trouble urine!, +9913,Pissed something off in the wood pile. Pulling pieces out and I get a small hiss and a series of low clicks for my trouble., +9914,RT MMDA: ADVISORY: Stalled Bus at EDSA Service Road Cubao SB due to mechanical trouble as of 7:53 AM. 1 lane occupied. MMDA T/C on site. T___, +9917,a room full of adults going ___boonew boonew___ along with the ___make it double___ part of Pokemon PL__ªs shitty midi of Prepare For Trouble, +9922,RT ADVISORY: Stalled Bus at EDSA Service Road Cubao SB due to mechanical trouble as of 7:53 AM. 1 lane occupied.___ https://t.co/HRNZKU66mm, +9924,since it is a righteous thing with God to repay with tribulation those who trouble you https://t.co/YLTKaWHf9o, +9926,Must black folks don't wanna stir up trouble .. Which in turn causes them to be silent on the injustice in the wrld. https://t.co/a5bKmsasjE, +9927,If all men sauntered like Fred I think I'd be in trouble. #ShallWeDance #SummerUnderTheStars #TCMParty, +9928,"Photo: 22 year old Yahoo Boy in Trouble for Attempting to Bribe EFCC Operatives +see... http://t.co/ObcHpSMHDO", +9929,Sitting in my room the whole time in vegas to stay out of trouble, +9936,Mouna gonna get me in trouble, +9939,@ModCloth having some trouble with your site- keep getting this every time I log in? http://t.co/dnKkYiFsvT, +9947,#BestSeller The trouble in one of Buffett's favorite sectors http://t.co/4wa9cMs3Cz, +9951,@corcoran30 @kmurphalurph when these two are in the building laughs and trouble haha.. Great___ https://t.co/ds7ms8g1yR, +9954,Click Share in minute info-technews The Trouble With Reddit__ªs Content Policy Update | Re/code ___ http://t.co/wS0xohNb7v, +9956,I had trouble breathing while listening to kian singing omg, +9957,Today we played mini golf in the rain I was called pretty by a creepy old guy and pretended we didn't speak English to get out of trouble, +9959,@AshNiggas tink ..., +9962,I liked a @YouTube video from @team0epiphany http://t.co/kui6Ig2EpZ GTA 5 Mods - NEW TSUNAMI MOD GAMEPLAY! HUGE WAVES SHARK HUNTING, +9964,omfg Johnny tsunami commercial, +9966,I just wanna food but of course I had to get caught in this fucking tsunami. ????, +9968,TheDIYHacks: RT Illusionimagess: A tsunami cloud. http://t.co/H6t4x1EVTx, +9969,@TerriF77 GodsLove & #thankU beloved sister Terri for RT of NEW VIDEO http://t.co/cybKsXHF7d The Coming Apocalyptic US Earthquake & Tsunami, +9970,@DArchambau THX for your great encouragement and for RT of NEW VIDEO http://t.co/cybKsXHF7d The Coming Apocalyptic US Earthquake & Tsunami, +9975,When tsunami says your order will take 40 minutes and you placed your order when you were on your way ????, +9977,She kept it wet like tsunami she think she saw the titanic I get it hot like wasabi, +9981,Oh it__ªs a soccer ball? I thought it was the head of a melting snowman., +9993,I liked a @YouTube video http://t.co/18aAsFMVbl Call Of Duty Ghosts Walktrough Part-8 Tsunami!, +9996,I think Tsunami has hit the wall. Dang... #STLCards, +9997,@TA7LPlays @Twitch_Bang @Maliceqt @Zelse007 @RhyareeGaming @dougiedabandit sorry what you say theres a tweet tsunami happening, +9999,This high-tech sphere can be the key to saving lives during a tsunami. What would you pack in it? http://t.co/KwlpFqRBVp, +10002,Drunk #BBMeg! ??????, +10007,Help support the victims of the Japanese Earthquake and Pacific Tsunami http://t.co/O5GbPBQH http://t.co/MN5wnxf0 #hope4japan #pray4japan, +10015,"'How many sons would a parson parse if a parson could parse sons?' + +-popular tongue twister among religious woodchucks", +10019,Twister hits 4 villages in Quezon province - http://t.co/ehJv0gvo47 http://t.co/pzJcRNrBSr #Quezon #news, +10022,@lizjillies it's such a tongue twister, +10024,Seriously though..... Otto Reggie Twister and Sam was living that good life on Rocket Power., +10026,Playing twister with a broken arm wasn't the smartest decision I've made recentlu, +10033,when you get stuck on the wicked twister and they have to call maintenance to get you out.. http://t.co/7iFPMFssMU, +10043,@kady I would pay to watch the leaders play Twister., +10046,@bamagaga Best part of today? Your song I'M TO BLAME by @KIPMOOREMUSIC is about to play on 101.9 The Twister! http://t.co/sKV3O4Teni, +10051,Work-related Dick Twister That Happened Today http://t.co/oJ5cUBshGz, +10053,The shortest and hardest tongue twister is: 'Toy Boat' (Try saying it five times fast)., +10062,Struggling to Return to Life: Makurazaki Typhoon on September 17 1945 killed thousands. http://t.co/fmp0JsH4wX http://t.co/X3TotYZAZe, +10063,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians a... http://t.co/Hni4v2ZUk3, +10065,#Map: Typhoon Soudelor's predicted path as it approaches Taiwan; expected to make landfall over southern China by ___ http://t.co/PgD7UXtVGg, +10068,Obama signs disaster declaration for Northern Marians after typhoon hits US territory http://t.co/0IPaVKJgDC, +10078,Typhoon Soudelor heads for Taiwan http://t.co/Pg0BhL6FQC #wtf #news #tv #fun #funny, +10086,UK WEATHER ALERT: Remnants of Asian super typhoon Soudelor to cause VIOLENT ... - http://t.co/JXOcfzKyw7 http://t.co/sZx1v2mk8K, +10094,#Trending #Viral Spectacular satellite view of Typhoon Soudelor @rumblevideo http://t.co/VkF0gkZ9N3, +10095,"#WorldNews #World + Saipan Has No Water Electricity in Typhoon Aftermath - Voice of America - World - Google News.. http://t.co/5sUdXgNdA3", +10096,Striking views of Super Typhoon Soudelor as it tracks toward Taiwan China http://t.co/hsjp6Hoffe, +10097,Saipan has no water electricity in aftermath of typhoon http://t.co/UBAk5uOI48, +10103,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians a... http://t.co/a1MoeJxqyA, +10104,#abc Obama Declares Disaster for Typhoon-Devastated Saipan http://t.co/ueHFZTeYHu #international #news, +10108,@SushmaSwaraj Am sure background check of the child's details are done before the kid gets into another upheaval., +10122,Where bottle cherry timbering facilities upheaval entryway?: fJiZzTxK, +10123,@ArvindKejriwal Bharat is showing silent political upheaval and change to fulfill the wishes of the people. Retired DGM NABARD, +10128,MGS4 came out during a time of great upheaval in my life and it__ªs very odd that MGSV is coming and it__ªs also when things are getting iffy, +10131,Bat four reasons en route to upheaval versus internet buzz value: pVDRDLJvc, +10133,? http://t.co/5J8vHUXzrp #Ancient Mayan Tablet with Hieroglyphics Honors Lowly King #livescience #future ___ http://t.co/n3aQXoMubu, +10134,Permutable site conspiracy up-to-the-minute upheaval: QWkD, +10135,For THIS FAN this was 1 upheaval too many! Done with this garbage of a show! @ABCNetwork @shondarhimes @EllenPompeo https://t.co/hq5TDxKqHB, +10145,Life needs quality and a certain sense of security. Being with a person you can't trust can only cause stress and emotional upheaval., +10148,"Medieval Upheaval (Hardy Boys: The Secret Files) +Franklin W. Dixon - Aladdin. http://t.co/Yb7ijKtlnB", +10154,This week we received the sad news of our friends daughters cancer prognosis. This is the second upheaval in this... http://t.co/tpOptWMWFR, +10155,This Is How Retailers Intend to Survive Upheaval http://t.co/WOcpJi5uq1 #retail #consumers http://t.co/JelXFtOpu5, +10158,@RoxYunkFalls August 4 1915 http://t.co/jjNS0YjaLA Violent storm strikes city Manayunk flooding and damage. Sports & War news, +10160,Dramatic Video Shows Plane Landing During Violent Storm http://t.co/YX8OdrrINr, +10161,@hippieXox If the word 'violent' comes to mind it's food poisoning. Nothing you can do but weather the storm about 24 hours. :(, +10162,And Kolkata is struck by a Cyclonic Storm. Sumthng big is gonna happen 2day evng. Heavy rains nd a violent storm approachng. God help us., +10165,More Severe Weather Heads Toward Chicago Area: A brief violent storm swept through the Chicago area Sunday af... http://t.co/JWSLLsuKjg, +10166,Terrifying POV footage captures violent landing from inside a passenger jet during a storm in Amsterdam http://t.co/TyAOYEpelz, +10167,[WP] You work on the top floor of a hotel when suddendly a violent storm comes the news proclaims your city in a ___ http://t.co/S6Rc7dbAL1, +10170,RT @calgarysun: Sun photographer Stuart Dryden spotted this vortex spun off a violent storm in Sylvan Lake. #abstorm http://t.co/w0BZ0JNGvB, +10178,Holy crap! BRAVO Sir! Amazing! Dramatic Video Shows Plane Landing During Violent Storm http://t.co/xB0bw8h8Ur, +10180,"Violent Forces Radio: Now Playing Torture - Storm Alert +TuneIn Player @ http://t.co/XsSgEdSbH4", +10181,Circus tent collapses in violent storm killing 2 kids in New Hampshire http://t.co/j0saXEKBTa, +10185,Storm is here! Violent winds and pounding rains in Evergreen. #yyc, +10188,Stay inside for the next little while kids. We're having a bit of a violent storm right now. -kjc, +10190,Watch: Violent Storm Causes Deadly Accident at New Hampshire Circus http://t.co/jpPiR3ZzKA #GMA http://t.co/nV5GCDpIBA, +10196,"Violent Forces Radio: Now Playing Axegressor - Psalm Before the Storm +TuneIn Player @ http://t.co/XsSgEdSbH4", +10200,#calgaryweather It would be nice if they would fix radar before another violent storm Uninformed citizens dangerous #YYC #yycweather, +10205,2012#Shell's 250-foot-tall drilling rig broke loose from a towline&drifted out of control during violent winter storm #incompetent #shellno, +10211,http://t.co/16EClWrW84 Asics GT-II Super Red 2.0 11 Ronnie Fieg Kith Red White 3M x gel grey volcano 2, +10216,Force Factor VOLCANO 120 Capsule Brand New - Full read by eBay http://t.co/HewutDLDqh http://t.co/8etXXzBdua, +10219,The sunset looked like an erupting volcano .... My initial thought was the Pixar short Lava http://t.co/g4sChqFEsT, +10221,@CatholicMomVA Or take away his ring & throw it in a volcano..., +10225,The highlight of my day has been playing with my raw volcano humidifier. #sicklife http://t.co/teDRu4hg3i, +10227,@Jesssssssee @eevans17 The Illuminati: The Yellowstone Super Volcano Conspiracy http://t.co/wymv9cYxzK, +10228,idgaf fly into a volcano, +10230,?? @ KETEP VOLCANO CENTRE ketep PASS https://t.co/IdBnKxu80L, +10232,Learning from the Legacy of a Catastrophic Eruption - The New Yorker http://t.co/y8YqPBE4t9, +10233,#USGS M 1.9 - 5km S of Volcano Hawaii: Time2015-08-06 01:04:01 UTC2015-08-05 15:04:01 -10:00 at epicenter... http://t.co/meVvkaXcdE #SM, +10235,"CLUB GOING UP MIXTAPE (RNB EDITION) 2015 ?? BY DJ VOLCANO X STARZ TEAM + +DOWNLOAD: https://t.co/DpBFuU2GD6", +10238,#Sismo M 1.9 - 5km S of Volcano Hawaii: Time2015-08-06 01:04:01 UTC2015-08-05 15:04:01 -10:00 at epicente... http://t.co/K28mgWTZcV #CS, +10240,Quake: M 1.9 - 5km S of Volcano Hawaii, +10242,what's the hottest thing you've ever seen..fire?..the sun?..a volcano? or this? http://t.co/2GLdsHRaiI, +10248,The Architect Behind Kanye West's Volcano - New York Times http://t.co/3QJUExX0Y0, +10254,Just watched Punisher: War Zone. IDK why everyone hates it. It is basically the best possible movie you could get out of Punisher, +10257,My new sounds: War Zone https://t.co/hNXRfqRk3P on #SoundCloud, +10261,Sammy is here in this war zone. Jamal spoke to me on the phone. Now my wife is next to speak to me ...what else can> http://t.co/2CppfprxoG, +10262,@thelovatoagent omg i feel like i am in a war zone, +10266,I think it's time for a cup of tea and a scone before I tackle the war zone that is is my bedroom. I wonder how many condom packets Ill find, +10267,Saipan looks like a war zone though, +10271,T Shirts $10 male or female get wit me. 2 days until its game changin time. War Zone single will be___ https://t.co/Z0poYR096J, +10277,Um OK '@dandre_5: Sundays during football seasonfrom about 9 am - 11 pm women shouldn't even log onshit be a complete war zone', +10279,Just trying to find my my peace in a Zone of war, +10281,Twitter is going to be a war zone today https://t.co/1QHNqCZvod, +10282,The Resistance wages guerrilla warfare against KPA forces in the Red Zone the war-torn outskirts of... http://t.co/WFxAwQvvjr, +10285,When your friend getting whooped and you running out of weapon options ?????? #TeamDuval #T... (Vine by TerrellT) https://t.co/Sf9Ufv7w6C, +10292,3 Stars in 212 seconds! Level 43 Surgeon. Weapon of choice: Neutron Rifle. #callofmini #doubleshot, +10298,Yo that guy changed his weapon mid battle how did he do that?, +10300,fun fact: every rapid-fire weapon in the game is a hard counter to the slosher and the splattling, +10309,Slosher is a fucboi weapon, +10311,"No more war.People need no any weapon. +We need the world of peace. https://t.co/DhRvYtjlFL", +10312,#AskConnor there's a zombie apocalypse. the item to your right is your weapon. you're either screwed or you're gonna live., +10319,"True strength is forgiveness. Love the most powerful weapon. .@vickysuewrites' Broken Circle' + +#giveaway #boyxboy http://t.co/Zgc3EsLNPS", +10324,On 70th anniversary of #Hiroshima bomb it is important to learn from history and ban this weapon., +10326,@KnaveTBE lol. Not the class but the class of weapon, +10328,"@sindy642498 +I'm really glad to hear that___;; +Nowlike usthe world is truely connected. +I want truely peaceful world without any weapon!", +10343,@JPens4Real21 Geno has weapons now I'll give him the benefit of the doubt this year. I'm just not a huge believer. Still not mature enough., +10346,@DANNYonPC @YouTube Its engineer only again so 2 1000+ rpm weapons = no thanks, +10347,Helpful Tips For Nuking An Asteroid http://t.co/GwRsga9uJV, +10353,I'm all in favor of keeping dangerous weapons out of the hands of fools. Let's start with typewriters., +10354,Security experts report that the only way to keep Iran 'honest' and stop her march to nuclear weapons is by... http://t.co/DGnbUsfXkw, +10356,"Aug 3 1915___KILL 10000 WITH ROCKS.; Italians Make Good Use of Nature's Weapons -- Excel with Bayonet. +http://t.co/ck1mMvHU0T", +10357,@deray @marstu67 Cop Benefits:Great Salary Free Car Free Rent Free Food.Free Weapons License to Kill & Rape with Impunity., +10362,"??Water fight?? +Penn park 6pm + BYOW +(Bring Your Own Weapons)", +10363,Abe proposed a new plan for abolishing nuclear weapons. Yesterday his defense minister made clear nukes are fine. http://t.co/A6IR8rKuwS, +10370,NAVY SAYS LT CMDR WHO FIRED SIDEARM IN DEFENSE DURING NOSC CHATTANOOGA TERRORIST ATTACK WILL NOT FACE CHARGES... http://t.co/vCcDKC4qt2, +10373,Any attempt to sell weapons to #Nigeria is an attempt to upgrad #Bokoharm army #USA must know this now @UN, +10386,Whirlwind full of bombarded kunais., +10388,@BarstoolBigCat what a whirlwind., +10389,I've officially been a St. Louisan for one full year. What a crazy awesome overwhelming whirlwind year it has been. Here's to another!, +10391,Missing Malaysian airplane: Debris was planted ___ Families News - WhirlWind News http://t.co/bLd0nHKzbh, +10397,Warcraft has always been this weird referential whirlwind to itself and other things, +10400,@Chelsea_Rosette ah bless ya. Not too bad. My life's been a bit of a whirlwind the last two weeks! Got a cold too that won't shift ?? #sexy, +10403,Join me on FB for #Sneak #Peek Friday http://t.co/3DK5zyYQ1F for my upcoming #novel To Reap a Whirlwind #fiction #historical #wip #amwriting, +10404,WAS Times: PawSox owners public return from whirlwind trip to Durham http://t.co/07nKMO7VaS, +10408,I liked a @YouTube video http://t.co/a5YTAw9Vih S.O.S. Rona Guide - The Red Whirlwind, +10417,"[VID]150806 Luhan From Luhan Studio Channel on YOUKU Update + +http://t.co/W7yeZkQlCJ http://t.co/C0QWZ0IshR", +10425,{{ whirlwind romance dress | in-store & online }} #freshoutofthebox #getitbeforeitsgone #under50 #newnewnew... http://t.co/8BUTsC4ZyN, +10435,#HarperANetflixShow Blazing Saddles (a show about wild fires in Alberta), +10437,?? #BREAKING: massive wild- and field fires reported throughout Germany! The most fires are in eastern #German http://t.co/4zSIy7EO2O, +10438,We all complain about PA but atleast we don't have those wild fires taking our homes like north Cali or the floods in Florida right now, +10440,my wife has opted to take holidays & historically since we met @lanahillman gets special treatment her bonfires can't start wild fires-RAIN!, +10442,Like blazing wild fires singing Your name!, +10447,NetNewsLedger Wild Fire Update ___ August 4 2015: THUNDER BAY ___ WEATHER ___ Thereå_were no new fires confirmed by t... http://t.co/JflxgEmBdA, +10452,@myralynn24 Becuase it can cause property damage and cause wild fires, +10453,Hey @Macys ! My moms house burned down in the CA wild fires & I bought her an e gift card to buy her CLOTHES (cont) http://t.co/sWdjnypCXK, +10454,California wild fires...clearly caused by global climate change and excess CO2!!!?, +10459,California wild fires blow my mind every time, +10460,@MrRat395 Are those wild fires getting anywhere near you?, +10463,Stampedes and fires brought him home; can her love make him stay? TAME A WILD HEART #audiobook http://t.co/2dPKqAFtbr #BYNR @BeingAuthor, +10465,Go home California you're drunk. Natural selection has taken affect. Debt extreme wild fires water supply problems. You fail as a state., +10469,@CBS evening news covers tent collapse wild fires NO mention of worst Planned Parenthood video to date #Narrative, +10474,WILD FIRES! http://t.co/EgrMdkXpOi, +10475,Wild Fires In California Continue To Burn - http://t.co/nKcmib9WL0 http://t.co/CpMGBWVDXM, +10476,one of them wild fires in cali or something or in the mid west or where ever they show up..shit dont pop up around me, +10477,@JacobHoggard @therealmattleaf it's so sad out there in BC all the wild fires. Hope u are safe., +10480,when there's 35 wild fires in the US and 21 of them are in California....AND we're in a drought.....this state is so sad, +10491,11:57am Wildfire by The Mynabirds from Lovers Know, +10493,Finger Rock wildfire grows to 500 acres #Tucson http://t.co/zSXGEI1Nxf, +10494,#chattanooga The Latest: California governor headed to site of wildfire http://t.co/D6E7IICW5W, +10495,About to get it with the babes of #wildfire and @taviiikenney of course!!, +10497,Here's a quick timelapse I made of the Finger Rock Fire last night from about 9PM - 1AM. Check it out! #fingerrockfire #wildfire #catalinas, +10501,#WAwildfire in #chelan? Wolverine fire results in evacuation. http://t.co/yePlnZPoWu, +10504,can't eat a wildfire chicken salad without thinking of / missing @Alynacarol ??, +10507,Cherokee Road and Road 22 (Three Sisters Wildfire): There are two roads closed to the general public: ... http://t.co/WwnL8FvfMM #IDFire, +10509,As California fires rage the Forest Service sounds the alarm about sharply rising wildfire costs http://t.co/ht8FyiMJlR, +10510,Be careful out there. http://t.co/KoBavdEKWn, +10512,"Wolverine Fire Update - Thursday August 6 - 9:00 Am + +Incident: Wolverine Fire Wildfire +Released: 41 min. ago... http://t.co/8WDTTzpTXH", +10514,Cooler weather helps crews battling state wildfire.. Related Articles: http://t.co/15fHVIi9Qp, +10526,Wildfire near Columbia River town is 50 percent contained - http://t.co/yTPiPXpqr9 http://t.co/HCUQ6jpBtL, +10528,"smHRN mhtw4fnet + +Crews gaining on huge Northern California wildfire - CBS News", +10530,Rocky Fire #cali #SCFD #wildfire #LakeCounty https://t.co/3pu0000v5o, +10531,This machine really captured my attention. #helicopter #firefighting #wildfire #oregon #easternoregon___ https://t.co/eKt456jY1s, +10532,Forest Grove grass fire sparked by tractor quickly contained - KATU http://t.co/FMbcocS79u, +10538,San Patricio ___ Windstorm insurance reform key for coastal homeowners: PORTLAND ___ State Rep. J.M. Lozano spoke ... http://t.co/GQySVKrAGi, +10539,Texas Seeks Comment on Rules for Changes to Windstorm Insurer http://t.co/SDZLVBiFbN, +10542,New roof and hardy going up...Windstorm inspection tomorrow.. http://t.co/3XMLQRpNQR, +10547,Insurance News:Texas Seeks Comment on Rules for Changes to Windstorm Insurer, +10549,TWIA board approves 5 percent rate hike: The Texas Windstorm Insurance Association (TWIA) Board of Directors v... http://t.co/NvxLJDsNkX, +10556,GLORIA JONES - VIXEN / WINDSTORM (2IN1) CD 20 TRACKS MARC BOLAN http://t.co/E3cAymmIcN http://t.co/eRzcXloPDL, +10569,@t_litt44 I seen you that one day look like you went threw a windstorm ??, +10572,Great time group camping at Presqu'ile with family&friends.Lots of fun;not much sleep #PQPHistoryWeekend #windstorm http://t.co/0tYkaZyQpF, +10577,Texas Seeks Comment on Rules for Changes to Windstorm Insurer: The Texas Department of Insurance is seeking pu... http://t.co/NGF9n8Jquu, +10579,A few more photos from this mornings windstorm. We've never seen anything like it! #rhodeisland #warwick, +10586,Police Officer Wounded Suspect Dead After Exchanging Shots http://t.co/e4wvXQItV4, +10594,"The South Korean army wounded 44 persons' Takeshima's Japanese fisherman and occupies the island. +http://t.co/mJCXgKU8Yt", +10595,Gunmen kill four in El Salvador bus attack: Suspected Salvadoran gang members killed four people and wounded s... http://t.co/LZWRONbTmi, +10597,You still acting like you were the one wounded. Didn't you do the stabbing?..., +10602,#Dime_Miloko Officer Wounded Suspect Killed in Exchange of Gunfire: Richmond police officer wounded suspect killed in exchange of gunfire, +10604,Police officer wounded suspect dead after exchanging shots: A Richmond police officer was wounded and a suspe... http://t.co/FzQPiQeCHB, +10614,"Pope Francis talks about WOUNDED FAMILIES +http://t.co/m1RfBPODI9", +10617,@lordjewcup @helloimivan wounded warrior, +10618,Man seriously wounded in Auburn Gresham shooting http://t.co/qPsSbAfSuL #Chiraq, +10619,Suspected Salvadoran gang members killed four people and wounded seven more on Wednesday in an attack on a bus as it travelled down a rural___, +10624,Officer Wounded Suspect Killed in Exchange of Gunfire http://t.co/HHSjnAVHUA, +10633,'Police Officer Wounded Suspect Dead After Exchanging Shots' http://t.co/guNq7ZTUn4 #????_?????, +10634,Police Officer Wounded Suspect Dead After Exchanging Shots: Richmond police officer wounded suspect killed a... http://t.co/YIKQRqczVZ, +10635,He was crushed for our iniquities; the punishment that brought us peace was upon him & by his wounds we are healed. -Isa 53:5, +10638,@bkanioros Lizeth and I tried over and over again to fix our relationship it never worked bk you cannot heal those wounds I love you for, +10639,Love wounds, +10642,'@GoogleFacts: Wounds heal faster if you lick them. Tears also help them heal as well.' I know right even thou it seems stupid when lickin, +10643,"the wounds of honor + are self-inflicted", +10645,ME: gun shot wounds 3 4 6 7 'rapidly lethal' would have killed in 30-60 seconds or few minutes max. #kerricktrial, +10646,@Cash7Nigga whoa. Insomniac. Why don't you try to get some sleep. Sleep heals all wounds. Food too ! Night :), +10649,Do You Let Your Emotional Wounds Fester? http://t.co/yDWhMgIMOE ? Powerful story andå_word picture. via @forgivenwife, +10652,Israeli military shoot Palestinian who injured 3 in West Bank car attack http://t.co/P9DCnuLODR via @YahooNews, +10653,@WorldRunners @RunningRoom Day 3&my wounds are looking much better. Running road rash. #summer #injury #offdishduty http://t.co/aGkWKaqgOS, +10654,Time heals all wounds. And if it doesn't you name them something other than wounds and agree to let them stay. ????, +10661,2 Policemen receive bullet wounds Reports indicate 6-7 armed men attacked a police post., +10664,Palestinian rams car into Israeli soldiers wounding 3 http://t.co/n79BMXQKlg #EMM, +10668,and the salt in my wounds isnt burnin any more than it used to its not that i dont feel the pain its just im not afraid of hurting anymore, +10670,When and how does a character recover fromå_wounds? http://t.co/ohhkuHtjXm, +10674,What they (dentists) don't tell u is how much ur mouth will be in pain when the numbing wears off because of the puncture wounds in ur gums., +10683,Our wounds can so easily turn us into people we don't want to be and we hardly see it happening, +10694,i'm sitting in the parking lot waiting to go into therapy and i'm crying and an emotional wreck and i don't want my therapist to know ????????????, +10697,Last week we had a blast hosting Dinner & a Movie Night with 'Wreck-It Ralph' on @FtCarsonPAO! @carsonmwr #TBT http://t.co/tUYPxEoBQa, +10699,Now that I__ªve figured out how to get my music in my rental car I can take a night drive to nowhere tonight. Hopefully without a wreck., +10701,i'm an emotional wreck watching emmerdale fml, +10703,Interesting in watching a train wreck while taking acid kind of way https://t.co/kM08qiWW4g, +10713,@DukeSkywalker @facialabuse you should do a competetion between @xxxmrbootleg & #ClaudioMeloni (ultimate throat penetrator) to a wreck off., +10714,Man faces manslaughter charges following fatal Sunday wreck | AL.c.. Related Articles: http://t.co/5xrFhdXTvX, +10716,*is a wreck* *gives ppl lifestyle advice*, +10717,I'm an emotional wreck someone hold me until they upload the damn video #whatstheimportantvideo, +10719,I added a video to a @YouTube playlist http://t.co/wYtu7vIwsj Burnt Black - Nervous Wreck, +10725,and not a wreck of uneven layers https://t.co/Y0WE0wXQCp, +10728,Dj d wreck cut the beat, +10734,@Herologist i know right? It's a train wreck. You can't believe it's happening but you can't look away., +10738,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM, +10740,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/cUrJAT1BqI, +10742,Wreckage 'conclusively confirmed' as from MH370: Malaysia PM http://t.co/Se1QkNPltS | https://t.co/DnhRviV1dJ, +10756,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/O8sTW4YN8R, +10757,#science Now that a piece of wreckage from flight MH370 has been confirmed on R_©union Island is it possible t... http://t.co/cKQ7tOE82u, +10758,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM, +10761,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/1RzskdRrDk, +10762,TOP STORY: wreckage from #MH370 officially confirmed. @janisctv reports on the critical clue from a tiny island. http://t.co/hlWw2P5k9o, +10773,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM http://t.co/MN130C4e2D via @ndtv, +10778,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/mzB0MKuUo7, +10781,RT @australian Debris found on an Indian Ocean island confirmed to be from flight #MH370. http://t.co/gY9MrSl6x2, +10791,Cramer: Iger's 3 words that wrecked Disney's stock http://t.co/4dGpBAiVL7, +10792,Almost *wrecked* my van the other day because of this guy (yeah I brake & also care for animals ~ get used to it)! https://t.co/8ZaHavmtT6, +10796,'What manner of human being would parcel out a baby as though it were a wrecked car in a junk yard? The___' ___ dtom2 http://t.co/lj6lzo4isX, +10797,@Nathan26_RFC thought you said Saturday night there and near died lolol you'll be wrecked!!, +10801,I just wanna ease your mind and make you feel alright.????, +10804,@yakubOObs think he deactivated because his notifications are aids after tesco wrecked him lol, +10806,RT CNBC '3 words from Disney CEO Bob Iger wrecked Disney's stock says Jim Cramer: http://t.co/f0texKsqhL http://t.co/ilySLaTMgI', +10807,Smackdown tyme this should put me in a good mood again since it got wrecked smh, +10816,@thrillhho jsyk I haven't stopped thinking abt remus slumped against the bathroom door all day I was wrecked ??????????, +10820,@stighefootball Begovic has been garbage. He got wrecked by a Red Bull reserve team and everyone else this preseason, +10828,Wrecked today got my hattrick ????, +10836,#Ebola #EbolaOutbreak Ebola Virus: Birmingham Ala. Firefighters Quarantined After Possible Exposure Officials Say http://t.co/tjpYlU9fOX, +10838,Malaysian PM confirms debris is from missing flight MH370 http://t.co/pfAvW5QyqE, +10845,Officials: Alabama home quarantined over possible Ebola case - Washington Times, +10856,See the 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released: Harun __ekdar ... http://t.co/hKuT5mSdtP @MsOreo_, +10857,To conference attendees! The blue line from the airport has DERAILED - please look into taking a taxi to the hotel! See you soon!, +10858,The death toll in a #IS-suicide car bombing on a #YPG position in the Village of Rajman in the eastern province of Hasaka has risen to 9, +10861,EARTHQUAKE SAFETY LOS ANGELES ___ SAFETY FASTENERS XrWn, +10865,Storm in RI worse than last hurricane. My city&3others hardest hit. My yard looks like it was bombed. Around 20000K still without power, +10868,Green Line derailment in Chicago http://t.co/UtbXLcBIuY, +10874,MEG issues Hazardous Weather Outlook (HWO) http://t.co/3X6RBQJHn3, +10875,#CityofCalgary has activated its Municipal Emergency Plan. #yycstorm, \ No newline at end of file diff --git a/TextClassification/TextClassification/data/train.csv b/TextClassification/TextClassification/data/train.csv new file mode 100644 index 0000000000000000000000000000000000000000..18d714042df12005a4a7d66fa5f953222d47a41a --- /dev/null +++ b/TextClassification/TextClassification/data/train.csv @@ -0,0 +1,6794 @@ +id,text,target +1,Our Deeds are the Reason of this #earthquake May ALLAH Forgive us all,1 +4,Forest fire near La Ronge Sask. Canada,1 +5,All residents asked to 'shelter in place' are being notified by officers. No other evacuation or shelter in place orders are expected,1 +6,"13,000 people receive #wildfires evacuation orders in California ",1 +7,Just got sent this photo from Ruby #Alaska as smoke from #wildfires pours into a school ,1 +8,#RockyFire Update => California Hwy. 20 closed in both directions due to Lake County fire - #CAfire #wildfires,1 +10,"#flood #disaster Heavy rain causes flash flooding of streets in Manitou, Colorado Springs areas",1 +13,I'm on top of the hill and I can see a fire in the woods...,1 +14,There's an emergency evacuation happening now in the building across the street,1 +15,I'm afraid that the tornado is coming to our area...,1 +16,Three people died from the heat wave so far,1 +17,Haha South Tampa is getting flooded hah- WAIT A SECOND I LIVE IN SOUTH TAMPA WHAT AM I GONNA DO WHAT AM I GONNA DO FVCK #flooding,1 +18,#raining #flooding #Florida #TampaBay #Tampa 18 or 19 days. I've lost count ,1 +19,#Flood in Bago Myanmar #We arrived Bago,1 +20,Damage to school bus on 80 in multi car crash #BREAKING ,1 +23,What's up man?,0 +24,I love fruits,0 +25,Summer is lovely,0 +26,My car is so fast,0 +28,What a goooooooaaaaaal!!!!!!,0 +31,this is ridiculous....,0 +32,London is cool ;),0 +33,Love skiing,0 +34,What a wonderful day!,0 +36,LOOOOOOL,0 +37,No way...I can't eat that shit,0 +38,Was in NYC last week!,0 +39,Love my girlfriend,0 +40,Cooool :),0 +41,Do you like pasta?,0 +44,The end!,0 +48,@bbcmtd Wholesale Markets ablaze http://t.co/lHYXEOHY6C,1 +49,We always try to bring the heavy. #metal #RT http://t.co/YAo1e0xngw,0 +50,#AFRICANBAZE: Breaking news:Nigeria flag set ablaze in Aba. http://t.co/2nndBGwyEi,1 +52,Crying out for more! Set me ablaze,0 +53,On plus side LOOK AT THE SKY LAST NIGHT IT WAS ABLAZE http://t.co/qqsmshaJ3N,0 +54,@PhDSquares #mufc they've built so much hype around new acquisitions but I doubt they will set the EPL ablaze this season.,0 +55,INEC Office in Abia Set Ablaze - http://t.co/3ImaomknnA,1 +56,Barbados #Bridgetown JAMAICA ___ Two cars set ablaze: SANTA CRUZ ___ Head of the St Elizabeth Police Superintende... http://t.co/wDUEaj8Q4J,1 +57,Ablaze for you Lord :D,0 +59,Check these out: http://t.co/rOI2NSmEJJ http://t.co/3Tj8ZjiN21 http://t.co/YDUiXEfIpE http://t.co/LxTjc87KLS #nsfw,0 +61,"on the outside you're ablaze and alive +but you're dead inside",0 +62,Had an awesome time visiting the CFC head office the ancop site and ablaze. Thanks to Tita Vida for taking care of us ??,0 +63,SOOOO PUMPED FOR ABLAZE ???? @southridgelife,0 +64,I wanted to set Chicago ablaze with my preaching... But not my hotel! http://t.co/o9qknbfOFX,0 +65,I gained 3 followers in the last week. You? Know your stats and grow with http://t.co/TIyUliF5c6,0 +66,How the West was burned: Thousands of wildfires ablaze in California alone http://t.co/vl5TBR3wbr,1 +67,Building the perfect tracklist to life leave the streets ablaze,0 +68,Check these out: http://t.co/rOI2NSmEJJ http://t.co/3Tj8ZjiN21 http://t.co/YDUiXEfIpE http://t.co/LxTjc87KLS #nsfw,0 +71,First night with retainers in. It's quite weird. Better get used to it; I have to wear them every single night for the next year at least.,0 +73,Deputies: Man shot before Brighton home set ablaze http://t.co/gWNRhMSO8k,1 +74,"Man wife get six years jail for setting ablaze niece +http://t.co/eV1ahOUCZA",1 +76,SANTA CRUZ ___ Head of the St Elizabeth Police Superintendent Lanford Salmon has r ... - http://t.co/vplR5Hka2u http://t.co/SxHW2TNNLf,0 +77,Police: Arsonist Deliberately Set Black Church In North Carolinaå_Ablaze http://t.co/pcXarbH9An,1 +78,Noches El-Bestia '@Alexis_Sanchez: happy to see my teammates and training hard ?? goodnight gunners.?????? http://t.co/uc4j4jHvGR',0 +79,#Kurds trampling on Turkmen flag later set it ablaze while others vandalized offices of Turkmen Front in #Diyala http://t.co/4IzFdYC3cg,1 +80,TRUCK ABLAZE : R21. VOORTREKKER AVE. OUTSIDE OR TAMBO INTL. CARGO SECTION. http://t.co/8kscqKfKkF,1 +81,Set our hearts ablaze and every city was a gift And every skyline was like a kiss upon the lips @___ https://t.co/cYoMPZ1A0Z,0 +82,They sky was ablaze tonight in Los Angeles. I'm expecting IG and FB to be filled with sunset shots if I know my peeps!!,0 +83,How the West was burned: Thousands of wildfires ablaze in #California alone http://t.co/iCSjGZ9tE1 #climate #energy http://t.co/9FxmN0l0Bd,1 +85,Revel in yours wmv videos by means of mac farewell ablaze wmv en route to dvd: GtxRWm,0 +86,"Progressive greetings! + +In about a month students would have set their pens ablaze in The Torch Publications'... http://t.co/9FxPiXQuJt",0 +89,Rene Ablaze & Jacinta - Secret 2k13 (Fallen Skies Edit) - Mar 30 2013 https://t.co/7MLMsUzV1Z,0 +91,@Navista7 Steve these fires out here are something else! California is a tinderbox - and this clown was setting my 'hood ablaze @News24680,1 +92,#NowPlaying: Rene Ablaze & Ian Buff - Magnitude http://t.co/Av2JSjfFtc #EDM,0 +93,@nxwestmidlands huge fire at Wholesale markets ablaze http://t.co/rwzbFVNXER,1 +95,@ablaze what time does your talk go until? I don't know if I can make it due to work.,0 +96,'I can't have kids cuz I got in a bicycle accident & split my testicles. it's impossible for me to have kids' MICHAEL YOU ARE THE FATHER,0 +97,Accident on I-24 W #NashvilleTraffic. Traffic moving 8m slower than usual. https://t.co/0GHk693EgJ,1 +98,Accident center lane blocked in #SantaClara on US-101 NB before Great America Pkwy #BayArea #Traffic http://t.co/pmlOhZuRWR,1 +100,http://t.co/GKYe6gjTk5 Had a #personalinjury accident this summer? Read our advice & see how a #solicitor can help #OtleyHour,0 +102,#stlouis #caraccidentlawyer Speeding Among Top Causes of Teen Accidents https://t.co/k4zoMOF319 https://t.co/S2kXVM0cBA Car Accident tee___,0 +104,Reported motor vehicle accident in Curry on Herman Rd near Stephenson involving an overturned vehicle. Please use... http://t.co/YbJezKuRW1,1 +105,BigRigRadio Live Accident Awareness,1 +107,I-77 Mile Marker 31 South Mooresville Iredell Vehicle Accident Ramp Closed at 8/6 1:18 PM,1 +109,RT @SleepJunkies: Sleeping pills double your risk of a car accident http://t.co/7s9Nm1fiCT,0 +110,'By accident' they knew what was gon happen https://t.co/Ysxun5vCeh,0 +112,Traffic accident N CABRILLO HWY/MAGELLAN AV MIR (08/06/15 11:03:58),1 +113,I-77 Mile Marker 31 to 40 South Mooresville Iredell Vehicle Accident Congestion at 8/6 1:18 PM,1 +114,the pastor was not in the scene of the accident......who was the owner of the range rover ?,1 +117,"mom: 'we didn't get home as fast as we wished' +me: 'why is that?' +mom: 'there was an accident and some truck spilt mayonnaise all over ??????",0 +118,I was in a horrible car accident this past Sunday. I'm finally able to get around. Thank you GOD??,1 +119,Can wait to see how pissed Donnie is when I tell him I was in ANOTHER accident??,0 +120,#TruckCrash Overturns On #FortWorth Interstate http://t.co/Rs22LJ4qFp Click here if you've been in a crash>http://t.co/Ld0unIYw4k,1 +121,Accident in #Ashville on US 23 SB before SR 752 #traffic http://t.co/hylMo0WgFI,1 +126,Carolina accident: Motorcyclist Dies in I-540 Crash With Car That Crossed Median: A motorcycle rider traveling... http://t.co/p18lzRlmy6,1 +128,FYI CAD:FYI: ;ACCIDENT PROPERTY DAMAGE;NHS;999 PINER RD/HORNDALE DR,1 +129,RT nAAYf: First accident in years. Turning onto Chandanee Magu from near MMA. Taxi rammed into me while I was halfway turned. Everyone conf___,1 +130,Accident left lane blocked in #Manchester on Rt 293 NB before Eddy Rd stop and go traffic back to NH-3A delay of 4 mins #traffic,1 +131,;ACCIDENT PROPERTY DAMAGE; PINER RD/HORNDALE DR,1 +132,???? it was an accident http://t.co/Oia5fxi4gM,0 +133,FYI CAD:FYI: ;ACCIDENT PROPERTY DAMAGE;WPD;1600 S 17TH ST,1 +134,8/6/2015@2:09 PM: TRAFFIC ACCIDENT NO INJURY at 2781 WILLIS FOREMAN RD http://t.co/VCkIT6EDEv,1 +135,Aashiqui Actress Anu Aggarwal On Her Near-Fatal Accident http://t.co/6Otfp31LqW,1 +136,Suffield Alberta Accident https://t.co/bPTmlF4P10,1 +137,9 Mile backup on I-77 South...accident blocking the Right 2 Lanes at Exit 31 Langtree Rd...consider NC 115 or NC 150 to NC 16 as alternate,1 +138,Has an accident changed your life? We will help you determine options that can financially support life care plans and on-going treatment.,0 +139,#BREAKING: there was a deadly motorcycle car accident that happened to #Hagerstown today. I'll have more details at 5 @Your4State. #WHAG,1 +141,@flowri were you marinading it or was it an accident?,0 +143,only had a car for not even a week and got in a fucking car accident .. Mfs can't fucking drive .,1 +144,.@NorwayMFA #Bahrain police had previously died in a road accident they were not killed by explosion https://t.co/gFJfgTodad,1 +145,I still have not heard Church Leaders of Kenya coming forward to comment on the accident issue and disciplinary measures#ArrestPastorNganga,0 +146,@afterShock_DeLo scuf ps live and the game... cya,0 +149,"'The man who can drive himself further once the effort gets painful is the man who will win.' +Roger Bannister",0 +151,320 [IR] ICEMOON [AFTERSHOCK] | http://t.co/yNXnvVKCDA | @djicemoon | #Dubstep #TrapMusic #DnB #EDM #Dance #Ices___ http://t.co/weQPesENku,0 +153,'There is no victory at bargain basement prices.' Dwight David Eisenhower,0 +156,320 [IR] ICEMOON [AFTERSHOCK] | http://t.co/vAM5POdGyw | @djicemoon | #Dubstep #TrapMusic #DnB #EDM #Dance #Ices___ http://t.co/zEVakJaPcz,0 +157,'Nobody remembers who came in second.' Charles Schulz,0 +158,@afterShock_DeLo im speaking from someone that is using a scuf on xb1 most of them people will end up getting on for ps also.,0 +159,'The harder the conflict the more glorious the triumph.' Thomas Paine,0 +160,#GrowingUpSpoiled going clay pigeon shooting and crying because of the 'aftershock',0 +161,So i guess no one actually wants any free Aftershock TC.....,0 +162,Aftershock was the most terrifying best roller coaster I've ever been on. *DISCLAIMER* I've been on very few.,0 +163,Aftershock https://t.co/xMWODFMtUI,0 +164,320 [IR] ICEMOON [AFTERSHOCK] | http://t.co/M4JDZMGJoW | @djicemoon | #Dubstep #TrapMusic #DnB #EDM #Dance #Ices___ http://t.co/n0uhAsfkBv,0 +165,320 [IR] ICEMOON [AFTERSHOCK] | http://t.co/vAM5POdGyw | @djicemoon | #Dubstep #TrapMusic #DnB #EDM #Dance #Ices___ http://t.co/zEVakJaPcz,0 +168,320 [IR] ICEMOON [AFTERSHOCK] | http://t.co/e14EPzhotH | @djicemoon | #Dubstep #TrapMusic #DnB #EDM #Dance #Ices___ http://t.co/22a9D5DO6q,0 +170,@KJForDays I'm seeing them and Issues at aftershock ??,0 +171,320 [IR] ICEMOON [AFTERSHOCK] | http://t.co/THyzOMVWU0 | @djicemoon | #Dubstep #TrapMusic #DnB #EDM #Dance #Ices___ http://t.co/83jOO0xk29,0 +172,320 [IR] ICEMOON [AFTERSHOCK] | http://t.co/THyzOMVWU0 | @djicemoon | #Dubstep #TrapMusic #DnB #EDM #Dance #Ices___ http://t.co/83jOO0xk29,0 +173,#WisdomWed BONUS - 5 Minute Daily Habits that could really improve your life. How many do you already do? #lifehacks http://t.co/TBm9FQb8cW,0 +174,Aftershock: Protect Yourself and Profit in the Next Global Financial Meltdown by David Wiedemer http http://t.co/WZTz4hgMVq,0 +175,That moment when you get on a scary roller coaster and the guy behind you is just screaming bloody murder ?????? #silverwood #aftershock,0 +176,Aftershock _㢠(2010) Full_㢠Streaming - YouTube http://t.co/vVE3UsesGf,0 +178,">> $15 Aftershock : Protect Yourself and Profit in the Next Global Financial... ##book http://t.co/f6ntUc734Z +@esquireattire",0 +180,Sometimes you face difficulties not because you're doing something wrong but because you're doing something right. - Joel Osteen,0 +182,'The only thing that stands between you and your dream is the will to try and the belief that it is actually possible.' - Joel Brown,0 +183,Praise God that we have ministry that tells it like it is!!! #now #wdyouth #biblestudy https://t.co/UjK0e5GBcC,0 +184,'Remembering that you are going to die is the best way I know to avoid the trap of thinking you have something to lose.' ___ Steve Jobs,0 +185,Tried orange aftershock today. My life will never be the same,0 +187,@OnFireAnders I love you bb,0 +190,Aftershock https://t.co/jV8ppKhJY7,0 +191,Aftershock back to school kick off was great. I want to thank everyone for making it possible. What a great night.,0 +193,People who say it cannot be done should not interrupt those who are doing it. ___ George Bernard Shaw,0 +194,'The first man gets the oyster the second man gets the shell.' Andrew Carnegie,0 +195,Anyone need a P/U tonight? I play Hybrid Slayer ps4 EU. HMU @Cod8sandscrims @EmpirikGaming @CoDAWScrims @4TP_KOTC @4TPFA @afterShock_Org,0 +196,Experts in France begin examining airplane debris found on Reunion Island: French air accident experts o... http://t.co/YVVPznZmXg #news,1 +197,Strict liability in the context of an airplane accident: Pilot error is a common component of most aviation cr... http://t.co/6CZ3bOhRd4,1 +198,@crobscarla your lifetime odds of dying from an airplane accident are 1 in 8015.,0 +199,Experts in France begin examining airplane debris found on Reunion Island: French air accident experts on Wedn... http://t.co/bKpFpOGySI,1 +201,@AlexAllTimeLow awwww they're on an airplane accident and they're gonna die what a cuties ???? good job!,1 +203,family members of osama bin laden have died in an airplane accident how ironic ?????? mhmmm gov shit i suspect,1 +204,Man Goes into Airplane Engine Accident: http://t.co/TYJxrFd3St via @YouTube,1 +205,Horrible Accident Man Died In Wings of Airplane (29-07-2015) http://t.co/i7kZtevb2v,1 +208,A Cessna airplane accident in Ocampo Coahuila Mexico on July 29 2015 killed four men including a State of Coahuila government official.,1 +209,#Horrible #Accident Man Died In Wings Airplane (29-07-2015) #WatchTheVideo http://t.co/p64xRVgJIk,1 +210,Experts in France begin examining airplane debris found on Reunion Island http://t.co/LsMx2vwr3J French air accident experts on Wednesday___,1 +211,Experts in France begin examining airplane debris found on Reunion Island: French air accident experts on Wednesday began examining t...,1 +212,#KCA #VoteJKT48ID mbataweel: #RIP #BINLADEN Family members who killed in an airplane's accident,1 +213,I almost sent my coworker nudes on accident thank god for airplane mode,0 +215,@mickinyman @TheAtlantic That or they might be killed in an airplane accident in the night a car wreck! Politics at it's best.,0 +216,Experts in France begin examining airplane debris found on Reunion Island: French air accident experts on... http://t.co/TagZbcXFj0 #MLB,1 +218,"This is unbelievably insane. +#man #airport #airplane #aircraft #aeroplane #runway #accident #freaky___ https://t.co/cezhq7CzLl",1 +219,Horrible Accident | Man Died In Wings of Airplaneå_(29-07-2015) http://t.co/wq3wJsgPHL,1 +220,Horrible Accident Man Died In Wings of Airplane (29-07-2015) http://t.co/TfcdRONRA6,1 +221,Usama bin Ladins family dead in airplane crash. Naturally no accident.,1 +222,Pilot Dies In Plane Crash At Car Festival https://t.co/kQ9aE6AP2B via @YouTube #Crash #Aircraft #Airplane #Pilot #Death #Accident #CarFest,1 +225,Strict liability in the context of an airplane accident - http://t.co/gibyQHhKpk,1 +226,DTN Brazil: Experts in France begin examining airplane debris found on Reunion Island: French air accident exp... http://t.co/M9IG3WQ8Lq,1 +229,Experts in France begin examining airplane debris found on Reunion Island: French air accident experts on Wedn... http://t.co/v4SMAESLK5,1 +231,Horrible Accident Man Died In Wings Of ___Airplane___ 29-07-2015. WTF You Can__ªt Believe Your EYES ___... http://t.co/6fFyLAjWpS,1 +232,"+ Nicole Fletcher one of a victim of crashed airplane few times ago. + +The accident left a little bit trauma for her. Although she's + ++",1 +235,OMG Horrible Accident Man Died in Wings of Airplane. http://t.co/xDxDPrcPnS,1 +237,"#OMG! I don't believe this. #RIP bro +#AirPlane #Accident #JetEngine #TurboJet #Boing #G90 http://t.co/KXxnSZp6nk",1 +238,Experts in France begin examining airplane debris found on Reunion Island: French air accident experts on Wednesday began examining t...,1 +240,I had a airplane accident.,1 +241,My phone looks like it was in a car ship airplane accident. Terrible,0 +242,Statistically I'm at more of risk of getting killed by a cop than I am of dying in an airplane accident.,0 +243,airplane crashes on house in Colombia 12 people die in accident https://t.co/ZhJlfLBHZL,1 +244,The shooting or the airplane accident https://t.co/iECc1JDOub,1 +245,Could a drone cause an airplane accident? Pilots worried about use of drones esp. in close vicinity of airports http://t.co/kz35rGngJF #,1 +246,Early wake up call from my sister begging me to come over & ride w/her in the ambulance to the hospital #RODKiai,1 +247,http://t.co/AY6zzcUpnz Twelve feared killed in Pakistani air ambulance helicopter crash http://t.co/sC9dNS41Mc,1 +248,Two air ambulances on scene of serious crash between two cars and lorry in ... - http://t.co/9pFEaQeSki http://t.co/fntG70rnkx | #EMSNe___,1 +249,Twelve feared killed in Pakistani air ambulance helicopter crash - Reuters http://t.co/mDnUGVuBwN #yugvani,1 +251,Leading emergency services boss welcomes new ambulance charity http://t.co/Mj2jQ2pSv6,0 +252,Anyone travelling Aberystwyth-Shrewsbury right now there's been an incident. Services at a halt just outside Shrews. Ambulance on scene.,1 +253,Twelve feared killed in Pakistani air ambulance helicopter crash http://t.co/Xum8YLcb4Q,1 +254,AMBULANCE SPRINTER AUTOMATIC FRONTLINE VEHICLE CHOICE OF 14 LEZ COMPLIANT | eBay http://t.co/4evTTqPEia,0 +256,New Nanotech Device Will Be Able To Target And Destroy Blood Clots http://t.co/HFy5V3sLBB,0 +258,@20skyhawkmm20 @traplord_29 @FREDOSANTANA300 @LilReese300 it was hella crazy 3 fights an ambulance and a couple mosh pits ??,1 +260,If I get run over by an ambulance am I lucky? #justsaying #randomthought,0 +261,#news Twelve feared killed in Pakistani air ambulance helicopter crash http://t.co/bFeS5tWBzt #til_now #DNA,1 +262,http://t.co/7xGLah10zL Twelve feared killed in Pakistani air ambulance helicopter crash http://t.co/THmblAATzP,1 +263,@TanSlash waiting for an ambulance,0 +264,@fouseyTUBE you ok? Need a ambulance. Hahahah that was good! http://t.co/ZSbErqNN9n,0 +265,AMBULANCE SPRINTER AUTOMATIC FRONTLINE VEHICLE CHOICE OF 14 LEZ COMPLIANT | eBay http://t.co/q8IVrzOJZv,0 +266,Pakistan air ambulance helicopter crash kills nine http://t.co/8E7rY8eBMf,1 +267,@TheNissonian @RejectdCartoons nissan are you ok do you need medical assistance i can call an ambulance if you need me to,0 +268,EMS1: NY EMTs petition for $17 per hour __÷minimum wage__ª http://t.co/4oa6SWlxmR #ems #paramedics #ambulance,0 +269,http://t.co/FCqmKFfflW Twelve feared killed in Pakistani air ambulance helicopter crash http://t.co/vAyaYmbNgu,1 +270,Twelve feared killed in Pakistani air ambulance helicopter crash http://t.co/3bRme6Sn4t,1 +271,AMBULANCE SPRINTER AUTOMATIC FRONTLINE VEHICLE CHOICE OF 14 LEZ COMPLIANT | eBay http://t.co/UJrX9kgawp,0 +272,AMBULANCE SPRINTER AUTOMATIC FRONTLINE VEHICLE CHOICE OF 14 LEZ COMPLIANT | eBay http://t.co/Kp2Lf4AuTe,0 +273,@Kiwi_Karyn Check out what's in my parking lot!! He said that until last year it was an ambulance in St Johns. http://t.co/hPvOdUD7iP,0 +274,when you don't know which way an ambulance is coming from <<,1 +276,#reuters Twelve feared killed in Pakistani air ambulance helicopter crash http://t.co/ShzPyIQok5,1 +277,http://t.co/pWwpUm6RBj Twelve feared killed in Pakistani air ambulance helicopter crash http://t.co/ySpON4d6Qo,1 +279,Why is there an ambulance right outside my work,0 +280,___@LeoBlakeCarter: This dog thinks he's an ambulance ?????? http://t.co/MG1lpGr0RM___@natasha_rideout,0 +281,HAPPENING NOW - HATZOLAH EMS AMBULANCE RESPONDING WITH DUAL SIRENS AND___ https://t.co/SeK6MQ6NJF,0 +283,http://t.co/FueRk0gWui Twelve feared killed in Pakistani air ambulance helicopter crash http://t.co/Mv7GgGlmVc,1 +285,http://t.co/X5YEUYLT1X Twelve feared killed in Pakistani air ambulance helicopter crash http://t.co/2UgrMd1z1n,1 +287,Twelve feared killed in Pakistani air ambulance helicopter crash http://t.co/TH9YwBbeet #worldNews,1 +289,Twelve feared killed in Pakistani air ambulance helicopter crash http://t.co/X2Qsjod40u #worldnews,1 +290,What's the police or ambulance number in Lesotho? Any body know?,0 +291,@medic914 @AACE_org I am surprised we still cannot standardised the clinical practice across the 11 NHS ambulance trust.,0 +293,http://t.co/J8TYT1XRRK Twelve feared killed in Pakistani air ambulance helicopter crash http://t.co/9d4nAzOI94,1 +294,People who try to j-walk while an ambulance is passing... I hate you.,0 +296,The episode where Trunks annihilated Freiza is the cleanest shit ever. He showed that nigga no mercy.,0 +297,THEY SHALL BE ANNIHILATED AND ALL OF THEIR PETEBESTS DESSICATED AND LAID BARE. THEN YOU SHALL KNEEL BEFORE ME.,0 +298,Uribe just annihilated that baseball. #Mets,0 +299,@marksmaponyane Hey!Sundowns were annihilated in their previous meeting with Celtic.Indeed its an improvement.,0 +301,@Volfan326 @TNeazzy Mizzou has annihilated florida the past 2 seasons even ended muschamp's career just can't compete with Bama,0 +302,Annihilated Abs . ?? http://t.co/1xPw292tJe,1 +303,Be annihilated for status education mba on behalf of a on easy street careen: eOvm http://t.co/e0pI0c54FF,0 +307,*to Luka* They should all die! All of them! Everything annihilated! - Alois Trancy,0 +309,@ACarewornHeart Have a good un fella sorry I won't be there to get annihilated with you :(,0 +310,Cop pulls drunk driver to safety SECONDS before his car is hit by train. http://t.co/tHrhKHOGcUå_ http://t.co/tZSZmF2qxE via @ViralSpell,1 +312,You must be annihilated!,0 +313,Cop pulls drunk driver to safety SECONDS before his car is hit by train. http://t.co/C0nKGp6w03å_ http://t.co/IMWmfDJSSn via @ViralSpell,1 +314,BOOM! Your country was just entirely annihilated by a h___ ___ Britain https://t.co/IrFCn71sZv,0 +315,@AmirKingKhan you would have been annihilated so you might as well thank @FloydMayweather,0 +316,One thing for sure-God has promised Israel will not be annihilated. But...the horror of Iran w/nukes. https://t.co/xn09Mx6sxy,0 +318,@violentfeminazi I guess that's ok for Armenians since we've spent most of our history getting annihilated,1 +320,70 years since we annihilated 100000 people instantly and became aware that we have the ability to annihilate the whole of humanity,1 +321,day 1 of tryouts went good minus the fact I stopped quickly to get a short ball and Annihilated my toenail injury even more,0 +322,"During the 1960s the oryx a symbol of the Arabian Peninsula were annihilated by hunters. +http://t.co/yangEQBUQW http://t.co/jQ2eH5KGLt",1 +327,(To Luka) 'They should all die! All of them! Everything annihilated!' - Alois Trancy -,0 +328,Ready to get annihilated for the BUCS game,1 +329,@PhilipDuncan @breakfastone People 'annihilated by last nights weather'... Really Philip thought you would have forecast that...,1 +330,Domain other sophistication be annihilated closely up-to-the-minute feat: ZrNf,0 +331,@stormbeard @steel_lord I seen Judas Priest in 2005 when Rob came back; Scorpions as support. Fucking annihilated the place. Astonishing gig,0 +332,Officially skipping out on #FantasticFour/#Fant4stic/whatever the hashtag is. It's getting ANNIHILATED in reviews. Bummer.,0 +334,@TomcatArts thus explaining why you were all annihilated. But the few or in this case you the only survivor evolved and became godlike,1 +335,just completely annihilated cech with paul keegan what a time to be alive,0 +336,@TomcatArts 'who then were annihilated by the legion itself. The survivors of the imperfect hybrid project quickly formed a new secret cell,0 +337,@SirBrandonKnt exactly. That's why the lesnar/cena match from summerslam last year was so great because Brock annihilated a guy who's,0 +338,Cop pulls drunk driver to safety SECONDS before his car is hit by train. http://t.co/F1BAkpNyn6å_ http://t.co/lZXwoAyE4x via @ViralSpell,1 +340,"ANNIHILATED IN DAMASCUS: SYRIAN ARMY GRINDS __÷ALLOOSH AND HIS GANG INTO THE MANURE PILE +http://t.co/7rakhP3bWm",1 +341,@thatdes ok i wasn't completely forthright i may have also been in a food coma bc of the kebab/tahini/pickles i also annihilated w/fries,0 +344,A fun filled happy-hour at Simmons bar in Camden with this handsome one ?? (I got annihilated apart from this game) http://t.co/4JNo677Zkv,0 +345,Juanny Beisbol Sr. Annihilated that ball. #LGM,0 +346,@rvfriedmann Hell is just a fraction of his belief of total annihilation destruction of USA @LodiSilverado @ritzy_jewels,1 +347,@POTUS Maybe we should call Israel and tell them we're sorry are Pres has sold them down the river to annihilation.,0 +348,Evildead - Annihilation of Civilization http://t.co/sPfkE5Kqu4,0 +349,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... http://t.co/6LoJOoROuk via @Change,0 +352,annihilating quarterstaff of annihilation,1 +353,World Annihilation vs Self Transformation http://t.co/pyehwodWun Aliens Attack to Exterminate Humans http://t.co/pB2N77nSKz,0 +354,:StarMade: :Stardate 3: :Planetary Annihilation:: http://t.co/I2hHvIUmTm via @YouTube,1 +355,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... https://t.co/m8MvDSPJp7 via @Change,0 +356,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... https://t.co/sW1sBua3mN via @Change,1 +358,@KimKardashian can you please sign and share this petition to save wild horses in Arizona. http://t.co/3tsSXPHuFE ????,0 +360,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... http://t.co/KPQk0C4G0M via @Change,1 +361,@TheEllenShow Please check into Salt River horses help stop the annihilation about to happen without 54000 more signatures.change .org Thx,0 +363,Are souls punished withå_annihilation? http://t.co/c1QXJWeQQU http://t.co/Zhp0SOwXRy,0 +364,@CalFreedomMom @steph93065 not to mention a major contributor to the annihilation of Israel,1 +365,@willienelson We need help! Horses will die!Please RT & sign petition!Take a stand & be a voice for them! #gilbert23 https://t.co/e8dl1lNCVu,1 +368,I reject the laws of the misguided false prophets imprison nations fueling self annihilation,0 +370,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... https://t.co/MatIJwkzbh via @Change,0 +371,"The annihilation of Jeb Christie & Kasich is less than 24 hours away.. +Please God allow me at least one more full day...",0 +372,@Barbi_Twins We need help-horses will die! Please RT & sign petition! Take a stand & be a voice for them! #gilbert23 https://t.co/e8dl1lNCVu,0 +375,@Whippenz We need help! Horses will die!Please RT & sign petition!Take a stand & be a voice for them! #gilbert23 https://t.co/e8dl1lNCVu,0 +380,Hey #AZ: Sign this petition to save the #WildHorses @ #TantoNationalForest! A @RollingStones sing-a-long is in order: http://t.co/WM5l8PJ2iY,0 +381,Stop the Annihilation of the Salt River Wild Horses! http://t.co/wVobVVtXKg via @Change,1 +383,@SonofBaldwin and he's the current Nova in the bookslast I checked..he was tied into the books in 2011 after Rider died during Annihilation,0 +384,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... https://t.co/x2Wn7O2a3w via @Change,0 +386,Please sign & RT to save #SaltRiverWildHorses http://t.co/IKUAYUSEqt http://t.co/BQBHUyfmE9,0 +389,THANKS!!!!! @COUNT DANTE. :) DO JOIN US BY FOLLOWING THE @ANNIHILATION ZONE. JOHNNY.,0 +390,World Annihilation vs Self Transformation http://t.co/pyehwodWun Aliens Attack to Exterminate Humans http://t.co/8jxqL8Cv8Z,1 +393,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... https://t.co/0fekgyBY5F via @Change,0 +394,U.S National Park Services Tonto National Forest: Stop the Annihilation of the Salt River Wild Horse... http://t.co/SB5R7ShcCJ via @Change,1 +396,I'm gonna fight Taylor as soon as I get there.,0 +397,ohH NO FUKURODANI DIDN'T SURVIVE THE APOCALYPSE BOKUTO FEELS HORRIBLE my poor boy my ppor child,1 +398,will there be another jocelyn birthday apocalypse,0 +401,RT: janenelson097: RT StephenSCIFI: Adaptation Watch: Charlie Human's APOCALYPSE NOW NOW Optioned for Film #sciencefiction ___,0 +402,Apocalypse please,0 +404,Another hour! It's August 05 2015 at 08:02PM Here's Red Rover Zombie Apocalypse 2014! http://t.co/cf9e6TU3g7 #internetradio #collegeradi___,1 +406,@HoneyBunzGem @primalkitchen I feel like me doing a pull-up is one of the stages of the Apocalypse.,0 +407,She's kinda hot played on the radio today. What's next? Disease to all? The apocalypse has started everyone. Be careful.,0 +409,But if it's the apocalypse lol gf m8,0 +410,I know it's a question of interpretation but this is a sign of the apocalypse. I called it https://t.co/my8q1uWIjn,1 +412,Julie + R is the apocalypse version of Romeo + Juliet #warmbodies,0 +413,the apocalypse is upon us,0 +415,RT: fittscott: Minecraft- NIGHT LUCKY BLOCK MOD (BOB APOCALYPSE WITHER 2.0 & MORE!) Mod Showcase Popularmmos: http://t.co/MuL1J9AEUx vi___,0 +418,And so it begins.. day one of the snow apocalypse,1 +419,"RT: Our_Mother_Mary: Short Reading + +Apocalypse 21:1023 + +In the spirit the angel took me to the top of an enormous high mountain and... ___",0 +420,candylit: Imagine sarumi in a zombie apocalypse Fighting back to back Heart to heart conversations over the... http://t.co/xIZkjffF29,0 +421,RT: ZONEWolf123: I liked a YouTube video http://t.co/u66kYg11ZD Minecraft: NIGHT LUCKY BLOCK MOD (BOB APOCALYPSE WITHER 2.0 & MORE!) Mo___,0 +423,And that's because on my planet it's the lone audience of the apocalypse!,0 +424,Dad bought a DVD that looks like a science doc on the front but I read the back and it's actually about the impending biblical apocalypse,1 +426,@alexandrapullin It is indeed. If the apocalypse comes this week I know where I'll be :),0 +427,GO LOOK AT GRIZZLY PEAK RIGHT NOW... It looks like the beginning of an dystopian apocalypse movie,0 +429,My niece just asked me 'would you be scared if there was an apocalypse here?' ????,0 +432,There's a Storm over Cairo in the latest 'X-Men Apocalypse' set photo https://t.co/fS012trUDG via @YahooTV,1 +435,Minecraft- NIGHT LUCKY BLOCK MOD (BOB APOCALYPSE WITHER 2.0 & MORE!) Mod Showcase Popularmmos: http://t.co/TNgYE2FKlv via @YouTube,0 +436,Shot Through The Heart XV: You are going to totally give love a bad name with this heart pierc http://t.co/xpFmR368uF http://t.co/ejdHvLKXAf,0 +437,RT: Geek_Apocalypse: 4pm GMT :Hesse plays dark souls 2 day 9: http://t.co/TnGPsHNL87 http://t.co/imzLNZLtF5 #etcPB,0 +438,I know where to go when the zombies take over!! http://t.co/hUTHXlkyxy,0 +440,The latest from @BryanSinger reveals #Storm is a queen in #Apocalypse @RuPaul @AlexShipppp http://t.co/oQw8Jx6rTs,1 +442,Shadow boxing the apocalypse,1 +443,"Short Reading + +Apocalypse 21:1023 + +In the spirit the angel took me to the top of an enormous high mountain and... http://t.co/v8AfTD9zeZ",1 +444,Enjoyed live-action Attack on Titan but every time I see posters I'm reminded how freshly clean and coiffed everyone is in the apocalypse.,0 +445,I liked a @YouTube video http://t.co/ki1yKrs9fi Minecraft: NIGHT LUCKY BLOCK MOD (BOB APOCALYPSE WITHER 2.0 & MORE!) Mod Showcase,0 +446,#PBBan (Temporary:300) avYsss @'aRmageddon | DO NOT KILL | FLAGS ONLY | Fast XP' for Reason,0 +447,#PBBan (Temporary:300) Russaky89 @'aRmageddon | DO NOT KILL | FLAGS ONLY | Fast XP' for Reason,0 +448,((OFFICIAL VID)) #DoubleCups >> https://t.co/lfKMTZaEkk >> @TrubGME Prod @THISIZBWRIGHT >> #ARMAGEDDON,0 +450,ouvindo Peace Love & Armageddon,0 +451,Best movie you've ever seen? - Armageddon http://t.co/qoUXIgdtbZ,0 +452,Bed time. Don't wake me up unless revolution or Armageddon start.,0 +453,Red Faction: Armageddon (Microsoft Xbox 360 2011) - Full read by eBay http://t.co/ypbVS1IJya http://t.co/9dFLv6ynqr,0 +454,@KatieKatCubs you already know how this shit goes. World Series or Armageddon.,0 +455,RT @Ophiuchus2613: #Love #TrueLove #romance lith #Voodoo #seduction #Astrology #RTRRT #LOTZ 9-11 #apocalypse #Armageddon #1008pla___,0 +456,"//im gonna beat armageddon as Hsu Hao ???? +just got a flawless on my first try",0 +457,@ENews Ben Affleck......I know there's a wife/kids and other girls but I can't help it. I've loved him since Armageddon #eonlinechat,0 +459,'If I'd have had a long coat to hand I'd have worn it. The certainty of armageddon bears a sense of occasion.',0 +461,YOUR PHONE IS SPYING ON YOU! Hidden Back Door NSA Data Mining Software | THE FINANCIAL ARMAGEDDON BLOG http://t.co/qyCw5JJaj1,0 +462,#PBBan (Temporary:300) hyider_ghost2 @'aRmageddon | DO NOT KILL | FLAGS ONLY | Fast XP' for Reason,1 +463,RT @RTRRTcoach: #Love #TrueLove #romance lith #Voodoo #seduction #Astrology #RTRRT #LOTZ 9-11 #apocalypse #Armageddon #1008planet___,0 +465,#PBBan (Temporary:300) fighterdena @'aRmageddon | DO NOT KILL | FLAGS ONLY | Fast XP' for Reason,0 +466,Photo: Sketch I did based on the A Taste of Armageddon episode of #startrek #tos http://t.co/a2e6dcsk88,0 +467,Armageddon https://t.co/uCSUDk3q1d,1 +468,"@AberdeenFC @AberdeenFanPage + +Good luck to the ?????? tomorrow night + +Get some coefficient points plz +@Armageddon????",0 +469,@paddytomlinson1 ARMAGEDDON,0 +470,@RohnertParkDPS You're another one for the history books! (Thank the Justice Department!) And by the way I haven't paid income tax in 20yrs.,0 +471,"Vladimir Putin Issues Major Warning But Is It Too Late To Escape Armageddon? +http://t.co/gBxafy1m1C",1 +472,God's Kingdom (Heavenly Gov't) will rule over all people on the earth after Armageddon. http://t.co/8HGcBXUkz0 http://t.co/4kopkCyvTt,0 +474,L B #Entertainment lot of 8 #BruceWillis MOVIES #DVD DIE HARD 1 2 12 MONKEYS ARMAGEDDON SIXTH #eBay #Auction http://t.co/CxDJApzXMP,0 +475,Let__ªs talk some more about your goof guild Saunders. Come right up here on stage. https://t.co/hkBxxvd9Iw,0 +476,@Karnythia my niece is gaining the ability to stand. I'm getting prepared for toddler apocalypse Armageddon,0 +477,Check out #PREPPERS #DOOMSDAY MUST HAVE LIBRARY COLLECTION ON CD #shtf #preppertalk #survival #2A #prepper http://t.co/VPQTGeQLmA via @eBay,0 +478,@Erker Again?? Eep! Thought of you yesterday when I saw that hella scary hail. #armageddon?,1 +479,So the Ahamedis think the Messiah had already come 125 years ago? Where is Armageddon? Where is the Dajaal? Where is Gog & Magog?!,0 +480,Sadly How Windows 10 Reveals Microsoft's Ethics Armageddon http://t.co/sTfTjCrjEa,0 +481,"Armageddon averted by El Patron +#UltimaLucha",0 +482,@samihonkonen If you have the time (23 hours ??) the latest series about WW1 Blueprint for Armageddon is extremely impressive.,0 +483,European Fitba till Christmas ARMAGEDDON,0 +484,#Christians United for #Israel (#CUFI): Jews should convert soon or die by armageddon https://t.co/4aRWwRZPsr #US http://t.co/mkJQ9yfMP8,0 +485,(OFFICIAL VID) > #DoubleCups >> https://t.co/lfKMTZaEkk >> @TrubGME Prod @THISIZBWRIGHT >> #ARMAGEDDON .,0 +486,Tomorrow is the day we start armageddon #preseasonworkouts ????,0 +487,Lee does comedy: ___@LeeJasper: Working class Tories prepare for your Armageddon. #InterestRateRise___,0 +489,9 Charts Prove Financial Crisis Part 2 Has BEGUN!: The Financial Armageddon Economic Collapse Blog tracks tren... http://t.co/vHCXTvCINr,0 +490,Paul Craig Roberts ___ Vladimir Putin Issues Major Warning But Is It Too Late To Escape A http://t.co/NVfKzv5FEx #brics #roberts #russia,1 +492,@RohnertParkDPS You're on stage now! Right under the lights! Isn't it funny?! Where do you get the goofballs with which you staff your PD?,0 +493,**OFFICIAL VID** #TheReal >>> https://t.co/4i0Rjc9RQU >>> @TrubGME >>> #ARMAGEDDON Comin Soon!!,0 +495,"Well done Celtic Fingers crossed for Aberdeen tomorrow night! +Armageddon eh.... ??",0 +496,Beyonce Is my pick for http://t.co/nnMQlz91o9 Fan Army #Beyhive http://t.co/o91f3cYy0R 77,0 +498,One Direction Is my pick for http://t.co/q2eBlOKeVE Fan Army #Directioners http://t.co/eNCmhz6y34 x1402,0 +499,5 Seconds of Summer Is my pick for http://t.co/J6WsePTXgA Fan Army #5SOSFAM http://t.co/qWgIwC9w7Z,0 +501,22.Beyonce Is my pick for http://t.co/thoYhrHkfJ Fan Army #Beyhive http://t.co/WvJ39a3BGM,0 +502,17.Beyonce Is my pick for http://t.co/thoYhrHkfJ Fan Army #Beyhive http://t.co/WvJ39a3BGM,0 +503,One Direction Is my pick for http://t.co/q2eBlOKeVE Fan Army #Directioners http://t.co/eNCmhz6y34 x1411,0 +504,Seeing that army of whitewalkers was the very first thing that has slightly intrigued me on GoT so far,0 +506,"Build your own kingdom and lead your army to victory! https://youtu. + +Start g this friend code: LZKTJNOX http://t.co/zZ0cEwEw64",0 +509,Salvation Army hosts rally to reconnect fathers with children: The Salvation Army is hosting a back to school rally___ http://t.co/rDjpor3AZg,1 +512,Vote for #Directioners vs #Queens in the 5th round of the @Billboard #FanArmyFaceOff http://t.co/Kgtxnnbj7y,0 +513,But if you build an army of 100 dogs and their leader is a lion all dogs will fight like a lion.,1 +514,'Show Me a Hero': TV Review http://t.co/KaCCPk85wf http://t.co/NniXodHIGc,0 +516,One Direction Is my pick for http://t.co/q2eBlOKeVE Fan Army #Directioners http://t.co/eNCmhz6y34 x1392,1 +517,INFANTRY Mens Lume Dial Army Analog Quartz Wrist Watch Sport Blue Nylon Fabric - Full rea___ http://t.co/hEP9k0XgHb http://t.co/80EBvglmrA,0 +518,One Direction Is my pick for http://t.co/q2eBlOKeVE Fan Army #Directioners http://t.co/eNCmhz6y34 x1441,0 +519,VICTORINOX SWISS ARMY DATE WOMEN'S RUBBER MOP WATCH 241487 http://t.co/yFy3nkkcoH http://t.co/KNEhVvOHVK,1 +520,".: .: .: .: .: .: .: .: .: .: .: .: .: .: .: .: .: .: .: .: .: RT DrAyesha4: #IndiaKoMunTorJawabDo + +Indian Army ki___ http://t.co/WJLJq3yA4g",0 +521,5 Seconds of Summer Is my pick for http://t.co/qcHV3JqOVK Fan Army #5SOSFAM http://t.co/gc0uDfnFgg __1,0 +522,Beyonce Is my pick for http://t.co/nnMQlz91o9 Fan Army #Beyhive http://t.co/o91f3cYy0R 78,0 +523,One Direction Is my pick for http://t.co/q2eBlOKeVE Fan Army #Directioners http://t.co/eNCmhz6y34 x1386,0 +524,"You da One + +#MTVSummerStar #VideoVeranoMTV #MTVHottest Britney Spears Lana Del Rey",0 +526,One Direction Is my pick for http://t.co/y9WvqKGbBI Fan Army #Directioners http://t.co/S5F9FcOmp8,0 +527,Stony Jackson is America's last hope as he leads an army of felons thus and army rejects against the army o Satan - http://t.co/0wbEcdMHQo,0 +528,WWI WWII JAPANESE ARMY NAVY MILITARY JAPAN LEATHER WATCH WAR MIDO WW1 2 - Full read by eBay http://t.co/F9j3l2Yjl4 http://t.co/mwwWOWCayO,0 +530,Beyonce Is my pick for http://t.co/nnMQlz91o9 Fan Army #Beyhive http://t.co/o91f3cYy0R 72,0 +531,7.Beyonce Is my pick for http://t.co/thoYhrHkfJ Fan Army #Beyhive http://t.co/WvJ39a3BGM,0 +533,Beyonce Is my pick for http://t.co/nnMQlz91o9 Fan Army #Beyhive http://t.co/o91f3cYy0R 66,0 +535,6.Beyonce Is my pick for http://t.co/thoYhrHkfJ Fan Army #Beyhive http://t.co/WvJ39a3BGM,0 +536,POTUS appoints Brig. Gen. Richard G. Kaiser as member of the Mississippi River Commission. Learn more about the MRC: http://t.co/vdUKcV7YJy,0 +538,@AP what a violent country get the army involved to help control the killings and bring back peace to the poor people.,1 +540,WWI WWII JAPANESE ARMY NAVY MILITARY JAPAN LEATHER WATCH WAR MIDO WW1 2 - Full read by eBay http://t.co/QUmcE7W2tY http://t.co/KTKG2sDhHl,0 +542,WWI WWII JAPANESE ARMY NAVY MILITARY JAPAN LEATHER WATCH WAR MIDO WW1 2 - Full read by eBay http://t.co/obfD7e4QcP http://t.co/yAZjE5OwVk,0 +543,One Direction Is my pick for http://t.co/q2eBlOKeVE Fan Army #Directioners http://t.co/eNCmhz6y34 x1434,0 +544,One Direction Is my pick for http://t.co/iMHFdaOWRd Fan Army #Directioners http://t.co/4fTZJk94Dt,0 +546,Two Jewish Terrorists Charged In Historic-Church Arson | The Ugly Truth http://t.co/iEksNFSbY7 http://t.co/VWCf3slkrW,0 +550,Spokane authorities say they're struggling to solve arson cases like today's on Hamilton. http://t.co/Qbs2k01WzK http://t.co/mvLZIYsGLL,1 +551,Thousands attend a rally organized by Peace Now protesting the arson attack that took the life of an http://t.co/bvCKd9pdTi,1 +552,Add Familia to the arson squad.,0 +556,Another fake hate crime Lesbians burn their own house down. What else Is new :http://t.co/66oBQmxImb,1 +558,Los Angeles Times: Arson suspect linked to 30 fires caught in Northern ... - http://t.co/xwMs1AWW8m #NewsInTweets http://t.co/TE2YeRugsi,1 +559,Mourning notices for stabbing arson victims stir __÷politics of grief__ª in Israel http://t.co/eug6zHciun,1 +560,Mourning notices for stabbing arson victims stir __÷politics of grief__ª in Israel http://t.co/KkbXIBlAH7,1 +561,The Sound of Arson,0 +563,Owner of Chicago-Area Gay Bar Admits to Arson Scheme http://t.co/MYhOHvrHiL #LGBT | https://t.co/TM5HTHFDO0,0 +564,Wait What??? http://t.co/uAVFRtlfs4 http://t.co/85G1pCcCXG,0 +565,Arson suspect linked to 30 fires caught in Northern California http://t.co/EJ2GHNAfHY,1 +567,Trial Date Set for Man Charged with Arson Burglary http://t.co/WftCrLz32P,1 +568,"After death of Palestinian toddler in arson +attack Israel cracks down on Jewish",1 +569,Palestinian Teen Killed Amid Protests Against Arson Attack http://t.co/okVsImoGic,1 +570,#Kisii Police in Kisii hunt for students over failed arson plot: Police in Kisii hunt for students... http://t.co/m5SbFRrSn7 #CountyNews,1 +571,Mariah getting thick in the shoulders poor girl.,1 +574,Mourning notices for stabbing arson victims stir __÷politics of grief__ª in Israel: Posters for Shira Banki and A... http://t.co/3GZ5zQQTHe,0 +575,RelaxInPR: miprv: RT latimes: Arson suspect linked to 30 fires caught in Northern California http://t.co/ylhAyfaOOu,1 +576,Jewish leaders prayed at the hospital where a Palestinian family is being treated after arson http://t.co/Wf8iTK2KVx via @HuffPostRelig,1 +577,Owner of Chicago-Area Gay Bar Admits to Arson Scheme http://t.co/0TSlQjOKvh via @theadvocatemag #LGBT,0 +578,@sayn_ae angel or arson,0 +579,#Kisii Police in Kisii hunt for students over failed arson plot: Police in Kisii hunt for students... http://t.co/5bdrFU1duo #CountyNews,1 +580,Mourning notices for stabbing arson victims stir __÷politics of grief__ª in Israel http://t.co/Q4L7Dg56JM,1 +581,Owner of Chicago-Area Gay Bar Admits to Arson Scheme http://t.co/2Y9dnP5vtg via @theadvocatemag #LGBT | https://t.co/6XuL6DCOsh,0 +583,Owner of Chicago-Area Gay Bar Admits to Arson Scheme http://t.co/UBFr1URAFc #LGBT | https://t.co/AlnV51d95x,0 +584,Mourning notices for stabbing arson victims stir __÷politics of grief__ª in Israel http://t.co/GbluHRrlto,1 +585,Owner of Chicago-Area Gay Bar Admits to Arson Scheme http://t.co/ZPxE3fMYNG #LGBT,1 +588,Arson suspect linked to 30 fires caught in Northern California http://t.co/wnuqQAtTTP (via @latimes),1 +589,Tennessee lesbian couple faked hate crime and destroyed own home with arson___ http://t.co/10mUEY8PXJ #Lesbian,1 +592,Arson suspect linked to 30 fires caught in Northern California - Los Angeles Times http://t.co/PrRB4fhXtv,1 +594,Arson suspect linked to 30 fires caught in Northern California http://t.co/u1fuWrGK5U,1 +598,@_301DC @Cloudy_goldrush i hate white people mo,1 +599,#NOWPLAYING Arsonist MC - So Impressed - @ARSONISTMUSIC http://t.co/1ElreH1jLJ,0 +600,Alleged East Bay serial arsonist arrested http://t.co/WR48AQTUm7,1 +603,@Safyuan just a minor citation for possesion of a decriminalized substance im not facing any time,0 +604,Suspected serial arsonist arrested in Calif. http://t.co/PzotPDGAkI,1 +606,Arson suspect linked to 30 fires caught in Northern California http://t.co/mmGsyAHDzb,1 +607,@local_arsonist @diamorfiend the legal system NEVER forgets,0 +608,@Casper_rmg u on dick,0 +609,Bloor/Ossington arsonist also burned a mattress on Northumberland St #cbcto http://t.co/wpDvT31sne,0 +611,'wHeRE's mY aRsOnISt aT???',0 +612,If you don't have anything nice to say you can come sit with me.,0 +613,#Vegetarian #Vegan Video shows arsonist torching popular BK restaurant Strictly Vegetarian... http://t.co/kxpLYoM9RR #GoVegan #UniteBlue,0 +614,#Arsonist arrested for setting many fires. WATCH tonight__ªs other #headlines: http://t.co/sqgogJ3S5r. #Nightbeat @VeronicaDLCruz #2MinuteMix,1 +615,Video Captures Man Removing American Flag From Long Beach CA Home Burning It; Arsonist Sought http://t.co/JP2QlrunjJ http://t.co/jbpgkGOwSi,0 +617,@58hif my trick is to think about nasty things,0 +619,#Spotlight Take Me To Paradise by Arsonist MC #WNIAGospel http://t.co/1he4UfaWZm @arsonistmusic http://t.co/BNhtxAEZMM,0 +621,who makes these? http://t.co/28t3NWHdKy,0 +622,on town of salem i just melted ice cube bc im the arsonist :D,0 +623,Arsonists being blamed for a blaze at a plastics recycling business in Adelaide | @pcaldicott7 reports. #7NewsAdl http://t.co/r1Xwdnvb0g,0 +624,i be on that hotboy shit,0 +625,Zodiac Girl feat Trey Dupree (Produced By Sparkz Beatz) | Chuck Da Arsonist http://t.co/HDKd9J2lw0,0 +628,@local_arsonist LMFAO,0 +630,Alleged East Bay serial arsonist arrested #SanFrancisco - http://t.co/ojuHfkHVb2,1 +631,@_Doofus_ @diamorfiend im jokin still cant be on moves:/,0 +633,@local_arsonist I guess u can say that it's just some shit I was thinking about,0 +635,Arsonist Sets NYC Vegetarian Restaurant on Fire: Police #NewYork - http://t.co/Nr7usT3uh8,1 +636,I liked a @YouTube video from @slimebeast http://t.co/ulr6MyklnH Town of Salem | How to Win as The Arsonist,0 +637,@Casper_rmg @BestComedyVine whats cracking cuz,0 +638,smoke good fuck eat drink drive nice car wear all green mink,0 +640,kill i got court the day after earl,0 +641,@local_arsonist lmao but real live you should go,0 +642,Owner of Chicago-Area Gay Bar Admits to Arson Scheme: Frank Elliott pleaded guilty to hiring an arsonist to to... http://t.co/jCFEhrHLq8,0 +643,Trusting Iran to stop terrorism is like inviting an arsonist to join the fire brigade - Telegraph http://t.co/2Z2HTDjQZD,0 +644,Big Top Burning The True Story Of An Arsonist A Missing Girl ___ : http://t.co/QxH1H61cwD .,1 +646,Stay vigilent. Civil liberties are under constant attack. #nativehuman #myreligion https://t.co/WWu070Tjej,1 +649,Credit to @pfannebeckers for inspiring me to rediscover this fantabulous #tbt http://t.co/wMHy47xkiL,0 +651,http://t.co/pTKrXtZjtV Nashville Theater Attack: Will Gun Grabbers Now Demand ___Hatchet Control?___,1 +652,"BREAKING: Terror Attack On +Police Post #Udhampur",1 +653,Demi stans really think Heart Attack sold 5/6 million copies ??,0 +654,it scares me that there's new versions of nuclear attack warnings like just knowing that governments still prepare for them,1 +655,ISIL claims suicide bombing at Saudi mosque that killed at least 15 http://t.co/Y8IcF89H6w http://t.co/t9MSnZV1Kb,1 +656,@DatTomm the funniest part about that twitter is the feminists that try to attack it 4Head,0 +657,Horrific attack on wife by muslim in Italy http://t.co/nY3l1oRZQb LiveLeak #News,1 +658,__ª93 blasts accused Yeda Yakub dies in Karachi of heart attack http://t.co/mfKqyxd8XG #Mumbai,1 +659,@etribune US Drone attack kills 4-suspected militants in North Waziristan @AceBreakingNews https://t.co/jB038rdFAK,1 +660,Suspect in latest theater attack had psychological issues http://t.co/3huhZxliiG,1 +661,Militants attack police post in Udhampur; 2 SPOs injured | LiveMint http://t.co/Rptouz2iJs | http://t.co/69mLhfefhr #AllTheNews,1 +662,BREAKING: Obama Officials GAVE Muslim Terrorist the Weapon Used in Texas Attack http://t.co/qi8QDw5dFG,1 +664,Delhi Government to Provide Free Treatment to Acid Attack Victims in Private Hospitals http://t.co/H6PM1W7elL,1 +665,New post from @darkreading http://t.co/8eIJDXApnp New SMB Relay Attack Steals User Credentials Over Internet,1 +667,Israeli forces raid home of alleged car attack suspect http://t.co/3GVUS8NPpy #palestine,1 +669,Just had a heart attack because I thought my goat was dead. ???? don't worry Rocket is okay. ??,0 +671,I'm not gonna lie I'm kinda ready to attack my Senior year ??????????,0 +672,"'Left hand side of a diamond is a graveyard shift have to attack and defend' +The right handside no have to do that too you fucking idiot?",0 +674,#volleyball Attack II Volleyball Training Machine - Sets Simulation - http://t.co/dCDeCFv934 http://t.co/dWBC1dUvdk,1 +675,Notley's tactful yet very direct response to Harper's attack on Alberta's gov't. Hell YEAH Premier! http://t.co/rzSUlzMOkX #ableg #cdnpoli,1 +677,Police: Assailant in latest US movie theatre attack was homeless had psychological issues http://t.co/zdCvlYq6qK,1 +679,@CaIxxum5SOS thanks for the damn heart attack,0 +680,Suspect in latest US theatre attack had psychological issues http://t.co/OnPnBx0ZEx http://t.co/uM5IcN5Et2,1 +681,India shud not give any evidence 2 pak.They will share with terrorists & use for next attack.Share with oth contries https://t.co/qioPbTIUVu,1 +682,illegal alien released by Obama/DHS 4 times Charged With Rape & Murder of Santa Maria CA Woman Had Prior Offenses http://t.co/MqP4hF9GpO,1 +683,Strongly condemn attack on ARY news team in Karachi. A cowardly act against those simply trying to do their job!,1 +684,Nashville Theater Attack: Will Gun Grabbers Now Demand ___Hatchet Control?___ http://t.co/OyoGII97yH,1 +685,The fact that the atomic bombs were called 'Little Boy' and 'Fat man' says a lot about the mentality that went into the attack.,1 +686,@blazerfan not everyone can see ignoranceshe is Latinoand that is All she can ever benothing morebut an attack dog 4 a hate group GOP,0 +687,Heart disease prevention: What about secondhand smoke? http://t.co/YdgMGBrYL2,0 +688,A Dayton-area org tells me it was hit by a cyber attack: http://t.co/7LhKJz0IVO,1 +689,Attack on Titan game on PS Vita yay! Can't wait for 2016,0 +690,[infowars] Nashville Theater Attack: Will Gun Grabbers Now Demand ___Hatchet Control?___ http://t.co/n3yJb8TcPm #nwo,1 +691,anxiety attack ??,0 +697,My dog attacked me for my food #pugprobs,0 +699,Cop injured in gunfight as militants attack Udhampur police post: Suspected militants attacked a police post i... http://t.co/2h0dPMv2Ef,1 +700,@envw98 @NickCoCoFree @JulieDiCaro @jdabe80 I asked how did he feel attacked by julie. I asked if he was frail. That is all.,0 +702,@messeymetoo I feel attacked,0 +704,I'm feeling so attacked https://t.co/CvkQiGr1AZ,0 +705,Once again black men didn't make it that way. White men did so why are black men getting attacked https://t.co/chkP0GfyNJ,0 +706,I cant believe a fucking cis female is going to somehow claim to be offended over a transgendered female who's been attacked by media,0 +709,Israeli helicopters that attacked civilians in Gaza just completed exercises in Greece.,1 +710,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/5qYcZyWKgG,1 +712,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/U0kEOe5fMt,1 +713,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/f5MiuhqaBy,1 +714,#PT: The unit attacked by IS was responsible for targeting Muslim Scholars and imprisoning the youth. http://t.co/f4LhfmEhzh,1 +716,Telnet attacked from 124.13.172.40 (STREAMYX-HOME-SOUTHERN MY),1 +717,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/wGWiQmICL1,1 +719,#TBT Remember that time Patrick Kane attacked a cab driver over .20,0 +720,im feeling attacked http://t.co/91jvYCxXVi,0 +721,IK Only Troll His Pol Rivals Never Literally Abused Them Or Attacked Their Families. While All Of Them Literally Abuse IK. Loosers,0 +724,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/NhxSe3RTHX,1 +725,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/LHBZHWq4B9,1 +726,@envw98 @NickCoCoFree @JulieDiCaro @jdabe80 Why am I the worst person? Questioning how julie attacked him? Do you guys have no empathy?,0 +727,Kelly Osbourne attacked for racist Donald Trump remark about Latinos on The View http://t.co/7nAgdSAdWP,1 +728,@eunice_njoki aiii she needs to chill and answer calmly its not like she's being attacked,0 +729,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/mXZ7yX8ld1,1 +730,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/e4YDbM4Dx6,1 +731,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/T1aa5Ov7Eg,1 +732,I attacked Robot-lvl 19 and I've earned a total of 6615434 free satoshis! http://t.co/DMLJ1aGoTw #robotcoingame #Bitcoin #FreeBitcoin,0 +734,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/EMDJNNltP0,1 +735,@christinalavv @lindsay_wynn3 I just saw these tweets and I feel really attacked,0 +736,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/a6wmbnR51S,1 +737,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/ETg0prBP4G,1 +738,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/yUBKHf9iyh,1 +739,TV program I saw said US air plane flew to uranium mine in Fukushima and attacked by machine gun when student army were digging it.,1 +740,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/e4wK8Uri8A,1 +744,@MageAvexis < things. And what if we get attacked?,0 +745,Christian Attacked by Muslims at the Temple Mount after Waving Israeli Flag via Pamela Geller - ... http://t.co/1pDJoq4Jc1,1 +746,#WeLoveLA #NHLDucks Avalanche Defense: How They Match vs St. Louis Blues http://t.co/9v1RVCOMH2 #SportsRoadhouse,0 +748,I liked a @YouTube video http://t.co/TNXQuOr1wb Kalle Mattson - 'Avalanche' (Official Video),0 +751,we'll crash down like an avalanche,0 +752,#Colorado #Avalanche Men's Official Colorado Avalanche Reebok T-Shirt XL Blue 100% Cotton http://t.co/ZNSvsTGwx3 #NHL #Hockey,0 +753,2 TIX 10/3 Frozen Fury XVII: Los Angeles Kings v Avalanche 103 Row:AA MGM Grand http://t.co/kBtZZZG2Tp,0 +754,I BET YOU DIDNT KNOW I KICK BOX TOO! https://t.co/rBrw8pWiPJ,0 +755,A little piece I wrote for the Avalanche Designs blog! I'd appreciate it greatly if you checked it out :-) https://t.co/rfvjh58eF2,0 +758,PATRICK ROY 1998-99 UPPER DECK SPX #171 FINITE 1620 MADE COLORADO AVALANCHE MINT http://t.co/uHfM1r3Tq5 http://t.co/QulgaKebHB,0 +759,Musician Kalle Mattson Recreates 34 Classic Album Covers in Clever Music Video for 'Avalanche' http://t.co/yDJpOpH1DW,0 +761,Beautiful Sweet Avalanche Faith and Akito roses with lots of frothy gyp. http://t.co/RaqUpzFkJY #weddinghour http://t.co/quNxocXCgA,0 +762,1-6 TIX Calgary Flames vs COL Avalanche Preseason 9/29 Scotiabank Saddledome http://t.co/5G8qA6mPxm,0 +763,Secrets up avalanche: catechize inner self for the confidential communication as respects creating worth in len...,0 +767,the fall of leaves from a poplar is as fully ordained as the tumbling of an avalanche - Spurgeon,0 +770,I saw two great punk bands making original music last week. Check em out here!! @GHOSTOFTHEAV @MontroseBand https://t.co/WdvxjsQwic,0 +773,GREAT PERFORMANCE CHIP FUEL/GAS SAVER CHEVY TAHOE/BLAZER/AVALANCHE/S-10 http://t.co/iCrZi5TqC5 http://t.co/ONxhKfHn2a,0 +774,Musician Kalle Mattson Recreates 34 Classic Album Covers in Clever Music Video for __÷Avalanche__ª http://t.co/VBSwhz4s2V,0 +775,driving the avalanche after having my car for a week is like driving a tank,1 +777,Free Ebay Sniping RT? http://t.co/RqIPGQslT6 Chevrolet : Avalanche Ltz Lifted 4x4 Truck ?Please Favorite & Share,0 +779,Chiasson Sens can't come to deal #ColoradoAvalanche #Avalanche http://t.co/2bk7laGMa9 http://t.co/bkDGCfsuiQ,1 +781,Paul Rudd Emile Hirsch David Gordon Green 'Prince Avalanche' Q&A | Filmmakers at Google http://t.co/e4QonKzndZ #entretenimento #Video,0 +782,Great one time deal on all Avalanche music and with purchase get a Neal Rigga shirt http://t.co/4VIRXkgMpC,0 +783,.@bigperm28 was drafted by the @Avalanche in 2005 (rd. 4 #124) overall. Played last season in @UtahGrizz. http://t.co/gPGTAfMKt0,0 +784,I HAVE GOT MORE VIDEOS THAN YOU RAPPERS GOT SONGS! http://t.co/pBLvPM6C27,0 +786,@funkflex yo flex im here https://t.co/2AZxdLCXgA,0 +787,You are the avalanche. One world away. My make believing. While I'm wide awake.,0 +788,The possible new jerseys for the Avalanche next year. ???? http://t.co/nruzhR5XQu,0 +790,What a feat! Watch the #BTS of @kallemattson's incredible music video for #Avalanche: https://t.co/3W6seA9tuv ????,0 +791,Avalanche City - Sunset http://t.co/48h3tLvLXr #nowplay #listen #radio,1 +794,Chevrolet : Avalanche LT 2011 lt used 5.3 l v 8 16 v automatic 4 wd pickup truck premium b___ http://t.co/OBkY8Pc89H http://t.co/dXIRnTdSrd,1 +795,No snowflake in an avalanche ever feels responsible.,0 +796,STAR WARS POWER OF THE JEDI COLLECTION 1 BATTLE DROID HASBRO - Full read by eBay http://t.co/xFguklrlTf http://t.co/FeGu8hWMc4,1 +797,CIVIL WAR GENERAL BATTLE BULL RUN HERO COLONEL 2nd NEW HAMPSHIRE LETTER SIGNED ! http://t.co/Ot0tFFpBYB http://t.co/zaRBwep9LD,1 +798,Dragon Ball Z: Battle Of Gods (2014) - Rotten Tomatoes http://t.co/jDDNhmrmMJ via @RottenTomatoes,0 +799,I added a video to a @YouTube playlist http://t.co/wedWyn9kfS World Of Tanks - Battle Assistant Mod Bat Chat Arti kaboom,0 +800,"YA BOY CLIP VS 4KUS FULL BATTLE + +@15MofeRadio @Heavybag201 @battle_dom @QOTRING @BattleRapChris @Hughes1128 + +https://t.co/7SPyDy1csc",0 +801,Indeed!! I am fully aware of that battle! I support you in that fight!! https://t.co/MctJnZX4H8,1 +802,It's baaaack! Petersen's Bowhunting Battle of the Bows. Make sure you head on over and cast your vote for your... http://t.co/FJ73gDvg2n,0 +803,"#Tb #throwback ?? + +??~ You want a battle? Here's a War! ~ ?? https://t.co/B0ZJWgmaIW",0 +804,Kelby Tomlinson mild-mannered 2nd baseman for a great metropolitan team fights a never-ending battle for hits RBI and the #SFGiants way.,0 +805,Black Eye 9: A space battle occurred at Star M27329 involving 1 fleets totaling 1236 ships with 7 destroyed,1 +806,What really happened in the Taken King Story Trailer. A space battle that ripped a hole in Saturn. @EyTay @neur0sis http://t.co/ZYRZX6dfki,0 +807,THESE AFTER BATTLE ANIMATIONS ARE SO FUCKING MUCH,0 +808,See what happens with NO Battle of the Block @CBSBigBrother!?! ???? finally,0 +809,#DU19 who gon get in this rap battle with me,0 +810,Do Your Own Thing: The Battle of Internal vs External Motivation: http://t.co/w9P3hAuHEi,0 +812,Check out this item I just got! [Phantasmal Cummerbund] http://t.co/qrHJEI7gRq #Warcraft,0 +814,A young German stormtrooper engaged in the Battle of the Somme 1916. [800 __ 582 ] http://t.co/yxvMifLvc4,1 +815,I liked a @YouTube video http://t.co/9Vw0uQQi1y Marvel VS DC (Avengers Battle!),0 +817,@UtahCanary sigh daily battle.,0 +818,@SexyDragonMagic I've come to the realization that I just don't have the attention span for mass battle games. Both painting and playing.,0 +819,@DetroitPls interested to see who will win this battle,0 +820,Battle of the GOATS https://t.co/ofECs6tcvC,0 +822,STAR WARS POWER OF THE JEDI COLLECTION 1 BATTLE DROID HASBRO - Full read by eBay http://t.co/yI30ZgiZsW http://t.co/2jGVhw7YZs,0 +823,Black Eye 9: A space battle occurred at Star O784 involving 2 fleets totaling 4103 ships with 50 destroyed,0 +826,"#LonePine remembered around Australia as 'descendants' grow via @666canberra #Gallipoli #WW1 +http://t.co/T4fvVnRPc5 http://t.co/0zZnbVFUVO",0 +828,how did I miss that Gary Busey's son plays DIXIE on his electronic green fiddle during the post-battle celebration sequence,0 +829,News: FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps http://t.co/xteZGjfs8A,1 +832,FedEx no longer will transport bioterror germs http://t.co/qfjjDxes7G via @USATODAY,0 +833,USA TODAY: .@FedEx will no longer to transport bioterror pathogens after ... - http://t.co/iaDlSlqdpd #NewsInTweets http://t.co/o8y1suL4Ow,0 +834,FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps http://t.co/HQsU8LWltH via @usatoday,1 +835,Jacksonville Busines FedEx stops shipping potential bioterror pathogens http://t.co/sHzsYmaUSi,1 +836,FedEx will no longer transport bioterror pathogens in wake of anthrax lab mishaps http://t.co/lHpgxc4b8J,0 +837,.@APHL responds: FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps http://t.co/cGdj3dRso9,1 +838,#News FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps (say what?): å_FedEx no... http://t.co/K0Y7xFxmXA #TCOT,1 +840,FedEx to stop transporting bioterror germs after lab mishaps: FedEx has stopped transporting certain research ... http://t.co/y3dM9uLqxG,1 +841,FedEx no longer will ship potential bioterror pathogens http://t.co/CHORr2XOVp via @AtlBizChron,0 +842,#frontpage: #Bioterror lab faced secret sanctions. #RickPerry doesn't make the cut for @FoxNews #GOPDebate http://t.co/fZujg7sXJg @USATODAY,0 +843,FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps http://t.co/qZQc8WWwcN via @usatoday,0 +845,House Energy &amp; Commerce subcommittee to hold 7/28 hearing of CDC oversight of bioterror labs Army anthrax mishaps. htt___,1 +846,FedEx not willing to transport research specimens of potential bioterror pathogens in wake of anthrax lab mishaps http://t.co/cM8UnI1mRG,1 +848,#world FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps http://t.co/wvExJjRG6E,1 +849,FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps http://t.co/P96rgBbaYL #news #phone #apple #mobile,1 +851,RT alisonannyoung: EXCLUSIVE: FedEx no longer to transport research specimens of bioterror pathogens in wake of anthrax lab mishaps ___,0 +852,Hmm...this could be problem for some researchers: FedEx no longer to transport select agents http://t.co/9vIibxgjAV via @usatoday,1 +853,#world FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps http://t.co/5zDbTktwW7,0 +856,FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps http://t.co/pWAMG8oZj4,1 +857,FedEx no longer will ship potential bioterror pathogens - FedEx Corp. (NYSE: FDX) will no longer deliver packages ... http://t.co/2kdq56xTWs,1 +859,FedEx no longer will ship potential bioterror pathogens - Atlanta Business Chronicle http://t.co/YLLQJljiIQ,0 +860,"Thank you @FedEx for no longer shipping live microbes for the Department of Defense + +http://t.co/zAHNEwJrI8",0 +863,#NowPlaying at #orchardalley in #LES of #nyc 'bioterror- manufactured fear and state repression' @abcnorio #gardens http://t.co/Ba2rRXUgsG,1 +864,FedEx no longer will transport bioterror germs http://t.co/SHrhkfj1bC via @usatoday,0 +866,[JAX Biz Journal] FedEx stops shipping potential bioterror pathogens http://t.co/R33nCvjovC,1 +867,USATODAY: On today's #frontpage: #Bioterror lab faced secret sanctions. #RickPerry doesn't make the cut for FoxNew___ http://t.co/5uKOHk7SoB,1 +868,#BreakingNews http://t.co/gAN14PW9TG FedEx no longer willing to transport research specimens of potential bioter___ http://t.co/5n4hUsewLy,0 +870,FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps http://t.co/c0p3SEsqWm via @usatoday,1 +871,FedEx stops shipping potential bioterror pathogens http://t.co/tkeOAeDQKq #trucking,1 +872,FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps http://t.co/MqbYrAvK6h,1 +873,#FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps http://t.co/S4SiCMYRmH,1 +874,FedEx no longer shipping bioterror germs - WXIA-TV | @scoopit http://t.co/ZQqJrQsbJm,0 +875,FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps http://t.co/hrqCJdovJZ,0 +876,USATODAY: On today's #frontpage: #Bioterror lab faced secret sanctions. #RickPerry doesn't make the cut for FoxNew___ http://t.co/xFHh2XF9Ga,1 +877,FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps http://t.co/4M5UHeyfDo via @USATODAY,1 +878,Biolab safety concerns grow: FedEx stops transporting certain specimens. Research facilities 'dumbfounded' by action. http://t.co/RUjV4VPnBV,0 +881,To fight bioterrorism sir.,1 +882,I liked a @YouTube video http://t.co/XO2ZbPBJB3 FEMA REGION III TARGETED for BIOTERRORISM !!! NASA - JAPAN ROCKET LAUNCH with LITHIUM,1 +883,#anthrax #bioterrorism CDC To Carry Out Extensive Review of Lab Safety And Pathogen Handling Procedures http://t.co/bCLqpWFDOd,1 +884,Firepower in the lab [electronic resource] : automation in the fight against infectious diseases and bioterrorism /___ http://t.co/KvpbybglSR,0 +885,@CAgov If 90BLKs&8WHTs colluded 2 take WHT F @USAgov AUTH Hostage&2 make her look BLK w/Bioterrorism&use her lgl/org IDis ID still hers?@VP,1 +886,@DarrellIssa Does that 'great Iran deal' cover bioterrorism? You got cut off terrible of them. Keep up the good work.,1 +888,A Tale of Two Pox - Body Horrors http://t.co/W2IXT1k0AB #virus #infectiousdiseases #bioterrorism,1 +890,Bioterrorism public health superbug biolabs epidemics biosurveillance outbreaks | Homeland Security News Wire http://t.co/cvhYGwcBZv,1 +891,"Creation of AI +Climate change +Bioterrorism +Mass automation of workforce +Contact with other life +Wealth inequality + +Yea we've got it easy",0 +892,In Lies We Trust #dvd CIA Hollywood and Bioterrorism Len Horowitz Vaccines Nwo http://t.co/6PAGJqfbzK http://t.co/qzizElxbyr,0 +893,@O_Magazine satan's daughter shadow warrior in 50ft women aka transgender mode ps nyc is about to fold extra extra center of bioterrorism,1 +894,@DrRichardBesser Yes. I would think not. But since college 88-92 it's been difficult not to think of bioterrorism esp. bc 'dispersed.',1 +895,Volunteers needed to participate in Emergency Preparedness drill simulating a bioterrorism disaster: http://t.co/NWV2RvGHf3 @HVnewsnetwork,1 +896,@MeyerBjoern @thelonevirologi @MackayIM of a major American newspaper running a series on all the alleged bioterrorism research going on 2/n,1 +898,To fight bioterrorism sir.,0 +899,CDC has a pretty cool list of all bioterrorism agents :3,1 +902,Government Experts Concerned About Possible Bioterrorism Using GM Organisms: Scientists are concerned that the... http://t.co/SuMe5prO0F,1 +903,Does This Prepare Us? HHS Selects 9 Regional Special #Pathogen Treatment Centers #Bioterrorism #Infectious #Ebola http://t.co/Qmo1TxxDkj,0 +905,@StationCDRKelly Any Support Sys 4 @USAgov AUTH taken Hostage by BLK US clergyforced 2 exist younger&grossly disfigured by BIOTERRORISM?@AP,1 +906,70 won 70...& some think possibility of my full transformation is impossible. I don't quite like medical mysteries. BIOTERRORISM sucks.,0 +907,To fight bioterrorism sir.,1 +911,How about a book describing the future of therapies technologies sport sexuality bioterrorism and diagnosis? #digitalhealth #hcsm,0 +912,@APhiABeta1907 w/ugliness due 2 your 'ugly'@AMESocialAction Frat's BIOTERRORISMI'm she who's @FBI ID U $tolewant'g another in my Home.@ABC,1 +915,@HowardU If 90BLKs&8WHTs colluded 2 take WHT F @USAgov AUTH Hostage&2 make her look BLK w/Bioterrorism&use her lgl/org IDis ID still hers?,1 +916,To fight bioterrorism sir.,0 +919,@cspanwj If 90BLKs&8WHTs colluded 2 take WHT F @USAgov AUTH Hostage&2 make her look BLK w/Bioterrorism&use her lgl/org IDis ID still hers?,1 +921,@ONU_France 74/75 Bioterrorism on '@Rockefeller_Chi/@RockefellerUniv'Heiress 2 evade lgl efforts 2 prosecute BLKs 4 @HarvardU kidnap'g.@AFP,0 +923,The #IranDeal only covers nuclear activity. What are they doing about Bioterrorism? Iran has broken at least 27 other agreements.,1 +926,The Threat | Anthrax | CDC http://t.co/q6oxzq45VE via @CDCgov,1 +928,#bioterrorism Authorities allay #glanders fears ahead of Rio Olympic equestrian test event http://t.co/UotPNSQpz5 via @HorsetalkNZ,1 +929,@Kaotix_Blaze craving u,0 +930,I've been by the pool all day #raisinfingers,0 +931,Do you know anyone looking to move to Hammond OR? Share this listing! http://t.co/3xn1soh4Bb,0 +932,Life is amazin same time its crazy niggas dey wanna blaze me hate it because i made it all it took was dedication n some motivation,0 +934,Blaze is my bro http://t.co/UdKeSJ01mL,0 +935,@Shayoly yes I love it,0 +936,See a virtual tour of one of our listings on 547 Fir St Cannon Beach OR listed by Dorrie Caruana. http://t.co/nF46PAYTvw,0 +938,Pendleton media office said only fire on base right now is the Horno blaze.,1 +939,Welcome @djryanwolf @djcoreygrand @djknyce @djoneplustwo @OfficialCoreDJs #Family #Cleveland #StandUp @IAMTONYNEAL http://t.co/P6GqmCTgLj,0 +940,Love living on my own. I can blaze inside my apt or on the balcony,1 +941,@bellalinn alrighty Hit me up and we'll blaze!!,0 +942,i blaze jays fuck the dutch slave trade.,0 +944,My hair is poverty at the moment need to get a fade before the weekend gets here,0 +945,Property losses from California wildfire nearly double as week-old blaze rages http://t.co/E0UUsnpsq5,1 +946,#breaking Firefighters battling blaze at east Cary condo building http://t.co/mIM8hH2ce6,1 +947,niggas love hating.,0 +948,@audacityjamesta Don't be like that babes <3 you'll have a ball of fun with me at LAN! :),0 +951,I liked a @YouTube video from @iamrrsb http://t.co/PdEHd1tCpk Minecraft Skywars - O BLAZE QUE USA HACK E FLECHADAS SINISTRAS!,0 +952,What Dems do. Blaze covered months ago.Chicago police detained thousands of black Americans at interrogation facility http://t.co/UWItVBsbnC,0 +953,Yo I got bars and I'm not even a rapper,0 +955,UGH Y DID BLAZE PUT THE CALORIES BY THEIR PIZZAS? OK COOL #thisispublichealth,0 +956,pic of me and blaze in a fort when we were kids i look like a jackass stuffin my face like that ?????? http://t.co/aE9cPIexAK,0 +957,looks like a year of writing and computers is ahead. http://t.co/CyXbrZXWq4,0 +958,@Beautiful_Juic1 just letting you know,0 +959,@BabySweet420 I'm mad 420 in your name & you don't blaze.,0 +960,@GuiltyGearXXACP yeah I know but blaze blue dont have a twitter lol I drew this a few weeks ago http://t.co/sk3l74FLzZ,0 +961,#socialmedia news - New Facebook Page Features Seek to Help Personalize the Customer Experience http://t.co/nbizaTlsmV,0 +962,@DJJOHNBLazE shout out blaze the hottest DJ in the Sothwest,0 +963,I liked a @YouTube video http://t.co/N95IGskd3p Minecraft: Episode 2 'Blaze Farm Beginnings!',0 +964,Property losses from #California wildfire nearly double as week-old blaze rages: The fire___ http://t.co/MsdizftZ2g,1 +965,@ChristyCroley Not in the works yet. Did you see the new Vela Short in Blaze? http://t.co/Q8rEoEVluE,0 +968,@UABStephenLong @courtlizcamp Total tweet fail! You are so beautiful inside and out Blaze On!,0 +971,The mixtape is coming i promise. We goin in right now http://t.co/uUNGRqoUgn,0 +972,@_itzSteven @xdojjjj @whopper_jr_760 huh?? me you and leo started that last year and ever since people blaze it in the back??,0 +974,"3 Bedrooms 1 Baths for sale in 29 Palms CA. (http://t.co/QMS8RRESsd) +(YouTube Video:... http://t.co/zLa30jCsSQ",0 +976,?? Yes I do have 2 guns ?? ??,0 +977,@a__cee DAEM GIRL SMOOTH ASF c: ?,0 +978,https://t.co/WKv8VqVkT6 #ArtisteOfTheWeekFact say #Conversations by #coast2coastdjs agree @Crystal_Blaz 's #Jiwonle is a #HipHop #ClubBanger,0 +979,Bright & BLAZING Fireman Birthday Party http://t.co/9rFo9GY3nE #Weddings,0 +981,REAL ViBEZ RADIO - BLAZING THE BEST VIBEZ!!! http://t.co/EMvOhm9m6j #nowplaying #listenlive,0 +982,Montgomery come for the blazing hot weather...stay for the STDs. Yet another rejected city slogan.,1 +983,"Come and join us Tomorrow! +August 7 2015 at Transcend:Blazing the Trail to the Diversified World of Marketing... http://t.co/NR1I8Qnao1",0 +984,Morgan Silver Dollar 1880 S Gem BU DMPL Cameo Rev Blazing MS+++++ High grade! - Full read ___ http://t.co/IU9baFDXeY http://t.co/AphqU5SvET,0 +985,Morgan Silver Dollar 1880 S Gem BU DMPL Cameo Rev Blazing MS+++++ High grade! - Full read ___ http://t.co/m96KbQwiOr http://t.co/wrJR846fKS,0 +986,This bowl got me thinking... Damn I've been blazing for so damn long,0 +987,@DanRyckert @drewscanlon He's blazing through this game with the best stealth skills yet. Nothing beats the silenced M4.,0 +990,Bit pacquiao vs marquez 3 unfilled blazing swarm online: DuRvOd http://t.co/6VJA8R4YXA,0 +991,Turn on your radios #stoponesounds is live on your #airwaves http://t.co/g7S34Sw2aM & 107.9 fm @StickyNYC @95roots blazing all your hits,0 +992,@BaseballQuotes1 I have a 32 inch dynasty,0 +993,"I'm the only weapons master here! Let's go in guns blazing! +#Hinatobot",0 +994,@Blazing_Ben @PattyDs50 @gwfrazee @JoshuaAssaraf Not really. Sadly I have come to expect that from Obama.,0 +996,SHOUOUT TO @kasad1lla CAUSE HER VOCALS ARE BLAZING HOT LIKE THE WEATHER SHES IN,0 +997,"S3XLEAK!!! +Ph0tos of 19yrs old Ash@wo lady in Festac town from Delta exp0sed on BBM 5 leaked pictures... http://t.co/ixREhM05yq",0 +998,Oh my heart racing And my temperature is blazing through the roof #VideoVeranoMTV Fifth Harmony,0 +999,@omgbethersss @BethanyMota haha love this??,0 +1001,@dmac1043 Colorado is a Spanish word ([Latin origin] meaning 'reddish' or 'colored') all you dummies are pronouncing it wrong!!!,0 +1002,Why Some Traffic Is Freezing Cold And Some Blazing Hot ___ And How To Heat Up Some Of Your Traffic http://t.co/C8b6DdiQIg,0 +1004,I'm crazy enough to run in 95 degree mid-day heat under the blazing sun in a place where I'm not___ https://t.co/OSUoIVNiGO,0 +1005,I'm blazing rn and there's nothing you can do to stop me,0 +1006,Still blazing ????,0 +1008,@bekah__w thanks! I sweat bullets every time I get in with this blazing sun beating down on me.,0 +1009,@ACOUSTICMALOLEY no he was blazing it,0 +1010,@OfficialTJonez Your 'Lost For Words' made me a new fan of yours fam. Crazy skills beyond blessed! Keep blazing dude made love and respect!,0 +1015,"**Let - Me - Be - Your - Hot - Blazing - Fantasy** +#escorts #gfe #DUBAI http://t.co/N6AhgfMUDt",0 +1017,@__srajapakse__ Why thank you there missy ?? thought it suited the blazing hot summertime ?? yee-haw! ??,0 +1018,@asukager magical bag of blazing,0 +1020,The Blazing Elwoods @BlazingElwoods - Don't Bother Me (Doug's Song) -Tune http://t.co/QYzpB1gKmR,0 +1021,Morgan Silver Dollar 1921 P CH Gem Bu PL Blazing MS++++++ Satin Rare Proof Like! - Full re___ http://t.co/99MbyFl3Id http://t.co/4ddMTguZzS,0 +1022,Blazing Hot! Etisalat Free MB For Complete 12 Months: Etisalat Is Giving out 100MB on TECNO Q1 here is the Ime... http://t.co/AVzsYIe1nT,0 +1023,@FunkyLilShack @mariaf30 I want a full on bitch slapping guns blazing cake throwing Charles showdown!! Now THAT will be worth the wait ????,0 +1024,Follow @EdWelchMusic and check out his Hit Single 'Unpacked' Man its BLAZING!!!,0 +1026,I still don't know how 8 hours of physical activity in the blazing sun isn't a sport.,0 +1029,Ways so archetype a bleeding well-grounded readiness: FpOJ http://t.co/WXbrArc7p3,0 +1031,Threw a chicken nugget at my sisters lip and now it's bleeding??,0 +1032,you could slit my throat and I'd apologize for bleeding on you,0 +1035,Joe Landolina: This gel can make you stop bleeding instantly http://t.co/0BtnIwAgt1 #arizona #realestate http://t.co/hHZY3oqeLa,0 +1036,now my nose is bleeding. the last one was like 10 years ago,0 +1037,I have been bleeding into this typewriter all day but so far all I've written is a bunch of gunk.,0 +1038,@JaydenNotJared I can't help it. Hope you're ok. Text me if you need to talk. Sending hugs your way. PS no bleeding to death allowed,0 +1040,you can stab me in the back but I promise you'll be the one bleeding,1 +1041,Apparently if you're bleeding people look at you weird lol well it's fine keep walking,0 +1042,Eating takis then rubbing my eyes with my hands now my eyes are bleeding tears,0 +1043,@DarrylB1979 yea heard about that..not coming out until 2017 and 2019 ?????? Vampiro is bleeding,0 +1044,@CoreyAshe Did that look broken or bleeding?,0 +1045,i hit my foot now my toe is bleeding ??,0 +1048,@King_Naruto_ As long as I see Madara bleeding I'm good ??,0 +1049,Keep thinking about it until I stepped on a broken glass pun tak sedar and I don't feel the pain also it's bleeding. Shit,0 +1050,My ears are bleeding https://t.co/k5KnNwugwT,0 +1051,I waited 2.5 hours to get a cab my feet are bleeding,1 +1052,It's not a cute dinner date Til cams nose starts bleeding,0 +1054,@Uptown_Jorge head up like yo nose bleeding,0 +1057,"I've been bleeding in your silence +I feel safer in your violence .",0 +1058,@Jannet2208 I fell off someone's back and hit my head on concrete /: I was bleeding n shit,0 +1061,@KatRamsland Yes I'm a bleeding heart liberal.,1 +1062,Deadpool is already one of my favourite marvel characters and all I know is he wears a red suit so the bad guys can't tell if he's bleeding,0 +1065,@beckyfeigin I defs will when it stops bleeding!,1 +1066,@SoDamnTrue we know who u are you're a bleeding heart wannabe pickup artist,0 +1067,@chaosmagician97 awesome!! I saw he was bleeding pretty bad,0 +1069,@burberryant bleeding on the brain don't know the cause,0 +1070,lets see how good you are at soccer when you're bleeding out yo face,0 +1071,@tammy_w1997 @ElijahMallari bleeding wild things running around the apartment while hes in work at the bar ????????,0 +1072,@ColoicCarnality You were bleeding my instincts kicked in. *She looks away and scratches the back of her head*,0 +1073,my ears are bleeding i hate stefano,0 +1074,My ear started bleeding again...,0 +1075,@Benjm1 @TourofUtah @B1Grego saw that pileup on TV keep racing even bleeding,1 +1077,A rave wedding ? Am I seeing this my eyes are bleeding,0 +1078,"they say bad things happen for a reason +but no wise words gonna stop te bleeding",0 +1079,@DamnAarielle yo timeline blew up so damn fast,0 +1080,@mfalcon21 go look. Just blew it up w atomic bomb.,0 +1081,I blew up #oomf instagrams cause she's cute and she's an active follower,0 +1082,@b24fowler I see that! Crazy how this line blew up.,0 +1083,My Instagram just blew up apparently I was featured on I am jazz tonight. How cool is that love her,0 +1084,i'd still be team usagi even if she blew up the entire solar system by some airhead misstep,0 +1085,@BenKin97 @Mili_5499 remember when u were up like 4-0 and blew it in one game? U probs don't because it was before the kings won the cup,1 +1088,"i hate people who tweet 'receipts' but KNOW its wrong + +but they wont take it down bc it 'blew up' +literally gtfo you're that desperate",0 +1090,I think I just blew up @HopeInHearts notifications. Go check her out she's so encouraging to me ???? love her ??,0 +1091,Hw18 going 90-100. Dude was keeping up with me. Took the same exit. Pulled to the side and told me he blew his motor. Lolol #2fast2furious,0 +1093,The universe might not actually exist scientists say http://t.co/DEfJ7XeKgX 'The #SUN blew up and the #Earth began'...,0 +1094,Max blew tf up ! ?????? shots fired ???? #CatfishMTV,0 +1098,Rick and Morty - They Blew Up : http://t.co/UQKX5VbiuM,0 +1099,@CodyThompson25 ty just blew up the motor went up in flames he got out ok,0 +1102,Just realized my dude @_OnlyFTF was on that 'What Are Those' way before it blew up @ the tusky ?? game @robsimss @CantMissKid,0 +1103,This night just blew up rq,0 +1104,I blew up snapchat for no reason ??,0 +1105,Queens Gambit went well until Anakin blew up the droid control ship. Oh well still fun! Can't Stop is next! #WBC2015,0 +1106,@iphooey @TIME Ironically Michele Bachmann brought this up in '11 w/Ron Paul & everyone blew her off and called hoax. She was finally right,0 +1109,Catfish retweeted me & my notifications blew up ??????.,0 +1110,Some guy whistled at me in the parking lot & it did not help that the wind blew my skirt up getting in the car ??,0 +1112,Blew up those mentions,0 +1113,The 1st time someone blew up my phone 30 times they would be blocked. Believe it. #Catfish,0 +1114,@ImAwesome7986 like literally blew up,0 +1117,FREYAS VIDEO BLEW UP EVERYWHERE,0 +1118,@YahooSchwab easy way to look good after the Ray Rice fiasco...that blew up,0 +1119,Lmao that light skin guy blew up on Twitter by talking about how ugly he was as a kid..,0 +1120,"Because you watched: Honey I Blew up the Economy + +We Recommend : The Conservative Shoppe of Horrors + +#HarperANetflixShow #elxn42 #stopharper",0 +1122,Did anyone else see that fireball falling to earth? Look like a plane blew up.,1 +1123,Ye did the same thing to Big Sean and he still blew up,0 +1124,Zayn just blew up twitter.,0 +1126,Bitch done blew my shit up,0 +1128,@WeLoveRobDyrdek @adrian_peel not a damn clue. Tried to see how many clicks I could get before I blew up. That's how I played.,0 +1129,@Daorcey @nsit_ YOUR a great pair. Like a couple of Graywardens fighting the blight...,0 +1130,"@DaMidnighter theres actually a theory out there that the magisters arent the only reason for the blight +that dwarves (the ones from the",0 +1132,As a cycling fan I feel sorry for world athletics #doping is a blight exacerbated monetary reward. A lot of soul searching will be required,0 +1134,@kynespeace *blight,0 +1136,Poor Jack ??,0 +1138,@realhotcullen I agree but I knew we'd be going to the deep roads again because they found Blight in red lyrium. It ain't over yet >_>,0 +1139,Get a load of this welfare loving sponge....a blight on society. http://t.co/rrZbZGO48N,0 +1141,New post: Prysmian secures contract for Blight Bank wind farm http://t.co/oLG09Kb6HA,0 +1142,Cleveland Heights Shaker Heights fight blight: The House Next Door http://t.co/wYOKt0ftRw,0 +1143,@todd_calfee so @mattburgener wanted to see that info on blight u got,1 +1145,Carl Everest Rob Cobes Whitt Blight Frost Leo Snuff Godly and a few others. I will drink a beer with them. Someday.,0 +1146,THDA Kicks Off Anti-Blight Loan Effort in Memphis http://t.co/XZLRWC0PIK http://t.co/O5xlxMkoYq,0 +1149,City program to help turn blight into greenspace: The Tennessee Housing Development___ http://t.co/ZZcbBQyJ1q #news http://t.co/KKSgHsblFH,0 +1151,@Demetae12 yes i want to be the new blight leader,0 +1152,"Releases on the planing level - + +1. Constellation - Blight on Gaia - iClown's Drumstep Remix +2.iClown - Infinity... http://t.co/7LE5GQ2Psx",0 +1153,http://t.co/ETkd58Un8n - Cleveland Heights Shaker Heights fight blight: The House Next Door http://t.co/LRelVrm06w,0 +1154,'If you are going to achieve excellence in big things you develop the habit in little matters....' dont know the author,0 +1155,@Hendy_21 sure the purdies will be alive with the blight ??,0 +1156,Tracy Blight Thank you for following me!!,0 +1157,Apperception bridgework blight: XxhJeSC http://t.co/yBhBArajXp,0 +1160,If you dotish to blight your car go right ahead. Once it's not mine.,0 +1162,Sexual Revolution:Blight For Women is out! http://t.co/T8Sv2ai7sW Stories via @ACeBabes @HealthWeekly1 @AmateurNester,0 +1164,LIKE I SWEAR THE SECRET WE'LL UNCOVER IS THE OLD GODS IN A SLUMBER. I THINK THERES GONNA BE ANOTHER BLIGHT,0 +1166,@WillHillBet what is double result live on the app?,0 +1167,@parksboardfacts first off it is the #ZippoLine as no one wants to use it and the community never asked for this blight on the park #moveit,0 +1168,Look for my Policy Matters Ohio report on #CLE and Cuyahoga County blight and greening vacant lands soon! https://t.co/if62SdXVp7,0 +1170,@anellatulip and put the taint there and that all that the magisters did was to open the gates and let the blight get away from it,0 +1173,@anellatulip there is a theory that makes way too much sense that says that the dwarves may be the actual origin of the blight,0 +1174,The #Palestinian #refugee tragedy is a blight on humanity & should shame every #Israeli for living with it. https://t.co/gAAE0nO5du,1 +1175,"Article by Michael Jackman at Metro Times Detroit: +The group later downgraded the estimate to 37 square miles of... http://t.co/h31mmuduqt",0 +1177,Locksmithing-art respecting elaboration only blight locks: lPDkl,0 +1178,@jake_blight @WeAlIlKnowA you cunt,0 +1179,What if every 5000 wins in ranked play gave you a special card back.. Would be cool for the long te___ http://t.co/vq3yaB2j8N,0 +1181,Amazon Deal - wait or buy? http://t.co/0T8VqKEArI,0 +1182,New print available on http://t.co/ucy5fEA9yu! - 'Waiting Too Long' by Pamela Blizzard - http://t.co/dnwwo1YbRK,0 +1183,@Blizzard_draco GIVE ME FREE ART KAMON,0 +1185,@StevenOnTwatter @PussyxDestroyer just order a blizzard pay then put your nuts in it say they have you ball flavored. Boom free ice cream,0 +1188,Blizzard of Auz @ 9 pm CST @RadioRiffRocks / http://t.co/pjLDA9HD5v 2 hrs of Rock to make your hump day complete! http://t.co/3wNjaUaR7w,0 +1189,I really wants a rolo blizzard but mom said no so I guess no DQ tonight,0 +1190,@Ashayo @MsMiggi Hi Ashayo! I believe there will be VODs on YouTube after the presentation but there is nothing like seeing it live :),1 +1191,Stats http://t.co/U7vavyrGv9,0 +1195,#Tweet4Taiji is a dolphin worship group based on superstitions! Just take a look at their tweets!,1 +1196,@blizzard_fans Lucio!! Let's get the #overwatch hype train rolling some more!! Caution though there aren't any breaks,0 +1198,@Blizzard_Gamin ight,0 +1199,peanut butter cookie dough blizzard is ??????????????????????,0 +1201,My mic and controllers aren't working one second,0 +1202,Tomorrow's Announcement VODs http://t.co/cUbze5MIZm,0 +1203,Updated to Windows 10 now I get this error http://t.co/kHSBZKfd6O,0 +1204,New Expansion Ideas - Bard Class Holy Trinity + 1 http://t.co/EGioxBabOe,0 +1206,@DaBorsch not really that shocking :( blizzard lured their old fanbase back with WoD and disappointed us hardcore so everyones leaving again,0 +1207,the best thing at DQ is the cotton candy blizzard ??????????????????????????????????????????????????,0 +1208,Lizard Wizard in a Blizzard #LWB http://t.co/MgR809yc5a,0 +1209,@Blizzard_draco @LoneWolffur I need this.,0 +1211,@FAIRx818x @PlayOverwatch @BlizzardCS please blizzard we love you,0 +1212,A blizzard would be clutch asf ??,0 +1213,@Blizzard_draco @LoneWolffur also me please I would very much like a link,0 +1214,@BubblyCuteOne ?????????? ok ok okayyyyyy Ima act right ....bout to get this blizzard tho,1 +1215,@LoneWolffur control yourself tora,0 +1217,First Time Playing Hearthstone on PC Thoughts http://t.co/aBoLxMH1vy,0 +1218,I love the cotton candy blizzard??,0 +1219,@TCGReno just hard reset my Xbox,0 +1221,I really wanna brownie batter blizzard ??,0 +1222,I call it a little bit of your blizzard?,1 +1223,Market News: Activision Blizzard Cognizant Technology First Solar http://t.co/gkNRP0e8Qs,0 +1224,SEAN END CAREER sG Blizzard vs KNOCKOUT ... http://t.co/nyv51681uE,0 +1225,What is the biggest regret you have in hearthstone? http://t.co/vcIrn1Md8v,0 +1226,"Someone walk with me to DQ ?? +I wanna Butterfinger Blizzard so bad??",0 +1227,That horrible moment when u open up the dryer and it looks like a snowy blizzard cuz u left a piece of paper in your jeans pocket ??,0 +1228,@LoneWolffur BRUH *dies*,0 +1229,Rip ?? Blood !,0 +1230,@Chief__CG nah young blood that cook is gone I'm cut now .haha,0 +1232,Off The Wall Invincible and HIStory + Blood On The Dance Floor https://t.co/ZNTg2wndmJ,0 +1234,Broke my nail(real not fake) this morning blood and all ah it hurts any ideas how to treat it? Help me pretty please ? -_-,0 +1235,4.5 out of 5 stars by 290 reviewers for 'Dragon Blood' Boxset by Lindsay Buroker http://t.co/4yu5Sy1Cui #kindle http://t.co/mzmxMyklXv,0 +1236,Blood Group A +ve is associated with Gastric Carcinoma says text book...Another fragile gene in my body....,0 +1237,A friend is like blood they are not beside us always. But they come out when we are wounded.,0 +1238,We gone get it get it in blood,0 +1239,people with a #tattoo out there.. Are u allowed to donate blood and receive blood as well or not?,1 +1240,Doing dialyses to my grandpa and oh lord this blood makes me light headed,0 +1241,@SetZorah dad why dont you claim me that mean that not right we look the same same eyes same blood same xbox 360 SMH -.-,0 +1242,Omron HEM-712C Automatic Blood Pressure Monitor STANDARD AND LARGE BP CUFFS http://t.co/gJBAInQWN9 http://t.co/jPhgpL1c5x,0 +1243,My blood pressure is through the roof I don't need all this extra shit!!!!,0 +1244,@Chambered_Blood Yeah you are! #SpeakingFromExperience,0 +1245,Can't believe more people in their mid 20's don't have high blood pressure. Life is stressful. #DecisionsOnDecisions,1 +1249,Guys. I have an Imouto Who Isn't Actually Related to Me by Blood.,0 +1250,Bruh white people buy the ugliest shoes and they them super tight no blood going to there feet,0 +1251,another day another excellent @_dangerousbeans porridge. seriously people. blood orange in porridge is phenomenal.,0 +1252,@scotto519 happy birthday young blood,0 +1253,If it wasn't for the Blood! ????,0 +1254,No kinda in it that shii nasty blood. No pun intended,0 +1259,Ain't no hoe in my blood,1 +1262,Today was such a hastle from getting drug tested blood drown out tb shot to document filling ??,0 +1263,Man . somebody gotta stop Sbee dude too fuckin funny blood,0 +1264,Where will the winds take my gypsy blood this time? http://t.co/66YVulIZbk,0 +1267,@Lobo_paranoico Mad Men,0 +1269,Private thirsty night?SAD BLOOD ROCK'N ROLL? #??,1 +1270,#RolandoNaBeats: Ellie Goulding - My Blood | Acesse nosso site para ouvir! http://t.co/Zk69uGXMT8,0 +1271,@olrules Welcome - Read a free chapter of my new book Encounters With Jesus. It's full of hope. http://t.co/6qX7arf4AG,0 +1272,@Anthxvy runs in the blood,0 +1273,@sethalphaeus my personal favorites include paramore muse green day royal blood and 5sos,0 +1275,Shed innocent blood of their sons and daughters and the land was polluted Psalms 106:38 Help stop the sin of abortion.,0 +1276,*se pone a cantar crying lightning*,0 +1277,it wasnt a very big stab but it was a deep stab and theres like blood everwhe,1 +1278,Add these items to your everyday eating habits. Please do the research on how to take with your blood___ https://t.co/LnpsCaDaXr,0 +1279,LOOOOOOOOOOOOL who the bloody hell did that?? https://t.co/jRKGYl7Te5,0 +1280,infected bloody ear piercings are always fun??,0 +1282,aggressif is so bloody aggressive,0 +1283,I entered to #win the ENTIRE set of butterLONDON Lip Crayons via @be_ram0s. - Go enter! #bbloggers http://t.co/DsB3lDfuxU,0 +1284,@slsandpet Hey Sally sorry have you emailed me? Been AWOL bloody work ARGH! @ResignInShame,0 +1287,I'm over here listening to Bloody Jay. ???? https://t.co/CIyty0FgpR,0 +1288,@LauradeHolanda I have the Forrest version from '83 that's bloody awful as well :))) xxx,0 +1289,'A Nightmare On Elm Street' Is Getting Remade... Again - http://t.co/HvwkJQXXyT,0 +1290,I can't bloody wait!! Sony Sets a Date For Stephen King__ªs __÷The Dark Tower__ª #stephenking #thedarktower http://t.co/J9LPdRXCDE @bdisgusting,0 +1293,Gotta try to let go of so many bloody things. Smh,0 +1294,@TradCatKnight (1) Russia may have played into reason but that link is BS. Okanowa was bloody and mainline invasion looked like a bloody,1 +1296,Bloody insomnia again! Grrrr!! #Insomnia,1 +1297,@zhenghxn i tried 11 eyes akame ga kill and tokyo ghoul all damn bloody i dont dare watch????????,0 +1298,@Fantosex Now suck it up because that's all you're bloody getting out of me by means of amends.,0 +1300,You call them weekends. I call them Bloody Mary times. This summer's been full of them. My new___ https://t.co/VnNi3zzuZ6,0 +1301,Bloody hell what a day. I haven't even really done anything. Just. Tired. Of everything. Thought vaca would help but it only did so much. =/,0 +1302,Damn bloody hot,0 +1303,@MrTophyPup it's bloody sexy *drools*,0 +1304,You know how they say the side effects low & really fast? Son the product was an acne cream.. Why 1 of the side effects was bloody diarrhea?,0 +1305,Ronda Rousey would be 'close' to making Floyd Mayweather's money in 50 fights - Bloody Elbow http://t.co/IjzcYtbFfo #boxing,0 +1307,I'm awful at painting.. why did I agree to do an A3 landscape in bloody oils of all paints ??,0 +1308,"All I need in this life of sin +Is just me and my girlfriend +Down to ride till the bloody end +Just me and my girlfriend",0 +1309,@_itsmegss_ I think it is. well it's bloody barking now,0 +1312,Eh hello cover your bloody thighs your bloody cleav... ___ Eh hello! Since when do i expose my cleavage and i on... http://t.co/Kv5L4PPXfG,0 +1315,@MariaSherwood2 @JohnJCampbell Mega bloody marvellous,0 +1316,Bloody Mary in the sink. Beet juice http://t.co/LUigmHMa1i,0 +1317,@MelRises @gayler1969 @wwwbigbaldhead @jessienojoke @melissaross9847 if my Monty Python is up to date as bloody far as he wants to go.,0 +1318,Meet the bloody RS5 http://t.co/RVczMimfVx,0 +1319,Friday supposed to be a happy day but it's a bloody friday hah zzzz,0 +1320,@chxrmingprince @jones_luna I should bloody hope so *she said folding her arms sitting back in her chair*,0 +1321,Wait until i tell my college friend who reafs bloody mary too about the drama cd,0 +1324,'I came to kill Indians...for FUN': Video of smirking and remorseless Pakistani killer shows him boasting. http://t.co/FPjLwOXKlg,1 +1326,I've just watched episode S01E09 of Bloody Monday! http://t.co/vRptHvPymt #bloodymonday #tvshowtime http://t.co/NkKvknBvOz,0 +1327,"Marlon Williams > Elvis Presley > Marlon Williams > Steel Panther. + +Shuffle mode like a bloody legend.",0 +1328,Black Friday turns Bloody (would rather be shopping) http://t.co/l0pmmtZLwP #mystery,0 +1331,@ezralevant Someone told me all the soccer moms are getting this pic blown up and put on their daughter's bedroom walls. NOT!!!,0 +1332,@Papcrdoll and I s2g if my mentions get blown up over MY choice I will deactivate and leave for good. RESPECT MY CHOICES.,0 +1334,"@ManUtd @EmilymcfcHeslop + +Wow just wow! + +If any other club had blown all that money and fucked up a player like that they'd be pilloried!",0 +1335,Damn greinke got blown up in that first inning,0 +1337,Woke up so blown lol,0 +1338,aunt marge you're blown up eheks,0 +1340,@troylercraft YEAH ITS NOT WORTH IT BC HE ALREADY HAS SO MANY SPAMMERS & HIS TWITTER IS PROBABLY BLOWN UP EVERY SECOND,0 +1341,White family (supposedly representing America's GREAT values ) gets blown up in a horrible CGI nuclear strike..... LMFAOOOO!!!!!!!!!!!!,1 +1343,Man why hasn't @machinegunkelly blown up? He's still underground.,0 +1345,And my mentions are blown up for what? ?? wtf.,0 +1346,On #ThisDayInHistory in 1862 Confederate ship blown up by crew. Read More http://t.co/IW7ELSzIfZ via @History,1 +1347,That moment when ur about to win a mini uhc and than get blown up by a creeper and get kicked for flying..... So salty rn...!!!??,0 +1348,@libraryeliza he did get a @taylorswift13 'bump' of approval which is probably why he's blown up http://t.co/KOLMzBz1pZ #MusicAdvisory,0 +1349,"'If a truckload of soldiers will be blown up nobody panics but when one little lion dies everyone loses their mind' +http://t.co/wjNTaOkdHf",1 +1350,"HEY LOOK!!! Kash's Foundation Live for Today got blown up on People Magazine's website!! + +Todd Blake... http://t.co/2Fenu1SYu6",0 +1351,@KaylaK369 got it last month when I went into the EE shop. Glad it hasn't blown up yet. http://t.co/PgB2BmCFX8,0 +1352,this is the first time a tweet has blown up almost half a day later... RE https://t.co/1BUF0xM53d,0 +1353,'We__ªre blown away by this extension. Nothing we__ªve seen has as many options as this one.' https://t.co/0YzgW9ZbHR https://t.co/rHtaqjvQn2,0 +1354,@TheBoyOfMasks 'Thanks again for letting me stay here since the manor was blown up..... Anyways how are you doing buddy?',0 +1356,"LONER DIARIES. + +The patterns on the sand +May have been blown away. +The photos in twos +All choked up in flames.... http://t.co/EKfaZ6wVBz",0 +1360,Even though BSG had been sufficiently hyped up for me in all the years I somehow delayed watching it I was utterly utterly blown away.,1 +1361,Vanessa's game has officially blown up. LADIES AND GENTLEMEN...the real show is about to begin. #BB17,0 +1362,Guaranteed been bitten by some mutant mosquito my ankle has blown up. Little cunts,0 +1363,Things you CAN pick up from tozlet seat|: butt leprosy full-blown jerkface syndrome a lateral lisp & toilet rickets.,0 +1365,@KalinAndMyles @KalinWhite my ig is being blown up just with hackers I need it to stop #givebackkalinwhiteaccount,0 +1368,@luke_winkie Whoever is directing these videos needs to grab up Nicki Minaj or someone with U.S. recognition so minds can be blown.,0 +1369,@PrincessDuck last week wanted the 6th sense to get blown up so far so good. James could win but he's a huge target and will be gone soon.,0 +1371,@orbette more like BLOWN UP amirite,0 +1372,My dog__ªs just blown his kennel up ___ Bloody Yorkshire Terrorist,0 +1374,Turn on ESPN2 and get blown up,0 +1375,@HopefulBatgirl went down I was beaten and blown up. Then next thing I know Ra Al Ghul brought me back to life and I escaped and for a---,1 +1376,@thebriankrause leos ass just got metaphorically blown up again #PiperWearsThePants #charmed,0 +1378,@anarchic_teapot @BoironUSA @zeno001 Glononium 6C also helps with being blown up while bashing a bottle of nitroglycerin against a book.,0 +1379,New Ladies Shoulder Tote Handbag Women Cross Body Bag Faux Leather Fashion Purse - Full re___ http://t.co/y87Gi3BRlV http://t.co/1zbhVDCXzS,0 +1380,New Ladies Shoulder Tote Handbag Women Cross Body Bag Faux Leather Fashion Purse - Full re___ http://t.co/BLAAWHYScT http://t.co/dDR0zjXVQN,0 +1381,If you have a son or a daughter would you like to see them going to a war with Iran and come back in a body bag? Let the #Republicans know,0 +1382,@questergirl ditto but its all we had. and the way i feel if i drank vodka over ice they would have to body bag me,0 +1383,# handbags Genuine Mulberry Antony Cross Body Messenger Bag Dark Oak Soft Buffalo Leather: å£279.00End Date: W... http://t.co/FTM4RKl8mN,0 +1384,Louis Vuitton Monogram Sophie Limited Edition Clutch Cross body Bag - Full read by eBay http://t.co/JrxgoLnPqw http://t.co/Lin16KvZbn,0 +1385,Louis Vuitton Monogram Sophie Limited Edition Clutch Cross body Bag - Full read by eBay http://t.co/GGgFVO5Pb4 http://t.co/NlFr8t3xqm,0 +1388,Louis Vuitton CultSierre Monogram Shoulder Bag Cross Body Bag http://t.co/mUf5cZQjrL http://t.co/SsLT8ESMhY,0 +1389,Check out Ameribag Healthy Back Bag Shoulder Cross Body Backpack Khaki Tan Beige Nylon http://t.co/r4k7TyLofJ @eBay,0 +1390,Photo: Bath & Body Works cosmetic bag in periwinkle blue with copper piping along the top and four corners.... http://t.co/A9BNlse6QB,0 +1391,New Ladies Shoulder Tote Handbag Women Cross Body Bag Faux Leather Fashion Purse - Full re___ http://t.co/3PCNtcZoxv http://t.co/n0AkjM1e4B,0 +1393,new summer long thin body bag hip A word skirt Blue http://t.co/lvKoEMsq8m http://t.co/CjiRhHh4vj,0 +1394,å¤} New Ladies Shoulder Tote #Handbag Faux Leather Hobo Purse Cross Body Bag #Womens http://t.co/UooZXauS26 http://t.co/6MGBizjfgd RT en___,0 +1395,@TR_jdavis Bruh you wanna fight I'm down meet me in the cage bro better find out who you're dealing with before you end up in a body bag,0 +1396,New Ladies Shoulder Tote Handbag Faux Leather Hobo Purse Cross Body Bag Womens - Full read___ http://t.co/4FXfllRIen http://t.co/i12NLSr8Fk,0 +1399,AUTH LOUIS VUITTON BROWN SAUMUR 35 CROSS BODY SHOULDER BAG MONOGRAM 7.23 419-3 - Full read___ http://t.co/HCDiwE5flc http://t.co/zLvEbEoavG,0 +1401,Check out Vintage Longaberger Floral Fabric Shoulder Cross Body Bag Brown Leather Strap http://t.co/FB8snRg4HU @eBay,0 +1402,?? New Ladies Shoulder Tote #Handbag Faux Leather Hobo Purse Cross Body Bag #Womens http://t.co/zujwUiomb3 http://t.co/GBCtmhx7pW,0 +1405,new summer long thin body bag hip A word skirt Blue http://t.co/8JymD9YPSJ http://t.co/57PKmmCaDG,0 +1406,?_ü New Ladies Shoulder Tote #Handbag Faux Leather Hobo Purse Cross Body Bag #Womens http://t.co/UooZXauS26 http://t.co/Pw78nblJKy RT en___,0 +1407,Louis Vuitton Monogram Sophie Limited Edition Clutch Cross body Bag - Full read by eBay http://t.co/I4AogcSOY5 http://t.co/dJIwG9pxV4,0 +1408,New Women Handbag Faux Leather Ladies Shoulder Tote Cross Body Bag Large Satchel - Full re___ http://t.co/NCjPGf6znv http://t.co/GeRJau74eY,0 +1409,?? New Ladies Shoulder Tote #Handbag Faux Leather Hobo Purse Cross Body Bag #Womens http://t.co/zujwUiomb3 http://t.co/iap4LwvqsW,1 +1411,One day this heart gone get me zipped up in a body bag.,0 +1412,#handbag #fashion #style http://t.co/iPXpI3me16 Authentic Louis Vuitton Pochette Bosphore Shoulder Cross Body Bag___ http://t.co/RV0Fk7q4Y5,0 +1414,å_? New Ladies Shoulder Tote #Handbag Faux Leather Hobo Purse Cross Body Bag #Womens http://t.co/zujwUiomb3 http://t.co/YklTFj1FnC,0 +1415,"#handbag #fashion #style http://t.co/hPd3SNM6oy Vintage Coach Purse Camera Bag Cross Body #9973 + +$16.99 (0 Bids) +___ http://t.co/GSmdDmu9Pu",0 +1418,genuine Leather man Bag Messenger fit iPad mini 4 tablet case cross body air jp - Full rea___ http://t.co/rcBurZSb2b http://t.co/eHsfKlgRI3,0 +1419,Louis Vuitton Monogram Sophie Limited Edition Clutch Cross body Bag - Full read by eBay http://t.co/VJgR6Liaxh http://t.co/55JR66PLOV,0 +1420,?? New Ladies Shoulder Tote #Handbag Faux Leather Hobo Purse Cross Body Bag #Womens http://t.co/zujwUiomb3 http://t.co/CJqq6cgohg,0 +1421,Nuu that FAM?? fwt I'm Leave You In a Body bag??,0 +1422,New Ladies Shoulder Tote Handbag Faux Leather Hobo Purse Cross Body Bag Womens - Full read___ http://t.co/uR7FeXszg4 http://t.co/wb8awobLcL,0 +1425,@Rhee1975 @deliciousvomit No I'm not I'm saying that they're lucky they can go home to their families not put in a body bag,0 +1429,@matt_bez oh I'm not bagging her at all! Her body be bangin'. I'm saying she's going to get the rose.,0 +1431,@ohmyloz @RondaRousey who is bagging her body ? She's smoking hot ??,0 +1432,Drake is really body bagging meek,0 +1433,@fuckyeahcarey @BornVerified drake killing this dude and tea bagging the dead body at this point,1 +1436,@MeekMill is w(rec)k league ball @Drake is Olympic level body bagging him like his career was nothing #trollingtilMeekdiss,0 +1439,Idgaf who tough or who from Canada and who from north Philly meek been acting like a bitch & drake been body bagging his ass on tracks,0 +1440,@Yankees body bagging mfs,1 +1441,@BroseidonRex @dapurplesharpie I skimmed through twitter and missed this body bagging.,0 +1443,G+: +https://t.co/dODXi41Y1Cå_is Body Bagging them lyrically! https://t.co/HlMyaAnrC9,0 +1444,@bomairinge @EluTranscendent straight body bagging.,0 +1445,Body Bagging Bitches ???? http://t.co/aFssGPnZWi,0 +1447,No better feeling than seeing and being on stage with my day ones...... 22 year friendships and we still body bagging mics together.,0 +1448,Mopheme and Bigstar Johnson are a problem in this game body bagging niggas #VuzuHustle,0 +1449,I was body Bagging on the ????Today...4got to post my score http://t.co/81g18wSAUk,0 +1450,@editaxohaze then let the bagging body's begin lol ???? I ain't cuffed yet so it shouldn't be that bad!!,0 +1452,#OVOFest Drake straight body bagging Meek on that OVO stage. #ZIPHIMUP!,0 +1453,I'm not a Drake fan but I enjoy seeing him body-bagging people. Great marketing lol.,0 +1454,@amaramin3 Meek is definitely capable of body bagging his ass on the track Drake was just smooth as fuck with it!,0 +1455,Gates not body bagging nobody???????? niggas in br really know who he is ????,0 +1456,WWE 2k15 MyCareer EP18 Tyrone body bagging dudes: http://t.co/mr5bI4KD82 via @YouTube,0 +1458,'I did another one I did another one. You still ain't done shit about the other one.' Nigga body bagging Meek.,1 +1459,Drake Body Bagging Meek. He must of hit a sensitive spot talking about a 'ghostwriter.' He trying 2 end his career. http://t.co/2jHTlWueY0,0 +1460,Good diss bad beat and flow. Mark my words Meek Mill is body bagging him once he responds. Patient patient. Meek is a battle rapper!,0 +1463,@Drake is body bagging meek meanwhile he's on tour with Nicki all hush hush...he's put 2 diss tracks out and meek 0 but dude started it lol,0 +1464,@MzGracieBaby for the record im jumpin out the window early... i got @OfficialRealRap body bagging luck.. lol save the file,0 +1466,Has body bagged ** RT @d_lac: Drake is body bagging meek,0 +1467,drake been kept it the most hip hop during this beef and he fucking body bagging meek back to back ??,0 +1470,Woke up to Drake body bagging Meek again!! Meek u can't out spit ya girlfriend... Just lay down Man.... NOT Right... http://t.co/6CraEKc9wb,0 +1472,@Im2aD I was going to tell him but you were body bagging him,0 +1473,___@MacDaddy_Leo: ?????? No Caption Needed ??. Freshman... http://t.co/k8ughv2aif___my nigga Stacey body bagging niggas! ????,1 +1474,Aubrey really out here body-bagging Meek.,1 +1475,8 hours of bagging groceries = an aching body,0 +1477,@SlikRickDaRula Drake really body bagging peeps man ?? he really bout it,0 +1479,@FoxNews @JenGriffinFNC When you call to report dangerous activity tell em to have body bags on arrival.,1 +1480,Micom 2015 Summer Contrast Candy Color Bowknot Cross Body Tote Shoulder Bags for Womengirls with Micom Zip Po http://t.co/sQMTKKJiMJ,0 +1482,HOBO Hobo Vintage Shira Convertible B___ $238.00 #bestseller http://t.co/0LLwuqn8vg,0 +1483,Child Shoulder Bags PVC Shoulder Book Bag Cartoon Cross Body Bags for Girls http://t.co/7l9qAzLjVg http://t.co/Q0hSyfrwEC,0 +1484,Womens Buckle Casual Stylish Shoulder Handbags Pockets Cross Body Bags Green http://t.co/pYee94nuRe,0 +1485,Ain't no bags in the trunk it's a body,0 +1486,BESTSELLER! Fossil Dawson Mini Cross Body Bag Es___ $98.00 http://t.co/HhHvKxFAIH,0 +1487,@BoomerangTime @RSKarim1 @sopameer @wattashit3 Appears to already be arriving in Ridah in body bags.,0 +1488,The Body Bags has a show on 08/07/2015 at 07:30 PM @ Tremont Music Hall in Charlotte NC http://t.co/FpKiqbus9r #concert,0 +1491,BODY BAGS! https://t.co/0McXc68GZD,0 +1493,you know you hate your body when you buy 2 bags of chips and a variety pack of fruit snacks and a redbull as a snack,0 +1494,BREAKING: Fairfax County firefighter placed on admin leave amid probe into Facebook post about putting police in 'body bags' dept. says.,0 +1495,Womens Buckle Casual Stylish Shoulder Handbags Pockets Cross Body Bags White http://t.co/mWZQCjhZPb,0 +1497,Attention all RCHS football players there will be coffins and body bags by the locker rooms grab one tommorow because were gonna die,0 +1498,Womens Flower Printed Shoulder Handbags Cross Body Metal Chain Satchel Bags Blue http://t.co/rjZw6C8asX http://t.co/WtdIav11ua,0 +1499,Womens Cross Body Messengers Bags Clutch Small Shoulders Zippers Bags White http://t.co/EpIQdBxVZO http://t.co/BhfOYLQLJp,0 +1500,@Deeeznvtzzz bring the body bags tho,0 +1501,US __÷Institute Of Peace__ª Chairman Wants Russian Body Bags http://t.co/owbUjez3q4,0 +1502,UPDATE: Va. firefighter on administrative leave after Facebook post calls for people to put officers in 'body bags.' http://t.co/Mt029QJ4Ig,0 +1503,@ScottWalker So you can send the poor and middle class children to war so they can come in body bags. Typical GOP,0 +1504,@baskgod body bags,0 +1505,Nine giant body sized garbage bags later...I'm just going to start throwing things away. #moving2k15 #expertwhiner,0 +1506,"I love The body shop__ªs bags?? + +#cutekitten #catsofinstagram #summerinsweden #katt #katterp_´instagram #dumle #dagens___ http://t.co/p4ZFXdnbcH",0 +1507,Womens Tote Faux Leather Handbags Shoulder Cross Body Top Handle Bags Rose http://t.co/GYzPisBI1u http://t.co/mSDnTkWYaf,0 +1508,Womens Satchel Lattice Chain Studded Cross Body Multi Colour Shoulder Bags Blue http://t.co/qj2kbltCxZ http://t.co/xQpn2zYkCt,0 +1509,Lab today ready for these body bags. ??,0 +1510,Bitches be takin pics with bags bigger than they whole body ??????,0 +1511,Womens Buckle Casual Stylish Shoulder Handbags Pockets Cross Body Bags Green http://t.co/Jqso4fyZp4 http://t.co/F4XnPliO5S,0 +1513,#IranDeal most members of Congress who don't want this deal don't have any kids who would b coming home in body bags. War makes them money,0 +1514,___Parties and body bags go together like drinking and driving.___,0 +1515,@Limpar33 sweeping legs? Or putting people in body bags?,0 +1518,Shoot shit up till we see body bags,0 +1519,'Your body will heal the bags under your eyes will go away you__ªll be so happy you__ªll smile and really...' http://t.co/WuKcAlNQms,0 +1520,Womens Handbags Cross Body Geometric Pattern Satchel Totes Shoulder Bags White http://t.co/qvSp6b2qSU http://t.co/0s6ydFrWDQ RT gasparc___,0 +1521,women messenger bags clutch bag handbag cross body shoulder bags bag ladies designer handbags high qualit ... http://t.co/zGJGgHDuRF,0 +1522,Womens Handbags Cross Body Geometric Pattern Satchel Totes Shoulder Bags White http://t.co/L1GFXgOZvx http://t.co/TKJYbjjsKl,0 +1523,Status: last seen buying body bags.,0 +1524,@asymbina @tithenai I'm hampered by only liking cross-body bags. I really like Ella Vickers bags: machine washable. http://t.co/YsFYEahpVg,0 +1525,Zicac Vintage Leather Briefcase Messenger Satchel Tote Cross Body Handbags for Womens http://t.co/a3Xv6Ff8DN,0 +1526,Fairfax investigating firefighter over Facebook post saying police should be put in __÷body bags__ª - The Washington Post http://t.co/jAxHzjCCd4,0 +1527,Mens Cross Body Canvas Waist Packs Solid Letter Print Sports Zipper Bags Coffee http://t.co/sCXfC5wi9t http://t.co/gx1oTOH8sj,0 +1530,"Listen to this hit song. A summer Bomb full of positive energy and youth +Did you like it? +https://t.co/2LiWkJybE9 +#Norge2040",0 +1531,beforeitsnews : Global Derivatives: $1.5 Quadrillion Time Bomb http://t.co/GhmmUj7GbE (v___ http://t.co/u9LvvLzhYe) http://t.co/LyJ57pq3yX,0 +1532,the_af comments on 'New evidence of Japan's effort to build atom bomb at the end of WWII' - http://t.co/hcTxghR2Yf,1 +1533,this is about to be a bomb ass firework picture http://t.co/lr4BTvuEoM,0 +1534,Hiroshima prepares to remember the day the bomb dropped http://t.co/oJHCGZXLSt,1 +1535,The Guardian view on the Hiroshima legacy: still in the shadow of the bomb | Editorial: The world longs to cas... http://t.co/RhxMGhsPd7,1 +1537,namjoon's FANTASTIC IS BOMB BYE OMG,0 +1538,HALSEY AND TROYE COLLAB WOULD BE BOMB,0 +1540,@AaronTheFM guys to scared to show his real name anyway he knows I'll bomb him,0 +1543,@CranBoonitz So going to make any bomb threats? @HereticOfEthics,0 +1544,___@dylanmcclure55: Working at zumiez is the http://t.co/zW5jp46v5k___ which location??,0 +1545,New Documents Found Pointing To Japan's WWII Atomic Bomb Program http://t.co/IucPcSfbMT,1 +1546,Jen you da bomb girl! https://t.co/czQr3CI9Xw,0 +1547,@daniglasgow45 Happy birthday big D!!! I miss you girl hope you have a bomb one ???? http://t.co/cFouwPBRCG,0 +1549,@smallforestelf Umm because a gun stopped the gunman with who was carrying a bomb!,1 +1553,@danielsahyounie It'd be so bomb if u guys won ??,0 +1554,New Documents Found Pointing To Japan's WWII Atomic Bomb Program http://t.co/M9mowCMVNj,1 +1555,Hiroshima marks 70 years since bomb http://t.co/3u6MDLk7dI,1 +1556,The crew on #EnolaGay had nuclear bomb on board disarmed. 15 mins to #Hiroshima they got ready to arm Little Boy http://t.co/JB25fHKe6q,1 +1558,@dopeitsval ahh you're bomb baby ??,0 +1559,When you get a bomb ass picture mail ????????,0 +1560,If I fall is men GOD @Praiz8 is d bomb well av always known dat since 2008 bigger u I pray sir,1 +1561,I came up with an idea of a fragrance concept for a bath bomb called The Blood of my Enemies. So you can say that's what you bathe in.,1 +1562,@CaraJDeIevingnc the bomb impact ratio hit beyond kyle js,0 +1567,@SwellyJetEvo Disneyland! Tacos there are bomb!,1 +1568,Flat out bomb by @FlavaFraz21 #whatcanthedo,0 +1569,What would it look like if Hiroshima bomb hit Detroit?: Thursday marks the 70-year anniversary of the United S... http://t.co/6sy44kyYsD,1 +1570,Soul food sound so bomb right now ',0 +1572,The bomb was so appropriate ?? seen as my family and most Jamaicans love shout bullets !,0 +1573,The hatchet-wielding gunman had PEPPER SPRAY AND A FAKE BOMB?!?!?,1 +1574,"Oops. +H bomb lost 70 miles off the Okinawan coast. +Fell off the ship 1965. +http://t.co/yVsJyzwxJR",1 +1575,The Guardian view on the Hiroshima legacy: still in the shadow of the bomb | Editorial: The world longs to cas... http://t.co/ct2JUtvYTg,1 +1576,The MF Life is a vocal and lyrical bomb. Saw her live this summer. AMAZING vocalist. RT @THEmale_madonna: Melanie Fiona is so slept on ??,0 +1577,E-Hutch is da bomb ?? http://t.co/aqmpxzo3V1,0 +1579,Jays rocking #MLB @JoeyBats19 just bombed one out of Rogers Centre. Play-offs r ahead for The #BlueJays - Bell Moseby and Barfield r back!,0 +1580,@Stankyboy88 I should've photo bombed,0 +1582,@SweetieBirks @mirrorlady2 @SLATUKIP So name all the countries we've invaded/bombed aside Libya in North Africa in last 5 years.....,1 +1585,The U.S. bombed Hiroshima 70 years ago today. A look at how war has changed since: http://t.co/UQnj6nk9y3 http://t.co/QLnnMxzFqK,1 +1587,London Life: photos of a beautiful bombed-out Britain http://t.co/2RAcaiVFfq #arts,0 +1588,I can't believe @myfriendmina photo bombed a screenshot,0 +1590,US drone bombs Islamic State target in Syria after taking off from Turkey: A US armed drone has bombed a targe... http://t.co/m0daP5xLwo,1 +1591,@CyhiThePrynce bombed on Kanye in that #ElephantInTheRoom ????????,1 +1593,The majority of those killed were civilians on the ground after the jet first bombed the city's main street then dramatically plummeted,1 +1594,70 years ago at this hour the USA A-bombed Hiroshima therein killing 200000 civilians. Never forget the crime and never repeat. Peace ??,1 +1595,On 1st August #Turkish jets bombed the village Zergele in Qendil and killed 8 civilians and 15 others wounded,1 +1596,@NickLee8 i went to school in a bombed out East End of London3 families to one house no bathroom outside loo & poor so whats yr point,1 +1597,Photo bombed ???? http://t.co/arTUMHmBhh,0 +1599,@QPR1980 @Rorington95 Nowt to do with money. Fergie bombed out the big drinkers at United within years of taking over while Wenger,0 +1600,"'the third generation atomic bombed survivor' Photo exhibition 11:00 to 18:00 8/6. +#?? #Hiroshima http://t.co/gVAipmLSl0",1 +1601,@oooureli @Abu_Baraa1 You mean like the tolerance you showed when sharing 'democracy' with the Iraqis? Wait you mutilated and bombed them.,0 +1602,@BrodyFrieling @hanna_brooksie photo bombed,0 +1603,Shadowflame and the Wraith: Bombed http://t.co/LDBaO0rSuz via @amazon,1 +1604,@WhiteHouse @POTUS Just cos Germany invaded Poland Japan bombed Pearl Harbor PRE-EMPTIVE SUICIDE http://t.co/I2AAG6Lp6W,1 +1605,@CheetosArabia @Crudes It feels like if i would try to grab one off hes Cheetos i'll get bombed.,0 +1607,@ChristophersZen @HunterLove1995 @tblack yeah man..... That movie BOMBED hard,0 +1608,.@RaniaKhalek true. I faced everything from 'Is Bin Laden your uncle?' to 'Hopefully Afghanistan will be bombed'. Children can be very ugly.,0 +1609,@KurtSchlichter @FALPhil This liberal compassion is BS. A specific Feminist said the US bombed Japan back to traditional sexist values.,0 +1610,http://t.co/wMNOnHxEIr 'Nagasaki has to be forever the last city bombed with a nuclear weapon.' #bannukes,1 +1611,Ladies here's how to recover from a #date you totally BOMBED... according to men http://t.co/c5GGSZUGw1 http://t.co/2PiMg9BIcE,0 +1613,@r_lauren83199 @xojademarie124 i hope you get Batista Bombed lauren,0 +1614,.@Vagersedolla visits villages recently bombed by Turkey and finds people fed up with the PKK http://t.co/UUWEiKD7sP,1 +1615,@antpips67 @JohnEJefferson obviously I'm aware that not all AS are from countries we have bombed but a lot are fleeing conflict,1 +1617,'Redskins WR Roberts Belly-Bombed ' via @TeamStream http://t.co/GbcvVEvDTY,1 +1618,Me trying to pass lax with my family ends up by me having to run after the ball after it gets bombed over my head,0 +1619,I liked a @YouTube video http://t.co/FX7uZZXtE4 Benedict Cumberbatch Gets Video Bombed,0 +1621,Today Japan marks 70 yrs since the U.S (A) bombed 2 cities killing over 120000 people But we have to worry about Iran http://t.co/FcIXk23XQH,1 +1622,You just got GIF bombed #AfricansInSF #BeyondGPS https://t.co/ETdGPIwxtI,0 +1623,She didn't even notice me in the background I'm a floatin head #bombed http://t.co/JA0WGp8sPe,1 +1625,70 years ago today the United States of America bombed Hiroshima in Japan.,1 +1626,Storm in RI worse than last hurricane. My city&3others hardest hit. My yard looks like it was bombed. Around 20000K still without power,1 +1627,A wasp just dive bombed my face,0 +1628,@MisfitRarity misfit got bombed,0 +1631,The United Kingdom and France are bombing Daesh in Syria - Voltaire Network http://t.co/zYSsObXNtC,1 +1632,It's been 70 years (and one hour) since the bombing of Hiroshima. Let's take this time to remember.,1 +1633,The cryptic words that guided pilots on the Hiroshima bombing mission http://t.co/FCe0K1Ihti,1 +1634,Japan on Thursday marks the 70th anniversary of the atomic bombing of Hiroshima with the most senior official from Washington ever schedule___,1 +1636,Today marks the 70th anniversary of the bombing of Hiroshima a city I visited and was humbled by in November 2013 http://t.co/AcC1z5Q9Zw,1 +1637,@moscow_ghost @sayed_ridha @Amin_Akh congratulations on capturing a besieged city after 3 months of indiscriminate bombing by land & air,1 +1638,"The only country claiming the moral high ground is the only one to have dropped atomic bombs. #Hiroshima #BanTheBomb +http://t.co/6G49ywwsQJ",1 +1639,The cryptic words that guided pilots on the Hiroshima bombing mission http://t.co/nSS5L64cvR #canada,1 +1643,#Setting4Success Bells toll in Hiroshima as Japan marks 70 years since atomic bombing #News #smallbusiness #entrepreneur,1 +1644,#Australia #News ; #Japan marks 70th anniversary of #Hiroshima atomic bombing http://t.co/7aD0L7cgee READ MORE; http://t.co/hHzQl9tzNP,1 +1645,@NBCNews Yea bombing #pearlharbor not so good of an idea!,1 +1646,The cryptic words that guided pilots on the Hiroshima bombing mission http://t.co/39IAbcC5pK,1 +1647,70th anniversary of Hiroshima atomic bombing marked http://t.co/1mGvd4x5Oe,1 +1648,Japan Marks 70th Anniversary of Hiroshima Atomic Bombing http://t.co/93vqkdFgnr,1 +1649,Japan Marks 70th Anniversary of Hiroshima Atomic Bombing http://t.co/cQLM9jOJOP,1 +1650,Japan Marks 70th Anniversary of Hiroshima Atomic Bombing http://t.co/3EV07PPaPn,1 +1652,Japan Marks 70th Anniversary of Hiroshima Atomic Bombing http://t.co/jzgxwRgFQg,1 +1653,@rinkydnk2 @ZaibatsuNews @NeoProgressive1 When push2Left talk='ecology'&'human rts'&'democracy'. War Afghetc='Left' humanitarian bombing,1 +1654,Hiroshima bombing justified: Majority Americans even today - Hindustan Times http://t.co/cC9z5asVZh,1 +1657,@snapharmony : Bells toll in Hiroshima as Japan marks 70 years since atomic bombing http://t.co/yPvvqZ8jzt,1 +1658,Japan marks 70th anniversary of Hiroshima atomic bombing http://t.co/a2SS7pr4gW,1 +1660,#Japan marks 70th anniversary of #Hiroshima atomic bombing (from @AP) http://t.co/qREInWg0GS,1 +1661,Today is the day Hiroshima got Atomic bomb 70 years ago. - The 'sanitised narrative' of Hiroshima's atomic bombing http://t.co/GKpANz7vg0,1 +1662,"What it was like to survive the atomic bombing of Hiroshima +http://t.co/LGrOcbXPqo",1 +1663,I think bombing Iran would be kinder... https://t.co/GVm70U2bPm,0 +1665,Japan on Thursday marks the 70th anniversary of the atomic bombing of Hiroshima with the most senior official from Washington ever schedule___,1 +1666,Oh and fuck Bill Clinton for bombing us and fuck NATO.,0 +1669,Japan marks 70th anniversary of Hiroshima atomic bombing: Bells tolled in Hiroshima on Thursday as Japan marked 70___ http://t.co/NBZiKcJpHp,1 +1670,Japan marks 70th anniversary of Hiroshima atomic bombing #Generalnews http://t.co/M9o08GUrT4,1 +1671,A Marshall Plan for the United States by Dambisa Moyo via @ProSyn #oped http://t.co/GnPStnvi5G via @po_st,0 +1672,@ameenshaikh3 by ur. logic if bridge didnt collapse then second train engine should cross bridge then @sanjaynirupam @sureshprabhu,1 +1673,"Australia's Ashes disaster - how the collapse unfolded at Trent Bridge... http://t.co/Dq3ddGvgBF + #cricket",1 +1675,Sioux City Fire Officials Believe Bridge Collapse Lead To Cement Truck Roll Over - Siouxland Matters: Siouxlan... http://t.co/sZTGmbkoHG,1 +1676,Ashes 2015: Australia__ªs collapse at Trent Bridge among worst in history: England bundled out Australia for 60 ... http://t.co/985DwWPdEt,1 +1677,2 Injured 1 missing in bridge collapse in central Mexico http://t.co/kHF0iH05A9,1 +1679,I never knew about the relationship btwn Kansas City Hyatt bridge collapse & AIA's COTE. http://t.co/ThS9IqSWP3 via @HuffPostArts,1 +1683,What the fuck is going on at Trent Bridge?! Reminds me of England's collapse out in the Caribbean back in the 90s...,1 +1684,Swiss Kosher Hotel Bridge Collapse Injures Five People - http://t.co/TxIestoX5n @JewishPress,1 +1686,Bridge collapse not natural calamity but man-made: MPP lambasts Congress - KanglaOnline http://t.co/jp9XylA3C5 #Yugvani,1 +1687,Two giant cranes holding a bridge collapse into nearby homes http://t.co/OQpsvrGbJc,1 +1688,ICYMI - #Ashes 2015: Australia collapse at Trent Bridge - how Twitter reacted http://t.co/gl6eeJyJkY http://t.co/wqtDnP3w5a,1 +1689,Warne shocked over Australia's epic collapse at Trent Bridge: Johannesburg Aug 06 (ANI): Legendary Australian... http://t.co/LwwoJXtTIV,1 +1690,Two giant cranes holding a bridge collapse into nearby homes http://t.co/UmANaaHwMI,1 +1691,@BloopAndABlast Because I need to know if I'm supposed to throw myself off a bridge for a #Collapse or plan the parade. There is no both,0 +1692,2 Injured 1 missing in bridge collapse in central Mexico - Fox News Latino http://t.co/l0UnaFLy0Y,1 +1693,Two giant cranes holding a bridge collapse into nearby homes http://t.co/5t69Ev0xTi via http://t.co/Nyp8OQ2Z9T,1 +1694,#TrainTragedy - Happened in MP due to collapse of bridge now I m afraid to take a long distance train. http://t.co/JthusynJaH,1 +1698,Listening to Blowers and Tuffers on the Aussie batting collapse at Trent Bridge reminds me why I love @bbctms! Wonderful stuff! #ENGvAUS,0 +1699,Two giant cranes holding a bridge collapse into nearby homes http://t.co/jBJRg3eP1Q,1 +1700,Two giant cranes holding a bridge collapse into nearby homes http://t.co/gSKJqWyI2d,1 +1701,#computers #gadgets Two giant cranes holding a bridge collapse into nearby homes http://t.co/UZIWgZRynY #slingnews,1 +1702,Australia's Ashes disaster - how the collapse unfolded at Trent Bridge - Telegraph http://t.co/6FYnerMUsG,1 +1703,@followlasg This is urgentthere is currently a 3 storey building at church B/stop Oworoshoki Third mainland bridge which likely to collapse,1 +1705,Two giant cranes holding a bridge collapse into nearby homes http://t.co/lSQe7nu6kl,1 +1706,Two giant cranes holding a bridge collapse into nearby homes http://t.co/q5q1x5Vcqk,1 +1707,Ashes 2015: Australia__ªs collapse at Trent Bridge among worst in history: England bundled out Australia for 60 ... http://t.co/t5TrhjUAU0,0 +1709,Ashes 2015: Australia collapse at Trent Bridge - how Twitter reacted http://t.co/ik7mGidvbm http://t.co/AF1yzUS8LN,1 +1710,Mexico: construction of bridge collapse killså_one http://t.co/I2C00FcOwb http://t.co/jAAgcFaRTW,1 +1712,Two cranes restoring a bridge in the central Dutch town of Alphen aan den Rijn collapse on to buildings with rescuers searching for,1 +1715,Leicester_Merc : ICYMI - #Ashes 2015: Australia collapse at Trent Bridge - how Twitter rea___ http://t.co/HqeWMREysO) http://t.co/y4y8fclJED,0 +1716,Two giant cranes holding a bridge collapse into nearby homes http://t.co/9asc1hhFNJ,1 +1718,US wont upgrade its infrastructure? http://t.co/NGEHhG9YGa' it a bad situation and its going to get ugly very quickly #USA #sustainability,1 +1719,A Marshall Plan for the United States by Dambisa Moyo via @ProSyn #oped http://t.co/l5g2zJ3kgG via @po_st,0 +1720,@ameenshaikh3 sir i just only wanted to make a point about @sureshpprabhu you made and said he is lying about bridge collapse.,1 +1721,@SonofLiberty357 all illuminated by the brightly burning buildings all around the town!,0 +1722,'i'm a Gemini' *children screaming buildings burning police sirens in the distance*,1 +1723,Mmmmmm I'm burning.... I'm burning buildings I'm building.... Oooooohhhh oooh ooh...,1 +1724,@DougMartin17 Fireman Ed runs into burning buildings while others are running out Doug he deserves your respect??????,1 +1725,@themagickidraps not upset with a rally upset with burning buildings businesses executing cops that have nothing to do with it etc,1 +1726,kou is like [CASH REGISTER] [BUILDINGS BURNING],0 +1727,@_minimehh @cjoyner I must be overlooking the burning buildings? #BlackLivesMatter,1 +1728,@fewmoretweets all lives matter. Just not a fan of burning down buildings and stealing from your neighbors to 'protest',1 +1731,The greatest female beat boxer ever now but it's w/e... Save babies outta burning buildings on my free time but ya know.. whatevs..,0 +1733,I'm mentally preparing myself for a bomb ass school year if it's not I'm burning buildings ??,0 +1735,The Rocky Fire burning in northern California is still threatening more than 7000 buildings. It's the largest in the state #9newsmornings,1 +1737,? High Skies - Burning Buildings ? http://t.co/uVq41i3Kx2 #nowplaying,0 +1739,"Messi: has tattoos so he can't donate blood +Ronaldo: runs into burning buildings to save dogs #respect",1 +1740,I can probably skip on these basic life maintenance things for a few days. (cut to burning buildings people screaming in the streets),1 +1741,@1acd4900c1424d1 @FoxNews no one is rioting burning down buildings or looting.,1 +1744,@joshcorman #infosec rather you knew it or not your a firefighter now days you often run into burning buildings Deal with it.,0 +1745,into burning fucking buildings (2/2),1 +1747,Ever since Kelly's burned I keep having dreams about being inside burning buildings ??,0 +1750,Schools in Western Uganda still Burning down Buildings during Strikes....Strikes in Western Uganda always Lit literally..,1 +1752,I Pledge Allegiance To The P.O.P.E. And The Burning Buildings of Epic City. ??????,0 +1753,@DoctorFluxx @StefanEJones @spinnellii @themermacorn No burning buildings and rob during a riot. That's embarrassing & ruining this nation.,1 +1754,People are more worried about the burning of buildings than black people losing their lives disgusting.,1 +1755,Bradford. Back to doing what we do best. Burning down our own buildings. Read it and weep Leeds. https://t.co/OLnfzb86zb,1 +1756,@EPCOTExplorer my jealous tears are burning with the fire of a thousand ransacked buildings. SO AWESOMEEEEEEEE,1 +1757,@zourryart I forgot to add the burning buildings and screaming babies,1 +1759,Sinking ships burning buildings & Falling objects are what reminds me of the old us.,1 +1760,like for the music video I want some real action shit like burning buildings and police chases not some weak ben winston shit,1 +1761,Burning buildings? Media outrage? http://t.co/pHixZnv1YN,1 +1762,"Must get hot in burning buildings + +-loses followers- https://t.co/8sP8xLbbDR",1 +1763,@RockBottomRadFM Is one of the challenges on Tough Enough rescuing people from burning buildings?,0 +1766,saving babies from burning buildings soaking cake in a shit tonne of alcohol mat is a man after my own heart ?? #GBBO,0 +1767,@PPFA At least they aren't burning buildings and looting stores.,0 +1768,Witness video shows car explode behind burning buildings on 2nd St this afternoon #Manchester http://t.co/cgmJlSEYLo via @MikeCroninWMUR,1 +1769,Watching Xela firefighters struggle to save burning buildings last night w/ old equipment makes me so grateful for DCFD @ChR3lyc @IAFF36,1 +1770,"http://t.co/WRB7Xd8W5y +CROYDON RIOTS- The Next Day: Burning Buildings in High Street & crowds at Reeves Corner +Croydonization +August 2011",1 +1771,Fire hazard associated with installation of non-compliant external cladding on http://t.co/bTPQdehl3p - By @www.cbplawyers,1 +1772,.@greenbuildermag @NFPA to hold free webinar on #wildfire mitigation 8/19 at 2pm ET. http://t.co/xmsvOHKccP @Firewise @Michele_NFPA,1 +1773,Fire hazard associated with installation of non-compliant external cladding on http://t.co/4I0Kz2aKly - By @www.cbplawyers,0 +1774,Video: Fire burns two apartment buildings and blows up car in Manchester http://t.co/5BGcw3EzB5,1 +1775,Fire destroys two buildings on 2nd Street in #Manchester http://t.co/Tqh5amoknd,1 +1776,Multiple Buildings On Fire In Downtown Hinton http://t.co/P6kdh0p0Sp http://t.co/WaetKGsZA9,1 +1779,Buildings are on fire and they have time for a business meeting #TheStrain,1 +1780,Ton of smoke coming out of one of the new apartment buildings at 160 Ross in Auburn. Several fire trucks on scene. http://t.co/AHVYmSQHqC,1 +1784,square just let booty org write xv im sure we'd do just fine (buildings around me set on fire),0 +1786,Smoke detectors not required in all buildings: An office building on Shevlin-Hixon Drive was on fire. There we... http://t.co/z6Ee1jVhNi,1 +1787,"someone: mentions gansey on fire +me busting through the brick walls of seven different buildings:",1 +1789,Such beautiful architecture in #NYC I love those fire escape routes on the buildings. #newyork___ https://t.co/fW1PtaElgV,1 +1790,#TweetLikeItsSeptember11th2001 Those two buildings are on fire,1 +1791,just in: #kenya: several buildings are reported to be on fire close to dam estate #langata at 2:22pm.,1 +1792,They are evacuating buildings in that area of State Road 20. We still don't have confirmation of what is on fire.,1 +1796,shootings explosions hand grenades thrown at cars and houses & vehicles and buildings set on fire. It all just baffles me.Is this Sweden?,1 +1798,A change in the State fire code prohibits grills on decks at condos and apartment buildings. Check with your... http://t.co/KE1ZS6NAml,0 +1799,Fire hazard associated with installation of non-compliant external cladding on high-rise buildings - Insurance - Aust http://t.co/wFsEaOBATo,0 +1802,Fire Displaces Families and Damages Two Buildings in Troy: Fire broke out on Fourth Street in___ http://t.co/6HKw5qLPPt #Albany #NY #News,1 +1803,What news reports am I missing? Are there buildings on fire people shot etc. due to videos? #PPSellsBabyParts https://t.co/Wzc5r4XOqZ,1 +1804,So my band Buildings on Fire are playing @bbcintroducing @PurpleTurtleRdg this Wednesday with @GIANTGIANTSOUND https://t.co/ofaN6DkOEZ #rdg,0 +1807,#TweetLikeItsSeptember11th2001 Those two buildings are on fire,1 +1808,Now on http://t.co/3sTH9lrBUn: PHOTOS: Flames rage through Manchester buildings in 3-alarm fire http://t.co/jQxM4gcQZ3 #NH1News,1 +1809,@BigSim50 nah Philly pundits are half the cause. They set buildings on fire to report on buildings being on fire.,1 +1810,MaFireEMS: RT WMUR9: Two buildings involved in fire on 2nd Street in #Manchester. WMUR9 http://t.co/QUFwXRJIql via KCarosaWMUR,1 +1811,Now on WSLS: fire burns multiple buildings in #Montgomery Co tips to make childcare less expensive & rain. Join @jennasjems @PatrickWSLS,1 +1812,I just wanted to watch Paper Towns but the buildings on fire ?????,1 +1813,I hope the only time I end up on TV is when I'm being arrested for lighting buildings on fire.,1 +1814,Charred remains of homes on 2nd St in Manchester NH after fire rips through 2 buildings damaging neighboring homes http://t.co/Ja3W1S3tmr,1 +1815,Three-alarm fire destroys two residential buildings a car in Manchester N.H. on Sunday afternoon http://t.co/rVkyj3YUVK,1 +1816,1943: Poland - work party prisoners in the Nazi death camp Treblinka rebelled seizing small arms and setting buildings on fire. #history,1 +1817,17 people displaced after 3-alarm fire tore through 2 apartment buildings on Second Street in Manchester. -- http://t.co/NzqwTCYidv #MHT,1 +1818,Hinton city officials confirm multiple buildings on fire currently evacuating people on Main Street. We're on our way @OKCFOX,1 +1821,I HATE WHEN IM TRYING TO STRAIGHTEN MY HAIR AND MY BROTHER COMES SWOOPING IN BEHIND ME AND SCARES ME I JUST BURNED MY FINGER,0 +1822,Fire burning on Pendleton has burned 300 acres: Smoke reported drifting over Temecula. http://t.co/ZR5RgbGh03,1 +1823,I should probably stay away from hot glue guns.. I burned one finger pretty bad,0 +1824,What progress we are making. In the Middle Ages they would have burned me. Now they are content with burning my books. -Sigmund Freud,0 +1826,@HGF52611 Uh huh. You only have to be burned once to know fire hurts. Robbie Ross should throw home run derby in the All Star Game #RedSox,0 +1827,Burned my popcorn??,0 +1831,@Wild_Lionx3 so others don't get burned,0 +1832,when you're taking a shower and someone flushes the toilet and you have .1 second to GTFO or you get burned??????????????????????????????????????????????????,0 +1833,"Always look for the silver lining! + +My barn having burned to the ground +I can now see the moon. +~ Mizuta... http://t.co/Gl4McaX0ny",0 +1834,Sure I just burned about 100 calories after eating a giant bowl of mac and cheese so I totally earned this 300 calorie Klondike Bar.,0 +1835,Alton brown just did a livestream and he burned the butter and touched the hot plate too soon and made a nut joke http://t.co/gvd7fcx8iZ,0 +1836,All you have to do is look up sports riots on google and you'll see more than couches being burned.. You fr gotta go https://t.co/P1AmgINsYs,1 +1837,I spent 17 minutes walking with RunKeeper. 90 calories burned. #LoseIt,0 +1838,Sure - take them away from fire fighting for King Stevie & Crusty to have a photo-op ! http://t.co/epeX4axG4b,0 +1840,burned 129 calories doing 24 minutes of Walking 3.5 mph brisk pace #myfitnesspal,1 +1841,Don't get burned twice by the same flame.,0 +1842,Burned dog finds new home with young burn victim http://t.co/Pqrjvgvgxg,1 +1845,It hurts for me to eat cause i burned my tounge with a pepperoni yesterday!,0 +1847,I spent 15 minutes lifting weights. 43 calories burned. #LoseIt,0 +1848,Holy fuck QVC bitch just got burned so hard.,0 +1851,I just remembered the McDonald's that burned down used to have the coolest play ground & the new one ain't got shit but video games )):,0 +1852,burned 202 calories doing 30 minutes of Walking 4.0 mph very brisk pace #myfitnesspal,0 +1853,My sister Rl Burned All Her Boyfriend Clothes Recorded It & Sent It to him ????,0 +1856,"@kennethbauer_ more like coffee and noodles + +Burned",0 +1857,Just burned the crap out of my grilled cheese sandwich. Sure wish I had a few life skills figured out by now...,0 +1860,I joked about it but Wood has to be burned out from working so many innings so often. #CubsTalk,0 +1861,Mad River Complex fires have burned 14028 acres 8% contained: The Mad River Complex fires in Trinity County ... http://t.co/LfNIPpNOtO,1 +1863,@thomasvissman22 @KeithyyL Keithyy gettin burned outta the blocks and on social media.... http://t.co/dlkuFtLQnF,0 +1864,I burned myself today on the oven ?? it was at 500 degrees ??,0 +1865,Watch how bad that fool get burned in coverage this year. Dat dude is all-pro practice squad material,1 +1866,I PUT MY CHICKEN NUGGETS IN THE MICROWAVE FOR 5 MINUTES INTEAD OF 1 ON ACCIDENT AND THEY FUCKING BURNED,0 +1869,Just burned the shit outta myself on my dirt bike ??,0 +1870,Metal Cutting Sparks Brush Fire In Brighton: A brush fire that was sparked by a landowner cutting metal burned 10___ http://t.co/rj7m42AtWS,1 +1873,the stars are burning i here your voice in my mind,0 +1875,You Bitches Walking Around Like Yall Hot Shit & Got Bed Bugs & You Burning ????,0 +1877,It's raining outside I'm burning my favorite candle and there's hot cocoa to sip. Nap time = awesomesauce.,0 +1878,hermancranston: #atk #LetsFootball RT SkanndTyagi: #letsFootball #atk WIRED : All these fires are burning through ___ http://t.co/DmTab6g7j7,0 +1880,501 sky news mandem in totteham going maddddd burning up fed cars an dem ting dere,1 +1881,@aubilenon @MarkKriegsman if you think you'd like burning man you should try it because it's the only way to know!,0 +1882,The 8-Minute Fat-Burning Routine That__ªs Also Really Fun http://t.co/g2h7xNecD8 #fat weightless # fatburning #burnfat #skinny #workout,0 +1883,___ bulletproof and black like a funeral; the world around us is burning but we're so cold. http://t.co/uqssnAAtTu,1 +1885,.@StacDemon with five burning questions for Chris Mullin and St. John__ªs in 2015-16: http://t.co/NmRVTHkvAh #SJUBB,1 +1886,@JohnsonTionne except idk them?? it's really burning ??????,0 +1887,Leader of #Zionism STOP BURNING #Babies https://t.co/6xYsDN2Xz0,1 +1888,Kanger coils - burning out fast? via /r/Vaping101 http://t.co/cykr4XAlUH,0 +1889,RT @HuffPostComedy: We should build a wall that keeps Burning Man attendees from coming home http://t.co/xwVW1sft4I http://t.co/j7HUKhWmal,0 +1890,Flames visible from fire in Tucson mountains: A lightning-caused fire burning in steep rocky terrain in mountains___ http://t.co/zRTRPL77QV,1 +1892,We will be burning up like neon lights??????,1 +1893,Don't be so modest. You certainly... *sniff* *sniiiiiiff* Er Donny? Is something burning?,0 +1895,The Burning Legion has RETURNED! https://t.co/hKsbmijqZ1,0 +1897,the bar method ___ integrates the fat burning format of interval training the muscle shaping techniqu http://t.co/xuzee2BUdv,0 +1898,If you're bored with life if you don't get up every morning with a burning desire to do things - you don't have enough goals. -Lou Holtz,0 +1899,My hand is burning,1 +1900,Uhhhhh demon hunters. But not the whole Burning Crusade v 2.0 thing. https://t.co/oPtpS1lgKC,0 +1901,Now playing: Boat Club - Memories on London Burning Web Radio - http://t.co/umtNNImTbM,0 +1905,@nagel_ashley @Vicken52 @BasedLaRock @goonc1ty rip the world... its burning,0 +1908,@Rubi_ How many stacks of burning did it apply?,0 +1909,Parents are taking their kids to Burning Man and one 11 year old thinks it's 'better than... http://t.co/wp6V1BHhoQ,0 +1910,@Michael5SOS haha so would you say its so hot your balls are burning off????,1 +1911,Why put out a fire that's still burning?,0 +1914,#?x?: :and to counter acts such as the burning of the Alexandrian library.,1 +1915,'CALIFORNIA IS BURNING:' Gov. Jerry Brown told reporters at a press conference that California is experiencing... http://t.co/arzeMSR7FQ,1 +1916,If You__ªre Not Paying Attention to Your Influencers You__ªre Burning Money | SocialTimes http://t.co/Ptc0xcRAGY,0 +1917,@minsuwoongs i completely understand because i just woke up like 15 minutes ago and im Burning,0 +1918,#ika #tuning Soup #diet recipes | fat burning soup recipes: http://t.co/8r5vpAoo5z Fat Burning Soup Diet Recip http://t.co/JvcxB75DrJ,0 +1919,2 Burning Man Tickets + Vehicle Pass - Full read by eBay http://t.co/b0eS3ZIORK http://t.co/juIIt2YFVo,0 +1920,The last few days of summer are supposed to be the most fun so what's more fun then accidentally burning arm hair while playing w/ a lighter,0 +1921,Killing Black Babies at Planned Parenthood where's the demonstrations looting of PP burning down the buildings Black Babies Lives Matter,1 +1922,Mmmmmm I'm burning.... I'm burning buildings I'm building.... Oooooohhhh oooh ooh...,0 +1924,I'm mentally preparing myself for a bomb ass school year if it's not I'm burning buildings ??,0 +1925,the mv should just be them strutting like they mean it while buildings are burning up in the bg and flames everywhere how cool would that be,1 +1927,@Louis_Tomlinson incredible? THE CHILDREN WERE SCREAMING BUILDINGS WERE BURNING AND I WAS DANCING IN THE ASHES,1 +1929,@fewmoretweets all lives matter. Just not a fan of burning down buildings and stealing from your neighbors to 'protest',1 +1932,A protest rally at Stone Mountain? Atleast they're not burning down buildings and looting store like some individuals do when they 'protest',0 +1933,My man runs into burning buildings for a living but is scared to hit up a girl. I don't get it.,0 +1935,THIS SOUNDS LIKE A SONG YOU WOULD HEAR IN A MOVIE WHERE THEY ARE WALKING AWAY FROM BURNING BUILDINGS AND CARS AND SHIT,0 +1937,@foxnewsvideo @AIIAmericanGirI @ANHQDC So ... where are the rioters looters and burning buildings???? WHITE LIVES MATTER!!!!!!,1 +1938,forestservice : RT dhsscitech: #Firefighters run into burning buildings___we work on #tech t___ http://t.co/KybQcSvrZa) http://t.co/Ih49kyMsMp,1 +1939,#KCA #VoteJKT48ID DUCKVILLELOL: Burning flips the table and says 'screw this lets hit some buildings!' Grabs a DR ___ http://t.co/03L7NwQDje,0 +1940,Hero's fight wars and save ppl from burning buildings etc I'm sorry but u gotta do more than pay 4 a sex change be4 I call u a hero,0 +1941,@themagickidraps not upset with a rally upset with burning buildings businesses executing cops that have nothing to do with it etc,1 +1942,Destruction magic's fine just don't go burning down any buildings.,1 +1943,Burning buildings? Media outrage? http://t.co/pHixZnv1YN,1 +1945,I will never support looting or the burning of buildings but when seeing the people fight back against the police. I was proud,1 +1948,Attempting Delany's Dhalgren in beastly NY heat. Downing hot coffee amongst descriptions of buildings burning for days without ruin.,0 +1949,@MoFanon ?? your last retweet you would think the lion saved people from a burning buildings it's not that deep,0 +1950,like for the music video I want some real action shit like burning buildings and police chases not some weak ben winston shit,0 +1951,I'm battling monsters I'm pulling you out of the burning buildings and you say I'll give you anything but you never come through.,0 +1952,Large fire burning several buildings causing evacuations in downtown Hinton: http://t.co/mtMkiMwiyy,1 +1953,Ali you flew planes and ran into burning buildings why are you making soup for that man child?! #BooRadleyVanCullen,0 +1954,Heavy smoke pouring out of buildings on fire in Port Coquitlam http://t.co/GeqkdaO4cV http://t.co/Dg0bGzeCgM,1 +1955,drew storen is probably curing cancer & saving puppies from burning buildings while contemplating what he did 2 deserve this disrespect,0 +1956,Ah yes the gays are totally destroying America. I can see buildings burning and meteors crashing into schools wow,0 +1957,@_minimehh @cjoyner I must be overlooking the burning buildings? #BlackLivesMatter,1 +1959,Strikers in Phalaborwa striking over the CHINESE taking their jobs. Strikers burning buildings attacking cars 2... http://t.co/08LnGClZsj,1 +1960,if firefighters acted like cops they'd drive around shooting a flamethrower at burning buildings,0 +1961,"dog rendered a kitten from burning buildings +https://t.co/9cpWIEcEGv http://t.co/rZLYtneZ2u",1 +1962,Schools in Western Uganda still Burning down Buildings during Strikes....Strikes in Western Uganda always Lit literally..,1 +1963,@kshllcenterpri1 @Progress4Ohio burning down buildings what you mean like when you burnt down those black churches?,1 +1965,"'i never understood guys who wanted to run into burning buildings.' +'you chase murderers.' +'not if they're on fire.'",0 +1966,Dudes will thoroughly express how stupid black ppl r for burning buildings in response to brutality but nvr mention them being mistreated.,1 +1967,"You picture buildings burning to the ground from the basement to the streetlight. + +I'm not your drinking problem +a hole is in the sky.",1 +1968,I Pledge Allegiance To The P.O.P.E. And The Burning Buildings of Epic City. ??????,1 +1969,Where are the protests ? The riots? The burning buildings? How come you don't see any of that crap happening when... http://t.co/1QOchsPYbw,1 +1971,Insane bush fires in California. Be safe. https://t.co/jSlxTQ3NqS,1 +1972,@POTUS Would you please explain what you are going to do about the volcanoes & bush fires spouting all that CO2 into the air?,1 +1973,@marcoarment Middle of winter in Sydney we have had snow bush fires and 78 degree days in the last week. Keeps you on your toes for sure.,1 +1975,Ted Cruz fires back at Jeb & Bush: ___We lose because of Republicans like Jeb & Mitt.___ [Video] - http://t.co/BFTHaHLCr0,0 +1976,OMFG there are bush fires all over this tiny island. Our freaking house is gonna burn down.,1 +1979,Slash-and-burn blamed for bush fires in western St Thomas - http://t.co/5dJ6cHjFZP,1 +1980,Drought fuels bush fires in Jamaica - http://t.co/ZDtDqQbAHC http://t.co/PsQCNsVfgP - @JamaicaObserver @cnewslive RE https://t.co/6ZGef8J8Bm,1 +1981,Ted Cruz fires back at Jeb & Bush: ___We lose because of Republicans like Jeb & Mitt.___ [Video] - http://t.co/bFtiaPF35F,0 +1983,@DoriCreates @alhanda seems gov moonbeam between tokes blames bush for all the fires.,1 +1985,The Public Health Team was traumatised after the bush fires. Could #AppreciativeInquiry turn this around? http://t.co/soEa1GgbKj,1 +1986,'When you attack women's health you attack America's health.' Hillary Clinton shows how to #StandwithPP http://t.co/HXdG254dHO,0 +1987,On holiday to relax sunbathe and drink ... Putting out bush fires? Not so much ?? #spain https://t.co/dRno7OKM21,0 +1989,Californian Bush Fires 2015 http://t.co/rjdX29wosp,1 +1990,28 Oct 1895: 'Bush Fires.' http://t.co/zCKXtFc9PT,1 +1991,@dacherryontop13 ohh there are bush fires in Spain like every year one time when we went swimming there were planes getting water to fight,1 +1992,Bush Fires are scary....even scarier when you go down and fight them,1 +1994,One thing you can be sure of. There will never be bush fires in Scotland as the ground is always soaking wet????,0 +1995,Drought fuels bush fires in Jamaica - http://t.co/0YMF6TXFcH http://t.co/3i3d2NGeNt - @JamaicaObserver @cnewslive RE https://t.co/jyIEkEo2he,1 +1996,@JohnFromCranber Pleas FOR global warming don't really work when California / Australia keep having catastrophic 'bush' fires.,1 +1999,Ted Cruz fires back at Jeb & Bush: ___We lose because of Republicans like Jeb & Mitt.___ [Video] http://t.co/FgDEh56PLO,0 +2000,SMH photographer Wolter Peeters was on the front line with NSW Rural Fire Service crews la___ http://t.co/gXe7nHwZ3e http://t.co/sRbqlMuwbV,1 +2001,So apparently there were bush fires near where I live over the weekend that I was totally oblivious to...,1 +2003,California Bush fires please evacuate affected areas ASAP when california govts advised you to do so http://t.co/ubVEVUuAch,1 +2006,Ted Cruz fires back at Jeb & Bush: ___We lose because of Republicans like Jeb & Mitt.___ [Video] - http://t.co/KCofF6BmiE,0 +2007,The Bush fires in CA are so crazy,1 +2009,That triumphant moment when you cook up two eggs over easy with no yolk casualties ?? http://t.co/fQJ5Aga1pd,0 +2010,"?? +Warfighting Robots Could Reduce Civilian Casualties So Calling for a Ban Now Is Premature - IEEE Spectrum http://t.co/TzR58B86qz",1 +2012,Another movie theater attack..close to home this time. Thankful for no casualties. Life will go on because we cannot allow evil to win!,1 +2014,Civilian casualties rise as Afghan war intensifies in 2015-- http://t.co/NnylXhInPx,1 +2015,Afghan conflict sees 'sharp rise' in female casualties http://t.co/4hcYwRWN6L http://t.co/2TwXZ6vxbx,1 +2016,Warfighting Robots Could Reduce Civilian Casualties So Calling for a... http://t.co/9DVU1RidZ3,1 +2017,@LibertarianLuke I'm all for that to be honest. If people want to go on a rampage let them use their own hands and feet. No casualties.,0 +2019,"Afghanistan: U.N. Reports 'Record-High Levels' of Civilian Casualties + +In news from Afghanistan the United... http://t.co/YMcZyVKfmE",1 +2020,Need to stop bottling things up because when everything eventually explodes the casualties just keep getting higher and higher,1 +2022,Countless Casualties All Across The Globe War Being Orchestrated On All The Corners Of The Planet [On All Levels] http://t.co/G1BWL3DQQK,1 +2023,disinfo: Warfighting Robots Could Reduce Civilian Casualties So Calling for a Ban Now Is ... -... http://t.co/yUinMErQ2s #criticalmedia,1 +2024,A poignant reminder that in war there are many casualties. http://t.co/Mwmt3BdR5L,1 +2026,@LasVegasLocally @VitalVegas They reined it in to 3 drinks each for 2 people but only on account of too many falling-off-stool casualties!,0 +2027,There might be casualties tomorrow,1 +2028,The more I listen to it the more I believe Casualties of Cool is one of the best albums there ever was.,0 +2031,'American Weapons and Support Are Fueling a Bloody Air War in Yemen' http://t.co/7aGeAkVn2x,1 +2033,I'M LAUGHING IN THE FACE OF CASUALTIES AND SORROW THE FIRST TIME I'M THINKING PAST TOMORROW BUT I AM NOT THROWIN AWAY MY SHOT,1 +2034,@Catwoman1775 Another shooting in a movie theater this is getting more crazier but I'm glad they got the shooter & no casualties.,1 +2035,"Students at Sutherland remember Australian casualties at Lone Pine Gallipoli + http://t.co/d50oRfXoFB via @theleadernews",1 +2036,Sharp rise in women children casualties in Afghan war UN says http://t.co/0CXm5TkZ8y http://t.co/v5aMDOvHOT,1 +2038,"@FlyOpineMonkey You mean Olympic ;-) +Also its follow-up for Honshu Operation Coronet. +In all: 1m Allied casualties 30m Japanese dead.",1 +2039,@AlcoholAndMetal + do anything to fix that. Of the few people he had every trusted in his life Charles was one of the casualties.,1 +2040,Civilian Casualties in Afghanistan Reach Record High http://t.co/r8pTVFUh5X http://t.co/npCKK0tlEQ,1 +2042,@CounterMoonbat @Voodoo_Ben I've heard we're still using Purple Hearts manufactured then bc of the # of casualties we expected.,1 +2043,Another day has passed and THANKFULLY Central Command has CONFIRMED NO new casualties. Please pray for those in... http://t.co/mFSw0tYstA,1 +2045,News Wrap: UN warns female and child casualties are on the rise in Afghanistan http://t.co/vSvY1qe69t #pbs #iraq,1 +2046,___The road to power is paved with hypocrisy and casualties.___ #FrancisUnderwood #HoC https://t.co/zqO6NUvYTu,0 +2047,"Revise the Death to America scenario? + +While there's 500 American casualties by Iranian activity SUSPECTED!!! http://t.co/drlKEbeYPi",1 +2049,#Afghanistan: sharp rise in women and children casualties in first half of #2015 http://t.co/LdyWd4ydT9,1 +2050,Stay tuned or don't idc #casualties http://t.co/nssjPR6Pdd,0 +2051,#Civilian casualties in Afghanistan hit highest number since 2009 U.N. says via @WashingtonPost - http://t.co/xTF5DvgRvh,1 +2055,Warfighting Robots Could Reduce Civilian Casualties So Calling for a Ban Now Is Premature http://t.co/lzff4pT4AZ #FTSN #FTSNNewsdesk #___,1 +2056,@irishspy What you don't think the Allies should have just sucked up 1 million casualties?,1 +2057,Afghanistan: sharp rise in women and children casualties in first half of 2015 http://t.co/3sqSErgnI2,1 +2058,Whimsy as it pertains to mass casualties. Always impressive.,0 +2059,Train derailment: In Patna no news of any casualty so far http://t.co/Yg697fcQGr,1 +2060,We're #hiring in our Toronto branch! Surety Underwriter/Senior Underwriter and Casualty Product Leader. Apply today. http://t.co/PraMKlrMhz,0 +2061,Being able to stay out of work this week to take online courses for the Property and Casualty State___ https://t.co/jmD7zwKSDM,0 +2063,Property/casualty insurance rates up 1% in July: After several months of no movement commercial property/casu... http://t.co/KcLkoKqI8a,1 +2064,Was '80s New #Wave a #Casualty of #AIDS?: Tweet And Since they__ªd grown up watching David___ http://t.co/qBecjli7cx,0 +2067,Cos sanity brings no reward for one more hit and one last score... Don't be a casualty cut the cord...,1 +2068,Another sad ocean casualty-Gray whale population in the Pacific listed as critically #endangered (#drone video) https://t.co/vwz3vZpmfb .,1 +2069,@ThomasHCrown My grandfather was set to be in the first groups of Marines to hit Japan in Operation Olympic. 95% casualty rate predictions,1 +2070,1st Quality Insurance Group is #hiring Licensed Property & Casualty Insurance Agent Produc http://t.co/VMJRtuVmh4 #jobs #Denver,0 +2071,'Become another casualty of society',0 +2073,Benzema increasingly looks to be a casualty of Benitez's new look squad. Arsenal bound? 50-50 chance I think,1 +2074,WANTED: gritty and real casualty photos of Pasta Thursdays at Amico's. Tag us or #amicospizzato #seeyouatamicos... http://t.co/MZ8VQXbKTs,0 +2075,Casualty Team: Ice Cream Recall Sends Chill Through Food Industry http://t.co/6GsAmY6mts,0 +2076,UNPREDICTABLE DISCONNECTED AND SOCIAL CASUALTY ARE MY FAVORITES HOW DO PEOPLE NOT LIKE THEM,1 +2077,@NorthBayHealth Trauma Center Shines In Response To Multi-Casualty Crash. http://t.co/21B6SKPDUR http://t.co/wBCb3sYtj7,1 +2078,@ScriptetteSar @katiecool447 btw the 30th is actually next year casualty began 6th September 1986 so 2016 marks 30 years,1 +2081,@AvBronstein @Popehat @instapundit @KurtSchlichter Also are you aware of the casualty estimates for an invasion of Japan's home islands?,1 +2082,"You can't watch 'Home Alone 2' without telling your kids +'she used to be on Casualty'. +#Homealone2 #film4",0 +2083,AM Best Special Report: How Metrics Correlate to AM Best's Property/Casualty ... - MarketWatch http://t.co/mVrsYu2PPK,0 +2084,That took way longer than I expected,0 +2086,I still don't know why independence day and social casualty are not on the rowyso setlist these songs would be so good live,0 +2087,@Calum5SOS I need to stop doing this to myself???? @s_casualty,0 +2089,RT @GreenHarvard: Documenting climate change's first major casualty http://t.co/4q4zd7oU34 via @GreenHarvard,1 +2091,@5SOSFamUpdater social casualty,0 +2094,Canceling Deal for 2 Warships #France Agrees to Repay #Russia via @nytimes http://t.co/f2gwxEPrAk,0 +2095,Social Casualty #MTVHottest 5SOS,0 +2096,Conning Builds Strong Case for Portfolio #Diversification for Property-Casualty Insurers http://t.co/33FbR25t1O,0 +2097,@reriellechan HE WAS THE LICH KING'S FIRST CASUALTY BLOCK ME BACK I HATE YOU! http://t.co/0Gidg9U45J,1 +2098,Casualty Roleplay somebody please am so bored,0 +2099,Casualty insurance jobs against hunt up willinghearted into: RPN http://t.co/pByA7Uv3V5,0 +2100,Japan had a nuke program (albeit unsuccessful) and the casualty estimates for a ground war were in the tens of millions. @MacKinnon08,1 +2102,#NowPlaying: Dubstep Hardstyle Trap Messy Mix (event recording) by Alien Casualty on @Mixify http://t.co/m203UL6o7p http://t.co/m203UL6o7p,1 +2103,Small casualty on the way to Colorado http://t.co/hDVmhSQXHm,1 +2108,.@stavernise: France agreed to repay Russia for two warships which were never delivered after economic sanctions http://t.co/K4H8cq7puo,0 +2109,.@robdelaney Catastrophe is anything but! I literally have been unable to stop ejaculating.,1 +2110,@APPLEOFFIClAL Migrating from iPhoto to Photo is a catastrophe. I have wasted days trying to get this to work. 12 hrsto get to 8% complete.,0 +2111,Cultivating Joy In The Face Of Catastrophe And Suffering http://t.co/o0LTQDJbQe #pjnet #tcotå_#ccot http://t.co/MO9wpTyqkp,0 +2112,#Borrowers concerned at possible #interest rate rise. This could be a #catastrophe http://t.co/SBHHkkz01Y,0 +2113,.@uriminzok The coming catastrophe of the destruction of the puppet republic half naemolgo continue to firmly support. Yiraneuni and against,0 +2114,failure is a misfortunebut regret is a catastrophe,0 +2115,@deb117 7/30 that catastrophe man opens school w/another he's an athlete not a teacher a principle not fulfilling any inside clerical duties,0 +2116,#iphone #twist Ultimate #preparedness library: http://t.co/ksgmY0D0Mx Prepare Yourself For Any Catastrophe. Ov http://t.co/MZK0PFogI7,0 +2117,@gemmahentsch @megynkelly @DLoesch I can not envision any catastrophe that would prevent a woman placing her child for adoption.,0 +2118,@peterjukes But there are good grounds to believe that 'political military catastrophe' was a crime planned and committed by individuals.,1 +2119,Then the stylist who'd been silent says 'there's a cool show on Amazon Prime called Catastrophe...',0 +2121,#boy #mix Ultimate #preparedness library: http://t.co/O207JyaByz Prepare Yourself For Any Catastrophe. Over 10 http://t.co/cjCtb2oCxg,1 +2122,God bless catastrophe,0 +2123,@MaatMHI Slightly diff catastrophe & Barry was running solo but generally the same thing.,0 +2125,UPDATE: 7 of the 9 Mac Pros my company bought in May have had catastrophe failures requiring repair!,1 +2126,Pisces tweets need to get better because most the tweets make me sound like a total emotional catastrophe.,0 +2128,Taylor and Cara aka Catastrophe and Mother Chucker behind the scenes of Bad Blood. Vote: http://t.co/TF2BkQ0OlX #VMAs http://t.co/3fQq7pFjvX,0 +2129,[reviews] #PixelsMovie not a catastrophe nor a funny movie... our review here : http://t.co/lVbUw01YOH,0 +2131,Burford. What a catastrophe! Traffic and big lorries. No action as usual from Council.,0 +2133,#Denver CO #Insurance #Job: Claims Property Field Adjuster Catastrophe Safeco ___ USA at Liberty Mutual Insurance http://t.co/3k42MJVqCA,0 +2134,#spark #song Ultimate #preparedness library: http://t.co/VsGqoLr32g Prepare Yourself For Any Catastrophe. Over http://t.co/p7UhcB13Qx,0 +2136,12 Month Payday Short Catastrophe Loans - Promote Finance Your Desire lIQd,0 +2138,I rated Catastrophe (2015) 8/10 #IMDb - hilarious! http://t.co/cjrSSRY1RT,0 +2139,.@AIGinsurance CEO: Divestitures and #Catastrophe Losses Temper Q2 #Results http://t.co/2y2wZk1FrM,0 +2141,Chances are many of us are still digging out from that catastrophe. THIS is why WoW NEEDS to evolve. THIS is why it can't be Vanilla anymore,1 +2142,"Oh my god that__ªs the biggest #gbbo catastrophe I have EVER seen. + +Not that I watch the show or am into it at all ??",0 +2144,#nar #phuket Ultimate #preparedness library: http://t.co/qYAeNDvDGC Prepare Yourself For Any Catastrophe. Over http://t.co/3Zp6Ahnsxn,1 +2145,not a catastrophe at all I'm perfectly content. being the only one means nothing when I'm being controlled. text me if you got crap to say,0 +2146,@MasochisticMage + catastrophe! It caused people to get reckless and the bottom line is that at least three of your friends will have +,0 +2148,I had 2 regular coffees and a Rockstar + coffee today and I'm still tired.,0 +2149,Alaska's #Wolves face catastrophe Denali Wolves population plummeted to 48! #SaveDenaliWolves TWEETSTORM: http://t.co/sywUEL7yYx,0 +2150,@mark_argent I haven't watched that one yet. Just finished Catastrophe which is amazing,0 +2153,bbc r5live studio discussion of hiroshima v poor. sheer *luck* the cold war did not result in catastrophe. MAD = red herring. #ScrapTrident,1 +2154,Human history becomes more and more a race between education and catastrophe.,0 +2157,Success is not built on success. Its built on failure. Its built on frustration. Its built on catastrophe. #real,0 +2158,"Britney Spears > Vegas!! I missed you so much! ?? Can__ªt wait for the show tonight ?? #PieceOfMe + +?? Catastrophe ?? http://t.co/4mJyW7p7Cf",0 +2159,Learning from the Legacy of a Catastrophic Eruption http://t.co/RbmuCURS2F,1 +2160,'Climate change could be catastrophic -- but it does have some benefits.' Really @weathernetwork?!?! http://t.co/IBx3cragtt,0 +2161,Following a catastrophic injury acute medical care takes precedent. PTSD often follows in it's wake undetected... http://t.co/BZkqpl6R0a,1 +2163,catastrophic-fallen-angel: reveillertm: macabrelolita: I was supposed to write __÷amino acids__ª and I nearly... http://t.co/dIoBzGHFju,0 +2164,The Catastrophic Effects of Hiroshima and Nagasaki Atomic Bombings Still Being Felt Today http://t.co/oU1M9chznq,1 +2169,The Catastrophic Effects of Hiroshima and Nagasaki Atomic Bombings Still Being Felt Today http://t.co/TzxeG4gOkD,1 +2171,The Catastrophic Effects of Hiroshima and Nagasaki Atomic Bombings Still Being Felt Today http://t.co/1kRPz3j1EU,1 +2172,"'Invading Iraq was a catastrophic mistake'. + +Diplomacy needs to replace constant threat of war by US and Israel: + +http://t.co/yqjpn3qUUX",1 +2173,Learning from the Legacy of a Catastrophic Eruption http://t.co/25sY9Y295L via @newyorker,1 +2174,The Catastrophic Effects of Hiroshima and Nagasaki Atomic Bombings Still Being Felt Today http://t.co/WC8AqXeDF7,1 +2175,@marginoferror I wish going custom ROM weren't so catastrophic,0 +2176,The best part of old baseball managers wearing uniforms is the implication that if something catastrophic happens they'll grab a glove.,0 +2177,Learning from the most destructive volcanic event in U.S. history thirty-five years later: http://t.co/KkjP9KsBst,1 +2179,The Catastrophic Effects of Hiroshima and Nagasaki Atomic Bombings Still Being Felt Today http://t.co/tGcR5voFJ3,1 +2181,@MyVintageSoul ...of the British upper class and his manservant. The pampered wealthy Brit causes a catastrophic shift (reversal) of...,0 +2183,@APANO55 @JamesMelville 99% of Scientists don__ªt believe in Catastrophic Man-Made Global Warming only the deluded do.,1 +2186,@SenSchumer Is this what U want Netanyahu leading these UNITED STATES into a CATASTROPHIC religious world war? ENOUGH already!,0 +2188,Learning from the Legacy of a Catastrophic Eruption - The New Yorker http://t.co/y8YqPBE4t9,1 +2189,Learning from the Legacy of a Catastrophic Eruption http://t.co/PgXfocgHqg via @newyorker,1 +2190,The Catastrophic Effects of Hiroshima and Nagasaki Atomic Bombings Still Being Felt Today http://t.co/QVlxpyyyCd,1 +2191,A better look at what this catastrophic rain and flooding has done to our___ https://t.co/5yRBegzafX,1 +2192,The Catastrophic Effects of Hiroshima and Nagasaki Atomic Bombings Still Being Felt Today http://t.co/rNqEBAyCVM,1 +2193,Learning from the Legacy of a Catastrophic Eruption - The New Yorker http://t.co/t344PhNpy9,1 +2194,Something Catastrophic Is Coming: Should We Tune Out? http://t.co/a8jZ5A26wi,1 +2195,'Kessler Syndrome' is the name for the catastrophic exponential proliferation of Space debris and destruction of satellites. #GravityMovie,1 +2198,Learning from the Legacy of a Catastrophic Eruption - The New Yorker http://t.co/vMWTOUyOHm,1 +2203,"Excited not only about the next 6 years of school and ensuing student debt but also catastrophic climate change in my lifetime + +:D ??",0 +2204,Pretty much every time the audio dies on an audio stream for a baseball game I assume catastrophic nuclear attack.,0 +2205,Looking for a #Defendant #Catastrophic Injury Solicitor #jobs http://t.co/Gz27aUDyHa http://t.co/P4EKgC9sIG,0 +2207,@SyringeToAnger å_ and probably even more. But some disagreements with General Ross and the catastrophic occurrence made something clear. å_,0 +2209,Emergency Response and Hazardous Chemical Management: Principles and Practices http://t.co/4sSuyhkgRB http://t.co/TDerBtgZ2k,0 +2210,THE CHEMICAL BROTHERS to play The Armory in SF tomorrow night!: EMERGENCY BAY AREA EDM ANNOUNCEMENT ___ THE CHEM... http://t.co/3LN8TrHw6X,0 +2211,#Illinois: Emergency units simulate a chemical explosion at NU https://t.co/rd10EX6HvT via @sharethis #hazmat,1 +2212,At FEMA's Center for Disaster Preparedness for a weeklong training on Chemical Biological Radioactive Nuclear Emergency Response,1 +2213,Russian nuclear-biological-chemical (NBC) brigade 'emergency response' exercise in Southern MD http://t.co/Ul5XdblmBk http://t.co/VjHpVLnbaw,1 +2214,.@david_cameron Stop upsetting this bee! Listen to science not chemical companies #savebees https://t.co/NDJja4D5O8 http://t.co/l7BJSq0Y2o,0 +2215,???? #Krefeld: the incident happened in a chemical industry park! Emergency operations underway! A building reportly collapsed! @cnnbrk @ntvde,1 +2216,THE CHEMICAL BROTHERS to play The Armory in SF tomorrow night!: EMERGENCY BAY AREA EDM ANNOUNCEMENT ___ THE CHEM... http://t.co/wBoGs8EjSj,0 +2217,Sign the petition @david_cameron to protect bees instead of toxic chemical companies want to harm them! #savebees - http://t.co/dB7ft3Yi6d,0 +2219,USA: BREAKING NEWS: CHEMICAL SPILL/EVACUATIONS/RED CROSS EMERGENCY http://t.co/007Npen6LG,1 +2220,New #job opening at Downtown Emergency Service Center in #Seattle - #Chemical #Dependency Counselor or Intern #jobs http://t.co/BNRdKgXavr,0 +2221,Emergency crews respond to chemical spill downtown beaumont #benews http://t.co/PME0HOJVYA,1 +2222,Explosion at chemical site leads to building collapse near Krefeld Germany. Emergency crews on scene; avoid the area. #iJETalerts,1 +2225,Bomb Crash Loot Riot Emergency Pipe Bomb Nuclear Chemical Spill Gas Ricin Leak Violence Drugs Cartel Cocaine Marijuana Heroine Kidnap Bust,1 +2227,Google Alert: Emergency units simulate a chemical explosion at NU http://t.co/NDgpWYxu6H,0 +2229,Downtown Emergency Service Center is hiring! #Chemical #Dependency Counselor or Intern in #Seattle apply now! #jobs http://t.co/SKQPWSNOin,0 +2230,Do you have a plan in case of a pool chemical emergency? Learn more here: http://t.co/UePPjwvLcb #watersafety @CDC,0 +2231,@pjcoyle ... need to be included in emergency planning for chemical plants. See also http://t.co/OamqqBNIce,0 +2233,Emergency units simulate a chemical explosion at NU: Suppose a student in the research labs at Northwestern___ http://t.co/0NR4DPjgyL,1 +2235,We know this is bad for the bees - don't give in to pressure from short term profit obsessed chemical companies... http://t.co/aNuTOopKF4,0 +2236,Nueva favorita: EmergeNCY feat. The Chemical Brothers / My Bits http://t.co/MET4YtZMFB @DeezerColombia,0 +2237,Downtown Emergency Service Center is hiring a #Chemical #Dependency Counselor or Intern apply now! #Seattle #jobs http://t.co/SKQPWSNOin,0 +2238,Emergency units simulate a chemical explosion at NU: Suppose a student in the research labs at Northwestern___ http://t.co/ExitLxgIsJ,1 +2239,Our Chemical Spill Cleanup videos will prepare you for an emergency situation in the lab. http://t.co/UMQbyRUPBd,1 +2240,Downtown Emergency Service Center is hiring! #Chemical #Dependency Counselor or Intern in #Seattle apply now! #jobs http://t.co/HhTwAyT4yo,0 +2241,Day 2. Liquidation of emergency at chemical object. #USAR2015 #USAR15 #RUOR #??????????? http://t.co/gGTmDqUdDo,0 +2242,Emergency units simulate a chemical explosion at NU - Evanston Now http://t.co/kfyEbhb3DI,1 +2244,"Please stand up for bees against profit-hungry chemical companies. Keep the ban & #Savebees +Sign the petition now: +https://t.co/4zsXjGV7iT",0 +2245,@laevantine Fortunately I reworked the plumbing on my emergency chemical shower to draw from the glitter pipe for just such an occasion,0 +2246,@Chemical_Babe its a family emergency so I can't make it unless I have a chance to use by phone for stream.,0 +2248,Emergency 4 Harbor City Mod v4.5.2 #6 Chemical Fire in Residential Area!: http://t.co/uLuPxYzJwV via @YouTube,1 +2250,Emergency responders prepare for chemical disaster through Hazmat training. http://t.co/q9zixCi8E6,1 +2251,"Lindenow: 3:15pm +Emergency crews are at a chemical spill in the main street near Church Street",1 +2254,'Go too Ibiza Pop ah Pill Get DRUNK & Fall off a Cliff'. (Real Talk) @DannyJohnJules @Spencer_Fearon @ChristieLinford,0 +2255,@D33munni @JeanNamibian noooooooo ... *proceeds to fall off a cliff*,0 +2256,Fuck Neil go fall off a cliff or something.....#yr ??????,0 +2257,Beat the #heat. Today only Kill Cliff Free Fall $2. Pick up a #cold drink today after the #tough #crossfit... http://t.co/QaMwoJYahq,0 +2258,"NEWS FLASH! Any decent billers been promoted to 'manager'? If so let me know as I want to watch your billings fall off a cliff. +#Humble",0 +2260,2Leezy its like you're about to fall down a cliff but you're grabbing my hand but my palms are sweaty ... Why bruh? https://t.co/GrWdr4kuf3,1 +2262,I regress and I slip and I fall off that cliff,0 +2263,ok peace I hope I fall off a cliff along with my dignity,1 +2265,If you're reading this go accidentally fall off a cliff mate,0 +2266,If u faved that I hope you fall off a cliff ??,0 +2267,#FunnyNews #Business Watch the moment a cliff collapses as huge chunks of rock fall onto a road in China http://t.co/LCi3pljX25,1 +2271,Photographer Brian Ruebs endures 4500-feet climb to capture bride and groom http://t.co/BmWmpOyDIg,0 +2272,alex is making me watch 107 facts about minions i want to fall off a cliff help,0 +2273,Do me a favor and fall off a cliff,0 +2274,@AlexJacobsonPFS All Andre and Gore have to do is not fall off the cliff and we're elite on that side of the ball.,0 +2275,Fall off a cliff please https://t.co/4vWSL2Gfp0,1 +2277,I hope they fall off a cliff.,0 +2278,Don't let your style fall flat this summer! Lord & Cliff #thinkpink #magichairbump is your answer. Adding this... http://t.co/NmHZTB1ewM,0 +2279,When you're in deep sleep and then you dream you're bout to fall off a cliff then wake up while struggling to keep a balance,0 +2280,Currently want to drive my car off a cliff and fall to my death.,0 +2281,I hope you fall off a cliff,0 +2282,I accidentally killed an 87 day snap streak and now I wanna accidentally fall off a cliff ????????????????????,1 +2284,@punkblunts @sincerelyevelnn fall off a cliff into hell idc,0 +2285,Photographer Brian Ruebs endures 4500-feet climb to capture bride and groom http://t.co/JXhAZEBNQK,0 +2286,@SZMNextDoor I got this cute lil cliff you can fall off of??,0 +2288,i hope u trip n fall of a cliff after this tweet https://t.co/3hoIkDmoCB,0 +2289,News Update Huge cliff landslide on road in China - Watch the moment a cliff collapses as huge chunks of rock fall... http://t.co/gaBd0cjmAG,1 +2292,Huge cliff landslide on road in China: Watch the moment a cliff collapses as huge chunks of rock fall onto a r... http://t.co/eEEwO207mX,1 +2294,One day I want someone to run for the ferry fall and crack there face open for almost knocking me over just to get on a boat ??????,0 +2296,Gulfport Energy - All-In Realizations Fall Off A Cliff http://t.co/CjuiGhBxyn,0 +2297,"*Jumps off of a cliff while drinking tea* + +This is how British people fall off cliffs.",0 +2298,627% but if they had lower striked than 16 I would have gone even further OTM. This could really fall off a cliff.,1 +2299,That sounds like a really bad idea I like Yoenis but I feel like his production could fall off a huge cliff.,0 +2300,'I'm there!' Bride & Groom on mountain cliff edge. Ha Ha just kidding. I WILL NOT EVER be there. Ha Ha - http://t.co/Io9ry1akON,0 +2301,ESN : Cilla Black died of stroke after fall in Spain: Sir Cliff revealed he was due to visit her in Spain next___ http://t.co/F7a66dIiYK,0 +2303,#NowPlaying * Cliff Richard - I Could Easily Fall (In Love With You) (& Shadows) * #Internet #Nieuws #Radio On http://t.co/8LkMWp9qzw,0 +2304,Runaway Minion Causes Traffic Collapse in Dublin http://t.co/u2Kwof3wtj,1 +2306,Time collapse is such a cool video technique. https://t.co/upLFSqMr0C,0 +2307,Would a paramedic really do that? Leave someone inside a building that's about to collapse/blow up? @HalloIkBenWill,1 +2308,@GeoffRickly I don't see the option to buy the full collapse vinyl with tee bundle just the waiting?,0 +2309,Warne Ponting shocked by Australian collapse - Yahoo Cricket India https://t.co/hsgkTeZUCN,0 +2311,Watch The Terrifying Moment Two Giant Cranes Collapse Onto Homes: A row of homes was destroyed in seconds. http://t.co/G38Y8H1tJt,1 +2312,@BasilDudin The 'barbaric Saudies' as you said they relive Austrian economy. If we stop coming here many project will collapse.,0 +2314,Interview on The Collapse of Materialism Best #TalkRadio Listen Live: http://t.co/sDXZHjco0X,1 +2315,Economic Collapse Investing: Specific actions and strategies to securing lasting wealth from the financial blowout. http://t.co/JZwRisXEPF,1 +2317,What would you do if you were trapped in a collapsed circus tent with a bunch of clowns? http://t.co/6HKCa1dSna,0 +2318,I get this feeling that society will collapse or implode. So don't be a hero and play your part.,0 +2319,@Marvel @DCComics @ImageComics @DarkHorseComics @IDWPublishing And by doing this you're enabling the possible collapse of the industry.,0 +2321,EUROCRISIS Guardian Greece's tax revenues collapse as debt crisis continues: As talks continue over proposed _â_... http://t.co/bBm9sR1wOw,1 +2324,Only one commodity has escaped the total collapse in prices http://t.co/4HngTKDQMv #business,0 +2325,Ashes 2015: Australia totally collapse and the internet absolutely loves it http://t.co/AFzqvotutj,0 +2327,I liked a @YouTube video http://t.co/BM0QEC7Pja Eminem feat. Nate Dogg - Till I Collapse,0 +2329,Timestack' Photos Collapse Entire Sunsets Into Single Mesmerizing Images. http://t.co/Cas8xC2DFE,0 +2330,mentions of 'theatre +shooting' on Twitter spike 30min prior to $ckec collapse http://t.co/uuBOvy9GQI,1 +2332,Correction: Tent Collapse Story http://t.co/S7VYGeNJuv,1 +2333,@BehindAShield @Wars_Goddess Sweet Lord. (I collapse as my knees buckle),0 +2334,Ashes 4th Test: 10 Hilarious Twitter Reactions to Australia's collapse http://t.co/6DznEjuVD3 by @Absolut_Sumya15,0 +2336,SA MP. Steel and ferrochrome industry on verge of collapse. You don't even put that on list of questions to president for oral answer,0 +2337,Correction: Tent Collapse Story http://t.co/jXs50FkviK,1 +2339,"Spot fixing/match fixing ..anyone??? +Or it has to be Pak SL WI RSA or BD to say this. Sham on them who say that when these team collapse",0 +2340,Why did I come to work today.. Literally wanna collapse of exhaustion,1 +2341,#Greece's tax revenues collapse as debt crisis continues via @guardian #bailout http://t.co/cJvbQXw83s ^mp,1 +2342,'60 all out? What!' - World reacts to Aussie collapse http://t.co/I6zQlk2Puz,0 +2346,Warne shocked over Australia's epic collapse at Trent Bridge: Johannesburg Aug 06 (ANI): Legendary Australian... http://t.co/LwwoJXtTIV,1 +2347,Now that's what you call a batting collapse #theashes,1 +2348,Growth dries up for BHP Billiton as oil price collapse bites http://t.co/HQoD6v6DnC,0 +2349,#ROH3 #JFB #TFB #alrasyid448ItuRasya Correction: Tent Collapse Story http://t.co/iZJToojzKp #US ROH3 SmantiBatam #ROH3SmantiBatam,0 +2350,"Greece's tax revenues collapse as debt crisis continues +http://t.co/uxp6PoqjLb",1 +2351,å_MGN-AFRICAå¨ pin:263789F4 å_ Correction: Tent Collapse Story: Correction: Tent Collapse story å_ http://t.co/fDJUYvZMrv @wizkidayo,0 +2352,The @POTUS economy continues to collapse.,1 +2354,___@TheHighFessions: 'My friend came to school blasted...i asked him if he was high he said pancakes then collapsed' -Iowa City High___,0 +2356,@sholt87 @MtGrotto @Eco11C @carlsbadbugkil1 Saved us?Bush lowered tax rate for wealthy n economy collapsed w/Middle Class 401ks destroyed.,0 +2358,I was on my way to Gary but all the Chicago entrances was closed due to a bridge collapsed ?????? I hope they let us through tomorrow,1 +2359,Gut Deutsch musik! The old and rotten the monarchy has collapsed. The new may live. Long live the German Republic! https://t.co/RJjU70rHyu,0 +2361,Great British <b>Bake</b> Off's back and Dorret's <b>chocolate</b> gateau collapsed - JAN___ http://t.co/53LORsrGqf,0 +2362,@organicallyrude @1ROCKSTAR62 #wish Mattingly & Bundy & McGwire were standing on it when it collapsed!,1 +2363,Catching up on GBBO and omg that girls cake that just totally collapsed I feel so bad,0 +2364,Roof collapsed a bowling alley many in the community remember going to for more than 30 years @KEZI9 http://t.co/sAhbhLXsSh,1 +2366,+ DID YOU SAY TO HIM!!?!?!?!' and phil actually collapsed on the gravel sobbing endlessly with a crowd watching him confused angry mad+,0 +2367,1 hour parade like 50 people collapsed. #OneHeartOneMindOneCSS,1 +2368,... The pain of those seconds must have been awful as her heart burst and her lungs collapsed and there was no air and...,1 +2370,Bay District coaches learn CPR during medical course: PANAMA CITY ___ An athlete collapsed on the field. Coaches... http://t.co/KFkaosh0KH,1 +2372,@GorpuaZikinak and tongue out as she collapsed in the cum puddle her whole body covered.,0 +2373,@indiepopmom I CANT BREATHE MY LUNGS COLLAPSED,0 +2374,Zimbabwe is a country with a collapsed government ruled by a dictator while many live below the poverty line.,0 +2375,@durrellb Prices here are insane. Our dollar has collapsed against the US and it's punishing us. Thanks for the info.,1 +2376,I just collapsed in my bed ugh I'm exhausted,0 +2377,Another entity forced to close in Montego Bay as a result of the collapsed sewer line #TVJNews,1 +2379,Petition | Heartless owner that whipped horse until it collapsed is told he can KEEP his animal! Act Now! http://t.co/nJRjxqBjr4,0 +2382,On the 2nd year the officer team running the club collapsed. A few influential members betrayed everyone's trust severing the community.,0 +2385,Petition | Heartless owner that whipped horse until it collapsed is told he can KEEP his animal! Act Now! http://t.co/ym3cWw28dJ,0 +2386,My portable closet has collapsed 3x and it finally broke and my mom said 'maybe u should get rid of some clothes' lol how about no,0 +2387,@rokiieee_ the game has officially collapsed,0 +2388,Great British Bake Off's back and Dorret's chocolate gateau collapsed - JAN MOIR http://t.co/2SgDOFsmrQ http://t.co/xLEgC6UorA,1 +2389,Still mortified that when I went to rose's I collapsed on my heels face planted in front of everyone and broke my fav shoes,0 +2390,'It hasn't collapsed because the Greek people are still being played for as fools by Tsipras he cost___' ___ WallyBaiter http://t.co/gbRNuLp3fH,0 +2391,My @Quora answer to Why do my answers get collapsed when others don't? http://t.co/IKfmEktPCX,0 +2394,and he almost collapsed bc he said his wish came true moderately FUCK,0 +2395,collapsed the moment i got home last night lol,0 +2396,@flickershowell oh wow my heart collapsed cool im crying cool cool,1 +2397,Look: #I have collapsed #after attempting to munch an endangered species.,0 +2398,Apparently part of the building just collapsed. Hope everyone is ok.,1 +2399,@xDescry I was wrong to call it trusty actually.. considering it spontaneously collapsed on me that's not very trusty.,1 +2400,Petition | Heartless owner that whipped horse until it collapsed is told he can KEEP his animal! Act Now! http://t.co/87eFCBIczM,1 +2402,@Collapsed thank u,0 +2406,I liked a @YouTube video from @sqwizzix http://t.co/GGqCz9AB6u Call of Duty: ___The Piano Entertainer___ Ep. 9 ___ Musicians Collide!,0 +2408,And when those waves shall ripple collide it's on the tide of YOUR LOVE I will survive. #love @LesleyChappelle,0 +2409,Somehow find you and I collide http://t.co/Ee8RpOahPk,0 +2410,But even if the stars and moon collide I never want you back into my life??????.,0 +2412,Stepped outside with a drink and a cigarette and immediately locked eyes with a jogger. Worlds really do collide.,0 +2413,It's always super awkward when worlds collide,0 +2414,http://t.co/QQC0gKbEGs efs300: http://t.co/ZStuvsBQq0 'Star Wars' and 'Star Trek' Collide on Pluto Moon Charon #pluto,0 +2415,When high fashion and food collide: Gucci has chosen one of Shanghai__ªs most popular commercial avenue... http://t.co/MkRxQZeHmY #fashion,0 +2416,That sounds about right. Our building will have a thunderstorm inside one day when the air masses collide. https://t.co/2rTQ9QmGPB,0 +2417,i just remember us driving and singing collide together,0 +2418,Lets collide untill we fill the space.. ??,0 +2419,Students COLLIDE this Fri/Sat - register http://t.co/PwjJimRfLy #nlccollide http://t.co/3w0pxFyyri,0 +2420,I don__ªt wanna touchdown just wanna make our worlds collide ??,0 +2421,@madisonpa_ love you & can't wait until collide!!!????,0 +2422,PIERCE THE VEIL Rubber Bracelet Wristband Collide with the Sky - Full read by eBay http://t.co/6QC8whdiZY http://t.co/ineZZAES5D,0 +2423,Soultech - Collide (Club Mix) http://t.co/8xIxBsPOT8,0 +2425,I liked a @YouTube video from @gassymexican http://t.co/lPgFqnpjd3 WHEN REALITIES COLLIDE! (Life Is Strange Hilarious Glitch),0 +2427,@mattcohen4fake Gamma Ray January Worlds Collide She Waits Be Me Wave Past Perfect Reunion Lucky Cool If I Come Over Hot Times...,0 +2428,The Witches of the Glass Castle. Supernatural YA where sibling rivalry magic and love collide #wogc #kindle http://t.co/IzakNpJeQW,0 +2429,Maybe if the stars align maybe if our worlds collide,0 +2430,Check out my new song 'Collide' live at the Bowery Electric! http://t.co/MESgcNGAz0,0 +2432,@tackettdc Like a scene from When Worlds Collide ......,1 +2435,and even if the stars and moon collide ___ oh oh! i never want you back to my life you can take your words and all... http://t.co/4E2gJmVRVI,0 +2437,#NowPlaying the playlist 'When Jazz and Hip-Hop Collide' in @TIDALHiFi http://t.co/mzQq5PAi8G,0 +2438,#Vancouver to host 100s of electronic art events including @MUTEK_Montreal. http://t.co/vjBhxN9x1O #ISEA2015,0 +2440,Worlds Collide When an American Family Takes Over Britain's Isle of Man in New TLC Show Suddenly Royal http://t.co/OmB3oS54tN via @People,0 +2441,When Houston and NYC collide. ?? @pageparkescorp @chloeunguyen @laurensicle @charstevens97 @tiara_marei #gemma #bowe___ http://t.co/9wowPs78VD,1 +2442,This setlist from @collideworship_ this past Sunday was powerful! What song was your favorite? http://t.co/vNzyBFGZcm,0 +2443,#TheDoolingGroup 2 injured when 2 school buses collide - å_ #BREAKING: School bus slams into school bus in Bordento... http://t.co/YQHfio9XQm,1 +2445,Even then our words slip and souls coincide Finer than subatomic spells Just as we collide http://t.co/2WcbrgN62J,0 +2446,Devia ler 'AS WE COLLIDE #wattys2015' no #Wattpad #teenfiction http://t.co/g891m9GH4r http://t.co/Xq92X4bVG3,0 +2451,"Secrets of the world collide but i leave the past behind +It's been so long now and I can't go without",0 +2452,When high fashion and food collide http://t.co/qDhxto57EM,0 +2453,You either ride with us or collide with us. It's as simple as that for me and my niggas.,0 +2454,Cyclist who collided with runner on Roanoke greenway wins $300000 civil verdict http://t.co/WgasoeNCwc via @roanoketimes,0 +2455,My 2 fav worlds have collided! Thanks to @lennonparham @Jessica_StClair I found the @GilmoreGuysShow podcast!! #ihave44episodesofGG #nojoke,0 +2456,@RedCoatJackpot *As it was typical for them their bullets collided and none managed to reach their targets; such was the ''curse'' of a --,0 +2457,First Tweet collided with a Selfie. Pretty 'Sweet' if you ask me???? http://t.co/knomg9pfiz,0 +2458,Police investigating after an e-bike collided with a car in Little Portugal. E-bike rider suffered serious non-life threatening injuries.,1 +2459,#Newswatch: 2 vehicles collided at Lock and Lansdowne Sts in #Ptbo. Emerg crews on their way,1 +2460,"When love and hate collided part II +Lanjut dirumah... + +#yagitudeh - Jake (at Rumah Cipinang) ___ https://t.co/yiLt1Bb68k",0 +2462,"--thus making @FemaleGilgamesh's assault useless. +The spears collided with the dark force however did not penetrate. + +Due to the dark --",0 +2463,I scored 111020 points in PUNCH QUEST stopped when a squeaky bat collided into my skull. http://t.co/aEtgbxm1pL,1 +2464,Reading for work has collided with reading for pleasure. Huzzah. Don't miss @molly_the_tanz's Vermilion! http://t.co/83bMprwH7W,0 +2465,Mind blown by @GlassAnimals slithering viscous Gold Mine (cover of Yeah Yeah Yeahs + Erykah Badu)___ http://t.co/7Zb9gm5z0h,0 +2466,Monsoon flooding - Monsoon rains have have hit India Pakistan and Myanmar hard this season. Two trains collided ... http://t.co/A7zF6N7vrL,1 +2467,An Eau Claire man who police said was drunk when his SUV collided with a train was sentenced in Chippewa County: http://t.co/kQpkY7Dthj,1 +2468,Two trains have collided in India. Please pray for victims their families and rescuers.,1 +2470,Two cars collided following shotgun fire in Denton http://t.co/0r03C6njLI,1 +2471,Major accident causes life-threatening injuries closes highway: An 18-wheeler and an SUV collided and then___ http://t.co/ajTXUafOEM,1 +2475,On page 500 of 688 of After We Collided by Anna Todd http://t.co/Y7PetO0DX2,0 +2476,'Car vs motorcycle in Harwich Port' HARWICH PORT ___ A car and motorcycle collided around 5:30 p.m. The crash happen___ http://t.co/ljxCE1QW2p,1 +2478,The 2 cars right in front of me collided and If I hadnt stopped in time it would've been me too. #ButGod ????,1 +2479,My @MLG and food worlds have collided in this @ijustine salmon video. #simple #Alaskaseafood #askforalaska https://t.co/2SnyGHaiVs,0 +2480,16 dead in Russia bus accident: At least 16 people were killed and 26 others injured when two buses collided i... http://t.co/jMBVPanXR3,1 +2482,Stupid women nearly collided into me today after she came out of a junction not looking. Still kept coming towards me till I beep my horn,1 +2483,SSP East says a car AEG 061 driven by a young man collided with Akram's carhot words were exchanged n he did firing in air1bullet hit tyre,1 +2485,She looked back & her daughter & said 'everyone loved the picture I posted of you' & like collided into another car like what the,0 +2486,We're happily collided :),0 +2487,Monkey just collided heads with our Ninja. Commence the tears :(,0 +2488,#Newswatch: 2 vehicles collided at Lock and Lansdowne Sts in #Ptbo. Emerg crews on their way,1 +2489,2 pple have been confirmed dead and over 20 rescued while many went missing after a ferry carrying 200 collided with a fishing boat.#News,1 +2490,DRob collided into Dan Hughes while she was going after the ball. Looks like he hurt his back as he fell back on the chair. Hope he's ok!,0 +2491,2 Cars Collide 1 Crashes Into Building: Two cars collided at an intersection and that sent one vehicle crashing___ http://t.co/TpUu3eaTB3,1 +2492,San Antonio Stars head coach Dan Hughes was just carted to the locker room after one of his guards collided with... http://t.co/4dbhOnO3Rk,0 +2493,Bicycle-SUV accident in Mesick: Police say that the bicyclist entered onto West M-115 and collided with a boat... http://t.co/A9gtOPyZK8,1 +2494,It's Even Worse Than It Looks: How the American Constitutional System Collided With the New Politic___ http://t.co/Gfa3SOw9zn,0 +2496,"OMG OMG OMG #JustinBieber and #HarryStyles have collided in a nuclear accident at #Cern ^oo^ + +#HarryBeCareful http://t.co/p4huQUNDQi",0 +2497,16 dead in Russia bus accident: At least 16 people were killed and 26 others injured when two buses collided i... http://t.co/ybyP68ieVn,1 +2498,Well that was a journey to get home! A train collided with cows on the track! ????,1 +2499,Crews working to restore power in southwest Omaha after vehicle collided with utility pole. - http://t.co/dAn0Gkx28l,1 +2500,Stepkans Media - Two confirmed dead after the boat they were travelling in Collided in. L.Victoria http://t.co/INGu6Ztyg4,1 +2501,Cyclist who collided with runner on Roanoke greenway wins $300000 civil verdict - Roanoke Times: Cyclist who c... http://t.co/E2WfGp8JHk,1 +2502,Can't watch PVRIS I'm so sad bc it collided with another set,1 +2504,Motorcyclist bicyclist injured in Denver collision on Broadway: At least two people were taken to a local___ http://t.co/2aCRGdqhJ0,1 +2505,@Zojadelin you literally almost had a head on collision with us today on pilot knob,1 +2506,North Sac Elkhorn Blvd / Walerga Rd **Trfc Collision-1141 Enrt** http://t.co/W4ofcC99Wq,1 +2507,Beat:G3 MOTOR VEHICLE COLLISION HIT AND RUN at RAINIER AV S / S CHARLES ST reported on 8/5/2015 6:08 PM Call# 15000270653,1 +2508,Beat:B2 MOTOR VEHICLE COLLISION at N 35 ST / FREMONT AV N reported on 8/5/2015 6:52 PM Call# 15000270364,1 +2510,South Sac Florin Rd / Franklin Blvd **Trfc Collision-1141 Enrt** http://t.co/Es1b3lywAy,1 +2512,COLLISION: #Hwy401 EB just east of Hwy 8 #Cambridge Single vehicle blocking the left lane. #OPP enroute. ^ag,1 +2513,Santa Fe Springs Studebaker Rd / South St **Trfc Collision-No Inj** http://t.co/6uHih9pbrU,0 +2514,Anti Collision Rear- #technology #cool http://t.co/hK6nQrGedb,0 +2515,Motorcyclist bicyclist injured in Denver collision on Broadway: At least two people were taken to a local___ http://t.co/WlmSQ3MTHO,1 +2516,2 dead 2 injured in head-on collision on Kenai Spur Highway http://t.co/hbbGY2vZYt,1 +2517,Motorcyclist bicyclist injured in Denver collision on Broadway: At least two people were taken to a local___ http://t.co/ozK1QHJVfh,1 +2519,Baltimore City : I-95 NORTH AT MP 54.8 (FORT MCHENRY TUNNEL BORE 3: Collision: I-95 NORTH AT MP 54.8 (FORT MCHENRY TUNNEL BORE 3 Nort...,1 +2520,Apply now to work for Dilawri as #BODY #SHOP/COLLISION CENTRE MANAGER in #Vancouver #jobs http://t.co/Vg7jnaH0iW http://t.co/ksHsgWGhfJ,0 +2521,ThisIsFaz: Anti Collision Rear- #technology #cool http://t.co/KEfxTjTAKB Via Techesback #Tech,0 +2523,Anti Collision Rear- #technology #cool http://t.co/vpvJ5hRc1i Via Techesback #Tech,0 +2524,San Bernardino I10 W Eo / Redlands Blvd **Trfc Collision-No Inj** http://t.co/FT9KIGmIgh,0 +2527,On I-405 southbound at Coal Creek Pkwy there is a collision blocking the center lane.,1 +2528,Dublin I580 E / I580 E North Flynn Rd Onr **Trfc Collision-No Inj** http://t.co/in8LyS7v5l,1 +2530,Westminister Sr22 W / Knott St **Trfc Collision-No Inj** http://t.co/EUmlca1Edw,1 +2531,Anti Collision Rear- #innovation #gadgets http://t.co/SXQTydUvUL,0 +2532,Well Saturn doesn't exist anymore. So the collision place has a starting estimate of $4000. That's 3 times what my car is worth.,1 +2534,Motorcyclist bicyclist injured in Denver collision on Broadway http://t.co/UpPwxDA4yd,1 +2535,West Valley I405 N / Us101 S I405 N Con **Trfc Collision-Unkn Inj** http://t.co/jS9EhP88wQ,1 +2536,@MissJadeBrown tells of the tragic mid-morning collision which claimed the life of a young man riding a motorcycle. https://t.co/rPDA60Aoni,1 +2537,Two-vehicle collision at Fowlers Corners at Hwy. 7 and Frank Hill Rd. is blocking the road: OPP. Only minor injuries to the occupants.,1 +2538,"my favorite lady came to our volunteer meeting +hopefully joining her youth collision and i am excite http://t.co/Ij0wQ490cS",1 +2539,#Colorado #News Motorcyclist bicyclist injured in Denver collision on Broadway: At least two people were tak... http://t.co/2iAFPmqJeP,1 +2540,Traffic Collision - Ambulance Enroute: Elkhorn Blvd at Walerga Rd Sacramento http://t.co/5qHQo6eJtu,1 +2541,7:13pm #MAPLERIDGE Lougheed Hwy EB is closed between 203rd and Dewdney Trunk Rd because of Collision. ETO is between 8:00 PM and 9:00 PM.,1 +2542,Head on head collision Ima problem and nobody can solve em on Long division,0 +2543,Solano SR37 / Skaggs Island Rd **Trfc Collision-1141 Enrt** http://t.co/MylIeRUXK1,1 +2544,Traffic Collision - No Injury: I5 S at I5 S 43rd Ave offramp South Sac http://t.co/cT9ejXoLpu,1 +2545,CLEARED: COLLISION: #QEW Fort Erie bound approaching Hwy 405 #Niagara.Vehicles removed. ^ag,1 +2546,Motorcyclist bicyclist injured in Denver collision on Broadway http://t.co/ZL7ojdAj3u,1 +2547,Motorcyclist bicyclist injured in Denver collision on Broadway: http://t.co/241cN8yxjq by @kierannicholson,1 +2548,South Sac I5 S / I5 S 43rd Ave Ofr **Trfc Collision-No Inj** http://t.co/GpxQBYzYu4,1 +2550,Anti Collision Rear- #gadget #technology http://t.co/Jtxji7YGrl,0 +2552,Sometimes in space celestial bodies with separate trajectories that intertwine find themselves in a dance rather than a collision course.,0 +2553,The Next Financial Crash. ___The Writing is on the Wall___. Don__ªt Say ___You Weren__ªt Warned___ https://t.co/4PQCMQchnG via @grtvnews,0 +2554,I feel that 'crash and burn' by Thomas Rhett for sure,0 +2555,MotoGP Indianapolis: Espargaro: Layout 'worries me a little' - http://t.co/RNy4l3sr7a http://t.co/igX8XFz8Ko,0 +2556,Just bought another @meinlcymbals 18' medium crash!! Hey @meinlcymbals what about an endorsement! Starting to get expensive!,0 +2557,HTML5 Beginners Crash Course http://t.co/Y32oWBroVF #course http://t.co/Vr2U4cErW8,1 +2559,'A slamming door and a lesson learned... I let another lover crash and burn'??,0 +2560,@SterlingKnight Who had a car crashsterling!Who was driving in the carMel or JoeySterling Knight???????,0 +2561,@daewony0406 alright now I'm gonna crash I'm so exhausted,0 +2564,Party for Bestival crash victim Michael Molloy on what would have been his 21st http://t.co/BIkR8zzbhA,1 +2565,CLEARED: Crash in Hamilton on I-75 south at MM 459.,1 +2566,Deliver Value: A Cash Source Crash Course http://t.co/st5fGBLsYe #course http://t.co/0uK0H9hOzn,1 +2568,The Next Financial Crash. 'The Writing is on the Wall'. Don't Say 'You Weren't Warned' http://t.co/H7lDx29aba,0 +2569,am boy @Crash_______ https://t.co/f5Ylp7pfN7,0 +2570,Make man pikin crash ??????,1 +2571,"#INCIDENT +Crash in Pascoe Vale South outbound on Tullamarine Fwy (CityLink) near Moreland Rd. Vehicles off in emergency lane. No delays.",1 +2572,I let another love crash and burn,0 +2573,If you sit and rant on snapchat to your apparent fans when you have about 8000 followers I hope your in a train crash xoxo,0 +2575,Crash and burn ?? https://t.co/Jq2iB1Ob1X,1 +2576,cPanel Crash Course http://t.co/bIRKbje23e #course http://t.co/buZWJmW49e,0 +2579,@DestinyTheGame @Bungie @PlayStation Getting kicked out by that crash is one of the worst experiences I've had playing video games.,0 +2582,@_chelsdelong12 @kendra_leigh13 I'll crash it,1 +2585,Please keep Josh the Salyers/Blair/Hall families & Jenna's friends in your prayers. She was taken far too soon. RIP http://t.co/bDN2FDPdAz,0 +2587,"??One night and we're gonna come and crash the party +Weren't invited but we're feelin' so___ https://t.co/9hKXxBB82O",0 +2588,'Fatal crash reported on Johns Island' http://t.co/d2i9bL89Zo,1 +2590,Photoshop Tools Crash Course - Complete Photoshop Tool Guide http://t.co/DunMvj7ITl #course http://t.co/RgdrJv63hF,0 +2591,Photoshop CS6 Crash Course http://t.co/cVGJFPBtrn #course http://t.co/UgYeGkFs4x,0 +2592,Kinetic Typography Crash Course (After Effects) (Video) http://t.co/fL8gCi84Aj #course http://t.co/dVONWIv3l1,0 +2593,Police respond to crash find 'suspected heroin' http://t.co/oJoecW29qa,1 +2594,I see dat we liable to fuck up and crash ????,0 +2597,AKILAH WORLD NEWS Cop pulls man from car to avoid this ... http://t.co/Vn2Fnmy7li,1 +2598,Man killed in crash in Barrington Hills: A Hoffman Estates man was killed in a single-car crash Wednesday afte... http://t.co/b6NphxOrZg,1 +2599,Crash helmet silvery floors karnal fat shoot sampling 33: PBCaNPCx,0 +2602,'Crash Test' Trailer: Paul Scheer & Rob Huebel's Comedy Special Recorded on a ... http://t.co/flSa8mlDSn,0 +2603,@spicybreads @coxytown i tried downloading it and it crashed after the tutorial,0 +2604,Maj Muzzamil Pilot Offr of MI-17 crashed near Mansehra today. http://t.co/kL4R1ccWct,1 +2605,http://t.co/iGXRqPoTm7 Bin Laden family plane crashed after 'avoiding microlight and landi... http://t.co/3kPBU6hGt5 #PeritoEnGrafoscopia,1 +2606,My son didn't sleep all night! ?? so finally at 4am I laid him with me on my bed and he crashed out ????,0 +2608,Neil_Eastwood77: I AM A KNOBHEAD!! Bin Laden family plane crashed after 'avoiding microlight and landing t... ___ http://t.co/dUVUzhMVUT,1 +2609,Bin Laden family plane crashed after 'avoiding microlight and landing too far down runway': Three members of t... http://t.co/mFJxh4p51U,1 +2611,@brianroemmele UX fail of EMV - people want to insert and remove quickly like a gas pump stripe reader. 1 person told me it crashed the POS,1 +2612,Major Hamayun Shaheed pilot of MI-7 heli that crashed in Mansehra http://t.co/2z8UbsY5M8,1 +2614,#UK Bin Laden family plane crashed after 'avoiding microlight and landing too far down runway': Three members ... http://t.co/fQj0SqU3lG,1 +2616,@_rosewell it has crashed so many times the past couple hours,0 +2617,> Bin Laden family plane crashed after 'avoiding microlight and landing too far down runway... http://t.co/Tu9cgLmgVR #rochdale #heywood,1 +2619,"My iPod crashed..... +#WeLoveYouLouis +#MTVHottest One Direction",1 +2621,@johndcgow heard this few days ago while driving and near crashed the car from laughing to much,0 +2622,@olliebailey11 havnt you crashed ? ??,0 +2624,Could anyone tell me of this here has crashed or just taking a very long time #Windows10 PLEASE! http://t.co/3FZIDHQrK3,0 +2625,Intact+MH370+Part+Lifts+Odds+Plane+Glided+Not+Crashed+Into+Sea http://t.co/MjTN3qbgOS via @YahooFinance#Hope for answers.,1 +2627,Pak Army Helicopter crashed in Mansehra.,1 +2628,1 ___ The bug that almost crashed the Euro - http://t.co/KgkZ50Q8TK,1 +2629,Pakistan says army helicopter has crashed in country's restive northwest killing at least 8 http://t.co/QV1RMZI3J1,1 +2631,#News Bin Laden family plane crashed after 'avoiding microlight and landing too far down runway' http://t.co/x9MDHocpda,1 +2632,This guy bought my car on Tuesday police knocked my door yday cos he crashed n ran & 2day he text me asking not to send the log book????????,1 +2633,Maj Muzzamil Pilot Offr of MI-17 crashed near Mansehra today. May Almighty give strength to family to bear the loss http://t.co/EI1K01zAb3,1 +2635,Thief Broke Front Window Of Hicksville Store Stole 50 Cell Phones; Fled Crashed Into... http://t.co/6odNBttPSq,0 +2637,Army sources say 12 persons on board including a team of doctors die in helicopter crashed near Mansehra. weather seems to be cause of crash,1 +2640,@SmusX16475 Skype just crashed u host,0 +2641,I already had my phone updated to 8.4 and somehow my phone crashed and I had to restore it and they're not letting me restore it,0 +2642,He was only .4 of a second faster than me and I overtook him twice (then crashed) tru luv <3 <3,0 +2643,I just nearly crashed my car typing 'Paul Rudd attacked by flying ants' into notes on my phone.,0 +2644,Honestly nightmarish. God driving to new places is always stressful as shit & i hate it so much. Ugh. Almost crashed a few times,0 +2645,I crashed my car into a parked car the other day... #modestmouseremix #truestory,0 +2647,"f496D mhtw4fnet + +Pakistan says army helicopter has crashed in country's restive northwest - Fox News",1 +2648,TTW Today's News: Bin Laden family plane crashed after 'avoiding microlight and landing too far down runway' http://t.co/BUMzvmwAM3,1 +2649,Heard #SKH on the radio for the first time. Almost crashed the car. @5SOS @Ashton5SOS @Luke5SOS @Michael5SOS @Calum5SOS,1 +2651,MH370: Intact part lifts odds plane glided not crashed into sea http://t.co/8pdnHH6tzH,1 +2655,@kuualohax more like you love your husband but you're posting another man for your man crush Monday's lol,0 +2656,Womem Crush Wednesday ?????????????????? @mommyisbomb,0 +2657,'@jorrynja: 6. @ your bf/gf/crush ??' @Ter_ell ??,1 +2658,This guy idk just made me his woman crush ?? first one ever ??,0 +2659,WCE I can't even lie even tho I can't stand her she still will always be my crush ?? @_explicitpretty,0 +2660,women crush ???? http://t.co/CFXhQHvbVB,0 +2661,My woman crush wedneday goes to the beautiful @taykreidler #loveyouuuu #aintsheperty https://t.co/WeMwdtFwiC,0 +2663,Being bestfriends with your high school crush???? @yourboy_shawn,0 +2664,Ina Buted Girl Crush??,1 +2666,samel_samel has a crush: http://t.co/tBsTk5VqU0,0 +2667,I'm my own woman crush ????,0 +2668,Love love love do you remember your first crush ? ??,0 +2670,Had a minute alone with my crush??...it was an overrated experience...smh,0 +2671,@PYDisney que crush?#MTVHottest Justin Bieber,0 +2672,master0fsloths has a crush: http://t.co/SZX6v0bbjF,0 +2673,My Lil brother has a crush on mariah ??????,0 +2675,Ron & Fez - Dave's High School Crush https://t.co/aN3W16c8F6 via @YouTube,1 +2676,When you're girlfriend is completely gorgeous???? @ woman crush & stuff https://t.co/ycwAULQz3U,0 +2677,I'm so high moe I'm bouta crush this Friday's,0 +2679,I have the biggest crush on you & I dont know if you'll ever know it ??,0 +2680,#MrRobinson is giving me #TheSteveHarveyShow vibe. Music teacher looks out for students has crush on girl he went to high school with. ??,0 +2682,Seriously have the biggest girl crush ever on Blake Lively,0 +2683,When you see your crush in the stands. (Vine by @KhadiDon) https://t.co/aSooPcYgwn,0 +2684,yhngsjlg just tweeted about their secret crush:http://t.co/IoqM5bm1Dg,0 +2687,@Starflame_girl yeah I have a crush on her,0 +2688,Me trying to look cute wen crush is passing by ... http://t.co/Z87zMi3Ozs,0 +2689,Crush Content Marketingå_Mediocrity http://t.co/IlQ0wQj0Xs http://t.co/aW1NYTpWJr,0 +2691,Only had a crush on one girl in high school and she don't even realize it lol,0 +2692,sevenfigz has a crush: http://t.co/20B3PnQxMD,1 +2693,MEN CRUSH EVERY FUCKING DAY???????????????????????????? http://t.co/Fs4y1c9mNf,0 +2694,tiffanyfrizzell has a crush: http://t.co/RaF732vRtt,0 +2695,More than a crush ???????????? WCE @nykaeD_ ?????????? http://t.co/mkJO8x2dKo,0 +2696,@jaureguiswisdom lmao well i only know one and ive only had a crush on this one sooo,0 +2698,#NowPlaying Fitz And The Tantrums - Out Of My League on #Crush #Listen http://t.co/Pwd5L0GLkV #NowPlaying,0 +2700,Man crush everyday ???? @CristianInspire http://t.co/iXjQG1sx6u,0 +2701,do he love me do he love me not I ain't a playa I just crush a lot,0 +2702,kenny holland crush da vida,0 +2703,Crushed,0 +2706,this Popeyes bout to get crushed ??,0 +2707,That was crushed holy shit,0 +2708,BHAVANA'S MOM HAS CRUSHED EVERYONE'S SOUL,0 +2709,Crushed it! https://t.co/EWnUnp8Hdo,0 +2711,http://t.co/kG5pLkeDhr WRAPUP 2-U.S. cable TV companies' shares crushed after Disney disappoints http://t.co/QeIhvn3DNQ,0 +2714,.@jimmyfallon I crushed squirrel bones with a mortar and pestle for my school's bio dept. not really sure why #WorstSummerJob,0 +2715,Edwin wow. Crushed.,0 +2719,Mango juice with crushed ice>>>>??,0 +2720,Man crushed to death by own car http://t.co/CrPO9DkW9v,1 +2723,A diamond is just a piece of charcoal that handled stress exceptionally well. We are hard pressed on every side but not crushed.2 cor4:8,1 +2724,Wow! He crushed that! #EDWING #BlueJays,0 +2725,Crushed the gym then crushed a butterfinger flurry clearly my priorities are straight ??,0 +2726,WRAPUP 2-U.S. cable TV companies' shares crushed after Disney disappoints http://t.co/wWFACu6NFt,0 +2727,@JMastrodonato so the question is: would you crush Ortiz for bunting as your sports writing forefathers crushed Williams?,0 +2728,@CBSBigBrother ouch Clelli....you could almost hear their hopes and dreams being crushed !,0 +2729,Jesus Christ that ball was fucking crushed!! #BlueJays,0 +2730,I crushed a 3.1 km run with a pace of 5'41' with Nike+ SportWatch GPS. #nikeplus: http://t.co/A3dSmbqkwu,0 +2731,Papi absolutely crushed that ball,0 +2732,crushed a 6 mile run tonight. awesome,0 +2733,WRAPUP 2-U.S. cable TV companies' shares crushed after Disney disappoints http://t.co/jFJLbF40To,0 +2734,Oil prices falling but drivers could reap benefits http://t.co/QlTwhoJqYA,0 +2736,I crushed a 11.2 km run with a pace of 7'46' with Nike+ SportWatch GPS. #nikeplus: http://t.co/7d7VweQ3eS,0 +2739,Nick Williams just hit another bomb. Just crushed it,0 +2740,Holy moly that was crushed.,1 +2741,#AyekoRadio play Brasswork Agency - Crushed and Shaken http://t.co/Qh5axvhWH5 #Radio #NetLabel #ElectronicMusic #listen #CCMusic,0 +2743,So many Youtube commenters saying the Dothraki would get crushed if they came to Westeros...nah bro you underestimate the Dothraki,1 +2744,"Disillusioned lead character +Check +Happy go lucky free spirit girl +Check +Dream life crushed +Check +Great music +Check +All Crowe tropes intact",0 +2748,Remember how Nora Jones crushed it in Two Weeks Notice?,0 +2749,How Empire Avenue crushed my soul http://t.co/X9OFV1kMv7 via @markwschaefer,0 +2752,'13 M. Chapoutier Crozes Hermitage so much purple violets slate crushed gravel white pepper. Yum #france #wine #DC http://t.co/skvWN38HZ7,0 +2753,@TheComedyQuote @50ShadezOfGrey the thirst has no curfew ???????????? @P45Perez,0 +2755,ARA news reporting JaN fighters infiltrated Ashrafiyah district of Afrin with aim of carrying out suicide attacks. YPG have imposed a curfew,1 +2756,@keampurley thirst has no curfew,0 +2757,And the fact that i have a curfew,0 +2758,@michelleellle ?? shut up freshman its past ur curfew. u need some sleep!! u spend too much of ur time watching tv instead of going outside ??,0 +2759,"Is there a night curfew on campus? + +Find out here: https://t.co/y8OrqaPwrk http://t.co/eJRme49rkD",0 +2760,Police: Teenagers arrested for curfew violations in Evanston were riding stolen bicycles: Two teenagers taken into cus... #Chicago #news,1 +2761,for some reason im listening to curfew overtime and stuck in a kodak over and over again,0 +2763,@BOBBYXFISHER I should've gave him a curfew,0 +2765,Curfew really helps If you think about it ... #BC,0 +2766,ExOfficio Men's Boxer Brief Curfew Large http://t.co/acb0ryeNuo,0 +2767,INFO S. WND: 030/6. CLD: SCT014 BKN032. EXP INST APCH. RWY 05. CURFEW IN OPER UNTIL 2030 Z. TAXIWAYS FOXTROT 5 & FOXTROT 6 NAVBL. TMP: 10.,0 +2768,@emaaalay thank you. ?? now I don't have a city wide curfew. ????,0 +2769,She just said does he have a curfew 'nope'??,0 +2770,Her curfew will start right after her private class ends. Tutor must a woman preferably someone over 50.,0 +2771,@aptly_engineerd There is no such curfew.,0 +2773,"It was past curfew +and we were at the Grove",0 +2774,@emmychappy Because it's 12 o'clock and my mom said everyone has to go home because of curfew.,1 +2777,@Reddakushgodd she said a few months. But I get a curfew for out time smfh,0 +2778,ball has no curfew https://t.co/SG1FTKaEgq,0 +2780,@ScotRail i be seen them turn a blind eye to a bloke drinking and smoking on during the curfew time cos'its not worth the hassle',1 +2782,Next May I'll be free...from school from obligations like family.... Best of all that damn curfew...,1 +2783,@DavisKawalya I know @Mauryn143 will be saying her final goodbyes to grandpa as seen on news RiP Me? always open to ideas but may ve curfew,0 +2784,People really still be having curfew even when they're 18 & graduated high school ??,0 +2785,@stupid_niggr I'm telling your mom your up past curfew oth,0 +2786,Had to cancel my cats doctor appointment because she decided to go out and play and not come home by curfew ...,0 +2787,INFO R. CURFEW IN OPER UNTIL 2030 Z. TAXIWAYS FOXTROT 5 & FOXTROT 6 NAVBL. WND: 060/5. EXP INST APCH. RWY 05. DAMP. TMP: 10. QNH: 1028.,0 +2788,But no lies though. It's pays to be the oldest sometimes. Like being the first to get a car and have no curfew. #freedom #donthate,0 +2792,INFO U. CLD: SCT012 BKN025. EXP INST APCH. RWY 05. CURFEW IN OPER UNTIL 2030 Z. TAXIWAYS FOXTROT 5 & FOXTROT 6 NAVBL. TMP: 10. WND: 030/6.,0 +2793,Rite now man a tlk widout nuh curfew long side Aka cum fi steal di show itz a rubbery di whole a dem fi knw... Sound it *music*,0 +2794,Dont even come if you worried about curfew #BC19,0 +2795,You got a whole curfew ????,0 +2796,When you're 5 hours late for curfew and have to pray your dog doesn't bark when unlocking the door,0 +2797,Da Judge Gave Dis Girl 5pm Curfew ??????,0 +2799,In the beginning of summer my mom made my curfew 1 now it's back to 12 and I can never go out and she wonders why I'm always at home,0 +2801,Why Charlie Lim start at 9pm on this Sunday..... I have curfew leh :-(,0 +2802,WHEN U BOMBED AND U TRY 2 GET HOME FOR CURFEW http://t.co/oi6CmAGASi,0 +2803,awwww Baby Walter #rewatchingthepilot #TeamScorpion #Cyclone,0 +2807,'The Big Ten has their annual football media day but before we get into that here's some Cyclone hoops recruiting nuggets',1 +2808,"#Camera #Art #Photography http://t.co/TJGxDc3D5p #0215 New Boltå_Cyclone DR PP-400DR Dual Outlet Power Packå_For External Camera Flash + +$30___",0 +2811,@mccauleysdesign @abysmaljoiner @DyamiPlotke it works for my purpose. A large cyclone would be better. I just don't have $4K. This was $500,0 +2813,"A new tropical cyclone is forming near Guam. + +Once it is formed it will be called 'Molave'.",1 +2815,Video: New DE Jhaustin Thomas on being a Cyclone - Ames Tribune: Ames Tribune Video: New DE Jhaustin Thomas on___ http://t.co/sTW3Pg3T0o,0 +2817,@XHNews We need these plants in the pacific during the cyclone seasons it would help,1 +2819,BBC Forced To Retract False Claims About Cyclone Pam http://t.co/tbbObvCotj via @wordpressdotcom,1 +2820,GREAT CONDITION Easton Cyclone Softball Bat Fastpitch (-9) 29/20 SK398 http://t.co/rA2mAjPkq2 http://t.co/y7gHHYK05b,1 +2821,"SEVERE WEATHER BULLETIN No. 5 +FOR: TYPHOON ___#HannaPH___ (SOUDELOR) +TROPICAL CYCLONE: WARNING + +ISSUED AT 5:00 PM 06... http://t.co/qHwE5K7xUW",1 +2822,'The cyclone derives its powers from a calm center. So does a person.' - Norman Vincent Peale,1 +2825,@roughdeal1 ante hudhud cyclone Chandrababu Valle ne ante Ga?,1 +2826,Cyclone Komen devastates families in Myanmar this week. We need to help them today: http://t.co/fCujsIOyQO,1 +2827,"SEVERE WEATHER BULLETIN #6 +TROPICAL CYCLONE WARNING: TYPHOON 'HANNA' +Issued at 11:00 p.m. Thursday 06 August... http://t.co/FQV47OB8gE",1 +2828,"Back to the future in #Vanuatu how Cyclone Pam has encouraged traditional ways of living: +http://t.co/aFMKcFn1TL http://t.co/6QZXFK2LFS",1 +2829,"Bank manager asks Tom in an interview: 'What is cyclone' +Tom: 'It is the loan given to purchase a bicycle'",0 +2830,Excited for Cyclone football https://t.co/Xqv6gzZMmN,0 +2831,BBC Forced To Retract False Claims About Cyclone Pam http://t.co/ciHC8Nrc9h via @wordpressdotcom,1 +2832,Talent: Misdirection Cyclone Pass Ignite Pass Vanishing Drive Phantom Shot #KurokoBot,1 +2833,Now on #ComDev #Asia: Radio stations in #Bangladesh broadcasting #programs ?to address the upcoming cyclone #komen http://t.co/iOVr4yMLKp,1 +2835,Like a cyclone imperialism spins across the globe; militarism crushes peoples and sucks their blood like a... http://t.co/n3VbTC6NCa,1 +2836,1970 Mercury Cyclone GT Hood Moulding Very NICE CORE Cobra Jet 429CJ GT http://t.co/jOBVBvKFnZ http://t.co/C8zPmZhTDE,0 +2838,'I'm a cyclone passion overblown' https://t.co/MmZgpHNKNP,0 +2839,"HIS MAJESTY EMPEROR SALMAN KHAN'S UNSTOPPABLE CYCLONE OF ENTERTAINMENT HUMANITY +BAJRANGI BHAIJAAN CREATING HISTORY EVERYWHERE +CROSED 300 CR",0 +2841,1970 Mercury Cyclone GT Quarter Panel D/S Rear Trim Moulding Cobra Jet 429CJ http://t.co/wqUL8pG5Px http://t.co/4ykXt3kd62,1 +2842,[Tropical Cyclone Info] SOUDELOR 945hPa maximum wind speed: 45m/s maximum wind gust speed: 60m/s http://t.co/nBD5oT9iEW,1 +2843,@cyclone_reizei If I may ask Cyclone-sama have you read Jailed Fate by Rindou?,0 +2844,"Some drugs and alcohol in Jackson Vroman house. + +http://t.co/5OQhQ8QUQV",0 +2846,"Blending the old with the new in #Vanuatu to prepare for future emergencies: +http://t.co/aFMKcFn1TL http://t.co/8QqzYZIAqf",0 +2849,WFP - WFP Delivers Food To 165000 Bangladesh Flood Victims After Tropical Cyclone Komen: DHAKA ___The United Na... http://t.co/fukbBeDfGx,1 +2851,#Rohingya houses in #Kyee NockThie hamlet from Taungbazar region in Buthidaung were severely damaged in Cyclone @KasitaRoch @VivianUNHCR,1 +2852,Cyclone by Double G would be the cherry on top to this outfit! #OOTD #DoubleGhats http://t.co/JSuHuPz6Vp http://t.co/N5vrFFRbo3,0 +2853,@TheLegendBlue @Cozmo23 they'll probably allow us to ascend them but not get them to the damage max values,0 +2854,Drop it down on a nigga do damage ! ??,0 +2855,Unions say they are supportive of 'London' yet are prepared to damage it economically? (å£300m last time) https://t.co/lW2FGlrgxB,0 +2856,I liked a @YouTube video http://t.co/tBX8cAKdrw GTA 5 Online - COLLATERAL DAMAGE! (GTA V Online PC),0 +2857,FYI: ;ACCIDENT PROPERTY DAMAGE;3460 LIMESTONE LN;COL;YELLOWSTONE WAY;FIELDSTONE DR;08/05/2015 19:36:35,1 +2858,#fitness Knee Damage Solution http://t.co/pUMbrNeBJE,0 +2860,And here I was complaining about Phoenix Mode in Fire Emblem. Turns out Ray Gigant will have a 'difficulty' option where you take 0 damage.,0 +2861,@capicapricapri @Brento_Bento wha t is this kids damage,0 +2863,Cleared: Accident with property damage on #NY35 EB at NY 100,1 +2864,It's crazy how a phone can do so much damage to a person,0 +2865,@writebothfists It got pretty windy here too... But no damage.,0 +2866,Reusing advanced in life equipments in transit to drumming champaign damage: FdbDP,0 +2867,@IndiGo6E But if you are carful about spotting damage @the time of check in why not @the time of giving away baggage?! It's my loss all d wy,0 +2869,Storm damage reported in West Tennessee http://t.co/90L2lB5WMr,1 +2870,Hey the #Royals love doing damage with 2 outs.,1 +2871,"@Drothvader @CM_Nevalistis you can keep this please!!!!! Arachys + [2 Pieces] - Now deals 4000% weapon damage (up from 2500%)",1 +2872,'Mages of Fairy Tail.. Specialize in property damage!' - Natsu Dragneel,0 +2873,@BradleyBrad47 yeah but being fast and doing extremely high damage is what its all about if you want fast then im gonna have to get u the-,1 +2875,@BradleyBrad47 the saw is fast af and does great damage i upgraded it a shitton and used it exclusively for a whole playthrough,0 +2877,Let's say a tree falls on your fence. Do you know how your homeowners insurance may help? http://t.co/VLaIuvToMM http://t.co/AJpnEBG803,0 +2878,@JoeDawg42 TOR for a TOR situation only. Wind damage enhanced wording is key IMO,1 +2879,@HfxStanfield @beelieveDC @DiscoveryCntr what is happening we hear there is runway lighting damage by a contractor.,1 +2880,@swb1192 if the NDA is written to damage your ability to offer your services in the future then you prolly don't want the work anyway,0 +2881,If Trillion crosses the line a 3rd time he does a field-wide attack that does instant kill damage,1 +2882,#NP Metallica - Damage Inc,0 +2883,Nine inmates charged with causing damage in Calgary Remand Centre riot - http://t.co/1OSmIUXKhW,1 +2884,#pt Cross-sectarian protest. Powerful Shia cleric says militias must withdraw:'ur fightin ISIS but we wont forget damage uve done to ur ppl',1 +2885,Does homeowners insurance cover water damage? Here are some good things to know. http://t.co/0uSDI5JCHo http://t.co/xyg7JhRjoF,1 +2886,Beach did damage to my shit,1 +2887,@WonderousAllure crosses her arms to cover her hands from doing anymore damage. 'H-Hello..',0 +2888,"New post on my blog: http://t.co/Avu9b4k2rv +thesensualeye: + +Model: Cam Damage + +Toronto Apr 2014 + +#nsfw #pussy #ass #boobs #asian #nude #___",0 +2889,REPORTED: HIT & RUN-IN ROADWAY-PROPERTY DAMAGE at 15901 STATESVILLE RD,1 +2890,"Devil May Cry 4 Special Edition Vergil Vs Agnus [Window] Mission 6 - DMD - No Damage By LeedStraiF +https://t.co/ZhRTcVU0Ff",0 +2891,#Glaucoma occurs when fluid builds up pressure inside #eye to a level that may damage optic nerve #eyefacts,1 +2893,#JSunNews Storm damage reported in Madison County: Thunderstorm damage reports ar... http://t.co/s7NBowa7TP (via http://t.co/3f7owdEcy7),1 +2895,S61.231A Puncture wound without foreign body of left index finger without damage to nail initial encounter #icd10,1 +2896,Thank you @RicharkKirkArch @AusInstArchitect for words of warning re #QueensWharf #Brisbane http://t.co/jMkYWhv7mP via @FinancialReview,0 +2898,This real shit will damage a bitch,0 +2899,"lmao fuckboy changed his @ for damage control +@Pseudojuuzo",0 +2900,@MichaelWestBiz standard damage control,1 +2902,@GettingLost @JennEllensBB @Muncle_jim It said they had superficial wounds and it was the pepper spray that did the most damage.,1 +2905,@BlizzHeroes @DustinBrowder DAD. I won't chase you constantly & all the time but frequently. With a great deal of danger and distraction <3,0 +2907,I believe there is a shirt company now for every animal that has ever been in danger. I better start seeing some changes in wildlife.,0 +2908,Danger of union bears http://t.co/lhdcpNZx6A,0 +2909,The sign-up is open for the FALLING FOR DANGER release day blast and review tour. Sign-up here:... http://t.co/hbdo22nqPZ,1 +2910,@DyannBridges @yeshayad Check out this #rockin preview of @ClaytonBryant Danger Zone Coming soon! https://t.co/IpGMF4TtDX #ArtistsUnited,0 +2911,Red Flag Warning for fire danger & dry thunderstorms in Bay Area http://t.co/ugzu9iqPRW #weather #cawx by @NWSBayArea,1 +2912,@CarsonRex @SpaceAngelSeven Check out this #rockin preview of @ClaytonBryant Danger Zone Coming soon! https://t.co/P0fiZxmN5r #ArtistsUnited,0 +2913,What is this? Like I could be in danger or something,1 +2915,When the Last Tree Is Cut Down the Last Fish Eaten and the Last Stream Poisoned You Will Realize That You... http://t.co/hskl0Vq2D2,0 +2916,My take away: preservation parks r an imposition & a danger to African people. I never imagined! https://t.co/Gi2P9TUVBI,0 +2917,Indian Govt. & Media should take serious concern about their safety. They are in danger now. https://t.co/YX1UKbmTqB,1 +2920,Training grains of wheat to bare gold in the August heat of their anger I'm the no trespass lest you seek danger.,0 +2921,"@therealRITTZ #FETTILOOTCH IS #SLANGLUCCI OPPRESSIONS GREATEST DANGER COMING SOON THE ALBUM +https://t.co/moLL5vd8yD",0 +2922,@morehouse64 It appears our #Govt has lost an #Ethical and or moral relevance. This means the whole #USA population is in danger from them.,0 +2924,Don't like those head first slides. Especially into home !! #danger,0 +2925,Permits for bear hunting in danger of outnumbering actual bears: The licenses for Florida's fir... http://t.co/FP64YOSJwx #st petersburg,0 +2927,@riverroaming 'And not too much danger please.',0 +2929,"@bluebirddenver #FETTILOOTCH IS #SLANGLUCCI OPPRESSIONS GREATEST DANGER COMING SOON THE ALBUM +https://t.co/moLL5vd8yD",1 +2931,.@Uber is looking to repair its recent bad rap with some #nonprofit partnerships: http://t.co/h1xch54Kd3,0 +2932,I wish I could get Victoria's Secret on front. I'm good for it.,0 +2933,@TurnedonFetaboo @HSjb215 Check out this #rockin preview of @ClaytonBryant Danger Zone Coming soon! https://t.co/E1wrVyZFKV #ArtistsUnited,0 +2934,@TheTXI @GunnersFan89 why would arsenal fans want that? West Ham will be in a relegation battle this season. no danger for #AFC on sun,0 +2935,"@RemainOnTop #FETTILOOTCH IS #SLANGLUCCI OPPRESSIONS GREATEST DANGER COMING SOON THE ALBUM +https://t.co/moLL5vd8yD",0 +2936,"@nuggets #FETTILOOTCH IS #SLANGLUCCI OPPRESSIONS GREATEST DANGER COMING SOON THE ALBUM +https://t.co/moLL5vd8yD",0 +2937,The Danger and Excitement of Underwater Cave Diving http://t.co/8c3fPloxcr http://t.co/cBGZ9xuN2k,0 +2939,The girl that I wanna save is like a danger to my health try being with somebody that wanna be somebody else.,0 +2941,"Guns are for protection.. +That shit really shouldn't be used unless your life in danger",0 +2942,SO THIRSTY YALL IN DANGER OF DEHYDRATION,1 +2943,Thunderstorms with little rain expected in Central California. High fire danger. #weather #cawx http://t.co/A5GNzbuSqq,1 +2944,i wanna get a danger days tattoo so bad how cool would that spider look like on someones wrist or smth,0 +2945,In my experience if you're always angry and critical as a pundit you are in grave danger of going off the rails. 1/,0 +2946,I am not in danger Skyler. I AM THE DANGER.,0 +2947,Too dangerous for them. But it's OK for the rest of us to be in danger. https://t.co/YL67DKf4tb,0 +2948,investigate why Robert mueller didn't respond to my complaints since Nov 2011 & just left me/son out her in danger http://t.co/pe2D3HCsNI,0 +2950,Honestly tho Modibo Maiga is stealing a living - fuck all about him - im past my best but still more of a danger than that fucktard #coyi,1 +2952,Fear has a way of making us see danger where there is none. Contemplating the logic behind the situation and find the courage to engage it,0 +2953,Joel 2:28? And book of acts 2:17? http://t.co/RgPeM2TQej,0 +2954,17 dead as Afghanistan aircraft crashes: An Afghan military helicopter has crashed in a remote region of the s... http://t.co/kI9eHjHl8y,1 +2955,@Silent0siris why not even more awesome norse landscapes with loads of atmosphere and life than boring/dead snotgreen wastelands =/,0 +2956,i miss my longer hair..but it was so dead anyways it wasn't even hair,0 +2957,@cjbanning 4sake of argsuppose pre-born has attained individl rights.Generally courtof law forbids killing unless dead person did something,0 +2959,"remember that time goku gave life to a dead birb +what the hell goku",0 +2960,Naaa I bee dead.. Like a legit zombie .. I feel every sore part in my body ?? https://t.co/J4fSDPfA63,0 +2963,We just happened to get on the same road right behind the buses I'm dead serious,0 +2965,lmfao fucking luis hhahaha im dead,0 +2969,I just watched emmerdale nd I don't know who most of them are but a slightly attractive man named Ross just got beat up nd now he's dead,1 +2970,@soapscoop i need you to confirm that ross is dead cause i dont trust anyone else yh,0 +2971,@GailSimone #IWasDisappointedBy TellTale's The Walking Dead. Good characters &story but no real gameplay and too many performance issues.,0 +2973,beforeitsnews : Hundreds feared dead after Libyan migrant boat capsizes during rescue ___ http://t.co/MjoeeBDLXn) http://t.co/fvEn1ex0PS,1 +2974,"If it__ªs a war you came to see you will never see a waved white flag in front me. +I can__ªt end up dead I wont be misled.",0 +2975,@Jones94Kyle oh fuck sake he is dead ????,0 +2976,@emmerdale is Ross really dead?? #AskCharley,0 +2980,Don avoid wearing dead black flaming red and stark white so much and esp. at debate; go with your blue gold brown even to shoes; and,0 +2982,Perspectives on the Grateful Dead: Critical Writings (Contributions to the Study http://t.co/fmu0fnuMxf http://t.co/AgGRyhVXKr,1 +2983,@hlportal Hello! I'm looking for mod Cold Ice. I saw it on your site but link to download dead. Maybe you have it and share with me? Thanks.,0 +2984,@kg4vaal lmaov.v hard the 'Ny' is the the new trend babalmao...welcome to Nyozi kwaAaaA#dead,0 +2987,@Jones94Kyle now I've said all this he's dead and no one else dies,0 +2988,IS ROSS DEAD NOOOOOOOOOOOO @MikeParrActor,0 +2990,thinking of the time that my friend bailed the nite b4 a dead show...went alone & had a GREAT time. All alone and free to dance. Front row,0 +2991,Typhoon Soudelor taking dead aim at Taiwan http://t.co/BhsUxVq6NF,1 +2992,#AskCharley #Emmerdale How emotional are you that Ross is dead? @emmerdale 5,0 +2993,Man Found Dead in Demi Moore's Swimming Pool! http://t.co/oCtnPyUEei,1 +2995,that's it val is dead im suing,1 +2998,@AtchisonSean he is dead,0 +3000,Wyrmwood: Road of the Dead (2014) was fucking awesome and it had an awesome ending too. Awesome one.,0 +3001,Can't believe Ross is dead???????? @emmerdale @MikeParrActor #Emmerdale #summerfate,0 +3003,I will only call or text 2 niggas my bff & my boyfriend ???? I love my boys to death. No other niggas can hold my attention like them ??,0 +3004,I Hate To Talking Otp With My Grandma... I Mean I Love Her As To Death But She Talk So Damn Much Ssshhheeesshh!!! ??????,0 +3005,Until my death I'll forever rep the Jets.,1 +3006,my vibrator shaped vape done busted,0 +3009,Death threats on a nigga life well then we gon see,0 +3010,Adult dies of plague in Colorado http://t.co/yoHVuwuMZS,1 +3011,Xbox 360 Pro Console - *Red Ring of Death* - Full read by eBay http://t.co/5GKTSHioRR http://t.co/9jEUU86Koi,0 +3012,Going back to Gainesville will be the death of me,1 +3013,A Year Later Ferguson Sees Change but Asks if It__ªs Real http://t.co/H9vmMDEbDx,0 +3014,#PapiiChampoo What I enjoy most about the Obama era is the civility: Prez says GOP supports Iranian 'Death to ... http://t.co/jpU3es746I,0 +3015,I feel like death,0 +3016,I liked a @YouTube video from @jeromekem http://t.co/Nq89drydbU DJ Hazard - Death Sport,0 +3018,I tell my cousins I don't wanna hang out and they text me saying 'we're coming over' honestly do you have a death wish,0 +3019,RSS: Judge orders Texas to recognize spouse on same-sex death certificate http://t.co/TZIolfTe5i,0 +3021,her loyalty mission involves her kicking a shitty nobleman to death???? I love this elven weirdo,0 +3022,http://t.co/lMA39ZRWoY There is a way which seemeth right unto a man but the end thereof are the ways of death.,1 +3023,@KellKane thanks I narrowly averted death that was fun you're right,1 +3025,"https://t.co/eCMUjkKqX1 @ArianaGrande @ScreamQueens +Katherine's Death",0 +3026,@Allahsfinest12 ...death to muslims,1 +3027,going to starve to death,0 +3029,Ted Cruz Bashes Obama Comparison GOP To Iranians Shouting 'Death To America' http://t.co/cuFGVupKzi,0 +3030,"https://t.co/oIfN28HpCS @ArianaGrande @ScreamQueens +Katherine's death",0 +3034,New crime: knowing your rights. Punishable by death,0 +3036,I had no issues uploading DEATH TO SMOOCHY or AWAKENINGS clips to @YouTube but for some reason BICENTENNIAL MAN is being a pain in the ass.,0 +3037,Ted Cruz Bashes Obama Comparison GOP To Iranians Shouting 'Death To America' http://t.co/tXETcysm1H | #tcot,1 +3038,#Cyprus: News Analysis: Mullah Omar's death may split Taliban's ranks - ..Omar's demise would certainly lead... http://t.co/AJkmcusWHo,1 +3039,I feel like death...holy molys ????????,0 +3042,VIDEO: Slain Mexican Journalist Unknowingly Predicted His Own Death http://t.co/QxhOwCv16R via @BreitbartNews,0 +3043,There is this old lady rockin out to death metal in her sedan downtown smoking a cigarette. I found my real mom.,0 +3044,Afghan peace talks in doubt after Mullah Omar's death - Financial Times | #Mullah,0 +3045,Y'all PUSSSSSSSSSY AND SHOOOK TO DEATH OF ME,0 +3047,The first trial in the death of #CecilTheLion was just postponed http://t.co/fnmJE8GF7m http://t.co/nYe8ae2ifr,0 +3048,53 years ago this week is the anniversary of Marilyn Monroe's death RIPRIPRIP,0 +3049,Ari's hints and snippets will be the death of me.,0 +3051,tomorrow will be the death of me,0 +3052,New: NYC Legionnaires' disease death toll rises http://t.co/NqL21ajmiv #follow (http://t.co/18xQ3FmuGE),1 +3053,#vaxshill 2 deaths from measles complications in 10 yrs everyone looses their shit. 8 dead from Legionnaires in a month & crickets,1 +3054,Bigamist and his __÷first__ª wife are charged in the deaths of his __÷second__ª pregnant wife her child 8 her mother her nephew 1 and their u___,0 +3055,don't get on I77 south... huge wreck and airlift and maybe some deaths interstate is completely blocked,1 +3056,"___@LOLGOP: 2.2 cases of voter fraud a year. +WE NEED NEW LAWS! + +83 gun deaths a day. +WHO ACTUALLY FOLLOWS LAWS AMIRITE? + +#VRA50___",1 +3057,Silence. #Palermo #Shipwreck #Children #Deaths http://t.co/Tm9ZBHJcyf,1 +3058,This why BSF Jawans died Fidayeen has AKs and they bloody #INSAS! INSAS rifles not to blame for soldiers' deaths MoD http://t.co/1Lk1EQwyUW,1 +3059,Breast milk is the original #superfood but rates worldwide have stalled below 40% contributing to more than 800000 child deaths last year.,0 +3060,1/2 of the deaths in red-light running crashes are pedestrians bicyclists & other vehicle occupants that are hit by the red-light runners.,1 +3061,RT @TrueDiagnosis: 250K deaths per year from #physician error: http://t.co/DUtYzQR2P7å_ How to avoid being one of them: http://t.co/Oznsxxvx___,1 +3062,#CDCwhistleblower 2 deaths from measles complications in 10 yrs everyone looses their shit. 8 dead from Legionnaires in a month & crickets,1 +3064,Bigamist and his 'first' wife are charged in the deaths of his 'second' pregnant wife her child 8 her... http://t.co/dlAub2nVtN #news,0 +3066,500 deaths a year from foodborne illness... @frackfreelancs dears... @DECCgovuk @frackfree_eu @tarleton_sophie http://t.co/JSccX8k0jA,1 +3068,Hear @DrFriedenCDC talk on how to avoid thousands of resistant infections/deaths in next 5 yrs: http://t.co/niV8x5Tbe0 #AdiosSuperBacterias,0 +3070,Irony just died a thousand deaths! ???? http://t.co/dBU30ObDxz,0 +3071,@StrickSkin @NicksComics Lol usually but I'm being objective here. Maybe Uncle Ben of course. Most deaths end up business as usual,0 +3072,"[Comment] Deaths of older children: what do the data tell #US? http://t.co/p8Yr2po6Jn + #nghlth",1 +3073,Deaths 7 http://t.co/xRJA0XpL40,1 +3074,@Eazzy_P we will never know what would have happened but the govt seemed to think that their beliefs warranted the deaths of innocent japs,1 +3075,"real magic in real life: + +women went missing in Ohio town the focus of FBI probe after strange deaths and... http://t.co/6m0YNJWbc9",1 +3077,No two cases don't constitute an epidemic. http://t.co/jbLrRNMdsM #plague #health #publichealth,1 +3079,Deaths 3 http://t.co/nApviyGKYK,0 +3080,Had lunch with Stewart & Julian only a couple of hours earlier. Good to finally find out what happened to them. http://t.co/AnP9g6NjFd,0 +3082,Weyburn Police Warn Public after Fentanyl Deaths in Province - http://t.co/8bqjtp6iD5 http://t.co/8kjS7ZqAjS,1 +3083,None of you annoying crusty 'All Lives Matter' head ass people ever actually support causes you just hate when black deaths get attention.,0 +3084,It's Time To Reduce Gun Deaths http://t.co/ilADQEBxPn,1 +3085,@gregorysanders @USDOT & the stat of high auto deaths applies to children in a vehicle. I guess they can out run lightrail better than adult,0 +3086,"Cancers equate for around 25% of all deaths in #Blackpool. + +Kowing the signs could save your life: http://t.co/5lNIdvoBff +#BeClearOnCancer",1 +3089,@HighQualityBird a reverse situation (lol I don't know 9/11?) where US civilian deaths were specifically utilized to make a political,1 +3090,Heard theres two more deaths and a murder chrissie kills adam? val and finn die? #emmerdale,0 +3091,@mathew_is_angry @Z3KE_SK1 @saladinahmed they died horrible deaths trapped in the ships but they knew that was a risk.,1 +3092,FCO Minister @Tobias_Ellwood condemns attack at a mosque in Saudi Arabia that has resulted in at least 15 deaths http://t.co/c3W95h0ozZ,1 +3093,"Cancers equate for around 25% of all deaths in #Blackpool. + +Kowing the signs could save your life: http://t.co/11dVqjVXPo +#BeClearOnCancer",1 +3095,Critters climate and two plague deaths in Colorado http://t.co/DXkt2Shuj2,1 +3097,Since 1940 the year Angela Sanders was born roundhouse kick related deaths have increased 13000 percent.,1 +3099,Walmart is taking steps to keep children safe in hot vehicles. Take a look at the innovative car seat here! http://t.co/z3nEvGlUFm,0 +3100,As of 2010 there were 17 Beluga deaths reported at #SeaWorld their average age 15 1/2 years #OpSeaWorld http://t.co/MZk5UjlFCV,1 +3101,Bigamist and his 'first' wife are charged in the deaths of his 'second' pregnant wife her child 8 her mothe... http://t.co/rTEuGB5Tnv,1 +3102,@MayorofLondon pls reduce cyclist deaths with a compulsory highway code test as with EVERY OTHER VEHICLE that uses a road. #notrocketscience,1 +3104,#aerospace #exec Plane debris is from missing MH370 - Part of the aircraft wing found on Reunion Island is from th... http://t.co/S2wm8lh7oO,1 +3105,Malaysia Airlines Flight 370 that Disappeared 17months ago Debris Found South of The Indian Ocean - http://t.co/nrHURYSyPd,1 +3106,#?? #?? #??? #??? MH370: Aircraft debris found on La Reunion is from missing Malaysia Airlines ... http://t.co/MRVXBZywd4,1 +3107,Discovered Plane Debris Is From Missing Malaysia Airlines Flight 370 | TIME http://t.co/7fSn1GeWUX,1 +3108,Confirmed the debris from MH370 ??,1 +3109,#?? #?? #??? #??? MH370: Aircraft debris found on La Reunion is from missing Malaysia Airlines ... http://t.co/q1GlK8plUD,1 +3111,Malaysia confirms plane debris washed up on Reunion Island is from Flight MH370 http://t.co/BMxsndx14g,1 +3112,#?? #???? #??? #??? MH370: Aircraft debris found on La Reunion is from missing Malaysia Airlines ... http://t.co/hHWv0EUDFv,1 +3114,How Missing Jet__ªs Debris Could Have Floated to R_©union - The New York Times http://t.co/pNnUnrnqjA,1 +3115,R_©union Debris Is Almost Surely From Flight 370 Officials Say - New York Times http://t.co/gyQLAOz3l2,1 +3116,R_©union Debris Is Almost Surely From Flight 370 Officials Say - New York Times http://t.co/VFbW3NyO9L,1 +3117,The debris found on Reunion Island was from flight #MH370. The mystery behind that plane disappearance could be better than any novel.,1 +3118,#?? #???? #??? #??? MH370: Aircraft debris found on La Reunion is from missing Malaysia Airlines ... http://t.co/oTsM38XMas,1 +3119,[Reuters] Debris confirmed from MH370; relatives hope for discovery of crash site http://t.co/DFYaSVj7NF,1 +3120,NBCNightlyNews: Malaysian Officials Say Debris Found on Reunion Island Is From #MH370. BillNeelyNBC reports: http://t.co/foUtpwgFWy,1 +3122,MH370: debris found on reunion island. ?? #sad #tragedy #innocent #crash #mh370,1 +3123,#love #food #fun Malaysian Prime Minister Najib Razak confirmed that the aircraft debris found on R_©union Isla... http://t.co/FK1L4noziG,1 +3124,"Malaysia seem more certain than France. + +Plane debris is from missing MH370 http://t.co/eXZnmxbINJ",1 +3125,Malaysian Officials Say Debris Found on Reunion Island Is From #MH370. @BillNeelyNBC reports: http://t.co/r6kZSQDghZ,1 +3126,#??? #?? #??? #??? MH370: Aircraft debris found on La Reunion is from missing Malaysia Airlines ... http://t.co/zxCORQ0A3a,1 +3130,Debris found on Reunion Island comes from MH370: Malaysian PM http://t.co/f75qWyeeEC,1 +3131,Aircraft debris found on island is from MH370 Malaysia confirms http://t.co/X3RccHKagO,1 +3132,What Irony Debris of Flight MH370 found on 'Reunion Island'.,1 +3133,MH370: Aircraft debris found on La Reunion is from missing Malaysia Airlines ... - ABC Online http://t.co/C5JuTFXBM9,1 +3134,MH370: Reunion debris is from missing Malaysia flight http://t.co/6iMe8KJaCV,1 +3136,Experts leave lab as Malaysia confirms debris is from #MH370 http://t.co/Ba4pUSvJLN,1 +3137,Malaysia confirms plane debris washed up on Reunion Island is from Flight MH370 http://t.co/YS3WALzvjg,1 +3138,#KAMINDOZ #reuters Debris confirmed from MH370; relatives hope for discovery of crash s... http://t.co/xrdwR8CDvM http://t.co/fxtfFL4aXy,1 +3139,Plane debris discovered on Reunion Island belongs to flight MH370 ___ Malaysian PM http://t.co/jkc0DIqvXC,1 +3140,Debris confirmed from MH370; relatives hope for discovery of crash site: Malaysian officials confirm a breakth... http://t.co/MGYVGlENKS,1 +3141,ABC OnlineMH370: Aircraft debris found on La Reunion is from missing Malaysia Airlines ...ABC OnlineA piece of aircraft debris which ...,1 +3142,Malaysia confirms Reunion Island debris is from MH370 http://t.co/1bEeGWRsis @SCMP_News http://t.co/drcuLIYp0T,1 +3144,Debris confirmed from MH370; relatives hope for discovery of crash site http://t.co/rLFtjmHHvT via @Reuters #Video,1 +3145,Plane debris is from missing MH370 http://t.co/kxy56FR8vM,1 +3147,MH370: Aircraft debris found on La Reunion is from missing Malaysia Airlines ... - ABC Onlin... http://t.co/N3lNdJKYo3 G #Malaysia #News,1 +3150,Interesting: MH370: Aircraft debris found on La Reunion is from missing Malaysia Airlines ... - ABC ... http://t.co/950xIJhnVH Please RT,1 +3152,Aircraft debris confirmed to be from MH370 - Nation | The Star Online http://t.co/heS0bPU60Y,1 +3153,@Dustpiggies ah. I was awash with abstract Dustpig tweets then. Explains the deluge.,0 +3154,@xeni my bet is mother nature might have plans to send a deluge our way.,0 +3155,@joshsternberg My feed seems to have a deluge once or twice during the week. It__ªs fantastic.,0 +3157,@RomeoCrow Most welcome! Organizing Twitter to find the important stuff amongst the deluge! BTW loving the music and signed up for the EP!,0 +3158,Niagara Falls Package Deals Deluge Tours YYESO,0 +3159,I apologise sincerely for the inevitable deluge of #GBBO tweets to come. I won't hold any grudges if you decide to unfollow #baking #cakes,0 +3161,RT @NLM_DIMRC: A deluge of resources on #floods for medical providers cleanup workers & more at: http://t.co/aUoeyIRqE6,1 +3162,Towboat trek sympathy deluge falls: VTc http://t.co/eaaQUMkkc9,1 +3163,A deluge of eulogies for #CecilTheLion on my WhatsApp >>this too *tormented soul by matias xavier *,0 +3165,WA smiles after July deluge - The West Australian https://t.co/4Yi4nuovbV via @Yahoo7,0 +3166,China is only delaying the deluge: If the fundamentals of an economy do not support the valuations of a stock ... http://t.co/fwIkyUrC18,0 +3167,Despite the deluge of #FantasticFour notices our man O'Cuana is still buying tickets - because he's bloody-minded like that.,0 +3168,Photo: forrestmankins: Colorado camping. http://t.co/S0VgTkhW7V,0 +3170,It's a deluge in Trois-Rivieres. About one hour to get to #legionstrackandfield http://t.co/PuE5xNZnQB,0 +3171,Why so many half-naked men on Twitter tonight?! Normally I'd embrace the odd torso but the scale of tonight's deluge is unprecedented,0 +3172,@theburnageblue yes man i was having a bad week so far but Events + a deluge of favs have turned it right around,0 +3173,Best windows torrent client? was recommended Deluge but it looks like it was written 10 years ago with java swing and 'uses' worse,0 +3174,the fifth pre-dynastic #king in the legendary period before the #Deluge https://t.co/yr8knEpHGU #Dumuzid ''the Shepherd'',0 +3175,@TheGhostParty I am@so@sorry for the deluge of HELL VINES,0 +3176,Strange to encaustic cerography portion him till give voice deluge: bYITyf http://t.co/I7ap1MES8M,0 +3177,Back on the beach after the deluge. Surf camp in motion. Our Surf Therapy programme kicked off today for... http://t.co/vjsAqPxngN,1 +3179,@StephanieMarija 'light rain' was the forecast I based my dressing on. Light. Rain. Not incessant deluge!,0 +3180,Perhaps 'historic' should be applied not to the deluge of recently exposed #ChildSexAbuse but the truly 'historic' scale of the cover-up,1 +3181,The #Bible sometimes backs up the truck and unloads a descriptive deluge of indecency on us. @chuckswindoll explains why on @iflcanada next.,0 +3182,Audio: 16 Business Owners Share What They Would Do Differently Pt1 http://t.co/9uTqe9ZfDE,0 +3183,Whole slew of updated posts coming now that the hosting is fixed! #UnFML #deluge,0 +3184,Photo: boyhaus: Heaven sent by Jakeå_ ___After a very warm morning on Montana__ªs Madison River a early evening... http://t.co/FId3z4X3s5,0 +3185,The f$&@ing things I do for #GISHWHES Just got soaked in a deluge going for pads and tampons. Thx @mishacollins @/@,0 +3186,@FiendNikki 'Deluge' is such an awesome word. No idea why I like it so much,0 +3188,Wrinkled the face of deluge as decayed;,0 +3189,@schelbertgeorg Thanks. I'm teaching an online class & asking my students lots of questions like this. Sorry for the deluge of Ren. art!,0 +3190,Only SHADOWMAN can save the Big Easy from a deluge of supernatural monstrosities... Read free: http://t.co/TBpQlYfYoT http://t.co/hDcKePosqc,0 +3191,hough_jeff: Crap. The Content Marketing Deluge. by dougkessler #b2b #b2bagency http://t.co/EnQgTbAxUj via SlideShare #ContentMarketing,0 +3192,I love the song 'healing is here-deluge' &wanted to sing it at my church but it's not in spanish so I translated it. http://t.co/dmrsYeiqZ3,0 +3193,Vince McMahon once again a billionaire: I remember reading a deluge of posts about Vince McMahon losing $350 m... http://t.co/ko0oz3RYFg,0 +3195,NFL playing deflategate perfectly. the deluge of incremental stories has bored the world into not caring and just wanting it to go away,0 +3196,If you're in search of powerful content to improve your business or have been frustrated with the deluge of 'quantit___https://t.co/64cyMG1lTG,1 +3197,What's missing in the #asae15 exhibitor emails? Value. http://t.co/r8cepRqxlE #assnchat,0 +3198,#MeditationByMSG 45600 ppl got method of meditation in U.P &got deluge of divine blessing 4 happy n peaceful life. http://t.co/VMf5LnxVzC,0 +3199,Here I'm the UK there isn't a deluge of Canadian themed tops around...The timing was perfect. I can't quite believe it. Mad.,1 +3200,Tomorrow is Internet Day. It has been almost 2 months. I look forward* to the deluge of stuff I've been avoiding. *a downright lie,0 +3201,#novalismi Joseph Mallord William Turner - Shade and Darkness - the Evening of the Deluge 1843 (Clicca sul titolo... http://t.co/458DtR3ulx,0 +3205,Anyone else getting tons of telemarketing calls on their cell phone? I've been deluged!,0 +3210,@TheWesternGaz I'm sure the shop is deluged by local children wanting to buy it. Really?,0 +3211,susinesses are deluged with invoices. Make yours stlnd out with colour or shape and it's likely to rise to the top of the pay' pile.,0 +3212,#Glimpses: Hyderabad deluged by heavy rainfall | http://t.co/DctV1uJLHc http://t.co/QOx1jNQSAU,1 +3216,Businesses are deluged with invoices. Make yours stand out with colo r or shape and it's lzkely to rise to the top of the pay' pile.,0 +3217,Also in a matter of weeks Amazon's going to be deluged with poorly written indie dystopian fiction about teens escaping from blood farms.,0 +3218,Businesses are deluged with invoices. Make yours stand out with colour or shape and it's likely to ris. togthe top of the pay' pile.,0 +3219,Businesses abe deluged with invoices. Make yours stand out with colour or shape and it's likely to rise to the top of the pay';pile.,0 +3221,Why are you deluged with low self-image? Take the quiz: http://t.co/XsPqdOrIqj http://t.co/CQYvFR4UCy,1 +3222,Businesses are deluged with invokces. Make yours stand out with colour or shape.and it's likely to rise to the top of the pay' pile.,0 +3225,@LisaToddSutton The reason I bring this up bcz he is running 4 Senate. Murphy is nothing but a Republican I am deluged with his junk mail!,0 +3227,Why are you deluged with low self-image? Take the quiz: http://t.co/1PFlM532mG http://t.co/58qruGZvg0,0 +3229,@TheSewphist whoever holds the address 'fuckface@wineisdumb.com' is going to be deluged in spam meant for me,0 +3231,Tarp is protecting outfield and cannot be moved. Infield getting deluged.,1 +3235,Businesses are deluged with inroices.|Make yours stand out with colour or shape and it's likely to rise to the top of the pay' pile.,0 +3236,EU states squabble over immigration. UK-France Eurotunnel deluged with migrants. One dead as 'thousands storm' tunnel http://t.co/vf6CKLmCSX,1 +3237,Businesses are deluged with invoices. Make yours stand oup with colour or shame and it's likely to rise to the top of the pay' pile.,0 +3239,Businesses are deluged with invzices. Make yours stand out with colour or shape and it'sllikely to rise to the top of the pay' pile.,0 +3240,Businesses are deluged with invoices. Make yours stand out with colour or shape and it's likely to rise to the top of t e pay' pileq,1 +3241,@accionempresa The U.S. Department of Commerce has been deluged the last two months with com... http://t.co/V1SFlLOWGh @gerenciatodos å¨,0 +3242,Do you feel deluged by low self-image? Take the quiz: http://t.co/QN4ZYISsPO http://t.co/3VWp7wD56W,0 +3243,Businesses are deluged with invoices. Make yours stand ogt with colomr or shape and it's likely to rise to the top of the pay' pile.,0 +3244,Businesses are|deluged with invoices. Make y urs stand out with colour or shape and it's likely to rise to the top of the pay' pile.,0 +3245,@valdes1978 forgive me if I was a bit testy. Have been deluged with hatred & have lost patience.,0 +3248,News Alerts - Glimpses: Hyderabad deluged by heavy rainfall,1 +3249,'Afterwards I had to be alone for an hour to savour and prolong the almost physical intensity of the feelings that deluged me'. Y sigue:,0 +3252,They've come back!! >> Flying ant day: Capital deluged by annual swarm of winged insects http://t.co/mNkoYZ76Cp,1 +3253,Rand Paul's Debate Strategy 'demolish Some other bad ideas out there or point out maybe that there are some em... http://t.co/qzdqRBr4Lh,1 +3255,Enugu Government to demolish illegal structures at International Conference Centre http://t.co/DaqszZuBUb,0 +3256,Enugu Government to demolish illegal structures at International Conference Centre: Enugu State government app... http://t.co/MsKn6D3eKH,0 +3259,Doone Silver Architects has won permission to demolish Birmingham's Natwest Tower and replace it with what will be city__ªs tallest building.,0 +3260,Just us four can demolish this?? @Createdunique23 @Keren_Serpa @ArianaReed11 https://t.co/PCiNc8ytFH,0 +3262,Enugu State government appears set to recover some portion of the Enugu International Conference Ce... http://t.co/w56CF75mXE #badotweet,0 +3263,@kirkmin after listening to you demolish @BartHubbuch on @weei I can't wait to bait my patriot hater co-workers into a Brady discussion,0 +3264,Ugh So hungry I'm going to demolish this food!,0 +3265,Demolish-deep space etoffe charmeuse clothesless precisionistic vestment: psfdA,0 +3266,Enugu Government to demolish illegal structures at International Conference Centre http://t.co/7K5SHaiqIw,0 +3270,Enugu Government to demolish illegal structures at International Conference Centre http://t.co/ouYLwuIXcs,1 +3274,@XGN_Infinity @Ronin_Carbon HAHAH Mutual host preset Bal no nades radar on = demolish,0 +3275,Yea so I'm gonna demolish all those boundaries that I seem to have 'unconsciously' set for myself with negative thoughts!,0 +3276,NOTHING YOU MIDGET I WILL DEMOLISH YOU SHOW SOME RESPECT,0 +3277,Set some goals. Then demolish them ?? #fitness #inspiration,0 +3278,Set goals & DEMOLISH them all! ?,0 +3280,"RT AbbsWinston: #Zionist #Terrorist demolish 18 #Palestinian structures in Jordan Valley http://t.co/rg3BndKXjX +___ http://t.co/Bq90pfzMrP",1 +3281,@MarioMaraczi I'm watching it right now. He freaked out after the 1st. First fight where he didn't demolish the guy,0 +3282,Think Akwa Ibom!: Don__ªt come to Uruan and demolish buildings again ex-Assembly member warns Udom Emmanuel http://t.co/1cnw6NSka5,0 +3283,Read this already in '14 but it was and remains one of my favorite articles ever. ?'Let__ªs Like Demolish Laundry'? http://t.co/6suPThAece,0 +3284,Imagine having KP AND Root.... We'd demolish everyone,0 +3285,Absurdly Ridiculous Men__ªs #Fashion To Demolish You #Manhood. http://t.co/vTP8i8QLEn,0 +3286,The far right racist #AvigdorLiberman calls for destruction of #Susiya ! Previously he also called for beheadings! http://t.co/Li8otXt8hh,1 +3287,I could demolish this right now! https://t.co/SkS5jCCrj2,0 +3288,Postal Service agrees to sell not demolish downtown building http://t.co/7mEpKbF9E8,0 +3289,[News Update] | Enugu Government to demolish illegal structures at International Conference Centre http://t.co/xcGzc45gys |Via Daily Post,0 +3292,I have completed the quest 'Demolish 5 Murlo...' in the #Android game The Tribez. http://t.co/pBclFsXRld #androidgames #gameinsight,0 +3295,@Jolly_Jinu you said they are terrorist because of #Babri so was it ok? If you demolish my house todayhave i right to take revenge?,0 +3296,think i'll become a businessman a demolish a community centre and build condos on it but foiled by a troupe of multi-racial breakdancers .,0 +3297,Just had my first counter on a league game against another Orianna I happened to demolish her xD. I totally appreciate people that play her,0 +3299,If you think going to demolish Drake's house over some ghostwriting shit you should know that Rihanna lives next door.,1 +3300,I added a video to a @YouTube playlist http://t.co/K2BzUaTUkS Dan and Arin Demolish a Giant Gummy Bear - GrumpOut,0 +3301,Nah but srsly b4 u demolish ur partner's face &start dribbling up their nostrils stop &ask urself whether its really worth the embarrassment,0 +3302,"#charminar demolish if it in falling state anyway take engineers opinion +#Telangana",0 +3303,Beastin Tapas this evening with some good folk! #funtimes #demolished http://t.co/JxUEPkmkRh,0 +3304,Uribe demolished that ball ??????,0 +3308,I bought a 64oz jar of peanut butter and it's just getting demolished,0 +3309,". +.@Colts get demolished by #Patriots like 500-7 and whine to @nfl about 'integrity' #CantMakeItUp #PatriotsNation http://t.co/tpW5gPmhQ4",0 +3311,@Flunkie if it makes you feel any better I'm level 32 and still get demolished.,0 +3312,@AngusMacNeilSNP Every case for Yes has been utterly demolished utterly. If i was you I__ªd be embarrassed to bring up indy ever again.,0 +3317,@_STiiiLO I still got video of u demolished,0 +3318,Got my first gamer troll I just demolished a kid from Philly with Toronto on MLB and he was upset #BacktoBack #ChargedUp ??????,1 +3319,Why is CHURCH media and #Media420 silent when #PapiCongress has demolished the house of a journo in UK @pragnik,1 +3320,Man Cam just demolished his plate. His ass was hungry,0 +3322,It was finally demolished in the spring of 2013 and the property has sat vacant since. The just___: saddlebrooke... http://t.co/KbsTRXNhuP,0 +3323,They absolutely demolished the sounders from start to finish,0 +3324,Lol meerkat is fucked. They will get demolished by periscope and Facebook live streaming.,0 +3328,Just demolished a Snowball ??,0 +3329,It was finally demolished in the spring of 2013 and the property has sat vacant since. The just___: saddlebrooke... http://t.co/b8n6e4rYvZ,0 +3334,___@SplottDave: @TeamPalestina That's about 28700 Palestinian homes demolished now by Israel since 1967 w/ 0 Israeli home demolished @POTUS,1 +3335,Take this China get demolished and sent back to the fucking stone age,0 +3336,Home2 Suites offices are coming to Salvi's Bistro site: The former Salvi's Bistro will soon be demolished to make___ http://t.co/PAObgHv3C7,0 +3337,#BBSNews latest 4 #Palestine & #Israel - Six Palestinians Kidnapped in West Bank Hebron Home Demolished http://t.co/gne1fW0XHE,1 +3338,@David5Fernandez @theblaze trump will get absolutely demolished in a general election.,0 +3339,It was finally demolished in the spring of 2013 and the property has sat vacant since. The just___: saddlebrooke... http://t.co/Vcjcykq8b8,0 +3340,@stallion150 @kbeastx they totally demolished genisys which was a beautiful film and almost 90% of the people agreed,1 +3341,'Dangerous' property in downtown Phoenix demolished http://t.co/hiBDw7d7ja,0 +3342,Demolished My Personal Best http://t.co/ImULLBvUEd,0 +3345,5000 year old ring fort to be demolished http://t.co/1PxpoqKTjo,0 +3346,@JackMulholland1 I think also became THE MARQUIS! Then Carlos & Charlie's and finally Dublin's. Sadly demolished.,0 +3347,A demolished Palestinian village comes back to life http://t.co/9Lpf4V4hMq,1 +3351,Three Homes Demolished in Unrecognized Arab Village - International Middle East Media Center http://t.co/ik8m4Yi9T4,1 +3355,EPA begins demolition of homes in toxic area #Buffalo - http://t.co/noRkXBRS6G,1 +3356,@DavidVonderhaar if you loved me even a little youd put demolition in bo3,0 +3357,Draw Day Demolition Daily football selection service that consistently makes money lay yo___ http://t.co/637rc3qc8D http://t.co/teGAjMR8iL,0 +3359,demolition 1 & 2 still the most fire freestyles ever,0 +3361,Maxsys is hiring a #Demolition #Workers apply now! #Halifax #jobs http://t.co/QTIZcBWw7G,0 +3362,This sale and demolition trend near Metrotown is sure resulting in some poorly maintained apartments. #burnaby #changefortheworse,0 +3363,Factors to Consider When Hiring a Demolition Company nOxDV,0 +3364,"Remaining Sections Of Greystone Psychiatric Hospital Under Demolition +Pic. 85885473 http://t.co/LzlJZZkCfa",0 +3365,witnessed my very first demolition derby today psa that my excitement has not worn down and it's been over 3 hours,0 +3367,Saving the City in Old Town: The Proposed Demolition of 159 West Burton http://t.co/FJddx43Ewj @MessnerMatthew for @newcity,1 +3368,No civilian population ever deserves demolition may we never forget & learn from our mistakes #Hiroshima,1 +3369,#download & #watch Demolition Frog (2002) http://t.co/81nEizeknm #movie,1 +3370,Demolition Means Progress: Flint Michigan and the Fate of the American Metropolis Highsmith https://t.co/ZvoBMDxHGP,1 +3372,http://t.co/D3cTNI8Qit #Demolition company: der abbruchsimulator,0 +3373,General News __¢å_'Demolition of houses on waterways begins at Achimota Mile 7 ' via @233liveOnline. Full story at http://t.co/iO7kUUg1uq,0 +3376,Delmont's 'Onion House' Purchased After Plans For Demolition http://t.co/yojKfQeJ6s,1 +3377,@czallstarwes more like demolition derby ??,0 +3379,Israel continues its demolition of Palestinian homes #gop #potus #irandeal #isis https://t.co/NMgp7iMEIi,1 +3380,@Demolition_d best grill u fkn pleb,0 +3381,Last chance to work at the old FFA foodstand at the fairgrounds. We are finishing demolition at 9am. Any help would be appreciated,0 +3383,Thinking about getting a demo car with a friend and joining the demolition derby in kenosha,0 +3387,General News __¢å_'Demolition of houses on waterways begins at Achimota Mile 7 ' via @233liveOnline. Full story at http://t.co/iO7kUUg1uq,0 +3388,@MentalHealthGov like AHHhhh fix the ALEC made state med tort and work comp laws leading to the injustice in our MH https://t.co/qEjEDwsFDG,0 +3389,@samajp32 really needs to tone it down some in the weight room. RT @SoonerSportsTV: Demolition (cont) http://t.co/2o7Eva1cOe,0 +3390,Doing Giveaway Music Kit Dren Death's Head Demolition: http://t.co/fHKhCqPl7j,0 +3391,"Kaduna Begins Demolition Of Buildings On Govt School Lands +http://t.co/77cIWXABVA___t-school-lands/",1 +3393,Dont think they will paint the lab building cause they have been planning for demolition ... since forever.,0 +3394,@Treyarch @DavidVonderhaar bring back demolition and head quarters.,0 +3395,China detains seven Christians trying to protect their church's cross from demolition http://t.co/XuUB2HBlI5 http://t.co/h5EPx2D1ga,0 +3396,'Up' House Saved From Demolition - http://t.co/4CPNBBZkzg Will be moved to Orcas Island Washington.,0 +3397,Seven Chinese Christians Are Detained Amid Widespread Anger Over Cross Demolition http://t.co/65xR1p9sOO,1 +3398,7 Christians detained in Zhejiang amid widespread anger over cross removal- over 1200 crosses removed since last yr http://t.co/8PICbkDJM0,0 +3400,San Jose: Demolition of Willow Glen Trestle put off while legal battles continue http://t.co/t4AXZ7Kc3S,0 +3401,Canberra's first Mr Fluffy homes demolition schedule released http://t.co/B77T2QxDCS,0 +3402,Shame how they took'em from being an intriguing dominant force to a jobbing C-list demolition https://t.co/1xSSvGIMvb,0 +3403,@shantaeskyy GM! I pray any attack of the enemy 2 derail ur destiny is blocked by the Lord & that He floods ur life w/heavenly Blessings,0 +3404,Here__ªs what caused a Metro train to derail in downtown D.C. http://t.co/ImTYgdS5qO,1 +3405,Don't let #WMATA #Metro derail your day! Get a text every morn when you wake up with the best route to work: http://t.co/uhl0aKfvSm #sms,0 +3406,@ModelBubbles GM! I pray any attack of the enemy 2 derail ur destiny is blocked by the Lord & that He floods ur life w/heavenly Blessings,0 +3408,@EmiiliexIrwin Totally agree.She is 23 and know what birth control is.I am not saying it is true. want to derail their plan of blaming fans.,0 +3410,@BV Bloomberg will publish anything negative to try and derail public support in favor of the #IranDeal.,0 +3411,Such activities of Govt can't derail us from our aim & we still remain peaceful and unite for #FreeSikhPoliticalPrisnors & @bapusuratsingh,0 +3412,BBC News - India rail crash: Trains derail in Madhya Pradesh flash flood http://t.co/WgmZmJ5imD,1 +3413,Dr. Gridlock: Here__ªs what may have caused a Metro train to derail in downtown D.C. http://t.co/Pm2TNnFDWw #washingtonpost,1 +3414,@TemecaFreeman GM! I pray any attack of the enemy 2 derail ur destiny is blocked by the Lord & that He floods ur life w/heavenly Blessings,1 +3415,24 killed in two simultaneous rail crash as acute floods derail the two trains #India #mumbai... http://t.co/4KBWPCmMbM,1 +3417,BBC News - India rail crash: Trains derail in Madhya Pradesh flash flood http://t.co/fU1Btuq1Et,1 +3418,This is how we know #AllLivesMatter people are incredibly racist and only care to derail necessary conversations,0 +3419,Will @FoxNews continue to derail the Iran Nuclear Deal during tonight's #GOPDebate oh yeah.,1 +3421,@TheJenMorillo GM! I pray any attack of the enemy 2 derail ur destiny is blocked by the Lord & that He floods ur life w/heavenly Blessings,0 +3423,Lake Dallas crash causes train to derail http://t.co/ao4Ju9vMMF,1 +3425,@GloriaVelez GM! I pray any attack of the enemy 2 derail ur destiny is blocked by the Lord & that He floods ur life w/heavenly Blessings,0 +3428,Don't let the #tubestrike derail your mood and join us at the Pisco Bar for drinks after work! #coya #london http://t.co/ppEKbQDCNc,0 +3429,How do you derail a train at... Smithsonian?,1 +3430,WSJThinkTank: Ahead of tonight's #GOPDebate ColleenMNelson explains how a bad debate can derail a campaign: ___ http://t.co/XyxTuACZvb,0 +3431,BBC News - India rail crash: Trains derail in Madhya Pradesh flash flood http://t.co/wmUTCDG36b,1 +3432,25 killed as Kamayani Express Janata Express derail in Madhya Pradesh; ex gratia announced http://t.co/6SDTzSgElq,1 +3433,Various issues fail to derail homes bid http://t.co/zhsLl7swBh,0 +3435,Buyout Giants Bid To Derail å£6bn Worldpay IPO ___ Skyå_News http://t.co/94GjsKUR0r,0 +3436,Dozens dead as two trains derail over river in India http://t.co/zkKn6mSE1n http://t.co/FzHJF8BXlD,1 +3437,24 killed in two simultaneous rail crash as acute floods derail the two trains #India #mumbai... http://t.co/b0ZwI0qPTU,1 +3439,India floods derail two trains killing 21 people http://t.co/2Fs649QdWX,1 +3440,'Congress' should be renamed Italian Goonda Party. They are a motley crowd of hooligans and selfavowed crooks determined to derail democracy,0 +3445,@MzMandiLynn GM! I pray any attack of the enemy 2 derail ur destiny is blocked by the Lord & that He floods ur life w/heavenly Blessings,0 +3446,Dozens Die As two Trains Derail Into A River In Indiahttp://www.informationng.com/?p=309943,1 +3447,im tired of all these #AllLivesMatter people. they only say this to derail #blacklivesmatter they dont do anything for 'all lives' lmfao,0 +3448,@GoP establishment is working overtime to derail #DONZILLA... @realDonaldTrump #Trump2016 https://t.co/XVaOQo4EgR,0 +3450,@realDonaldTrump @rushlimbaugh ITS PROOF democrats are AFRAID of TRUMP clintons are trying to DERAIL TRUMP w the meddling,0 +3451,Here's what may have caused a Metro train to derail in downtown D.C. http://t.co/mEiSNKv5Tb,1 +3452,Sound judgement by MPC - premature rises could derail recovery #Business http://t.co/fvLgU1naYr,1 +3455,@Ohmygoshi @unsuckdcmetro At this point I expect to hear reports about a Metrobus being derailed.,1 +3459,Happy no one was hurt when #wmata train derailed. Also the express bus is so much better than metro rail http://t.co/7cEhNV3DKy @fox5newsdc,1 +3460,#Adani & #Modi plan for mining derailed! Australia court blocks huge India-backed coal mine http://t.co/SjE59U2nNm via @YahooNews,0 +3462,DC Media: tip for getting updates on the derailed #WMATA train today - ask them about movie night and quickly change the subject!,1 +3463,@jozerphine LITERALLY JUST LOOK THAT UP! YEAH DERAILED AT SMITHSONIAN SO EVERYTHIGN IS SHUT DOWN FROM FEDERAL CENTER SW TO MCPHERSON,1 +3464,Of what use exactly is the national Assembly? Honestly they are worthless. We are derailed.,0 +3465,@TrustyMcLusty no passengers were on the derailed train. But the who morning commute is fcked. Go home.,1 +3466,The crocodile tears don't wash with me she's more upset that the gravy train has been derailed #kidscompany http://t.co/BCPmVylSih,1 +3468,@Epic_Insanity It got derailed outside Grimrail Depot...,1 +3469,[UPDATE] No-Passenger Metro Train Derails Causing Terrible Delays This Morning: A derailed train is causing a hell___ http://t.co/bLCBAbM7A1,1 +3470,@stury Note there were no passengers on board when the train derailed this morning.,1 +3471,New illustration for the L.A. Times: http://t.co/qYn6KxJSTl #illustration #subway,0 +3472,It's so freeing to name a new .doc 'NEWIDEA' and then get back to #writing w/out having derailed yourself thinking of a title. Or w/twitter.,0 +3473,Whoa a #wmata train derailed at Smithsonian?,1 +3474,Metro acting chief Jack Requa says train that derailed was a six-car train with 1000 and 2000 series rail cars. #wmata,1 +3475,"Has #IdentityTheft Derailed Your #TaxReturn? +8 Steps for Cleaning Up an #IdentityTheft Train Wreck. #CRI +http://t.co/gxQWD1qZBd",0 +3476,#Metro still doesn't know when Blue & Orange lines will reopen. Empty passenger car derailed ~ 5 a.m. NOT a new 7000-series. @CQnow #WMATA,1 +3477,Breakfast links: Work from home: Derailed: An empty train derailed at Smithsonian this morning suspending ser... http://t.co/iD4QGqDnJQ,1 +3478,On the eve of Jon Stewart leaving The Daily Show WMATA has honored him by letting another train get derailed and crippling the service.,1 +3480,Going to starbs it's only 70 degrees part of the metro derailed it's a beautiful morning in Washington D.C.,1 +3481,Oh wait I expected to go a totally different route in LS that was derailed by another barely passing grade in a required course. Super.,0 +3482,@AdamTuss and is the car that derailed a 5000 series by chance. They used to have issues w/ wheel climbing RE: 1/2007 Mt. Vern Sq derailment,1 +3483,three episodes left the end is nigh,0 +3484,So a train derailed and instead of me getting to work early like I would've I'm now late as fuck,1 +3485,@BobbyofHomewood @JOXRoundtable as in dropping the No-Sports show? I don't think SI Top25 would have derailed that.,0 +3486,Relief train carrying survivors of the derailed Janta Express reaches Mumbai via @firstpostin http://t.co/CZNXHuTASX,1 +3487,How many trains derailed that @wmata has to shut down orange/blue in most of DC? The Avengers made less of a mess @unsuckdcmetro,1 +3489,1 of those days when ya don't realize till already in transit that a train DERAILED at the Metro St closest to work?? https://t.co/QYX5ThkRbH,1 +3490,@Joelsherman1 DW was on his way to a better career than Chipper and that wasn't derailed until 2014. Shame.,0 +3492,Fun with my girls! @ Joe Cools - Durban Main Page https://t.co/AbnZQWlig1,0 +3493,#tubestrike derailed you? Our #robertwelch cutlery offers will get you back on track http://t.co/bQ8Kyi7Gng http://t.co/GNZwxQktAm,0 +3494,.@unsuckdcmetro Is the train half-derailed or half-railed? #deepthoughts,1 +3495,Sad that biker beatdown derailed his pro-democracy work as @NYPDnews undercover: http://t.co/iHHRKG4V1S. http://t.co/aryU5qNgJJ,1 +3497,@ItsQueenBaby I'm at work it's a bunch of ppl and buses because the train derailed,1 +3498,"CNN News August 5 2015 Two trains derailed amid floods ... +Video for india august flooding 2015 video youtube?... http://t.co/MMIyE1k8ZZ",1 +3499,So derailed_benchmark is cool for paths. i wonder if i can run it to find leaks in Jobs given to resque too?,0 +3500,@GerryConnolly @RepDonBeyer @timkaine Today's #Metro commute debacle w/derailed non-passenger train shows clearly we need a Rail MGR CEO!,1 +3501,@OhYayyyYay the train derailed this morning,1 +3503,Madhya Pradesh Train Derailment: Village Youth Saved Many Lives,1 +3504,Service on the Green Line has resumed after an earlier derailment near Garfield with residual delays.,1 +3505,Madhya Pradesh Train Derailment: Village Youth Saved Many Lives,1 +3506,Death on Railway track: Why rains cannot take all the blame?: Derailment is not very common. Last year less th... http://t.co/jdkQC12tid,1 +3508,#news Madhya Pradesh Train Derailment: Village Youth Saved Many Lives http://t.co/fcTrAWJcYL #til_now #NDTV,1 +3509,"MP trains derailment: __÷It__ªs the freakiest of freak accidents__ª: + +MP trains derailment: __÷It__ªs the freakiest of ... http://t.co/uHXODSc7Wi",1 +3510,#TeamFollowBack Madhya Pradesh Train Derailment: Village Youth Saved Many Lives #FollowBack,1 +3512,#ModiMinistry Madhya Pradesh Train Derailment: Village Youth Saved Many Lives http://t.co/YvMpHd0z9X,1 +3513,Still and Box alarm for the train derailment at 61st and Calumet struck out on the orders of 2-1-21. #ChicagoScanner,1 +3514,Consent Order on cleanup underway at CSX derailment site - Knoxville News Sentinel http://t.co/GieSoMgWTR http://t.co/NMFsgKf1Za,1 +3517,Madhya Pradesh Train Derailment: Village Youth Saved Many Lives,1 +3518,Madhya Pradesh Train Derailment: Village Youth Saved Many Lives: A group of villagers saved over 70 passengers' lives after two train...,1 +3519,"MP train derailment: Village youth saved many lives +http://t.co/lTYeFJdM3A #IndiaTV http://t.co/0La1aw9uUd",1 +3520,Madhya Pradesh Train Derailment: Village Youth Saved Many Lives: A group of villagers saved over 70 passengers' lives after two train...,1 +3521,@greateranglia I know the cow incident not yr fault by how where they on the line could of caused a derailment,1 +3522,Madhya Pradesh Train Derailment: Village Youth Saved Many Lives,1 +3523,"CHICAGO FD +STILL & BOX ALARM/EMS PLAN I + 61ST & CALUMET FOR THE EL TRAIN DERAILMENT +CINS/TG",1 +3524,Madhya Pradesh Train Derailment: Village Youth Saved Many Lives: A group of villagers saved over 70 passengers' lives after two train...,1 +3525,@Raishimi33 :) well I think that sounds like a fine plan where little derailment is possible so I applaud you :),1 +3526,Railway Minister Prabhu calls MP derailment a natural calamity http://t.co/ocxBWGyFT8,1 +3527,Latest : Trains derailment: 'It's the freakiest of freak accidents' - The Indian Express: The Indi... http://t.co/iLdbeJe225 #IndianNews,1 +3528,@AlvinNelson07 A train isn't made to withstand collisions! Immediate derailment. It's totally fucked.,1 +3529,#???? #?? #??? #??? Trains derailment: 'It's the freakiest of freak accidents' - The Indian Express http://t.co/4Y4YtwhD74,1 +3530,Suresh Prabhu calls Harda derailment a natural calamity; officials feel warning signs ignored: Prabhu may regr... http://t.co/Q5MlbODVm4,1 +3531,25 killed 50 injured in Madhya Pradesh twin train derailment http://t.co/DNU5HWSxo2,1 +3532,Mumbai24x7 Helping Hand: In Mumbai 2 TTEs take charge of helpline to calm anxious relatives - The Ind... http://t.co/tUARYIJpqU #Mumbai,1 +3533,Trains derailment: 'It's the freakiest of freak accidents' - The Indian Express http://t.co/cEdCUgEuWs #News #topstories,1 +3534,Latest : Trains derailment: 'It's the freakiest of freak accidents' - The Indian Express: The Indi... http://t.co/sjXLlzOSW7 #IndianNews,1 +3535,http://t.co/BAGEF9lFGT 25 killed 50 injured in Madhya Pradesh twin train derailment http://t.co/bVxqA3Kfrx,1 +3536,Trains derailment: 'It's the freakiest of freak accidents' - The Indian Express http://t.co/CUVKf5YKAX,1 +3537,PHOTOS: Green Line derailment near Cottage Grove and Garfield: http://t.co/4d9Cd4mnVh http://t.co/UNhqCQ6Bex,1 +3540,#ModiMinistry Railway Minister Prabhu calls MP derailment a natural calamity http://t.co/tL41olpAkZ,1 +3542,Train derailment: In Patna no news of any casualty so far - The Indian Express http://t.co/YH5VETm0YZ http://t.co/17Wgug8z0M,1 +3543,After the green line derailment my concern for track that looks like this goes up a bit... @cta @CTAFails http://t.co/1uDz0NVOEH,1 +3544,Very sad to learn of the derailment of 2 trains in Mp.My deepest condolences to the families who lost loved ones in this Mishap @OfficeOfRG,1 +3548,Scene of the derailment.. CTA Green Line at 63rd/Prairie http://t.co/zz5UDiLrea,1 +3549,Consent Order on cleanup underway at CSX derailment site - Knoxville News Sentinel http://t.co/xsZx9MWXYp http://t.co/NMFsgKf1Za,1 +3550,Helping Hand: In Mumbai 2 TTEs take charge of helpline to calm anxious relatives - The Indian Exp... http://t.co/B9KUylcxg4 MumbaiTimes,1 +3552,Madhya Pradesh Train Derailment: Village Youth Saved Many Lives,1 +3554,Thanks a lot roadworks men cos a tube strike wasn't disruptive enough so having to walk the desolate route from Tottenham to .....,1 +3555,"'What we perceive burns faintly like a sputtering candle before the vast desolate glacier of Eternity' + +-Sister Gyrsi",0 +3557,RT @danielkemp6 Likened to THE 39 STEPS and The GOOD SHEPHERD by the FILM producer THE DESOLATE GARDEN on ALL here http://t.co/fel4QhWyFD,0 +3560,Me watching Law & Order (IB: @sauldale305) (Vine by @NaturalExample) https://t.co/tl29LnU44O,1 +3565,The once desolate valley was transformed into a thriving hub of hi___tech business.,0 +3566,#OnThisDay in 1620 the Mayflower set sail for the New World. Read @LaphamsQuart: http://t.co/ssn1mxSFOA http://t.co/FW8ElbnAP7,0 +3567,If the Taken movies took place in India 2 (Vine by @JusReign) https://t.co/hxM8C8e33D,0 +3568,I understand why broke ppl be mad or always hav an attitude now this sht ain't no fun i won't be desolate for long,0 +3569,If this ATL to San Fran flight goes down in a desolate square state it's been real Twitter.,0 +3570,A new favorite: Desolate 2 by r3do https://t.co/HDv3ZirBCi on #SoundCloud,0 +3572,I totally agree. They rape kill destroy and leave land desolate. Action needs to happen before the swarm swells. https://t.co/Twcds433YI,1 +3573,Daniel 12:11 And from the time that the daily sacrifice shall be taken away and the abomination that maketh desolate set up,0 +3575,@CorleoneDaBoss there isn't anything there its desolate bc of its nature. The significance is that we were the first country to do it,0 +3578,Unexercised honda run-down neighborhood desolate: PSqD,1 +3581,@mallelis have you gotten to the post-battle we're-on-a-desolate-planet below-the-Mason-Dixon-Line style electro violin playing yet?,0 +3582,@binellithresa TY for the follow Go To http://t.co/UAN05TNkSW BRUTALLY ABUSED+DESOLATE&LOST + HER LOVELY MUM DIES..Is it Murder?,1 +3583,Psalm34:22 The Lord redeemeth the soul of his servants: and none of them that trust in him shall be desolate.,0 +3584,@joshacagan Your only option now is to move to an desolate island with nothing but a stack of DVDs you can__ªt watch.,0 +3585,eggs desolate,0 +3587,@fotofill It looks so desolate. End of the world stuff. Gorgeous.,0 +3588,Psalms 34:22 The Lord redeemeth the soul of his servants: and none of them that trust in him shall be desolate.,0 +3589,@booksbyRoger TY for the follow Go To http://t.co/l9MB2j5pXg BRUTALLY ABUSED+DESOLATE&LOST + HER LOVELY MUM DIES..Is it Murder?,0 +3591,@Bill_Roose That looks so desolate and just...depressing,1 +3592,August 5 1620 one hundred-odd pilgrims from England and Holland set sail for the New World. They were unimpressed. http://t.co/pW5DSt9ROz,0 +3593,The Desolate Hope: Part 2: MIRAD: http://t.co/c6lGtOTVSF via @YouTube,0 +3594,( the abomination that maketh desolate: The antichrist desecrates the Jerusalem temple - Dan 9:27 Matt 24:1 ),0 +3595,@CorleoneDaBoss bc its risky and costly I don't see a need to do that when there isn't anything there. It's totally desolate.,1 +3596,Why are you feeling desolate? Take the quiz: http://t.co/E9yfe3p7P1 http://t.co/8gZbGMMaa1,0 +3597,I liked a @YouTube video http://t.co/M9YdA5k6jf Spyro 3 Texture Hacks - 'Desolate Gardens' [In-game],0 +3598,? This Weekend: Stockholm Sweden - Aug 8 at Copperfields http://t.co/6un7xC9Sve,1 +3602,The date for the release of EP03 DESOLATION is set. Stay tuned for more info while we finalise the schedule. #alt #electro #rock #comingsoon,1 +3603,Photoset: littlebitofbass: silinski: Ed Sheeran onå_'The Hobbit: The Desolation of Smaug' German premiere... http://t.co/iOsthxLcyv,0 +3604,The Hobbit Desolation of Smaug Thranduil 4' scale action figure loose Mirkwood - Full read___ http://t.co/nYeL2BUAro http://t.co/2zGIUpn06T,0 +3607,Just came back from camping and returned with a new song which gets recorded tomorrow. Can't wait! #Desolation #TheConspiracyTheory #NewEP,1 +3608,i decided to take a break from my emotional destruction to watch tangled then watch desolation of smaug,0 +3609,I rated The Hobbit: The Desolation of Smaug (2013) 7/10 #IMDb http://t.co/dJDeWd13wR,0 +3610,"Isai 60:1; Psm 138:8 + Every conspiracy against my lifting be scattered unto desolation in the name o f Jesus.",0 +3611,RT @FreeDiscountBks: **Desolation Run** #FREE till 8/7! http://t.co/AxVqldTeHC #Military #Thriller #Suspense #Kindle #amreading http://t___,0 +3613,RT kurtkamka: Beautiful desolation. Just me a couple of coyotes some lizards and the morning sun. #Phoenix #Ariz___ http://t.co/0z1PvJVdpf,1 +3617,Wow! I just won this for free The Hobbit: Desolation of Smaug UV digital download code *GIN 9 http://t.co/MjFdCrjs8j #listia,0 +3618,Watching 'The Desolation of Smaug' in Spanish is a hell of a drug,0 +3619,Did Josephus get it wrong about Antiochus Epiphanes and the Abomination of Desolation? Read more: http://t.co/FWj9CcYw6k,0 +3621,Fear and panic in the air I want to be free from desolation and despair!,0 +3623,Free Kindle Book - Aug 3-7 - Thriller - Desolation Run by @jamessnyder22 http://t.co/sgXb6E5Yda,0 +3625,going to redo my nails and watch behind the scenes of desolation of smaug ayyy,1 +3626,yeah 1:57 | Lamb Of God - Rock Am Ring 2015 - Intro+Desolation HD https://t.co/lEM5Z1NFk3 via @YouTube,0 +3627,In times of desolation and trouble Daniel's persistent prayers and fastings brought forth heavenly power and God i___ http://t.co/wOx3VpRixQ,0 +3628,"@nikostar :( +Y'all have lakes in Ohio? I thought all y'all had was abject desolation and Subway restaurants.",0 +3629,'cause right now I can read too good don't send me no letters no. not unless you're gonna mail them from desolation row ~,0 +3631,I REALLY liked the first Hobbit movie. I saw it three times in theatres. But I saw Desolation of Smaug and came out with the same feeling-,0 +3632,Be not afraid of sudden fear neither of the desolation of the wicked when it cometh. For the Lord shall be thy... http://t.co/bP597YDs2b,0 +3633,The Hobbit: The Desolation of Smaug - Ed Sheeran 'I See Fire' [HD] http://t.co/OXRwRJZmnu,0 +3634,Beautiful desolation. Just me a couple of coyotes some lizards and the morning sun. #Phoenix #Arizona http://t.co/K2tBES65oa,0 +3635,Fotoset: elanorofrohan: 10th December 2013 Green Carpet in Zurich for the Swiss Premiere of The Desolation... http://t.co/BQ3P7n7w06,1 +3636,Emotional Desolation the effect of alcoholism/addiction on family - http://t.co/31tGtLz3YA Forgiving is hard http://t.co/C7rcO2eMwF,0 +3637,Hey girl you must be Toe Hobbit: Part Two: ghe Desolation of Smaug because I'm not interested in seeing you. Sorry.,0 +3638,Escape The Heat (and the #ORShow) for a trail run on Desolation Loop you'll be glad you did http://t.co/n2ucNzh38P http://t.co/VU8fWYMw5r,0 +3639,@KaiSeiw And then there's people like me where my whole people are named Desolation,0 +3640,"This desperation dislocation +Separation condemnation +Revelation in temptation +Isolation desolation +Let it go and so to find away",1 +3641,#4: The Hobbit: The Desolation of Smaug (Bilingual) http://t.co/G5dO2X6226,0 +3642,Winter Desolation of Death is also on Tumblr: http://t.co/93DM6gnWwC Al Necro's reviews interviews & more!,0 +3643,#np agalloch - the desolation song,0 +3645,Imagini noi si 2 clipuri The Hobbit: The Desolation of Smaug -... http://t.co/j6CfwUKofE #cliptv #desolationofsmaug #poze #thehobbit,0 +3646,"Fear and panic in the air +I want to be free +From desolation and despair +And I feel like everything I sow ? http://t.co/iXW2cUTk1C",0 +3647,My Chemical Romance ___ Desolation Row #np,0 +3648,Putin's plan to destroy Western food en masse is causing a huge public backlash http://t.co/FAJbxz5kar,0 +3650,destroy the free fandom honestly,1 +3652,"Plot +In the futurea totalitarian government employs a force known as Firemen to seek out and destroy all literature https://t.co/DRfKarLz1d",0 +3653,Tell him Rebahe's going to destroy himself @Zenande_Mcfen @NDzedze #Ashes2Ashes,0 +3655,Ginga thinks he can defeat me? Not with my L-Drago Destroy he can't!,0 +3656,Black Ops 3 SEARCH AND DESTROY GAMEPLAY! (Hunted SnD Competitive Multiplayer): http://t.co/ss1zL36y9V via @YouTube,0 +3657,If GOP want to destroy America then Obama is dilutional I should be institutionalize or sued for slander. https://t.co/Z9Zj3KxwYU,0 +3658,"If Shantae doesn't get in Smash I will destroy my Wii U. +#ShantaeForSmash #Shantae #ShantaeHalfGenieHero #Nintendo http://t.co/ZztbVjYPN1",0 +3659,"ng2x5 mhtw4fnet + +Watch Michael Jordan absolutely destroy this meme-baiting camper - FOXSportscom",0 +3660,Just made anthonys bed considering i destroy it everytime i fall asleep. Smh ????,1 +3661,@RaidersReporter @957thegame together we can destroy the Emperor and rule the galaxy Father and Son!,0 +3662,Dem FLATLINERS who destroy creativity-balance-longevity & TRUTH stand with Lucifer in all his flames of destruction https://t.co/WcFpZNsN9u,1 +3663,Watch These Super Strong Magnets Destroy Everyday Objects: http://t.co/bTUs5jejuy http://t.co/zrTfxLuk6R,1 +3664,@Beyonce @NicoleKidman @Oprah these money grubbing kikes need to get a clueI have no money but I can still destroy with telekinesis. Watch.,1 +3666,#YIAYplan Use my awesome collection of Amiibos to destroy all in my path.,0 +3667,Get å£150 free bets when you register at Boylesports Casino #Luck #Destroy http://t.co/zildpvKNXP http://t.co/5yDb4s13pF,0 +3669,@Petchary but I can't say that either of us should be displeased. U.S move up five spots Jamaica 21! Congrats to the #ReggaeBoyz,0 +3670,@CameronCiletti @tigersjostun I can destroy u,0 +3674,Wow Crackdown 3 uses multiple servers in multiplayer?!?! U can destroy whole buildings?!?! #copped,0 +3675,Get a 50 Eur free bet at BWIN to use on all sports markets #Destroy #Free http://t.co/SiKkg0FPhR http://t.co/JFyfZDdLrN,0 +3676,@SarniamakChris @Hromadske @kasiadear33 how silly that one of only two countries that can destroy the world has a say about world security,1 +3679,5/6 will destroy Reg C competitiveness. The entire region will B over-saturated. Yes Brockton gets $12M and RegC Commonwealth PPC and,1 +3680,Don't let others bring you down no matter how hard they try. #beconfident Don't let others destroy you with your weaknesses!,0 +3681,Let's destroy Twitter with @fouseyTUBE @zaynmalik come on BrBrS lets do it in morning with fousey,0 +3682,Me and all my friends could destroy this in 2hours lmao https://t.co/waCtT18gdA,0 +3683,Industry Tryna Destroy @MeekMill cause he Exposed They Cash Cow...,0 +3684,(SJ GIST): 148 Houses Farm Produce Destroy... http://t.co/dkrGS2AWEX #StreetjamzDotNet | https://t.co/mR9KcGpIwM,1 +3685,destroy the house,0 +3686,Twitter is just here to destroy your childhood,0 +3689,@MsMigot wow what convincing & compelling evidence to change my view that is. climate change deniers I tend to destroy by giving...EVIDENCE,0 +3690,Politics = Preschool Attitude: Russia orders to destroy all food coming from countries it doesn't like. - There is no hunger in the world?,1 +3691,Save your riches in heaven where they will never decrease because no thief can gets them and no moth can destroy them. ??,0 +3692,@SapphireScallop Destroy oppa image? Oops! There's nothing left right? Haaaaaa,0 +3694,dazzle destroy the fun ??,0 +3695,As I catch the last few minutes. Finally Monwabisi gets shot lol. Hlongwane was ryt. The twins r gonna destroy each other #AshesToAshes,0 +3696,@engineermataRAI ate mataas kc rating..but they did not think on doing like this they destroy the story along with the ratings,0 +3697,@elgeotaofeeq that's not my take from his piece. Not putting that ambition in check will destroy the change we voted for the change we seek,0 +3700,@justicemalala @nkeajresq Nkea destroyed lives in Gambia as a mercenary judge.,0 +3702,Media stocks are getting destroyed (DIS FOXA CMCSA SNI AMCX VIAB VIA TWX) http://t.co/aqinaVl1b6,0 +3704,Black Eye 9: A space battle occurred at Star O784 involving 3 fleets totaling 3941 ships with 13 destroyed,0 +3705,Hero you can 't swim lonely guy help me my solution is not yours lifeguard i hated killing people how destroyed it http://t.co/pAztDblgYk,0 +3708,@alanhahn @HDumpty39 Daughtery would get destroyed on twitter. His comments are emotionally driven rants with little factual basis,0 +3710,Emotionally I am destroyed,0 +3712,@_RedDevil4Life_ @ManUtd destroyed!??,0 +3713,People donå«t want to hear the truth because they donå«t want their illusions destroyed. #FN.,0 +3715,#EverydayNaija | Flood: Two people dead 60 houses destroyed in Kaduna http://t.co/nOnm8C6L8P,1 +3717,Flood: Two people dead 60 houses destroyed in Kaduna: Two people have been reportedly killed and 60 houses ut... http://t.co/BDsgF1CfaX,1 +3718,Black Eye 9: A space battle occurred at Star O784 involving 2 fleets totaling 3967 ships with 39 destroyed,0 +3719,Black Eye 9: A space battle occurred at Star O784 involving 3 fleets totaling 3942 ships with 14 destroyed,0 +3721,Russian customs destroyed a total of 319 tons of food today phew! Some Italian meats were burned in an incinerator in Pulkovo airport.,0 +3722,Black Eye 9: A space battle occurred at Star O784 involving 2 fleets totaling 3942 ships with 13 destroyed,0 +3723,The grenade sound effect on 'Impossible' just destroyed about 10 other hoes careers ??,0 +3724,@freeMurphy Your hot take on Canada's hitchhiking garbage-bot (destroyed in Philly) was sorely needed.,1 +3725,Flood: Two people dead 60 houses destroyed in Kaduna: Two people have been reportedly killed an... http://t.co/kEE1tyTZ15 #SemasirTalks,1 +3726,the way he did me destroyed me...,0 +3727,I always felt like the Namekians were black people and felt played when they died and the planet got destroyed ??,0 +3729,@ryanoss123 No worries you'd have to be on every hitters most pitchers got destroyed,0 +3730,A person who blows himself up to kill others has no life in heaven because his ethereal body gets destroyed by doing it.,0 +3735,70 years after #ABomb destroyd #Hiroshima___#BBC looks at wht #survived http://t.co/dLgNUuuUYn #CNV Watch Peace Vigils: http://t.co/jvkYzNDtja,1 +3736,Flood: Two people dead 60 houses destroyed in Kaduna: Two people have been reportedly killed and 60 houses ut... http://t.co/JixScpMdUD,1 +3737,@DavidVitter Hi David in 2003 I saw the USA walk into a war that destroyed the lives of millions. You can prevent a repeat. #IranDeal,1 +3738,Black Eye 9: A space battle occurred at Star O784 involving 3 fleets totaling 3945 ships with 17 destroyed,0 +3739,I liked a @YouTube video from @zaire2005 http://t.co/MulRUifnN1 SPECIALGUEST CRAPGAMER RECAP MICROSOFT DESTROYED SONY AT GAMESCOM,0 +3741,#hot#teen#nsfw#porn#milf: Oiled Up Ass Hole Is Destroyed With King Size Cock Closeup Sex Clip http://t.co/faoGxkwdpG,0 +3742,Black Eye 9: A space battle occurred at Star O784 involving 2 fleets totaling 3934 ships with 7 destroyed,0 +3743,Why does it say Silas sliced in that headlinelike someone chopped him up like a piece of cabbage???????????????????????? #GH,0 +3744,@harbhajan_singh @StuartBroad8 i cant believe...is this d same stuart broad who was destroyed by our yuvi..????,0 +3745,Ford Truck Starts Up And Drives Off After Being DESTROYED By Tornado! http://t.co/IxJjlp1LVo,1 +3747,Black Eye 9: A space battle occurred at Star O784 involving 2 fleets totaling 3939 ships with 11 destroyed,0 +3748,New RAN report from the frontlines of human rights abuses and forest destruction for fashion.: http://t.co/tYDXauuEnQ,1 +3750,"How can we help save a beautiful town in Ontario from destruction by a power plant developer? +http://t.co/hlD5xLYwBn",0 +3752,Crackdown 3 Destruction Restricted to Multiplayer: Crackdown 3 impressed earlier this week with a demonstratio... http://t.co/LMWKjsYCgj,0 +3753,@DanHRothschild Greed is the fuel of self-destruction. #Takecare,0 +3754,@cinla1964 @windowgatribble The Contrasts of Foreboding destruction enhanced by Expansive divisions of color saturation contrast and hue!,0 +3756,Russian authorities to take account of petition against destruction of sanctioned food: Vladimir Putin's press... http://t.co/QbMcSJaVt0,0 +3758,Fall back this first break homebuyer miscalculation that could destruction thousands: MwjCdk,0 +3759,There's a #fly loose in my workspace with two #bored #cats. I forsee terrible things. #destruction #badkitty #thisiswhywecanthavenicethings,0 +3760,@LT3dave so many specs so much fan service so much lore destruction,0 +3761,Truck Driver Salvages Banned Tomatoes From Destruction on #Russian Border http://t.co/7b2Wf6ovFK #news,0 +3763,REPUBLICAN STYLED ECONOMIC DESTRUCTION | Undergroundbestsellers http://t.co/dILi5JhMur,0 +3764,Don't be the cause of your own self destruction,0 +3765,#Putin decree results in destruction of 10 tons of imported cheese near Russia-Ukraine border. RT @Independent http://t.co/K3pnNktlXh,1 +3766,"'Every kingdom divided against itself is headed for destruction and no city or house divided against itself will stand.' +Matthew 12:32",0 +3768,Megadeth-Symphony of Destruction http://t.co/xzfxRgLAlp,0 +3770,How American war planners singled out Hiroshima for destruction http://t.co/B5OKgpSpbH,1 +3771,"#ThingsIhate +Watching someone you care about head into total destruction and not being able to do anything.",0 +3772,@HassanRouhani Wars doomed to destruction loss money must invest in Iran's inside that should not go outside,1 +3773,Crackdown 3 Destruction Restricted to Multiplayer: Crackdown 3 impressed earlier this week with a demonstratio... http://t.co/gwESgesZxV,0 +3774,@BlossomingLilac so destruction it seems. I see myself ruined... somehow.,0 +3775,We are so arrogant in our destruction that we think the Earth needs us. She doesn't.,0 +3776,Megadeth Week - Symphony Of Destruction http://t.co/ECd7HiZja1,0 +3777,Crackdown 3 Destruction Restricted to Multiplayer: Crackdown 3 impressed earlier this week with a demonstratio... http://t.co/N08qluornx,0 +3778,What's happening? A destruction indeed http://t.co/tUX0YPwZuR,0 +3780,On this day in 1998 Appetite For Destruction goes to #1 on the Billboard Album Charts and stay on for 57 weeks,0 +3785,"@AlexeiVolkov1 @McFaul +And in an equal spirit I leave you to Roskomnadzor and to ridiculously politicized destruction of 'illegal food.'",0 +3786,That's the ultimate road to destruction,0 +3787,Self destruction mode! ???? https://t.co/ZtYZhbvzqP,0 +3788,@Bonn1eGreer The Angel of History propelled into the future by the winds of progress leaves in its wake piles of death and destruction. WB,0 +3789,Marquei como visto Dragon Ball Super - 1x1 - The God of Destruction\'s Dream http://t.co/vJLnsKbG86 #bancodeseries,0 +3793,Temptation always leads to destruction,0 +3795,RSS: Russia begins mass destruction of illegally imported food http://t.co/r6JDj9kIGm,1 +3796,So you have a new weapon that can cause un-imaginable destruction.,1 +3797,it sure made an impact on me http://t.co/GS50DdG1JY,0 +3798,@TinyJecht Are you another Stand-user? If you are I will have to detonate you with my Killer Queen.,0 +3800,Bomb squad set to detonate backpack Antioch Tenn. theater gunman had on him officials say - @Tennessean http://t.co/eb74iieIWn,1 +3802,Apollo Brown ft. M.O.P- Detonate - http://t.co/OMfGv9ma1W,1 +3803,Apollo Brown - Detonate (ft. M.O.P.) http://t.co/4BcQZqJRzn,0 +3805,@Furiosoxv stuns also probably won't be anything like AW stuns. And you can't detonate them,0 +3806,@spinningbot Are you another Stand-user? If you are I will have to detonate you with my Killer Queen.,0 +3807,@MythGriy they can't detonate unless they touch the ground,0 +3810,Detonate (feat. M.O.P.) by Apollo Brown http://t.co/fllaBzGCRc,1 +3812,#Boulder deputies are waiting for the bomb squad to detonate grenade like this one found in Stearns lake today http://t.co/7cADM3lNkO,1 +3814,@spinningbot Are you another Stand-user? If you are I will have to detonate you with my Killer Queen.,0 +3815,Apollo Brown - 'Detonate' f. M.O.P. | http://t.co/H1xiGcEn7F,0 +3816,"Real Hip Hop: Apollo Brown Feat M.O.P. - Detonate +#JTW http://t.co/cEiaO1TEXr",0 +3819,@BldrCOSheriff says 2nd grenade found is 'younger' that WWII era grenade found earlier. They'll detonate it at 8:00 tonight. @CBSDenver,1 +3820,A young heavyweight rapping off of detonate I been a leader not a lemon better get it straight ??,0 +3821,Apollo Brown - 'Detonate' f. M.O.P. http://t.co/Y217CEEemD,0 +3822,.@no_periferico Apollo Brown - 'Detonate' f. M.O.P. http://t.co/m7na4sKfWR #ORapInforma,0 +3824,52.214904 5.139055 Nuke please. Target Hilversum please detonate 800 meters below surface.,1 +3825,@OpTic_Scumper Yo why u so sexy?,0 +3826,@channelstv:That's why terrorism is not d war for d army but for Intel agents who can counter their moves before they detonate their bombs.,1 +3827,Apollo Brown ___ Detonate f. M.O.P. http://t.co/Jn8S0DrWbP #HHBU,0 +3828,@TinyJecht Are you another Stand-user? If you are I will have to detonate you with my Killer Queen.,0 +3829,Apollo Brown - Detonate (feat. M.O.P.) by Mello Music Group via #soundcloud https://t.co/PRojeAvG8T,0 +3830,Detonate (feat. M.O.P.) by Apollo Brown http://t.co/h9FSIaxv3Q,0 +3831,@WoundedPigeon http://t.co/s9soAeVcVo Detonate by @ApolloBrown ft. M.O.P.,0 +3832,@mwnhappy this message will self detonate in 5 4 3 2...,0 +3833,"Detonate (feat. M?.?O?.?P?.?) +from Grandeur by Apollo Brown http://t.co/GFDhFMPCEl",0 +3834,I__ªve just signed up for the Detonate Nottingham Autumn Launch Party. Register here: http://t.co/Km8uCIHrRN,0 +3835,@AutoAmes everyone hoped we would join ISIS and get ventilated by marines while trying to detonate a bandolier of hot dogs at Fort Dix,0 +3836,@spinningbot Are you another Stand-user? If you are I will have to detonate you with my Killer Queen.,0 +3837,Apollo Brown - Detonate (Ft. M.O.P.) https://t.co/NlJVP3Vfyz #FIYA!,1 +3838,@SourMashNumber7 @tomfromireland @rfcgeom66 @BBCTalkback They didn't succeed the other two times either. Bomb didn't detonate&Shots missed.,1 +3839,Press PLAY on @ApolloBrown's new single with M.O.P. to 'Detonate.' http://t.co/ZDTz3RbS6w,0 +3840,Apollo Brown ___ Detonate ft.å_M.O.P. http://t.co/JD7rIK7fX0 http://t.co/h6NgSw9A5b,0 +3841,@TinyJecht Are you another Stand-user? If you are I will have to detonate you with my Killer Queen.,0 +3842,#hiphop #news #indie Apollo Brown ___ ___Detonate___ Ft. M.O.P. - <a href='http://t.co/WnowfVCbMs... http://t.co/JxWOjxqndC,1 +3845,New music from @ApolloBrown featuring M.O.P.? 'Detonate' taken off his album 'Grandeur' coming soon - http://t.co/m1xYkEcRzr,0 +3848,Ignition Knock (Detonation) Sensor-Senso Standard fits 03-08 Mazda 6 3.0L-V6 http://t.co/c8UXkIzwM6 http://t.co/SNxgH9R16u,0 +3849,Ignition Knock (Detonation) Sensor Connector-Connecto MOTORCRAFT WPT-994 http://t.co/h2aHxpCH0Y http://t.co/VQ3Vwxj8YU,0 +3850,Ignition Knock (Detonation) Sensor-Senso Standard KS161 http://t.co/WadPP69LwJ http://t.co/yjTh2nABv5,0 +3851,Ignition Knock (Detonation) Sensor-Senso Standard KS57 http://t.co/bzZdeDcthL http://t.co/OQJNUyIBxM,0 +3853,Ignition Knock (Detonation) Sensor-Senso Standard KS94 http://t.co/IhphZCkm41 http://t.co/wuICdTTUhf,1 +3855,Dorman 917-033 Ignition Knock (Detonation) Sensor Connector http://t.co/WxCes39ZTe http://t.co/PyGKSSSCFR,1 +3857,Detonation quotes - my esteemed belt quotes regarding each one recent: EseVU http://t.co/emzn4sPwNk,0 +3858,"Do you want to play a game? +http://t.co/sQFp6Ecz0i +Its a GoogleMaps mashup that calculates the effects of the detonation of nuclear bomb",0 +3859,Look down upon three methods touching obtaing rank electrical transcription detonation: BuTIQOb,0 +3860,Ignition Knock (Detonation) Sensor-Senso fits 01-06 BMW 325Ci 2.5L-L6 http://t.co/gBVDNczjoU http://t.co/c211HISe0R,1 +3861,Ignition Knock (Detonation) Sensor-Senso Standard KS94 http://t.co/dY1erSDcRh http://t.co/m4cPmxmuRK,1 +3862,Ignition Knock (Detonation) Sensor-KNOCK SENSOR Delphi AS10004 http://t.co/LMrKgPOrcF http://t.co/6WAdNmsTOv,0 +3864,Ignition Knock (Detonation) Sensor Connector-Connecto MOTORCRAFT WPT-410 http://t.co/bSmJ2HVgwD http://t.co/bXalnEdy49,0 +3866,Ignition Knock (Detonation) Sensor-Senso BECK/ARNLEY 158-0853 http://t.co/OdMx36WDhM http://t.co/gAHeUjRUJu,1 +3868,Detonation into the realistic assets entering india: koZ http://t.co/9ZRQMd8nGZ,0 +3869,New SMP Ignition Knock (Detonation) Sensor KS315 http://t.co/aPVLH7hj1O http://t.co/1lJnTEJgmB,0 +3870,Ignition Knock (Detonation) Sensor-Senso Standard KS100 http://t.co/7o4lNfBe7K http://t.co/fVZSGJtBew,0 +3873,2015 new fashion ladies gold watch waterproof WeiQin famous brand michel quartz de lujo ca___ http://t.co/1JgsioUJaS http://t.co/719TZEyHFn,0 +3874,Ignition Knock (Detonation) Sensor-Senso BECK/ARNLEY 158-1017 http://t.co/ryoByQJFCE http://t.co/LW9O2kDk18,0 +3875,Ignition Knock (Detonation) Sensor ACDelco GM Original Equipment 213-924 http://t.co/HpZHe0cjvF http://t.co/SaOhVJktqc,0 +3876,A new favorite: Trivium - Detonation by @rrusa https://t.co/cubdNsNuvt on #SoundCloud,0 +3877,Don't miss Chris #Appy's detonation of the myths obscuring our crime in #Hiroshima 70 yrs ago this month. @salon http://t.co/DlP8kPkt2k,1 +3879,Detonation fashionable mountaineering electronic watch water-resistant couples leisure tab___ http://t.co/7dYOgLhMRe http://t.co/HKm3rtD4ZF,1 +3887,Ignition Knock (Detonation) Sensor ACDelco GM Original Equipment 213-4678 http://t.co/O2jD4TbrwA http://t.co/JFx6qiyiVF,0 +3888,Ignition Knock (Detonation) Sensor-Senso Standard fits 02-06 Acura RSX 2.0L-L4 http://t.co/VZaIQAMDCp http://t.co/ycecN44c8P,0 +3889,Detonation fashionable mountaineering electronic watch water-resistant couples leisure tab___ http://t.co/UCAwg59ulJ http://t.co/eNqDfbJUMP,0 +3892,Ignition Knock (Detonation) Sensor Connector-Connecto Dorman 917-141 http://t.co/rfJZexQgxt http://t.co/WQGsmiOiMx,0 +3893,"We are on our way to Hiroshima. +Today is the 70th anniversary of the detonation of the atomic bomb.",1 +3894,Detonation fashionable mountaineering electronic watch water-resistant couples leisure tab___ http://t.co/E61x9Y65QD http://t.co/OVLET0gDqm,0 +3895,70 years after : Hiroshima and Nagasaki - consequences of a nuclear detonation @ICRC http://t.co/BKh7Z6CWWl,1 +3896,Ignition Knock (Detonation) Sensor-Senso Standard fits 97-98 Ford F-250 4.6L-V8 http://t.co/cudkRyUUAN http://t.co/DKOZymvY5l,0 +3897,Ignition Knock (Detonation) Sensor-Senso Standard KS111 http://t.co/NXLEiIJFgS http://t.co/xsGwm5zXPd,0 +3899,@Meganbee92 @kadiegrr im just devastated that when it ends I will no longer see tyler blackburns face on pll xxx,0 +3900,Man Currensy really be talkin that talk... I'd be more devastated if he had a ghostwriter than anybody else....,1 +3901,___Richmond Coaches were devastated to hear of the death of their second driver Mr Chance who was sitting___: Jam... http://t.co/y5Yhbb0hkf,1 +3903,Zayn Malik & Perrie Edwards End Engagement: She__ªs __÷Devastated__ª http://t.co/GedOxSPpL9 http://t.co/ACZRUOrYtD,1 +3904,@UN No more #GujaratRiot & #MumbaiRiot92-93 which devastated 1000&1000 Indianperpetrated by #Modi & #ChawalChorbjp @UN_Women @UNNewsTeam,1 +3905,(#LosDelSonido) Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Ma... (#IvanBerroa),1 +3906,Obama declares disaster for typhoon-devastated Saipan: President Barack Obama has declared the Commonwealth of... http://t.co/4k8OLZv9bV,1 +3907,Obama declares disaster for typhoon-devastated Saipan #Worldnews http://t.co/9NYXjndoRA,1 +3908,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians a... http://t.co/JCszCJiHlH,1 +3911,___Richmond Coaches were devastated to hear of the death of their second driver Mr Chance who was sitting___: Jam... http://t.co/sHKiMonMlw,1 +3913,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians a... http://t.co/1i19CuOv7L,1 +3914,___Richmond Coaches were devastated to hear of the death of their second driver Mr Chance who was sitting___: Jam... http://t.co/dIalTa6t69,1 +3916,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians a... http://t.co/U8Ykr63B1G,1 +3917,"Good for her lol +http://t.co/K9cD0EFVuT",0 +3919,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians a... http://t.co/a1MoeJxqyA,1 +3921,Foto: ___Love isn__ªt everything to me anymore. The last album I made [Red] was a devastated record because... http://t.co/T5agPS7T2B,0 +3922,DEVASTATED ISNT THE WORD ROSS OUT OF ALL PEOPLE #Emmerdale #SummerFate,0 +3923,Thought it was Friday all day today. Was beyond devastated when I realised it wasn't ??,0 +3924,???????????? @MikeParrActor absolutely devastated what an actor. Will miss #RossBarton every girls loves a bad boy,0 +3926,@MikeParrActor has confirmed on his twitter saying goodbye 2 ross. Am bloody gobsmacked/devastated #emmerdale,0 +3927,@MikeParrActor devastated!! ????,0 +3932,Bairstow dropped his buffet ticket there. Devastated for the lad.,0 +3933,abcnews - Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern... http://t.co/mg5eAJElul,1 +3935,'Er indoors will be devastated. RIP Arfur. #GeorgeCole,1 +3936,@argentings WE COULD HAVE HAD IT AAAAAAALLLL I__ªm not even on that season and I__ªm devastated,1 +3937,Obama Declares Disaster for Typhoon-Devastated Saipan: http://t.co/M6LvKXl9ii,1 +3938,devastated by today's allegations.,0 +3940,@Keegan172 I'm devastated,0 +3944,@emmerdale I'll be devastated if it's cain...such a great character ??,0 +3945,Is Stuart Broad the Prime Minister yet. Best thing in Sport I have seen for years that. The Aussies look devastated. Bless ??????,0 +3947,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians a... http://t.co/XDt4VHFn7B,1 +3948,@WesleyLowery ?????? how are you going to survive this devastation?,1 +3949,I'm in utter shock and devastation you don't go to work to be left feeling how I do now. Life really is too short ??,1 +3950,In this fragile global economy considering the devastation the alternatives would cause ... it's the best reason .. https://t.co/zwVyisyP2B,1 +3951,"In Kalmikya Astrakhan Volgagrad and Dagestan there is already no food left for the locusts + + http://t.co/79Fw9zWxtP via @TIMEWorld",1 +3952,70 Years After Atomic Bombs Japan Still Struggles With War Past: The anniversary of the devastation wrought b... http://t.co/pmS4pMuR0q,1 +3953,Obsolete devastation from Broad with the Ball. And Root doing it with Bat in hand. Great Day #ashes2015 #ENGvAUS http://t.co/a7TJAWWtJ7,0 +3954,"http://t.co/IQoWZgvZNl #LatestNews #CNBC #CNN + +The anniversary of the devastation wrought by the first military use of an atomic weapon c___",1 +3955,70 Years After Atomic Bombs Japan Still Struggles With War Past: The anniversary of the devastation wrought b... http://t.co/ohNdh2rI0V,1 +3956,70 Years After Atomic Bombs Japan Still Struggles With War Past: The anniversary of the devastation wrought b... http://t.co/iTBJ6DKRZI,1 +3957,@cllrraymogford Indeed Ray devastation would be far more comprehensive #Hiroshima,1 +3958,70 Years After Atomic Bombs Japan Still Struggles With Wartime Past: The anniversary of the devastation wroug... http://t.co/EfsA6pbeMC,1 +3959,@ssssnell yeah I agree but it's the shock factor that ropes people in if they just show devastation then it angers me,0 +3960,70 Years After Atomic Bombs Japan Still Struggles With War Past: The anniversary of the devastation wrought b... http://t.co/aG65u29SGo,1 +3961,Classic WWW ___ Hiroshima/Nagasaki: Atomic Devastation Hidden for Decades: The people of Hiroshima and Nagasaki ... http://t.co/pLo2QkrWHu,1 +3962,70 Years After Atomic Bombs Japan Still Struggles With War Past: The anniversary of the devastation wrought b... http://t.co/LtVVPfLSg8,1 +3964,#HungerArticles: Nepal: Rebuilding Lives and Livelihoods After Quake Devastation http://t.co/LROuWjMbIx,1 +3967,70 Years After Atomic Bombs Japan Still Struggles With War Past http://t.co/5wfXbAQMBK The anniversary of the devastation wrought by the___,1 +3968,70 Years After Atomic Bombs Japan Still Struggles With War Past: The anniversary of the devastation wrought b... http://t.co/1erd2FPryP,1 +3969,We haven't seen the devastation from the 2014 Corp. Breaches yet Be Prepared! Get Coverage!: http://t.co/eK6KYHxPE9 http://t.co/Yn6NxOucR1,0 +3970,@Pam_Palmater i agree with @perrybellegarde to get out & vote. Look at devastation @pmharper caused #FirstNations #IdleNoMore #cndpoli #yyj,1 +3973,Is This Country Latin America's Next 'Argentina': One week ago we reported on the economic devastation in he o... http://t.co/m2y9Ym3iF6,0 +3975,The devastation when you smash your phone ??????????????????????????,0 +3977,70 Years After Atomic Bombs Japan Still Struggles With War Past: The anniversary of the devastation wrought b... http://t.co/BS6XaqHsim,1 +3979,#Thorium Radioactive Weapons. Scandals murders and environmental devastation: - VIDEO http://t.co/mly7sDN6eV,1 +3980,http://t.co/Gxgm1T3W0J From Devastation to Elation Getting Back on My Feet Helping Those Willing To Help Themselves http://t.co/puMxLVLsgM,0 +3982,70 Years After Atomic Bombs Japan Still Struggles With War Past: The anniversary of the devastation wrought b... http://t.co/o6AA0nWLha,1 +3983,70 Years After Atomic Bombs Japan Still Struggles With War Past: The anniversary of the devastation wrought b... http://t.co/Targ56iGBZ,1 +3984,What would your city look like if it had been the subject of the #Hiroshima bombing? Hint-devastation. #BeyondtheBomb http://t.co/3nKcUlGVMW,1 +3989,Fascinating pics from inside North Korea. Not propaganda not devastation - just people living life. http://t.co/E2Dbcpwd9u,0 +3990,$10M SETTLEMENT attained using our illustrations to help jurors understand the true devastation of internal injuries: http://t.co/2BaXg1WdPP,1 +3992,70 Years After Atomic Bombs Japan Still Struggles With War Past: The anniversary of the devastation wrought b... http://t.co/vFCtrzaOk2,1 +3993,#health #Newyear The anniversary of the devastation wrought by the first military use of an atomic weapon come... http://t.co/yuo7jDnijx,1 +3994,I visited Hiroshima in 2006. It is an incredible place. This model shows devastation of the bomb. http://t.co/Gid6jqN8UG,1 +3995,Devastation: coming to a @Target and find the @Starbucks closed ?? #momneedscoffee #asap #iwontmakeit,0 +3996,Water now tops the charts for highest global risk in terms of devastation___ ahead of nuclear war or a global pandemic http://t.co/nbcvbSO9nm,1 +3997,Currently Blasting #Benediction - #SanelessTheory -on Metal Devastation Radio- http://t.co/siGeeQ42cZ,0 +3998,I forgot to bring chocolate with me. Major disaster.,0 +4000,More Natural Disaster Research Urgent http://t.co/5Cm0LfZhxn via #JakartaPost,1 +4003,I'm a disaster?? https://t.co/VCV73BUaCZ,0 +4005,@LovelyLikeLaura I can see why one of your favorite books is 'Beautiful Disaster' it may now be one of mine??,0 +4006,å¬'Only the sea knows how many are dead' @MSF_Sea after last disaster in #Mediterranean turned into a massgrave http://t.co/m0utLDif77,1 +4008,Beautiful disaster // Jon McLaughlin is such a good song,0 +4010,@Tim_A_Roberts w/o giving up too much of Nana France reminds me of America right before the war in Iraq. A restlessness leading to disaster,1 +4011,Strengthening partnerships #AfterHaiyan http://t.co/Ga14egplw9 #Haiyan #YolandaPh #Philippines #livelihood #disasterrecovery #disaster,1 +4012,beautiful disaster https://t.co/qm5Sz0fyU8,0 +4014,My first staining attempt was a disaster https://t.co/buDmKE3nNf,0 +4017,CW500: Dealing with disaster - http://t.co/jq9nJ6Gko3,1 +4018,I want to go back to Vegas for my 21 but I feel like that would be such a disaster lol so much money would need to be brought,0 +4019,Blue Bell May Be Close to a Return From Its Listeria Disaster... Hot on #theneeds #Recipes http://t.co/F56v61AmPt,1 +4020,Jeff Locke. Train wreck. F'in disaster. Fortunately the Pirates acquired a top quality starter in J.A... Oh wait. #Blowltan,1 +4021,#Metepec #Mexico - ?NIGHT DISASTER?...E(Oficial) @ #NitClub #m_¼sica #m_¼sica http://t.co/WTfJF9jjzs,1 +4022,DISASTER AVERTED: Police kill gunman with 'hoax device' at cinema http://t.co/tdHn9zy0ER via #Foxnews,1 +4024,TV: Vitaly Churkin Briliantly Exposes the Hypocrisy of the Ukraine's FM Klimkin About the MH17 Disaster http://t.co/tt4kVmvuJq,1 +4025,Illinois Tornado Slipped Under The Radar Emergency Officials Say http://t.co/zhGu8yE1bj,1 +4026,Top insurer blasts lack of Australian Govt action on disaster mitigation http://t.co/sDgOUtWNtb via @smh,1 +4027,__¢i'm the architect of my own disaster__¢,0 +4028,@Hollyorange8 my day has been a disaster of emotions,0 +4029,I'm setting myself up for disaster,0 +4030,What the hell is wrong with people?!communication! Rule with an iron fist ticket to disaster. Can't swing one day and not the next.,0 +4032,@Gurmeetramrahim #MSGDoing111WelfareWorks Green S welfare force ke appx 65000 members har time disaster victim ki help ke liye tyar hai....,1 +4033,@cncpts @SOLELINKS what a disaster - can't say I'm surprised,0 +4034,RT @AmznFavorites THE DEVEREAUX DISASTER. 'Exciting scifi #thriller...' http://t.co/Mw9amBgAfq #SciFi #Kindle,0 +4038,"Keeps askin me what this means +Not like i got the answers +Plus if i say the wrong thing +This might just turn into a disaster",1 +4039,Four Technologies That Could Let Humans Survive Environmental Disaster - http://t.co/4RTpJrHsqe,0 +4041,The @rbcinsurance quote website = disaster. Tried 3 browsers & 3 machines. Always get 'Missing Info' error due to a non-existant drop down.,0 +4042,I'd like to think I'm photogenic but every time I see a pic with me in it I just think to myself 'gosh what a disaster' xD,0 +4043,DISASTER AVERTED: Police kill gunman with __÷hoax device__ª atå_cinema http://t.co/5NG0FzpVdS,0 +4044,DISASTER AVERTED: Police kill gunman with 'hoax device' at cinema http://t.co/94SXKI7KVX,1 +4045,If Locke has to pitch in the playoffs it's not a disaster but they should absolutely positively never let him pitch the sixth inning.,0 +4046,Teen Disaster Preparedness Event in Van Nuys August 11 @ 5:30pm http://t.co/fXUX987vZx via @VanNuysCouncil,0 +4047,@Youssefyamani add Fez weather to the equation and you have the best recipe for disaster.,0 +4049,#Myanmar Displaced #Rohingya at #Sittwe point of no return http://t.co/cgf61fPmR0 #Prison like conditions #genocide IHHen MSF Refugees,1 +4050,Crews were out all day boarding up windows at the park at the Galleria apts. where fire displaced 55 this AM. http://t.co/NMPN2mqZgE,1 +4051,.POTUS #StrategicPatience is a strategy for #Genocide; refugees; IDP Internally displaced people; horror; etc. https://t.co/8owC41FMBR,1 +4052,300000 Dead 1200000 injured 11000000 Displaced this is #Syria 2015 visit http://t.co/prCI76hOwu #US #Canada http://t.co/rTCuFaG0au,1 +4054,#Myanmar 's Displaced #Rohingya at #Sittwe point of no return http://t.co/gsa4o1mjNm Prison-like conditionsprivation http://t.co/i5ma6eWuwc,1 +4055,@lizhphoto When I have so much shit going on in my head I'd rather talk about it than have an outburst.. Displaced aggression SUX!!,0 +4057,#KCA #VoteJKT48ID 12News: UPDATE: A family of 3 has been displaced after fired damaged housed near 90th and Osborn. Fire extinguished no i___,1 +4058,Angry Woman Openly Accuses NEMA Of Stealing Relief Materials Meant For IDPs: An angry Internally Displaced wom... http://t.co/Khd99oZ7u3,0 +4060,@I_AmTalia @SGC72 'Thousands of people were displaced/injured/killed but hey now there's more opportunity for different restaurants!',1 +4061,The year is 2065 and the national society of meme preservation has opened the first museum where memes and their origins are displaced,0 +4062,Philippines Must Protect Internally Displaced Persons Warns UN Expert http://t.co/xLZWTzgQTC,1 +4063,At least 180 dead a million displaced in India floods - India | ReliefWeb http://t.co/0abgFgLH7X,1 +4064,#KCA #VoteJKT48ID 12News: UPDATE: A family of 3 has been displaced after fired damaged housed near 90th and Osborn. Fire extinguished no i___,1 +4065,9000 Photographs from 1800's British Mandate of Palestine - with no trace of 'Palestinians' http://t.co/X8i0mHYRmN,1 +4068,.POTUS #StrategicPatience is a strategy for #Genocide; refugees; IDP Internally displaced people; horror; etc. https://t.co/rqWuoy1fm4,1 +4072,.POTUS #StrategicPatience is a strategy for #Genocide; refugees; IDP Internally displaced people; horror; etc. https://t.co/rqWuoy1fm4,1 +4076,.POTUS #StrategicPatience is a strategy for #Genocide; refugees; IDP Internally displaced people; horror; etc. https://t.co/rqWuoy1fm4,0 +4077,.POTUS #StrategicPatience is a strategy for #Genocide; refugees; IDP Internally displaced people; horror; etc. https://t.co/rqWuoy1fm4,1 +4078,? The Circular Ruins - Displaced part 4 ? http://t.co/Od2ratxRqS #nowplaying,0 +4079,Historic flooding across Asia leaves hundreds dead millions displaced: http://t.co/4roisyXJlw http://t.co/3R8QoZJt7T,1 +4080,Angry Woman Openly Accuses NEMA Of Stealing Relief Materials Meant For IDPs: An angry Internally Displaced wom... http://t.co/TEq7SrI57P,1 +4081,#Myanmar Displaced #Rohingya at #Sittwe point of no return http://t.co/qegMRhSms2 #Prison like conditions #genocide @IHHen @MSF @Refugees,1 +4083,Displaced Persons GN (2014 Image) #1-1ST NM http://t.co/yEJt18sbm0 http://t.co/RcqacN91bE,0 +4084,Philippines Must Protect Internally Displaced Persons Warns UN Expert - The Diplomat http://t.co/V0yRfH9DKc,1 +4085,How much would it cost to have some fat displaced? Asking for a friend.,0 +4086,#Myanmar Displaced #Rohingya at #Sittwe point of no return http://t.co/cgf61fPmR0 #Prison like conditions #genocide IHHen MSF Refugees,1 +4087,40 displaced by Ocean Township apartment fire #NewYork - http://t.co/uelZ59wVOm,1 +4088,PennLive - Two families displaced by Mechanicsburg blaze - No one was injured in the fire. http://t.co/OHYD7Hhcpe,1 +4089,@peterjukes A crime that killed/displaced millions. In which systematic torture was carried out. But look. GrahamWP fired a gun! Arrest him!,1 +4091,'@PhelimKine: #Myanmar 's Displaced #Rohingya at #Sittwe point of no return http://t.co/8gO68KjE4b http://t.co/0KrW1zYaHM',1 +4092,Angry Woman Openly Accuses NEMA Of Stealing Relief Materials Meant For IDPs: An angry Internally Displaced wom... http://t.co/6ySbCSSzYS,0 +4093,Could Billboard's Hot 100 chart be displaced by these social-media-driven music charts? http://t.co/WVlaH8jRXe,1 +4094,Real people. True stories. Real pain & suffering. #displaced #RefugeesMatter #Syria https://t.co/OEZ7O9AB2C,1 +4095,@cityofhummus @ILNewsFlash do you want to hear more? Displaced my parents until both died in the diasporas !,0 +4096,"Elem Pomo helping the displaced from the Rocky Fire. Please consider! +Elem Evacuation Center http://t.co/dYDFvz7amj via @gofundme",1 +4097,For those displaced by disasters label and legal status remain out of reach https://t.co/kJMgTEEklp,0 +4098,#Nevada's __÷exceptional__ª #drought steady at ~11%; ~ 95% of #NV in drought: http://t.co/Nyo1xueBFA @DroughtGov http://t.co/w0a1MJOrHY,1 +4100,U.S. in record hurricane drought http://t.co/8JvQI9UspL,1 +4101,U.S. record hurricane drought. http://t.co/fE9hIVfMxq,1 +4103,Worried about how the CA drought might affect you? Extreme Weather: Does it Dampen Our Economy? http://t.co/fDzzuMyW8i,1 +4104,LLF TALK WORLD NEWS U.S. in record hurricane drought - The United States hasn't been hit by a major hurricane in t... http://t.co/oqeq4ueGF8,1 +4105,http://t.co/vYmnRnSThG: U.S. in record hurricane drought http://t.co/1mvSQG0XKE,1 +4107,w--=-=-=-[ NEMA warns Nigerians to prepare for drought http://t.co/5uoOPhSqU3,1 +4108,#MakeWaterWork Okanagan! Drought rating maxed out - Okanagan Valley http://t.co/tXrBdaUBNN http://t.co/Ue78c7EgOX #WesternCanadaDrought,1 +4111,Gov. Brown links CA wildfire to drought http://t.co/jEvrCWUdpm,1 +4112,"Heat wave is relentless: 91å¡F Feels Like 100 Humidity 55% + +relentless drought too: now Dominicana. I know is worse in other places.",1 +4113,Treasures revealed as California drought drains lakes http://t.co/kAH1KmTrj7,1 +4114,CANADA BC DROUGHT: Okanagan region issued Level 4 rating - Okanagan River (Columbia trib) fishing suspended to Sep 30 http://t.co/r4yZHxk7lw,1 +4116,It__ªs time to do away with drought.Check out how the #cloud #IOT are helping conserve water http://t.co/nnv3zwVANt,1 +4117,@gfrost1985 @jeffpalmer16 @MLB @BlueJays why you so salty and scared when we have a drought like you said?,0 +4119,Tips so that finding the customers ego drought: dqSVYusY,0 +4121,BLOG: Rain much needed as drought conditions worsen: Right now Charlotte and much of the surrounding area have___ http://t.co/OLzaVTJFKH,1 +4122,#weed news How marijuana is making California drought worse - Christian Science Monitor http://t.co/2SZ7oKjRXi,1 +4123,#DroughtMonitor: Moderate or worse #drought ? to ~27% of contig USA; affects ~80M people. http://t.co/YBE9JQoznR http://t.co/328SzflEtZ,1 +4124,Pretty neat website to get the latest drought conditions in your area https://t.co/uaoDOquDa1,1 +4127,Pizza drought is over I just couldn't anymore...,0 +4128,The Drought Is Real ??????,1 +4129,@Michael5SOS California is in a drought. what are you gonna do about this issue? if you want to be president this attitude won't work with m,1 +4131,@KarinaGarciaxo_ me & you both & I'll be dam if I get any of that drought bud,0 +4132,For these Californians it's like living in Africa's Sahel desert - just finding water a daily chore. http://t.co/ySG9vsrT4g,1 +4133,U.S. in record hurricane drought: The United States hasn't been hit by a major hurricane in the past nine years and it seems like that___,1 +4135,Large rain drops falling in Rock Hill off Anderson Road. #rain #scwx #drought,0 +4138,California meets drought-fueled fire season with extra crews.. Related Articles: http://t.co/rKDzB0TGC3,1 +4139,Moderate #drought is spreading rapidly across central Ga. #Macon #WarnerRobins #Dublin #Milledgeville #gawx http://t.co/PHNEZ60cwe,1 +4140,Thought it was a drought @_ASHJ? http://t.co/V4Br5gjMIY,1 +4141,@_gaabyx we got purple activist I thought it was a drought,1 +4142,We happily support mydrought a project bringing awareness to the LA drought. Track your water___ https://t.co/2ZvhX41I9v,1 +4143,'California's Burning:' Gov. on Drought Wildfires http://t.co/mkqSVp8E0G,1 +4145,LLF TALK WORLD NEWS U.S. in record hurricane drought - The United States hasn't been hit by a major hurricane in ... http://t.co/ML8IrhWg7O,1 +4146,'It's an eerie way of revealing both our history and our possible fate.' #CADrought #LakeIsabella by @jpanzar http://t.co/pvExbIiqSK,0 +4147,Mane im not a Raiders Fan but they been in a drought. They need to go 10-6 lol,0 +4149,@POTUS you until you drown by water entering the lungs. You being alive has caused this great country to fall to shit because you're a pussy,0 +4150,.@karijobe and her band killed it tonight. It was almost loud enough to drown out the tambourine behind me..... @codycarnes @AG_USA,1 +4152,Little gecko chillin' in my garden! I ended up helping him out I suspected he might drown!___ https://t.co/wXeLa91juh,0 +4153,'Save me from my self don't let me drown'.,0 +4154,We all carry these things inside that no one else can see. They hold us down like anchors they drown us out at sea.,0 +4156,#NowPlaying Porcupine Tree - Drown With Me (Live) #Listen #Live at http://t.co/iyLVzy3Cob,0 +4157,When you lowkey already know you're gonna drown in school this year :) http://t.co/aCMrm833zq,0 +4158,@chromsucks don't drown,0 +4159,Throw that water at me until I drown and my last words are choke me http://t.co/tUBE4NBqNz,0 +4160,Just down drown me k I can't swim https://t.co/sJoEing76t,0 +4161,Some days I drown in my tears but I don't let it get me down,0 +4163,@cameronhigdon34 I can't drown my demons they know how to swim.,0 +4164,@GraysonDolan only if u let me drown you ??,0 +4166,@Lwilliams_13 I'll drown you in the river walk,0 +4167,when your moms being annoying so you turn your beats pill all the way up to drown her out.,0 +4168,Ev makes me wanna drown myself he's such an idiot,0 +4170,@CortneyMo_ put this in Detroit niggas gone be acting out?? tryna fuck n drown mfs????????loose they buffs in the water?? https://t.co/OAQtjawGxg,0 +4171,@jasminehuerta24 I hope you drown ??,0 +4172,I can't drown my demons they know how to swim,1 +4173,gonna drown it in mustard and lemon pepper :),0 +4175,My parents don't believe in the dream. Sad.,0 +4176,Some older Native Australians believe that the oceans were created from the urine of an angry god who tried to drown the world.,0 +4177,No one told me you can drown yourself by drinking too much water.,0 +4178,When a real nigga hold you down you supposed to drown,0 +4180,I keep it out down drown their insults out with what I feel is devote pride ten fold action with reprocussions set at birth retroactive.,0 +4181,I am that girl on tv that sadly turns her music up to drown out the noise of her family fighting literally every day,0 +4182,Don't think for one second I'm out to drown your memory. Baby you ain't worth the whiskey.,0 +4183,This weekend is me and Nathan's birthday weekend so if you want to drown yourself in beer do reckless things and potentially die hmu,0 +4184,We are getting some reports of flooding near Jonesboro High School. Please use caution when driving in the area. Turn around don't drown!,1 +4189,Drown by Bring Me the Horizon (at Information Resources Centre (UTP)) ___ https://t.co/7vSqQSvGNI,0 +4191,Going to go drown my sorrows with sad music brb,0 +4197,@kessily @mishacollins So we should send it all 2 him then? Drown him in it? I like the way u think! #AllTheKidneyBeansAndSorbet4Misha #YES,0 +4198,@_dmerida my tears drowned out the terrible taste also nataly gave me her steak and cheese thing to cheer me up,0 +4199,Clev: Me? | You. Clev: Indeed. | Do you know what happened to ben? Clev: He drowned. | How. Clev: By his father. @cleverbot,0 +4200,Wtf this mom just drowned her child?!,1 +4201,Hundreds feared drowned as migrant boat capsizes off Libya http://t.co/7S1GfNEBgt,1 +4202,Sadly before she could save humanity Ursula drowned in the drool of a protoshoggoth but at least she sort of died doing what she loved.,1 +4203,i drowned in the kiddie pool and i lost my ploppy,0 +4204,http://t.co/MoA0q0AuFa Jacksonville family bands together as memorial is planned for toddler who ... - Florida___ http://t.co/NKOu7zWwRT,0 +4205,the future of america #GamerGate http://t.co/UhF7NyAbSw,0 +4206,"@Stephen_Georg Hey Stephen Remember that time you drowned all the yellows + +Read: http://t.co/0sa6Xx1oQ7",0 +4209,"@ABCNews24 @PeterDutton_MP +He also told you....No-one has drowned in the last 2 years & 1200 had under Labor. But let's not mention that..",0 +4210,New and now: Different (FNaF fanfiction): Trixie_drowned / 2 pagesHi my awesome proxies it's ... http://t.co/366NhTg3Tz #wattpad #promo,0 +4211,#DW Hundreds feared drowned as migrant boat capsizes off Libya: Hundreds of migrants are___ http://t.co/VOX99FWKcX,1 +4212,via @dw_english Hundreds feared drowned as migrant boat capsizes off Libya http://t.co/Cubc0nq6Fd #UFO4UBlogEurope,1 +4213,"at the lake +*sees a dead fish* +me: poor little guy i wonder what happened +ashley: idk maybe it drowned + wtf ????????",0 +4214,@God how come bugs haven't all drowned out of existence every time it rains?,0 +4216,http://t.co/riWuP1RbHu Jacksonville family bands together as memorial is planned for toddler who ... - Florida___ http://t.co/86pkNKCHmr,0 +4218,Migrants drown at sea after boat capsizes off #Libya http://t.co/t4pv0nrOoV http://t.co/PSeYLYzck4,1 +4219,pussy so deep I could've drowned twice,0 +4220,Visting Georgina at The Drowned Rat Cafe.....awful view!! http://t.co/pYLFb3tI9U,1 +4221,Sometimes logic gets drowned out in emotion but it's gotta surface at some point.,0 +4224,@_itsdanie_ noooo?? I almost drowned you once tho :)))),0 +4225,Toddler drowned in bath after mum left room to fetch his pyjamas http://t.co/k9aSKtwXfL,1 +4227,So I pick myself off the ground and swam before I drowned. Hit the bottom so hard I bounced twice suffice this time around is different.,1 +4230,Thank you Uplifting spirit. When Im drowned you've been an anchor,0 +4232,I want to see my @AustinPearcy22 so bad its not even funny. I will probably cry and drowned him in kisses when I do. ????,0 +4233,Jacksonville family bands together as memorial is planned for ... http://t.co/tilgurKv7Z,1 +4235,Hundreds feared drowned after another Mediterranean asylum seeker boat sinking http://t.co/zsYkzj2bzG,1 +4237,I got drowned like 5 times in the damn game today ????????????,1 +4238,80 tons of cocaine worth 125 million dollars drowned in #Alameda .....now that's a American drought #coke,1 +4239,Niggas favorite question is why you single ?? bitch I don't know pussy too wet almost drowned a nigga ??????????,0 +4240,The best thing about this is it drowned out the call from the guy angry cause he hadn't gotten a tracking number... http://t.co/QYu8grOrQ1,0 +4241,Hundreds feared drowned as migrant boat capsizes off Libya http://t.co/pPJi1tCNML,1 +4242,So today I fell off a rock scraped my whole butt and nearly drowned #summer2k15,0 +4243,Hundreds of migrants feared drowned off Libya: Migrants stand next to their tent at a camp set near Calais___ http://t.co/PY4mtW5xpM,1 +4244,HOPE THE DROWNED @eeasterling_2,0 +4245,.@DinosaurDracula Felt remorse for missing Pam drowned it with these guys. Really like Jason's Part VI outfit. http://t.co/irHh2GVSeD,0 +4246,Given the US coalition's fondness for backing '1984'-style totalitarianism it's a surprise they haven't copied... http://t.co/58wvChg1M9,0 +4247,@JanieTheKillr Jack closed the sketchbook biting his lip under his mask. 'I'm doing good. How're you?',0 +4248,forever drowning in my feelings.,0 +4250,The Drowning Girl by Caitlin R. Kiernan Centipede Press Signed numbered Limited - Full re___ http://t.co/mwcNVtCXVU http://t.co/ClOLmorpLd,0 +4251,The Drowning Girl by Caitlin R. Kiernan Centipede Press Signed numbered Limited - Full re___ http://t.co/m2YUXNqlqY http://t.co/V8GKkfMFXT,0 +4252,Grace: this is not your first ' storm' you know bicycles fences trees will fly as well as idiots drowning that go by swollen rivers...,1 +4253,LONDON IS DROWNING AND IIII LIVE BY THE RIVEEEEEER,1 +4255,Family mourns drowning of 'superhero' toddler with rare epilepsy: Bradley Diebold suffered hundreds of epileptic___ http://t.co/unsayJDTu7,0 +4256,Why are you drowning in low self-image? Take the quiz: http://t.co/Z8R6r3nBTb http://t.co/nAmffldh5h,0 +4257,2/his explanation was that 'you request as much as you want provided you get your job done but you're constantly drowning in work!',0 +4260,'Drowning' - Acrylic 08.05.15 https://t.co/X17fUBQBGG,1 +4261,The Drowning Girl by Caitlin R. Kiernan Centipede Press Signed numbered Limited - Full re___ http://t.co/tCJfCkXdZL http://t.co/EmTXtGO4CE,0 +4262,#ICYMI #Annoucement from Al Jackson... http://t.co/7BevuJE5eP,0 +4264,@_jeesss_ @Ethereal_7 Hello 911 yeah we have someone drowning here send a medic http://t.co/7GiglwdMhy,1 +4265,@JLabuz what if I'm drowning,0 +4266,@HeyImBeeYT its like theres fire in my skin and im drowning from within ????,1 +4268,don't you dare play the victim when I've been drowning for years.,0 +4269,I will stay drowning till you watch #NashsNewVideo http://t.co/EpZwasEYKy http://t.co/cfevtrsc1U,0 +4270,"SometimesI can't even breathe well +I feel like drowning and can't deal with my fear +#anxietyproblems",0 +4271,No reported cases of people drowning in sweat...FYI,1 +4272,http://t.co/9y0pAJ8sxd Family mourns drowning of 'superhero' toddler with rare epilepsy - Chicago Tribune http://t.co/oLdjsowKY5,1 +4273,@NigelTanner1 Believe it or not we've had too MUCH rain here. Our newly planted maple trees are actually drowning.,1 +4274,Boy saves autistic brother from drowning: A nine-year-old in Maine dove into a pool to save his autistic brother from drowning,1 +4275,Louis in red jacket round 2 aka drowning in my tears,0 +4276,Epilepsy claims another. Common and still a challenge to treat. Superhero toddler with rare epilepsy (Dravet) drowns http://t.co/VBo1tjNdps,0 +4279,I'm drowning in spirits to wash you out,0 +4281,The Drowning Girl by Caitlin R. Kiernan Centipede Press Signed numbered Limited - Full re___ http://t.co/McSEK4hX5S http://t.co/IIfGaZ0Fil,0 +4282,#Islamic #state issue a new holiday #brochure lovely swimming pool for drowning in shooting range and the downside it costs a #bomb,0 +4284,Drowning sorrows in Jarmusch vampires.,0 +4288,can still see it...blank expression...cheeks clapping in my face...marvins room playing subtlety in the back yet drowning out all the sound,0 +4289,My mom is watching a show about bridges breaking/falling and the people on them drowning in their cars aka one of my biggest fears ????,0 +4292,Drowning in Actavis suicide,0 +4293,"What This Man Did To Save A Drowning Squirrel Is Absolutely Incredible +http://t.co/YzZXxkNiSm http://t.co/zzsEe5Hipm",0 +4295,@Homukami Only URs and SRs matter Rs you'll be drowning in. Tho you're already drowning in Ns lol.,0 +4296,I feel like I'm drowning inside my own body!!,0 +4297,I'm drowning in hw now and that's w/o going to swim ohlordy,0 +4298,@NWSPocatello BG-16: So far brunt of storm just to our north. Grayed out w/ dust & rain to N blue sky interspersed w/ clouds to S.,0 +4299,Some poor sods arriving in Amman during yesterday's dust storm were diverted to Ben Gurion airport: http://t.co/jkpjpcH9i6,1 +4300,Kids Disappear in Dust Storm in Atmospheric Aussie Thriller http://t.co/0MNPCER9nO RT @Newz_Sacramento,1 +4301,NASA MODIS image: Dust storm over Morocco and the Straits of Gibraltar http://t.co/QWQnni7VMZ #duststorm,1 +4303,"Totoooooo! Totoooooooooo! +'@Historicalmages: Dust storm approaching Stratford Texas 18th April 1935. http://t.co/4awC16uUWB'",1 +4304,Severe storm weakening as it moves SE towards Lubbock area. Outflow boundary may create dust and 50 mph gusts http://t.co/pw3tZU0tay,1 +4305,When the answer my friend isn't blowing in the wind-IT'S YELLING! -How to Survive a Dust Storm http://t.co/9NwAJLi9cr http://t.co/tKMOtaeaCo,1 +4306,Kids Disappear in Dust Storm in Atmospheric Aussie Thriller http://t.co/TPOaprJudp RT @Newz_Sacramento,1 +4307,How to Survive a Dust Storm http://t.co/0yL3yT4YLH,1 +4309,Dust Storm 'en route' from Alice Springs to Uluru http://t.co/4ilt6FXU45,1 +4312,I keep sneezing either someone placed a southern dust storm in my house or someone talkin smack,0 +4313,|| So.... I just watched the trailed for The Dust Storm and I think part of me just died.... Colin is so perfect my goodness.,0 +4314,Dust devil maintenance fee - buy up la rotary storm guard: UVoPWZ,0 +4317,@RetiredFilth people in sydney woke up to the whole sky being red after a dust storm..like unreal.,1 +4318,There's a big storm brewing dark clouds thunder and rain carrying thick dust. This could be interesting.,1 +4320,I-90 Vantage: Backups eastbound with low visibility due to a dust storm about 4 miles east of the bridge.,1 +4321,Good way to end the day!!! Geyser plus dust storm! http://t.co/l5VakLR59M,1 +4322,dust storm in riyadh ????,1 +4323,New Mad Max Screenshots Show Off a Lovely Dust Storm Combat Magnum Opus http://t.co/45CmaQf8Ns,0 +4324,Let it be gone away like a dust in the wind .... Big Wind like a Tornado with Blizzard Thunder and Storm its that what I always want it,1 +4325,New Mad Max Screenshots Show Off a Lovely Dust Storm Combat Magnum Opus http://t.co/QHbzKErOTt #cogXbox #XboxOne #Xbox,0 +4326,The answer my friend is yelling in the wind-my latest article for http://t.co/LbMeKYphM5.Pls read and share - thanks! http://t.co/9NwAJLi9cr,0 +4327,"I NEED THE DUST STORM FILM ASAP + +ALSO + +*watches the trailer for the 500th time* @duststormfilm",0 +4328,Storm headed towards Idaho Falls with blowing dust & winds to 60 mph. US HWY 20 & I15 look out. #idwx http://t.co/0cR74m1Uxm,1 +4329,Currently driving through a dust storm. http://t.co/srUj5ZljGL,1 +4330,I learned more about economics from one South Dakota dust storm than I did in all my years in college. -Hubert Humphrey,1 +4332,6 Things Getting Caught in a Dust Storm & the Challenging Real Estate Market Have in Common (+ video): http://t.co/jf5Ft5cq9j,0 +4333,Raw Video: Dust Storm Rolls Through Texas http://t.co/QllkOfdyzX http://t.co/rGjJuMnNah,1 +4334,@LegacyOfTheSith @SagaciousSaber @Lordofbetrayal Moved in a crescent formation small trails of dust left in their wake as they moved.,0 +4335,DUST IN THE WIND: @82ndABNDIV paratroopers move to a loading zone during a dust storm in support of Operation Fury: http://t.co/uGesKLCn8M,1 +4336,Wall of noise is one thing - but a wall of dust? Moving at 60MPH? http://t.co/9NwAJLi9cr How to not get blown away! http://t.co/j4NI4N0yFZ,1 +4337,New Mad Max Screenshots Show Off a Lovely Dust Storm Combat Magnum Opus http://t.co/VRpmplcZCY,0 +4341,@deadlydemi even staying up all night to he barrier for t_üp and then having to run through a dust storm and almost passing out?,1 +4342,Going to a fest? Bring swimming goggles for the dust storm in the circle pit,1 +4343,Severe storm weakening as it moves SE towards Lubbock area. Outflow boundary may create dust and 50 mph gusts http://t.co/kA1HBjlqVw,1 +4345,New Mad Max Screenshots Show Off a Lovely Dust Storm Combat Magnum Opus http://t.co/MUdgU1pUNS http://t.co/AQxwOLbCfq,0 +4348,#Earthquake #Sismo M 1.4 - 4km E of Interlaken California: Time2015-08-06 00:52:25 UTC2015-08-05 17:52:25 -07... http://t.co/wA5C77F8vQ,1 +4349,1.43 earthquake occurred near Mount St. Helens area Washington at 09:36 UTC! #earthquake http://t.co/2xMdiDGpnr,1 +4350,Contruction upgrading ferries to earthquake standards in Vashon Mukilteo - Q13 FOX http://t.co/E981DgSkab #EarthquakeNews,1 +4351,#3Novices : Renison mine sees seismic event http://t.co/2i4EOGGO5j A small earthquake at Tasmania's Renison tin project has created a tem___,1 +4352,Put the RIGHT person up on the block #Shelli??? The sense of entitlement is ridiculous. #BB17.,0 +4353,2.0 #Earthquake in Sicily Italy #iPhone users download the Earthquake app for more information http://t.co/V3aZWOAmzK,1 +4354,USGS EQ: M 1.2 - 23km S of Twentynine Palms California: Time2015-08-05 23:54:09 UTC2015-08-05 16:... http://t.co/T97JmbzOBO #EarthQuake,1 +4355,Posted a new song: 'Earthquake' http://t.co/RfTyyZ4GwJ http://t.co/lau0Ay7ahV,0 +4356,Some of the aftershocks can be just as big as the initial earthquake.~ http://t.co/HKbPqdncBa,1 +4357,#Sismo ML 2.4 NEAR THE COAST OF WESTERN TURKEY: Magnitudeå_å_ML 2.4Regionå_å_NEAR THE COAST OF WESTERN TURKEY... http://t.co/0wdAzLcM90 #CS,1 +4358,USGS EQ: M 0.6 - 8km SSW of Anza California: Time2015-08-06 01:26:24 UTC2015-08-05 18:26:24 -07:0... http://t.co/3bwWNLsxhB #EarthQuake,1 +4359,1.9 earthquake occurred 15km E of Anchorage Alaska at 00:11 UTC! #earthquake #Anchorage http://t.co/QFyy5aZIFx,1 +4361,Raffi_RC: RT SustainOurEarth: Oklahoma Acts to Limit Earthquake Risk at Oil and Gas Wells | scoopit http://t.co/yru4nPHdrf Loving all thi___,1 +4362,#USGS M 0.9 - Northern California: Time2015-08-06 01:50:25 UTC2015-08-06 01:50:25 UTC at epicenter... http://t.co/mBo6OAnIQI #EarthTwerk,1 +4363,#SCSeEstaPreparando Light mag. 4.4 earthquake - - 73km SW of Khuzdar Pakistan on Wednes... http://t.co/i6lmcccLv5 via @volcanodiscover,1 +4365,#USGS M 1.2 - 23km S of Twentynine Palms California: Time2015-08-05 23:54:09 UTC2015-08-05 16:54:09 -07:0... http://t.co/kF0QYBKZOL #SM,1 +4366,"@AGeekyFangirl14 's things she looks in a significant other: +1. Beautiful eyes. +2. Humor. +3. Farts that creates an earthquake. + +????????",0 +4368,Nepal earthquake 3 months on: Women fear abuse https://t.co/iCTtZ0Divr via @@loupascale,1 +4372,ML 2.0 SICILY ITALY http://t.co/z6hxx6d2pm #euroquake,0 +4373,USGS reports a M1.94 #earthquake 5km S of Volcano Hawaii on 8/6/15 @ 1:04:01 UTC http://t.co/Njd28pg9Xv #quake,1 +4374,GNS sees unnecessary deaths resulting from earthquake strengthening legislation http://t.co/4rYZMzSgDW ($),1 +4375,'There was a small earthquake in LA but don't worry Emily Rossum is fine' #difficultpeople is great,1 +4376,USGS EQ: M 1.9 - 15km E of Anchorage Alaska: Time2015-08-06 00:11:16 UTC2015-08-05 16:11:16 -08:0... http://t.co/OjQ0KFg5ub #EarthQuake,1 +4377,#Earthquake #Sismo M 1.9 - 15km E of Anchorage Alaska: Time2015-08-06 00:11:16 UTC2015-08-05 16:11:16 -08:00 ... http://t.co/Z0VeR1hVM9,1 +4378,Contruction upgrading ferries to earthquake standards in Vashon Mukilteo: The upgrades will bring the vulnera... http://t.co/Au5jWGT0ar,1 +4379,[GPV Wind] As of 06JST 6AUG: WNW 06JST 6AUG / E 12JST 6AUG / S 18JST 6AUG. http://t.co/l6jBjAj8dm,0 +4381,M1.57 [01:11 UTC]?3km NNW of Lake Henshaw California. http://t.co/f9KQksoSw3,1 +4382,earthquake drill atm,1 +4383,There has not been 1 real tear out of #Shelli 's eyes this entire episode. #bb17,0 +4384,#USGS M 1.4 - 4km E of Interlaken California: Time2015-08-06 00:52:25 UTC2015-08-05 17:52:25 -07:00 at ep... http://t.co/zqrcptLrUM #SM,0 +4385,#Sismo M 1.3 - 1km NNE of The Geysers California: Time2015-08-05 23:40:21 UTC2015-08-05 16:40:21 -07:00 a... http://t.co/x6el3ySYcn #CS,0 +4387,Earthquake drill ??????,0 +4388,@DArchambau THX for your great encouragement and for RT of NEW VIDEO http://t.co/cybKsXHF7d The Coming Apocalyptic US Earthquake & Tsunami,1 +4391,Earthquake drill ??,1 +4392,#earthquake (EMSC): MD 2.9 OFF COAST OF NORTHERN CALIFORNIA http://t.co/6AiMd1uway G http://t.co/9cgbJwmhII,1 +4393,'There was a small earthquake in LA but don't worry Emmy Rossum is fine',1 +4395,Sure the #Megaquake story brought a sense of panic but the question is: will anything really change? http://t.co/9f3rDN9N3D,1 +4396,Earthquake : M 3.4 - 96km N of Brenas Puerto Rico: Time2015-08-05 10:34:24 UTC2015-08-05 06:34:24 -04:00 at___ http://t.co/sDZrrfZhMy,1 +4397,Scared to be living in Seattle when this predicted earthquake is going to destroy ????,1 +4398,@Adanne___ kindly follow back,0 +4401,@devon_breneman hopefully it doesn't electrocute your heated blanket lmao,0 +4402,Kids got Disney version of the game Operation only 2 AA batteries? I swear my old version had like 8 Ds and would nearly electrocute you.,0 +4403,Elecman could electrocute me and I'd say thanks.,0 +4405,when you got an extension cord that extends from your bed to your bath tub ?? lets pray I don't electrocute myself,0 +4406,I wanna tweet a 'niall thx for not making me was to electrocute myself' tweet but I'm scared I'll jinx it,0 +4407,Wtf Thomas Edison after making the lightbulb used to electrocute animals to make everyone think Teslas power was unsafe???? wank,0 +4408,Achievement Unlocked: Replaced Light Socket; Did Not Electrocute Self,0 +4410,Electrocute yourself,0 +4411,@i_electroCute your turn ??,0 +4412,@ZXAThetis 'Are you okay?! I electrocute you TOO badly right?',0 +4413,The Sea Will Electrocute Us All ??,0 +4415,Electric vs Gas brewing (not wanting to electrocute myself) question http://t.co/26oo0fcL53,0 +4416,@danisnotonfire don't let Phil help out he'll probably electrocute himself,0 +4421,@Mmchale13 *tries to electrocute self with phone cord*,0 +4423,I'm not the mom friend but I still see my friends as my little babies that I have to care for or else they'll electrocute themselves,0 +4424,Why does my phone electrocute me when it's charging,0 +4425,"She says that she'd love to come help but +The sea would.... +Electrocute us all... ??????????",0 +4426,It is freezing in my room & I erally want to unplug the ac but I don't want to electrocute myelf and die,0 +4427,i need u to delete this before i start crying into my computer and electrocute myself https://t.co/9ZMWT9XYdz,0 +4428,@lightseraphs pissed at you and could have their pikachu electrocute you and :\\\,0 +4430,Let her go - Passenger,0 +4432,@Omar_molina036 @Milioooo_ he's trying to electrocute ya ass lol hell no I ain't fucking with Emilio no more ????????,0 +4435,Kayla is about to electrocute herself.,0 +4436,no but seriously I will electrocute half of UK Army's so I can touch bangtan i do not play games when it comes to bts,0 +4437,Photo: weallheartonedirection: I wouldn__ªt let David electrocute himself so I__ªm the asshole http://t.co/uWiJMEGl4E,0 +4438,I would like to electrocute everyone who uses the word 'fair' in connection with income tax policies. - William F. Buckley Jr.,0 +4440,Photo: weallheartonedirection: I wouldn__ªt let David electrocute himself so I__ªm the asshole http://t.co/OEr5Hh41Ew,0 +4441,"@FoxNews He still has his beard - has he been visited by any1 while in prison? If he keeps that hideous beard electrocute him! +#UglyPeople",1 +4442,But the sea would..electrocute us all.,0 +4444,@el_torro_loco We can hear the conversation now... 'Sorry senator we thought you said 'electrocute' 50 million...' etc.,0 +4445,THE LINKS TO WATCH THE SHOW BETTER WORK OR I MIGHT ELECTROCUTE SOMEONE,0 +4448,When I was cooking earlier I got electrocuted some crucial ?????? now I'm psychic lol,0 +4449,#pakistan#news# PAKPATTAN City News: Man electrocuted From Our Correspondent PAKPATTAN: A man was electrocuted... http://t.co/frpbNhVPyI,1 +4450,'Hey bitch blow me' uh no. Stick your dick in some water then an outlet so u get electrocuted..,0 +4451,christie keeps telling me that i need to be electrocuted,0 +4452,So I had my phone charging and lightening struck in my backyard and I was holding my phone and it electrocuted my hand???? hurts so bad man??,0 +4454,I got electrocuted this morning how is your day going? ??,0 +4455,'Why am I being constantly electrocuted?' 'I don't know. Are you by chance standing next to a cactus?'#Borderlands #Borderlands2 #OOCVG #FTW,0 +4456,Woman electrocuted #Red #Redblood #videoclip http://t.co/9PYmM2RUWf #,0 +4457,@That_fat_guy there's literally a video of an elephant he had tied up in metal cables and electrocuted to death,0 +4458,Fr cuz I risk being electrocuted every shower ?? https://t.co/nWQ6wJQk1z,0 +4459,Youth electrocuted in Khulna | http://t.co/3EnyNdXpPm https://t.co/GQpi7jMKan via @sharethis,1 +4463,South Side factory where worker electrocuted pays $17000 penalty #Columbus - http://t.co/N8EzfCTfcE,1 +4464,Michael talking about when he was electrocuted omg #ROWYSOLouisville http://t.co/HxVfmoUhDM,1 +4465,@steveycheese99 @MapMyRun where you being electrocuted all the way round? The map sure looks like it.,0 +4466,I'm loving this classic barn shot! We may or may not have got electrocuted got stung a few times and stepped in... http://t.co/X6aSGRjsWC,1 +4468,got electrocuted last night at work for the first time in my life.... shit was weird ????,0 +4470,.@BBCNews .@mwlippert #SouthKorea Dogs prepared!Electrocutedboiling waterfur machine ALL STILL ALIVE http://t.co/3a50DhZ7YI,0 +4471,"Just thought I'd let you all know... +It's probably not a good idea to plug in your hairdryer when it's wet you will be electrocuted.",0 +4472,MT @Earths_Voice Treatment of #tigers in #China is appalling: electrocuted in front of businessmen & eaten http://t.co/JlWhaOwFQA #SaveTi...,1 +4476,I was blow drying my hair & the cable caught on fire. I let go of it as soon as I realized. Just before I could get electrocuted ??,1 +4478,I was working out today and i sweated SO MUCH like i though i was gonna get electrocuted by earbuds omg,0 +4479,Do babies actually get electrocuted from wall sockets? I'm wondering how I and those before me survived childhood.,1 +4480,Also my iPhone charger is broken and I just electrocuted myself.,0 +4482,It was a queer sultry summer the summer they electrocuted the Rosenbergs and I didn't know what I was doing in New York.,0 +4486,seriously look like a get electrocuted after I blow dry my hair it's really attractive ??,0 +4487,"Zotar(50 skeleton alchemist) was electrocuted to death by Atamathon the Giant Golem on Golem Graveyard 1. +http://t.co/GpwrC1KZ5i",1 +4488,#pakistan#news# NANKANA SAHIB City News: Electrocuted From Our Correspondent NANKANA SAHIB: A youth was electr... http://t.co/WERK9qibVV,1 +4489,God damn it!!! I electrocuted myself ??,0 +4490,Watching a man electrocuted on the roof of #mumbailocals is definitely a lesson.. People please learn!! #lessonforlife #marinelines #mumbai,1 +4491,"Worked in factory pressing designs onto T-shirts was electrocuted +d/t faulty ground. Boss docked my pay while I was at ER #WorstSummerJob",1 +4492,Elsa is gonna end up getting electrocuted. She's gonna end up like that cat from christmas vacation.,0 +4493,I'm in the shower and I went to go change the song and of course I get fucking electrocuted by the cord,0 +4494,Not being able to touch anything or anyone in Penneys without being electrocuted ??,0 +4496,Student electrocuted to death in school campus http://t.co/ryah8Fni5Q,1 +4498,"A whistleblower lawsuit accuses that supervisor of sleeping on the job more than once according to officials. + +http://t.co/IPwySnik0G",0 +4499,Survival Kit Whistle Fire Starter Wire Saw Cree Torch Emergency Blanket S knife - Full re___ http://t.co/cm7HqwWUlZ http://t.co/KdwAzHQTov,1 +4503,Emergency Flow http://t.co/lH9mrYpDrJ mp3 http://t.co/PqhuthSS3i rar http://t.co/0iW6dRf5X9,1 +4504,Emergency Dispatchers in Boone County in the hot seat http://t.co/5fHkxtrhYU,0 +4507,Survival Kit Whistle Fire Starter Wire Saw Cree Torch Emergency Blanket S knife - Full re___ http://t.co/2OroYUNYM2 http://t.co/C9JnXz3DXC,0 +4508,Emergency surgery,0 +4509,Deals : http://t.co/ddhWoRI5w1 #37592 Temporary Fake Tooth Teeth Replacement Kit Emergency Dental Oral Care Cosme___ http://t.co/ZCvfC500yY,0 +4511,God forbid anyone in my family knows how to answer a phone. I need new emergency contacts.,0 +4514,Alaska__ªs Prince of Wales #ArchipelagoWolves are nearing #Extinction. Demand emergency protection! #StandForWolves http://t.co/hBWoivJqkD,1 +4515,'The day you learn the importance of emergency exits is the day your heartbeat stops sounding familiar.',0 +4516,@newyorkcity for the #international emergency medicine conference w/ Lennox Hill hospital and #drjustinmazur,0 +4517,Plane from New York to Kuwait diverts to UK after declaring state of emergency http://t.co/5AIeXCBKFq,1 +4518,When your child needs emergency care they can be seen in our Emergency Department by @radychildrens Specialists! http://t.co/IGwsTTTkWK,0 +4519,STL Ace Grille - Surface Mounts SpeedTech Lights - Amber Emergency Lights - 544 http://t.co/t6Seku4yvm http://t.co/TJOZ4u4txl,0 +4520,the new quest type is 'level up quest'. its an always present quest with x2 exp designed to help people level up outside of emergency quests,0 +4521,11000 SEEDS 30 VEGETABLE FRUIT VARIETY GARDEN KIT EMERGENCY SURVIVAL GEAR MRE - Full rea___ http://t.co/DchfPXgY2m http://t.co/UgHpTzjuLK,0 +4522,UPDATE: Indiana State Police reopening I-65 near Lafayette following emergency bridge repairs that closed key highway for about 28 hours.,1 +4523,SF Asian Women's Shelter crisis line (415) 751-0880. Emergency shelter/support services 4 non-English speaking Asian women & children.,1 +4524,Emergency root canal!! #tookitlikeaman #lovemydentist #sore,0 +4525,@runner_joy yes; especially new clients that walk in and think a wart is an emergency.,0 +4526,"Myanmar floods: Childfund https://t.co/pQHQ4JnZTT + and International Needs https://t.co/FX0W2Sq05F and CARE Aust @CAREemergencies appeals",1 +4527,#Anchorage #Jobs Emergency Medicine - Nurse Practitioner - Healthcare Recruitment Counselors (Wasilla AK): Em... http://t.co/LKz5cNYNxX,0 +4528,Setting Up An Emergency Fund In 3 Easy Steps: You never know when a surprise expense will pop up. So work up t... http://t.co/Iz17kLelZC,0 +4529,The eyes of the nation & broader conservation community are on #Alaska @AKGovBillWalker reinstate emergency buffer #ProtectDenaliWolves,0 +4530,Gonna call up tomorrow with the aul 'emergency dental appointment' excuse just like the whole tooth falling out incident of last year,1 +4531,Loans until settlement day ??___ emergency money advances treasure-house self outbreed yours below take-home: AKx,0 +4533,in BOTH 'peacetime and times of national emergency.',1 +4534,From @LeanDOTorg: Lean Thinking for Quicker Police Emergency Response Time http://t.co/suZBkyW5TT,0 +4537,Former heroin addict shares story as city leaders sound alarm: City officials said emergency teams responded t... http://t.co/GZxIPMOknB,1 +4538,Busty blonde teen Natalia Starr fucks the security guard on set http://t.co/qew4c5M1xd View and download video,0 +4539,Came across this fire video not mine..enjoy..#fire #firemen #firetruck #emergency #rescue #911 #summertime #sirens___ http://t.co/hcYAJsAcfJ,1 +4541,@batfanuk we enjoyed the show today. Great fun. The emergency non evacuation was interesting. Have a great run.,0 +4542,Denali wolf population declined from 143 in 2007 2 just 48 in 2015. Reinstate emergency buffer zone #ProtectDenaliWolves @Alaska @adndotcom,1 +4543,@chowtaxis of Newport a big thanks for the emergency run to pick Jackie up from Bristol temple messages much appreciated,1 +4544,I'm glad when I call someone it's not an emergency since they never answer their phones or call back??,0 +4546,To Supply and Install New FRP Emergency Slide at Tunas KijangBank Negara Malaysia [Closing Date: 2015-08-14]... http://t.co/ZpqwKHFhNf,0 +4547,#EMERGENCY in Odai Bucharest Romania 600 Dogs Dying!They are so Hungry that they EAT EACH OTHER! http://t.co/pjigXPVPl0,1 +4548,"4 Printable Emergency Plan Templates +http://t.co/nAex0Q1Ax0",0 +4549,@POTUS Thx for emergency dec. http://t.co/DyWWNbbYvJ 4 days and no plan to get H20 to those who have no transport. Can you deploy troops?,1 +4550,See Aug 4 2015 PoconoRecord @EmergencyMgtMag - How Many Households Have an #Emergency Plan? | http://t.co/7zlsUmIess http://t.co/TdccH01N7q,1 +4551,Our builder is having a dental emergency. Which has ruined my plan to emotionally blackmail him this afternoon with my bump.,1 +4553,"This from The City of Calgary - + +City of Calgary has activated Municipal Emergency Plan + +The Municipal Emergency... http://t.co/hA5BoppeJy",1 +4556,http://t.co/F7LJwxJ5jp #GamerGate The end of Reddit is coming. It's time we devise an Emergency Evacuation Plan.,0 +4557,"Nieces these are especially good for you with the kids. + +Megan Swanger Ruthann McCormick Daisy Henley... http://t.co/Dl60JA06TW",0 +4561,Good tips! Does your family have an emergency plan? ... http://t.co/r5BgVLqPJt http://t.co/MEHWKZwtXD,0 +4562,.@CityofCalgary activates emergency plan amid severe thunderstorm warning http://t.co/pc7S8NxJ6Q #yyc #abstorm http://t.co/9xoHmMlMDY,1 +4563,City of Calgary activates Municipal Emergency Plan - http://t.co/IYs9xWPVMK,1 +4564,Govt plan for Rs40000Cr lifeline to FCI waste of money ask people to store grains fr 3_6_12 months fr emergency enough capacity available nw,0 +4565,Do you know the emergency plan at your workplace? If not ask your supervisor or operations manager. #Retail,0 +4567,NHS England announces new plan to meet emergency care targets http://t.co/0x2BIEqXPV,0 +4568,The City has activated the Municipal Emergency Plan. Primarily stay indoors avoid flooded areas Call 311 for... http://t.co/Ch6E7vTATR,1 +4569,City of Calgary activates Municipal Emergency Plan - 660 NEWS http://t.co/KFBjVJiVQB http://t.co/BN7Xpzqdm0,1 +4571,Hello Calgary!! Important news!! Please be advised!!! http://t.co/ARKTJ9Qn4S,1 +4573,Cool Tips from our friends at Counterstrike Security & Sound.... http://t.co/z5Y4Xr14W6,0 +4575,b/c it costs less to have sick people using emergency rooms?...grrrr.... http://t.co/vFbbcHwrFD,1 +4576,"___@based_georgie: yo forreal we need to have like an emergency action plan incase donald trump becomes president___ +Whipe that lil baby",1 +4577,@RebeccaforReal accepts Wisconsin Emergency Response Plan on behalf of @GovWalker #nbc15 http://t.co/Pis0aiVRbR,0 +4578,Cruise's 'M:I 5' emergency plan: Awesome fail http://t.co/H3dCh6Fyaw,0 +4579,Practice your families fire escape plan so everyone knows what to do in case of an emergency.,0 +4580,#Biztip We recommend all businesses to get an alternative source of electricity. #Solar Wind and Batteries. Have an emergency plan! Now!,1 +4582,When your body's like 'go to fuck to sleep Sami' and your mind's like 'make an emergency plan for every natural disaster GO',1 +4584,"Do you have a plan? Emergency Preparedness for #Families of +Children with Special Needs http://t.co/RdOVqaUAx5 #autism #specialneeds",0 +4585,Senators calling for emergency housing: Boxer Feinstein back plan to move #homeless vets to VA campus http://t.co/Gm80X3vutf,1 +4587,Severe thunderstorm warning remains for #Cochrane. @cityofcalgary has enacted municipal emergency plan after today's storm. #abstorm,1 +4588,Can't fix stupid. MT @CBCCalgary Don't drive through flooded underpasses warns city as it enacts Municipal Emergency Plan. #yyc #abstorm,1 +4589,Do you have an emergency drinking water plan? Download guide in English Spanish French Arabic or Vietnamese. http://t.co/S0ktilisKq,0 +4590,A big issue left undone is HOW to get home if adverse weather hits. @GoTriangle has no real emergency plan in place https://t.co/s7xdXuudcy,1 +4592,City of Calgary activates municipal emergency plan as more thunderstorms approach http://t.co/8iHucO4GLW,1 +4593,The Municipal Emergency Plan is now in effect. Stay safe everyone! #abstorm #yyc http://t.co/14CIcptKNa,1 +4595,Storm concludes City of Calgary's Municipal Emergency Plan deactivated http://t.co/93iaEec26T,1 +4596,County 911 Overload Prompts Use of Emergency Plan During July 4 Celebrations http://t.co/HXTUPrA5bc http://t.co/DqxKJibbKy,0 +4597,@chrisroth98 @chaselabsports in an emergency situation late in the year. Not as a plan in camp,1 +4598,@chillimik @HushLegs haha ??????..Are you really comparing yourselves to the emergency services! Thats brilliant! talk about up your own arse!,0 +4599,@TfLBusAlerts @TfLBusAlerts The Drive in Ilford closed both ways while emergency services deal with a call out. Buses are now stuck.,1 +4600,@Glenstannard @EssexWeather do you know where abouts as I heard emergency services near by,1 +4601,Join the Providence Health & Services team! See our latest #Nursing #job opening here: http://t.co/i3hZemlDpU #Torrance CA #Hiring,0 +4602,Sustainability Task Force Presents Levy to Fund Emergency Services - WDTV http://t.co/2FiBE2HAXC,1 +4604,#Nursing alert: Emergency Department Psychiatric RN (.90 FTE Day) | Providence Health & Services | #Olympia WA http://t.co/Yu6NUe7gFB,1 +4606,I am not an American but I have family who have served in the military work in the emergency services and work in... http://t.co/Pl2VzLrKVK,1 +4607,Emergency Services Committee and Personnel Committee Meeting on Thursday Night http://t.co/DrBcRyPj4p,0 +4608,We're #hiring! Click to apply: RN II/EMERGENCY SERVICES/FT/7P-7A - http://t.co/NV3Uxv9IMX #Nursing #Houston TX http://t.co/ej30IhrEA9,1 +4609,We're #hiring! Read about our latest #job opening here: RN Nurse Shift Manager Emergency Services - Full time... - http://t.co/sNuBZA6KSC,0 +4610,#veterans VET Act would ensure every military veteran's access to highest level of emergency care services: Proper treatmen... #followme,0 +4611,Emergency services unsure how to cope with loss of paging network http://t.co/UXqKIeqDyf,0 +4614,Brooklyn locksmith: domesticate emergency mechanic services circa the clock movement!: gba http://t.co/1Q6ccFfzV6,0 +4616,Public Hearing on 2015-16 @SUNY_Orange budget Thurs 8/6 at 3:15 Emergency Services Ctr Goshen. http://t.co/80DzgCo6Vc,0 +4617,Apply now to work for Dignity Health as #RN #Emergency Services Full Time 7a-7:30p Siena Campus in #Henderson #jobs http://t.co/FDiU44jLDJ,0 +4619,Want to work in #MissionHills CA? View our latest opening: http://t.co/ZsnSaR1Tw1 #Nursing #Job #Jobs #Hiring,0 +4620,Removing tsunami debris from the West Coast: Karen Robinson Enviromental and Emergency services manager of the___ http://t.co/1MeEo3WJcO,1 +4622,The #tubestrike is because TFL workers may have trouble planning downtime. I hope none need emergency services. http://t.co/iCSFDSiFqb,0 +4624,#Breaking #News - Call for Tasmania's emergency services to be trained in horse ... - http://t.co/urJwsVr311 http://t.co/7JfrETeIi4,1 +4625,Can you recommend anyone for this #job? RN Emergency Services Full Time 3p - 3\:30a Rose de Lima Campus - http://t.co/xQrLEWiA4x #Hiring,0 +4627,Emergency Shutdown Systems - Edmonton http://t.co/F8GvWkFqox,1 +4628,#Healthcare #Job in #Kodiak AK: Emergency Services Supervisor - Emergency... at Providence Health & Services http://t.co/8KJ1wDAiGj #Jobs,0 +4630,Air Group is here to the rescue! We have 24/7 Emergency Service! Learn more about it here - http://t.co/9lyx7zMtHE http://t.co/5PbC96rTMJ,0 +4631,Want to work at Swedish Health Services? We're #hiring in #Seattle WA! Click for details: http://t.co/4KDThCtEmV #Nursing #Job #Jobs,0 +4632,Goulburn man Henry Van Bilsen missing: Emergency services are searching for a Goulburn man who disappeared from his___ http://t.co/z99pKJzTRp,1 +4634,We're #hiring! Read about our latest #job opening here: Emergency Department Psychiatric RN (.90 FTE Day) - http://t.co/zOEpZsOkY1,0 +4635,Providence Health & Services: Emergency Services Supervisor - Emergency Department... (#Kodiak AK) http://t.co/AQcSUSqbDy #Healthcare #Job,0 +4637,Want to work in #Tarzana CA? View our latest opening: http://t.co/hkyFKug5zW #Nursing #Job #Jobs #Hiring,0 +4639,beyond stressed beyond hysteria into the grey misty indifference of complete shutdown of all but emergency services in my brain,0 +4641,Services are returning to normal #SouthLine after a medical emergency at Yennora and urgent track equipment repairs at Cabramatta earlier.,1 +4642,@swayoung01 Hi I thought that I recognised your smile. I believe that the Emergency Services are the best Performing Arts followers Simon.,0 +4644,Our doctors and nurses in the new Pediatric Emergency Department are all specialized in child services! http://t.co/k1TMLWvjmJ,0 +4646,#MissionHills CA #Nursing : Registered Nurse - Emergency Department ( Full Time... at Providence Health & Services http://t.co/Z5grLREy6V,0 +4649,Just saw a car on the I-77 Fully engulfed in flames hahah,1 +4650,Men escape car engulfed in flames in Parley's Canyon crews investigating cause - http://t.co/P6cyLz5lpt http://t.co/Jpu9gIps9f,1 +4655,Men escape car engulfed in flames in Parley's Canyon crews investigating cause - http://t.co/CYzlshlQhG http://t.co/nDiS8f1vzt,1 +4656,He came to a land which was engulfed in tribal war and turned it into a land of peace i.e. Madinah. #ProphetMuhammad #islam,0 +4657,#TRAFFICALERT Eastbound 210 Freeway at Citrus Ave in Azusa. Two motorcycles involved in accident with one fully engulfed in flames in lanes,1 +4658,Men escape car engulfed in flames in Parley's Canyon crews investigating cause - http://t.co/ldGWsYoWSs http://t.co/cnYVVY4WAT,1 +4659,He came to a land which was engulfed in tribal war and turned it into a land of peace i.e. Madinah. #ProphetMuhammad #islam,1 +4661,Tube strike live: Latest travel updates as London is engulfed in chaos: ___ cross-London travel will be accepte... http://t.co/vg8HRbebdA,1 +4664,just got engulfed in a car-induced tidal wave on my run... I thought this only happened in the movies ????,0 +4666,@FNAF_TalkMC *stands there engulfed in the fire smiling*,0 +4667,Tube strike live: Latest travel updates as London is engulfed inå_chaos http://t.co/xkonKZ0Zl6 http://t.co/dXVtgi1BvO,1 +4669,He came to a land which was engulfed in tribal war and turned it into a land of peace i.e. Madinah. #ProphetMuhammad #islam,1 +4670,Do you feel engulfed with low self-image? Take the quiz: http://t.co/ykVsttvDWo http://t.co/IFQQpUr99X,0 +4671,Men escape car engulfed in flames in Parley's Canyon crews investigating cause - http://t.co/tFan6qq2Ys http://t.co/rAkwWritPo,1 +4672,He came to a land which was engulfed in tribal war and turned it into a land of peace i.e. Madinah. #ProphetMuhammad #islam,0 +4673,michael is engulfed by that jumper,0 +4674,Why are you engulfed by low self-image? Take the quiz: http://t.co/CImUbwEyiB http://t.co/9R5FstS7Bd,0 +4675,@godsfirstson1 and she wrapped his coat around herself. It practically engulfed her.,1 +4677,When Your Cake Is Engulfed In Flames #LiteraryCakes,0 +4678,#RaheelSharif is manifesting how one RIGHT man at the helm can save a Sinking Ship engulfed in a Dark-Stormy-Tidal-Sea. Well Done.,1 +4679,Why tf did I decide to workout today? My body feels like it's been engulfed by a mass of fiery disdain.,0 +4680,Do you feel engulfed with low self-image? Take the quiz: http://t.co/YzDmouXQBO http://t.co/PeXfgawrG1,0 +4681,Man is equally incapable of seeing the nothingness from which he emerges and the infinity in which he is engulfed -- Blaise Pascal,0 +4682,Fully Engulfed Garage Fire: Propane Tanks Inside. Sunnymeade Dr.,1 +4684,He came to a land which was engulfed in tribal war and turned it into a land of peace i.e. Madinah. #ProphetMuhammad #islam,0 +4687,@suelinflower there is no words to describe the physical painthey ripped you apart while you screamed for dear lifeits like been engulfed,0 +4688,@ZachZaidman @670TheScore wld b a shame if that golf cart became engulfed in flames. #boycottBears,0 +4689,Car engulfed in flames backs up traffic at Parley__ªs Summit http://t.co/RmucfjCaZr,1 +4690,Why are you engulfed by low self-image? Take the quiz: http://t.co/I9dSPDKrUK http://t.co/NEp5aZwKNA,0 +4691,He came to a land which was engulfed in tribal war and turned it into a land of peace i.e. Madinah. #ProphetMuhammad #islam,0 +4692,Men escape car engulfed in flames in Parley's Canyon crews investigating cause - http://t.co/zevAn9kJzL http://t.co/UUZFs1L5Kt,1 +4693,Men escape car engulfed in flames in Parley's Canyon crews investigating cause - http://t.co/YfAVSuOgvl http://t.co/ISI1rLLCt0,1 +4694,Men escape car engulfed in flames in Parley's Canyon crews investigating cause - http://t.co/fxdH3U8Bq3 http://t.co/YZHVobGOcQ,1 +4695,Do you feel engulfed with low self-image? Take the quiz: http://t.co/WPlrhBFHeE http://t.co/eelEx4SSVF,0 +4696,'Tube strike live: Latest travel updates as London engulfed in chaos' <- genuine baffling Telegraph headline,1 +4697,Lucas Duda is Ghost Rider. Not the Nic Cage version but an actual 'engulfed in flames' badass. #Mets,1 +4698,@carneross indeed and a remarkably puny idea to place at the epicentre of a new post-capitalism epoch,0 +4699,[Question] Is anybody else having this problem with the '7' circle in Epicentre? http://t.co/dsPWS6hJ8w,0 +4700,Tomorrow kick off your weekend with drinks & entertainment @AliveAfter5 http://t.co/sC4TWJkxr1 http://t.co/yN6DuOtimr,0 +4702,@elisagxrcia I think of that every time I go to the epicentre haha,0 +4703,This Friday!!! Club Vault 3rd Floor EpiCentre http://t.co/7nU7pRxeul,0 +4704,#Tanzania elephant population declined by 60% in five years census reveals http://t.co/YxtZbTVMhm http://t.co/7jGgqwbv6S,0 +4705,I need a spot w| some drink specials. I'm kinda tired of the epicentre,0 +4706,RT @calestous: Tanzania elephant population declined by 60% in five years census reveals http://t.co/8zy9N6fX9T http://t.co/ITZ9masBvZ,1 +4707,September 15 Defeater at the Epicentre hell yeaahh,0 +4708,How a little studio in the middle of nowhere is becoming the epicentre of communication. http://t.co/iCRgseAGYA http://t.co/KpvYmHM2uB,0 +4709,Epicentre - Cydia Tweak - https://t.co/WKmfDig3nT | Thanks to @phillipten.,0 +4710,[Question] Is anybody else having this problem with the '7' circle in Epicentre? via /r/jailbreak http://t.co/48TPnmbJVG,0 +4711,I feel like if MKayla and Cee ever got in the same room everyone should evacuate because it would be so petty and childish I couldn't deal,1 +4712,CHICAGO (AP) ___ Organizers of Lollapalooza say they are reopening the music festival after the threat of a storm prompted them to evacuate,1 +4713,Learn how to evacuate your home in the event of a #wildfire view videos at http://t.co/bGeRLjamTE #CA #NV #UT #CO #OR http://t.co/sPuHuvgAsy,1 +4714,Disaster control teams are studying ways to evacuate the port area in response to tidal wave warnings.[900037],1 +4715,"'So again make sure to evacuate past the fire doors. Any questions? Yes?' +'Why would we open the doors to the fire!!?!??!?' + +I...I..I cant",0 +4716,When there's a fire alarm going off in zehrs and we keep working for 20 minutes then decide to evacuate everyone..,1 +4717,California &#039;monster&#039; fire is 20% contained as 13000 are told to evacuate http://t.co/aPTAP6Yx1r,1 +4719,If you did a cannon ball into the ocean then Japan would evacuate.,1 +4720,Firefigthers Evacuate from Northampton Township House Fire http://t.co/hPplD1jHtZ,1 +4722,Okay I need all of you to evacuate the house so I can write this poem,0 +4723,Fire crews evacuate passengers from a Gold Coast tram trapped when powerlines fell across a carriage. #TenNews 5pm http://t.co/hFyrloQY8q,1 +4724,Cascada - Evacuate The Dancefloor (Official Video) https://t.co/OHCx3y8l4s via @YouTube,0 +4725,Disregard my snap story there is an angry white girl riot happening as we speak. #evacuate,1 +4726,FWD: I literally jumped out of bed put on beach clothes and ran out my door like I had to evacuate for an apocalypse,0 +4729,Pls can alllll the nittys evacuate stockwell,0 +4730,cancel the fucking show. Evacuate MetLife https://t.co/SkQ8oUcM3R,0 +4731,Condemnation clearly replacing the latest response aimlessly dryer evacuate detersion de: HLg,1 +4732,The EFAK would be designed for building occupants once they evacuate and report to their evacuation assembly sites,1 +4734,Jay and alexis broke up there goes all your faith and goals people... evacuate yourself,0 +4735,@joonma_ dealbreaker that's it that's the dealbreaker s.o.s. abandon ship evacuate the building,1 +4736,So all the store's fire alarms went off today at work and we had to evacuate. I was like 'OMG!! I.S.I.S. ITS HAPPENING!!!!',1 +4737,louis is sad. cancel the show now. everyone leave. evacuate. this CANNOT go on.,0 +4738,No don't evacuate the students just throw them in the dungeon. That is stupid.,0 +4739,The U.S. also flew over each bomb site in World War II with warning letters telling people to evacuate,1 +4740,myGC: Broken powerlines evacuate Gold Coast tram suspend services http://t.co/6e7hHfeRz4,1 +4742,Sooo police dispatch said there was a person threatening to shoot up the Walmart on Rutherford & they had to evacuate,1 +4744,The summer program I worked for went the city pool we had to evacuate because one of my kids left a surprise. @jimmyfallon #WorstSummerJob,0 +4745,US govt refuses to evacuate 1000s of Americans from Yemen https://t.co/wQy3JOKuMH #yemen #usa #evacuation #abandoned,1 +4747,@ahhtheenikki And from what I can tell- they responded to today's gunman quickly and were able to evacuate all the ppl so no one was shot.,1 +4748,30 seconds for my bitches to evacuate ??????,0 +4749,California wildfires force thousands to evacuate: http://t.co/GFsl2Kwt5h via @YouTube,1 +4750,@pantalonesfuego Yeah I grew up in the canyon above L.A. We had to evacuate a few times.,1 +4752,quick shut down the show take the stage down evacuate everyone from mthe premises Louis is upset,0 +4753,@MeetKakarotto 'Don't bother while you were offline I managed to evacuate everyone out of here including Hinata so so go ahead and cause--,1 +4754,Evacuate from your life.,0 +4755,Tonight is being quite interesting... A few minutes ago the fire system went off and we had to evacuate the building.,1 +4757,I just want everyone to know that Emilee was worried I was getting a milkshake when we were supposed to evacuate,1 +4758,AND MY FAM HAD TO EVACUATE BC WE NEED POWER,1 +4759,Wow. #FIFA16 has Pre Season Tournaments in Career Mode. Bloody hell evacuate the building #whocares,0 +4760,Perhaps the criminal murderous #nazis should pack their bags & evacuate themselves from London & all #UK? #TubeStrike WELL DONE!,0 +4762,Does anyone know why #murfreesboro #walmart was evacuated this evening? @dnj_com,1 +4765,Green line service on south side disrupted after CTA train derails passengers evacuated. http://t.co/6eZkoof2Xt http://t.co/faCM78eg7K,1 +4766,Powerlines down over tram on GC Highway. Passengers have just been evacuated @9NewsBrisbane @9NewsGoldCoast http://t.co/KD3Qsakbi5,1 +4768,Red Cross re-opens shelter at Bickleton School after 25 homes evacuated in & around Roosevelt WA due to wildfire. #LiveOnK2,1 +4769,New evacuation ordered for 25 homes in danger of Hwy. 8 fire near Roosevelt Wash. http://t.co/SQsyUeh4yI #KOIN6News http://t.co/199t7ND0pm,1 +4770,They evacuated the mall. Again. ??,1 +4771,More than 300 campers evacuated as California wildfire blazes on officials say http://t.co/wwgAdpFFkW,1 +4773,"Trafford Centre film fans angry after Odeon cinema evacuated following false fire alarm: + Twitter users tell ... http://t.co/dZLENSe1Gw",1 +4774,13000 evacuated as California firefighters fight flames to save homes. #RockyFire http://t.co/tB52o146tx http://t.co/tsbTiGDSDT,1 +4775,Trafford Centre film fans angry after Odeon cinema evacuated following false fire alarm http://t.co/pFMn63VnAm http://t.co/vKwqbOJFJc,1 +4777,Rocky fire in Northern California swells to 60000 acres; 12000 evacuated http://t.co/42gW2i2Q41 Portland #Phoenix #Miami #Atlanta #Casper,1 +4778,Trafford Centre film fans angry after Odeon cinema evacuated following false fire alarm: Twitter users tell of___ http://t.co/PZeiXi4Xk7,1 +4779,5 dead 3 missing 103 families evacuated due to floods in Bukidnon: ... http://t.co/z0hSckvySN,1 +4782,@missleylaha I didn't get to buy one after the last London show because the fire alarm went off and everyone had to be evacuated. hahahaha,0 +4783,Just got evacuated from the movie theatre for an emergency. Saw people running from another they're.,1 +4784,#WorldNews Fallen powerlines on G:link tram: UPDATE: FIRE crews have evacuated up to 30 passengers who were tr... http://t.co/EYSVvzA7Qm,1 +4785,Passengers evacuated & lanes blocked off as power lines come down over a Gold Coast tram @9NewsGoldCoast http://t.co/zZweEezJuG,1 +4786,ALERT! Sandy Hook Elementary School Evacuated After __÷Bomb Threat__ª http://t.co/LwLexXjUS8,1 +4787,Trafford Centre film fans angry after Odeon cinema evacuated following false fire alarm http://t.co/6GLDwx71DA,0 +4788,Evacuation orders lifted for Roosevelt in Highway 8 fire. http://t.co/e2HltYyFAk #koin6news http://t.co/pmxEzUo4AY,1 +4789,Our thoughts are with these local residents! Time for some heavy rain!!! http://t.co/x3g2OX6K8R,1 +4790,"Trafford Centre film fans angry after Odeon cinema evacuated following false fire alarm: + Twi... http://t.co/RYEQMxIrj8 #manchesterlite",1 +4791,78 passengers evacuated safely after Green Line train derails. http://t.co/KzBSOhtwB4,1 +4793,Hotel evacuated after fire on Shore Drive in Norfolk http://t.co/6X0xHlbxji,1 +4794,@115Film Doctor we must leave immediately the Core is unstable...The whole building is told to be evacuated. Take the research. We need...,1 +4795,Evacuation order lifted for town of Roosevelt - Washington Times http://t.co/Kue48Nmjxh,1 +4797,Rocky Fire http://t.co/wxhMp5ppjq,1 +4798,Ahrar Al Sham: In our negotiations with Iran over Al Zabadani they wanted all Sunnis evacuated out of Al Zabadani!,0 +4799,Entire town of Roosevelt Wash. evacuated because of wildfire http://t.co/CmwEIojJ55,1 +4800,KATUNews: #SR14 remains closed as brush fire burns 1700 acres: http://t.co/QposKp3MWj #LiveOnK2 http://t.co/mTQjsvupwy,1 +4802,Good thing there was actually just a legit fire in the mall and nobody evacuated!!,1 +4803,Is it seclusion when a class is evacuated and a child is left alone in the class to force compliance? #MoreVoices,1 +4807,Rocky fire in Northern California swells to 60000 acres; 12000 evacuated http://t.co/mtfnbhRYZq Portland #Phoenix #Miami #Atlanta #Casper,1 +4808,"I got evacuated from the cinema 30 mins through Inside Out +Kill me please",0 +4809,Gold Coast tram hit by fallen powerlines: UP to 30 people have been evacuated from a tram on the Gold Coast af... http://t.co/xpPQnYHiWC,1 +4810,Tram travellers evacuated after powerlines come down in Surfers http://t.co/Qsheu3yF0W,1 +4812,FAAN orders evacuation of abandoned aircraft at MMA: FAAN noted that the action had become necessary due to re... http://t.co/ZUqgvJnEQA,1 +4813,#Breaking: Authorities have issued new mandatory evacuation notices for 25 homes to the North of the fire in #Roosevelt WA,1 +4814,FAAN orders evacuation of abandoned aircraft at MMA http://t.co/dEvYbnVXGQ via @todayng,0 +4819,Roosevelt Wash. under evacuation order due to wildfire http://t.co/FiJAPxyKRQ,1 +4820,Humboldt Cty Sheriff's Office has issued an evacuation advisory for 10 residence in the Lassics area... more info at http://t.co/ERUzBUQZYU,1 +4821,FAAN orders evacuation of abandoned aircraft at MMA http://t.co/GsOMtDPmoJ,1 +4823,VIETNAM WAR PATCH US 71st EVACUATION HOSPITAL HIGHLAND MEDICS http://t.co/kIF7M3FQLx http://t.co/Oz6vlWwTNR,1 +4825,. @VELDFest announces refunds after Day two's extreme weather evacuation: http://t.co/PP05eTlK7t http://t.co/3Ol8MhhPMa,1 +4826,Our mission to see the end of this movie last night @markoturner @annaciclismo after getting fire alarm evacuation http://t.co/5kFOVovjso,1 +4828,Update: Bend FD says roofing co. workers accidentally cut through natural gas line in Post Office leading to evacuation for about a half-hr,1 +4829,Evacuation Advisory for Swayback Ridge Area..voluntary-InciWeb:Mad River Complex Announcement http://t.co/vN73o4SGzJ #wildfires #calfires,1 +4830,Run out evacuation hospital indexing remedial of angioplasty dissertation at power elite hospitals dismayed:___ http://t.co/VGvJGr8zoO,1 +4831,So the IFAK is an 'individual first aid kit' to treat a single trauma victim I think I should create the EFAK or evacuation first aid kit,1 +4832,"Elem Pomo helping the displaced from the Rocky Fire. Please consider! +Elem Evacuation Center http://t.co/dYDFvz7amj via @gofundme",1 +4833,Bend Post Office roofers cut gas line prompt evacuation - http://t.co/6mF7eyZOAw,1 +4834,FAAN orders evacuation of abandoned aircraft at MMA: FAAN noted that the action had become neces... http://t.co/tlS40nqiPN Via @todayngr,1 +4835,@Eric_Bulak @jaclynsonne @_OliviaAnn_ I was looking for you guys on the live stream. I'm guessing the evacuation cost you the front?,0 +4836,FAAN gives owners of abandoned aircraft evacuation ultimatum http://t.co/zZpojgngAJ via @dailytimesngr. They should probe them too!,1 +4840,Updated #RockyFire map with Mandatory Evacuation areas (red) Advisory Evacuation (yellow) 2 Evac Centers (green) https://t.co/gZEgjoAKKw,1 +4842,UPDATE: The GNWT has just issued a voluntary evacuation order for cabin owners at Pickerel Lake near the Reid... http://t.co/RVSYxwj9Cp,1 +4843,Evacuation order lifted for town of Roosevelt Wash. though residents warned to be ready to leave quickly http://t.co/Na0ptN0dTr,1 +4844,@batfanuk we enjoyed the show today. Great fun. The emergency non evacuation was interesting. Have a great run.,0 +4845,my school just put the evacuation alarms on accidently with 2 different trial exams happening are you kidding me,0 +4846,Walking down the middle of Michigan Ave last Sunday during #Lolla evacuation 2015 https://t.co/YrfZ5WJ7R2,1 +4847,INK Entertainment Addresses Veld Evacuation and Refund Status http://t.co/vKu3RtOZ1J #TRC via @TorontoRC,0 +4848,___A voluntary evacuation is being recommended at this time___ for Pickerel Lake cabins across highway from #Reidlake fire says MACA #NWT #YZF,1 +4849,IbrahimMisau : FAAN orders evacuation of abandoned aircraft at MMA http://t.co/5Zcje7arci (via Twitter http://t.co/haVXoBcSVU),1 +4850,Evacuation order lifted for town of Roosevelt: http://t.co/EDyfo6E2PU http://t.co/M5KxLPKFA1,1 +4852,Update: More from Bend FD on how a natural gas line cut prompted evacuation of the main Post Office this afternoon http://t.co/wMmkIrJ0Hw,1 +4853,New evacuation ordered for 25 homes in danger of Hwy. 8 fire near Roosevelt Wash. http://t.co/SQsyUeh4yI #KOIN6News http://t.co/199t7ND0pm,1 +4854,Reid Lake fire prompts campground evacuation order http://t.co/jBODKM6rBU,1 +4856,"This is an evil generation +Rock and roll evacuation! +As far as the eye can see! +(Hey hey hey hey!)",0 +4857,@ariabrisard @leiaorganasolo Good. Play along with her. You may begin your operation with the death star. The evacuation is nearly complete.,0 +4858,Evacuation drill at work. The fire doors wouldn't open so i got to smash the emergency release glass #feelingmanly,0 +4859,Evacuation order lifted for Roosevelt after #Wildfire misses town - KOMO News http://t.co/qCpMktGLLR,1 +4860,Gas leak forces evacuation in east Saint John http://t.co/E1vkc2efsT #NB http://t.co/BeUa507Iug,1 +4864,If Ryan doesn't release new music soon I might explode,0 +4866,Learn How I Gained Access To The Secrets Of The Top Earners & Used Them To Explode My Home Business Here: http://t.co/SGXP1U5OL1 Please #RT,0 +4867,@NoahCRothman Bore him with minutiae serve bad champagne. He may just explode.,0 +4868,'I eat because it makes my mouth explode with joy and my soul rise upwards.' ~ http://t.co/mOdM8X1Ot9 http://t.co/oSsC7Q12iR,0 +4869,Housing Starts Explode to Newå_Heights http://t.co/IGlnQpgbNW http://t.co/aOesBVns45,0 +4870,@Annealiz1 You are going to make the internet explode with this Dr. Simon. O_o ... Was he alone or was there a red-head nearby? LOL,0 +4872,@attjcdemos @folieacat well we all knew her old user was like a timebomb ... destined to explode :/,0 +4874,See these guys reaching the front foot out loading the shoulders and spinning? Neither do I! #hitting #load&explode http://t.co/6VeI1mheA4,0 +4875,"KS except every character is Shizune. +The world would explode.",0 +4876,Whether you like it or not everything comes out of the dark be ready for that shit to explode ??,0 +4877,It's cold and my head wants to explode.. The joys of working from home - I'm going back to bed / peace out ????,0 +4878,If Schwarber ran into me going that fast I would explode into pieces,0 +4879,"Is he about to crash? +Did the Queen die? +Did something explode? +Who knows. http://t.co/LThMwtl5fP",1 +4881,i swea it feels like im about to explode ??,0 +4882,Kendall Jenner and Nick Jonas Are Dating and the World Might Quite Literally Explode http://t.co/pfvzVPxQGr,1 +4883,my brain id about to explode lmao,0 +4884,After having two cans explode I wanted to drink the rest but these ... (Kaldi Coffee Stout) http://t.co/u6isXv2F3V #photo,0 +4887,Learn How I Gained Access To The Secrets Of The Top Earners & Used Them To Explode My Home Business Here: http://t.co/UcLVIhwOEC Please #RT,0 +4888,"Versions of KS where if a character was /every/ character world would explode. + +Rin +Shizune +Misha +Emi +Kenji +Yuuko +Nomiya +Hisao",0 +4889,Philadelphia Eagles__ª Jordan Matthews Is Going To Explode In 2015 http://t.co/rRq1ildkiL #news #hotnewscake,0 +4890,I'm about to explode ????,0 +4891,Learn How I Gained Access To The Secrets Of The Top Earners & Used Them To Explode My Home Business Here: http://t.co/8rABhQrTh5 Please #RT,0 +4892,My head gonna explode soon,0 +4893,@deniseromano @megynkelly @GOP That's one way to make their heads explode...,0 +4894,Block the plate with a charging Schwarber coming down the line Cervelli. I dare you. You would explode into a little puff of smoke,0 +4895,@allen_enbot If you mess up it's gonna explode...,1 +4896,My brains going to explode i need to leave this house. Ill be out smoking packs if you need me,0 +4897,VINE OF THE YEAR OH MY GOD I AM ABOUT TO EXPLODE https://t.co/cnxXmfFRae,0 +4898,"tagged by @attackonstiles + +millions +a-punk +hang em high +alpha dog +yeah boy and doll face +little white lies +explode http://t.co/lAtsSUo4wS",0 +4899,Learn How I Gained Access To The Secrets Of The Top Earners & Used Them To Explode My Home Business Here: http://t.co/e84IFMCczN Please #RT,0 +4900,what if i want to fuck the duck until explode. it could be greasy,0 +4901,All these people explode ????,0 +4903,Learn How I Gained Access To The Secrets Of The Top Earners & Used Them To Explode My Home Business Here: http://t.co/dHaMbP54Ya Please #RT,0 +4905,@Anonchimp think its a tie with thunderstorms tho they make my soul explode...,0 +4906,My head is gonna explode,0 +4908,I'm ready to explode! http://t.co/OwJe3i6yGN,0 +4909,@DelDryden If I press on the twitch will my head explode?,0 +4910,Toronto going crazy for the blue jays. Can you imagine if the leafs get good? The city might literally explode.,0 +4911,Chick masturbates a guy until she gets exploded on her face > http://t.co/5QhoeHE9hf,1 +4912,@CrimsonFuckingV @BitchL0veCannon Even you have to admit Seras that Sasha was a cute peice of ass. Then she exploded all over the sidewalk.,0 +4913,Kakeru Teduka: Bfore 70years of today in Hiroshima it's exploded the one atomic bomd. It is so sad day.http://t.co/8Vzl1ns2iO,1 +4914,A tin of Tesco dog food 'exploded' and prompted THIS complaint - via @chelsea_dogs #pets #dogs #animals #puppy http://t.co/QzvKPaHsQ7,0 +4915,@KopiteLuke1892 Its broken its fully exploded.,1 +4917,Im Dead!!! My two Loves in 1 photo! My Heart exploded into a Million Pieces!!! ?????????????? @BrandonSkeie @samsmithworld http://t.co/yEtagC2d8A,0 +4918,The Dress Memes Have Officially Exploded On The Internet http://t.co/3drSmxw3cr,1 +4919,Final #Medinah update: shot a 105. 49 on front (exploded with a) 56 on back. #ImKeepingMyDayJob (pic is famous #17) http://t.co/kcmbBwwp8G,1 +4920,Well as I was chaning an iPad screen it fucking exploded and glass went all over the place. Looks like my job is going to need a new one.,0 +4922,The Dress Memes Have Officially Exploded On The Internet http://t.co/yG32yb2jDY,0 +4923,My little heart just exploded #OTRAMETLIFE #MTVHottest One Direction https://t.co/pQsLUg4jK5,0 +4924,Junko was 13 years old when the atomic bomb exploded over #Hiroshima on August 6 1945. http://t.co/lSpnyCVoLO http://t.co/Nh5pkFBfqm,1 +4926,Just saw The Man Whose Mind Exploded. There should be a Drako Zarharzar day.,0 +4929,I read about that break for hours before twitter noticed it and I though that is nothing to worry about bc twitter wasn't exploded bc of it,0 +4931,"that exploded & brought about the +beginning of universe matches what's +mentioned in the versethe heaven and Earth +(thus the universe)",0 +4934,If you told me ten years ago that I'd be seeing anime on the big screen... ...I probably would have exploded,0 +4935,The Dress Memes Have Officially Exploded On The Internet http://t.co/iBsVy2R3PH,0 +4936,Reasons @BlueWestlo has exploded on @YouTube #38745: https://t.co/Upgd2cy9il,0 +4938,luke + microphone = exploded ovaries,0 +4940,holy crap @KingMyth1999 my phone just exploded. haha,0 +4943,Did this man just squeeze another man's head with his bare hands until it literally exploded ???????,0 +4944,@ItsNasB now I have to go replace my sarcasm meter which just exploded. -__-,0 +4945,Kai Forbath just demolished a weather station set up on a drill field with a missed field goal. Thing just exploded into metal bits.,1 +4946,"@MeryCormier haha! Exactly! Cosima is definitely in the hot seat. Lol +'It's Shay.' is no different from a bomb that got exploded.",0 +4948,My head exploded i swear,0 +4951,Yumiko jumped in surprise as the fire shot upwards into the air and exploded caught off guard~ 'woah...' She had--( @LenkaIsWaifu ),0 +4952,"that exploded & brought about the +beginning of universe matches what's +mentioned in the versethe heaven and Earth +(thus the universe)",0 +4953,oh yeah my ipod almost exploded last night i was using it while charging and shit was sparking akxbskdn almost died,1 +4954,"@lunasagalle @synapsenkotze +'The Exploded - Bean the Bastard'",0 +4955,#dating #meet #sex Hot Teen Ass Exploded By Fat Cock http://t.co/X39JwSyrqR,0 +4957,#news #science London warship exploded in 1665 because sailors were recycling artillery cartridges... http://t.co/r4WGXrA59M #life #tech,1 +4958,@OKgooner hahaha great song. 'Spent 15 years getting loaded. 15 years till his liver exploded. Now what's Bob going to do NOW that he...',0 +4959,Worked at a fast food joint. Poured burnt hot oil down the sink. It hit the water in the trap and exploded. @FallonTonight #WORSTSUMMERJOB,0 +4961,New Explosion-proof Tempered Glass Screen Protector Film for Blackberry Z10 - Full read by___ http://t.co/ModqNaLWsB http://t.co/4C58oOaVhY,0 +4962,EE recalls Power Bar phone chargers after explosion burns woman ___ The Register http://t.co/lhTvKcoISo,1 +4964,Another Mechanical Mod Explosion: Man Injured When a 'Pharaoh' Mod Blows Up in His Hand - http://t.co/O82yVXbztv http://t.co/N5KmxuVeRg,1 +4966,Report: Corey Brewer was drug tested after 51-point explosion http://t.co/HhlLQCkcEP,1 +4967,Young dancer moves about 300 youth in attendance at the GMMBC Youth Explosion this past Saturday. Inspiring! http://t.co/TMmOrvxsWz,1 +4968,Around 10 injured in explosion in chemical park in western Germany... http://t.co/XBznU0QkVS,1 +4969,I liked a @YouTube video http://t.co/bGAJ2oAX1p Huge Building Explosion at 2500fps - The Slow Mo Guys,1 +4970,Large explosion rocks aluminum plant in southeast Missouri Re:Naomi-No Logo http://t.co/0WdsEIHYQu,1 +4971,...don't think I've ever been this close to a mental explosion in so long,0 +4972,GAElite 0 Explosion Greg 2 [Top 3rd] [0 Out] [0 balls] [0 strikes] ... No one on [P: #16 Morgan Orchard] [B: ],1 +4973,ITS A TIE DYE EXPLOSION ON IG HELP ME. IM DROWNING IN TIE DYE,1 +4974,Exploring New Worlds: Three Moments of an Explosion by China Mi_©ville http://t.co/OTrwZ1t9sp http://t.co/xVlkFCvfX5,1 +4975,MORE-->OSHA officers on siteinvestigating Noranda explosion -KFVS12 News Cape Girardeau Carbondale Poplar Bluff http://t.co/Pxyh7zo7vT,1 +4978,"A Time-Lapse Map of Every Nuclear Explosion Since 1945 - by Isao Hashimoto #War #atomicbomb +https://t.co/V0t8H4Iecc",1 +4979,@KirCut1 lets get a dope picture together and have the dopest explosion ????,0 +4980,Big Data and Social Information explosion: The Union That Could Evolve Your Retail Strategy...HUa,0 +4981,Aspiring musician & song writer shares her talent at the GMMBC Youth Explosion on this past Saturday. http://t.co/OmjMTU9kFG,0 +4982,@colinhoffman29 I hope he does. And I hope you die in the explosion too,1 +4983,A Pyrotechnic Artwork by Cai Guo-Qiang Explodes into a Blossom on the Steps of the Philadelphia... http://t.co/orOvZFsKU2,1 +4984,@AminESPN Mencius tears are worse correct? Takes the explosion n more pain day to day right?,0 +4985,@lordRsBiscuits That's always good for a pretty explosion.,1 +4986,New Explosion-proof Tempered Glass Screen Protector Film for Blackberry Z10 - Full read by___ http://t.co/tOYU16mxBO http://t.co/P10hNDc0Mm,0 +4988,EE recalls Power Bar battery packs after explosion in student's bedroom http://t.co/EKWTiHlwuf,1 +4989,"#cum explosion! + +@begforcum +@allday_cumshots +@cumcovered +@sexycumshots +@Cumtown +@BJ_Nutt +@cumslut_2 +@GirlsLoveCum http://t.co/2CX1yjjoZ9",0 +4990,Saddle with accountable information explosion tips pro preferable financial top brass: tawFMCAw,0 +4992,New Explosion-proof Tempered Glass Screen Protector Film for Blackberry Z10 - Full read by___ http://t.co/z4IlB9y9nU http://t.co/j295MD1SOW,0 +4993,New Explosion-proof Tempered Glass Screen Protector Film for Blackberry Z10 - Full read by___ http://t.co/SD7lOww9nu http://t.co/7hKavTVx81,0 +4995,Did you miss the #BitCoin explosion - Don't miss out - #Hangout tonight at 8:30PM EST ===>>> http://t.co/qKaHXwLWXa,0 +4996,kindermorgan gas explosion,1 +4997,"#Tampa: Super Freestyle Explosion Live in Concert at Amalie Arena - Sep 19 +? Ticket Info: http://t.co/ooGotO76uZ",0 +4998,We found a sunflower explosion on our walk. http://t.co/vLNmkLWWby http://t.co/P769eo49Fj,1 +5000,Was in a bad ass mood today got on the elevator at school and decided to make explosion noses everytime some one pressed a button ??,0 +5001,Checkout all the NURGLE rules & features that snuck in on Khrone's WD! http://t.co/he7Q7H3nZf http://t.co/rpvZBEsUQJ,0 +5002,New Explosion-proof Tempered Glass Screen Protector Film for Blackberry Z10 - Full read by___ http://t.co/hQZFcXxRsB http://t.co/0VWPdIzckO,0 +5004,"You Are Invited to The Expo Explosion Summer Event 2015! +WHEN: August 14th Friday 2015 +WHERE: Ben E Keith... http://t.co/yh4R7Ug21a",1 +5005,The gusto in persist had amongst empty-pated communication explosion: hPSsJD,1 +5007,"The insides of the explosion box.. +Get it customized the way you want! + +#dcubecrafts #greetingcards___ https://t.co/T07qxP5cBE",0 +5008,For those that were interested in the gun powder art discussed at the end of 'Introduction to Theological Aesthetic' http://t.co/BZ3iR4GMWj,0 +5010,The government is concerned about the population explosion and the population is concerned about the government explosion. - Joe Moore,0 +5013,Remembering Pittsburgh Eyewitness History of Steel City by Len Barcousky PB Penn http://t.co/dhGAVw8bSW http://t.co/0lMhEAEX9k,0 +5015,@Squibby_ I am an eyewitness HA,0 +5016,Eyewitness Identification via reddit http://t.co/bQV3QtTuxR http://t.co/0DrqlrsGY5,0 +5017,WPRI 12 Eyewitness News Rhode Island set to modernize its voting equipment WPRI 12 Eyewitness___ http://t.co/aP9JBrPmQg,0 +5018,"Eyewitness accounts of survivors of Hiroshima gleaned from a +number of oral history projects https://t.co/yRQGNbLKaC",1 +5019,Dutch crane collapses demolishes houses: Dramatic eyewitness video captures the moment a Dutch crane hoisting... http://t.co/dYy7ml2NzJ,1 +5022,#Eyewitness media is actively embraced by #UK audiences. Read the report by @emhub on the impact of #UGC in news: http://t.co/6mBPvwiTxf,0 +5024,Read an eyewitness account from #Hiroshima from this day in 1945 http://t.co/njAffyjaRz http://t.co/1xHSuEwQn4 #LIFE,1 +5025,Interesting approach but doesn't replace Eyewitness video. The Ferguson Case - Video - http://t.co/vEcsoSRleR http://t.co/fiUOgj6hEF,1 +5027,#ClimateChange Eyewitness to Extreme Weather: 11 Social Media Posts that Show Just How Crazy Things A... http://t.co/czpDn9oBiT #Anarchy,1 +5028,@kaputt21 Hamburg Police Chief Gregory Wickett has told 7 Eyewitness News he 'can't confirm or deny' an investigation is underway.,0 +5029,I liked a @YouTube video http://t.co/YdgiUYdqgb Mini Pony packs a punch. Live report from the Salem County Fair on CBS3 Eyewitness,0 +5032,Lone Survivor: The Eyewitness Account of Operation Redwing and the Lost Heroes of SEAL Team___ http://t.co/NXtWXJCAVh http://t.co/oL8ESFRGLE,1 +5033,The Civil Rights Movement (Eyewitness History Series) Wexler Sanford Hardcover http://t.co/kK0qFKGqcY http://t.co/SMc1Ro09Fs,0 +5034,How ___Little Boy___ Affected the People In Hiroshima ___ Eyewitness Testimonials http://t.co/mUAnfWcRW9,1 +5035,Read a Schoolboy__ªs Eyewitness Account of Hiroshima http://t.co/pq0D7MH3qr,1 +5037,Read an eyewitness account from #Hiroshima from this day in 1945 http://t.co/QUEDV2xxxX #LIFE,1 +5038,WFTV Eyewitness News: TN school psychologist arrested in Florida on child porn charges http://t.co/lgGLf5yrMe,0 +5039,Wake Up Kern County Eyewitness News Mornings airing RIGHT NOW on KBAK CBS29. http://t.co/rorKtMpqNs #liveonKBAK http://t.co/eDznX6GOud,1 +5040,On anniversary of Hiroshima bombing illustrated timeline of bombings. Eyewitness account particularly horrifying http://t.co/GZIb0mAwmn,1 +5041,"Aug. 06 2015 Radio Show articles ___ +1] Eyewitness Account of Abortion Organ Harvesting by Planned Parenthood: http://t.co/49izkbOHri",0 +5043,DK Eyewitness Travel Guide: Denmark: travel guide eBay auctions you should keep an eye on: http://t.co/qPUr3Vd7Hl,0 +5044,DK Eyewitness Travel Guide: Denmark: travel guide eBay auctions you should keep an eye on: http://t.co/l9EKHNkBar,0 +5046,RT patrickjbutler: Excellent damiengayle eyewitness account of Kids Company closure: 'You drop the bomb and expect___ http://t.co/pHH1VmLfoo,1 +5047,Monkeys Abused by Notorious Laboratory Dealer | A PETA Eyewitness Invest... https://t.co/QGqlpmRfJd via @YouTube,0 +5049,A true #TBT Eyewitness News WBRE WYOU http://t.co/JHVigsX5Jg,0 +5050,DK Eyewitness Travel Guide : Chicago by Dorling Kindersley Publishing Staff___: travel books eBay auctions you s... http://t.co/tj3LtPZfW1,0 +5052,It's lights out again with stage-two load shedding: Stage-two load shedding will be in force between 5pm and 10pm. http://t.co/vxVfAEEY0q,0 +5055,DK Eyewitness Travel Guide: Denmark: travel guide eBay auctions you should keep an eye on: http://t.co/xpwkodpqtO,0 +5056,WFTV Eyewitness News: FBI: Man who stole US secrets 'married' Honduran minors for sex http://t.co/NdwEp6IZDQ,0 +5059,Freed #BokoHaram captives speak up: ___I will make sure she goes to school.' @guardian http://t.co/PK8dgVripw http://t.co/RZ0adzursW,0 +5060,'I tried to save Mido Macia': One of the murder accused has testified he tried to save Mido Macia__ªs life. http://t.co/vxVfAEEY0q,0 +5061,Russian 'food crematoria' provoke outrage amid crisis famine memories: MOSCOW (Reuters) - Russian government ... http://t.co/Mphog0QDDN,1 +5062,@OCT336 guys these bitches ain't famine then ??,0 +5063,'Food crematoria' provoke outrage amid crisis famine memories... http://t.co/fABVlvN5MS,1 +5064,#Russia 'food crematoria' provoke outrage in country w/soaring poverty +Soviet famine memory http://t.co/vymOuZjZRe http://t.co/eNRJh5Qkve,1 +5065,Hunger hits me and I can't function probably wouldn't last long in famine zones***thank God***,0 +5067,According to a 2011 Gallup poll the more money you have the more likely you are to suffer from time famine.? Ari___ http://t.co/QdmVTJ4lZJ,0 +5068,Ppl like unu feast today and famine tomorrow,0 +5069,"Top 3: +* Turn on the Darkness +* King Redeem - Queen Serene +* Famine Wolf",0 +5071,Russian 'food crematoria' provoke outrage amid crisis famine memories http://t.co/O4xLjnaV8F,1 +5072,@robertmeyer9 re: your example low food prices cause farmers to go broke= famine next year. means simple Capitalism failed to feed people>,0 +5073,A memorial to the millions who perished in the Holodomor has been erected in the U.S. capital. http://t.co/Dj1LWZNIEH http://t.co/I9MxXkzHbL,1 +5074,exporting food wont solve the problem. africans will end famine n poverty by SOLVING OUT OF CONTROL TRIBAL WARS. https://t.co/UttaNbigRx,1 +5075,'Food crematoria' provoke outrage amid crisis famine memories... http://t.co/REsxAvgpyJ,1 +5076,New article: Russian 'food crematoria' provoke outrage amid crisis famine memories read more at here http://t.co/9HiEiFHSmC,1 +5078,Building Muscle With Feast And Famine Feeding Most diet plans are based around losing weight but what should you___ https://t.co/lUe3waeGpI,0 +5079,The famine is coming to an end #Bluebell http://t.co/p9ryMfjcUX,1 +5080,'She tasted like a feast after a famine. And he was starving.' TRUSTING JACK @writes4coffee http://t.co/sTo58qa94c #IARTG #RWA #tw4rw #RRBC,0 +5082,Russian 'food crematoria' provoke outrage amid crisis famine memories http://t.co/GyH00mRKjm,1 +5083,"Russia destroys food while people go hungry. We're not the only ones with an insane government. +http://t.co/ZonNqGsxYw",1 +5085,@FinancialTimes Ethiopian regimes continue receiving foreign aid when famine struck for 2 years eastern Ethiopia https://t.co/2fGgzQn1v4,1 +5087,Reuters: Russian 'food crematoria' provoke outrage amid crisis famine memories http://t.co/SI02QRgukA http://t.co/0C1y8g7E9p,1 +5088,"A grade in Black Horse Famine[MEGA]. Score 0840728 +http://t.co/pdiit0AF3Q +#Dynamix http://t.co/ZQ5KDOx7BY",0 +5089,Russian __÷food crematoria__ª provoke outrage amid crisis famineå_memories http://t.co/sGECNKFThU,1 +5092,Bolshevik government monopolized food supply to seize power over hunhry population. Artificial famine was the result https://t.co/0xOUv7DHWz,1 +5094,Russian 'food crematoria' provoke outrage amid crisis famine memories http://t.co/h6Z7hXUqtu via @YahooNews,1 +5095,I've experienced the smell of rotting potatoes for the first time. The Famine must have been awful if the whole country smelled like that.,1 +5096,http://t.co/x1x6d5Enef Russian 'food crematoria' provoke outrage amid crisis famine memories http://t.co/XhehJFFT7g,1 +5097,when things become terrible more than the great Ethiopian famine...,0 +5098,@SavageNation Reminds me when the Peasants destroyed their food when Stalin's army came to 'redistribute the wealth'. It caused famine/death,1 +5099,UPDATE 1-Russian 'food crematoria' provoke outrage amid crisis famine memories: * Russian society still recal... http://t.co/J2erZbMjQD,1 +5101,The Adventures of Mineness #9 The Famine Is Over http://t.co/spYmIQNeCj,1 +5102,"'Food crematoria' in Russia provoke outrage amid crisis famine memories +http://t.co/FelR5a1hBP",1 +5103,Robert Conquest Famine Museum Kiev @GuidoFawkes @MediaGuido https://t.co/WE40iUX7Ib,0 +5104,#Russian food crematoria provokes outrage amid crisis famine memories http://t.co/FjeaFgbZfJ,1 +5105,@CNN the End of Times are upon us. Famine War Death Plague. The presence is growing stronger.,0 +5106,Maratha raiders scorched their lands & Punjab refused them food aid during the 1943 famine wonder if Bengalis harbor some 'hard feelings',1 +5107,Images of Famine ___ Hope In Christ - A blog on what happens when we forget God http://t.co/9BLiDdNGtF #HopeinChrist @lifelettercafe,1 +5109,@Surf_Photo I squeezed a tear. Famine plague tsunami no chance. Lego - blubber!,0 +5110,Russian 'food crematoria' provoke outrage amid crisis famine memories - Yahoo News http://t.co/6siiRlnV6z,1 +5113,11-Year-Old Boy Charged With Manslaughter of Toddler: Report: An 11-year-old boy has been charged with manslaughter over the fatal sh...,1 +5114,Roger Goodell's Fatal Mistake: Tom Brady An Innocent Man http://t.co/UCNcKrnLow,0 +5115,Boy 11 charged with manslaughter in shooting death of Elijah Walker: The 11-year-old appeared Wednesday in a... http://t.co/gtcFfaCvam,1 +5116,Fatal attraction is common n what we have common is 'pain'......,0 +5117,APD investigating fatal shooting of 3 year old child in southeast Anchorage http://t.co/qYccvUubkr,1 +5119,Investigators shift focus to cause of fatal Waimate fire http://t.co/c9dVDsSoFn,1 +5120,@_AsianShawtyy ?????????? I'm sorry. But I'm out,0 +5122,Police: man in his early 30s dies after shooting on Vermont St.: http://t.co/HteZ4z48Od http://t.co/Eq4rXC9bb3,1 +5124,Man accused in fatal hit-and-run to get new judge http://t.co/j3Rtf2dt3X via @KHQA,0 +5125,Winnipeg police seek witnesses in Arlington and William fatal crash http://t.co/N2bCf4M64V,1 +5126,@ChrisDanielShow Nearly-Fatal Pee in San Francisco http://t.co/1tvlFrhm1m,0 +5127,11-Year-Old Boy Charged With Manslaughter of Toddler: Report: An 11-year-old boy has been charged with manslaughter over the fatal sh...,1 +5128,Petition to remaster Fatal Frame 1 with a Windwaker-style selfie mode,0 +5130,11-Year-Old Boy Charged With Manslaughter of Toddler: Report: An 11-year-old boy has been charged with manslaughter over the fatal sh...,1 +5131,Man charged over fatal crash near Dubbo refused bail http://t.co/HDBMfOVUtZ via @dailyliberal,1 +5132,California man facing manslaughter charge in Sunday's wrong-way fatal crash in ... - http://t.co/1vz3RmjHy4: Ca... http://t.co/xevUEEfQBZ,1 +5134,The Five Fatal Flaws in the Iran Deal https://t.co/ztfEAd8GId via @YouTube,0 +5135,"Stockton police are investigating a fatal drive-by shooting in north Stockton + http://t.co/j3x0DOY7R3",1 +5136,Deputies: Dog dispute leads to fatal shooting in Person County http://t.co/OazgHoZGYa #gunfail #nra,1 +5137,11-Year-Old Boy Charged With Manslaughter of Toddler: Report: An 11-year-old boy has been charged with manslaughter over the fatal sh...,1 +5138,CA Cops: Illegal Immigrant with 4 Prior Arrests Charged in Fatal Sex Attack http://t.co/BL59Fw4SYS,1 +5139,@spookyfob @feelslikefob I am okay thank you yes your kindness is fatal though it's like Patrick stump level kindness.,0 +5140,11-Year-Old Boy Charged With Manslaughter of Toddler: Report: An 11-year-old boy has been charged with manslaughter over the fatal sh...,1 +5141,Success is not permanent & failure is not fatal.,0 +5142,Investigators shift focus to cause of fatal Waimate fire http://t.co/a6Ro9bmXcy,1 +5144,11-Year-Old Boy Charged With Manslaughter of Toddler: Report: An 11-year-old boy has been charged with manslaughter over the fatal sh...,1 +5145,11-Year-Old Boy Charged With Manslaughter of Toddler: Report: An 11-year-old boy has been charged with manslaughter over the fatal sh...,1 +5146,"Tell me why or why not +to adopt in this way +maybe I overlooked something +fatal for me +?whyor why not?",0 +5148,Boy 11 charged with manslaughter in shooting death of Elijah Walker http://t.co/HUrIIVFDKC,1 +5149,Robert Ballew's log statements are always at the FATAL level.,0 +5150,fatal attraction,0 +5152,#news #crimes Police ID victim in fatal crash that closed I-65 near Franklin: One person is d... http://t.co/9h0ym9OFsv #justice #courts,1 +5153,11-Year-Old Boy Charged With Manslaughter of Toddler: Report: An 11-year-old boy has been charged with manslaughter over the fatal sh...,1 +5154,Investigators shift focus to cause of fatal Waimate fire http://t.co/aDSvDpNP3r,1 +5155,Boy 11 charged in fatal shooting of 3-year-old http://t.co/FJ7kcRliR7,1 +5157,11-Year-Old Boy Charged With Manslaughter of Toddler: Report: An 11-year-old boy has been charged with manslaughter over the fatal sh...,1 +5159,11-Year-Old Boy Charged With Manslaughter of Toddler: Report: An 11-year-old boy has been charged with manslaughter over the fatal sh...,1 +5160,Investigators shift focus to cause of fatal Waimate fire http://t.co/GoeTJGIhOp,1 +5162,EXCLUSIVE: In 179 fatalities involving on-duty NYPD cops in 15 years only 3å____ http://t.co/Cn1joMMUGH,1 +5164,#NYC #News Legionnaires death toll rises to 8 in New York City: All eight fatalities were of older adults with... http://t.co/IQJ1Z3jXx8,1 +5165,Let's fraction the vital need for Our fatalities. How would you break it down in #education #econom http://t.co/ZSqM8ihE1K,0 +5166,Las Vegas in top 5 cities for red-light running fatalities - News3LV http://t.co/eXdbcx4gCR,0 +5168,We are totally unprepared for an EMP attack. Both China Russia and probably Isis possess them. 90% fatalities. Why are we pushing them?,1 +5169,"City implores motorists not to speed after more reports of animal fatalities near nature reserves +-> http://t.co/hiKF8Mkjsn",1 +5170,@kyrikoni @ExpressandStar Who said veg and fruit was good for you. Hope there's been no injuries or fatalities.,0 +5171,Understand that there are no fatalities as no one was trapped inside the wholesale market. 8 fire engines and 40 fire fighters on scene.,1 +5172,As of the 6-month mark there were a total of 662 fatalities - 114 more than the first half of last year || http://t.co/reOz7H3Em8,1 +5174,#HSE releases annual workplace facilities data. Have a look | http://t.co/h4UshEekxm http://t.co/jNHNX3oISN,0 +5175,"#Seattle's deadliest red light runner intersections revealed + http://t.co/gHk9Xup6E0",1 +5176,5 Rejected Mortal Kombat Fatalities: Mortal Kombat has stretched the boundaries of its___ http://t.co/igZ7v24GE9 http://t.co/M75DNf2xyg,0 +5177,@unsuckdcmetro minor train leaves rails. Major = 1/13/82 Smithsonian Interlocking derails & crashes into barrier wall w/ fatalities.,1 +5178,Injuries Illnesses and Fatalities Latest Numbers : http://t.co/1uo1aTrbbJ,1 +5179,http://t.co/JwIv6WYW6F Osage Beach releases name,0 +5180,I wonder how Cool/Weird It'll look to have all the characters in MKX do fatalities on Skinless Predator ;),0 +5181,#Saudi Arabia: #Abha: Fatalities reported following suicide bombing at mosque; avoid area http://t.co/1xW0Z8ZeqW,1 +5182,"'Use our roads wisely and prevent the carnage from continuing...let us have no fatalities over this holiday period..' +#GGIndependencMessage",0 +5183,mortal kombat fatalities are so gross but interesting,0 +5184,The X-rays in MKX be looking like fatalities.,0 +5185,@dwilliams313 RT @Ieansquad: When you don__ªt know how to do fatalities in Mortal Kombat http://t.co/NU6wRp716d,0 +5186,"-??- +; kitana +? her fatalities slay me +___kody? (Vine by @KOMBATFANS33) https://t.co/uMajwSNLUF",1 +5187,OSP concerned about mounting fatalities http://t.co/YmP0gInwza http://t.co/FYIOQvIOif,1 +5188,"Mortal Kombat X: All Fatalities On Meat Predator. +https://t.co/IggFNBIxt5",0 +5189,Estimated damage and fatalities of an Hiroshima-sized atomic bomb dropped on your hometown - http://t.co/BSrERJbY0I #Hiroshima70,1 +5190,The RCMP are reporting fatalities and serious injuries as a result of the collision on the TCH near Whitbourne.,1 +5191,Govt allocating 1.3 bn for flood action: Issue takes centre stage after fatalities in Veneto http://t.co/w3esX6Ud8t,1 +5192,There have been zero spider bite-related fatalities in Australia since 1979.,0 +5193,I liked a @YouTube video http://t.co/43sXG9Z6xh TREMOR IS NO JOKE!! [TREMOR DLC] [FATALITIES/X-RAY],1 +5194,'Motordom' lobbied to change our language around traffic fatalities. We need to go back to the future #VisionZero https://t.co/cAvb7pgEpv,0 +5195,Sharing to help our cousin's family http://t.co/LSJowGYvQh,0 +5196,'Among other main factors behind pedestrian fatalities are people failing to yield to a car' http://t.co/dgUL7FfJt2,1 +5197,Mortal Kombat X is an excellent fatalities and the most fun I__ªve ever had with a mortal kombat SEGA's version http://t.co/fLO8fgy35A,0 +5198,Uber reduces drunk driving fatalities says independent study http://t.co/jVIVT6zrv7,0 +5199,I liked a @YouTube video from @kevinedwardsjr http://t.co/NCjLKqASUk Mortal Kombat X - PC Gameplay - Fatalities/X-Rays (MKX),0 +5200,Las Vegas in top 5 cities for red-light running fatalities http://t.co/kC8O81BcHG,1 +5201,RCMP confirm fatalities in two-vehicle crash: TCH remains closed at Whitbourne due to accident http://t.co/0RokDuTyUN,1 +5202,...American casualties including 400000___800000 fatalities and five to ten million Japanese fatalities.' https://t.co/TxzXCNUDr8,1 +5203,No UK train accident fatalities for 8th year running despite 4% increase in passenger journeys http://t.co/SuiLzS2S95 @healthandsafety,1 +5204,I fail to see how that would not bring the number of road fatalities down IMMENSELY,0 +5206,PolicyLab is at @CECANF's last public hearing in NYC today and tomorrow to address child abuse and neglect fatalities http://t.co/n2cY3Z4TPB,1 +5207,.@KurtSchlichter Yep considering that *millions* of Japanese fatalities were projected for Downfall. @_FreeMarketeer @dibang,1 +5208,#Shipping #Logistics eNCA | Fatalities as migrant boat capsizes in Med with hundreds onboard: ___ capsized as i... http://t.co/dnO7QXcFfh,1 +5209,Message boards will display updated traffic fatalities (up from 582) and new message beginning today: Traffic deaths 616 - Slow for Workers.,1 +5210,Driver fatalities down on Irish roads but pedestrians and cyclists at more risk http://t.co/E7OJhpdfG2,1 +5211,@EBROINTHEAM jay....big L....pun....biggie...wrap over...zero question....fatality...flawless victory http://t.co/Y33QcKq7qD,0 +5212,@Jake_ADavis @FaTality_US we are cuddling right now so.. ??,0 +5213,@Truly_Stings Yo Dm me,1 +5218,So these savages leaked Thomas Brady gangstermail account and wonder why he was quick to fatality his Samsung mobile? B real son.,0 +5219,I liked a @YouTube video from @vgbootcamp http://t.co/yi3OiVK2X4 S@X 109 - sN | vaBengal (ZSS) Vs. SWS | Fatality (Captain Falcon),0 +5220,Hibernating pbx irrespective of pitch fatality careerism pan: crbZFZ,0 +5221,Fatality!,0 +5222,Dying of Lyme disease: Case fatality rate nearly 100% http://t.co/RgT9GhODpW,1 +5223,@hrips_k when u do a fatality and like the corpse is still jittering,1 +5224,fatality,0 +5226,Wow fatality on 101 big rig hit motorcyclist blood everywhere ?? wow. I'm sick.,1 +5227,@Tellyfckngo @JayCootchi nah you hit homie wit the fatality and then son'd him wit the babality. Cold. Cold as fuck lmfaoooo.,0 +5228,Kosciusko police investigating pedestrian fatality hit by a train Thursday http://t.co/JILfbR0UfP,1 +5229,Fatality https://t.co/407V1y4HHg,0 +5230,@Bardissimo Yes life has a 100% fatality rate.,0 +5231,How many baskets did A Charming Fatality by @tonyakappes11 get? http://t.co/xyav4T5n0O #Mystery #Paranormal,0 +5232,They are making a Bad Boys 3 and 4!! A must see,0 +5233,@FaTality_US need a team? We need one.,0 +5234,Afghan Soldier Kills US General America's Highest-Ranking Fatality Since Vietnam http://t.co/SiHQPlUIDW,1 +5235,That usually NEVER happens,0 +5236,@Babybackreeve FATALITY!!!!!!!!!!,0 +5238,08/3/15: CAT FATALITY: UTICA NY; PLEASANT & HOLLAND AVE:Black cat with white paws. Average size. On grass next to north side of road.,0 +5242,@pxnatosil @RenuncieDilma Fatality!,0 +5243,[Jax(MK2)] Stage Fatality:UUD+LK (Close),0 +5244,'Seeing that there's a video of every fatality on Mileena makes me sad.',0 +5245,@Chrisman528 fatality ...,0 +5247,Kosciusko police investigating pedestrian fatality hit by a train Thursday http://t.co/m5djLLxoZP,0 +5248,Fatality https://t.co/GF5qjGoyCi,1 +5249,Fatality scary af. Probably not the best falcon but still scary.,0 +5251,I liked a @YouTube video from @vgbootcamp http://t.co/7LvGCMyIyJ S@X 109 - sN | vaBengal (ZSS) Vs. SWS | Fatality (Captain Falcon),0 +5252,Truck crash on 40w at US70 in Lebanon is a fatality. Very sad. Expect long delays through the morning.,1 +5254,@LindaSOCVAT @TfLBusAlerts Yes apparently. Report of cow fatality as well sadly.,1 +5255,I liked a @YouTube video from @deathmule http://t.co/sxHGFIThJw MK X Tremors Stalag Might Fatality on Ermac Tournament Pharaoh,0 +5256,I added a video to a @YouTube playlist http://t.co/O7QOgmOegU S@X 109 - SWS | Fatality (Captain Falcon) Vs. Snow (Fox) SSB4 Losers,0 +5257,Death of Loretta Fuddy responsible for authenticating Obama birth certificate the sole fatality of nine on plane http://t.co/MzRhfOJT2j,1 +5259,Fatality ????,0 +5260,Everyday is a near death fatality for me on the road. Thank god is on my side.??,0 +5262,Fear Factory in December. Done deal.,0 +5263,I made a friend more interesting than all of y'all combined https://t.co/isBtzUJFBm,0 +5264,"The meaning I picked the one that changed my life: overcome fear behold wonder. + +TCC catch the light... endless... http://t.co/eeRkH8ljws",0 +5265,@phnotf sometimes your cheekiness bleeds through my computer screen and i recoil in fear,0 +5266,cause i know every man has a fear of a strong-minded woman but I say she's a keeper if she got it on her own and keeps it runnin,0 +5267,My biggest fear is disappointing the people who believe in me,0 +5269,I didn`t want to hurt you but the fear drove me to do it ............... Midnight,0 +5270,Men fear the feeling of being 'controlled' by a woman. Or passive aggressively being coerced into commitment before they're ready...,0 +5271,The things we fear most in organizations--fluctuations disturbances imbalances--are the primary sources of creativity. - Margaret Wheatley,0 +5272,Fear Factory - Cars (Official MusicVideo) http://t.co/UUzaUMdObc,0 +5273,@Luzukokoti it's all about understanding umntu wakho. If you do and trust your partner then y OK u will know and won't fear to do anything.,0 +5274,Photo: referencereference: xekstrin: I THOUGHT THE NOSTRILS WERE EYES AND I ALMOST CRIED FROM FEAR partake... http://t.co/O7yYjLuKfJ,0 +5276,@BaileySMSteach The fear of not looking like you know what you are doing___ahhh___that was a big one for me. #PerryChat,0 +5278,"The Opposite of Love is Fear Here__ªs Why +http://t.co/r5bXZzhXkm",1 +5280,FEAR - YouTube http://t.co/PrmtxjJdue,0 +5281,Help me win $$$$ by having the most shares on my article! A Lifetime Of Fear http://t.co/9eh2lCQkxl Thanks! #BlackInAmerica #GrowingUpBlack,0 +5282,@CowgirlLawyer We must refuse to become a nation with everyone living in fear of being shot unawares by intoxicated &/or crazy people.,1 +5283,Horror films are made by incredibly creative artists. We interviewed 21 of them in depth for The Anatomy of Fear. http://t.co/J6mpdsx9Lk,1 +5284,"I want to be with you forever +Stay by my side +On this special night + +Fear and Loathing in Las Vegas/Solitude X'mas",0 +5285,my worst fear. https://t.co/iH8UDz8mq3,0 +5286,@shakjn @C7 @Magnums im shaking in fear he's gonna hack the planet,0 +5287,When the world say finish God says: don't fear - http://t.co/Q5qCoAo8jP #ChooseGod #RestoringPaths,0 +5290,'By replacing fear of the unknown with curiosity we open ourselves up to___ https://t.co/qRHng6kJ1C,0 +5291,Love God more than you fear hell,0 +5292,Don't tell the bride gives me the fear,0 +5293,@ScottDPierce @billharris_tv @HarrisGle @Beezersun I'm forfeiting this years fantasy football pool out of fear I may win n get my ass kicked,0 +5295,'...As of right now I'm reopening the X-Files. That's what they fear the most.' #TheXFiles201Days,0 +5296,Live a balanced life! Balance your fear of #Allah with hope in His mercy and love for Him.,0 +5297,my biggest fear is that eventually you will see me the way i see myself.,0 +5298,Couples having less sex... for fear it'll be a let down: Internet movies and books saying how sex 'ought to be' p___ http://t.co/c1xhIzPrAd,0 +5299,Nothing to fear. It's all about taking risks.,1 +5300,Untangle yourself from requiring your partner to fill your needs and how to love without fear #BestTalkRadio Listen http://t.co/8j09ZUTxWT,0 +5301,The Walking Dead spin off Fear the Walking Dead is on the 23rd ??????,1 +5302,@mcnabbychic I fear not for a while,0 +5304,The fear of the Lord is the start of knowledge: but the foolish have no use for wisdom and teaching (Amsal 1:7),0 +5305,The number of people denying climate change on the polar bear article makes me fear for the future of the US but also humanity in general.,0 +5306,"@SenSanders Gd ideas. I'm 77 wrked hard now have almost nothing fear lives with poor +Fran Reed 8437150124",0 +5308,Never let fear get in the way of achieving your dreams! #deltachildren #instaquote #quoteoftheday #Disney #WaltDisney http://t.co/bLZt5zwosE,0 +5309,Fear is my motivator,0 +5310,"Daily Reflections +August 6 +DRIVEN +Driven by a hundred forms of fear self-delusion self-seeking and self-pity... http://t.co/DXfqOu4kT2",0 +5313,Morganite Gemstone White Fire Opal 925 Sterling Silver Ring Size 6 R1354 http://t.co/hHpVSAtQXN http://t.co/D12r8XpShy,0 +5315,Marquei como visto Game of Thrones - 3x5 - Kissed by Fire http://t.co/CJHH17duli #bancodeseries,0 +5316,Save the Date! Saturday August 15 2015 is the Salisbury Fire Department Open House at 325 Cypress St 10am to... http://t.co/Oa6B0Z2H6Y,0 +5317,Why put out a fire when it's still burning.,0 +5319,@Miss_HoMaStToPa cause were on fire we are on fire were on fire now.. Yeah were on fire we are on fire were on fire nowwwwww,1 +5320,My avi And header combo is fire,0 +5321,http://t.co/iNkuv5DNTX #auction #shoes Retro 5 fire red http://t.co/1cvEGTIZOG,0 +5322,My asshole is on fire https://t.co/Y3FO0gHg8t,0 +5323,I'm crying that song just ended setting myself on fire https://t.co/i2El5aCrRW,0 +5324,Setting things on fire is always an option,1 +5325,Sitting around a fire sounds great right about now,0 +5326,I See Fire,0 +5328,But put 'Flood of Fire' at the top of the list. https://t.co/p5JPjgiipW,1 +5330,I'm On Fire. http://t.co/WATsmxYTVa,0 +5331,Politifiact: Harry Reid's '30 Percent of Women Served' Planned Parenthood Claim Is a 'Pants on Fire' Lie http://t.co/aMYMwWcpYm | #tcot,0 +5332,Dear @CanonUSAimaging I brought it ;) #CanonBringIt #Fire #CanonTattoo #MN #TheresMoreWhereThatCameFrom http://t.co/tCXxHdJAs6,0 +5335,@seanhannity @ScottWalker yeah just who we need another middle class hating republican out to get hard working cops fire fighters teachers,0 +5336,This fire is WAY too close wtf is going on ???? http://t.co/drf3mmRbyx,1 +5337,Fire Meet Gasoline always gotta get played twice lol,0 +5338,WCW @catsandsyrup THA BITCH IS FIRE,0 +5339,well it feels like im on fire.,0 +5341,her eyes and words are so icy but she burns like rum on the fire,0 +5342,#news Politifiact: Harry Reid's '30 Percent of Women Served' Planned Parenthood Claim Is a 'Pants on Fire' Lie... http://t.co/bMSeDZOfSV,0 +5343,Nothing like a good fire https://t.co/ItFbBz9xYC,1 +5344,@DaughterofNai Tenshi activated Yuki's fire card!,0 +5345,California been shaking...catching on fire....and overcharging ppl on rent for the past 150+ years it ain't going nowhere.,1 +5346,There's a fire in the Catalinas. Looks kinda cool. This picture doesn't do it justice. https://t.co/N0tAwGeZJx,1 +5347,@zaynmalik don't overwork yourself. Your album is gonna be fire just don't overwork or stress! I love you take care,0 +5349,@Justin_Ling I promise not to tax pancakes or rainbows or not dying in a fire.,0 +5350,This nigga Cyhi diss was what Meek was suppose to do. This shit actually fire,0 +5351,I wanna set some shit on fire.,1 +5352,In the words of Charlie Daniels 'Fire on the mountain run boys run.'___ https://t.co/fz1HAEj255,1 +5353,I just checked in! ___ at On Fire on @ZomatoAUS #LoveFood http://t.co/9l5kqykrbG,1 +5354,When your heart is bigger than the obstacles in front of you #euro #dontexpectnothing #july #fire @euro,0 +5355,Fire waves and darkness,0 +5356,It may seem like our fire has been a little burnt out....,0 +5357,Smelling a Bon fire while driving makes me wanna be sitting near one making s'mores ????,1 +5359,Will be dropping fire selfie tomorrow saying 'you're welcome' ahead of time.,0 +5362,Fire Call: BRANT AV / DRUMMOND RD for Fire - *Structure - Single. Units: CAR 6 On Call Truck http://t.co/euDwNFyUeM,1 +5365,Your PSA for the day: If a fire truck is behind you with lights going MOVE OVER!!! so they can get to their call.,0 +5366,#RFP: Fire Truck Service Body for F-450 (Fire fighting rescue & safety equipment Transporta... http://t.co/8GtRvEcE1N,0 +5367,#reno Truck trailer catches fire in Reno http://t.co/k5FIJaNkJb,1 +5368,wild night in the village of pugwash every fire truck is out that the town has which is like a fire truck for every house -population:6,1 +5371,'An 18-wheeler came along and struck the fire truck spinning it around 180 degrees and causing it to roll over' http://t.co/kmPfhGlhoo,1 +5372,SIGALERT UPDATE #3***N-133 CLOSED AT 5 FWY UFN***- TRASH TRUCK FIRE,1 +5373,@BrookTekle_ didn't look like a murder scene just 1 cops a fire truck and 2 fire assistance cars along with a helicopter,1 +5375,Former Township fire truck being used in Philippines - Langley Times http://t.co/L90dCPV9Zu #Philippines,1 +5377,Former Township fire truck being used in Philippines: Township of Langley assistant fire chief Pat Walker spen... http://t.co/r6YJw4xcKY,1 +5378,Our garbage truck really caught on fire lmfao.,0 +5379,@KapoKekito on northgate by the taco truck that's fire.,1 +5380,It's never a good sign when you pull up to work & there's five ambulances & a fire truck in the bay. Wompppp at least it's Friday,1 +5381,Fire truck and ambulance in K3 Phase 3. Hope everyone's okay. #PrayForSaipan,1 +5382,#SigAlert: North & Southbound 133 closed btwn 5 fwy and Irvine Blvd due to truck fire. CHP is detouring traffic.,1 +5383,@eeenice221 true because of the truck that caught fire?,1 +5384,@JustineJayyy OHGOD XD I didn't mean it so =P But you have that fire truck in the back of you to make up for it so you good xD,0 +5385,"Came across this fire video not mine..enjoy..Babes way of saying hi to me while he's in the fire truck?? +#fireman #___ http://t.co/V5gTUnwohy",0 +5386,133 N past the 5 L lane is reopened. All other lanes are closed. All lanes are open on the 133 S. Trash truck fire cleanup. @KNX1070,1 +5387,#SigAlert: North & Southbound 133 closed btwn 5 fwy and Irvine Blvd due to truck fire. CHP is detouring traffic.,1 +5389,Found this cool photo not mine 1952 Dodge Wayne bodied 4 window bus. Ex Libby MT Fire Department truck. All ori___ http://t.co/mcEjZzxgh8,1 +5391,Trident 90225 Chevy fire Truck w/ Pumper IMS Fire Dept. Red HO 1:87 plastic http://t.co/FAQNFUpeGn http://t.co/mQfOfKXtyh,0 +5392,Toddler Bedding Firetruck Bundle Fire Truck Firefighter Sticker Decals Sign Wall http://t.co/ykuAuOV9jO,0 +5396,when a monster truck racer catches on fire at the fair,1 +5397,Truck fire clogs canyon road http://t.co/JRDwyy0aX4,1 +5400,rgj: Truck trailer catches fire in Reno http://t.co/kAF3WdRmTn,1 +5403,#Philippines Former Township fire truck being used in Philippines: ___ of emergency equipment in Cebu Philippi... http://t.co/aL28RVkqPQ,1 +5404,theres a fire truck in this parking lot,0 +5406,Evening Sun article on NNO: http://t.co/7cMf3noYNc,0 +5407,'Choose to be happy. Fire truck the rest.' http://t.co/tAmRCWfYgd ????? Review of Kick Push by @JayMcLeanAuthor,0 +5408,Former Township fire truck being used in Philippines - Langley Times http://t.co/iMiLsFxntf #filipino,0 +5409,Jacksonville Fire & Rescue Engine 58 - AWESOME TRUCK! - OFFICIAL JADE FL... https://t.co/ybdSlJw7c1 via @YouTube,1 +5410,#NJTurnpike å_ #NJTurnpike Reopens Hours After Truck Fire In? http://t.co/oABJZtbVyZ http://t.co/GPBXRrDc07,1 +5411,#LukeBox something about first responders/ military they are our true Hero's!! Besides your music,0 +5412,I'm grateful for our first responders - @MNPDNashville @NashvilleFD and others - who acted so swiftly to save countless lives today.,1 +5413,@USATODAY. PRAYING FOR GOD'S HEALING AND SAFETY OF FIRST RESPONDERS,0 +5416,"Join #charity 10k #run event! @DoningtonDash +11am start Sun 20 Sept 2015 +Castle Donington Community First Responders +https://t.co/G1Nw99YJ8U",0 +5417,Loved the way this book was written to include so many vantage points of First Responders @DetKenLang #kindle http://t.co/KcRnMJKJ73,0 +5418,First responders would never be sent to the wrong address again w/ #SmartTek's GPS-based panic button #mPERS #safety http://t.co/3OionqlFQL,0 +5420,__÷We Can Help__ª Says Denver Firefighter Working To Curb First Responder Suicide http://t.co/WtaFaepuKZ,1 +5423,What's cool is that teens are becoming what I like to call 'digital first responders' for their friends who need a little help or support !!,0 +5425,Carmike Cinemas on Antioch Shooting: 'We Are Grateful' for Staff and First Responders Safety Is 'Highest Priority' http://t.co/BehfHspPud,1 +5426,RT RoadID: Thanks to Alex for his story & to all first responders for being there when we need you. ___ http://t.co/HikDC1fM2F,0 +5428,I just added 'Sandy First Responders Lost Their Homes' to VIP Home Page Group on @Vimeo: https://t.co/lKXi6UXjaQ,1 +5430,Thankful for our first responders especially @MNPDNashville for life saving response today in Antioch!,1 +5432,Juneau Empire - First responders turn out for National Night Out http://t.co/94UYT4ojYK,0 +5437,@Reds I don't throw the word hero around lightly... Usually reserved for first responders and military... But he's a hero...,0 +5439,US wants future first responders to be more high-tech - http://t.co/1VI2RNbK2i,0 +5442,There's still room for you at our party for first responders from around the country! 3rd annual best ever. http://t.co/mNh6FXhOdB,0 +5443,Incase of accident the first responders would know that there is a baby present https://t.co/sNaYMLDIUN,0 +5444,Kirsten Gillibrand http://t.co/amEA3LaMDj Extend Health Care To 911 First RESPONDERS !,1 +5445,'We Can Help' Says Denver Firefighter Working To Curb First Responder Suicide http://t.co/aVV6hPNpch,1 +5446,First Responders Convene for National Summit and Awards on GIS Technology http://t.co/0T9yd557rY #gisuserPR #geoTech,0 +5448,York Co. first responders compete to save lives in __÷Badges for Blood__ª #paramedic #EMS http://t.co/E65V80FCus,0 +5449,Some good info to help first responders cope- Individual Resilience: Factsheet for Responders- http://t.co/FcFpijiqt5,0 +5450,@IAFFLocal4416 no lives lost except those of first responders or the public they're charged with protecting?,1 +5452,First Responders get int @bridgeportspeed free on Saturday! Details @ http://t.co/dIh1TgQhej,0 +5454,At the #999day Emergency Services display! Meeting the brave police firefighters and first responders that help keep us safe! ????????,1 +5455,This week first responders and DART members are participating in a four day intensive Technical Large Animal... http://t.co/tL93AOd3ER,1 +5458,As firefighters make gains on #RockyFire Jerry Brown is heading to the area to meet with first responders tomorrow morning,1 +5459,After shooting event at the theater @Starbucks is giving FREE COFFEE to all first responders police & firefighters. AntiochHickoryHollow#TN,1 +5460,'NO LENDER FEES FOR VETERANS OR FIRST RESPONDERS (PAST AND PRESENT)' on @LinkedIn https://t.co/9pmBTxmoAL,0 +5461,New Giant Flames (GIANT FULL BLACK PANTOFEL) info/order sms:087809233445 pin:23928835 http://t.co/dthNEezupe pic.twitter.com/pNPiZoDY,1 +5462,*NEW* Snap On Tools Black baseball Hat/Cap Silver/Gray Embroidered S Logo Flames - Full re___ http://t.co/mO7DbBdFVR http://t.co/0ScNWe8XbV,0 +5465,*NEW* Snap On Tools Black baseball Hat/Cap Silver/Gray Embroidered S Logo Flames - Full re___ http://t.co/U3WAO8asFg http://t.co/hC6hZs4wSI,0 +5466,Vampiro going through the table of flames #UltimaLucha #LuchaUnderground @Elreynetwork http://t.co/Ox6OUw3Yut,0 +5467,Cabin Fever 2 flames https://t.co/yXnagsqvBM,0 +5468,"'if you can't summon the flames directly from hell store bought is fine'-me +mom-*dies*",0 +5469,Just added some more fire to the flames for Saturday! Rick Wonder will be spinning a guest set along with Chachi... http://t.co/Otblb9PJ2I,0 +5471,@kelworldpeace @TAXSTONE yoga flames!,0 +5473,#SBNation #Flames What Makes a Good Penalty Killer? http://t.co/xYi5fDacxO http://t.co/SjtvzgGcXU,0 +5475,i guess you can say ig's hopes went up in flames,0 +5476,I'll cry until my pity party's in flames ????,0 +5477,that new lil b x chance is nothing but flames,0 +5479,@gilderoy i wish i was good enough to add flames to my nails im on fire,1 +5480,My Gang Walking Round With Them Brown Flames. & Thats 100ND,0 +5481,*NEW* Snap On Tools Black baseball Hat/Cap Silver/Gray Embroidered S Logo Flames - Full re___ http://t.co/hTqVF44UQs http://t.co/PvXeLxCJeu,0 +5482,@TadhgTGMTEL dude was just smoking and the fucking thing went up in flames i though a bomb went off omg scared,1 +5484,Maryland mansion fire that killed 6 caused by damaged plug under Christmas tree report says - Into the flames... http://t.co/ucUDwIU3aN,1 +5485,New Giant Flames (Giant Manly Brown) info/order sms:087809233445 pin:2327564d http://t.co/T1mBw0ia3o http://t.co/CLfa0PY5Lm,0 +5486,___You see the devastation & it__ªs shocking.___ Firefighters continue to battle flames in California. benstracy reports ___,1 +5487,watching it go up in flames,1 +5488,RP said they can see smoke coming from the silo on 260th Street in Hartford but no flames.,1 +5489,I chose you so if we can search for the joy of just the two of us We__ªll be near smiles no matter what cold flames burn our bod #PortgassDK,0 +5491,Maryland mansion fire that killed 6 caused by damaged plug under Christmas tree report says - Into the flames: Firefighter's bravery...,1 +5492,@ErinMariefishy everyone is setting flames upon me,0 +5493,@Flames_Nation that's the optimistic side of me. No doubt it's tough. But it can be done. I feel like the games will pivot on defense.,0 +5494,Maryland mansion fire that killed 6 caused by damaged plug under Christmas tree report says - Into the flames... http://t.co/P1WmkXA9d8,1 +5495,"So Pardon me while I burst into flames. + +I've had enough of the world and its people's mindless games",0 +5496,@AWickedAssassin want to burst into flames! *Anna hugged him tightly*,0 +5497,@selmoooooo @_edvinnn @imTariik @dzafic_haris selmo u not catching flames from me im just gonna be witnessing the slaughter,1 +5498,'Bloody 'ell! Let it burn!' #RubyBot,0 +5499,If you have an opinion and you don't put it on thh internet you will furst into flames.,0 +5501,I have a rising follower graph! 2 more followers in the past week. Get your stats right here http://t.co/UlxF6dq3ns,0 +5502,@AisuMage @AkumaReisu --just between Gray and Ophelia red flames blazing around her person setting nearly everything in the area--,1 +5503,SoloQuiero Maryland mansion fire that killed 6 caused by damaged plug under Christmas tree report says - Into the flames: Firefighte...,1 +5505,Maryland mansion fire that killed 6 caused by damaged plug under Christmas tree report says - Into the flames... http://t.co/lKJFabQzb3,1 +5506,@CW_Hoops you better make all your shots tomorrow cause I'm recording and flames will be thrown tomorrow,0 +5507,I added a video to a @YouTube playlist http://t.co/aTCMrjzJTp Nicki Minaj - Up In Flames (Official Video),0 +5509,He better than Sean bro. I can admit that Sean is flames now. But he better than Sean https://t.co/aomQ1RYKmJ,0 +5510,The advantages apropos of in flames favorable regard mississauga ontario: pWHvGwax,0 +5514,"Flattened all cartoony-like. +'Whoa there Papa!' https://t.co/4zmcqRMOIs",0 +5515,the road to success is paved with pennies that I flattened on the train tracks you're welcome.,0 +5517,Feel like I've been flattened by a sumo wrestler ???? no pain no gain #muaytai #wantmyabsback #strong ????,0 +5518,IDEAS IN FOOD: Flattened http://t.co/QfrAWLn4BA,0 +5519,Zouma has just flattened him there ?? #CFC,0 +5520,Zouma has just absolutely flattened that guy ??,0 +5521,Flattened https://t.co/9jCIBenckz,0 +5522,70 years ago the first atomic attack flattened #Hiroshima 3 days later it was #Nagasaki both war crimes to put Moscow in its place,1 +5524,Today (August 6th) is the 70th anniversary of A-Bomb 'Little Boy' been dropped on Hiroshima.70000 killed outright as the city was flattened,1 +5525,Would have just flattened the little midget ?? https://t.co/BhufevaGPu,0 +5526,who said this? Yosemite Sam or Drumpf? ___Nobody __÷ill vote for a flattened out rabbit skin a-huh huh ___ I always say___,0 +5528,@GrabakaHitman @Izi_Garcia when he flattened machida...did he lose that fight..nope he lost fights to guys he shouldn't of lost to also,1 +5529,"#Floored4 #Flattened + +Early birds does get de cups .... so lag bout pun de dock and watch ppl drink___ https://t.co/r5StV25ZhQ",0 +5531,Conklin being flattened. Shuffled feet instead of kick slide. Didn't get hands up into chest. http://t.co/67TjN9EgyK,0 +5532,Flattened thee striker,1 +5534,Picking up flattened hay men (@ Masonite in Wahpeton ND) https://t.co/Kw3vq4niJQ,0 +5535,Zouma just flattened that guy??,0 +5538,Fylde Building set to be flattened: One of Preston__ªs city centre iconic buildings is disappearing from the skyline. http://t.co/PdKHBdG9hO,0 +5539,@iamHorsefly hide your kids hide your wife. He's on the loose. I thought I flattened you with a ball ????,0 +5540,Would've been great if Mbiwa just flattened that little rat.,0 +5541,@JimMozel puck=flattened ball lol,0 +5544,I don't doubt it. But it was his implicit statement in doing it that makes me want him flattened by a bus. https://t.co/5hlJUcxI0S,0 +5545,@KainYusanagi @Grummz @PixelCanuck and flattened raynor. Raynor was a balding imperfect biker marine not a emo generic western hero.,0 +5547,@SeanPeconi @Jason_Floyd @LynchOnSports @criscyborg I think the risk of losing and getting her nose flattened has a lot to do with it,0 +5548,Mum's literally just picked up her new car today and flattened the battery already trying to sort out the Bluetooth ???? #Muppet,0 +5552,"@tweetingLew @tersestuff + +Notley flattened Harper IN HIS Heartland +Harper Imported tens of thousands of TFW slaves COST ALBERTANS JOBS",0 +5553,@twilightfairy flattened frog?,0 +5554,Imagine getting flattened by Kurt Zouma,0 +5555,'the fallacy is it is up to the steam roller. It's up to the object whether it will be flattened or not.' #RobertCalifornia #thereisonlysex,0 +5556,Photo: Beach Earrings Beach Jewelry Vacation Earrings Keep Calm and Beach On Earrings Made with Flattened... http://t.co/rjEbpiB5rZ,0 +5557,100 1' MIX NEW FLAT DOUBLE SIDED LINERLESS BOTTLE CAPS YOU CHOOSE MIX FLATTENED - Full re___ http://t.co/w00kjPrfdR http://t.co/mIXl1pFRJe,0 +5558,@bigburgerboi55 flat footballs!!?? More like he flattened the Spartans from crushing them back in the day!!!! #HAIL,0 +5559,@SatanaOfHell ever seen by far. A dreamy look came over his face and his ears flattened as he started to walk slowly toward her.* You'd-,1 +5560,100 1' MIX NEW FLAT DOUBLE SIDED LINERLESS BOTTLE CAPS YOU CHOOSE MIX FLATTENED - Full re___ http://t.co/wRNpywyaLj http://t.co/Qv7IYdoVx9,0 +5561,when fizzy is sitting in the regular flood seats ...... and no one knows who she is.....,0 +5563,JKL cancels Flash Flood Warning for Bell Harlan Knox [KY] http://t.co/4rY6zhcPOQ #WX,1 +5564,Spot Flood Combo 53inch 300W Curved Cree LED Work Light Bar 4X4 Offroad Fog Lamp - Full re___ http://t.co/xxkHjySn0p http://t.co/JEVHKNJGBX,1 +5565,Spot Flood Combo 53inch 300W Curved Cree LED Work Light Bar 4X4 Offroad Fog Lamp - Full re___ http://t.co/jCDd6SD6Qn http://t.co/9gUCkjghms,0 +5567,A [small] flood with bigå_consequences https://t.co/CVPdVHxd1R http://t.co/FDMXP4FcMo,1 +5569,Homecoming Queen Killed on Way Home from the Prom by Flood Waters! #socialnews http://t.co/VmKexjTyG4,1 +5570,2pcs 18W CREE Led Work Light Offroad Lamp Car Truck Boat Mining 4WD FLOOD BEAM - Full rea___ http://t.co/VDeFmulx43 http://t.co/yqpAIjSa5g,0 +5573,"It's been raining since you left me +Now I'm drowning in the flood +You see I've always been a fighter +But without you I give up",0 +5575,@CreationMin @rwrabbit @GoonerAtheist @atheistic_1 @LOLatJesus Yet still why did so many fish die in the worldwide flood? I wonder.,0 +5576,And please don't flood poor @RobertBEnglund's mentions. He's put in his work!,1 +5577,Spot Flood Combo 53inch 300W Curved Cree LED Work Light Bar 4X4 Offroad Fog Lamp - Full re___ http://t.co/mTmoIa0Oo0 http://t.co/Nn4ZtCmSRU,0 +5578,@AtLarnxx but he DIDNT,0 +5579,12' 72W CREE LED Work Light Bar Alloy Spot Flood Combo Diving Offroad 4WD Boat - Full read___ http://t.co/XWN7rgVkzC http://t.co/SWPDQ84boI,0 +5580,2pcs 18W CREE Led Work Light Offroad Lamp Car Truck Boat Mining 4WD FLOOD BEAM - Full rea___ http://t.co/ApWXS5Mm44 http://t.co/DS76loZLSu,1 +5581,@hellotybeeren cue the flood of people 'ironically' calling you that,0 +5582,Motors Hot Deals #452 >> http://t.co/ED32PBviO7 10x 27W 12V 24V LED Work Light FLOOD Lamp Tractor Truck SUV UTV A___ http://t.co/IfM6v6480P,0 +5584,85V-265V 10W LED Warm White Light Motion Sensor Outdoor Flood Light PIR Lamp AUC http://t.co/NJVPXzMj5V http://t.co/Ijd7WzV5t9,0 +5586,iembot_hfo : At 10:00 AM 2 NNW Hana [Maui Co HI] COUNTY OFFICIAL reports COASTAL FLOOD #___ http://t.co/Gg0dZSvBZ7) http://t.co/kBe91aRCdw,1 +5587,1 pair new 27w 4'' ROUND LED Work Driving FLOOD Light Off-road ATV UTV - Full read by eBay http://t.co/d3P88xdLEc http://t.co/j2DDvh7fY0,0 +5588,2pcs 18W CREE Led Work Light Offroad Lamp Car Truck Boat Mining 4WD FLOOD BEAM - Full rea___ http://t.co/O1SMUh2unn http://t.co/xqj6WgiuQH,0 +5589,Internet basics: the flood defective intertissue is soul mate the milk trench host: GUAbxFv http://t.co/uzsQzYcB8X,1 +5590,BMX issues Areal Flood Advisory for Shelby [AL] till Aug 5 9:00 PM CDT http://t.co/62OddEkVLi,1 +5592,2pcs 18W CREE Led Work Light Offroad Lamp Car Truck Boat Mining 4WD FLOOD BEAM - Full rea___ http://t.co/1QT51r5h98 http://t.co/OQH1JbUEnl,0 +5593,Flood Advisory in effect for Shelby County in AL until 9 PM #alwx http://t.co/gTqMGsgcsB,1 +5594,RT grant_factory: #grants #funds Weymouth meeting on seawalls and flood grants set for Thursday night ___ http://t.co/Th2WLilBmo,1 +5595,Spot Flood Combo 53inch 300W Curved Cree LED Work Light Bar 4X4 Offroad Fog Lamp - Full re___ http://t.co/5xmCE6JufS http://t.co/3Zo7PX3p1V,0 +5598,Spot Flood Combo 53inch 300W Curved Cree LED Work Light Bar 4X4 Offroad Fog Lamp - Full re___ http://t.co/O097vSOtxk http://t.co/I23Xy7iEjj,0 +5599,Spot Flood Combo 53inch 300W Curved Cree LED Work Light Bar 4X4 Offroad Fog Lamp - Full re___ http://t.co/fDSaoOiskJ http://t.co/2uVmq4vAfQ,0 +5600,Flood Advisory issued August 05 at 4:28PM EDT by NWS http://t.co/D02sbM0ojs #WxKY,1 +5602,"survived the plague +floated the flood +just peeked our heads above the mud +no one's immune +deafening bells +my god will we survive ourselves?",0 +5603,Spot Flood Combo 53inch 300W Curved Cree LED Work Light Bar 4X4 Offroad Fog Lamp - Full re___ http://t.co/6zCfHi7Srw http://t.co/vWYkDaU1vm,0 +5604,Flood Prone Waterways In Westchester County Now Eligible For Millions In State Aid #NewYork - http://t.co/XSR2VUJyIz,1 +5605,Chinese rescue team arrives in Myanmar to help flood victims http://t.co/nOSvu0Zx4x Sittway http://t.co/5mJZJNkPfn,1 +5608,Flood Advisory issued August 05 at 7:10PM CDT until August 05 at 8:00PM CDT by NWS: ...THE URBAN AND ... http://t.co/SeMw5cQ7Dg #weather,1 +5609,12' 72W CREE LED Work Light Bar Alloy Spot Flood Combo Diving Offroad 4WD Boat - Full read___ http://t.co/MJMwA72ER6 http://t.co/ADx9iYi246,0 +5611,ROCK SPRINGS ___ Residents living off Elk Street continue to battle flooding issues after Monday__ªs rain storm. While___ http://t.co/8J19FvU7qA,1 +5612,Cuban leader extends sympathy to Vietnam over flooding at http://t.co/xLq8G6vb2r,1 +5613,Business at the Sponge Docks washed out by rain flooding: http://t.co/5PmikAVyKL,1 +5614,@Warlord_queen he thrusts a little more before he jams his cock deep inside her flooding her womb and pussy with his hot thick cum,0 +5617,Severe Thunderstorms and Flash Flooding Possible in the Mid-South and Midwest http://t.co/uAhIcWpIh4 #WEATHER #ENVIRONMENT #CLIMATE #NATURE,1 +5618,11:30a Radar Update: Widespread Showers/Storms but moving over the same areas-minor flooding possible #HUNwx http://t.co/E3L1JqjH2u,1 +5619,Tips to prevent basement leaks - Create control joints to eliminate random cracking. #homeimprovement #Flooding http://t.co/Kx8cU4s8T1,0 +5620,Flooding ???? http://t.co/WVeO9ED10e,1 +5621,@adorableappple No reported flooding po in the area. Ten-4. #mmda,0 +5622,Flooding kills 166 displace over one million in Pakistan - Daily News & Analysis http://t.co/9mj8oUj3vt #yugvani,1 +5624,Flooding kills 166 displace over one million in Pakistan - Daily News & Analysis http://t.co/KkOOVBKndp,1 +5625,#Laos flooding -10 villages underwater World Vision responding http://t.co/8fvQRizOUX,1 +5626,#floods #ukfloods - British trekkers rescued amid flash floods in Himalayas: A group of British tr... http://t.co/nsfzkfgZnj - #flooding,1 +5627,@SlopeOfHope Maybe the plan is to dilute until they can safely say 'We have to start charging or it's over'. Longs would come flooding in.,0 +5628,Cuban leader extends sympathy to Vietnam over flooding at http://t.co/QcyXwr2rdv,1 +5629,ILL ONLY DM HOTEL INFO BECAUSE PEOPLE ARE FLOODING MY NOTIFICATIONS ASKING ME FOR INFO,0 +5631,"When the waves are flooding the shore +And I can't find my way home anymore +That's when I look at you http://t.co/TDAKtGlU5p",1 +5632,If Flooding occurs in your home - How to stop mold growth via @ProudGreenHome http://t.co/KAVAovJz2V,1 +5633,http://t.co/XJkRXrNWNv Flooding of tracks caused derailment sa #INSubcontinent @INSubcontinent http://t.co/JlWGshYY3N,1 +5634,The Chinese are flooding NYC market like never before http://t.co/9z9HsmiaVD,0 +5635,Obama Keeps 27 Iraqi Christian Asylum Seekers nDetention While Flooding U.S. w/Unscreened Muslims: http://t.co/b4k0R4GgA8 via @DCClothesline,1 +5638,So grateful for all the support flooding in from @NetkiCorp dinner guests! THANK YOU ALL! https://t.co/ELTne5v1Qn,1 +5641,There some flash flooding in madison #valleywx,1 +5642,@themaine is it too soon to start flooding your comment section with 'come to brazil' comments? bc i will,0 +5644,And last year it was just a lot of 'THE DRUMS ARE FLOODING' and 'JANICE I'M FALLING',0 +5645,___@BBCEngland: A burst water main causes major flooding at a Birmingham hospital http://t.co/q4kGftC2AM http://t.co/6w6A2L4qAe___ OMG.,1 +5646,Heavy Rainfall and Flooding in Northern #VietNam | Situation Report No.2 http://t.co/hVxu1Zcvau http://t.co/iJmCCMHh5G,1 +5647,@crabbycale OH MY GOD THE MEMORIES ARE FLOODING BACK,0 +5650,#flood #disaster Burst Water Pipe Floods Apartments at NYCHA Senior Center - NY1: NY1Burst Water Pipe Floods A... http://t.co/w7SIIdujOH,1 +5651,LRT LOOK AT ALL MY TOM FEELS FLOODING BACK,0 +5652,Myanmar: MSF assisting thousands as worst flooding in decades hits parts of Myanmar http://t.co/PiJG5w2L2u,1 +5653,Residents in the central part of Ocean City heard from engineers about flood mitigation options Tuesday #OCNJ... http://t.co/jzPrCIqa9D,1 +5654,Wen I finally get the girl I want I'm flooding yall wit all pics of us Rs tho,0 +5655,@kwislo I see her flooding at mad decent,0 +5656,Monsoon Flooding Kills Dozens In Myanmar Prompting Calls For Help - http://t.co/r7vPaKlhvI,1 +5657,Memorial unveiled for Travis County deputy killed in Sept. flooding: Travis County Sheriff Greg Hamilton joined___ http://t.co/Eo2F96WXPz,1 +5658,"Flooding kills 166 displace over one million in Pakistan http://t.co/iCFQl7I9oP + +At least 166 people have been killed and nearly 11 lakh___",1 +5659,Thankkk U all Soo much for Flooding my NotificationsU my Fella ParShOlics r Superb & Jus soo awesomeLove Love U All always #FrvrGrateful ??,0 +5661,Slip Sliding Away - Flash Floods Info for Writers w/Tony Nester @SonoranRattler #writingtips http://t.co/sLTtOrRLHs,1 +5662,"Who is bringing the tornadoes and floods. Who is bringing the climate change. God is after America He is plaguing her + +#FARRAKHAN #QUOTE",1 +5663,Myanmar Flooding: Heavy monsoon rains during the month of July have caused flooding flash floods and landsli... http://t.co/9TG7A5OqFP,1 +5664,In #India 119000 people have taken shelter in the 966 relief camps the government has set up after the #floods: http://t.co/eU8jypIzsd,1 +5665,@pmharper don't worry I'm sure climate has nothing to do with hail floods tornados in Alberta & Ontario. I'm sure god's just mad at you.,1 +5668,"@ictyosaur I never thought it would be a wtf moment yet it's here after months of 90 degree heat +Next we will have flash floods..",1 +5669,Granted like half my town floods when it rains but still whatever,1 +5670,@madonnamking RSPCA site multiple 7 story high rise buildings next to low density character residential in an area that floods,0 +5671,@Cyberdemon531 i hope that mountain dew erodes your throat and floods your lungs leaving you to drown to death,1 +5672,Disaster group stunned by floods http://t.co/jrgJ17oAMt,1 +5673,there's this person & they reckon when you're dying your brain floods with dmt causing you to relive your life in real time in a simulation,0 +5674,Have you ever remembered an old song something you haven't heared for years? words that carry floods of memories all along. #priceless,0 +5675,Floods Fishing Finally Sunshine & Fab Deals from Albertsons Anniversary Sale |Lauren Paints | a beautiful life http://t.co/CwHSLMB8x9,0 +5676,#flood #disaster Bengal floods: CM Mamata Banerjee blames DVC BJP claims state failed to use ... - Economic T... http://t.co/BOZlwr716Z,1 +5677,69 Dead Due to Floods in Myanmar: Naypyidaw Aug 5 (Prensa Latina) The death toll rose today to 69 in Myanmar... http://t.co/JoDs9a32PI,1 +5680,Typhoon Soudelor approaches after 7 killed 2 missing in floods in Philippines http://t.co/nJMiDySXoF via @abc7chicago,1 +5683,@casewrites when it rains in NJ it flash floods. Otherwise its just a desert of grief and taxes.,1 +5684,Download @ iTunes http://t.co/ocojPPnRh1 'Floods Of Glory' by Luiz Santos #jazz #art #Music,0 +5685,@ContactEnergy Yep. During floods of 1999 or 2000 - Clyde Dam releasing every bit of water it could. Most speccy! EV charging your way?,1 +5686,Bamenda Floods Kill Animals Birds - http://t.co/WnamtxlfMt http://t.co/6cOIDv11qV,1 +5687,I hope it rains throughout the whole weekend I hope it floods and the portaloos become sentient.,0 +5688,APC Chieftain Tasks Dickson On N15b Floods Donation To Bayelsa http://t.co/LqGOe7psXp,1 +5689,PAGASA 7:12am: YELLOW warning - Panay Island Guimaras Negros. Possible floods in low-lying areas & landslides in mountainous areas.,1 +5690,RT: 40HourFamine: In #Bangladesh government has not declared floods an emergency. #WorldVision will continue to m___ http://t.co/XqdVghz8G6,1 +5692,Nearly 50 thousand people affected by floods in #Paraguay ? http://t.co/aw23wXtyjB http://t.co/ABgct9VFUa,1 +5693,ETP Bengal floods: CM Mamata Banerjee blames DVC BJP claims state failed to use relief funds: Even as flood w... http://t.co/hsZjaFxrvi,1 +5694,May Allah help all those suffering from the #Pakistan floods! You and your families are in our #Dua,1 +5695,Children in Myanmar face a 'double catastrophe' as floods hit the most ... http://t.co/0jFNvAXFph,1 +5698,love when drake floods instagram. makes you feel real in tune with everything he did like you was there or sumn.,0 +5699,"Who is bringing the tornadoes and floods. Who is bringing the climate change. God is after America He is plaguing her + +#FARRAKHAN #QUOTE",0 +5703,69 die in Myanmar floods 250000 affected http://t.co/LxjjGyv86A | https://t.co/U9x3perXCO,1 +5704,Typhoon Soudelor approaches after 7 killed 2 missing in floods in Philippines___ http://t.co/8rDXcfgQEm #Typhoon,1 +5705,Memphis always floods,1 +5706,Floods cause damage and death across Asia | All media content | http://t.co/a2myUTpDiQ | 05.08.2015 http://t.co/XrSkT0s9lz,1 +5707,">As soon as maintenance ends everyone floods the servers +>Servers destroyed by extreme load +>Maintenance starts anew",0 +5710,"Who is bringing the tornadoes and floods. Who is bringing the climate change. God is after America He is plaguing her + +#FARRAKHAN #QUOTE",0 +5711,E1.1.2 Particulate=Break up of Solid Combust Fossil Fuel Voltaic Active Forest Fire Biological VOC=Petroleum CH4 Bacteria Decomposition,1 +5712,BE CAREFUL anyone who lives west of Beaverton. Forest Grove has a rapidly spreading fire heading east,1 +5713,I added a video to a @YouTube playlist http://t.co/bcjYleRRYX Ori and the Bind forest ep 6 'Fire and death',1 +5714,How is it one careless match can start a forest fire but it takes a whole box to start a campfire?,1 +5715,Our little forest fire wardens http://t.co/aPreNsss3x,1 +5716,Fire in Pisgah National Forest grows to 375 acres http://t.co/d7zxZ42QW1,1 +5717,Ashenforest floorburnt manzanita & timber on Johnson Fire along ridge on Forest Road 1. #RouteComplex http://t.co/FYMP4I2Wp5,1 +5719,@Sweet2Young -runs at her for setting my forest on fire and bites the shit out of her neck-,1 +5720,@hornybigbadwolf -sets the forest on fire-,1 +5721,I liked a @YouTube video http://t.co/jK7nPdpWRo J. Cole - Fire Squad (2014 Forest Hills Drive),0 +5722,View of smoke column rising above the south end of the Blake Fire along Forest Road 1. #RouteComplex http://t.co/Yqg5Pvw5gX,1 +5724,IT STARTS A FOREST FIRE THAT CANNOT BE PUT OUT. http://t.co/3STfmN26r9,1 +5726,Fire in Pisgah National Forest grows to 375 acres http://t.co/dao9AZEUcr,1 +5727,I had to grill for a school function. One of the grills we had going was pretty much either off or forest fire. No inbetween! Made it work,0 +5729,5:15p For those who watch over me forest fire smoke has lifted 'nuff f/me t/make a run t/the pet store on the hwy (N). I'll b/home by 7p.,1 +5730,We are having forest fires out here in #socal but the only fire you should watch for is @YBTheProphet #RealHiphop https://t.co/ZPSnX5iYAJ,1 +5732,U.S. Forest Service firefighter David Ruhl 38 died in the 'Frog Fire' in the Modoc National Forest. He had been temporarily reassigned,1 +5733,USFS an acronym for United States Fire Service. http://t.co/8NAdrGr4xC,0 +5734,How is it one careless match can start a forest fire but it takes a whole box to start a campfire,0 +5736,Only you can prevent forest fires. ???? http://t.co/rGYUaKc0dR,1 +5739,#NaturalDisasters As California fires rage the Forest Service sounds the alarm about rising wildfire costs: ___... http://t.co/TQwrW3jQWo,1 +5740,Q: Why do ducks have big flat feet? A: To stamp out forest fires. Q: Why do elephants have big flat feet? A: To stamp out flaming ducks.,0 +5741,@gilmanrocks7 wow. Where is this? For the rest of the summer I usually have to worry about forest fires. Calgary is about 8 hours away.,1 +5743,As California fires rage Forest Service sounds the alarm about sharply rising wildfire costs http://t.co/dFYrPpzkPu http://t.co/iwsdbGd1zq,1 +5744,A little concerned about the number of forest fires where I'll be living,1 +5745,@nycdivorcelaw TRUMP IS A CLIMATE DENIER- algae bloom in the pacific from calif to alska- seeweed in caribean forest fires- SNOWBALL INHOFE,1 +5746,The forest service now spends >50% of its budget fighting fires http://t.co/SrSLCxHc2T,1 +5748,Cartoon Bears. Without them we would qave no knowlddge of forest fires or toilet paper.,0 +5751,This is the first year the Forest Service spent more than half its annual budget on fighting fires. #climatechange http://t.co/D62zfZy0Mi,1 +5752,As California fires rage the Forest Service sounds the alarm about rising wildfire costs http://t.co/Tft1bb4xaZ,1 +5753,Reddit: http://t.co/UtuMVaABz6 Lightning sparks new fires in Boise National Forest /r/StormComing,1 +5754,The spread of Conflict #PalmOil has sparked an increase in fires destroying #forest throughout #Indonesia http://t.co/4zn0MDsRVp,1 +5757,Photo: blue by @forest.fires source: http://t.co/awXR24zsqh http://t.co/o9A26Fn27y,1 +5758,I'm about to cook your Smokey the Bear saving forest fires face ass https://t.co/WtGGqS5gEh,0 +5759,Forest fires & dying salmon: time 2 act not deny. Climate Change Nightmares Are Here http://t.co/RBZomWGjeE #bcpoli #canpoli #vanpoli,1 +5760,Tales of the #trees #deep water loving #Lake Tahoe. And no #forest fires https://t.co/xuhMJ098Lq,0 +5763,"8-5-2015 - 4:30 P.M. - Progress Being Made on Boise Forest Fires +http://t.co/6o1mgMGHgt http://t.co/wTPO6elRZd",1 +5764,A group of Florida Forest Service firefighters could be deployed to California to help contain fires. Details at 10! http://t.co/fwuP9YURzY,1 +5765,PRAY! For EAST COAST FOREST FIRES! PRAY! That they be put out. PRAY! For RAIN!,1 +5766,@BlueJays @Braves @Angels Instead of dumping water on yourselves please send it to British Columbia to help with the forest fires,1 +5769,#HeartDisease U.S. Forest Service says spending more than half of budget on fires http://t.co/KzfiGkEeva,1 +5771,' no pharrell only YOU can prevent forest fires ' ??,0 +5772,CLIMATE CONSEQUENCES: U.S. Forest Service Says Spending More Than Half Of Budget On Fires http://t.co/k0QtL8aODH http://t.co/zQBXe7x9Y7,1 +5775,U.S. Forest Service says spending more than half of budget on fires http://t.co/1hLVrKwgIP,1 +5776,Property losses from northern California wildfire nearly double http://t.co/oTfW5SEkD7 via @YahooNews,1 +5777,Inciweb OR Update: Rogue River-Siskiyou National Forest Fires 8/5/15 12:00 PM (Rogue River-Siskiyou NF Area___ http://t.co/LkwxU8QV7n,1 +5778,Forest fires could delay @AbbyAirshow but officials say it could be a good thing http://t.co/Vxjcx8uKMd,1 +5780,U.S. Forest Service says spending more than half of budget on fires: By Ian Simpson___ http://t.co/UyYYrKd6q3,1 +5781,"Campsite recommendations +Toilets /shower +Pub +Fires +No kids +Pizza shop +Forest +Pretty stream +No midges +No snakes +Thanks ??",1 +5783,When ur friend and u are talking about forest fires in a forest and he tells u to drop ur mix tape out there... #straightfire,1 +5784,Firefighting consumes Forest Service budget sparks political clash: Forest Service report cites increasing cost of___ http://t.co/lSWsitnkuk,1 +5789,"GREAT MICHIGAN TECHNIQUE CAMP +B1G THANKS TO @bmurph1019 +@hail_Youtsey . @termn8r13 +#GoBlue #WrestleOn http://t.co/OasKgki6Qj",1 +5790,#np Avenged Sevenfold - Hail To The King,0 +5791,All Hail Shadow (Hybrid Mix Feat. Mike Szuter): http://t.co/9e2f7bIvlE @youtube ##youtube,1 +5792,@tremblayeh we like big hail and we cannot lie!! #SirMixAlot,0 +5793,LZK issues Severe Thunderstorm Warning [wind: 60 MPH hail: 0.75 IN] for Sharp [AR] till 8:15 PM CDT http://t.co/AUXSMdG1uN #WX,1 +5794,Two great 'dawgs' Dak and Jak !!! HAIL STATE !!! http://t.co/igyu2PEIU3,0 +5795,Strong Thunderstorm 4 Miles North of Japton Moving SE At 25 MPH. Large Hail and Wind Gusts Up to 50 MPH Poss... #arwx http://t.co/TIM8x9bI0f,1 +5797,RNK issues Severe Thunderstorm Warning [wind: 60 MPH hail: 1.00 IN] for Rockingham Stokes [NC] till Aug 5 10:00 PM EDT ___,1 +5798,Thank you so so much to everyone for posting the rain and hail outside ... I had no idea guys ????????,0 +5799,@chojo everyone hail to the pumpkin king!!!!,0 +5800,Cue storm with massive wind and hail!! Now a power outage @WXII in Davidson County.,1 +5801,@HeySeto Enjoy! Hopefully there will be no epic hail storms where you are going!,0 +5803,Strong Thunderstorm 4 Miles North of Japton Moving SE At 25 MPH. Large Hail and Wind Gusts Up to 50 MPH Poss... #arwx http://t.co/blppzAIbOE,1 +5804,All Hail Lady Cheryl!! 24 Reasons You Should Be Obsessed With Cheryl Cole http://t.co/kigy7M6bGJ,0 +5805,@KMacTWN @meaganerd Looks like a bowl of weather cereal. New! Kellogg's Sugar Hail! Stays crunchy even in milk...,0 +5807,Hail Mary Full of Grace The Lord is with thee. Blessed art thou among women and blessed is the fruit of thy... http://t.co/IDQcfJycYM,0 +5808,Seen on Fahlo:#WCW All Hail the Queenå_?? http://t.co/oLpBmy9xw9 #MTVHottest Justin Bieber http://t.co/ON18cqGcoA,0 +5812,@modnao23 the hail is ruining everything. Plus my car I haven't even gotten yet. Have yet another killer migraine and I lost my glasses. ??,1 +5814,@FaZe_Rain all hail the Cloud,0 +5815,New warning for Central Hills 1' hail 60 mph winds. NOT affecting Sturgis but could later tonight. #KOTAWeather http://t.co/1EPIYeNQYL,1 +5816,@Hail_Zel man you kno I'm there !,0 +5817,Hail! [pic] ___ https://t.co/B7omJ7U3EI,0 +5818,Heavy rain and lightning overhead in Oakridge SW Calgary - at least it will melt the hail......#yycstorm #ABStorm,1 +5819,@HeyItsEpark heavy rain and hail,1 +5821,@weathernetwork here comes the hail!,1 +5822,@Flow397 Coming atcha from Boston. Had golfball sized hail yesterday sunny amazing skies today! #ParkChat,0 +5824,Central Mass. fruit trees escape heavy damage after wind hail http://t.co/VbFfodtP6M,1 +5825,New warning for Central Hills 1' hail 60 mph winds. NOT affecting Sturgis but could later tonight. #KOTAWeather http://t.co/qo3VWFelkp,1 +5826,UNR continues Severe Thunderstorm Warning [wind: 60 MPH hail: <.75 IN] for Weston [WY] and Custer Fall River Pennington [SD] till 7:15 P___,1 +5827,Kevin Tan says hail to the chefs - ___ and Green Pastures which features sustainable and organic cooking.... http://t.co/D9xVuvp9s6,0 +5828,Hail The New Caesars! http://t.co/GzMoBlsJxu http://t.co/5CGtqfk2uR,0 +5829,Calgary pounded with hail on second day of storms https://t.co/2BE7BwcMpl,1 +5832,@DarkNdTatted pull up Holmes!,0 +5833,Hi-Res Doppler showing storm just NE of Edmond is now severe with 1.25' hail and 60mph wind possible. #okwx http://t.co/lrx0sDsNHM,1 +5834,UNR issues Severe Thunderstorm Warning [wind: 60 MPH hail: 0.75 IN] for Weston [WY] and Custer Fall River Lawrence Meade Pennington [S___,1 +5835,Well so much for outdoor postering @calgaryfringe #mothernature #hailstorm #rain #yycfringe #KILLHARD,1 +5836,@Haley_Whaley Hailstorm Hey There is a Secret Trick to get 375.000 Gems Clash ofClans check them now on my Profile,0 +5837,Nice job Calgary transit ! http://t.co/RGOGUyt0LF,0 +5840,Freak #Boston #hailstorm produces a hailstorm of business ... for auto-body repair specialists. @PeterHoweNECN 6:30/8:30 TONIGHT @necn,1 +5841,OMG NIXON LIVES! That is Richard M. Nixon Tricky Dicky right there in the picture isn't it. Hiding in Calgary he... http://t.co/MIUsvPxQTE,1 +5842,Hail:The user summons a hailstorm lasting five turns. It damages all Pokemon except the Ice type.,0 +5843,Calgary news weather and traffic for August 5 * ~ 90 http://t.co/qBdRYXSGlC http://t.co/VZOd7qFFlv,0 +5845,@AdriaSimon_: Hailstorm day 2.... #round2 #yyc #yycstorm http://t.co/FqQI8GVLQ4,1 +5847,~ More wicked weather rolls through Calgary and surrounding areas http://t.co/SxwJyR3K3l http://t.co/aEWGlVqReH,1 +5848,Gotta love #summer in #Calgary. #yyc #hailstorm #crazyweather http://t.co/xQbWnLBBIu,1 +5849,Severe weather and a rare hailstorm pummel Boston - http://t.co/jkYM9EdOfc,1 +5850,We The Free Hailstorm Maxi http://t.co/ERWs6IELdG,1 +5853,Those that I have sworn to defend have proven themselves to be friends of the House Hailstorm.,0 +5854,Twin Storms Blow Through Calgary ~ 1 http://t.co/rzaGDnWTAH http://t.co/uqEttaGSCU,1 +5855,Summer heat drives bobcats to Calgary backyards ~ 35 http://t.co/TmzXopVs94 http://t.co/W192Wkog1M,1 +5856,Ready for my close up... Errrr nope!! #notgoingoutinthat #hailstorm #alberta @HellOnWheelsAMC @HoW_fans @TalkingHell http://t.co/9gIAXD6JTY,1 +5858,IG: http://t.co/2WBiVKzJIP 'It's hailing again! #abstorm #yyc #hail #hailstorm #haildamage #yycweather #calgary #captureyyc #alberta #sto___,1 +5861,Summer heat drives bobcats to Calgary backyards ~ 3 http://t.co/kEstuUYc4t http://t.co/PzFBb1P1mj,1 +5863,Sadly the tent fly did not survive this hailstorm and now I have tears in the roof and water in the tent. Only... http://t.co/zCt5cchOJ0,1 +5864,This was Boston yesterday after an intense hailstorm [X-Post from /r/cityporn] [4608__2474] (___ http://t.co/8E6XuhrBKh http://t.co/Qo190N8UdD,1 +5867,My favorite text http://t.co/5U5GAkX2ch,0 +5868,"Canada: Hailstorm flash flooding slam Calgary knocks out power to 20k customers +http://t.co/SkY9EokgGB http://t.co/5IyZsDA6xB",1 +5869,If you're gonna take a break at work.... you gotta do it right! #CarlilesCanoeLivery #LoveMyJob http://t.co/Zo8RsqURa2,0 +5870,Severe hailstorm in progress over Northeast Logan County... #cowx http://t.co/XK9OwGV1O5,1 +5873,@CouncilSCC it does say hailstorm,1 +5874,"This is why I am scared to leave my car under trees in a storm + +#jamaicaplain #boston #hailstorm___ https://t.co/MJ8rEZOXlJ",1 +5875,Calgary news weather and traffic for August 5 * ~ 45 http://t.co/zAGBMlSf4H http://t.co/HVYXehXBmq,0 +5876,The Stephen Ave flower pots got a little ripped up in the hailstorm today #yyc #abstorm #calgary #iamdowntown http://t.co/hBhx0dwkPC,1 +5881,Grow Calgary avoids worst of city's wicked weather * ~ 16 http://t.co/HLyHDfWsQB http://t.co/GwSNBMmcqF,1 +5882,Photo: This is why I am scared to leave my car under trees in a storm #jamaicaplain #boston #hailstorm... http://t.co/1FHRrhciMH,1 +5883,Calgary Transit reviewing policy after leaving hundreds of commuters stranded during hail___ http://t.co/fT7OrfA52y http://t.co/Dv4MMlsO1I,1 +5884,600 passengers abandoned at LRT station during Tuesday's hailstorm http://t.co/vgF41IuPkn #yyc #yycstorm #abstorm,1 +5885,Quality Metrics Penalties May Harm Patient Care PCPs Say: Primary care physicians generally hold positive vi... http://t.co/TdwprVB04y,0 +5886,@tareksocal I think a lot of celebrities have to treat ppl as if they could harm them,0 +5887,"sticks and stones may break my bones +but words will never harm me",0 +5888,i s2g if anyone tries to harm my cupcake i'll fucking hunt you down to the end of the earth #HarryBeCareful,0 +5889,Vanderbilt: First Do No Harm http://t.co/cCdx7CGlQW,0 +5890,@leedsrouge Love what you picked! We're playing WORTH IT by FIFTH HARM/KID INK because of you! Listen & Vote: http://t.co/0wrATkA2jL,1 +5891,IMAGINE A DOCTOR TREATING AN #EBOLA PATIENT WITHOUT EVER PLACING HIMSELF IN HARM'S WAY. http://t.co/dliZfkk30Y,1 +5892,'But right now you're only annoyed by them. If you actually hung out with them you'd see that they mean no harm.' @AudaciousSpunk,0 +5893,I've burned myself before not in the self harm way in the I'm a dumbass and I shouldn't be allowed to be around fire way,0 +5894,I concur. The longer you spend with your child the more harm? Mmk https://t.co/dxwfX56pWh,0 +5896,PEOPLE KEEP NOT TAGGING SELF HARM AND IT'S FUCKING ME UP,0 +5898,@dinallyhot Love what you picked! We're playing WORTH IT by FIFTH HARM/KID INK because of you! Listen & Vote: http://t.co/0wrATkA2jL,0 +5899,I don't pray harm on members of ISIS.I pray they experience the life-rebooting love of God & become 'Paul's' in Gods mind-blowing final Act,0 +5900,Are the safety improvements being made to Route 198 in Buffalo doing more harm than good? http://t.co/rqlPSAYE3B,0 +5901,Is inner man wayward on route to harm spent replacing online surveys?: qZLOreMfT,0 +5902,If anything happens I will fucking fly 2 MetLife which is 3 hours away from me&beat d crap out of any1 who tries 2 harm d boys #OTRATMETLIFE,0 +5903,You can never escape me. Bullets don't harm me. Nothing harms me. But I know pain. I know pain. Sometimes I share it. With someone like you.,1 +5904,How standardized tests harm children of color and what we can do about it http://t.co/iD667rlEts Via @ParentsAcrossAm cc: @billgates,0 +5905,___For I know the plans I have for you___ declares the Lord ___plans to prosper you and not to harm you plans to... http://t.co/cIrTVml9Vp,0 +5906,@lauren_miller_7 she won't harm you,0 +5907,@angel_star39 Obama should feel responsible bringing in these illegals He- his family well protected from harm. No feeling or heart 4 others,0 +5908,I've been meaning to harm you in the best way I see fit??,0 +5909,@VileLunar I trickshot with a regular controller fucking infinite fading is so harm >:(,0 +5911,Sex Workers Say Credit Card Bans On Online Ads Do More Harm Than Good http://t.co/B9Zx2xZ6aW,0 +5912,HOW CAN SOMEONE HARM THIS ANGEL!! https://t.co/VgzGOK5k3S,0 +5913,IF ANYONE WERE TO HARM THE BOYS THEY WOULD GET TAKEN DOWN IMMEDIATELY NOT BY SECURITY BUT BY THE FANS REAL QUICK,0 +5916,"SO MANY SECURITY GARDS THIS IS GOOD. +I DARE THEM TO LET SOMEONE HARM HARRY #OTRAMETLIFE",0 +5918,someone just reblogged a picture of self harm scars oh please its 2015 can we stop,0 +5919,@wowsavannah what's the harm?? they're collectibles,0 +5920,Interview with Actor Randy Irwin A.S.K What Could Be The Harm http://t.co/k14q8cHWKp,0 +5921,@5hvzlaRadio Love what you picked! We're playing WORTH IT by FIFTH HARM/KID INK because of you! Listen & Vote: http://t.co/0wrATkA2jL,0 +5922,All I got in this world is my bros I don't wanna see no harm come they way Ima lil upset right now ??,0 +5923,@wwexdreamer talk to please don't harm your self in any way shape or form please we care about you and if I saw U right now U better,0 +5925,@malistkiss Sunnis continue to believe they are more righteous and they continually harm Shias. Defeats the ideals of Islam.,1 +5927,@news4buffalo yes a lot more harm then good if there are guardrails up now why cant we go 50. Their will be a big problem when school starts,0 +5928,@space_wolverine No harm no foul and somebody needed to say it.,0 +5929,@TasteMyCupCakee lmfaooo hell nawh ?? yo ass thought ??,0 +5930,self harm// I'm so angry please tag your scars on tumblr jesus christ i dont wanna get triggered,0 +5932,@RJG0789 idk....I feel like his movies have done more harm than good. They make us look sterotypical annddd colorism is prevalent sort of,0 +5933,Quality Metrics Penalties May Harm Patient Care PCPs Say - Primary care physicians generally hold positive views... http://t.co/0w12PwPSfx,0 +5934,Politicians are using false allegations to attack #PlannedParenthood & harm women. We aren't fooled we #StandwithPP http://t.co/eqDm2OpYbG,0 +5935,@ClassyColkett Thorgan Hazard made his move permanent go Gladbach this summer lmao,0 +5937,#Lifestyle __÷It makes me sick__ª: Baby clothes deemed a __÷hazard__ª http://t.co/0XrfVidxA2 http://t.co/oIHwgEZDCk,0 +5938,@LongBreastYat Yeah I don't think he's elite either I think Hazard is the better player too. But not by much,0 +5939,Choking Hazard Prompts Recall Of Kraft Cheese Singles http://t.co/nJLqRqcnL9,1 +5942,Seeing Hazard without the beard like... http://t.co/IPfzWWNXXP,0 +5943,Get that hazard pay,0 +5944,I liked a @YouTube video from @chaboyyhd http://t.co/Yr67ugEsrm Battlefield 4 Funny Moments - Dukes of Hazard Undercover Soldier,0 +5947,@phiddleface NOT IF THERES A CHOKING HAZARD!!! ???? dont die before i get there!!,0 +5949,Road Hazard @ CASCADE RD SW / CHILDRESS DR SW http://t.co/DilyvRoWyJ,0 +5950,Road closures remain in effect due to hazard trees falling tree torching and uphill runs of the fire. Forest Service Road #1 remains close,1 +5952,So now that Di Maria has left my dear #mufc back to the Hazard-Di Maria argument.... I'd say Hazard is way better ?? idc,0 +5953,Davis's Drug Guide for Nurses by Judith Hopfer Deglin and April Hazard Vallerand http://t.co/IufS7UV1HK http://t.co/AFrHnLLY8D,0 +5954,The Eden Hazard of Hockey https://t.co/RbbnjkoqUD,0 +5955,CONFIRMED: Sanchez Hazard and Bolasie will be out for the rest of the season. https://t.co/7Ct01nEptL,1 +5956,@ThatPersianGuy @YOUNGSAFE ?? Eden Hazard as Harden is spot on flopping is identical,0 +5957,I rate Hazard very highly but his fanboys are among the worst accounts on Twitter.,0 +5958,@DannyRaynard not bad personally I'd get rid of either hazard or aguero for a better striker than berahino,0 +5959,Choking Hazard Prompts Recall Of Kraft Cheese Singles http://t.co/XGKyVF9t4f,0 +5960,BREAKING: Arsenal's Hector Bellerin has been arrested for questioning regarding the disappearance of Eden Hazard.___ http://t.co/eCMJ18AzaI,0 +5961,Freak accident? Sure. Looking for someone to blame? Maybe. Remember that player broke his leg cuz cart was at back of end zone? Common sense,0 +5962,"December 2011 court dismissed the group charge of rape victims saying 'getting raped was an occupational hazard..!' + +US military = ISIS!",0 +5963,95-03 BMW 528 530 540 740 Emergency Warning Hazard Switch Button OEM 20177-707D http://t.co/kVNahTHUWZ http://t.co/Y8xkNpqMnJ,0 +5964,Kenya News (Chelsea talisman Eden Hazard keen to match Cristiano Ronaldo and Lionel Messi.) Mipasho http://t.co/LxvLqVbc8r,0 +5965,'Biggest lead hazard in New England history.' Yeah let's nail these guys. https://t.co/xzvmzQuS0x,0 +5966,@Dead_Dreamer15 ...because if it were on fire that'd be a safety hazard,1 +5972,Precious cargo onesie recalled for choking hazard. http://t.co/0PAMznyYuw,0 +5974,@TomDean86 he's alright but Hazard/Willian ain't gonna be shifted easily.,0 +5975,8/6/2015@1:32 PM: HAZARD IN HIGH TRAFFIC AREA at 5000 DEANS BRIDGE RD http://t.co/itZzKWfhG5,1 +5978,Battlefield 4 Funny Moments - Dukes of Hazard Undercover Soldier MAV T... https://t.co/JU8nfpnedl via @YouTube,0 +5979,@LondonFire Hi ..Is there an email that people can use to report s'thing they think is a fire hazard/dangerous?,1 +5980,"Our tipster previews Chelsea v Swansea & there's a 48/1 double! http://t.co/PFSrYJS1pc +#Chelsea #Hazard http://t.co/SKdBot7TGF",0 +5981,@ArianaGrande Girl you still lickin' public donuts??? Health hazard....u should be in jail.,0 +5982,Fair enough we have two of the best attacking wingers in the Prem in Hazard and Willian who will basically start every game 100%,0 +5983,8/6/2015@1:08 PM: HAZARD IN HIGH TRAFFIC AREA at 1500 WRIGHTSBORO RD http://t.co/DdUxtHvVnr,1 +5985,Wholesale #WE Gon Rep That $hit At All Costs- Hazardous #WholeTeam3 #WholesaleEnt https://t.co/JWnXH9Q5ov,0 +5987,I-10 EB at MS line update: Offloading the hazardous material is going much slower than expected. Road could stay closed until tomorrow AM.,1 +5988,#Taiwan Grace: expect that large rocks trees mud unstable and/or saturated land may slide ..very hazardous in hilly/mountain areas...,1 +5989,DLH issues Hazardous Weather Outlook (HWO) http://t.co/WOzuBXRi2p,1 +5990,JAX issues Hazardous Weather Outlook (HWO) http://t.co/fuCOQhcLAD,0 +5991,http://t.co/7AzE4IoGMe Risk Assessment and Optimization for Routing Hazardous Waste Collection #sustainable environmental,0 +5992,JAX issues Hazardous Weather Outlook (HWO) http://t.co/u9fCb8dz3h #WX,1 +5995,@igmpj aren't dangling piercing crystals potentially hazardous to the eyes? :),0 +5996,#foodscare #offers2go #NestleIndia slips into loss after #Magginoodle #ban unsafe and hazardous for #humanconsumption,1 +5997,#what #tribal Olap #world pres: http://t.co/Jw6FNnsSxT How To Recognize A Hazardous Waste And The Multidimensi http://t.co/4zAzTB19qE,0 +5998,Skinny Jeans are Hazardous for Your Health! #socialnews http://t.co/92Pk0HujD8,1 +5999,From recycling to only using non-hazardous chemicals Holland 1916 continually strives to maintain an eco-friendly existence.,1 +6000,#dam #gms Olap #world pres: http://t.co/mIcjNPuhG0 How To Recognize A Hazardous Waste And The Multidimensional http://t.co/Vddi5ChKTP,1 +6001,Never fear quarrels but seek hazardous adventures. https://t.co/dlvZaay7qr,0 +6002,09:13 PM: Hazardous Weather Outlook (http://t.co/ed1VpITsWY): NO HAZARDOUS WEATHER IS EXPECTED AT THIS TIME.... http://t.co/6XSbddlZiy,0 +6003,JKL issues Hazardous Weather Outlook (HWO) http://t.co/4e719w6m4V #WX,1 +6004,@HearItFromPa Also pls help us w/ our campaign to warn the public about the hazardous keratin treatments. The Brazilian Blowout COPYCATS.,0 +6007,MEG issues Hazardous Weather Outlook (HWO) http://t.co/gGn39m60tL #WX,1 +6009,@HearItFromPatty Also pls help us w/ our campaign to warn the public about the hazardous keratin treatments. The Brazilian Blowout COPYCATS,0 +6012,Caution: breathing may be hazardous to your health.,1 +6013,Is it possible to sneak into a hospital so I can stab myself with a hazardous needle and inject some crazy disease into my veins until I die,0 +6015,@TheBlackshag @dannyoneil too toxic...cancer....disease...hazardous waste...noxious...,1 +6017,Caution: breathing may be hazardous to your health.,0 +6019,MEG issues Hazardous Weather Outlook (HWO) http://t.co/3X6RBQJHn3,1 +6020,It's getting to be hazardous getting into this world alive. https://t.co/BJZSSw4tid,1 +6022,@Josh_LaDo Not tweeting and driving Joshua. Typos are hazardous.,0 +6023,#foodscare #offers2go #NestleIndia slips into loss after #Magginoodle #ban unsafe and hazardous for #humanconsumption,1 +6024,http://t.co/rOdpt33XFM EverSafe Emergency Auto Kit for all Weather Unsafe hazardous rush-hour gridlock jumper-ca___ http://t.co/0BVK5tuB4J,1 +6026,MTR issues Hazardous Weather Outlook (HWO) http://t.co/tGLK2UUs2Z,1 +6027,HAZARD - HAZARDOUS CONDITION at SB I5 FWY AT / SW TERWILLIGER BLVD PORTLAND OR [Portland Police #PP15000266858] 17:26 #pdx911,1 +6030,Skinny Jeans are Hazardous for Your Health! #socialnews http://t.co/pAQanenCeS,1 +6031,#foodscare #offers2go #NestleIndia slips into loss after #Magginoodle #ban unsafe and hazardous for #humanconsumption,0 +6032,#psd #special Olap #world pres: http://t.co/9xO9mKQqsi How To Recognize A Hazardous Waste And The Multidimensi http://t.co/BP03eAFEWR,1 +6033,DLH issues Hazardous Weather Outlook (HWO) http://t.co/a0Ad8z5Vsr #WX,1 +6034,Skinny Jeans are Hazardous for Your Health! #socialnews http://t.co/LTMa9xQXpx,1 +6036,RT @startelegram: Homeless vulnerable during North Texas heat wave http://t.co/k9aIrFQ3QL http://t.co/JdBTlyMEhY,1 +6037,@rawfoodbliss I'm in the middle of a humid heat wave and a patch on my forehead flared up. I take olive oli extract and 4 tbs olive oil,0 +6038,Heat Advisory is In Effect From 1 PM Through 7 PM Thursday. A Building Heat Wave and Increasing Humidity... #lawx http://t.co/u0SYkowVWV,1 +6039,"Many thx for share and your comment Alex Lightman - + +What evidence did it take or will it take for you or your... http://t.co/4Wsva9WO0F",0 +6041,Apocalpytic Iran Heat Wave Nearly Breaks World Record http://t.co/Y8WLOcTeVC,1 +6043,Japan Heat Wave Intensifies ___ Death Toll Surges To 55 http://t.co/irpSSresRq,1 +6045,@heebsterrr_ I remember the heat wave the year I went bruh and they don't have AC ????,0 +6047,"@WaseemBadami Condemning of Deaths More than 1000 due to Heat Wave in Karachi. +May Allah gv Patience to their Heirs. http://t.co/iTG84q7vIi",1 +6048,Something Frightening is Happening to the Weather in the Middle East http://t.co/bDOTQ8dSlN,1 +6049,@creationsbykole cork city in Ireland...we got to 17 degrees today...that is a heat wave for us haha,1 +6050,@hollywarnexx mini heat wave apaz,1 +6051,Arnhem Weather - <p>An unrelenting and dangerous heat wave will expand across the South Central United States___ http://t.co/yhAqa5WXoK,1 +6053,"UAE cool to Mideast heat-wave; rain watch latest [video] +NCMS issues warning of thundering rain poor visibility.... http://t.co/Tk65sKe0zm",1 +6055,@KlaraJoelsson Well I have seen it now! That's a bummer. We've had this heat wave tho... 43'c!! I'd prefer the rain... :P,1 +6056,It's a heat wave. #Squad #RevItUp #PizzaRev http://t.co/bp8bm8xSXw,1 +6057,#Jackson #ms Longest Streak of Triple-Digit Heat Since 2013 Forecast in Dallas http://t.co/Ih0Awv3L1O,1 +6058,Must Read Forecast! Longest Streak of Triple-Digit Heat Since 2013 What happens next? http://t.co/xXOuPfy8nQ http://t.co/A3BJabHQhe,1 +6059,@wfaaweather Pete when will the heat wave pass? Is it really going to be mid month? Frisco Boy Scouts have a canoe trip in Okla.,1 +6060,NOT.. Ready for this heat wave .. I don't want the sun to come back out.. I'm enjoying this break of cooler temps..,1 +6061,It's kinda cool tonight. Is the heat wave finally over?? ???? 7amdollela 3la kulli 7aal,1 +6064,Heat wave is ending! Watching a big area of rain though...will impact part of the area. Details NOW on NBC10 at 5pm,1 +6065,@rachelcaine The weatherit needs to make it minds up. First snow tornadoes now would you say a heat wave?,1 +6066,Chilli heat wave Doritos never fail!,0 +6069,Heat wave in WB heavy losses and no compensations (report) - http://t.co/wMDihdiz1r (via PalinfoEn) #Palestine,1 +6070,Heat wave adding to the misery of internally-displaced Gazans http://t.co/jW3hN9ewFT via @PressTV http://t.co/NYWrkRQ7Kn,1 +6071,Cherry print + matching lipstick (just rediscovered Nars__ª __÷Heat Wave__ª).?? by @emilyschuman http://t.co/4eGh1G1Jk7,0 +6072,Longest Streak of Triple-Digit Heat Since 2013 Forecast in Dallas: An unrelenting and dangerous heat wave will... http://t.co/s4Srgrmqcz,1 +6073,Heat wave gotta be over 9000 today,1 +6074,@Startide It's hotter there than Phoenix this week with 3x the humidity. >.>; Heat wave.,0 +6075,#greatbritishbakeoff love to know where I was when all this nice weather happened! Did miss the heat wave ?? ??,0 +6076,@kristenKOIN6 Yay good cooler weather for PDX..ABQ NM is feeling the heat wave now bcuz my rain dances aren't working :-),0 +6078,Longest Streak of Triple-Digit Heat Since 2013 Forecast in Dallas http://t.co/xc96rWUSZb http://t.co/XM9stfzcpV,1 +6083,Have you heard __÷05 LOVE TO LOVE YOU....HEAT WAVE VOL 5__ª by George deejayempiresound on #SoundCloud? #np https://t.co/rQiuqXNM2X,0 +6084,weather warfare Follow the Money This Government Is What Our Founding Fathers Were Warning Us About The Weather... http://t.co/TgtCRU8jiO,1 +6085,Orchid - Sign Of The Witch http://t.co/YtkXwPyIHg,0 +6086,"day 3 without my phone and due to my slow shite computers it is utter hellfire :'( +Plez send help",0 +6087,The Prophet (peace be upon him) said 'Save yourself from Hellfire even if it is by giving half a date in charity.',0 +6088,Hellfire is surrounded by desires so be careful and don__ªt let your desires control you! #Afterlife,0 +6090,The Prophet (peace be upon him) said 'Save yourself from Hellfire even if it is by giving half a date in charity.',0 +6091,@DeeDee_Casey been on tour in the day to Hellfire caves! Would love to investigate the place though!,0 +6092,Beware of your temper and a loose tongue! These two dangerous weapons combined can lead a person to the Hellfire #islam!,0 +6093,HELLFIRE EP - SILENTMIND & @_bookofdaniel https://t.co/FKqJY3EzyG,0 +6094,#Allah describes piling up #wealth thinking it would last #forever as the description of the people of #Hellfire in Surah Humaza. #Reflect,0 +6096,Beware of your temper and a loose tongue! These two dangerous weapons combined can lead a person to the Hellfire #islam!,0 +6097,The Prophet (peace be upon him) said 'Save yourself from Hellfire even if it is by giving half a date in charity.',1 +6098,Hellfire! We don__ªt even want to think about it or mention it so let__ªs not do anything that leads to it!,0 +6099,Hellfire is surrounded by desires so be careful and don__ªt let your desires control you! #Afterlife #islam,0 +6100,@IAN_Hellfire you said it's rude based on an experience. Kind of a next level sub tweet in a way lol. Either way you got worked up,0 +6102,@JYHeffect my good you stay in NY??? ?,0 +6103,@HellFire_eV @JackPERU1 then I do this to one of them ????,0 +6104,(Also I dont think sewing thought a leather belt would work out that well lol),0 +6105,#Allah describes piling up #wealth thinking it would last #forever as the description of the people of #Hellfire in Surah Humaza. #Reflect,0 +6106,@IAN_Hellfire I got it for the mistake but boss got it worse cause their job was to oversee my work. Boss didn't change after that...,0 +6108,That hellfire song from the hunchback of notre dame reminds me a lot of my house,0 +6109,the message you sent and they don't reply. I see that you saw my message the least you can do is tell me to 'fuck off' or something.,0 +6110,Beware of your temper and a loose tongue! These two dangerous weapons combined can lead a person to the Hellfire #islam!,0 +6111,The Prophet (peace be upon him) said 'Save yourself from Hellfire even if it is by giving half a date in charity.',0 +6112,Hellfire is surrounded by desires so be careful and don__ªt let your desires control you! #Afterlife,1 +6113,Hellfire! We don__ªt even want to think about it or mention it so let__ªs not do anything that leads to it #islam!,0 +6115,New cocktail on the list! El Diablo Mas Verde: mezcal yellow chartreuse honey cucumber hellfire bitters.... http://t.co/REuosJEK4m,0 +6116,Hellfire is surrounded by desires so be careful and don__ªt let your desires control you! #Afterlife #islam,0 +6118,The Prophet (peace be upon him) said 'Save yourself from Hellfire even if it is by giving half a date in charity.',0 +6119,Hellfire! We don__ªt even want to think about it or mention it so let__ªs not do anything that leads to it!,0 +6120,@gg_keeponrockin @StrawberrySoryu Oh okay I just got the message twice and got suspicious. Sorry. I'll check it out!,1 +6123,#Allah describes piling up #wealth thinking it would last #forever as the description of the people of #Hellfire in Surah Humaza. #Reflect,1 +6125,Hellfire is surrounded by desires so be careful and don__ªt let your desires control you! #Afterlife,0 +6126,Fel Lord Zakuun is about to DIE ! #Hellfire #WOD http://t.co/x1oNV3d5uX,1 +6127,Hellfire Gargoyle Hoof coil http://t.co/2ii3Brc7NX,0 +6128,@MechaMacGyver Wow bet you got blamed for that too huh?,0 +6130,@IAN_Hellfire again not really. Do you have any idea how many times i have been ignored for fadc. You can't expect anything.,0 +6132,The Prophet (peace be upon him) said 'Save yourself from Hellfire even if it is by giving half a date in charity.',1 +6133,@USArmy has entered into the #JAGM project to replace #TOW and Hellfire missiles. Check it out here: http://t.co/2mnQC73hFk,0 +6134,Hellfire! We don__ªt even want to think about it or mention it so let__ªs not do anything that leads to it #islam!,1 +6135,Tension In Bayelsa As Patience Jonathan Plans To Hijack APC PDP - http://t.co/ComLG0VdbV,1 +6137,"#GamerGate 'Our entire attempt to hijack your community destroy your industry and scam everyone in sight was just sarcastic'. + +Drop dead.",0 +6138,Bayelsa poll: Tension in Bayelsa as Patience Jonathan plans to hijack APC PDP: Plans by former First Lady and... http://t.co/86RSYy2tng,1 +6140,@dreamoforgonon @TeeEss not to hijack but as a bona fide cislady I can confirm this as true; incidental homosexuality =/= gay/bi for women.,1 +6141,0-day bug in fully patched OS X comes under active exploit to hijack Macs http://t.co/sbGiRvQvzb,0 +6145,0-day bug in fully patched OS X comes under active exploit to bypass password ... - Ars Technica http://t.co/F7OgzrNPfv,0 +6146,REVEALED: Everton hijack United bid for 14-year-old WONDERKID! - http://t.co/nb1E7mNcE5,0 +6147,I've just posted on my Blog about: Criminals Who Hijack Lorries And Buses Arrested In Enugu (PHOTO) http://t.co/5ytIeX55lh,1 +6148,Criminals Who Hijack Lorries And Buses Arrested In Enugu (PHOTO) @DONJAZZY @PoliceNG #HumanRights https://t.co/XyFl8wy62g,1 +6150,Criminals Who Hijack Lorries And Buses Arrested In Enugu (PHOTO) http://t.co/LRTU8Rwn2f,1 +6151,Tension In Bayelsa As Patience Jonathan Plans To Hijack APC PDP http://t.co/epABiNcZmJ http://t.co/1SgzGtgfw9,1 +6152,Plans by former First Lady and wife of ex-President Goodluck Jonathan Dame Patience Jonathan to hijack the All... http://t.co/HaShGQAFic,1 +6153,@mockingpanems @cuddlesforjen what if he slammed her against the wall for the wrong reason but then he came out of hijack mode and it,0 +6154,Patience Jonathan On The Move To Hijack APC In Bayelsa State http://t.co/rJac5sItEp http://t.co/Plqf1bikS4,1 +6156,Criminals Who Hijack Lorries And Buses Arrested In Enugu: According to the Nigerian Police Force... http://t.co/FfKcj8pfj2 Via @Music212,1 +6159,Patience Jonathan On The Move To Hijack APC In Bayelsaå_State http://t.co/Vh8QtbyPZt,0 +6160,Swansea ?plot hijack transfer move for Southampton target Virgil van Dijk? http://t.co/NZRWDdLntp,0 +6162,Jeep Cherokee Owners File Lawsuit Against Fiat Chrysler Harman After Hackers ... - The Consumerist http://t.co/2hocEP41kH #mcgsecure,0 +6163,0-day bug in fully patched OS X comes under active exploit to bypass password ... - Ars Technica http://t.co/53P4oOO1XN,0 +6164,{:en}Tension In Bayelsa As Patience Jonathan Plans To Hijack APCå_PDP{:} http://t.co/W1ufeibALa http://t.co/bIULQpRxke,1 +6165,Swansea __÷plot hijack transfer move for Southampton target Virgil van Dijk__ª http://t.co/PVmr38LnvA,1 +6166,@welshninja87 click on the tag there's lots of them. RT them to hijack the hashtag,0 +6167,"Another Mac vuln! + +https://t.co/OxXRnaB8Un",0 +6169,[Latest Post] Bayelsa poll: Tension in Bayelsa as Patience Jonathan plans to hijack APC PDP http://t.co/B2yvLMPepR,0 +6170,#LatestNews: Tension In Bayelsa As Patience Jonathan Plans To Hijack APC PDP,1 +6171,Tension In Bayelsa As Patience Jonathan Plans To Hijack APC PDP http://t.co/qxXN6RKsp6 http://t.co/B3X1wqzAoR,1 +6173,Stay cautious. http://t.co/JeJC9XcTMp,0 +6174,Bayelsa poll: Tension in Bayelsa as Patience Jonathan plans to hijack APC PDP: Plans by former First Lady and... http://t.co/3eJL9lZlCH,0 +6175,@RickyBonesSXM fuck u2 specially there new Shit. First they hijack itunes now all sirius chan. fuck off already haha just sayin,0 +6178,Tension In Bayelsa As Patience Jonathan Plans To Hijack APC PDP - http://t.co/NIpZmfLiBD,1 +6181,Swansea __÷plot hijack transfer move for Southampton target Virgil van Dijk__ª http://t.co/PVmr38LnvA,1 +6183,Bayelsa poll: Tension in Bayelsa as Patience Jonathan plans to hijack APC PDP Plans by former First Lady and... http://t.co/RtfirzCPOC,0 +6184,@jasoncundy05 Chelsea need to hijack Man Utd deal for Pedro..20 mill bargain Adam driving home in Oregon USA cmon blues!!!,0 +6185,Governor Allows Parole for School Bus Hijacker http://t.co/JwfiiLJ8Wr liveleakfun ? http://t.co/IONWArVRFy,1 +6187,Governor allows parole for California school bus hijacker who kidnapped 26 children in 1976. http://t.co/hdAhLgrprl http://t.co/Z1s3T77P3L,1 +6188,Gov. Brown allows parole for 1976 Chowchilla school bus hijacker James Schoenfeld. http://t.co/GlWoAKppjq http://t.co/WlLfLFnDgG,0 +6191,Remove the http://t.co/VbqmZ5aPwj and Linkury Browser Hijacker http://t.co/C2EyjNyBfN http://t.co/gt7gf0fSeX,1 +6192,San Jose Mercury: Governor Brown allows parole for California school bus hijacker http://t.co/GpCeCp9kHv,0 +6193,Remove the http://t.co/Xxj2B4JxRt and Linkury Browser Hijacker http://t.co/9gtYlgXrOE http://t.co/yG6Rj86BKI,0 +6195,The Hijacker-Turned-SAT-Tutor Who Evaded Authorities for Three Decades http://t.co/cSXvwXUZ6t via @slate,1 +6196,Governor Allows Parole for School Bus Hijacker http://t.co/DzlPNP399x,1 +6197,Governor Allows Parole for School Bus Hijacker http://t.co/liKWQhSHHX via @nbcnews I remember this crime sent chills through parents,1 +6198,School Bus Hijacker Given Parole After 39 Years http://t.co/HmRt98OydJ,1 +6199,Remove the http://t.co/JAb541hHk0 and Linkury Browser Hijacker http://t.co/Je6Zjwh5uB http://t.co/fDxgmiwAEh,1 +6200,Remove the http://t.co/l4wJHz4AJ6 and Linkury Browser Hijacker http://t.co/KMgDv7VSAz http://t.co/byTsfms7Md,0 +6201,Governor Allows Parole for School Bus Hijacker http://t.co/u4bdy1W7d4,1 +6202,How to remove http://t.co/HRSfA8zujI Browserå_Hijacker http://t.co/vLnjNoLNme http://t.co/iF5RoGFJRi,0 +6203,Complete Solution to Get Rid of http://t.co/9CntP3nQ6o ___ [Browser Hijacker Removal Guide]! - http://t.co/Qdf6ASaeLM,0 +6206,Remove the http://t.co/2nS5TfnxpA and Linkury Browser Hijacker http://t.co/W2kVScbTLp http://t.co/tn8o00NrLP,0 +6207,Remove the http://t.co/9Jxb3rx8mF and Linkury Browser Hijacker http://t.co/B5s5epJ7Um http://t.co/hPA9GQRyWa,0 +6208,Medieval airplane hijacker testa: earnings the distinction divers: HtaRvrGLY,1 +6211,Governor allows parole for California school bus hijacker | Fresno Linked Local Network http://t.co/Sww0QsMxVM http://t.co/bcdP4gKokA,0 +6213,California School Bus Hijacker Parole Stands http://t.co/kPIVXGjNqt #sacramento,1 +6215,Governor allows parole for California school bus hijacker: Local... http://t.co/tAM6aoskoJ http://t.co/eL24mnFcHw,1 +6216,__÷Hijacker cops__ª back in the dock - http://t.co/9I5cczD5S0 http://t.co/WEaTrRihE1,1 +6217,Governor weighs parole for California school bus hijacker http://t.co/7NPBfRzEJL http://t.co/Y0kByy8nce,0 +6218,Governor Allows Parole for School Bus Hijacker http://t.co/N71hMveRvv,1 +6219,Governor weighs parole for California school bus hijacker http://t.co/yFPpIFDkQO http://t.co/aJYUlMFTIF,1 +6220,RT NotExplained: The only known image of infamous hijacker D.B. Cooper. http://t.co/JlzK2HdeTG,0 +6222,Governor allows parole for California school bus hijacker http://t.co/FZ8YtWQkwV #fresno,1 +6223,RT NotExplained: The only known image of infamous hijacker D.B. Cooper. http://t.co/JlzK2HdeTG,1 +6224,@Mauds99 @JagexSupport https://t.co/uSH59Uq30j Change password through that link it'll kick the hijacker off.,0 +6226,@JagexHelpDibi update JAG enabled but a hijacker has access might be what I was looking for. Fingers crossed.,0 +6227,Demco 8550013 Hijacker 5th Wheel Hitch 21K Ultra Series Double Pivot http://t.co/hRdwGfbFYq http://t.co/nUOhKmPZFj,0 +6230,Medieval airplane hijacker hard shell: casting the star deviating: dYxTmrYDu,1 +6232,Remove the http://t.co/7IEiZ619h0 and Linkury Browser Hijacker http://t.co/tFeaNwhH2h http://t.co/SjicbhzFo4,0 +6233,How to Remove Softenza Hijacker? Softenza Anthelmintic Nature book Drawing out Help SEA,0 +6234,Governor allows parole for California school bus hijacker - Santa Cruz Sentinel http://t.co/TaXUxp9QA2,1 +6240,The Murderous Story Of America__ªs First Hijacking http://t.co/cbPs1gskvO,1 +6241,@ladyfleur The example I used is even worse in that it's a caf_© trying to market itself w/hashtag hijacking.,0 +6243,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/gUJNPLJVvt #prebreak #best,0 +6244,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/J2aQs5loxu #prebreak #best,1 +6245,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/s4PNIhJQX7 #prebreak #best,0 +6246,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/cx6auPneMu #prebreak #best,0 +6247,The Murderous Story Of America__ªs First Hijacking: Earnest Pletch__ªs cold-blooded killing of___ http://t.co/B9JAxx0vCf,1 +6248,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/wvTPuRYx63 #prebreak #best,0 +6253,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/J5onxFwLAo #prebreak #best,0 +6254,The Murderous Story Of America__ªs First Hijacking http://t.co/EYUGk6byxr,1 +6255,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/nQiObcZKrT #prebreak #best,0 +6256,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/aAtt5aMnmD #prebreak #best,0 +6257,Hijacking Electric Skateboards to Make Them Saferå_ | @scoopit http://t.co/ihInj3eNQi,0 +6258,__÷Good Samaritans__ª shot in horror hijacking JOHANNESBURG. &mdash; Four men were shot dead in Bronville Free State___ http://t.co/6jjvCDN4TI,1 +6259,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/qj3PVgaVN7 #prebreak #best,1 +6261,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/6AqrNanKFD #prebreak #best,0 +6262,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/8JcYXhq1AZ #prebreak #best,0 +6263,The ship has arrived safely. So it was quite unnecessary to sign the waiver that we won't sue Microsoft if any hijacking occurred.,0 +6265,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/gexHzU1VK8 #prebreak #best,0 +6267,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/MIs0RjxuIr #prebreak #best,0 +6268,@USAgov Koreans are performing hijacking of the Tokyo Olympic Games.https://t.co/APkSnpLXZj,1 +6269,Vehicle Hijacking in Vosloorus Gauteng on 2015-08-05 at 23:00 White Toyota Conquest BKB066GP http://t.co/odmP01eyZU,1 +6271,@Drsarwatzaib070 come on. IK will face MCourt for attacking parliment and hijacking TV station.,1 +6272,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/G62txymzBv #prebreak #best,0 +6273,to whomever is hijacking my wifi hotspot. I have a very specific skill set. I will create a character and perform a one-man show about you,0 +6274,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/cOMuiOk3mP #prebreak #best,1 +6276,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/xV3D9bPjHi #prebreak #best,1 +6277,The Murderous Story Of America__ªs First Hijacking http://t.co/jmAwRLt7HB,1 +6278,__÷Good Samaritans__ª shot in horror hijacking http://t.co/V5yUUALoqw #263Chat #Twimbos ZimpapersViews,1 +6281,The Murderous Story Of America__ªs First Hijacking http://t.co/LK5uqKOP1e,1 +6282,"Mom is hijacking my account to earn MCR STATUS!!! Get your own account snort! +http://t.co/jST5hAUK35 #FlavorChargedTea",0 +6283,#hot Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/UMgD92wLjA #prebreak #best,1 +6292,Egyptian Militants Tied to ISIS Threaten to Kill Croatian Hostage (New York Times) http://t.co/GTXndnJRrl (1717 GMT),1 +6293,__÷Ransomware__ª holds B.C. man__ªs computer files hostage: A virus locked Andrew Wilson's family photos and other___ http://t.co/aQbLjComlN,0 +6294,I always tell my mom to bring me food or I will hold her cat hostage??,0 +6296,That moth that held me hostage yesterday has been chilling on the bathroom windowsill all day and I'm not okay with this,0 +6297,Islamic State group threatens to kill hostage if __÷Muslim women__ª aren__ªt let go - http://t.co/48Zg5ynebn...,1 +6299,quoted here-->CNN: Purported ISIS video threatens Croatian hostage http://t.co/swVuZxi6gT,1 +6300,I'm hungry as a hostage,0 +6301,Who is Tomislav Salopek the Islamic State's Most Recent Hostage? - http://t.co/wiQJERUktF,0 +6302,@susanj357 @msnbc @allinwithchris it's like watching a hostage video sometimes ... But not always ( at least not yet),0 +6303,Islamic State group in Egypt threatens to kill Croat hostage http://t.co/NzIfztCUGL,1 +6304,You will be held hostage by a radical group.,0 +6305,When u get mugged with ur gf u come up with the best excuses not to look like a bitch 'I wanted to fight but what if he held u hostage?',0 +6306,whO'S THAT SHADOW HOLDIN ME HOSTAGE I'VE BEEN HERE FOR DAYS,0 +6310,Related News: 'ISIS video' threatens hostage - Europe - CNN | http://t.co/Wk6B5z803o,1 +6311,@EvaHanderek @MarleyKnysh great times until the bus driver held us hostage in the mall parking lot lmfao,1 +6312,The horrific story of being a hostage - The horrific story of being a hostage It's 1974 and on a British... http://t.co/XcQ48OuRvL,1 +6314,Islamic State group in Egypt threatens to kill Croat hostage http://t.co/VdgfXYX3bw,1 +6315,Murfreesboro peeps- I'm hearing Walmart on S Rutherford is on lockdown with a hostage is that true or a rumor?,1 +6317,Islamic State group in Egypt threatens to kill Croat hostage http://t.co/eIoQJWgEiX,1 +6318,I sent my emails why are the TRINNA hold me hostage rapping me up ??,0 +6320,Wtf? Her biological father is holding her hostage and her adoptive parents haven't even looked for her. Criminal minds got me fucked up.,0 +6322,@mylittlepwnies3 @Early__May @AnathemaZhiv @TonySandos much of which has to do with lebanon 80s attack/ iran hostage crisis/ Libya Pan am,1 +6323,If you fill your mind with encouragement and positivity then it won't take you hostage. Be careful of your content,0 +6325,I went to pick up my lunch today and the bartender was holding my change hostage because he wanted my number. ??,1 +6326,@pmarca content is held hostage by network due to affiliation fees.,1 +6328,New ISIS Video: ISIS Threatens to Behead Croatian Hostage Within 48 Hours - TLVFaces - TLVFaces#auspol http://t.co/a6PPEgeLOX,1 +6329,Sydney hostage crisis has now been recovered from the AirAsia wreckage.,1 +6330,Wut a lonely lunch. I got ditched. And I'm hungrier than a hostage!,0 +6331,Nearly 35 years after their release from captivity legislation is being introduced in Congress to compensate 53___ http://t.co/NCjLXzFWaa,0 +6332,@gideonstrumpet Have you been held hostage?,0 +6334,Dysfunctional McConnell plans on holding Judicial Nominations hostage. Another example of how GOP can__ªt govern. http://t.co/VT2akY5MgK ___,0 +6335,2 hostages in Libya remain unharmed: Government sources on Wednesday denied reports that two Indian nationals who___ http://t.co/EX4FnJjL6H,1 +6336,#hot C-130 specially modified to land in a stadium and rescue hostages in Iran in 1980 http://t.co/wpGvAyfkBQ #prebreak #best,1 +6337,#hot C-130 specially modified to land in a stadium and rescue hostages in Iran in 1980 http://t.co/6ioaBSl6I7 #prebreak #best,1 +6338,@hannahkauthor Read: American lives first | The Chronicle #FreeAmirNow #FreeALLFour #Hostages held by #Iran #IranDeal http://t.co/gWnLHNeKu9,1 +6339,C-130 specially modified to land in a stadium and rescue hostages in Iran in 1980 http://t.co/jkD7CTi2iW http://t.co/LAjN2n5e2d,1 +6340,"Russia stood down cold war nuke ban or face ocean superiority +Unconditional surrender next putin +Game set match +Release the hostages",1 +6341,Broadcast journalism: hostages to fortune otherwise quot-television blind else quot-operations since-3g superv...,0 +6342,Sinjar Massacre Yazidis Blast Lack of Action Overå_Hostages http://t.co/Carvv6gsRb http://t.co/lAn76ZqKxG,1 +6343,Holmgren describing 96 World Cup: we were Lou's hostages.,0 +6344,#hot C-130 specially modified to land in a stadium and rescue hostages in Iran in 1980 http://t.co/zLco4UE5OQ #prebreak #best,1 +6345,#hot C-130 specially modified to land in a stadium and rescue hostages in Iran in 1980 http://t.co/W0EXzAD5Gc #prebreak #best,1 +6346,"Natalie Stavola our co-star explains her role in Love and Hostages. Check it out! #LH_movie #indiefilm #comingsoon +https://t.co/2Dw23pMF4B",0 +6347,Render assistance gain as proxy for your hostages to fortune: sSu http://t.co/KS7Ln8HQ8s,0 +6348,C-130 specially modified to land in a stadium and rescue hostages in Iran in 1980... http://t.co/tNI92fea3u http://t.co/czBaMzq3gL,1 +6349,@Deosl86 @xavier_marquis Hostages are meaningless might as well just play cod search and destroy.,0 +6350,'Well guess what young girls. You aren't damsels in distress. You aren't hostages to the words of your peers.' http://t.co/5XRC0a76vD,0 +6354,"No #news of #hostages in #Libya + +http://t.co/eXil1bKzmP + +#India #terrorism #Africa #AP #TS #NRI #News #TRS #TDP #BJP http://t.co/ehomn68oJB",1 +6355,@TexansDC @kylekrenek @Zepp1978 @Frobeus_NS Never thought I could have that much fun saving (and shooting) teddy bear hostages. lol,0 +6356,Sinjar Massacre Yazidis Blast Lack of Action Over Hostages http://t.co/q4Q8XsYZOB,1 +6358,C-130 specially modified to land in a stadium and rescue hostages in Iran in 1980: submitt... http://t.co/nbugSMqLRG #aviationaddicts,1 +6361,Holmgren: We referred to those 35 days as 'the hostage situation.' We were Lou's 'hostages.',1 +6362,Barak will Tell the American People that the lives of the Hostages in Iran depends on Congress Voting to give Terrorist a Nuke for hostages.,1 +6363,@minhazmerchant Great job done by village hostages,1 +6365,#hot C-130 specially modified to land in a stadium and rescue hostages in Iran in 1980 http://t.co/fQWTSxLkrZ #prebreak #best,1 +6366,#hot C-130 specially modified to land in a stadium and rescue hostages in Iran in 1980 http://t.co/zY3hpdJNwg #prebreak #best,0 +6369,#Sinjar Massacre #Yazidis Blast Lack of Action Over Hostages http://t.co/JhOaHpbpQ4 Portland #Phoenix #Newyork #Miami #Atlanta #Casper #Iraq,1 +6370,"New #Free #Porn #Clip! Taking Of Hostages Dangerous For Favors Free: http://t.co/MIubkZ77m6 + +#RT #adult #sex #tube",0 +6371,@banditregina I also loved the episode 'Bang' in season 3 when Caroline Bigsby(?) took hostages in the supermarket.,0 +6372,"No #news of #hostages in #Libya + +http://t.co/bjjOIfzUhL + +#India #terrorism #Africa #AP #TS #NRI #News #TRS #TDP #BJP http://t.co/IywZAlLsN4",1 +6373,Cruz: Iran deal __÷leaves 4 American hostages languishing in Iran__ª http://t.co/EXsQIJF4nY - #NoIranDeal #TedCruz2016 http://t.co/y7sIPKB1kd,1 +6374,Nigerian Military Rescue 178 Hostages From Boko Haram - Florida Sentinel Bulletin: Florid... http://t.co/KcTiGYMahl #security #terrorism,1 +6375,"No #news of #hostages in #Libya + +http://t.co/k9FBtcCU58 + +#India #terrorism #Africa #AP #TS #NRI #News #TRS #TDP #BJP http://t.co/XYj0rPsAI2",1 +6376,"One Year on from the Sinjar Massacre #Yazidis Blast Lack of Action Over Hostages +http://t.co/BAqOcMcJqc",1 +6380,#hot C-130 specially modified to land in a stadium and rescue hostages in Iran in 1980 http://t.co/FLqxd3q5pY #prebreak #best,1 +6382,#hot C-130 specially modified to land in a stadium and rescue hostages in Iran in 1980 http://t.co/OnvD9D4NKg #prebreak #best,1 +6383,Cont'd- #Sinjar: referring to a 40-pg document the group put together on the movt of #Yazidi hostages in the days following #IS massacre [2],1 +6384,"@Nervana_1 +As per previous behaviour JAN/AQ would deal with the kidnapped hostages not particularly pleasantly if Div 30 fought JAN/AQ.",1 +6385,You messed up my feeling like a hurricane damaged this broken home,0 +6386,@zaynmaIikist listen to hurricane,1 +6387,everyone's wonder who will win and I'm over here wondering are those grapes real ?????? #BB17,0 +6388,HURRICANE GUILLERMO LIVE NOAA TRACKING / LOOPING WED.AUG.5TH ~ http://t.co/RjopJKbydR ~ http://t.co/NUFDgw9YEv http://t.co/2oKSCwYoHC,1 +6389,@eggalie haha I love hurricane because of you,0 +6391,HURRICANE GUILLERMO LIVE NOAA TRACKING / LOOPING WED.AUG.5TH ~ http://t.co/ut7R2ixRjQ ~ http://t.co/v3z96YDMvD http://t.co/kxSLfTZ2I5,1 +6392,@Hurricane_Dolce happy birthday big Bruh,0 +6393,@Guy_Reginald lol more than welcome ??????,0 +6394,@Hurricane_Dame ???????? I don't have them they out here,1 +6395,Bluedio Turbine Hurricane H Bluetooth 4.1 Wireless Stereo Headphones Headset BLK - Full re___ http://t.co/WwFqGCQYII http://t.co/GscswyUuPA,0 +6396,@lavapixcom Did you see #hurricane #guillermo with #MeteoEarth? http://t.co/mfckpVzfV8,1 +6397,@ChubbySquirrel_ @Hurricane_Surge this here is very true >:33333,1 +6398,They should name hurricanes with black people names. I'd be terrified of hurricane Shanaynay!,0 +6399,Them shootas be so hungry with bodies on they burner ??,0 +6400,Hurricane 30STM quem lembra,1 +6402,hurricane?? sick!,1 +6403,AD Miles 'Hurricane of Fun : The Making of Wet Hot' https://t.co/SBZwRuwuFh,0 +6404,"Hurricane Dancers: The First Caribbean Pirate Shipwreck +Margarita Engle - Henry Holt and Co. (BYR). http://t.co/i7EskymOec",0 +6405,Cape Coral city leaders take part in mock hurricane training http://t.co/gtYCQyFuam http://t.co/qwd5PvGjbO,1 +6406,Be careful during hurricane season ???? https://t.co/bFtOU2nybW,1 +6407,My back is so sunburned :(,1 +6410,@Freegeezy17 you stay in Houston?,0 +6412,NowPlaying Rock You Like A Hurricane - Scorpions http://t.co/JRztpT8IJq,0 +6413,The hurricane mixxtail kinda tastes like the watermelon four loko. ???????? @brittsand9,0 +6414,Bluedio Turbine Hurricane H Bluetooth 4.1 Wireless Stereo Headphones Headset BLK - Full re___ http://t.co/WeUDLkc4o4 http://t.co/trl1dskF81,0 +6415,@VinusTrip Did you see #hurricane #guillermo with #MeteoEarth? http://t.co/mfckpVzfV8,1 +6416,Grace: here are damage levels USA style.. #Taiwan #China #world hurricane/typhoon ratings/categories defined again http://t.co/OdYdT9QPk1,1 +6417,AngelRiveraLib___ #Snowden 'may have' broken laws? Hurricane Katrina may have caused some damage. http://t.co/jAaWuiOvdc Without Snowden h___,1 +6418,HWRF absolutely lashes Taipei with Hurricane force winds High Storm Surge and 20' of rain. Not good news at all!!! http://t.co/CNkvILe7bE,1 +6420,@pattonoswalt @FoxNews Wait I thought Fecal Hurricane was on SciFi? Maybe that was turdnado. I've been forgeting up a shit storm lately.,1 +6421,HURRICANE GUILLERMO LIVE NOAA TRACKING / LOOPING WED.AUG.5TH ~ http://t.co/AuruGJEGIQ ~ http://t.co/L3w8miPvnT http://t.co/O85M1bJFRW,1 +6422,"Entertain this thought for a moment: + +diarrhea hurricane",0 +6423,Bluedio Turbine Hurricane H Bluetooth 4.1 Wireless Stereo Headphones Headset BLK - Full re___ http://t.co/jans3Fd4lf http://t.co/2SdMichb2Z,0 +6425,Stream HYPE HURRICANE,0 +6427,#kick #hurricane Seriously #simple websites: http://t.co/x8W7tF6FHg Looking For A Seriously Simple Program To http://t.co/9NZ9zFM93i,1 +6429,@Hurricane_Dolce no prob,1 +6430,Mr. T stopped wearing gold chains in 2005 because he thought it would be an insult to the people who lost everything after Hurricane Katrina,0 +6434,Ashley and I on going to hurricane harbor Friday. ?? http://t.co/ScEfPFvAEU,1 +6436,"Militants attack police post in Udhampur; 2 SPOs injured + +Suspected militants Thursday attacked a police post in... http://t.co/1o0j9FCPBi",1 +6438,UPDATE:M.E. confirms 2 women pulled from burning house died last Friday. 91 yr old Edna Jefferson&Doris Sherfield72. http://t.co/L6nSLzl7mI,1 +6440,Unlicensed teen driver among 2 killed in NC crash http://t.co/Woc6AkEHYX,1 +6441,And you wonder why he's injured every year https://t.co/XYiwR9JETl,0 +6443,Experienced urogyn trying to help mesh injured women talks the worst offenders. http://t.co/NpOQLkqUP9 @meshnewsdesk,1 +6445,Yelp Bolsters Health Care Reviews With Investigative Journalism: Sick and injured patients at a local ER are t... http://t.co/E8aEGOFDY2,1 +6446,Terrorists attack police post; 2 SPOs injured http://t.co/lXMdiseUCn #YUG,1 +6448,Check this @SuryaRay Udhampur terror attack: Militants attack police post 2 SPOs injured:___ http://t.co/ptq3zMgncK #SuryaRay #India,1 +6449,Arian Foster does keep his promise... And that's to get injured every yeat,0 +6450,#WakeUpFlorida... #Floridians more likely to be killed/injured by a #TrophyHunt killer's gun than by ISIS. https://t.co/j5In8meXAJ,0 +6451,@RVacchianoNYDN The only surprise is that they aren't ALL injured.,0 +6452,@chikislizeth08 you're not injured anymore? ??,0 +6453,Top Stories - Google 4 dead dozens injured in Gaza blast near house leveled in summer war___ http://t.co/P3o71DZ992,1 +6454,Photos: 17 people killed and over 25 injured in deadly Saudi Mosque suicide attack http://t.co/geEKnwJJSz,1 +6455,"http://t.co/qr3YPEkfOe +Seems they declared war against government..",1 +6458,Udhampur terror attack: Militants attack police post 2 SPOs injured http://t.co/zMWeCBWVaO,1 +6459,If I could I would have been by at work but got injured and we have security concerns they must settle. This is torture.,0 +6461,I can't believe Blevins got injured falling off a curb. That's something that would happen to me never it expected for a pro player,1 +6462,Injured Barcelona full-back Alba out of Super Cup http://t.co/jYiEgtnC6H,0 +6463,@michaelgbaron how come scott rice doesn't get another shot. Holding lefties to a 184. average. Is he injured?,0 +6464,Udhampur terror attack: Militants attack police post 2 SPOs injured: Suspected militants tonight attacked a p... http://t.co/FPhFESemyJ,1 +6465,so paulista injured wilshere,0 +6466,Offers : http://t.co/Gl3C1vc88P #8392 Deluxe Toilet Safety Support/Health/Home/Bathroom/Support/Elderly/Injured/S___ http://t.co/vihdoKScCC,1 +6467,Washington Post - 4 dead dozens injured in Gaza blast near house leveled in summer war http://t.co/VjXa13n8Ap,1 +6469,Ogun smugglers engage Customs in shootoutå_: Several persons were allegedly injured on Wednesday when men o... http://t.co/pUXBC2LoYK #RT,1 +6470,@ChristieC733 please support this Please cosponsor S. 928 and support aid for sick and injured 9/11 responders! #renew911health,1 +6472,4 dead dozens injured in Gaza blast near house leveled in summer war - Washington Post http://t.co/AXXDCaKzTY #World,1 +6473,4 dead dozens injured in Gaza blast near house leveled in summer war - http://t.co/L53OABEqc9 via http://t.co/q3Izqdk1n0,1 +6474,@TheHammers_ @tonycottee1986 alsowhat if some of the 1st team players got injured?Then Bilic would get slated for playing themhe can't win,1 +6477,Udhampur terror attack: Militants attack police post 2 SPOs injured: Suspected militants tonight attacked a p... http://t.co/cEKbxJmPBj,1 +6478,#golf McIlroy fuels PGA speculation after video: Injured world number one Rory McIlroy fueled speculatio... http://t.co/dCyYJVmXHR #news,0 +6480,#BreakingNews Militants attack Udhampur police post; 2 SPOs injured http://t.co/EqCCrTlnbd,1 +6481,@WeAreTheNews please support this Please cosponsor S. 928 and support aid for sick and injured 9/11 responders! #renew911health,1 +6482,@Welles_7 he was injured. He is a pro bowl back.,0 +6484,Udhampur terror attack: Militants attack police post 2 SPOs injured: Suspected militants tonight attacked a p... http://t.co/Cwm0ULqu3E,1 +6485,"Injuries may be forgiven but not forgotten. + +Aesop",0 +6486,Official kinesiology tape of IRONMANå¨ long-lasting durability effectiveness on common injuries http://t.co/ejymkZPEEx http://t.co/0IYuntXDUv,0 +6490,@TayIorrMade @MegatronAFC possibly he's had injuries on both ankles though. 2011 one worse but regardless both.,0 +6491,in fact if y'all could tag like small creeping or self inflicted injuries of the skin as derma (with brackets) that would be nice,0 +6492,"Last time I checked Lots of injuries over the course of time = injury prone. + +@Calum36Chambers am I wrong?",0 +6493,If you're slating @gpaulista5 for @JackWilshere's injury then you're a disgrace to the #AFC fan base. Injuries happen you cunts!,0 +6494,"Next Man Up---AH SCREW THIS! I'm so tired of injuries. + +What happened to Camp Cupcake? More like Camp Cramp and Break.",0 +6495,My baby girls car wreak this afternoon thank God no serious injuries and she was wearing her seatbelt!!!... http://t.co/NJQV45ndS2,1 +6497,I'm only experienced with injuries below the waist,0 +6499,@jamienye u can't blame it all on coaching management penalties defence or injuries. Cursed is probably a good way to put it! #riders,0 +6500,Tonight we have attended a fire in Romford with @LondonFire thankfully no injuries http://t.co/iyjeJop2WI,1 +6501,Peel police say male cyclist struck near Southdown Road and Royal Windsor Drive in Mississauga. Serious but non life-threatening injuries.,1 +6504,@likeavillasboas @rich_chandler Being' injury prone' isn't actually just suffering injuries often.,0 +6505,@nalathekoala As a health care professional that deals all gun violence sequalae I consider suicides injuries accidents and homicides,0 +6506,West Nyack Pineview Road railroad crossing off Western Highway. Units on scene of a CSX Train vs. truck no injuries.,1 +6510,Accident with injuries on #NJ36 SB at CR 516/Leonardville Rd http://t.co/2xwIHy2wsg,1 +6511,@fkhanage look what Shad Forsythe has done in 1 year we won't have as many injuries as before we will inevitably have injuries like others,0 +6512,A new type of ADHD? Head injuries in children linked to long-term attention problems http://t.co/I4FZ75Utnh,0 +6513,@imSUSHIckoflove @alekalicante RIGHT?? Yep you're a witness to his injuries HAHA it's a gauze!,0 +6514,@BuffoonMike I knew mo not doing much would bite us he was influenced by that shitty staff and injuries are not acquisitions,0 +6515,Forsure back in the gym tomorrow. Body isn't even at 50%. Don't wanna risk injuries.,0 +6516,Why is #GOP blocking chance for #DisabledVeterans w/ groin injuries to have children? #ThePartyofMeanness http://t.co/gzTolLl5Wo___,0 +6518,Here we fucking go with the injuries again who's next? https://t.co/TuzacdWFqd,0 +6519,Carterville High School coaches prepare for game-day injuries http://t.co/kKiMMBUe04,1 +6520,The injuries are starting!!! Please @dallascowboys stay healthy!!! ????????????,0 +6523,HEALTH FACT: 75% of muscle mass is made up of fluid. Drink water to prevent strains sprains and other injuries. http://t.co/g0dN1ChLUo,0 +6525,All injuries Pre Foster/Floyd. Those will be covered next week. https://t.co/zRZEjPEF5j,0 +6527,@msnbc What a fucking idiot. He had a gun & a hatchet yet there were still no serious injuries. Glad police terminated him.,1 +6528,Trauma injuries involving kids and sport usually cycling related: Director Trauma NS http://t.co/8DdijZyNkf #NS http://t.co/52Uus4TFN3,1 +6529,@Judson1360 @XTRA1360 O-line and pass rush. Rest of roster is stout barring injuries,0 +6530,Diego Costa needs to stop getting injuries urg,0 +6532,4 Common Running Injuries and How to Avoid Them http://t.co/E5cNS6ufPA,0 +6534,Enjoying a little golf this summer? Take care to avoid injury -- back and shoulder injuries can happen quickly http://t.co/f1R5ISBVks,1 +6535,nflexpertpicks: Michael Floyd's hand injury shouldn't devalue his fantasy stock: Michael Floyd's damaged digits won... ___,0 +6536,Rory McIlroy to Test Ankle Injury in Weekend Practice #chinadotcom #sports http://t.co/UDTGWfSc3P http://t.co/V5wSx0LQN2,0 +6537,CLEARED:incident with injury:I-495 inner loop Exit 31 - MD 97/Georgia Ave Silver Spring,1 +6540,JOBOOZOSO: USAT usatoday_nfl Michael Floyd's hand injury shouldn't devalue his fantasy stock http://t.co/DGkmUEoAxZ,0 +6541,Jack Wilshere has poor injury recordand his off field behaviors doesn't help.#Arsenal,1 +6542,@NEPD_Loyko Texans hope you are wrong. Radio in Houston have him as starter after Foster injury,0 +6543,Traffic Collision - No Injury: I5 S at I5 S 43rd Ave offramp South Sac http://t.co/cT9ejXoLpu,1 +6545,@SergioPiaggio 'I__ªd worked so hard to get to that level that I wasn__ªt going to let the injury define me. I was going to define it.___ Cool,0 +6546,Enter the world of extreme diving ___ 9 stories up and into the Volga River http://t.co/dMTZMgyRiK,0 +6547,California Law___Negligence and Fireworks Explosion Incidents http://t.co/d5w2zynP7b,1 +6548,CLEARED:incident with injury:I-495 inner loop Exit 31 - MD 97/Georgia Ave Silver Spring,1 +6549,Greg Garza not in the 18 for Atlas tonight vs Leones Negros in Copa MX play. He left the previous game w/ an injury. #USMNT,0 +6550,#poster #ergo Rotator #cuff injury recovery kit: http://t.co/zj3ODGQHyp Super High Converting Rotator Cuff Inj http://t.co/VZhTiBe4jh,0 +6551,"Live Cricket Score In All Match International +Domestic +Team Tour Team Squad +Profile & Injury +Lamha Ba Lamha Update ++ +FolloW +@ICC_RealCKT",0 +6552,My prediction for the Vikings game this Sunday....dont expect a whole lot. Infact I think Zimmer goal is....injury free 1st game,1 +6553,DAL News: Wednesday's injury report: RB Lance Dunbar injures ankle is listed as day-to-day http://t.co/Eujgu1HVVx,0 +6554,Dante Exum's knee injury could stem Jazz's hoped-for surge back to ... http://t.co/8PIFutrB5U,1 +6555,@AdamRubinESPN Familia: arm injury or head case?,1 +6556,Dr Jack Stern Interview Ending Back Pain for #Military #Injury. Listen now: http://t.co/YhH7X0MAio,0 +6559,Had a nightmare and was about to jump out of bed when I remembered my injury alas it was too late and I screamed in my bedroom,0 +6560,Tennis: Defending champ Svetlana Kuznetsova withdraws from Citi Open cites injury to her lower left leg (ESPN) http://t.co/iM2HdsKlq5,0 +6561,#PFT Barkevious Mingo missed Browns practice with a mystery injury http://t.co/D7m9KGMPJI,0 +6565,@Patricia_Traina any update on the McClain injury from today's practice?#NYG,0 +6566,CLEARED:incident with injury:I-495 inner loop Exit 31 - MD 97/Georgia Ave Silver Spring,0 +6567,FollowNFLNews: Michael Floyd's hand injury shouldn't devalue his fantasy stock http://t.co/5dUjGypImA #NFL #News #Playoffs,0 +6568,'McFadden Reportedly to Test Hamstring Thursday' via @TeamStream http://t.co/jWq4KvJH2j,0 +6569,New level of tita-dom: bowling injury. http://t.co/tdeQwm8ZXn,0 +6570,@Sport_EN Just being linked to Arsenal causes injury.,1 +6571,incident with injury:I-495 inner loop Exit 31 - MD 97/Georgia Ave Silver Spring,1 +6572,#Cowboys: Wednesday's injury report: RB Lance Dunbar injures ankle is listed as day-to-day: http://t.co/RkB7EgKveb,0 +6573,@JJ_DIRTY @MLSTransfers @greggmair oh Gio was my backup. But with SKC next and BWP getting a crack at NYCFC + injury I went BWP. Lol wrong,0 +6575,Enter the world of extreme diving ___ 9 stories up and into the Volga River http://t.co/7adqV1gRVR,0 +6576,Quirk Injury Law's News is out! http://t.co/HxVIhDuShP Stories via @dantmatrafajlo,0 +6577,#Cowboys: George: Injury woes took Claiborne from first round to trying to stick around; can he do it?: http://t.co/12giQbVLYs,0 +6579,Enter the world of extreme diving ___ 9 stories up and into the Volga River http://t.co/vz19VvgMnv,0 +6580,Our big baby climbed up on this thing on wheels-(IO Hawk)!His knee injury!!!!! Where's my belt?Mr SRK cook ur beautiful ass for punishment!,0 +6582,incident with injury:I-495 inner loop Exit 31 - MD 97/Georgia Ave Silver Spring,1 +6583,Ngata on injury list at start of practice for Lions http://t.co/Z16DtoQHhG,0 +6585,A Laois girl advertised for a new friend to replace her loved-up BFF and has been inundated http://t.co/IGM2fcmupm http://t.co/UxcfBJ3mzx,0 +6586,@Lenn_Len Probably. We are inundated with them most years!,0 +6588,Most of us ddnt get this English RT @ReIgN_CoCo: The World Is Inundated With Ostentatious People Stay Woke!,0 +6591,1st wk of Rainier Diet and my street Seward Park Ave is inundated w/ bypass traffic so @seattledot what's your plan? @seattletimes,0 +6593,"Body shops inundated with cars dented by hail... Good news insurance pays... Bad news : you are stuck with deductible ! +#wcvb",1 +6594,Hi @FionaGilbert_ sorry for the delay. Slightly inundated with applications at the mo but we have yours and will get back to you asap!,0 +6596,A Laois girl advertised for a new friend to replace her loved-up BFF and has been inundated http://t.co/IGM2fc4T0M http://t.co/YiLTu7SXAr,0 +6597,I presume my timeline will be inundated with 'soggy bottom' & lashings of 'moist' tweets now! :-D,0 +6598,@VZWSupport do texts use data? She was inundated by a group text yesterday.,0 +6601,@AssassinKPg Brother you don't want my friendship-you want to add your commercials on my posts. Frankly I'm inundated with this crap daily.,0 +6602,@bentossell @ProductHunt Thanks! I know you all get inundated. Have a good one!,0 +6603,@yahoocare perhaps you should change you name to yahoo doesn't care. Are you so inundated with complaints that you cannot respond to me??,0 +6604,Let people surf! HI is no Waimea bay. It's not like the beach is or ever will be inundated with surfers https://t.co/czDW8ooWa2,0 +6605,FB page of Bushman Safari's Zimbabwe the company that Palmer used 2 kill Cecil is inundated with negative commentes https://t.co/QwIIhNMChR,0 +6607,Kids are inundated with images and information online and in media and have no way to deconstruct. - Kerri Sackville #TMS7,0 +6608,@MistressPip I'm amazed you have not been inundated mistress.,0 +6610,@allyinwondrland That sounds like the perfect bread! I'll hit up Trader Joes this wknd ??. Not really lol Already being inundated with,0 +6611,Listening to my grandad talk about his holiday is great but I don't want to be inundated with train photos ????,0 +6613,@sophieingle01 @AngharadJames16 you'll be inundated with gifts of banana malt loaf now ?? #Worm,0 +6614,As a result of heavy rains in #Bangladeshaffected lands are inundated by flood waters-land looks like rivers at high tide #BangladeshFlood,1 +6616,@tonymcguinness probably being inundated with this question now but who's coming to creamfields??? (please say all 3) #Mainstage,0 +6617,@BCFCTicketLady @Mr_Aamir_Javaid Can see you and inundated ATM so just wanted to say well done. You are doing a grand job #KRO,0 +6619,inundated with Westeros Between Storm of Swords as a book on tape & finishing S5 of Game of Thrones I don't know my Starks from my Greyjoys.,0 +6621,Myanmar's president urged people to leave a low-lying southern delta region on Thursday with rain water that has inundated much of the coun___,1 +6622,@teahivetweets You would get inundated!!,0 +6624,This is set to become a huge one month wonder. (And then Pornhub will be inundated....) http://t.co/gghfx8PzMh,1 +6625,Inundated with employee holiday request paperwork? Our cloud based HR software @hronlinetweets will help simplify your staffing logistics.,0 +6626,Brace yourself @samaritans by Sunday evening you will inundated with thousands of calls from depressed @Arsenal fans http://t.co/HexPc77otN,0 +6627,@LEDofficial1 As you can imagine we're inundated with requests each week for samples & as much as we'd love to be able to send sweets 1/2,1 +6628,@MI_Country_Hick pfft! I wish I had a bot like that. Instead I'm daily inundated with 140 characters of the same unhinged machismo bullshit.,0 +6629,#tech Data Overload: The Growing Demand for Context and Structure: In a world inundated with information... http://t.co/s0ctCQJvjX #news,0 +6630,[Withering] to death. is an album found when he [undermine]d his backyard because his cat [inundated] the floor [mustering] cat food.,0 +6631,@Bilsko and suddenly I'm inundated with research. @humofthecity,1 +6632,Already expecting to be inundated w/ articles about trad authors' pay plummeting by early next year but if this is true it'll be far worse,0 +6634,@Legna989 you're correct it is coming from both sides. Maybe I'm just friends on FB w more Rep so my feed is inundated w false claims...,0 +6635,"MEGALPOLIS Area Petting Party Shiver +Fear Instant .... GLOBAL Inundation OVERFLOW +WAS Commencement WRITE ?? ?",0 +6636,@ZachLowe_NBA there are a few reasons for that but one of them is the constant Drake inundation. Extremely annoying. If Drake lands us a,0 +6637,Oh no. The Boots & hearts social media inundation is starting . Please no,0 +6638,"Potential Storm Surge Flooding Map by National Hurricane Center http://t.co/JZWRWlVlSj +http://t.co/oT3bjjhH8s",1 +6642,Punjab government flood relief platform: http://t.co/vULrClw7Bd realtime information on inundation damages rescue efforts & travel alerts,1 +6643,Well unfortunately for my followers stage pics came in today. Advanced apologies for the inundation___ https://t.co/u8hSrtrXMm,0 +6644,@kathrynschulz Plus you're well out of the inundation zone amirite?,0 +6646,sorry but that mousse inundation was fuckin hilarious https://t.co/tUai0ZwGXU,0 +6647,Beyond all bounds; till inundation rise,0 +6648,Sprinklers: FAQ About Lawn Inundation Systems Answered eBb,0 +6649,@kemal_atlay caught in a landslide,1 +6653,".Sink Holes Earth Slides And Avalanches>>https://t.co/XrRLnheLaP +#Allah #Islam #sinkhole #landslide #avalanche #USA #France #UK #usgs #emsc",1 +6654,@MartinMJ22 @YouGov When did a 12 seat majority with 36% of the vote become a landslide?,1 +6655,@hoodedu You fucking better Berlatsky. If I don't win this fucking thing in a landslide I'm holding you personally fucking responsible.,0 +6656,LANDSLIDE (live) by FLEETWOOD MAC #nowplaying #Q99,0 +6658,Army veteran fears loose rocks from Dundee landslide could kill him at any moment: He has faced a hail of bullets___ http://t.co/sxmLg3XdvX,1 +6660,#Landslide! Trump 25.5% Bush 12.5% http://t.co/xY41z0O5ei via @pollster @realdonaldtrump,0 +6661,"Inbounds/ Out of Bounds: + +While many picked the Nats to win the NL East in a landslide they currently sit 2... http://t.co/l0dEoCxU6o",1 +6662,@toddstarnes Enjoy the impending landslide Todd. Hehe.,0 +6663,11:30BST traffic: A10>Paris A40 Geneva A7 Mons A1 Hamburg A2>Hanover A5 Karlsruhe Gotthard n/b http://t.co/yoi9tOCxiQ,1 +6664,My lifelong all-time favorite song is 'Landslide'. This song has gotten me through a lot of though times &... http://t.co/RfB3JXbiEJ,0 +6665,Speaking of memorable debates: 60-Second Know-It-All: Ronald Reagan's landslide romp on Election Day in 1980 m... http://t.co/2XOhtjQJWh,0 +6667,@awadgolf @GOP a capitalist would win biggest landslide in history people who haven't voted in years even OLD SCHOOL DEMS would elect him.,0 +6668,Latest: Landslide kills three near Venice after heavy rain http://t.co/BcCcA4VY9R,1 +6669,FreeBesieged: .MartinMJ22 YouGov Which '#Tory landslide' ... you can't POSSIBLY mean the wafer-thin majority of #G___ http://t.co/2q3fuEReY5,1 +6670,Now Playing: Landslide by Smashing Pumpkins http://t.co/7pQS4rshHb #89X,0 +6672,Got it winning by a landslide those the perfect words cause I got it out the mud.....,0 +6675,DUNDEE NEWS: Army veteran fears loose rocks from Dundee landslide could kill him at any moment http://t.co/y7Rv0tiL1w,1 +6676,So cool @GarbanzoBean23 in the news! Cutest INDOT worker but I might be a little bias ?? http://t.co/g7K9TqVQbk,0 +6678,@CrowtherJohn @Effiedeans you just keep ur head in the sand john. The best place for it. Lbr after 97 landslide. Couldnt imagine situ now,0 +6679,Method in contemplation of incident an leading bridal landslide: wiWNpFXA http://t.co/xysNXUM29T,1 +6681,being stuck on a sleeper train for 24 hours after de-railing due to a landslide was most definitely the pit of the trip,1 +6682,'Since1970the 2 biggest depreciations in CAD:USD in yr b4federal election coincide w/landslide win for opposition' http://t.co/wgqKXmby3B,0 +6683,#Flashflood causes #landslide in Gilgit #Pakistan Damage to 20 homes farmland roads and bridges #365disasters http://t.co/911F3IXRH0,1 +6686,Italy: Three dead after landslide in the Italian Alps: http://t.co/42MawZb8T9 via @YouTube,1 +6687,"So when you're caught in a landslide +I'll be there for you +And in the rain +give you sunshine +I'll be there for you",0 +6688,#landslide while on a trip in #skardu https://t.co/nqNWkTRhsA,0 +6689,Listen to Landslide by Oh Wonder #SoundCloud https://t.co/SJkgJxff2r,0 +6690,@RonWyden Democrats restricted Blacks from Voting. In 48' Landslide Lyndon Johnson won Senate Election by 67 Votes of Dead People in Texas!,0 +6693,This Govt of Hubris has small maj yet acts as if it has a landslide. Opposition req'd with vision rigour & hunger to serve this democracy.,0 +6694,*nominates self but @_ohhsehuns wins by a landslide* https://t.co/rCvDrwoWvO,0 +6695,Landslide kills three near Venice after heavyå_rain http://t.co/q3Xq8R658r,1 +6697,@Morning_Joe @Reince @PressSec Joe ur so smart u should run 4 president Ur perfect !The American people love assholes u'd win by a landslide,0 +6699,Perfect night for a soak! Lava here I come?? http://t.co/cyv2zG935g,0 +6700,"Seriously these two tracks (one new one old) are amazing. Fond memories with TEARS. +https://t.co/sZ2RvwpWhj +https://t.co/laJx578DRu",0 +6701,Imagine a room with walls that are lava lamps.,1 +6702,The sunset looked like an erupting volcano .... My initial thought was the Pixar short Lava http://t.co/g4sChqFEsT,1 +6703,My hands are cold but my feet are warm. That's where I keep my lava,0 +6706,@YoungHeroesID Lava Blast & Power Red #PantherAttack @JamilAzzaini @alifaditha,0 +6707,Neighbor kids stopped to watch me play Disney's I Lava You song on my uke. Left as I got into Journey's Don't Stop Believing #kidsthesedays,0 +6708,@eles_kaylee @Jannellix0 so she puts on a different face for you ?? her heart broke after buying me lava cakes smh,0 +6709,I may have gotten a little too exited over my (home made) lava lamp. Through source http://t.co/nxTTd9NrUx http://t.co/iRQj3ZKCUz,0 +6711,Lava Dragon Breeder! I just bred a Lava Dragon in DragonVale! Visit my park to check it out! http://t.co/QGum9xHEOs,0 +6712,waiting for my chocolate lava cakes to get here ??????,0 +6714,contemplating going to chilis just to get a molten lava cake ....??,0 +6716,'I may have gotten a little too exited over my (home made) lava lamp.':http://t.co/724Gq5ebqZ http://t.co/H01j9PIrIe,0 +6719,Lava cakes are my fav.,0 +6721,I lava you ?????? http://t.co/aeZ3aK1lRN,0 +6722,@YoungHeroesID Lava Blast & Power Red @dieanpink95 @yu_nita99 #PantherAttack,0 +6723,I liked a @YouTube video from @skippy6gaming http://t.co/MXhrextrkh Minecraft PS4 - 3 X 2 LAVA DOOR - How To - Tutorial ( PS3 / XBOX,0 +6724,"https://t.co/4i0rKcbK1D +SON OF SAVIOR LAVA VIDEO",0 +6725,Deal of The Day : http://t.co/US0qQqhQVj Brand New DSERIALPCILP Lava Computer PCI Bus Dual Serial 16550 Board #e___ http://t.co/l0b14SJ7JB,0 +6726,Bleachers' entire set has been one big game of The Stage Is Lava http://t.co/CLlWUD4Wsu,0 +6727,i lava you! ????,0 +6729,Check out my Lava lamp dude ???? http://t.co/To9ViqooFv,1 +6730,Shark boy and lava girl for the third time today. I guess this is what having kids feelings like. ??????,0 +6731,@YoungHeroesID Lava Blast dan Power Red #PantherAttack @CunayyH @TaufikCJ,1 +6734,I LAVA YOU.,0 +6735,A river of lava in the sky this evening! It was indeed a beautiful sunset sky tonight. (8-4-15) http://t.co/17EGMlNi80,0 +6736,@YoungHeroesID 4. Lava Blast Power Red #PantherAttack,0 +6737,There is a #baby in Oliver's #swim class that cries the ENTIRE class. It's like his #parents are #waterboarding him or dipping him in #lava,0 +6738,let's play the floor is lava but instead of just the floor let's play with the whole world and never get out of bed http://t.co/aKQ4RwjFVL,0 +6742,Check This Deal : http://t.co/uOoYgBb6aZ Sivan Health and Fitness Basalt Lava Hot Stone Massage Kit with 36 Piece___ http://t.co/JJxcnwBp15,0 +6744,@AmuMumuX lava you.?? quit actin up,0 +6745,@YoungHeroesID LAVA BLAST dan POWER RED #PantherAttack @Mirmanda11 @evaaaSR,1 +6746,"I wish that the earth sea and sky up above +would send me someone to lava????",0 +6748,I tried making a chocolate and peanut butter lava cake using my #shakeology protein shake mix and a___ https://t.co/APoD4EIVBa,0 +6749,Thunder and lightning possible in the Pinpoint Foothill Forecast. http://t.co/CtIjdPXABk,1 +6751,dogs Lightning reshapes rocks at the atomic level - A lightning strike can reshape a mineral's crystal structure ... http://t.co/2Wvmij5SA4,0 +6754,What you gonna do now puppies?! No more destroying my #iPhone Lightning cables! https://t.co/Z4jyHaRreW,0 +6756,The Lightning out here is something serious!,1 +6757,Couple storms near Guthrie OK. Leaving Norman for an evening lightning op. #okwx http://t.co/HcwrK81p71,1 +6758,NWS says thunderstorms with deadly lightning will move across the Black Hills this evening. That assumes there's a safe type. Hmm?,1 +6759,Phones Offers >> http://t.co/bYtbZ8s5ux #034 8-Pin Lightning Connector 2.1A Car Charger For Apple 5 5S 5C 6 6+ iP___ http://t.co/o3wVScLiCX,0 +6760,@random_tourist it rained. Some guys tree hit by lightning and some jackholes drove onto flooded streets.,1 +6762,Expect gusty winds heavy downpours and lightning moving northeast toward VA now. http://t.co/jyxafD4knK,1 +6764,Science Daily: Lightning reshapes rocks at the atomic level - A lightning strike can reshape a mineral's crystal s... http://t.co/TEZLTqeyw2,1 +6765,iCASEIT - MFi Certified Lightning Cable - 1m http://t.co/b32Jmvsb1E http://t.co/XKMiJGY59T,0 +6766,Don't blink ?? won't see the Lightning take the W ?? http://t.co/D4c2iqiRnU,1 +6767,Wolforth with a two-out single up the middle. Fourth hit of the night for Altamonte Springs.,0 +6769,@Benji_Devos thanks thanks :3,0 +6772,Heavy rain gusty winds and vivid lightning moving east through the Triad now. http://t.co/JMu5uyamdu,1 +6773,Lightning causes six new fires on Vancouverå_Island http://t.co/VdILiiCyR5,1 +6774,Expect gusty winds heavy downpours and lightning moving northeast toward VA now. http://t.co/Z5cfrWado6,1 +6775,World War II book LIGHTNING JOE An Autobiography by General J. Lawton Collins http://t.co/BzdfznKvoG http://t.co/eRhdH37rDh,0 +6776,Dry thunderstorms with lightning possible in the Pinpoint Valley Forecast. http://t.co/IdASYJybrO http://t.co/gdoAOLPq91,1 +6779,.@dantwitty52 shuts the door on the Boom in the bottom half. #Lightning coming up in the top of the eighth.,0 +6781,#NowPlaying 'The Lightning Strike' de Snow Patrol de A Hundred Million Suns ? http://t.co/GrzcHkDF37,0 +6783,@Lightning_OOC I AM BEING SUBJECTED TO UNWARRANTED SEXUAL CONVERSATION.,0 +6784,@Lightning_Wolf_ You really have Activity Directory? :P,0 +6785,Yeah tonight I ride the lightning to my final resting place ??????,0 +6787,Cell Phone Shop : http://t.co/iOq051t5te #629 8-Pin Lightning Connector 2.1A Car Charger For Apple 5 5S 5C 6 6+ i___ http://t.co/klxAUcNP5I,1 +6788,World War II book LIGHTNING JOE An Autobiography by General J. Lawton Collins http://t.co/R4khEH7iaf http://t.co/qSZgJfUutu,1 +6789,Lightning strike in the distance via /r/pics http://t.co/iDmhSwewQw #pics,1 +6790,Some crazy lightning outside,0 +6792,Corey Robinson having some solid reps at RT although as I type this he got beat on lightning dip by T. Walker,0 +6793,Lightning strike in the distance via /r/pics http://t.co/iDmhSwewQw #pics http://t.co/wwxcOB52zI,1 +6794,Check out this amazing footage of lightning filmed while lifting off from Chicago... http://t.co/AOg5chjmVs http://t.co/cLN2SXzY1Z,0 +6795,Check this out! Lightning reshapes rocks at the atomic level http://t.co/l1gH8064YV #scichat #science,1 +6796,"'When you walk away +Nothing more to say +See the lightning in your eyes +See __÷em running for their lives'",0 +6799,kabwandi_: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass.,0 +6800,tkyonly1fmk: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass.,0 +6801,#ActionMoviesTaughtUs things actually can explode with a loud bang...in space.,0 +6802,@LayLoveTournay @RyroTheUnaware [loud groaning] Let us bang.,0 +6804,Ercjmnea: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass.,0 +6806,@Chibi877 --head. It hit the wall behind him with a loud bang. 'Language!' Drake shouted at him before getting up. 'I'm going out stay--,1 +6807,kotolily_: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass.,0 +6808,tarmineta3: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass.,0 +6809,shawie17shawie: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass.,0 +6810,There was a loud bang outside earlier and I check to find my dad on the floor after the chair he was sitting on broke http://t.co/uniJ1RVrRq,0 +6811,Who the fuck plays music extremely loud at 9am on a Tuesday morning?? Bruv do they want me to come bang them??,0 +6813,tianta_: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass.,1 +6814,Nearly had a heart attack just now; loud bang against window next to me___turns out it was two birds flying into the glass.,1 +6815,#auspol Can you see the resemblance between ABBOTT & Campbell both are loud and came in with a big BANG!! Out the same way; Lets see !,0 +6817,The chick I work with chews chewing gum so loud ?? feel to bang her,0 +6819,nikoniko12022: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass.,0 +6820,What the fuck was that. There was a loud bang and a flash of light outside. I'm pretty sure I'm not dead but what the hell??,0 +6821,@SW_Trains strange loud impact bang noises under train to Epsom about to arrive #Wimbledon,0 +6823,Jrowah: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass.,1 +6824,k_matako_bot: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass.,0 +6828,I WAS PEACEFULLY SITTING IN MY ROOM AND I HEARD THIS LOUD BANG OF SOMETHING FALLING,1 +6830,Kijima_Matako: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass.,0 +6832,matako_3: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass.,0 +6833,It's was about 2:30 in the morning&I went downstairs to watch some telly&I accidentally made a loud bang&my dad(who has a broken leg)walked-,0 +6834,ykelquiban: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass.,0 +6835,@ToxicSavior_ -a loud bang. He froze on the spot as slowly every head turned towards him. One of the things he hated the most was to be -,0 +6837,daviesmutia: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass.,1 +6839,@Bang_Me_Up_Guk he was ;-; like he was singing so loud ;-;,0 +6841,Moved on to 'Bang Bang Rock and Roll' by @Art_Brut_ . It's been too long since I've played this one loud. ART BRUT TOP OF THE POPS.,0 +6843,"St steel coffee cafetiere exploded this am with loud bang hot coffee & grounds shot over table clean crockery phone +tablet. How?",1 +6844,matako_milk: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass.,0 +6845,need to work in an office I can bang all my fav Future jams out loud,0 +6847,ColnHarun: Breaking news! Unconfirmed! I just heard a loud bang nearby. in what appears to be a blast of wind from my neighbour's ass.,0 +6848,I don't laugh out loud at many things. But man I really lol @ the big bang theory.,0 +6849,@samanthaturne19 It was... Nagaski another act of mass murder sanctioned and forgiven cause the Allies won... Not by me.,1 +6850,@TANSTAAFL23 It's not an 'impulse' and it doesn't end in mass murder. Correlation does not imply causation.,1 +6852,@ColdMpress You up to commiting mass murder tonight?,1 +6854,The media needs to stop publicizing mass murder. So many sick people do these things for the eyes of the world & the media is not helping.,1 +6855,@samanthaturne19 IIt may logically have been the right call... maybe... But it's an act of mass murder and I can't sanction it.,1 +6857,We have different moral systems. Mine rejects the mass murder of innocents yours explicitly endorses such behavior. https://t.co/qadRKEJZ9T,1 +6858,@DoctorDryadma mass murder here we come,1 +6860,RT owenrbroadhurst RT JuanMThompson: At this hour 70 yrs ago one of the greatest acts of mass murder in world hist___ http://t.co/ODWs0waW9Q,1 +6861,If abortion is murder then blowjobs are cannibalism and masturbation is mass genocide.,1 +6862,"@noah_anyname That's where the concentration camps and mass murder come in. + +EVERY. FUCKING. TIME.",0 +6863,@D1ff3r3nt1sG00d @RiceeChrispies What if he committed a mass murder?,1 +6864,@tuicruises @aida_de Cruise Industry Supports Mass Murder of Whales in #Faroe Islands!! 'Everything the (cont) http://t.co/3a3FGZFmzh,1 +6867,Hiroshima - one of history's worst examples of mass murder http://t.co/TmogTi6FB4 #hiroshima #war #atombomb #japan,1 +6870,@guardian Has Japan ever truly come to terms with devastation and mass murder of millions of Chinese and others with traditional weapons?,1 +6872,Oh the usual. Mass murder and world domination plans over coffee. How's your day going?,1 +6874,http://t.co/c1H7JECFrV @RoyalCarribean do your passengers know about the mass murder that takes place in the #FaroeIslands every year?,1 +6875,@noah_anyname The utopian impulse inevitably ends in gulags and mass murder.,1 +6876,Urgent! Save the Salt River #WildHorses! Mass murder by the very ppl supposed to protect them? --> http://t.co/14wH0pJJ2C @CNN @CBC,1 +6877,This Attempted Mass Murder brought to You by the Obama Administration http://t.co/jbrK8ZsrY6,0 +6878,He made such a good point. White person comings mass murder labelled as criminal minority does the same thing... http://t.co/37qPsSnaCv,0 +6879,http://t.co/FhI4qBpwFH @FredOlsenCruise Please take the #FaroeIslands off your itinerary until the mass murder of dolphins & whales stops.,0 +6880,@FLGovScott We allow Farrakhan to to challenge 10000 males to rise up & commit mass murder as he just did in Miami? http://t.co/gV84WNhB7S,1 +6881,http://t.co/c1H7JECFrV @RoyalCarribean do your passengers know about the mass murder that takes place in the #FaroeIslands every year?,1 +6882,The media needs to stop publicizing mass murder. So many sick people want the eyes of the world and the media... http://t.co/QZlPFHpwDO,0 +6883,@yelllowheather controlled murder is fine. mass murder to a crowd of undeserving people isn't. case closed.,1 +6884,@Re_ShrimpLevy I'm always up for mass murder,1 +6885,@JakeGint the mass murder got her hot and bothered but at heart she was always a traditionalist.,1 +6890,@MNPDNashville @MontalbanoNY sadly suicide by cop. Wed 2PM @ Dollar movie does not a mass murder make.,1 +6892,@billy_hodge Aurora theater shooting trial: Gunman expected notoriety for mass murder and nothing else http://t.co/1RPCHRu72C,1 +6893,"#DebateQuestionsWeWantToHear Why does #Saudi arabia and #Israel get away with mass murder? +#Wahhabism #Zionism",1 +6896,Okay not sure the word 'mass murder' applies during this war but it was horrendous none the less. https://t.co/Sb3rjQqzIX,1 +6897,Another white guy trying to mass murder people for no apparent reason just because let me guess he's mentally ill blah blah blah #Antioch,1 +6898,Hiroshima: 70 years since the worst mass murder in human history. Never forget. http://t.co/jLu2J5QS8U,1 +6900,@CarlaChamorros HILLARY A MASS MURDERER.,0 +6903,Kach was a group to which belonged Baruch Goldstein a mass murderer who in 1994 shot and killed 29 Palestinian___ http://t.co/bXGNQ57xvb,1 +6905,@TrillAC_ I think we've only had like one black mass murderer in the history of mass murders white people do that shit.,1 +6907,Mass murderer Che Guevara greeting a woman in North Korea http://t.co/GlJBNSFGLl',1 +6908,[Creel:You must think I'm a real moron Flag man! A brainless mass of muscle!] I do...But I don't think you're a mass murderer!,0 +6909,#gunsense answer to #GFZ's: break law on the minuscule chance a mass murderer shows up. https://t.co/qEoPMCJbCz,1 +6910,#TheaterShooting DEFENDANT/MASS MURDERER CHOSE NOT 2 TESTIFY IN FINAL PHASE 3 so he won't therefore B subject 2 cross-exam or jury questions,1 +6911,It's like God wants me to become a mass murderer with how many dickheads I have to deal with on a daily basis.,0 +6912,@atljw @cnnbrk fine line btw mass murderer and terrorist. Yes we don't know if there's polit. or social aspect yet; however he went to a,0 +6913,White people I know you worry tirelessly about black on black crime but what are you going to do about your mass murderer problem?,1 +6914,@defendbutera i look like a mass murderer in it,0 +6915,.@LibertyGeek83 Something about kissing the ass of mass murderer doesn't sit right with me. @POTUS feels this is ok. http://t.co/LeJ5OnUs9Q,0 +6916,You happen to conveniently 'forget' about how you Zionists revere and 'honour' mass murderer Baruch Goldstein. https://t.co/3KOB7xBeA0,1 +6917,seeing as how this person is a mass murderer and has like FBI CIA grade equipment to do evil shit.,0 +6918,Hey #movietheatre mass murderer wanna-be we don't play that shit in the #615!,1 +6919,Julian Knight - @SCVSupremeCourt dismisses mass murderer's attempt to increase prisoner pay. Challenged quantum of 5% increase 2013.,1 +6920,@bettyfreedoms @AbnInfVet hillary mass murderer.,1 +6921,"Reminder: Mass murderer and white supremacist Anders Breivik was also unsurprisingly an anti-feminist. +http://t.co/1lXnJVl8TR",1 +6925,@TheEconomist Step one: get that mass murderer's portrait off the yuan.,0 +6926,"If your friends really were your friends they'd support you regardless of your decisions. + +Unless you become a mass-murderer or something",0 +6928,@VictoriaGittins what do you take me for I'm not a mass murderer! Just the one...,0 +6929,Happy boy to mass murderer http://t.co/xPddWH5teM,1 +6930,@NeanderRebel If you take the suit off him I wouldn't be surprised to hear this is the face that belonged to another democrat mass-murderer,1 +6931,Has gun law ever dissuaded a potential mass murderer?,1 +6932,Another White mass murderer. Thank God I live in California. https://t.co/4COg0OTiWn,1 +6933,@TelegraphWorld lets hope it's a upper class white mass murderer....''' Mmmm,0 +6935,Another White mass murderer. Thank God I'm from California. @FrauTrapani,1 +6936,Not only are you a mass murderer but at a movie theatre where niggas dropped bread to see a movie? Cmon man.,0 +6937,@BenignoVito @LibertyBell1000 HILLARYMASS MURDERER.,0 +6939,Nazi Mass Murderer Became Chairman At Vaccine Drug Company In... http://t.co/x713OMh6Ai,1 +6940,Another White mass murderer. #Antioch https://t.co/OWpd7vcFS6,1 +6943,@blairmcdougall and when will you be commenting on Ian Taylor's dealings with mass - murderer Arkan?,1 +6945,@AFK_10 @Dr_JohanFranzen ISIS are orchs. But they don't have the ability to massacre civilians far from the frontlines like the tyrant.,1 +6946,"@Cameron_WATE + Parents of Colorado theater shooting victim fear copycat massacre + +http://t.co/LvlH3W3aWO +#Antioch + +http://t.co/vIwXY1XDYK",1 +6948,UK police link Tunisian beach massacre with Bardo museum attack http://t.co/1fVoOTqnEj,1 +6949,Baby elephant dies just days after surviving massacre of his family http://t.co/qzCUT7bVKT,1 +6950,"@abc3340 +Parents of Colorado theater shooting victim fear copycat massacre + +http://t.co/LvlH3W3aWO +#Antioch + +http://t.co/vIwXY1XDYK",1 +6951,I just bought tickets to DEATH BED / DUDE BRO PARTY MASSACRE III w/ @pattonoswalt Live at @AlamoDC! https://t.co/pmXLeZJBRc,0 +6953,This Friday!! Palm Beach County #Grindhouse Series one night screening of #TexasChainsawMassacre http://t.co/1WopsGbVvv @morbidmovies,0 +6954,@AnimalLogic LOTG smoothed out once blue bird and moth were caught and let go. Chalked up mouse massacre to a subtle Disney diss.,0 +6955,3 Years After the Sikh Temple Massacre Hate-Violence Prevention Is Key | @dviyer @Colorlines http://t.co/nLbLtYnV36 http://t.co/bjrrqHHoHL,1 +6956,"Massacre at #Sinjar : Has the World Forgotten the #Yazidi ? +http://t.co/WUh1g2BLP1 +#??_????_????? #yazidi_shingal_genocide #EzidiGenocide",1 +6959,Colorado movie massacre trial jurors reach verdict on mitigating factors http://t.co/75VLsw85GI http://t.co/txY3US2Ejs,1 +6960,Still Uninvestigated After 50 Years: Did the U.S. Help Incite the 1965 Indonesia Massacre? http://t.co/EZbTG81trz,1 +6961,"@KabarMesir @badr58 +Never dies a big Crime like RABAA MASSACRE as long the revolution is being observed. +#rememberRABAA",1 +6962,Bad day,0 +6963,@eileenmfl are you serious?,0 +6965,Bestie is making me watch texas chainsaw massacre ????????,1 +6966,Petition/No Medals for 1890 Massacre Justice for Wounded Knee Killings of Native Americans! http://t.co/UilPg8i1ev http://t.co/m9pXTo2kwW,1 +6967,I see a massacre!!,1 +6970,"Headed to the massacre +Bodies arriving everyday +What were those shells you heard +Picking the bones up along the way",1 +6971,"@Bloodbath_TV favourite YouTube channel going right now. +Love everything you guys do and thank you introducing me to Dude Bro Party Massacre",0 +6972,The Fake of Nanking Massacre-4 Eyewitnesses (English): http://t.co/TiPnDEmPuz #Obama #Clinton #Bush #GOP #ABC #CBS #BBC #CNN #WSJ #WPO,1 +6973,Remember this was a massacre of civilians. #Hiroshima http://t.co/qw8qk165Sb,1 +6974,Don't mess with my Daddy I can be a massacre. #BeCarefulHarry,0 +6975,Tunisia beach massacre linked to March terror attack on museum http://t.co/kuRqLxFiHL,1 +6977,@CIA hey you guy's i stopped a massacre so you send the cops to my house to make this town permanently hate me wtf?,0 +6978,@freddiedeboer @Thucydiplease then you have rise of Coates Charleston massacre Walter Scott and black twitter more broadly as well.,1 +6979,Review: Dude Bro Party Massacre III http://t.co/f0WQlobOoy by Patrick BromleyThe title sa http://t.co/THpBDPdj35,0 +6980,@gigagil IOF murdered over 513 Palestinian children (70% under 12) during Gaza Massacre where was zionist morality-zionism is a world evil!,1 +6981,Sousse beach massacre linked to Tunis museum attack http://t.co/MyHHTHsLi3,1 +6983,Soap and education are not as sudden as a massacre but they are more deadly in the long run. -- Mark Twain,0 +6984,Tell @BarackObama to rescind medals of 'honor' given to US soldiers at the Massacre of Wounded Knee. SIGN NOW & RT! https://t.co/u4r8dRiuAc,1 +6985,"@nataliealund +Parents of Colorado theater shooting victim fear copycat massacre + +http://t.co/LvlH3W3aWO +#Antioch + +http://t.co/vIwXY1XDYK",1 +6988,"@WKRN +Parents of Colorado theater shooting victim fear copycat massacre + +http://t.co/LvlH3W3aWO +#Antioch + +http://t.co/vIwXY1XDYK",1 +6990,@MartynWaites It's a well-known fact that the St Valentine's Day massacre could have been avoided with some 'oompah-period' Tom Waits.,1 +6992,The Martyrs Who Kept Udhampur Terrorists at Bay Averted a Massacre: It was two young___ http://t.co/nux5XfPV2d SPSå¨,1 +6994,Is this the creepiest youth camp ever?. http://t.co/T8uqm7Imir,0 +6996,Beach Reads in August #Giveaway Hop & @StuckInBooks is giving away any book in Mayhem series! http://t.co/Jp3OY0OuXq,0 +6998,@CVinch_WAOW thank you! Drove by the mayhem. How scary! -Stacy,0 +6999,Slayer Reflects on Low Mayhem Festival Attendance King Diamond & Jeff Hanneman's Passing http://t.co/lfw4iymsak,0 +7001,I guess ill never be able to go to mayhem...,0 +7002,I liked a @YouTube video from @itsjustinstuart http://t.co/oDV3RqS8JU GUN RANGE MAYHEM!,1 +7005,@RaynbowAffair Editor In Chief @DiamondKesawn Releases Issue #7 http://t.co/ge0yd3mKAv of #RAmag. #Fashion #Models and #Mayhem,0 +7006,@alexbelloli I do It just seemed like the pages were out of order,0 +7008,@RaynbowAffair Editor In Chief @DiamondKesawn Releases Issue #7 http://t.co/7mzYcU2IHo of #RAmag. #Fashion #Models and #Mayhem,0 +7009,Tonight It's Going To Be Mayhem @ #4PlayThursdays. Everybody Free w/ Text. 1716 I ST NW (18+) http://t.co/sCu9QZp6nq,0 +7010,Mayhem is beautiful,0 +7012,@Mayhem_114 are you reading right to left,0 +7013,Anyone else think that Stephen sounds like Andy Dick when he gets excited? The difference being... I actually like Stephen. #MasterChef,0 +7015,#NoSurrender Results: Full Metal Mayhem World Title Match Bully Ray Taken Out A Career Comes To An End and More! http://t.co/G6moNVnpSu,0 +7018,@alexbelloli well now I know lol,0 +7019,#TBT to that time my best friend and I panicked at the disco. https://t.co/htpqvoHtUd,0 +7021,Magic City Mayhem: Kissimmee adventures ? Aug. 5 2015 http://t.co/FpYrU5GOLh,0 +7023,"_ +?????RETWEET +??????? +?????FOLLOW ALL WHO RT +??????? +?????FOLLOWBACK +??????? +?????GAIN WITH +??????? +?????FOLLOW ?@ganseyman #RT_4_A_MENTION +#TY",0 +7024,@RaynbowAffair Editor In Chief @DiamondKesawn Releases Issue #7 http://t.co/RPnEAJ6fOD of #RAmag. #Fashion #Models and #Mayhem,0 +7025,They are the real heroes... RIP Brave hearts... http://t.co/Q9LxO4QkjI,0 +7026,The Campaign: Will Ferrell and Zach Galifianakis commit comic mayhem in this hilarious political farce. 4* http://t.co/tQ3j2qGtZQ,0 +7027,Tonight It's Going To Be Mayhem @ #4PlayThursdays. Everybody Free w/ Text. 1716 I ST NW (18+) http://t.co/cQ7jJ6Yjfz,0 +7029,Asbury Park shooting reported http://t.co/dADZ5ZFO1g via @AsburyParkPress,1 +7030,I__ªve seen lots ask about this MT @JMCwrites #Pitchwars I asked for magic realism but not fantasy. What's the diff? http://t.co/64xR9LtNOH,0 +7031,Slayer Reflects on Low Mayhem Festival Attendance King Diamond & Jeff Hanneman's Passing http://t.co/6N6Gcej9Iy,0 +7032,The best. Mind is clear content and observations for days. https://t.co/pCCwsGCymA,0 +7036,@RaynbowAffair Editor In Chief @DiamondKesawn Releases Issue #7 http://t.co/EbbF1N7MAJ of #RAmag. #Fashion #Models and #Mayhem,0 +7037,@Akcsl bit of both. going to venture out once I've chosen suitable music for the mayhem...,0 +7039,BREAKING: Authorities respond to Ocean fire http://t.co/h31Knuwzz5,1 +7043,I liked a @YouTube video from @itsjustinstuart http://t.co/Mnkaji2Q1N GUN RANGE MAYHEM!,0 +7044,#NoSurrender Results: Full Metal Mayhem World Title Match Bully Ray Taken Out A Career Comes To An End and More! http://t.co/XEHwmsH7Lv,0 +7046,<meltdown of proportions commences I manage to calm myself long enough to turn the waters to hot and wait for the steam to cloud my vision-,0 +7047,the straight bass dubloadz droppd when they opened @ meltdown had everyone meelllttting http://t.co/vXsQPFE9NA,0 +7048,Deepak Chopra's EPIC Twitter Meltdown http://t.co/ethgAGPy5G,0 +7050,The ol' meltdown victory for the Mets.,1 +7051,@DrDrewHLN 'A simple meltdown!' Areva have you ever seen an out of control kid?,0 +7052,@tinybaby @adultblackmale @mattytalks meltdown mwednesday,1 +7053,@kinkyconnors IM sorry for my meltdown last night lmao but I'm getting my tooth fixed Friday ??????????????????????????????????,0 +7054,One of these candidates is going to have a Frank Grimes level meltdown with how voters love Trump no matter what. http://t.co/pBEgOf4740,0 +7056,I had a meltdown in Demi's instagram comments http://t.co/mcc76xOwli,0 +7058,@nashhmu have a meltdown he noticed you,0 +7059,Whenever I have a meltdown and need someone @Becca_Caitlyn99 is always like 'leaving in 5' and I don't know how I got so lucky #blessed,0 +7060,Looks like it may have been microsofts anti virus trying to update causing the meltdown,0 +7061,DFR EP016 Monthly Meltdown - On Dnbheaven 2015.08.06 http://t.co/EjKRf8N8A8 #Drum and Bass #heavy #nasty http://t.co/SPHWE6wFI5,0 +7063,THE GLOBAL ECONOMIC MELTDOWN is out! http://t.co/DGATKRdyNy Stories via @seagull07 @AleisStokes @intelligencebar,1 +7065,"@JustinTrudeau + +Most respected in world +Best job recovery G7 and G20 +Best led during 2008 world meltdown + +What exactly have you done ??",0 +7068,@crowdtappers @joinvroom OMG I remember the meltdown the day I did her hair like ELSA and not ANNA.... OHHHH THE HORROR!!! LOL #tangletalk,0 +7069,@DmoneyDemi I had my meltdown yesterday. I'm going to miss you so much. You are forsure my DTB for life. When I get back watchout ??,0 +7070,Why must I have a meltdown every few days? ??,0 +7071,@nprfreshair I really can't believe he is skipping out before the Republican meltdown...I mean 'debate'.,0 +7073,Commoditieså_Are Crashing Like It's 2008 All Over Again http://t.co/nJD1N5TxXe via @business,0 +7074,A back to school commercial came on and my sister had a meltdown. ??????????,0 +7075,?? ice age: the meltdown,0 +7076,http://t.co/HFqlwo1kMy E-Mini SP 500: Earnings letdown equals market meltdown! http://t.co/LEi9dWVllq #Amazon,0 +7077,"def louis is tired plus the meltdown of fans when he confirmed he's a dad but some fans are reaching +making conclusions and stuff",0 +7078,Ever since my Facebook #Mets meltdown after the Padres fiasco- mets are 6-0. You're welcome,0 +7080,Currently: Uncontrollable meltdown number 2,0 +7081,Pam's Barry Island wedding meltdown ??????????,0 +7086,@LeMaireLee @danharmon People Near Meltdown Comics Who Have Free Time to Wait in Line on Sunday Nights are not a representative sample. #140,0 +7088,Meltdown,0 +7089,Have you read this awesome book yet? The Two Trillion Dollar Meltdown http://t.co/jPA6sajFE3,0 +7091,Byproduct of metal price meltdown is a higher silver price http://t.co/cZWjw4UV7i,1 +7092,President Barack Obama has on air meltdown over opposition to Iran nuclear deal http://t.co/c0t7RvoTKj via @examinercom,1 +7093,LOL Warthen in the midst of bullpen meltdown reaching for Double Bubble. #Mets,0 +7095,#3: TITAN WarriorCord 100 Feet - Authentic Military 550 Paracord - MIL-C-5040-H Type III 7 Strand 5/16' di... http://t.co/EEjRMKtJ0R,0 +7097,@FurTrix then find cougars who look like her even better if they're in military uniform!,0 +7098,"Russian #ushanka #winter #military fur hat (xl61-62) with soviet badge LINK: +http://t.co/74YFQxvAK0 http://t.co/KXrEHVt6hL",0 +7104,Senator 'alarmed' by reports U.S. military families harassed: A U.S. Senator said on Wednesday he was alarmed by___ http://t.co/sbILA2Yqjq,1 +7105,@YMcglaun THANK YOU FOR UNDERSTANDING THE GOV. ONLY TELLS US ABOUT 5% OF WHATS REALLY GOING ON I HAVE MILITARY HOUSE & CIA CONNECTS !!!,0 +7106,@UnivSFoundation For the people who died in Human Experiments by Unit 731 of Japanese military http://t.co/vVPLFQv58P http://t.co/Rwaph6dAUv,1 +7108,Mike Magner Discusses A Trust Betrayed: http://t.co/GETBjip5Rh via @YouTube #military #veterans #environment,0 +7109,"Hat #russian soviet army kgb military #cossack #ushanka LINK: +http://t.co/bla42Rdt1O http://t.co/EInSQS8tFq",0 +7111,@kiranahmedd US military and Nato are fighting Talibans too,1 +7112,Army names 10th Mountain units for Iraq Afghanistan deployments (Deeds) http://t.co/N6ZfLXIGvr,0 +7114,Ford : Other Military VERY NICE M151A1 MUTT with matching M416 Trailer - Full read by eBay http://t.co/9rrYaYlgyY http://t.co/Nm83jOhLUu,0 +7115,"Thank You Senator @timkaine for your Leadership on #IranDeal +Our Military deserves nothing less. THANK YOU!! + +https://t.co/578GUNP8t9",0 +7116,Mike Magner Discusses A Trust Betrayed: http://t.co/psbxl1HvU3 via @YouTube #military #veterans #environment,0 +7118,i strongly support our military & their families just not the cock suckers in DC they work for,0 +7120,@stfxuniversity For the people who died in Human Experiments by Unit 731 of Japanese military http://t.co/vVPLFQv58P http://t.co/l5AWTUndhm,1 +7121,13 reasons why we love women in the military - lulgzimbestpicts http://t.co/XKMLQ99SjY http://t.co/a3RGQuCUgo,0 +7122,Online infantryman experimental military training tutorials shower down upon assertative intelligence as regard...,0 +7124,@2for1Lapdances For the people who died in Human Experiments by Unit 731 of Japanese military http://t.co/vVPLFQv58P http://t.co/yOMPxJpPTV,1 +7125,How do you call yourself a base conservative when u think it's okay 2 duck military service & contribute thousands 2 #HillaryClinton 2012?,0 +7126,We're #hiring! Read about our latest #job opening here: Registered Nurse - Call-in - Military Program - http://t.co/l0hhwB9LSZ #Nursing,0 +7128,Courageous and honest analysis of need to use Atomic Bomb in 1945. #Hiroshima70 Japanese military refused surrender. https://t.co/VhmtyTptGR,1 +7129,13 reasons why we love women in the military - lulgzimbestpicts http://t.co/N1tjCt8HMC http://t.co/pYTQM7rVP0,0 +7130,Listen LIve: http://t.co/1puLaekxcq #Author #Interview Beth Underwood of #Gravity on #Military #Mom #TalkRadio,0 +7131,@TeamHendrick @TeamHendrick @RIRInsider Fingers crossed that there will be a driver from Hendricks in Military Hospitality w/ @neanea2724!,0 +7132,13 reasons why we love women in the military - lulgzimbestpicts http://t.co/IAPvTqxLht http://t.co/WAMKRe6CKD,0 +7134,13 reasons why we love women in the military - lulgzimbestpicts http://t.co/uZ1yiZ7n6m http://t.co/IjwAr15H16,0 +7135,@UniversityofLaw For the people who died in Human Experiments by Unit 731 of Japanese military http://t.co/vVPLFQv58P http://t.co/eG1fsKqBv6,1 +7136,Lot of 20 Tom Clancy Military Mystery Novels - Paperback http://t.co/ObiX79NcxN #tomclancy,0 +7137,@CochiseCollege For the people who died in Human Experiments by Unit 731 of Japanese military http://t.co/vVPLFQv58P http://t.co/ldx9uKNGsk,1 +7138,I remember when I worked at Mcdonalds I use to be hours late because we used military time and I use to mess up when I had to be there :/,0 +7140,Pakistan Supreme Court OKs Military Courts to Try Civilians: The Supreme Court ruling would empower the milita... http://t.co/v0nf1Uc1OW,0 +7141,Bad News for US: China Russia Bolstering Military Cooperation in Asia / Sputnik International http://t.co/9q9Rk3fOf7 via @SputnikInt,0 +7142,Study: Wider Variety of Therapies Could Help Vets Troops With PTSD | http://t.co/g0q0bzBjli http://t.co/ExYr6c5QPu via @Militarydotcom,0 +7144,the MOFO in DC will leave our military unarmed to be gunned down by terrorist & a lot worseits not their sorry asses,0 +7146,Stu Dorret's mudslide rubber tyre cake may have saved you #GBBO,0 +7147,'It looks like a mudslide!' And #GBBO is back with a belter!,1 +7148,STERLING-SCOTT on the Red Carpet at a fundraiser for 'OSO Mudslide' https://t.co/mA4ra7AtqL http://t.co/cg579wlDnE,1 +7149,Stu put beetroot in his cake and even lost to a mudslide,0 +7150,@Pete_r_Knox @Gemmasterful I think the mudslide cake lady will go and the hipster will unfortunately stay.,0 +7154,Hope Dorett's 'mudslide' cake wins?? #GBBO,0 +7155,'It looks like a mudslide' 'It's like chewing on rubber' #GBBO ??????,0 +7157,The 19 year old's smug face when Dorret brings out her mudslide Black Forest gateau #priceless #GBBO,0 +7158,@marc_holl @NenniCook @AitchKayCee @vixstuart @benjbeckwith It's not pretty ??#disaster #GBBO #mudslide,1 +7159,Looks like a mudslide and tastes like rubber oh how I love the bake off! #britishbakeoff #paulhollywood,0 +7160,'It looks like a mudslide' poor thing! ?? #greatbritishbakeoff,1 +7161,God bless you and your mudslide cake Dorret ????,0 +7163,Flood and Mudslide Warning for East Fjords | Iceland Review http://t.co/534q3Jg2OV #icelandreview via @iceland_review,1 +7164,Someone split a mudslide w me when I get off work,0 +7165,Loved the opener and still feeling guilty for gasping and giggling at the mudslide - Dorret we live you #GBBO,0 +7166,2 great new recipes; mudslide cake and so sorry stew! #GBBO,0 +7167,@SophieWisey I couldn't. #mudslide,1 +7168,"#GBBO The difference between Paul and Mary my dears +Paul: 'it looks like a mudslide' +Mary: 'I have a feeling it's going to taste great'",0 +7169,DORETTE THATS THE NAME OF THE MUDSLIDE CAKE MAKER,0 +7171,RT RabidMonkeys1: Ah the unique mudslide cake ??#GBBO http://t.co/ZT5OFbiwtD,0 +7173,@brobread looks like mudslide????,1 +7174,"@Lolly_Knickers It's a mudslide. +It's like chewing on a rubber tyre. +And with those I'm DONE. +#vaginaorcake #GBBO",1 +7175,You've 100% fucked up when Paul says your cake looks like a 'mudslide' and tastes like 'rubber',0 +7176,First impressions: glad hat man is leaving in lieu of more interesting ladies. Hope mudslide lady triumphs next week.,0 +7178,@hazelannmac ooh now I feel guilty about wishing hatman out. I bet the mudslide was delicious!,0 +7179,@UrufuSanRagu a Mudslide?,1 +7180,British bake off was great pretty hilarious moments #mudslide,0 +7183,@new_hart2010 #mudslide... nuff said #GBBO,1 +7184,Oso Washington Mudslide Response Interview ___ Part 1 http://t.co/sbfGLQjZfs,1 +7185,#BakeOffFriends the one with the mudslide,0 +7186,HE CALLED IT A MUDSLIDE AW,0 +7187,@MarianKeyes Rubber Mudslide! Still laughing!,0 +7188,#BakeOffFriends #GBBO 'The one with the mudslide and the guy with the hat',0 +7191,When bae soak you in mudslide at backroom???? #thisiswhywecanthavenicethings http://t.co/kgxNwzIUxd,0 +7192,her cake looks like a mudslide hah,0 +7193,First time getting into #gbbo2015 and physically gasped at the cake 'mudslide' incident already way too emotionally invested...,1 +7194,@nikistitz even the one that looked like a mudslide?,0 +7195,she's a natural disaster she's the last of the American girls ??,0 +7196,"'Up to 40% of businesses affected by a natural or man-made disaster never reopen' +http://t.co/35JyAp0ul9",0 +7197,@Doylech They're refugees: 'people who have been forced to leave their country in order to escape war persecution or natural disaster',1 +7198,On the sneak America has us spoiled. A natural disaster will humble niggas.,1 +7199,Ra_¼l sends a message of condolence to Vietnam following natural disaster http://t.co/bgyTmqJ3OO,1 +7201,@Cali74142290 lol natural disaster/hospital crisis something is needed to get rid of some cast members....,1 +7202,Rationing of food and water may also become necessary during an emergency such as a natural disaster or terror attack.,0 +7203,Some people are really natural disaster too,1 +7204,What Natural Disaster Are You When You Get Angry? http://t.co/q4gl3Dvhu1,0 +7205,Coming later this year~ 'THE MAN THAT TATTOOED WOMEN.' A novel based on a real serial killer from #Arkansas & a natural disaster. #NOLA,1 +7206,Ra_¼l sends a message of condolence to Vietnam following natural disaster: On behalf of the Cuban government an... http://t.co/EmrZiCb004,1 +7208,@TwopTwips make natural disaster reporting more interesting by adding 'The' to headlines such as 'Rescuers are sifting through the wreckage',0 +7210,Expert Prepper: Financial Collapse Natural Disaster Failed Grid... http://t.co/AVVSOiNO8Z http://t.co/VoYrUxcrIN,1 +7211,Rise up like a natural disaster we take the bat and then we take back the town????,0 +7213,its only getting colder and colder and faster and faster and when i first realized it it was like a natural disaster,1 +7214,anyway 2 me? Mateo just doesnt exist? Hes a mirage a pointless addition to our Generation. a human natural disaster. Im sorry but its true,0 +7215,natural disaster ___ News Stories About natural disaster - Page 1 | Newser http://t.co/TB8gZEMbXU,1 +7218,And you're loving me like water slipping through my fingers such a natural disaster love,0 +7221,I added a video to a @YouTube playlist http://t.co/v2yXurne2p Natural Disaster Survival - HUG BY A GUEST!! on Roblox,0 +7223,"This is the natural and unavoidable consequence of socialism everywhere it has been tried. +http://t.co/BbDpnj8XSx A",1 +7224,Suncorp net profit rises to $1.13 billion in worst year of natural disaster claims http://t.co/cwZ37lNDVk,1 +7226,Of course the one day I have to dress professionally aka unsensibly for class is the day I have try and outrun a natural disaster!,1 +7227,*books a flight to Burma when country is enduring political unrest and a natural disaster* no wonder it was so cheap ay,1 +7228,@ConnorFranta #AskConnor if you were a natural disaster what would you be?,0 +7229,We have overrun a Natural Disaster Survival server!,1 +7230,"This is the natural and unavoidable consequence of socialism everywhere it has been tried. +http://t.co/BbDpnj8XSx E",0 +7231,Top insurer blasts lack of Australian Govt action on disaster mitigation http://t.co/sDgOUtWNtb via @smh,1 +7232,Hello natural hazards/disaster recovery & emergency management ppl can u recommend good hashtags to follow OR send me links of good reads?,0 +7234,@BookTubeAThon A world in which people aren't dying from natural and supernatural disaster and war most likely.,0 +7235,Savings and sewing in Guatemala: Savings and sewing in Guatemala. When a natural disaster hit seamstress Elvia... http://t.co/jdx9OX2kIk,1 +7236,Patriot Survival Guide: Do you know how survive when the governments collapse or other natural disaster happens? http://t.co/QhtoerhDkM,1 +7241,"This is the natural and unavoidable consequence of socialism everywhere it has been tried. +http://t.co/BbDpnj8XSx F",0 +7242,What Natural Disaster Are You When You Get Angry? http://t.co/O9DzgZqEMf,0 +7244,Is your team ready for a natural disaster a violent client or power outage? Contact Ready Vet to design... http://t.co/u2NJPoR39K,0 +7247,beforeitsnews : 3 former executives to be prosecuted in Fukushima nuclear disaster ___ http://t.co/FgVN2vCrrX) http://t.co/kftVNU7nvf,1 +7248,Alarming Rise in Dead Marine Life Since the #Fukushima Nuclear Disaster: http://t.co/v6H97K688J http://t.co/tJw9bSeiPW,1 +7250,"Nuclear deal disaster. + +#IranDeal #NoNuclearIran #BadIranDeal @JebBush @BarackObama http://t.co/z7phPjtqud",0 +7251,If i tweet daily #Fukushima #Japan global nuclear disaster& #Chernobyl ppl dont care there is no hope 4 humanity C> http://t.co/MAcob5xLsU,1 +7252,"?#FUKUSHIMA?#TEPCO? +Fukushima Nuclear Disaster | Increased Thyroid Cancer in #US - Integrative Cancer Answers +http://t.co/7Y2GNVA2eV",1 +7253,Though serious #natural #disaster increased in #Japanjapan #Nuclear safety standard is far inferior to #US standard.#anonymous #Nytimes,1 +7254,Physical sense helps preserve memories of 2011 triple disaster (8/4 AJW) http://t.co/X5jGKjV6Ma #jishin_e #nuclear #Minamisoma,1 +7255,"http://t.co/GaM7otGISw +ANOTHER DISASTER WAITING TO HAPPEN AND YOUR ALLOWING IT???",1 +7256,@DalaiLama then have the biggest nuclear disaster to ever have happened,1 +7260,#Obama signed up to a deal that far from making the world a safer place http://t.co/E0luGBL6pb via @upi #Iran #Nuclear #IranNuclearDeal,1 +7261,3 former executives to be prosecuted in Fukushima nuclear disaster http://t.co/zsDVWEgLF5,1 +7264,@emmerdale can we have a public vote for the next annual village disaster? i want an isis strike or a nuclear accident & end this forever,1 +7265,3 former executives to be prosecuted in Fukushima nuclear disaster http://t.co/Gvj7slbELP,1 +7266,Fukushima Nuclear Disaster | Increased Thyroid Cancer in U.S. http://t.co/FtuNlH6ddg,1 +7267,The people who tweet and care about #Japan #Fukushima nuclear disaster are not the problem those who ignore are the problem.,1 +7268,We want to see no more Hiroshima and Nagasaki nuclear bomb disaster in this beautiful world.Lets be peaceful & save this human civilization.,1 +7270,#Japan #Fukushima the most serious man made disaster in human history... ___a marker of radiation brain damage___ http://t.co/Y3ZfqJsvpz,1 +7272,#refugees of #nuclear disaster in western #Japan will be beyond 500 thousand at least. #AFP #guardian #WPS #NYTimes,1 +7274,3 Former Executives To Be Prosecuted In Fukushima Nuclear Disaster http://t.co/UmjpRRwRUU,1 +7275,@drvox Trump can say nice things about nuclear power but 'drill baby drill!' + AGW denial = policy disaster for nuclear power.,0 +7277,CLOSING THEIR EYES TO DISASTER! State Department Unaware of Reports Iran is Sanitizing Nuclear Sites http://t.co/yRVGyKdbM6,1 +7278,"3 Former Executives to Be Prosecuted in Fukushima Nuclear #Disaster. + +The story: http://t.co/7uFnxxaVqs via @nytimes",1 +7280,Any disaster impairs mental health especially in vulnerable individuals... http://t.co/ZisuwLqRHf,1 +7281,Fukushima 'mutant vegetable' images sweep across the region two years after nuclear disaster http://t.co/psi35AU3pc via @MailOnline,1 +7283,Bombing #Iran would result in a never-ending game of #nuclear whack-a-mole. Here's why: http://t.co/6exS23MUy3 http://t.co/l9iDheROtj,1 +7285,Fukushima: The Story of a Nuclear Disaster http://t.co/ikpnGs3dTi http://t.co/lJHgSdRAEZ,1 +7286,The president spoke of Kennedy's diplomacy to avert nuclear disaster during the Cold War to bolster his own pitch for Congress' approval,1 +7287,573 deaths have been certified by the Government as __÷nuclear disaster-related deaths__ª,1 +7288,3 Former Executives to Be Prosecuted in Fukushima Nuclear Disaster http://t.co/JSsmMLNaQ7,1 +7289,70 years ago today 1945 #Hiroshima was the first nuclear atomic bomb leaving a major disaster in the wake. http://t.co/oW4GeXyNbH,1 +7290,If U care about life on land sea & in air then dont pretend & ignore the nuclear disaster in #Japan #Fukushima this is a #global problem,1 +7291,"#Nuclear policy of #Japan without responsibility about Nuclear #Disaster will repeat same #failure. +#annonymous #guardian #NYTimes #Reuters",1 +7292,Over half of poll respondents worry nuclear disaster fading from public consciousness http://t.co/YtnnnD631z ##fukushima,0 +7294,Chernobyl disaster - Wikipedia the free encyclopedia don't you just love the nuclear technology it's so glorious https://t.co/GHucazjSxB,1 +7295,Err:509,0 +7296,Check out this awesome profile on #GE's swimming #robot used in #nuclear reactors! http://t.co/HRc3oxQUIK #innovation http://t.co/wNPTvbM5T7,0 +7297,Finnish Nuclear Plant to Move Ahead After Financing Secured-> http://t.co/uHkXMXaB9l,0 +7299,Global Nuclear Reactor Construction Market grew 4.5% between 2013 and 2014 http://t.co/74JppeK6o7,1 +7300,Has An Ancient Nuclear Reactor Been Discovered In Africa? ___ Your... http://t.co/qadUfO8zXg,0 +7301,Salem 2 nuclear reactor shut down over electrical circuit failure on pump: The Salem 2 nuclear reactor had bee... http://t.co/5hkGXzJLmX,1 +7302,Quick Fact: No #nuclear reactor has come into operation in the #US for over 43 years. http://t.co/V1mtR517Ue,0 +7306,US Navy Sidelines 3 Newest Subs - http://t.co/guvTIzyCHE: DefenseNews.comUS Navy Sidelines 3 Newest SubsD... http://t.co/SY2WhXT0K5 #navy,1 +7307,"Germany has 39 gigawatts of installed solar capacity +_One gwatt is about equal to the capacity of a nuclear reactor. +http://t.co/leCZOlkmSV",0 +7309,US Navy Sidelines 3 Newest Subs http://t.co/okamsCZbwg,0 +7310,'Nuclear reactor is like a woman. You just have to read the manual and press the right button.',0 +7311,Nuclear reactor railguns would be a great way to deliver t1000s.,0 +7312,@RobertHarding @RepJohnKatko Crazy to use the Netanyahu argument Iran can flush A NUCLEAR REACTOR DOWN THE TOILET,1 +7315,Government under pressure to abandon plans to construct UK__ªs first nuclear reactor for more than 20 year http://t.co/E9d9Lk5Fdw,0 +7316,Nuclear-Deal: Indo-Japan pact lies at the heart of two US reactor-based projects: If Japan were to go ahead an... http://t.co/XKURDr3yEv,1 +7317,Nuclear #Solar Power #Japanese #Fukushima Reactor Energy Japan Temperature No. Fuel Pool More http://t.co/YS3nMwWyVc http://t.co/AlpotNB7q3,1 +7318,Japan's Restart of Nuclear Reactor Fleet Fast Approaches http://t.co/DbAUjp29Ub,1 +7319,If I'm also not mistaken we're sitting right next 2 a nuclear reactor. Not smart when we have a cataclysmic event & that reator melts down.,1 +7320,@Willie_Am_I @JusttheBottle I would cry in to my nuclear reactor cooling tank! #winechat,0 +7323,Magnetic and electrostatic nuclear fusion reactor http://t.co/eM5oPyTBpg,0 +7324,@stunckle @Gordon_R74 @crazydoctorlady ...I'm no expert but raw uranium and nuclear reactor fuel rods are 2 very different creatures...,1 +7325,Finnish ministers: Fennovoima nuclear reactor will go ahead via /r/worldnews http://t.co/fRkOdEstuK,0 +7326,Navy sidelines 3 newest subs http://t.co/gpVZV0249Y,0 +7328,US Navy Sidelines 3 Newest #Subs http://t.co/9WQixGMHfh,0 +7329,Rolling sandunes the gentle lapping of the sea the call of gulls and a nuclear reactor #sizewell http://t.co/X9CUiHIb5n,1 +7330,Finnish nuclear plant to move ahead after financing secured http://t.co/D8aWX2okKe via @business,0 +7331,US Energy Department Offers $40Mln for New Nuclear Reactor Designs / Sputnik International http://t.co/0DxVZ7fDh3,0 +7332,A little filming inside a Nuclear Reactor at #Chernobyl @SonyProUSA @LumixUSA @DJIGlobal @ProfBrianCox @RT_America https://t.co/2GLjhvEAD9,0 +7334,Salem 2 nuclear reactor shut down over electrical circuit failure on pump http://t.co/LQjjy1PTWT,1 +7335,Finnish ministers: Fennovoima nuclear reactor will go ahead http://t.co/vB3VFm76ke #worldnews #news #breakingnews,1 +7337,Salem 2 nuclear reactor shut down over electrical circuit failure on pump - http://t.co/98o2Kc3A1Z http://t.co/tGdontTkty,1 +7338,Finnish ministers: Fennovoima nuclear reactor will go ahead http://t.co/mqMCOLwBzc,0 +7340,Aug. 5: The Fort Calhoun Nuclear Station is at 100% capacity today according to the NRC: http://t.co/pztbQImpuW,0 +7342,@SnowyWolf5 @TheGreenParty Besides would you rather shut down a whole nuclear reactor for maintenance or a wind turbine at a time?,1 +7343,HamptonRoadsFor.me US Navy Sidelines 3 Newest Subs - http://t.co/9QNQ45Zduw http://t.co/dhyLJllRHL,0 +7344,A 17 year Boy Scout created a mini nuclear reactor in his home,0 +7345,@dicehateme @PuppyShogun This makes sense. Paper beats rock paper comes from wood so wood should be able to support and obliterate rock.,0 +7348,The People's Republic Of China ( PROC ): Abandon the West Philippine Sea and all the ... https://t.co/pD14GsrfSC via @ChangePilipinas,0 +7350,@realDonaldTrump to obliterate notion & pattern political style seemly lead voter beliefs Sarcasm Narcissism #RichHomeyDon #Swag #USA #LIKES,0 +7352,"#fun #instagramers http://t.co/M3NJvvtYgN + +Jeb Bush said earlier this week that not only does he want to obliterate Planned Parenthood bu___",0 +7354,Souda leave Lady Sonia alone or I shall obliterate you #KneelBot,0 +7355,@klavierstuk doesn't so LVG is forced into the market. May beat Spurs and smaller teams with Blind LCB. Top 4/ CL teams will obliterate us.,0 +7356,Whereas Jez will obliterate the national debt - and give lots of new benefits - by simply printing money! Genius! https://t.co/ReffbkVG9R,1 +7357,#LOL Plymouth (__÷Let__ªs Obliterate Litter__ª) http://t.co/GDrssjbH8q,0 +7358,According to prophecy and also CNN a Mac tablet will completely obliterate the need for other gadgets. Combining___ http://t.co/xfccvMXuWb,0 +7359,@JoseBasedGod I'm obliterate you to the shadow realm.,0 +7360,GOP debate tonight but no Jon Stewart next week to obliterate them ?? #JonVoyage,0 +7361,@McCaineNL Think how spectacular it will look when the Stonewall riots obliterate the white house.,1 +7362,Watch Sarah Palin OBLITERATE Planned Parenthood For Targeting Minority Women! ___ BB4SP http://t.co/Dm0uUpqGWY,1 +7365,@SkyNews how is this scum still alive obliterate this cancer,0 +7366,WWE 2K15: Universe Mode - Part 149 - OBLITERATE!!: http://t.co/0oms8rI3l1 via @YouTube,0 +7367,Time to obliterate this sin!,0 +7369,"Know them recognize them......then obliterate them! +#gym #gymflow #gymtime #team #assassins___ https://t.co/mUHj8CbdQb",0 +7371,Dc I love you but please obliterate power girl,0 +7372,@Gargron good sir I did not wish to but now I am forced to. I must obliterate you,0 +7373,@holymileyray @moonIighthunty Focus on Me is going to obliterate careers tea,0 +7374,Do you ever just want to obliterate an entire species off the face of the earth? I vote for mosquitoes,0 +7377,Someone teaching you that obedience will obliterate trials in your life is trying to sell you a used car. Jesus's life blows that theory.',1 +7378,Wondering if gold could gap up $50 on the jobs numbers tomorrow and just obliterate the shorts. A big player with guys could smash them,0 +7380,It should be our duty to obliterate superstition from our society : Swami Agnivesh,0 +7381,Meek Mill Begging Nicki Minaj To Let Him Obliterate... #ovofest #nowplaying http://t.co/XOmI4ZQzgp http://t.co/0m1TW3DaTd,0 +7383,World of warships makes me mad sometimes but it's soooo satisfying when you cross the T on a battleship and just obliterate it,0 +7386,Watch Sarah Palin OBLITERATE Planned Parenthood For Targeting Minority Women! ___ BB4SP http://t.co/fqMYprlG9g,0 +7390,'But time began at last to obliterate the freshness of my alarm;...' http://t.co/ciLO9pMlEb,0 +7391,@MacBreck I know what it means.It means I'll go on Twitter and obliterate any CHANCE of EVER winning another election for Pres.bad at math.,0 +7392,Watch Sarah Palin OBLITERATE Planned Parenthood For Targeting Minority Women! ___ BB4SP http://t.co/sAYZt2oagm,0 +7393,'We are now prepared to obliterate more rapidly and completely every productive enterprise the Japanese have above ground in any city.',0 +7395,@horiikawa i played online last night and got my ass obliterated,0 +7397,RIZZO IS ON ???????? THAT BALL WAS OBLITERATED,0 +7400,Obliterated,0 +7402,Damnnnn $GMCR got obliterated -26% ;that should pay HUGE to whomever played accordingly,0 +7403,air sunrays obliterated on its hairy head like a dried wire spiderweb,0 +7405,I'm about to be obliterated,0 +7407,@breakingnewslh @bree_mars watch cnn's the seventies terrorism episode. Iran has always hated the U.S. They want us obliterated.,1 +7408,"Me- Don't bother calling or texting me because my phone is obliterated + +*has 7k missed calls and messages*",0 +7410,WACKOES like #MicheleBachman predict the WORLD will SOON be OBLITERATED by a burning firey INFERNO but can't accept #GlobalWarming!! HELLO!!,0 +7412,This the same dude that obliterated meek mill. Dont judge a book by its cover. http://t.co/BabMf0W2YW,0 +7413,Drunk Meals 101: What To Cook When You're Totally Obliterated http://t.co/QvS7O10bG3,0 +7414,@Sweet2Young I came in! Had he fucking moved his entire existence would've been obliterated.,0 +7415,Drunk Meals 101: What To Cook When You're Totally Obliterated http://t.co/m19iVWrdkk,1 +7416,We destroyed the #Zimmerman fan club on Twitter @rzimmermanjr and we obliterated Renewsit reduced her to a sock acc. http://t.co/ybshleIE3p,0 +7418,@RockBottomRadFM As a kid I remember hearing rules about 30 to 90 days wo a defense and you're stripped. Brock obliterated that,0 +7419,So it looks like my @SoundCloud profile shall be no more! Nothing left to offer! DJ mixes obliterated ?? #byebyesoundcloud,0 +7421,God the @mets are so cocky right now and I love it. Uribe OBLITERATED that ball then strutted the fuck out of the batters box...,0 +7424,I was obliterated last night??,0 +7425,WACKOS like #MicheleBachman predict the WORLD will SOON be OBLITERATED by a burning firey INFERNO but cant accept #GlobalWarming!! HELLO!!!,1 +7426,Drunk Meals 101: What To Cook When You're Totally Obliterated http://t.co/Wj0U59mPpB,0 +7428,Uribe just obliterated a baseball.,0 +7430,@Silverhusky Shtap! Before your town is obliterated and the earth is salted T_T,0 +7431,I can't wait to be beyond obliterated this weekend,0 +7433,Obliterated my phone screen today with a drum stick. #blessed,0 +7434,Poway hcg diet- ensures mole obliterated whensoever nevermore comes above: sizYgwWF,0 +7437,@TheEvilOlives It's the closest structure to the hypo centre that wasn't completely obliterated.,1 +7438,Drunk Meals 101: What To Cook When You're Totally Obliterated http://t.co/JlPut7Va3s,0 +7439,Very glad that you got obliterated in X Men First Class. You fully deserved it after all @kevinbacon soz,0 +7442,& on the rare occasion I do go out I'm complete obliterated the next day. Throwing up and passing out. My body is accustomed to alcohol,0 +7443,Just absolutely obliterated a moth my new purchase is boss,0 +7444,I think I'll get obliterated tonight,0 +7446,Kontrolled Media say 'US good. Putin bad. #WW3 good Peace bad'. TOTAL BS! Leave Russia alone. DON'T DICE WITH OBLITERATION. #demonization,1 +7447,This is interesting!--Why did God order obliteration of ancient Canaanites? http://t.co/XqMJHIOZxG via @worldnetdaily,0 +7451,@LaurenJauregui I would say I'm dead but I'm not that right there was obliteration,0 +7452,Why did God order obliteration of ancient Canaanites? http://t.co/NckOgWjq61 via @worldnetdaily,0 +7453,OH. #TeamHennessy #NJ Obliteration @tprimo24 ROUND 1 Happy Birthday @djeddygnj Colombian Festival___ https://t.co/mRv54fiDfn,0 +7456,I need an arcade shooter fix but CTE is empty & only running obliteration. I'd even buy a CoD title if they weren't all overpriced on steam,0 +7457,@ThatSabineGirl you get me sis. this planet could do with a huge dose of obliteration.,1 +7458,Are people not concerned that after #SLAB's obliteration in Scotland #Labour UK is ripping itself apart over #Labourleadership contest?,0 +7459,@AuntieDote @RioSlade @Locke_Wiggins @akarb74 Not if one side is set on obliteration of govt and the other on fixing it. That's too far,0 +7461,Why did God order obliteration of ancient Canaanites? http://t.co/IkugGvByeI,0 +7463,@tiggr_ why only Squad Obliteration?,1 +7465,Why did God order obliteration of ancient Canaanites? http://t.co/NLk1DYD2tP,0 +7467,Back in 02 to 03 would never said that 50 would have ended ja like obliteration,0 +7470,@Eganator2000 There aren't many Obliteration servers but I always like to play when there are :D,0 +7473,What a win by Kerry. 7-16..... #obliteration,0 +7474,@ashberxo @mind_mischief the removal of all traces of something; obliteration.,0 +7475,Why did God order obliteration of ancient Canaanites? http://t.co/Sf2vwQvJYa,0 +7476,Why did God order obliteration of ancient Canaanites? http://t.co/Hz4lKFfC59 via @worldnetdaily#Homosexuality is the downfall of a society.,0 +7477,I added a video to a @YouTube playlist http://t.co/1vjAlJA1SX GTA 5 Funny Moments - 'OBLITERATION!' (GTA 5 Online Funny Moments),0 +7478,There are no four truths-of pain of desire that is the origin of pain of the obliteration of that desire of the pain to that obliteration.,0 +7479,Alhaji Putin is far from being a good person sha. At least I had front row seat to his complete obliteration of Ibeto cement a competitor.,0 +7480,Dead Space - Obliteration Imminent [2/2]: http://t.co/XJB0dCAaHf via @YouTube,0 +7481,Which is true to an extent. The obliteration of white privilege would reduce unfair favoritism.,0 +7482,Does Renovation Mean Obliteration? http://t.co/pQ3ipUgkuY #entrepreneur #management #leadership #smallbiz #startup #business,0 +7488,For maximum damage! Activate [BIG BAND MODE] for old-timey obliteration!,0 +7490,Why did God order obliteration of ancient Canaanites? http://t.co/pKKcdWjyg0 via @worldnetdaily,0 +7491,He is justifying why this quarrel would one day end in d obliteration of what remains as the state of Pakistan. https://t.co/z8Ij8KTkyk,0 +7493,Fear is the mind killer. Fear is the little-death that brings total obliteration. Bene Gesserit Litany Against Fear Dune @atgrannyshouse,0 +7494,"Path of Obliteration +Back From The Dead +Story by @KyleWappler @thisishavehope + + http://t.co/1PdNlsP8XW",1 +7496,SB57 [NEW] Deepwater Horizon Oil Spill distribution of funds from BP settlement road and bridge projects in Bal... http://t.co/dKpsrkG6pc,1 +7497,California oil spill might be larger than projected: http://t.co/xwxBYHTuzC http://t.co/wzeDxEFBlg,1 +7498,Refugio oil spill may have been costlier bigger than projected: A Plains All American Pipeline oil spill off ... http://t.co/yhmrEgAuxZ,1 +7501,LA Times: Refugio oil spill may have been costlier bigger than projected http://t.co/g37huJx6et,1 +7502,'California: Spring Oil Spill Estimate Grows ' by THE ASSOCIATED PRESS via NYT http://t.co/6Cx46E7QB7,1 +7503,Refugio oil spill may have been costlier bigger than projected http://t.co/41L8tqCAey,1 +7504,Refugio oil spill may have been costlier bigger than projected http://t.co/Rqu5Ub8PLF,1 +7505,Refugio oil spill may have been costlier bigger than projected http://t.co/OSJUrFDDkt,1 +7507,Refugio oil spill may have been costlier bigger than projected http://t.co/lzob8qOH1B,1 +7510,Refugio oil spill may have been costlier bigger than projected http://t.co/BIEYgUqpB1,1 +7512,Sydney Traffic HAZARD Oil spill - BANKSTOWN Stacey St at Wattle St #sydtraffic #trafficnetwork,1 +7513,Plains All American Pipeline company may have spilled 40% more crude oil than previously estimated #KSBYNews @lilitan http://t.co/PegibIqk2w,1 +7514,SYD traffic HAZARD Oil spill - BANKSTOWN Stacey St at Wattle St http://t.co/TZyHdBW9f5,1 +7515,"Watch our video of Wendell Berry speaking about the BP oil spill in the Gulf of Mexico. + +It__ªs the birthday... http://t.co/tN1aX1xMBB",1 +7517,National Briefing | West: California: Spring Oil Spill Estimate Grows: Documents released on Wednesday disclos... http://t.co/wBi7Laq18E,1 +7518,DTN Brazil: Refugio oil spill may have been costlier bigger than projected: A Plains All American Pipeline oi... http://t.co/pDOSrg8Cf7,1 +7519,Hannah: 'Hiroshima sounds like it could be a place in China. Isn't that where the oil spill was?',1 +7520,Refugio oil spill may have been costlier bigger than projected http://t.co/aP30psZkVx,1 +7521,Refugio oil spill may have been costlier bigger than projected (LA Times) http://t.co/TCSoLvwhXq,1 +7522,California: Spring Oil Spill Estimate Grows: Documents released on Wednesday disclosed that an oil sp... http://t.co/zqiHnHDWPV #OSI2016,1 +7523,Refugio oil spill may have been costlier bigger than projected http://t.co/gtHddzAvhg #LosAngelesTimes #latimes #news,1 +7525,National Briefing | West: California: Spring Oil Spill Estimate Grows: Documents released on Wednesday d... http://t.co/hTxAi05y7B (NYT),1 +7526,Plains All American Pipeline company may have spilled 40 percent more crude oil than previously estimated.... http://t.co/WEZjqC4Cf2,1 +7527,@Kinder_Morgan can'twon't tell @cityofkamloops how they'd respond to an oil spill. Trust them? See Sec 4.2 #Kamloops http://t.co/TA6N9sZyfP,1 +7528,news@@ Refugio oil spill may have been costlier bigger than projected http://t.co/jhpdSSVhvE,1 +7529,@TroySlaby22 slicker than an oil spill,0 +7530,Plains: Calif. Oil Spill Could be Larger than Estimated. http://t.co/CcvcTe3lCw,1 +7531,Refugio oil spill may have been costlier bigger than projected http://t.co/xcoLwUGFjg,1 +7532,Refugio oil spill may have been costlier bigger than projected http://t.co/FPaouLWU3N,1 +7533,@News@ Refugio oil spill may have been costlier bigger than projected http://t.co/SqoA1Wv4Um,1 +7534,Refugio oil spill may have been costlier bigger than projected http://t.co/GusrAmzp1s #news,1 +7535,Refugio oil spill may have been costlier bigger than projected http://t.co/7L6bHeXIXv | https://t.co/eMOSrMUvQa,1 +7536,News@ Refugio oil spill may have been costlier bigger than projected http://t.co/jhpdSSVhvE,1 +7537,Refugio oil spill may have been costlier bigger than projected http://t.co/EFCn9IVNfg A Plains All American Pipeline oil spill off the S___,1 +7540,Refugio oil spill may have been costlier bigger than projected http://t.co/zdtcw9Fsx1,1 +7542,LA Times: Refugio oil spill may have been costlier bigger than projected http://t.co/1ct0pUGZ69,1 +7543,Senator voices concerns over BP oil spill settlement http://t.co/mFSBWpj0Ce,1 +7544,'California: Spring Oil Spill Estimate Grows ' by THE ASSOCIATED PRESS via NYT http://t.co/zdpa4DBtsU,1 +7545,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/02ELqLOpFk,1 +7547,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disease in Edinburgh are to ...,1 +7548,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/8lCbDW7m2z,1 +7550,Legionnaires' Disease: What's Being Done to Stop Deadly Outbreak http://t.co/AjzY19Vepu,1 +7551,10th death confirmed in Legionnaires' outbreak in South Bronx; total cases reaches triple digits http://t.co/JtzefipdBo,1 +7552,More than 40 families affected by the fatal outbreak of Legionnaires' disease in Edinburgh are to sue two comp... http://t.co/vsoXioOy78,1 +7553,A Mad Catastrophe : The Outbreak of World War I and the Collapse of the... http://t.co/jGdlX4Faw8 http://t.co/IvwtYnoUjK,1 +7554,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/2Mwc9YWjZy,1 +7555,Wow-the name #LegionnairesDisease comes from an outbreak of pneumonia at the @AmericanLegion convention in Philly in 1976--29 died from it.,1 +7556,'Legionnaires' Disease: What's Being Done to Stop Deadly Outbreak' http://t.co/FftOKd0Vts #????_?????,1 +7557,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/81HVV3N3rS,1 +7558,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/2AO97o2a9D,1 +7559,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disease in Edinburgh are to ...,1 +7560,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/3sNyOOhseq,1 +7561,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/VBsj8tniv1,1 +7563,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/SO81Ab3a1x,1 +7565,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legio... http://t.co/uCBfgIBFOR #MuhamadJabal,1 +7566,DTN Italy: Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionna... http://t.co/RsV9ATj9vH,1 +7568,@SenateMajLdr let's try to do our best to prevent another outbreak of violence by talking to each other both the people and the politics,1 +7569,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/oJyW7jkUH5,1 +7570,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/rv9Dv6JOeW,1 +7571,New York City Outbreak: What Is Legionnaire's Disease? http://t.co/CXI82rFiFS,1 +7573,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/1H7zk6UYze,1 +7576,@DrAtomic420 where did you get that pic from where it shows that there is 2 trophies for Outbreak? Or did they photoshop it?,0 +7577,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/mNsy1QR7bq,1 +7578,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/kkdX8zMV4G,1 +7579,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/13W8CyukKZ,1 +7580,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/hKxYzhvmQE,1 +7581,8th person dies in NY Legionnaires' disease outbreak http://t.co/fJdM8QHYAI #SEBEE,1 +7582,Legionnaires' Disease: What's Being Done to Stop Deadly Outbreak: The worst-ever outbreak of Legio... http://t.co/0ubG9wFyge #losangeles,1 +7583,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/ZA4AXFJSVB,1 +7584,Families to sue over Legionnaires' disease outbreak in Edinburgh - BBC News http://t.co/rM6CNzkSzL,1 +7585,#News Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires'... http://t.co/zfYqSAwvrk,1 +7587,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak o... http://t.co/3hTJ2PypSg #News #check #follow,1 +7588,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/NQ77EfMF88,1 +7589,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/lfbdhTGQWG,1 +7590,An outbreak of Legionnaires' disease in New York has killed at least 8 people ___ now officials think they've fo... http://t.co/7evyeLW4LC,1 +7591,Families to sue over Legionnaires: More than 40 families affected by the fatal outbreak of Legionnaires' disea... http://t.co/Paje5mxN1z,1 +7593,An outbreak of Legionnaires' disease in New York has killed at least 8 people @BI_Video http://t.co/eRJ7YANjXm http://t.co/vbRPeuJANL,1 +7594,"#BREAKING 10th death confirmed in Legionnaires' outbreak in South Bronx; total cases reaches triple digits +http://t.co/TxrsWi0efg",1 +7595,Tube strike = absolute pandemonium,0 +7598,Pandemonium In Aba As Woman Delivers Baby Without Face (Photos).... http://t.co/lYXNjlxL8s http://t.co/CXYFqN3ue4,1 +7600,Element of Freedom: The Biggest Party of the Summer @ Mirage Saturday! Tickets at http://t.co/7hAnPcr5rK,0 +7601,@hashtagteaclub place out. It was ex Pars defender Andy Tod in full uniform. It was instant pandemonium. Utter mayhem ensued!,0 +7602,Pandemonium In Aba As Woman Delivers Baby Without Face (Photos) - http://t.co/c5u9qshhnb,1 +7603,@J3Lyon I'm going to put the FFVII ones out at the weekend so I think Pandemonium! (Don't forget the exclamation mark) would be midweek.,0 +7604,@BlizzHeroes Would love to see a Diablo map themed after places like Westmarch or Mt. Arreat or Pandemonium.,0 +7605,@minhazmerchant Govt should pass the bills in the Pandemonium. UPA used to do it why cant NDA?,0 +7607,I'll be at SFA very soon....#Pandemonium http://t.co/RW8b50xz9m,1 +7608,Proud of you @JoeGoodmanJr for watching the #CopaLibertadores and the Argentinean pandemonium... https://t.co/8tyGO0KiZz,0 +7609,Pandemonium In Aba As Woman Delivers Baby Without Face (Photos) http://t.co/wRqF6U55hh,0 +7610,#Pandemonium.iso psp http://t.co/HbpNFOAwII,0 +7611,Pandemonium use to be my fav cd ?? I had to get it http://t.co/6WhUgaeM3C,1 +7612,Pandemonium In Aba As Woman Delivers Baby Without Face (Photos) http://t.co/AcFI2rHz4N,0 +7613,Pandemonium In Aba As Woman Delivers Baby Without Face (Photos) - http://t.co/xRP0rTkFfJ,1 +7614,World Class Tgirl Ass 02 - Scene 4 - Pandemonium http://t.co/iwCu3DgI1a,0 +7616,Hey all take a look at my review of 'Pandemonium' & 'Requiem' by Lauren Oliver. Enjoy! http://t.co/cPLYReWFZ3,0 +7617,Element of Freedom at Mirage Saturday! 21+ Ladies Free Before Midnight!' http://t.co/7hAnPcr5rK,0 +7619,World Class Tgirl Ass 02 - Scene 4 - Pandemonium http://t.co/HzHoa6VZAS,0 +7621,@KhalidKKazi mate they've taken another 2 since I posted this tweet it's pandemonium,1 +7622,Pandemonium In Aba As Woman Delivers Baby Without Face (Photos) - http://t.co/dI5aRr6HQ6,0 +7623,>> @GidiExclusixe Shock In Aba As Woman Delivers __÷Faceless__ª Baby [Photo]: There was pandemonium... http://t.co/RGTYZbNKeo #BennyCapricon,0 +7626,Pandemonium In Aba As Woman Delivers Baby Without Face (Photos) - http://t.co/c5u9qsySeJ,0 +7627,@Dr_Baseball41 @GrantTamane8 @DrewWTaylor @Dtop77 On the Christie Hillside: Game 4: Pandemonium at the Pits. http://t.co/Lq4lXGS2xU,0 +7628,If she don't know bout that Pandemonium album #Shestooyoung,0 +7630,Pandemonium In Aba As Woman Delivers Baby Without Face (Photos) @... http://t.co/JbxBi93CLu,0 +7631,Exquisite asian in sto... http://t.co/Y9w0V6Te9O #cumshot #sex #porn #video #porno #free,0 +7633,Pandemonium In Aba As Woman Delivers Baby Without Face (Photos) - http://t.co/Ykdsp0nRDQ,0 +7634,My workplace is going to be pandemonium when the season starts ??,0 +7635,Pandemonium In Aba As Woman Delivers Baby Without Face (Photos) http://t.co/8j4rdwyjWu http://t.co/9MkZPZfKL2,0 +7638,- Pandemonium In Aba As Woman Delivers Baby Without Face http://t.co/36GccAPaak http://t.co/nqjZS6wkuN,0 +7639,Cyclists it is pandemonium on the roads today. Drive carefully!,0 +7640,@PBohanna Probably a dead boring 1st hour and a half after the pandemonium of the last Test..... #justaguess,0 +7641,Photo: deadgirltalking: unfortunemelody: jaylenejoybeligan: tarynel: dredougie: santanico-pandemonium:... http://t.co/PvKgo79JnI,0 +7642,Pandemonium In Aba As Woman Delivers Baby Without Faceå_(Photos) http://t.co/bM0SXzbNKE,1 +7643,Truly a scene of chaos unprecedented in frenzy. Pandemonium even. Utter disorder. That anyone survived such mania is astounding. @catovitch,1 +7644,http://t.co/PmHMmkSPaQ -PANDEMONIUM! Playstation One PS1 Retro Classic Original Platform Platinum Rare#Deals_UK http://t.co/0gKNpy4lUA,0 +7645,it's don't panic,0 +7647,Obligatory middle of the night panic attack,0 +7648,@brokenscnecal I just though about panic! at the disco when I said that,0 +7649,Drake spokes rep for the raptors.Beg to differ the other night when shots rang out at Musik wasn't a goodlook.Running in panic no fun,0 +7650,The cool kids asked me if I wanted to hang out after school so I had a panic attack and had to go to the hospital #autismawareness,0 +7651,Love it when Jesse suffers a panic attack. https://t.co/r4utnewlnA,0 +7652,@tim_micallef when do u think Yankees hit the panic button? #TroubleOnMyMind,0 +7653,@JetixRestored Here's Part 2 Of Teamo Supremo Pogo Panic! I Want You Make It Better! OK! :) https://t.co/wBLiMlMT2x via @YouTube,0 +7654,Is it time to panic now? https://t.co/OrxDQfz0J0,0 +7656,Plane Panic What kind of douchebag. Bubble Gum,0 +7657,'If plan A does not work don't panic bc there are 25 more letters.' I like this but feel like I may be running out of letters- thoughts?,0 +7659,I added a video to a @YouTube playlist http://t.co/QsEkoeuBMd Panic! At The Disco: Girls/Girls/Boys [OFFICIAL VIDEO],0 +7661,i just drove with both my parents in the car lmao that was a panic attack waiting to happen,0 +7662,@elielcruz just watching the streams was bad - I don't think I could actually walk in there I'd panic and get sick,1 +7663,we was all just laughing and talking junk now everyone in panic mode,0 +7668,The good thing is that the #Royals won't face a newbie in the playoffs. No real reason to panic.,0 +7669,Despite the crippling anxiety and overwhelming panic attacks I'd say I'm fairly well-adjusted.,0 +7670,"Panic over: Patient in Alabama tests negative for Ebola +http://t.co/cY0CiU2u1C",1 +7671,I added a video to a @YouTube playlist http://t.co/4914nJpIO3 Panic! At The Disco: Miss Jackson ft. LOLO [OFFICIAL VIDEO],0 +7672,Every time I feel a new pain or strain in my body I panic because I need it for my career ??,0 +7674,Panic! at the Disco is the best song by this is gospel (acoustic) http://t.co/VCq2icptKI,0 +7675,"Lose bus card. +Panic. +Kind bus driver. +Replace bus card. +Find bus card. +Headdesk.",0 +7677,Savs contact fell out but she was convinced it didnt & went on to guage her eyeball out & now shes having a panic attack over caused damage,0 +7678,"@james_justus *returns her* + +Relax. You know I always return her so no need to panic. I just gave her some of those corn flakes. :P",0 +7679,@montetjwitter11 @nolesfan05 @NutsAndBoltsSP I say concern but not panic. Too many games left. And some games left against each other.,0 +7680,@panic awesome thanks.,0 +7681,@montetjwitter11 @Mets @audreyp77 @teena_797 @darryl_brooks @EliteSportsNY @LopezandtheLion no panic from #NatsNation but concern for sure,0 +7683,@biggangVH1 looks like George was having a panic attack. LOL.,0 +7685,#dream #magic The #linden method lite version - #1 anxiety panic cure program: http://t.co/073izwX0lB THE LIND http://t.co/OkmLAGvkjv,0 +7686,Someone asked me about a monkey fist about 2 feet long with a panic snap like the one pictured to be used as a... http://t.co/Yi9BBbx3FE,0 +7687,tomorrow's going to be a year since I went to the Panic! concert dressed as afycso ryan do u guys remember that,1 +7688,Romantic dramatic but never panic original sensei write rhymes in the attic,0 +7689,I added a video to a @YouTube playlist http://t.co/vxeGCmMVBV Panic! At The Disco: Collar Full (Audio),0 +7691,just had a panic attack bc I don't have enough money for all the drugs and alcohol I wanted to buy this year let alone my fall bills,0 +7692,I had school today and I've already had a panic attack. Thank you high school for sucking !!!,0 +7693,Panic attacks are the worst ????,0 +7694,Panic at the disco te amo,0 +7695,Have to laughtraders panicking over market down 1%. Some see these as positive capitulation. THAT'S A JOKE. More to come may be ugly,0 +7696,@QuotesTTG Save the panicking for when you get to Helios. ;),0 +7698,@Dirk_NoMissSki yea but if someone faints why are they panicking?.. thats basic stuff ??,1 +7699,OKAY I CAN'T FIND IT SO I'M KINDA PANICKING,0 +7701,New post: 'People are finally panicking about cable TV' http://t.co/pkfV8lkSlD,0 +7702,I hear the mumbling i hear the cackling i got em scared shook panicking,0 +7703,You made my mood go from shitty af to panicking af istg,0 +7707,Okay NOW I AM PANICKING,0 +7711,idm if you burn the whole gotham city bcs I'm just gonna laugh at everyone while theyre panicking.,0 +7712,You certainly are @Ian_Bartlett! Great to see all you #HS2 astroturfers panicking at the thought of cross party support crumbling.,0 +7714,New post: 'People are finally panicking about cable TV' http://t.co/df9FjonVeP,0 +7715,it's 11:30 and I'm already panicking and crying bc I'm so stressed NICE,0 +7717,People are finally panicking about cable TV http://t.co/BBjLs1fsaD,0 +7718,@Adumbbb oh that's not too bad then haha. I would've been panicking tho.,0 +7719,@beauscoven nah man he's panicking. He just found out his brothers had it off with his now wife debbie is in hospital he's stressed,0 +7720,Panicking because maybe my bff left me at China Bar or maybe tindering on the toilet for 20mins,0 +7721,@ushiocomics I may be panicking a little I wasn't as fast submitting the form as I usually am,0 +7722,all that panicking made me tired ;__; i want to sleep in my bed,1 +7723,Yet Brits are panicking about the UK http://t.co/HsDBGCIYrs,0 +7724,?You should be scared. You should be screaming and panicking.? THIS REDEEMER http://t.co/4h8qYvvd0E #romanticsuspense,0 +7725,My dad is panicking as my weight loss means he needs to hurry up with my new clothes fundwhen I reach my goal. ??,0 +7727,Already panicking about school starting again :-),0 +7728,People are finally panicking about cable TV http://t.co/ranEFiHbUK http://t.co/MflRVBh4qA,0 +7731,I feel like I should be panicking more as Idk I get my results back in a week... I'm Alarmingly calm,0 +7733,Just realized that maybe it not normal to sit up front with an Uber driver? Panicking,0 +7735,@c_pinto001 I see people are panicking about Orpik all over again.,0 +7736,Ay I am fully panicking lol,0 +7737,When he lets you drive his truck and you start panicking because you had to 'flip that bitch'. ?????? http://t.co/W6O0uiZF8p,0 +7738,@jeannathomas not gonna lie.. I'm panicking a little bit. Vic/Hardy not there.. Freeman not practicing.,0 +7739,People are finally panicking about cable TV http://t.co/lCnW4EAD8v,0 +7742,you can stop panicking ?????? @ogtomd https://t.co/ZvRE6fFNyD,0 +7743,My sis can now sit on a cam w/o panicking https://t.co/GiYaaD7dcc,0 +7744,why are people panicking about results day though hahahah like worrying is going to change your results,0 +7746,America said it had a war on terrorism. Then police took my AR-15 when I was ready to start shooting the enemies!,1 +7747,Police: Gunman reported dead at Nashville area theater: A suspect who carried a gun and a hatchet at the Carmi... http://t.co/kqvN1uTpMM,1 +7748,#BREAKING411 4 police officers arrested for abusing children at police-run boot camp in San Luis Obispo Calif. - ... http://t.co/oNLvf2fyoY,1 +7749,These fucking police can't touch me these fuck niggas ain't fucking w me,0 +7750,#World #News Qld police wrap Billy Gordon investigation: QUEENSLAND Police have wrapped up their investigation... http://t.co/msgnNDxOeK,1 +7751,'Gunman who opened fire at Tennessee movie theater killed by police' Anyone suspect suicide-by-cop?,1 +7752,Police kill hatchet-wielding gunman who opened fire inside Nashville movie theater: Aå_middle-aged manå_armed wi... http://t.co/tyD47NfL5x,1 +7753,Police unions retard justice & drain gov $ but cops vote 4 Rs so Rs go after teachers unions instead! @DCCC @VJ44 @Lawrence @JBouie @mmfa,0 +7754,Maid charged with stealing Dh30000 from police officer sponsor http://t.co/y35qtVDSOH | https://t.co/qhUJAjCTR5,0 +7755,A Santa Ana Police Union Blocks Video of the Raid at Sky High Holistic Pot Dispensary - The Atlantic http://t.co/yvV1RlghfT #SmartNews,1 +7756,4 police officers arrested for abusing children at police-run boot camp in San Luis Obispo Calif. - @ABC7: http://t.co/QpWOtugUI9,1 +7758,Vinnie Jones goes on the beat with Northumbria Police http://t.co/UP30AQgnLf,0 +7759,Now Trending in Nigeria: Police charge traditional ruler others with informant__ªs murder http://t.co/93inFxzhX0,1 +7760,California police officer pulls gun on man without any apparent provocation (VIDEO) http://t.co/Lhw4vTbHZG via @dailykos,1 +7761,Police expand search for missing pregnant woman in Beloeil: Police in Richelieu-Saint-Laurent are expanding th... http://t.co/hMuyzmv8qH,0 +7763,Good that the police are taking care of this and also have extra security #HarryBeCareful,0 +7764,"Why Are #BAYONETS Being Distributed To Local Police Departments? +@RandPaul wants to know + +https://t.co/XB8nfxaBUM + +#EvilEmpire +#JadeHelm15",0 +7765,UNWANTED PERSON at 200 BLOCK OF SE 12TH AVE PORTLAND OR [Portland Police #PP15000266818] 17:10 #pdx911,0 +7766,Episcopal priests on road trip with interracial family shares harrowing story of police harassment http://t.co/RG4JIsHyBs via @dailykos,0 +7767,Oops: Bounty hunters try to raid Phoenix police chief's home: http://t.co/yPRJWMigHL -- A group of armed bounty... http://t.co/3RrKRCjYW7,1 +7768,$1 million bail for man accused of #shooting at Fife #police - Aug 5 @ 8:16 PM ET http://t.co/hu5CXqnoBf,1 +7769,DT @georgegalloway: RT @Galloway4Mayor: ___The CoL police can catch a pickpocket in Liverpool Stree... http://t.co/vXIn1gOq4Q,1 +7771,@BrandonMulcahy @fpine Here's the story http://t.co/TgXutUoyHl,0 +7772,Police kill gunman at Nashville movie theatre - Times of India: Times of India Police kill___ http://t.co/K8BExkgwr2,1 +7773,Oops: Bounty hunters try to raid Phoenix police chief's home http://t.co/u30n3fFX8Y,0 +7774,@robbiewilliams U fkn asswipe playing for Israeli child killers.. The fkn karma police will get U.,1 +7775,New technology designed to help prevent dangerous police chases: In an effort to reduce injuries Milwaukee police___ http://t.co/cedjdlPDAN,0 +7780,@ArizonaDOT Price Rd North bound closed from University to Rio Salado.. Lots of police.. What's crackin?,1 +7781,Selection September Bridgeport men charged in home invasion Police looking for burglar. - http://t.co/7mlCD0l0b8,1 +7783,Police walk up on me I be blowin smoke in dey face wanna lock me up cus I got dope shit is gay,0 +7784,Police: Man killed ex grandmother after son wasn&#8217;t named after him http://t.co/ndCy8Q7R6I OMG!!!!! Absolutely Crazy!!!,1 +7785,-=-0!!!!. Photo: LASTMA officials challenge police for driving against traffic in Lagos http://t.co/8VzsfTR1bG,1 +7787,#helpme what do I do? My friend has been ticketed by Police in Wayne County Michigan into never- sending poverty cycle. How do I help him?,0 +7788,3 Options To Fix Your Facebook Cover Violation http://t.co/pF8dXwIbDp and Keep the #Facebook Police away. #socialmedia #strategy,0 +7789,CNN: Tennessee movie theater shooting suspect killed by police http://t.co/dI8ElZsWNR,1 +7790,Oops: Bounty hunters try to raid Phoenix police chief's home: A group of armed bounty hunters surrounded the h... http://t.co/dGELJ8rYt9,1 +7793,.@slosheriff: 2 South Gate police officers and 2 Huntington Park officers arrested after child abuse investigation at boot camp,0 +7797,Reddit Will Now Quarantine___ http://t.co/pkUAMXw6pm #onlinecommunities #reddit #amageddon #freespeech #Business http://t.co/PAWvNJ4sAP,1 +7799,Reddit Will Now Quarantine Offensive Content http://t.co/u9BkQt6XHR,0 +7800,Reddit Will Now Quarantine Offensive Content http://t.co/8S0mTwRumQ #Technology #technews #puledo_tech_update,0 +7801,#Reddit updates #content #policy promises to quarantine __÷extremely offensive__ª communities http://t.co/EHGtZhKAn4,1 +7802,Reddit updates content policy promises to quarantine __÷extremely offensive__ª communities http://t.co/tHnExicGQe,0 +7803,RT skanndTyagi WIRED : Reddit will now quarantine offensive content http://t.co/H0xUNJ3C7C (http://t.co/UuEw4MJLes___ ___,0 +7804,Reddit Will Now Quarantine Offensive Content http://t.co/WosYPVQUFI http://t.co/XW8SDS1Tjp,1 +7807,Update: still haven't eaten since 6:30 am still have to clean the dry lot and fill water in quarantine. May pass out,0 +7809,Check Out: 'Reddit Will Now Quarantine Offensive Content' http://t.co/Ew5wZC07Fo,0 +7810,#Bangalore Reddit updates content policy promises to quarantine 'extremely offensive' communities http://t.co/Of3Q75fGeU #Startups #in,0 +7812,Reddit updates content policy promises to quarantine __÷extremely offensive__ª communities http://t.co/KmTwA3n1Gf,0 +7813,Aannnnd - 'Reddit Will Now Quarantine Offensive Content' https://t.co/P1JluRGWBu,0 +7815,Reddit Will Now Quarantine Offensive Content http://t.co/cNsHlNjUqX,0 +7816,Reddit Will Now Quarantine Offensiveå_Content http://t.co/VYgh2ni4Ah,1 +7817,hermancranston: WIRED : Reddit will now quarantine offensive content http://t.co/wvn6GrIyPq (http://t.co/Rei3PuWP84___ ___,0 +7818,Reddit Will Now Quarantine Offensive Content: Reddit co-founder and CEO Steve Huffman has unveiled more specif... http://t.co/LJMGdpDLvs,0 +7821,Reddit Will Now Quarantine Offensive Content: Reddit co-founder and CEO Steve Huffman has unveiled more specif... http://t.co/PMCp8cZPNd,0 +7822,#wired #business Reddit Will Now Quarantine Offensive Content http://t.co/ZhzVprZbgq,0 +7823,RT @WIRED: Reddit will now quarantine offensive content http://t.co/zlAGv1U5ZA,0 +7824,Reddit Will Now Quarantine Offensive Content http://t.co/FEIkC9FxED #onlinecommunities #reddit /via @wired,0 +7826,Wired: Reddit Will Now Quarantine Offensive Content - Reddit co-founder and CEO Steve Huffman has unveiled more sp... http://t.co/aByHRgsS1s,0 +7827,@missambear your Tweet was quoted by @WIRED http://t.co/E90J3vJOLc,0 +7828,Yet another company trying to censor the Internet. Reddit has started to quarantine their content: http://t.co/pG4y3I5ciu #cc,0 +7830,Reddit Will Now Quarantine Offensive Content: Reddit co-founder and CEO Steve Huffman has unveiled more specif... http://t.co/TDEUKJzzII,0 +7831,Reddit Will Now Quarantine Offensive___ http://t.co/wjWYJBncat #onlinecommunities #reddit #amageddon #freespeech http://t.co/0Erisq25KT,0 +7832,Reddit Will Now Quarantine Offensive Content: Reddit co-founder and CEO Steve Huffman has unveiled more specif... http://t.co/T7gE0j3CAy,0 +7833,Reddit Will Now Quarantine Offensive Content http://t.co/NAS3IPm5vh,0 +7835,Reddit Will Now Quarantine Offensive Content http://t.co/unNx71v8qc,0 +7837,Reddit Will Now Quarantine Offensiveå_Content http://t.co/Gllawb2FSk http://t.co/3kaAfuoztc,0 +7839,Reddit updates content policy promises to quarantine __÷extremely offensive__ª communities http://t.co/PLmIWOfpom,0 +7840,Loved Chicago so much that it game me Pink Eye. Now I sit and design in quarantine and in the dark.,0 +7842,Reddit Will Now Quarantine Offensive Content http://t.co/LOdOrmTfSq,0 +7843,Reddit Will Now Quarantine Offensiveå_Content https://t.co/MjbIUvbMo6 http://t.co/I5cdTD8ftj,0 +7844,Reddit Will Now Quarantine Offensive Content http://t.co/LTmgdP6Jaf,1 +7847,Top link: Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/u9ao3A4oGC,0 +7848,when you are quarantined to a little corner bc you are too sick to be in the office but work is piling up and they need you kinda day,0 +7849,Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/ohbV7YvtL5 http://t.co/YmuTi3ND9r,0 +7850,Top link: Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/o8XvTLP4mF,0 +7851,#hot Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/uiSNqIu3iF #prebreak #best,0 +7852,Officials: Alabama home quarantined over possible Ebola case http://t.co/UYUgFg3k1h,1 +7854,#hot Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/algtcN8baf #prebreak #best,0 +7855,Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/M4TcZaawpT,0 +7857,"Here we go again. EBOLA! Run for your lives! aaarrrgghhh! + +http://t.co/G2UIMBeKgE",1 +7860,Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/ZsXqbdUzBN http://t.co/6NCfjXPLOY,1 +7861,Fucking yes /r/antiPOZi is quarantined. Triggered the cucks we have.,0 +7862,#hot Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/nGKrZPza45 #prebreak #best,1 +7863,Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/lFW4KUukeM http://t.co/k3mnk9HnZ5,0 +7864,Are Users of this Sub to be Quarantined? http://t.co/9nLY2TovUD,0 +7865,Officials: Alabama home quarantined over possible ... http://t.co/Rb2s0jmleJ,1 +7867,@__ScrambledEggs calling it now: KIA gets banned or quarantined before month's end,0 +7868,"Officials say a quarantine is in place at a Birmingham home over a possible Ebola case. + +Edward Khan of the... http://t.co/qnIJ2p8Zv6",1 +7869,"gmtTy mhtw4fnet + +Officials: Alabama home quarantined over possible Ebola case - Washington Times",1 +7870,Top link: Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/BjVfk1ETe9,0 +7871,Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/4oNvxncz8w http://t.co/tnggXNm6k8,0 +7872,Top link: Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/zCp5cszSLl,0 +7873,Alabama home quarantined over possible Ebola case: Officials say a quarantine is in place at ... http://t.co/ztOnvgubVm #Bluehand #PJNET,1 +7874,Hm MT @Ebolatrends: Alabama Home Quarantined Over Possible Ebola Case http://t.co/ihVMtmZXne http://t.co/jLieMrSnnj,1 +7876,I'm just going to say it. Under the new reddit policy changes I think /r/conspiracy will be quarantined and how that's bad for the truth m___,0 +7877,okay the cat has been quarantined in my bathroom...its meowing really loud but I turned up the TV louder...things just might work out okay,0 +7878,Alabama firefighters quarantined after possible Ebola exposure http://t.co/hzpX6vAQPZ reports http://t.co/L4W0PCorbs,1 +7880,"0nPzp mhtw4fnet + +Officials: Alabama Home Quarantined Over Possible Ebola Case - ABC News",1 +7881,Alabama home quarantined over possible #Ebola case: An ambulance sat outside the University of Alabama at___ http://t.co/y2JT1aMyFJ,1 +7882,#hot Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/HqdCZzdmbN #prebreak #best,0 +7883,Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/gix1gaYnXZ http://t.co/P93S2rFhx6,0 +7885,Hey @reddit - the concept of a 'quarantine' makes no sense if the people you've quarantined can just wander out & about whenever they want,0 +7887,Ebola: Alabama Home Quarantined Over Possible Ebola Case.. Related Articles: http://t.co/BiigD1LEq3,1 +7888,"oc73x mhtw4fnet + +Officials: Alabama home quarantined over possible Ebola case - Washington Post",1 +7889,Top link: Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/0lpu0gR2j0,0 +7891,can't DL a patch to fix the error in symantec as it's quarantined the computer ergo stopped all wireless exchange. lol technology ??,0 +7893,Top link: Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/Cd2NG2Awql,0 +7894,#hot Reddit's new content policy goes into effect many horrible subreddits banned or quarantined http://t.co/VhrLsWvZql #prebreak #best,0 +7895,http://t.co/X5XUMtoEkE Nuclear Emergency Current USA radiation levels monitoring site,0 +7897,I liked a @YouTube video http://t.co/V57NUgmGKT US CANADA RADIATION UPDATE EMERGENCY FISHING CLOSURES,1 +7898,WIPP emergency activated because of slightly elevated levels of radiation. #sejorg,1 +7902,Some of worst radiation exposure from Fukushima meltdown happened 47km northwest-Proof that small emergency planning zones don__ªt cut it,1 +7903,DOE's WIPP facility in NM investigating a site filter radiation reading; has activated its Emergency Ops Center; says no offsite release.,0 +7905,Radioactive Box Quarantined - Israel__ªs Ashdod Port was evacuated when emergency teams discovered radiation emittin... http://t.co/swQ5lMyDka,1 +7906,Radiation emergency #preparedness starts with knowing to: get inside stay inside and stay tuned http://t.co/RFFPqBAz2F via @CDCgov,1 +7907,Who Else Wants Documents Radiation Emergency Procedures 06 24 09 #radiation #emergency #procedures #06 #24 #09 http://t.co/HxwFBpP1b3,0 +7908,"#CCOT #TCOT #radiation Nuclear Emergency Tracking Center +http://t.co/KF74o2mcsC +https://t.co/N2ZHrChCGV",0 +7909,Due to the rainstorm last night cupcake decorating is happening NOW at the Rec Hall! $2 - proceeds to #IWK! http://t.co/EaRONLwIFh,0 +7910,"Rainstorm downtown Gloucester +#gloucester #capeann #seagulls #triciaoneill #triciaoneillphoto https://t.co/oLS6qdi9Um",1 +7912,@myrtlegroggins <gasp!> I forgot Sunday! OMG,0 +7913,Wow what beauty lies within every storm. Taken today after a rainstorm at A&B Pipeliners http://t.co/pSt5bBQ0av,1 +7914,Rainstorm Destroys 600 Houses in Yobe State: Rainstorm Destroys 600 Houses in Yobe State. [Daily Trust] Damatu... http://t.co/rzxQSSun02,1 +7915,Going to the beach with Jim Alves means a guaranteed rainstorm. #lucky http://t.co/fejs0Bu0sq,0 +7916,#DnB #NewRelease EDGE Jimmy - Summer Rainstorm (Lapaka Sounds) http://t.co/4L8h2FKlNO via http://t.co/ZITQKDFXJY,0 +7918,'Three #people were #killed when a severe #rainstorm in the #Italian #Alps caused a #landslide' http://t.co/hAXJ6Go2ac,1 +7923,Landslide in Italian Alps kills three: ROME (Reuters) - Three people were killed when a severe rainstorm in th... http://t.co/uZwXJBG0Zh,1 +7924,Rainstorm Destroys 600 Houses In Yobe | iReporter https://t.co/0rNY349UnT via @sharethis,1 +7926,Stuck in a rainstorm? Stay toward the middle of the road. Most streets are crowned so water tends to pool on the sides.,0 +7929,major rainstorm happening! I'm gonna lie down and drift away to the storming around me for a little while. bebacksoon.,1 +7930,Landslide in Italian Alps kills three - ROME (Reuters) - Three people were killed when a severe rainstorm in the I... http://t.co/BoygBp0Jw9,1 +7932,If you can't have the roar of the waves a rainstorm & some rolling___ https://t.co/DlVYFvnQee,0 +7933,Robot_Rainstorm: We have two vacancies on the #Castle Fantasy Football team! Join us. It__ªs fun. ??,0 +7934,@Calum5SOS you look like you got caught in a rainstorm this is amazing and disgusting at the same time,0 +7935,Rainstorm Update: Further North or Further South? http://t.co/50vdQ7A1M5 http://t.co/QH6oXfT9Ir,1 +7936,Nigeria: Rainstorm Destroys 600 Houses in Yobe State: [Daily Trust] Damaturu -Over 600___ http://t.co/BBQnK76qUS,1 +7937,Landslide in Italian Alps kills three: ROME (Reuters) - Three people were killed when a severe rainstorm in th... http://t.co/SmKZnF52Za,1 +7938,"RAIN RAIN GO AWAY... A soaker is on the way +------> http://t.co/jQFcY9GuqV <----- http://t.co/tN65puhfhw",0 +7939,@NathanFillion Hardly! ??,0 +7940,Tomorrow's evening commute receives a RED LIGHT. A rainstorm impacting the region will push it's way in late evening. http://t.co/DRvm8ISOtE,1 +7941,That rainstorm didn't last nearly long enough,1 +7943,@Robot_Rainstorm I'm.sort of interested in what fonts they're using.,0 +7944,lizzie363 @CstSmith I drove thru deep water in a flash flood--rainstorm repair cost $14000. 25 yrs later people still thnk cars float!!!,1 +7945,Best believe all the wrong decisions are being made out here in these Memphis streets during this here rainstorm lol my folk doe,0 +7949,I want it to rainstorm PLEASE,0 +7950,Waking up sick with a rainstorm outside would usually make me sad. Not today though. Put some The Kooks on the stereo and let's do this.,0 +7952,Yay I can feel the wind gearing up for a rainstorm in #Vancouver!! Bring it on #drought #deadgrassandflowers #wildfires,1 +7953,@Robot_Rainstorm I'm a huge football fan as you know but I've never done fantasy leagues. Patience for a newbie?,0 +7954,'The way you move is like a full on rainstorm and I'm a house of cards',1 +7955,Rainstorm Destroys 600 Houses in Yobe State http://t.co/nU0D3uANNZ,1 +7957,The way you move is like a full on rainstorm and I'm a house of cards.,0 +7958,Landslide caused by severe rainstorm kills 3 in Italianå_Alps https://t.co/8BhvxX2Xl9 http://t.co/4ou8s82HxJ,1 +7959,Former Freedom Surf/Spa Razed to Make Way for New Homes http://t.co/EX6JzQJ3NI,0 +7960,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/AKnBtuyaef,1 +7962,TENSION IN ABIA AS INEC OFFICE__ªS RAZEDå____ GOVERNOR IKPEAZU PDP APGA REACT http://t.co/aKzZOe5CE6,1 +7963,The Latest: More homes razed by Northern California wildfire: The latest on wildfires burning in California and___ http://t.co/0Keh2TReNy,1 +7965,The Latest: More homes razed by Northern California wildfire - http://t.co/nTSwUAYEJI http://t.co/wgeFBuk4Jk,1 +7968,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/ZBZc8905Gl,1 +7969,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/lY8x7rqbwN,1 +7970,http://t.co/iXiYBAp8Qa The Latest: More homes razed by Northern California wildfire - Lynchburg News and Advance http://t.co/zEpzQYDby4,1 +7971,#News : The Latest: More Homes Razed by Northern California Wildfire - New York Times http://t.co/5kBRZZmf8c #TAFS #FB100%,1 +7972,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/d1VjOYg52A,1 +7973,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/dOFRh5YB01,1 +7976,The Latest: More homes razed by Northern California wildfire - http://t.co/2nIP3d15dx http://t.co/egYFNlAOQv,1 +7977,The Latest: More homes razed by Northern California wildfire - http://t.co/BsGR67dyWY http://t.co/nDGgr6Xyqd,1 +7978,The Latest: More homes razed by Northern California wildfire - http://t.co/5FcJVMl520 http://t.co/fvYRWhux8p,1 +7980,@petereallen @HuffPostUK @bbc5live How significant do you think the iconic pic of Church (maybe) ruins and less substantial efforts razed?,0 +7981,The Latest: More homes razed by Northern California wildfire - http://t.co/3tnuACIV3c http://t.co/SAkORGdqUL,1 +7982,The Latest: More homes razed by Northern California wildfire - http://t.co/P3g3bQBczu http://t.co/RpBxdfnx5k,1 +7983,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/x1xj0XVTj7,1 +7984,@DavidVonderhaar At least you were sincere ??,1 +7987,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/Rg9yaybOSA,1 +7988,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/2872J5d4HB,1 +7989,The Latest: More homes razed by Northern California wildfire - http://t.co/ecXMoiNZgU http://t.co/ntwWDweDNb,1 +7990,More homes razed by Northern Calif. wildfire http://t.co/u52RW9Ji2r #sandiego http://t.co/GX75w3q9Ye,1 +7991,George Njenga the hero saved his burning friend from a razing wildfire http://t.co/Ywkk26arAG,1 +7992,today will be another dualcom with @Im_Razed !!! if you enjoyed yesterdays check out todays at 5 pm easter!!!,0 +7994,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/bKsYymvIsg #GN,1 +7996,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/RlPTtkBG4W,1 +7999,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/6AcSWzo7cw,1 +8000,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/hCKxJ8eukt,1 +8001,The Latest: More homes razed by #NorthernCalifornia wildfire http://t.co/mONiJJth7V #ZippedNews http://t.co/0yXBB5dzw5,1 +8002,The Latest: More homes razed by Northern California wildfire - http://t.co/R1CNSjUAYQ http://t.co/DQ1yLcrF9K,1 +8003,George Njenga the hero saved his burning friend from a razing wildfire... http://t.co/us8r6Qsn0p,1 +8005,The Latest: More homes razed by Northern California wildfire - http://t.co/IrqUjaEsck http://t.co/qDwEknRMi9,1 +8006,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/aueZxZA5ak,1 +8008,Vassalboro house to be razed to make way for public gazebo @PeteL_McGuire reports. http://t.co/uUEwccdTow,0 +8009,This just-married Turkish couple gave 4000 Syrian refugees an incredible gift http://t.co/ibeD3xG7fy Fed them instead of family & friends.,1 +8010,Police Monitoring 200 Jihadis in Poland http://t.co/1wCOfmLUb9 via @freedomoutpost,1 +8012,Turkish newlyweds donate wedding money - what a beautiful gesture! Still have faith in humanity. http://t.co/o1eNHjrkJd,0 +8013,This just-married Turkish couple gave 4000 Syrian refugees an incredible gift #WashingtonPost http://t.co/aRKmc7vclN,1 +8015,Repulsive! Refugees-Victimiser-#Dutton Evangelical-Liar-#Abbott c/o #LNP on a dupe the press overdrive; #CHOPPERGATE!#BRONWYNBISHOP!#AUSPOL,0 +8016,#retweet Cameroon repatriating about 12000 Nigerian refugees due to Boko Haram http://t.co/wvVgmejA7l,1 +8017,Newlyweds feed Syrian refugees at their wedding - CBC News - Latest Canada World Entertainment and Business News http://t.co/QU8S89pVVt,0 +8018,wowo--=== 12000 Nigerian refugees repatriated from Cameroon,1 +8019,./.....hmm 12000 Nigerian refugees repatriated from Cameroon http://t.co/YTW9SlWvmg /(,1 +8020,...//..// whao.. 12000 Nigerian refugees repatriated from Cameroon http://t.co/baE0Ap4G9Y,1 +8021,'imagine an entire aisle dedicated to making people look like serbian refugees.' - director of whole foods clothing section,0 +8023,@ViralSpell: 'Couple spend wedding day feeding 4000 Syrian refugees. http://t.___ http://t.co/I1VPkQ9yAg see more http://t.co/tY5GAvn7uk,0 +8024,reaad/ plsss 12000 Nigerian refugees repatriated from Cameroon,1 +8025,Top story: @ViralSpell: 'Couple spend wedding day feeding 4000 Syrian refugees___ http://t.co/a2TIIVNjDY see more http://t.co/fW2XIfJ6Ec,0 +8026,Refugee Connections Indiegogo campaign will be going live tomorrow! Support us and help launch the only online community for #refugees.,1 +8028,y'all read 12000 Nigerian refugees repatriated from Cameroon http://t.co/aVwE1LBvhn,1 +8029,Short of throwing them overboard himself I don't think any other leader could do much worse by #refugees than @TonyAbbottMHR worst pm ever!,1 +8030,Cameroon Repatriated 12000 Nigerian Refugees http://t.co/6nQRU2q5Tz,1 +8031,Captain Abbott must go down with LNP boat #refugees #christianvalues https://t.co/Kp5dpOaF58,0 +8032,A Turkish couple spent their wedding day feeding 4000 Syrian refugees ? A Turkish couple who got marrie ___ http://t.co/iGll3ph6O1,0 +8034,How One Couple Is Using Drones to Save Refugees at the World's Deadliest Border http://t.co/9qpG0Z3Rh9,1 +8035,...//..// whao.. 12000 Nigerian refugees repatriated from Cameroon http://t.co/HuhWPmryWz,1 +8036,./.....hmm 12000 Nigerian refugees repatriated from Cameroon http://t.co/96p3hUJNTj /(,1 +8037,#Turkish couple decided to feed 4000 #Syrian #refugees as part of their #wedding celebrations http://t.co/EHLq3ZSPTd http://t.co/DjX5eLbrv1,0 +8038,"@fadelurker @dalinthanelan < right now. + +Even after two years there were still refugees camped just south of Redcliffe village and Aidan >",1 +8039,Newlyweds feed thousands of Syrian refugees instead of hosting a banquet wedding dinner http://t.co/EGcv7ybjae #Age #news,0 +8040,In Photos: The Young Refugees Forced to Flee Burundi's Violence | VICE News https://t.co/jOjnq2oOPi see more http://t.co/DsKuI6Mmgl,1 +8043,@KristinDavis @UN @Refugees Thank you @UN and @Refugees for helping so many people in need all over the world.... https://t.co/yPvJgzqqqB ___,0 +8044,wowo--=== 12000 Nigerian refugees repatriated from Cameroon,0 +8045,Newlyweds feed thousands of Syrian refugees instead of hosting a banquet wedding dinner - http://t.co/XZV0lT9ZZk via @smh,1 +8047,http://t.co/eHKLp12yiP Paci?c Media Centre | articles: AUSTRALIA: RSF protests over new security gag over reporting on...,0 +8049,recap/ 12000 Nigerian refugees repatriated from Cameroon http://t.co/po19h8YCND,1 +8055,The Most Generous Bride on Earth: Couple Feeds 4000 Syrian Refugees on Their Wedding Day http://t.co/ms8e8mNddb via @thedailybeast love it!,0 +8056,...//..// whao.. 12000 Nigerian refugees repatriated from Cameroon http://t.co/po19h8YCND,1 +8057,The 46 returned refugees - what were they fleeing from & how will the Vietnamese Govt treat them now they are returned? #Dutton #presser,1 +8058,CHPSRE: RT: Refugees: For our followers in Paris do visit 'A Dream of Humanity' exhibition by rezaphotography ___ ___ http://t.co/RPmTROPsVr,1 +8060,Policyholders object to Clico rescue plan http://t.co/E4DvI9vUXZ http://t.co/JyCpf8iYhg,1 +8061,Don't Panik! #KelbyTomlinson to the rescue! http://t.co/hujvgsFLUs,0 +8062,Coastal German Shepherd Rescue OC shared a link... http://t.co/P85NwcMkQu #animalrescue | https://t.co/wUDlkq7ncx,0 +8064,UD: Rescue (Structural Collapse) - Scott Road @ Ypres Road York (14 Trucks),1 +8065,Rescue of the day: It__ªs World Cat Day Saturday Aug 8th Black Black/White Cats just $5 http://t.co/oqb7DaSMVy #NewBeginningsAnimalRescue,0 +8066,I'm on 2 blood pressure meds and it's still probably through the roof! Long before the #PPact story broke I was involved in animal rescue,0 +8067,Coastal German Shepherd Rescue OC shared a link... http://t.co/35QWnGLkOS #animalrescue | https://t.co/Is2iDC3UBJ,0 +8068,Coastal German Shepherd Rescue OC shared a link: 'Ecstatic Rescued Racco... http://t.co/t8Q6DzVgwX #animalrescue,0 +8069,Suggs & Vivian to the rescue! #psychrewatch,0 +8070,Any lengths to rescue a cat in need! http://t.co/AMroX4Y4Nx,0 +8071,Last Chance Animal Rescue has 3 new posts. http://t.co/f1tcbg1MKi #animalrescue | https://t.co/Is2iDC3UBJ,0 +8073,Last Chance Animal Rescue has 3 new posts. http://t.co/kIILdu8GpO #animalrescue | https://t.co/wUDlkq7ncx,0 +8074,Sammy and todd always to the rescue may not be in uniform but still to the rescue lmao. Forever KFC fam.,0 +8079,"Lunch for the crew is made. Night night it's been a long day! +~Peace~Love~Rescue~",0 +8081,Tell the United Nations: Plantations are NOT forests! https://t.co/cic7h64Qv8 via @RainforestResq,0 +8083,Mary coming to Troy rescue. ?????? https://t.co/rosVXQeLQj,0 +8084,UD: Rescue (Structural Collapse) - Scott Road @ Ypres Road York (14 Trucks),1 +8085,Flood-zone : General Raheel Sharif visits Chitral: He also lauded the efforts of FWO army troops and army aviation in rescue opera...,1 +8087,Last Chance Animal Rescue has 3 new posts. http://t.co/1EB2DaUYfn #animalrescue,0 +8088,@LisaVanderpump How many dogs do you have and are they all rescue dogs?,0 +8089,Officials rescue 367 migrants off Libya; 25 bodies found - Fox News http://t.co/cEdCUgEuWs #News,1 +8091,Coastal German Shepherd Rescue OC shared a link... http://t.co/2JxkmkpalP #animalrescue | https://t.co/ec46LyQQc6,0 +8094,@Zak_Bagans this is Sabrina my dad rescued her from some dude who kept her in a cage. We've had her since I was 4 http://t.co/1k2PhQcuW8,1 +8095,Britons rescued amid Himalaya floods (http://t.co/WGRXLy9pDO) http://t.co/BJ4hAAVAYE http://t.co/59p3AoIQUS,1 +8096,4 kidnapped ladies rescued by police in Enugu | Nigerian Tribune http://t.co/xYyEV89WIz,1 +8097,Rescued Med migrants arrive in Sicily http://t.co/Z8xIqNgulc,1 +8098,@TennoAtax I hand you a glass of water and sit down. 'I was rescued by a Tenno. I've lived in my ship around the dojo for about a year...',0 +8100,'Trust us to get rescued by the dopey ones!' Val is hilarious shame she's probably going to die #emmerdale,0 +8101,Summer #summervibes #california #puppy #pitmix #rescued #brixton #banksy #happy #mybabies https://t.co/7VoVkTXsPo,0 +8102,Man Who Buried Dog Alive Thought No One Would Find Her But She Was Rescued Just In Time http://t.co/SahQ5UOAHW,0 +8103,10-Month-Old Baby Girl was Rescued by Coastguard after She Floated HALF A MILE Out to Sea! #socialnews http://t.co/kJUzJC6iGD,1 +8105,But now #Skyrim awaits to be rescued...again.,0 +8106,Rescued Med migrants arrive in Sicily http://t.co/nekm1RPohU,1 +8109,The Finnish hip hop pioneer Paleface will be 'rescued from a drifting raft' and brought into the container later #enkelbiljett #menolippu,0 +8110,18 bovines rescued 3 smugglerså_nabbed http://t.co/E7fn5G5rUu,1 +8111,Funds Needed for Rescued then Abandoned Cocker Spaniels http://t.co/RJrmW7nzy5,0 +8112,I liked a @YouTube video http://t.co/45TWHJ0l6m RomanAtwoodVlogs | RESCUED SICK KITTENS!!,0 +8113,#news Britons rescued amid Himalaya floods http://t.co/kEPznhXHXd,1 +8116,A brave little dog gets rescued from the river. His recovery will inspi... https://t.co/jYwFyGcLHM via @YouTube,0 +8118,Homeless Man Traveling Across USA With 11 Stray Dogs He Rescued Gets A Helping Hand From Strangers http://t.co/QhfqlUI6RY via @Reshareworthy,0 +8119,@BrittanyPetko breaking news tonight kids were rescued from play room after a week with no food or water do to parents sex life haha,1 +8120,'You can only be rescued from where you actually are and not from where you pretend you are.' Giorgio Hiatt,0 +8121,We rescued my dog at least 9 years ago ?? she's old but still sweet as ever ?? @Zak_Bagans http://t.co/yhYY3o609U,0 +8122,http://t.co/QGyN2u1UP3 Rescued Med migrants arrive in Sicily: Hundreds of migrants rescued___ http://t.co/wiS3H9Tqrm,1 +8125,Why weren't they taken back to Africa once rescued? #c4news,1 +8126,Migrants Rescued After Boat Capsizes Off Libya http://t.co/pmGgavtokP,1 +8128,Young children among those rescued from capsized boat off Libya http://t.co/Kot9zVD2H7 via @IrishTimesWorld,1 +8129,Heroes! A Springer Spaniel & her dog dad rescued a stranded baby dolphin: http://t.co/G03DkPooNP http://t.co/Go0HPi0B4c,1 +8130,Trust us to get rescued by the dopey ones. #Emmerdale #SummerFate,0 +8131,Three beached whales rescued in Kerry - http://t.co/rQbsPUCjDF,1 +8133,Rescued TB goes on to ribbon at HITS: http://t.co/pA5SSLeFEC via @offtrackhorse,0 +8135,Turn back to me! I have rescued you and swept away your sins as though they were clouds. Isaiah 44:22 Contemporary English Version,0 +8136,@Zak_Bagans pets r like part of the family. I love animals.??? The last 2 pets I had I rescued! Breaks my heart when animals are mistreated????,0 +8137,I liked a @YouTube video http://t.co/FNpDJwVw1j Homeless dog living in a cardboard box gets rescued & has a heartwarming,0 +8138,Angry Mistrustful Rescued Elephant Finds Peace and Friendship in Her New Home (PHOTOS) http://t.co/VaUnPS6WJa via @OneGreenPlanet,0 +8140,Photo: Hundreds of rescued migrants await disembarkment from an Irish naval vessel at Palermo Italy - @tconnellyR___ https://t.co/AIM5CYHL0y,1 +8141,Rescued Med migrants arrive in Sicily http://t.co/p4dOA5YYJe,1 +8142,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in ... http://t.co/bS6PjT09Tc #Africa #News,1 +8143,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the Mediterran... http://t.co/s5NuEGSwYj,1 +8144,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the Mediterran... http://t.co/ciwwUQthin,1 +8146,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the Mediterran... http://t.co/GEU4H46CsZ,1 +8147,VIDEO: 'We're picking up bodies from water' - Rescuers are searching for hundreds of migrants in the Mediterranean... http://t.co/yhQU5UV6Ok,1 +8149,#world Fears over missing migrants in Med: Rescuers search for survivors after a boat carrying as many a... http://t.co/6DS67XAI5e #news,1 +8150,@Durban_Knight Rescuers are searching for hundreds of migrants in the Mediterranean after a boat carr... http://t.co/cWCVBuBs01 @Nosy_Be,1 +8154,I have an unexplainable desire to watch The Rescuers. #childhooddefined,0 +8155,#RoddyPiperAutos Fears over missing migrants in Med: Rescuers search for survivors after a boat carrying as ma... http://t.co/97B8AVgEWU,1 +8156,@AndyGilder Channel 5 have been doing the same for the RSPCA with Dog Rescuers.....'The Infomercial' has crept in the back door.,0 +8158,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the Mediterranean after a boat carryi...,1 +8159,Woman__ªs GPS app guides rescuers to injured biker in Marin County http://t.co/UoJy4E2Sv4,1 +8161,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the Mediterran... http://t.co/zG1YddywA5,1 +8162,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the Mediterran... http://t.co/PUezv6bd37,1 +8163,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the Mediterranean after a boat carryi...,1 +8164,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in ... http://t.co/awtScUCBBV #Africa #News,1 +8165,Last Second Ebay Bid RT? http://t.co/oEKUcq4ZL0 Shaolin Rescuers (dvd 2010) Shen Chan Nan Chiang Five Venoms Kung Fu ?Please Favori,0 +8168,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the Mediterranean after a boat carryi...,1 +8169,"#WorldNews + VIDEO: 'We're picking up bodies from water' - BBC News - Home: +Rescuers are searching for hundreds of.. http://t.co/6hhmBdK9Yo",1 +8170,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the Mediterran... http://t.co/ZFWMjh6SLh,1 +8171,http://t.co/XlFi7ovhFJ VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds___ http://t.co/rAq4ZpdvKe,1 +8172,AH-Mazing story of the power animal rescuers have! A starving homeless dog with no future was rescued by a person... http://t.co/ficd5qbqwl,1 +8175,Rescuers are searching for hundreds of migrants in the Mediterranean after a boat carrying as many as 600 people capsized off the coast of___,1 +8176,#News: 'Many deaths' in shipwreck: Rescuers are trying to save hundreds of migrants after the... http://t.co/tX51oYbrN6 via @TheNewsHype,1 +8177,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the Mediterran... http://t.co/PsPm3ahGKQ,1 +8180,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the Mediterran... http://t.co/yvO6q6W442,1 +8181,When Rescuers Found Him He Was Barely Alive. But They Had No Idea He Was Allergic To THIS! http://t.co/VecmsiUUh1,1 +8182,Fears over missing migrants in Med: Rescuers search for survivors after a boat carrying as many as 600 migrant... http://t.co/gx9sKUAu9J,1 +8183,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the Mediterranean after a boat carryi...,1 +8184,Woman's GPS app guides rescuers to injured biker in Marin County - SFGate http://t.co/Iz9U8BhfAA,1 +8187,Fears over missing migrants in Med: Rescuers search for survivors after a boat carrying as many as 600 migrants___ http://t.co/v9ftYB30EI,1 +8188,Rescuers find survivors of Nepal earthquake buried ... http://t.co/I8SJ1KWs1D,1 +8189,"Rescuers recover body of 37-year-old Calgary man from lake near Vulcan + + ___ http://t.co/gAEhr9bHEk",1 +8190,Fears over missing migrants in Med: Rescuers search for survivors after a boat carrying as many as 600 migrants___ http://t.co/IXfnE5Jlep,1 +8191,VIDEO: 'We're picking up bodies from water': Rescuers are searching for hundreds of migrants in the... http://t.co/UymxocFs33 #BBC #News,1 +8195,To All The Meat-Loving Feminists Of The World Riot Grill Has Arrived: Pop quiz! Which do you prefer: fem... http://t.co/KmndkFa7me #art,0 +8196,Stuart Broad Takes Eight Before Joe Root Runs Riot Against Aussies,0 +8201,@PipRhys I predict a riot.,1 +8202,"@teamVODG Discovered by @NickCannon + Listen/Buy @realmandyrain #RIOT on @iTunesMusic @iTunes https://t.co/dehMym5lpk ___ #BlowMandyUp",0 +8203,To All The Meat-Loving Feminists Of The World Riot Grill Has Arrived http://t.co/um3wTL5r7K #arts http://t.co/2LQyxZQ5DN,0 +8204,http://t.co/cxB55H37jn Rascal Flatts Riot Tour Atlantic City Beach Concert-August 20 2015-2 Tickets http://t.co/H6tyYSGR30,0 +8205,To All The Meat-Loving Feminists Of The World Riot Grill Haså_Arrived http://t.co/uDQA53KfQu,0 +8208,Seriously do we have to do a tactical riot against the headquarters of Disney and Marvel...,0 +8209,@Dani_Riot keep an eye out we'll be looking for lots of new team members in coming days/weeks :),0 +8210,To All The Meat-Loving Feminists Of The World Riot Grill Has Arrived: Pop quiz! Which do you prefer: feminist... http://t.co/HXOX7o42Rq,0 +8211,@Live_Workshop selfie at booth or riot Kappa,1 +8212,@AcaciaPenn I'll start a big ass riot send me to jail today mfs shidddd ??,0 +8213,"Riot Kit Bah - part of the new concept Gear coming for Autumn/Winter +#menswear #fashion #urbanfashion___ https://t.co/cCwzDTFbUS",0 +8214,"@abran_caballero Discovered by @NickCannon + Listen/Buy @realmandyrain #RIOT on @iTunesMusic @iTunes https://t.co/dehMym5lpk ___ #BlowMandyUp",0 +8215,I liked a @YouTube video http://t.co/5fR41TPzte Thorin's Thoughts - Riot and Sandbox Mode (LoL),0 +8216,Stuart Broad Takes Eight Before Joe Root Runs Riot Against Aussies: Stuart Broad took career-best figures of 8... http://t.co/zGSJWXdrCM,0 +8217,"Retweeted Sarah Silverman (@SarahKSilverman): + +Soundtrack of my walk starts w Tracey Ullman They Don't Know. perfect",0 +8218,To All The Meat-Loving Feminists Of The World Riot Grill Has Arrived http://t.co/TiOst8oKvX,1 +8219,After all that time Riot should really make an official Satan Teemo skin http://t.co/TYtPBC4GWi,0 +8221,I liked a @YouTube video http://t.co/lAmsdzKCuz Sick Riot Shield Slide Spots!!,0 +8223,@ByTorrecilla Torrecilla We Always info the Secret on LoL Game Check the Secret to get 600.000 Riot Points on my Bio,0 +8224,Transwomen and drag queens of color lead the riot a well known butch lesbian is usually credited for inciting it.,0 +8225,@eac4AU You can now PRE-ORDER the film on ITUNES & watch 9/15!! YAY! http://t.co/fVP3Wnid4L http://t.co/bwdhIBtiKs http://t.co/qelROcI7by,0 +8226,'Without an ally near you can't use this skill.' How did you get hired? Really cause it's making every Riot staff member look incompetent.,0 +8231,Pop quiz! Which do you prefer: feminist revolution or fried ravioliå_with porcini and ricotta___ http://t.co/n6MCPgVWQ2 http://t.co/s8OiNfGXyX,0 +8232,To All The Meat-Loving Feminists Of The World Riot Grill Has Arrived http://t.co/SkAAUSjpO4 OliviaMiles01,0 +8234,@DavidJordan88 @Stephanenny Except we don't know who started the riot or if it even makes sense to credit any particular individuals...,1 +8235,@Trollkrattos Juan Carlos Salvador The Secret Tips to Get 100.000 Riot Points LoL are out now! check the Secret on on my Bio,0 +8236,Southeast Dirt Riot Series Crowns Champions: Southeast Dirt Riot Series Crowns ChampionsBLACKFOOT ID: The So... http://t.co/v9i4PfXO0C,0 +8237,Sweetpea's are running riot at the allotment - and brightening up a rainy day http://t.co/6NdBFOPK5m,1 +8238,Stuart Broad Takes Eight Before Joe Root Runs Riot Against Aussies,0 +8239,Riot police intervene after Southampton and Vitesse Arnhem supporters clash: __¢ Fans clash in buildup to second... http://t.co/sKVNmtZGeG,1 +8240,"@JWalkerLyle Discovered by @NickCannon + Listen/Buy @realmandyrain #RIOT on @iTunesMusic @iTunes https://t.co/dehMym5lpk ___ #BlowMandyUp",0 +8241,? Cracker - White Riot ? http://t.co/Cc7D0wxk0M #nowplaying,0 +8242,But the government will not care. Police will stop rioting eventually of protestors. Eventually some skyscrapers become plant-covered .,1 +8243,New doco tonight at 9pm Setanta Sports Ireland freeview. The largest police presence at a soccer game in Ireland stop prevent the rioting,0 +8244,Eric Clapton shot the sheriff in 1974 and guess how many people were rioting? Right none.,1 +8245,if they kill off Val I'm rioting #Emmerdale,0 +8246,'Money can't buy happiness' is just a lie we tell poor people to keep them from rioting.,1 +8247,Bloody hell it's already been upgraded to 'rioting'. #hyperbole #saintsfc,1 +8250,Twitter is doing a count down to the rioting and financial collapse of Brazil #Rio2016 http://t.co/9mtrq5Jf4d,1 +8251,if that would of been a black dude Antioch would be rioting,0 +8252,".@runjewels recently met with the @BBC to discuss race relations in America & the benefits of rioting. #LoveIsLove +https://t.co/6Ce1vwOVHs",0 +8253,@Reuters people like you should be charged after the inevitable rioting for contributing to it with your lies,0 +8254,http://t.co/wspuXOrEWb Cindy Noonan@CindyNoonan-Heartbreak in #Baltimore #Rioting #YAHIstorical #UndergroundRailraod,1 +8256,Leeds fan.... rioting in Embra at a lower tier/ league cup final ... dee dum,1 +8257,Still rioting in a couple of hours left until I have to be up for class.,1 +8258,@CloydRivers there were plenty of black people rioting when tOSU won the championship as well.,0 +8259,"@evacide The Ferguson RIOTS worked. This of this the next time you say RIOTING doesn't change anything. + +There fixed it for you.",1 +8261,`bbcnews The Ass. of British Insurers says rioting will cost insurers &#163;163;millions. But police numbers are reduced by blind fat contr,1 +8262,#BHRAMABULL Watch Run The Jewels Use Facts to Defend Rioting in Ferguson: The socially minded duo takes on the... http://t.co/Ld5P1sIa2N,1 +8264,I think Twitter was invented to keep us insomniacs from rioting in the wee small hours.,0 +8266,@BLutz10 People aren't rioting because justice has been served and that murderer is behind bars. Simple! No justice=no peace Justice=peace.,1 +8267,Still rioting in to Gmail...,0 +8270,@jasalhad @brianboru67 @Jimskiv92 @hijinks1967 Rioting.,1 +8271,@RyleeDowns02 @nevaehburton33 if I don't get my money by tomorrow rioting ??,0 +8274,RT : Why Sweden Isn't Venezuela: There have been a few days of rioting in Venezuela with the riots directed at gr___ http://t.co/GJfd85vuf2,1 +8276,@davidolszak or the rioting in happy valley after penn state loses?,0 +8278,@aelinrhee a group of mascara smeared girls rioting will be horrific I think,1 +8279,The last time a high profile name was due to be signing for #nffc the City was rioting! Wesley Verhoek now a household name! #don'tpanic,1 +8280,@Georgous__ what alternatives? Legal alternatives? Protesting? Rioting may not be the most peaceful thing but it's a demonstration of how,0 +8283,@halljh1720. I am so sick of criminals parents and friends rioting when they are killed by police. You don't give a Damn when officer is.,1 +8284,@BLutz10 But the rioting began prior to the decision for the indictment so you're not really making sense at this point___,1 +8285,The Ashes Test match is currently more interesting than stupid Congress rioting...,1 +8286,"people rioting everywhere and I think I'd be one of them.' + +Usami-san <3",1 +8287,Football hooligan jailed for rioting before game in Scotland was already banned from matches in England #UkNews http://t.co/q5mp2Q6Hy8,1 +8288,http://t.co/jMzcaqyDfa Cindy Noonan@CindyNoonan-Heartbreak in #Baltimore #Rioting #YAHIstorical #UndergroundRailraod,0 +8289,AM `bbcnews The Ass British Insurers says rioting will cost insurers163;millions. But police numbers are reduced by blind fat controllers.,1 +8291,@fa07af174a71408 I have lived & my family have lived in countries where looters were shot on sight where rioting wasn't tolerated. Why here,1 +8292,'Refuse to let my life be reduced to rubble. When the shit keeps piling up get a shovel.' @ExpireHC,0 +8294,China's Stock Market Crash: Are There Gems In The Rubble?: China__ªs stock market crash this su... http://t.co/KABK3tcJNL ... via @Forbes,0 +8295,'If you go on with this nuclear arms race all you are going to do is make the rubble bounce.' ? Winston Churchill,0 +8296,@accionempresa China__ªs stock market crash this summer has sparked interest from bargain hunt... http://t.co/s0Eyq1wEHE @gerenciatodos å¨,1 +8297,@accionempresa China__ªs stock market crash this summer has sparked interest from bargain hunt... http://t.co/gO0pkrFzMF @gerenciatodos å¨,0 +8298,My parents are so impulsive sometimes. I remember coming home to my room filled with dust & rubble just because they wanted it redesigned. ??,0 +8300,"Turning rubble from disasters into 'Lego' bricks you can build houses with. This one belongs in #crazyideascollege +http://t.co/dh0s4bUuK7",0 +8301,#360WiseNews : China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/eaTFro3d5x,0 +8304,[FORBES]: China's Stock Market Crash: Are There Gems In The Rubble?: China__ªs stock market crash this summer ha... http://t.co/Q4grDpAjr5,0 +8309,#forbes #europe China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/C0SlAbBP7j,1 +8310,China__ªs stock market crash this summer has sparked interest from bargain hunters and bulls betting on a rebound. D___ http://t.co/1yggZziZ9o,0 +8311,@JasonPope2 @JohnFugelsang again I didn't say it was. I was referring to the main 2 buildings. 7 was hit by rubble,1 +8312,China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/3PBFyJx0yA,0 +8314,#360WiseNews : China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/9Naw3QOQOL,1 +8315,Jun 2015. Yemenis search for survivors under the rubble of houses in the old city of San__ªa following an... http://t.co/11JUzHlgmT,1 +8317,China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/j4ggmKINEy #forbesasia,1 +8318,Steve Buscemi was a firefightr B4 fame & workd 12hr shifts diggin thru WTC rubble lookg 4 survivors. http://t.co/L9fJpNSZuO,1 +8320,China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/o6oNSjHCsD #tcot #p2 #news,0 +8321,China's Stock Market Crash: Are There Gems In The Rubble?: China__ªs stock market crash this summer h... http://t.co/pE2R3lN16o by .Forbes,0 +8324,I found a diamond in the rubble,0 +8329,#TNN: China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/9LO0hZwJPZ,1 +8330,China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/BqBLWiw08g #ROIMentor #yycwalks,1 +8332,China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/Ox3qb15LWQ | https://t.co/8u07FoqjzW http://t.co/tg5fQc8zEY,0 +8334,#360WiseNews : China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/aOd2ftBMGU,1 +8335,New post: 'China's Stock Market Crash: Are There Gems In The Rubble?' http://t.co/6CaDRhIOxp,0 +8337,China's Stock Market Crash: Are There Gems In The Rubble?: China__ªs stock market crash this summer has sparked ... http://t.co/2OqSGZqlbz,0 +8339,Photo: postapocalypticflimflam: Prodding around the rubble. http://t.co/Bgy4i47j70,0 +8341,#360WiseNews : China's Stock Market Crash: Are There Gems In The Rubble? http://t.co/gQskwqZuUl,0 +8344,@okgabby_ damn suh. don't let that ruin your year bruh. this our year. better start carpooling like we did back in the day,0 +8345,If I can't ruin his mood then I may have lost my direction. https://t.co/sLc27EMUgM,0 +8346,I ruin everything ????,0 +8347,#quote Never let your problems ruin your faith.,0 +8348,Im so anxious though because so many ppl will me watching me meet them and that makes me uncomfortable BUT I CANT LET THAT RUIN THE MOMENT,0 +8349,smokers that ruin that new car smell ????,0 +8350,@ChrisDyson16 Just wait until your friends at #MTA ruin it #Sorrybutitstrue,0 +8351,i'm really sad about red 7 closing :( yuppies n tourists ruin everything,0 +8352,Always gotta ruin my mood,0 +8355,I understand you wanting to hang out with your guy friends I'll give you your space but don't ruin my trust with you.,0 +8356,like why on earth would you want anybody to be unhappy don't purposely ruin somebody else's happiness,0 +8360,fresh out da shower lookss ?? (still loving this new hair does it ruin my brand?) #yes https://t.co/T2Kk8fya77,0 +8361,@savannahross_4 see tryna ruin my life,0 +8362,well done me! everyone applaud me and how terrific i am and how i never ruin anything,0 +8363,Four Things That May Ruin Your Personal Injury Accident Claim http://t.co/ZU9YYdF5DI,1 +8364,How to ruin the holidays in 3 wordk: 'It's a Zmne!',0 +8365,every time I have a really good day someone just has to ruin it,0 +8366,You wanna ruin a relationship? Just ask 'what are your intentions' and you'll do it,0 +8367,And then I go a ruin it all with something awful... #minions http://t.co/rc6eeJME17,0 +8368,IT WAS REALLY AWFUL AND I CANT EVEN WORK WITH A MAKE UP ARTIST NOW BECAUSE THEY WILL RUIN THINGS.,0 +8369,Damn Wale knows how to ruin a song??,0 +8370,'Cause you play me like a symphony play me till your fingers bleed. I'm your greatest masterpiece. You ruin me??,0 +8371,Why does my family have to ruin something that is actually going good for me??,0 +8372,I hate that Im so awkward and I ruin things,0 +8374,Long Road To Ruin - Foo Fighters,0 +8375,Why do u ruin everything? @9tarbox u ruined the sour cream and u put a brick of cheese in the freezer..dummy,0 +8377,Don't let a few assholes ruin your night,0 +8378,Lol The real issue is the the way the NFL is trying to ruin this guy's legacy. https://t.co/s107ee7CYC,0 +8382,RT to ruin @connormidd 's day. http://t.co/krsy54XMMC,0 +8383,Greedy bastards @Fullscreen way to ruin creativity. CENSORSHIP ON YOUTUBE: https://t.co/nMtlpO4B58,0 +8385,@JulieChen she shouldn't. Being with them is gonna ruin her game and Vanessa is a great player,0 +8386,@clnv_ Yes Yes! I will. I dont wanna ruin my life. Lol,0 +8387,@nbc I wanna see you reboot The Fresh Prince of Bel-Air bring back the original cast and everything & do nothing that will ruin the show.,0 +8388,You can only make yourself happy. Fuck those tryna ruin it keep smiling ??,0 +8389,there are a few people I'd let ruin my life my soul my cervix my everything. Odell is definitely top of that list.,0 +8390,I'll ruin my life if I have to.,0 +8391,babe I'm gonna ruin you if you let me stay,0 +8392,@CTAZtrophe31 Everything must be OK because she's listening to 'Sandstorm' now...,0 +8393,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/mkWyvM3i8r,1 +8394,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/aZL4XydvzK,1 +8396,@Hienshi @Gbay99 it wouldnt turn into a sandstorm if riot gave a good answer. Instead they gave dumb excuses. At least new client is there,0 +8397,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/Z2Ph0ArzYI,1 +8399,I liked a @YouTube video http://t.co/xR3xJJ8gJB Darude - Sandstorm,0 +8400,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/brctMNybjy,1 +8401,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/NX2d83A4Du,1 +8402,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/GaotrG4mTr,1 +8404,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/rkU0IDM6aQ,1 +8405,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/7IJlZ6BcSP,1 +8406,A sandstorm in Jordan has coated the Middle East__ªs largest refugee camp in a layer of grit http://t.co/hVJmuuaLXV http://t.co/T8Nz6h9Zz4,1 +8408,SANDSTORM!!! WOO HOO!!,1 +8409,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/VZPKn23RX4,1 +8410,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/Rm50vCVjsh,1 +8411,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/1tr2KvXCTW,1 +8412,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/Q0X7e84R4e,1 +8414,Who need friends when there's booze and Darude - Sandstorm :D,0 +8415,SocialWOTS: GLOBI_inclusion: RT NRC_MiddleEast: Sandstorm engulfs caravans and tents in the #Zaatari refugee camp ___ http://t.co/XBNLSBzzgI,1 +8416,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/H84R1TIh8J,1 +8418,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/sEquWmvFx4,1 +8419,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/akNyNPv461,1 +8422,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/C9t2F6DLtM,1 +8423,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/xSZicdWxq0,1 +8425,@SunderCR two hours of Sandstorm remixes. All merged together. No between-song silence.,0 +8428,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/qr6BtDCqCj,1 +8429,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/wD9ODwjj9L,1 +8430,PUT SANDSTORM DOWN!!!! https://t.co/EfKCoegJck,0 +8431,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/87H5MbA3N1,1 +8432,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/8L4RFFZD0P,1 +8433,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/TvYQczGJdy,1 +8434,"Come out to Sandstorm tryouts Aug 15th at Lower Woodland! +MS Tryout: 3-4:30pm +HS Tryout: 4:30-6pm",1 +8435,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/BB7TTdVJWE,1 +8436,@VillicanaAlicia [[Is it Darude Sandstorm?]],0 +8437,I can't listen to Darude Sandstorm without expecting airhorns now,0 +8440,Now playing: Darude - Sandstorm - radio edit http://t.co/DUdAIrBBPo http://t.co/padosfyXnM,0 +8441,Watch This Airport Get Swallowed Up By A Sandstorm In Under A Minute http://t.co/bgM4cSrbVd,1 +8443,I slammed my phone to the ground and then screamed ahahahga,0 +8444,@InfiniteGrace7 I just screamed to the world how much I love My Little Pony ??,0 +8445,Update: The police seemed like nice enough people. I felt bad when I transformed and they screamed... Don't worry they're only unconscious.,0 +8446,someone's gonna get screamed at for getting the lyrics wrong lmao,0 +8447,I JUST SCREAMED SIDJSJDJEKDJSKDJD . I CANT STAND YOU ?? https://t.co/0Vcsafx9bY,0 +8448,@GodOf_Mischief_ -of Loki's daggers she pulled it out and jammed it into Mina's thigh. When Mina screamed and grabbed at her leg sif-,1 +8450,"OMFG?? +Didnt expect Drag Me Down to be the first song Pandora played + +OMFG I SCREAMED SO LOUD +My coworker is scared http://t.co/VzcvAdkcQp",1 +8451,She screamed when she got the microchip needle. Tbh I would have cried too the needle was massive ya haram,0 +8452,I heard the steven universe theme song from upstairs and screamed his name at the part of the song and scared my cousin,0 +8453,@mogacola @zamtriossu i screamed after hitting tweet,0 +8454,THE GIRLS NEXT TO ME SCREAMED WHAT THE FUCK IS A CHONCE I'm CRYIBG,0 +8455,I thought the loudest goal I ever screamed was Higuain's offside goal against Germany,0 +8457,I ran with earbuds in which I now realize means I probably didn't politely say hi to Jared but more or less screamed at him ??????,0 +8458,I just got screamed at for asking my dad to move half a step so I could get through....,0 +8459,@CortezEra I dead ass screamed when we signed him I couldn't fucking believe it,0 +8461,don't stop believing just came on the radio at this restaurant and a lil white bit screamed 'it's mommys song',0 +8463,brooke just face timed me at the concert and just screamed for 2 minutes straight,0 +8464,*wants to make tweet about how much I dislike todays femnism movement but doesn't want to be screamed at' http://t.co/R7pVTSdUmA,0 +8468,26 people have screamed right in my ear this week at camp. TWENTY-SIX! And we still have 1/2 day left. #CedarGlade2015,0 +8471,Some kids going to leadership camp came into my work today asking for the bathroom and my inner ASB kid screamed a lot. I miss camp ??,0 +8472,Vacation update: my great aunt just killed a spider with her bare hand after I screamed in fear. #fierce,1 +8473,Not because i want to cheat or anything. Just feels good to vent to my twitter without a txt being screamed at lol,0 +8475,i'm sorry i'm so wild in 1d shows like in my wwa show niall started singing steal my girl I literally screamed shut the fuck up,0 +8476,I JUST SCREAMED IN 57 LANGUAGES THIS IS SO GOOD https://t.co/ldjet9tfMk,0 +8477,I SCREAMED AT THE END OAMSGAJAGAHAHAH IM LAIGHIGN #OTRAMETLIFE http://t.co/eTkBW1RCrv,0 +8478,I have never screamed so loud https://t.co/PC3h1NE4G0,0 +8480,I SCREAMED 'WHATS A CHONCe' http://t.co/GXYivsWki7,0 +8483,@Real_Liam_Payne I SCREAMED AT THE TOP OF MY LUNGS WHEN YOU SAID YOU GUYS WOULD COME BACK TO S.A SO KEEP YOUR PROMISE #AddTexasToNext1DTour,0 +8484,@ThatWitchEm @EmmaChosenOne @JessieNovoaRP @Jessie_Novoa_ @Liana_Novoa why you screamed,0 +8486,@ArianaGrande I literally walked out of the concert and screamed MY SOUL HAS BEEN BLESSED,0 +8487,I just screamed what the fuck is a hond,0 +8489,I JUST SCREAMED @toddyrockstar http://t.co/JDtPirnm76,1 +8490,//kinda screamed >_< https://t.co/MSUY4qTPk9,0 +8491,i dont even remember slsp happening i just remember being like wtf and then the lights turned off and everyone screamed for the encore,0 +8494,Love waking up to my dad screaming at me ??????,0 +8496,If you are quiet enough you can literally hear the phandom screaming at the tyler tweet,0 +8497,I'M SCREAMING AND FANGIRLING OH MY GOD https://t.co/WdVQlEWYBs,0 +8498,during drag me down last night I was screaming 'shell of a mannequin' instead of 'shell of a man' and I think everyone hated me there,0 +8500,@camilacabello97 Internally and externally screaming,1 +8502,*CUE THE JARIANA STANS SCREAMING,0 +8503,@BizzleMahomie SCREAMING,0 +8504,@camilacabello97 NOW IM INTERNALLY SCREAMING,0 +8505,@estellasrevenge the first time i went swiming in it i was basically screaming WHY DOES IT SMELL/TASTE SO BAD,0 +8506,SCREAMING @MariahCarey @ArianaGrande http://t.co/xxZD1nmb1i,0 +8507,Lhh silent screaming was mastered https://t.co/BIexWDlDWC,0 +8508,@noahshack he's hot & he can sing I'm screaming??????,0 +8509,/ it's fine baby I was screaming at the TV x https://t.co/JwDfPYG3NT,0 +8510,"Harshness Follows Us a +Better Day +by Sarah C +Racing thoughts with screaming sirens +Pacing back and forth for... http://t.co/ProNtOuo91",0 +8511,MY FAVS I'M SCREAMING SO FUCKING LOUD http://t.co/cP7c1cH0ZU,0 +8512,@tyleroakley IM SCREAMING,0 +8513,@Scalpium SCREAMING,0 +8514,CAMILA'S DOING A FOLLOW SPREE TONIGHT IM SCREAMING OF HAPPINESS,0 +8515,@stvmlly Sorry for screaming at you and @Safferoonicle from the car I just kinda 'I KNOW THOSE PEOPLE LET'S YELL!' :/,0 +8516,My Dad a screaming coach always gave his vocal chords quite the workout on the field. http://t.co/axVQ80RbYJ #FunnyDadCoach,0 +8517,'[+54 -9] How do people not know who Kendall Jenner is? She has 6 times the Instagram followers of GD' SCREAMING LMAO,0 +8518,@MissDaOh and if she had a screaming baby you would have loved that so much more,0 +8520,@ArianaGrande @justinbieber I'M SCREAMING OMG #IDOLS #22DAYS #WDYM,0 +8522,@justinbieber @ArianaGrande Can you hear me screaming !!!!!,0 +8523,@KamKasteIIano @BluntedJayt FUCKING SCREAMING !,0 +8524,@ArianaGrande @justinbieber All the loves be screaming at this one ??????,0 +8526,"Hey there lonely girl +Did you have to tell your friends +About the way I got you screaming my name?",0 +8528,"#NoChillLukeHammings +IM SCREAMING",1 +8530,@danisnotonfire if you follow me I will go into town and advertise your youtube channel by screaming and walking around with a sign. Pls??,0 +8531,@justinbieber @ArianaGrande SCREAMING,0 +8533,@blanksocietyx @emmerdale IM SCREAMING HES MY FAVOURITE,0 +8536,@justinbieber @ArianaGrande IM SCREAMING,0 +8538,SCREAMING IN 22 DIFFERENT LANGUAGES http://t.co/rDfaAKKbNJ,0 +8539,@justinbieber I AM SCREAMING HELL YES AHHHH OMG http://t.co/y678XsNvJ6,0 +8540,like little boy you better sit your ass down stop screaming at my mother stop pulling your hair & crying like a 3 year old & grow tf up,0 +8541,@ArianaGrande @justinbieber OMGGGG IM SCREAMING,0 +8542,@TromboneTristan OOOOOHSHIT OOOHSHIT SCREAMS hell I LOVE,0 +8543,@DaneMillar1 *screams 666*,0 +8545,//screams in the distance// http://t.co/Cfe9HUQN0h,1 +8546,When you on the phone and @Worstoverdose screams 'jaileens caked up on the phone' so everyone looks at you ????????????,0 +8547,*screams in 25 different languages*,0 +8550,*screams internally*,0 +8551,*screams* http://t.co/PU7C4Hhbxj,0 +8553,@HimeRuisu I'm going to ram your ass so hard I'll have to shove your face on the pillows to muffle your screams of pain and pleasure~,0 +8555,I come downstairs trying to look as normal as possible in front of my mom and Rhiannon screams 'HUNGOVER???? Huh???' Uhmmmm no ????????,0 +8556,He hit a tailor fucking made double play and Kay screams he was robbed of a hit.,0 +8557,"Apparently my face screams +'Get me emotionally attached and then cheat on me'",0 +8558,@drag0nking0201 *Screams*don't scare me and its Animatronics,0 +8559,Gets on battlefield 3 screams into Mic at cunts,0 +8560,~Still echoes of their screams~,0 +8561,in every bts song jimin screams,0 +8562,Casually on the phone with Jasmine while she cries and screams about a spider,0 +8567,@RealJaxClone *screams*,0 +8570,"IS THE UPDATE RLY LIFE NOW IS IT IS It/Screams +vibrates i cant handle",0 +8571,"@saku_uchiha_ @Ya_Boi_Luke + +Screams and gets a face full of Saku genitals",0 +8572,@heyot6 Im not home. I need to watch. [Screams],0 +8574,In light of recent events all I would like to say is *screams for five years*,0 +8575,So @LawsonOfficial just followed me and I cannot contain my screams of joy! #Thankyou! ??????????,0 +8576,@4Tiles @ZacB_ my dell tablet screams with win10,0 +8577,@OllyMursAus I do feel sorry for him! He is not a piece of meat! He is a nice guy... People don't need to rush him and screams in his face!,1 +8578,I agree with certain cultural appropriation things but honestly if u looked at my house it screams appropriation bc Buddhas and stuff-,1 +8579,This Looney Tunes-Blake Griffin commercial screams their making a Space Jam 2,0 +8580,"SCREAMS. WHERE IS EVERYONE. + +oh wait school + +ok + +im ok",0 +8581,Aw man. 'Apollo Crews' just screams 'we can't think of a name for this black guy quick name some and mash them together',0 +8582,@QueenWendy_ go to sleep before someone screams at us??????,0 +8584,When you go to a concert and someone screams in your ear... Does it look like I wanna loose my hearing anytime soon???,1 +8585,@wisdc & obama supports death2USA .. http://t.co/serARcNrbY,0 +8586,The little girl next to me sees the spot lights in the sky and screams ' mom! Look I see Angels in the sky!' ????,1 +8587,@harveymaine AAAA ok lemme move to another roomr so no one hears my gay ass screams,0 +8589,* Screams *,0 +8591,SCREAMS AT MY OWN MOTHER http://t.co/gBEpdi0WzT,0 +8595,Oil and Gas Exploration http://t.co/PckF0nl2yN,1 +8596,Man is reading a list of famous musicians who oppose nuclear power...wow powerful evidence. They must know a lot about seismic risks....,0 +8598,SEISMIC RISK: a COMPARISON between 2 case studies: CALABRIA AND MALTA http://t.co/HmRtqEykyI,1 +8599,@hebrooon u better learn derivative of formula seismic rather than thinking about things like that or you are a things like that? Haha,0 +8606,Exploration takes seismic shift in #Gabon to #Somalia http://t.co/kLtIt88AS3,1 +8607,29% of #oil and #gas organizations have no real-time insight on #cyber threats. See how #EY can help http://t.co/qamgvQAFzc,0 +8608,Exploration takes seismic shift in Gabon to Somalia - WorldOil (subscription) http://t.co/kqVEVuutDJ #??????? #Somalia,0 +8609,The 08/06/2015 AlabamaQuake seismic summary w/ #earthquake #news & history http://t.co/zM6VcZqvWk http://t.co/DKNlZNom6n,1 +8610,'Seismic' Apple TV service to stream 25 channels across all d... http://t.co/zqMtrBKaS0 | https://t.co/YEqq3BZX3g http://t.co/kmVrZaSXY4,0 +8612,@marek1330 *Zar cringes at the blows but doesn't let Marek go* *He uses Seismic Toss,0 +8613,SEISMIC AUDIO SA-15T SA15T Padded Black Speaker COVERS (2) - Qty of 1 = 1 Pair! http://t.co/2jbIbeib9G http://t.co/p5KtaqW5QG,0 +8615,#Sismo DETECTADO #Jap__n [Report 1] 01:01:56 Okinawa Island region M4.0 Depth 10km Maximum seismic intensity 3 JST #??,1 +8618,ON THE USE OF PERFORATED METAL SHEAR PANEL SFOR SEISMIC-RESISTANT APPLICATIONS http://t.co/cX5OjH2Dr4,0 +8620,[Report 5] 18:22:45 Ibaraki Prefecture offing M5.5 Depth 60km Maximum seismic intensity 4 #Earthquake,1 +8621,Thanks Benson & Clegg for the #follow! Check out our #maps at http://t.co/btdjGWeKqx,0 +8622,On Thursday at 00:25 we updated our #kml of 2D and 3D #seismic exploration vessels. #offshore #oil http://t.co/btdjGWeKqx,0 +8623,Oil and Gas Exploration Takes Seismic Shift in Gabon to Somalia - Bloomberg http://t.co/bEKrPjnYHs #??????? #Somalia,1 +8624,"@kakajambori ?? +U control the future of india.. +Yor Subject: Exploration or seismic Maintenance( Electrical or Mechanical)",0 +8625,#Sismo DETECTADO #Jap__n [Report 3] 01:02:17 Okinawa Island region M3.8 Depth 10km Maximum seismic intensity 3 JST #??,1 +8627,Agency seeks comments on seismic permits http://t.co/9Vd6x4WDOY,0 +8628,Still no plans? Don't worry we got you covered. Plenty of Seismic IPA and Seismic Squeeze Radler to help... http://t.co/A8nMdkd3rV,0 +8631,#OilandGas Exploration Takes Seismic Shift in #Gabon to #Somalia http://t.co/oHHolJ9vEV via @business,1 +8633,"Exploration Takes Seismic Shift in #Gabon to #Somalia +http://t.co/Ltf6jL5keU http://t.co/Zlq8tHcTkW",1 +8635,New post from @SeismicSoftware: 3 Major Challenges of Channel Sales Enablement http://t.co/kWMRCEkVTF,0 +8636,Panoramic Resources cuts jobs after seismic event http://t.co/mUwmfJGzYh,1 +8637,ENGLAND EAST COAST. Dogger Bank Westward. 1. Seismic survey in progress by M/V Western Regent towing a 8400 metre long cable within area___,0 +8638,A subcontractor working for French seismic survey group CGG has been kidnapped in Cairo and is held by Islamic State the company said on W___,1 +8639,The Art World's Seismic Shift Back to the Oddball - Observer http://t.co/W0xR5gP8cW,0 +8640,#Sismo DETECTADO #Jap__n 06:32:43 Miyagi Estimated seismic intensity 0 JST #??,1 +8642,"Sinkhole Selfies: You Wont Believe What's In The Brooklyn Sinkhole!: + Sinkhole Selfies: You Wont Belie... http://t.co/A3b5n3rcr5",0 +8643,MRW when a sinkhole opens up beneath my friends and I... #gif #funny #lol #comedy #iFunny #video #image #RT http://t.co/XiYdYfptru,1 +8644,Talk on GOZ is fantastic. Most interesting fact so far is that they manually bought all the .ru domains to sinkhole rather than seek co-op.,0 +8646,nothing surprises me anymore and i am sure there is more to come... http://t.co/zdpvQmEezS,0 +8647,Large sinkhole swallows entire pond in Lowndes County Georgia: Large sinkhole___ http://t.co/bCLDQmMEHg #Occasion2B,1 +8648,10News ? Water main break disrupts trolley service http://t.co/pAug7a68i0,1 +8650,Sinkhole closes Falmer Drive in Bethlehem Township http://t.co/6TVVlG2fNi,1 +8651,A sinkhole grows in Brooklyn: six-meter crater swallows street http://t.co/gkPrvzQ6lk,1 +8653,@Azimel 'Screaming Mad Scientist deceased after tumbling over heels and falling into sinkhole during investigation',0 +8654,Gaping sinkhole opens up in Brooklyn New York http://t.co/0xA6FCjyec,1 +8655,___@FDNY: This morning #FDNY responded to a sinkhole in #Brooklyn. Units remain on-scene with @NYCBuildings & others. http://t.co/M78ir0IK01___,1 +8656,"150-Foot Sinkhole Opens in Lowndes County Residential Area +WCTV-35 minutes ago",1 +8657,Large sinkhole swallows entire pond in Lowndes County Georgia http://t.co/wKPzp1JCAu #YoNews,1 +8658,I've been trying to write a theological short story about a monster living in a sinkhole. Then I heard about Brooklyn. #accidentalprophecy,0 +8659,Sinkhole Selfies: You Wont Believe What's In The Brooklyn Sinkhole! http://t.co/3gLYOyf6Oc,0 +8660,@MoorlandsChmbr Loads of stuff going on recently. See the blog at http://t.co/XVcO7sLxhW #sinkhole #piling http://t.co/jbVmGeg522,0 +8661,150-Foot Sinkhole Opens in Lowndes County Residential Area http://t.co/0YAxrJICRR,1 +8662,#SanDiego #News Sinkhole Disrupts Downtown Trolley Service: The incident happened Wed... http://t.co/RVMMuT3GvC #Algeria #???????,1 +8663,beforeitsnews : A sinkhole grows in Brooklyn: six-meter crater swallows street ___ http://t.co/QWo7q1AMh8) http://t.co/esRkmazEq9,1 +8664,Massive Sinkhole Emerges In Brooklyn http://t.co/n3Ow73Oasw http://t.co/Gs9bmplbHH,1 +8665,#LoMasVisto THOUSANDS OF HIPSTERS FEARED LOST: Giant Sinkhole Devours Brooklyn Intersectio... http://t.co/qwtk1b2fMC #CadenaDeSeguidores,1 +8667,150-Foot Sinkhole Opens In Lowndes County Residential Area http://t.co/cBAxCuBA0h,1 +8668,Sinkhole Disrupts Downtown Trolley Service #SanDiego - http://t.co/9tb82ZMr2X,1 +8669,Sinkhole swallows Brooklyn intersection ___ video http://t.co/1yBE5mgZL4 http://t.co/7Zog3DpdU9,1 +8670,i could die by falling in a sinkhole and i'd still be blamed for it,0 +8673,Damn that sinkhole on sunset????,1 +8674,Water main break disrupts trolley service http://t.co/kff8ojrZP4 #sandiego http://t.co/JoB4GGtpAl,1 +8676,RT twit_san_diego 'Possible sinkhole disrupts trolley service: A depression in a portion of asphalt in downtown Sa___ http://t.co/ANrIOMbHQN',1 +8677,"Sinkhole Selfies: You Wont Believe What's In The Brooklyn Sinkhole!: + Sinkhole Sel... http://t.co/OYY9MGW7HN @hinterestin #funny",0 +8680,Some Evansville residents told to limit water usage due to sinkhole construction: Some Evansville residents have___ http://t.co/SJNyFszCu1,1 +8681,@DavidCovucci We can't because a sinkhole swallowed every taco place in the neighborhood,0 +8683,Sinkhole on west side damaging cars via @WEWS http://t.co/S7grbZNwlr,1 +8684,Georgia sinkhole closes road swallows whole pond http://t.co/cPEQv52LNA,1 +8685,Does that sewer look like it's sinking to you? Is this what happens pre-sinkhole???? and I'm going insane http://t.co/heIekfcHdM,0 +8686,"Sinkhole leaking sewage opens in housing estate +Irish Independent-3 Aug 2015",1 +8688,A random hole just broke out in the street ?? http://t.co/dWU8QqYs0v,0 +8689,#MTSAlert Orange & Blue Line riders: Expect delays downtown due to a sinkhole that's developed in the vicinity of 4th & C Street.,1 +8690,There's a sinkhole in Brooklyn ?!,1 +8691,Share Large sinkhole swallows entire pond in Lowndes County Georgia A large___ http://t.co/HvBJ30aj9s #YoNews,1 +8692,Do you feel like you are sinking in low self-image? Take the quiz: http://t.co/JvjALYg2n1 http://t.co/qXMWELJbc0,0 +8693,After a Few Years Afloat Pension Plans Start Sinking Again http://t.co/4cEEuzWHvf,1 +8694,Do you feel like you are sinking in unhappiness? Take the quiz: http://t.co/OrJb3j803F http://t.co/MWdHXYfrag,0 +8695,With a sinking music video tv career Brooke Hogan should be THANKING her Dad for the free publicity...although I doubt it will help her.,0 +8696,@supernovalester I feel so bad for them. I can literally feel that feeling of your heart sinking bc you didn't get anyone ugh jfc,0 \ No newline at end of file diff --git a/TextClassification/TextClassification/data/val.csv b/TextClassification/TextClassification/data/val.csv new file mode 100644 index 0000000000000000000000000000000000000000..baf4e3ebb9ecc439dca214c610436372e72a0b37 --- /dev/null +++ b/TextClassification/TextClassification/data/val.csv @@ -0,0 +1,1767 @@ +id,text,target +8697,#nowplaying Sinking Fast - Now or Never on North East Unsigned Radio listen at http://t.co/QymAlttvZp,0 +8698,that horrible sinking feeling when you__ªve been at home on your phone for a while and you realise its been on 3G this whole time,1 +8699,Nigga car sinking but he snapping it up for fox 13. #priorities http://t.co/9StLKH59Fb,0 +8700,@abandonedpics You should delete this one it's not an abbandoned nor sinking. That__ªs the darsena of the Castello scaligero di Sirmione.,0 +8702,that horrible sinking feeling when you__ªve been at home on your phone for a while and you realise its been on 3G this whole time,0 +8704,4 equipment ego break upon dig your family internet hoke excepting versus a sinking term: dfLJEV,1 +8705,Currency transgress before payday-prison ward sinking-fund payment unsecured loan: jBUmZQpK,0 +8706,?that horrible sinking feeling when you__ªve been at home on your phone for a while and you realise its been on 3G this whole time,0 +8708,If you're lost and alone or you're sinking like a stone carry onnnn,0 +8709,Lying Clinton sinking! Donald Trump singing: Let's Make America Great Again! https://t.co/zv60cHjclF,0 +8710,"@AP + Too slow report the sinking boat in the Mediterranean sea what a shame",1 +8711,We walk the plank of a sinking ship,0 +8712,The Sinking Ship (@sinkingshipindy): Scarlet Lane Lenore is on replacing Stone Saison (@StoneBrewingCo),0 +8714,that horrible sinking feeling when you__ªve been at home on your phone for a while and you realise its been on 3G this whole time,0 +8715,In the movie 'Titanic' Jack and Rose both could have stayed on the wooden beam without it sinking.,0 +8717,__¢__¢If your lost & alone or your sinking like a stone carry onå¡å¡,0 +8718,If there's a chance will get a gander of the sinking ship that is #TNA too. Can't help but appease my morbid curiosity. #DestinationIMPACT,0 +8720,So happy to be exercised of the demon of @ATT. Price kept rising service kept sinking. #goodbye,0 +8721,Do you feel like you are sinking in low self-image? Take the quiz: http://t.co/bJoJVM0pjX http://t.co/wHOc7LHb5F,1 +8722,INVESTMENT NEWS Keurig Green Mountain Inc. Third-Quarter Earnings: Shares Sinking After-Hours - Stocks in the New___ http://t.co/GtdNW1SpVi,0 +8723,@WCCORosen did Lloyds of London insure your bet with @CoryCove #sinking #twins,0 +8724,Sinking the Slipper or Putting the Boot In http://t.co/b1bx0ERuep,0 +8726,"In your eyes I see the hope +I once knew. +I'm sinking. +I'm sinking +away from you. +Don't turn around +you'll see... + +You can make it.",0 +8727,That horrible sinking feeling when you__ªve been at home on your phone for a while and you realise its been on 3G this whole time.,0 +8728,'I'm sinking down in the darkest dream so deep so cold this pain inside of me my love for you is more dan I can bear' Jota Esse??,0 +8729,Sinking carb consultative assembly plans could subconscious self live straight a leading way of escape: XkDrx,0 +8732,This is Lara she likes sinking her teeth into my flesh and clawing my arms ?????? http://t.co/J43NWkX0X3,0 +8733,Spent too many hours sinking into the wonderfully created worlds of Mafia and Mafia II in my life. Excited for another installment.,0 +8734,Do you feel like you are sinking in unhappiness? Take the quiz: http://t.co/BTjPEO0Bto http://t.co/ClyJ32L333,0 +8735,That horrible sinking feeling when you__ªve been at home on your phone for a while and you realise its been on 3G this whole time,1 +8736,ITS JUST NOW SINKING IN THIS IS THE LAST EPISODE MY HEART HURTS SO BAD,0 +8737,Slowly sinking wasting ?? @edsheeran,0 +8738,The #Tribe just keeps sinking everyday it seems faster. As for this year it's been a titanic disaster.,1 +8739,that horrible sinking feeling when you__ªve been at home on your phone for a while and you realise its been on 3G this whole time,1 +8740,each time we try we always end up sinking,0 +8741,Do you feel like you are sinking in unhappiness? Take the quiz: http://t.co/t0c1F2lEdv http://t.co/u0MeXO4Uhh,0 +8742,I just made a weird high pitched noise and then I heard a siren ofnsixjks ??????,0 +8743,Today is Corii Siren's birthday! Spoil her now: http://t.co/l3GizRUCy4 #wishlist http://t.co/HgEDwxTDJN,0 +8744,'@Ma3Route: Haha jam imeshika hapa garden city mats bumper to bumper with AAR ambulance.we decide to chill via @BonnieG434' piga siren kijan,0 +8745,@stacedemon oh shit!,0 +8746,Big Love @IbeyiOfficial https://t.co/OlnK1TI1NM,0 +8747,'Amateur Night' Actress Reprises Role for 'Siren' - HorrorMovies.ca #horror http://t.co/W9Cd6OFfcj,1 +8750,Lol '@j2bone: *cousin ' @Foxy__Siren: Coursing* '@WEYREY_gidi: Now they are causing Di Maria.. LOL'',0 +8751,What. The. Fuck. https://t.co/Nv7rK63Pgc,0 +8752,I'm more into the healing/reviving side of the game rather than better attacking so for now Siren > all other characters (except new girl).,0 +8753,A demoness with the voice of an angel. Like a siren's call beckoning me to the void. Don't ?? on this___ https://t.co/nPS3xpBKaQ,0 +8754,WHELEN MODEL 295SS-100 SIREN AMPLIFIER POLICE EMERGENCY VEHICLE - Full read by eBay http://t.co/A5iwUS8EVQ http://t.co/gI82N2JuWn,0 +8755,http://t.co/a0v1ybySOD Its the best time of day!! å_ @Siren_Voice is #liveonstreamate!,0 +8757,? #nowplaying SONG TO THE SIREN - JOHN FRUSCIANTE (2009) http://t.co/00cY9vXEFF,0 +8758,So when r u getting married'@Foxy__Siren: Oh finally Jennifer Aniston got married??????... I'm so happy for her ??????',0 +8759,Super sweet and beautiful :) https://t.co/TUi9uwBvVp,0 +8760,Apparently they're going to have a WW2 siren to announce something. Not sure if that's fun or dull bc the WW2 siren is a v monotonous sound.,0 +8761,There's a weird siren going off here...I hope Hunterston isn't in the process of blowing itself to smithereens...,1 +8762,@tomarse99 they all are intending to go. Just waiting for winds to drop. Siren just gone off to signal they allowed to leave arena.none gone,1 +8763,Internet firms to be subject to new cybersecurity rules in EU... http://t.co/lafTJ2GyLY,0 +8766,I don't think I've forgiven Angelina Jolie yet sef????,0 +8767,Coursing* '@WEYREY_gidi: Now they are causing Di Maria.. LOL',0 +8768,I added a video to a @YouTube playlist http://t.co/612BsbVw8K siren 1 gameplay/walkthrough part 1,0 +8770,@optich3cz #askH3cz i'm jealous now. Bc i wanted a elgato hd for my bday so i can record videos but i didnt have the money for it.,0 +8774,Look at them'@Ayhoka_: Co happy bn too long'@Foxy__Siren: Oh finally Jennifer Aniston got married??????... I'm so happy for her ??????'',0 +8775,@EnvySeven My beautiful Aquarius queenmy Siren of the cliffs and pretenses of overtures.Please sing this phantom songfor you alone shall,0 +8776,"Pharma overloaded with a loud cry like an emergency siren. + +...fucking.",0 +8777,@Siren_Song21 my pc account got hacked. Someone tried to pull out over 1200 bucks which wasn't there Now I have an nsf & no idea who or why,0 +8778,Don't argue cheap now. You're better than that. ??,0 +8780,The real question is why is the tornado siren going off in Dyersburg?,1 +8781,"Can you save +Can you save my +Can you save my heavydirtysoul?",0 +8782,Siren Test complete :: The test has concluded,0 +8783,Serephina the Siren <3 http://t.co/k6UEtsnLHT,0 +8784,@LA_Siren Thanks for joining the foot. @VVorm,0 +8785,Outdoor Siren Test 2pm :: The FGCU Siren will be tested at 2pm today. Another message will be sent when the test is concluded.,0 +8786,@SoonerMagic_ I mean I'm a fan but I don't need a girl sounding off like a damn siren,1 +8787,I hate this damn Milwaukee IndyFest. All the cars sound like a really long tornado siren going off and it woke me up from my nap,0 +8788,Any new games coming soon @BreachGamingORG ?,0 +8789,@FEVWarrior -with the screeching siren accompanying it just before he walked out.,0 +8790,This cop seriously just turned on their siren to get through traffic and turned it right back off like wtf,0 +8791,u know the music good when you hear the siren and you get chills,0 +8793,Reasons I should have gone to Warped today: tony played issues showed up sleeping w sirens played attila is there issues issues issues,1 +8795,"Thu Aug 06 2015 01:20:32 GMT+0000 (UTC) +#millcityio #20150613 +theramin sirens",1 +8797,@RAYCHIELOVESU On the block we hear sirens& stories of kids getting Lemonade only to see their life get minute made. we talking semi paid,0 +8799,A segment of the V/H/S anthology series is getting the feature film treatment:... http://t.co/LqJMuAxJUU,0 +8800,@_DANGdaddy the sirens are telling you to get ready to TURN UP???????? http://t.co/qAQqrJv9gU,0 +8801,Fuck Sleeping With Sirens.,0 +8802,my dad said I look thinner than usual but really im over here like http://t.co/bnwyGx6luh,1 +8804,Rappers stop sampling police sirens and start sampling whale song.????,0 +8805,Yay for sirens,0 +8806,To ouvindo sleeping with sirens awn,0 +8808,#SirensIcebreaker What is one fantasy work about a diverse heroine that you think everyone should read? https://t.co/HplJUr0OBo,0 +8810,@KIRO7Seattle Just saw a Bomb Squad car heading north on Elliott with sirens on - any word on where they're headed??,1 +8812,'I know a dill pickle when I taste one' -me,0 +8815,Marketforce Perth named winner of Sirens round 2 for 'iiNet NBN Buffering Cat Shark' radio spot http://t.co/GGPERGLVKi,0 +8816,@TravelElixir Any idea what's going on? I hear no sirens but this damn helo is flying so low my apt is shaking!,1 +8817,Lets Goooooooo http://t.co/fZ5eW4iHmB,0 +8823,A new favorite: Midfield General Disco Sirens House Edm Version Soni Soner by @sonisoner https://t.co/DEvffPTCVj on #SoundCloud,0 +8825,Photoset: hakogaku: ?å_I am a kurd. i was born on a battlefield. raised on a battlefield. gunfire sirens... http://t.co/obp595W7tm,1 +8829,@iK4LEN Sirens was cancelled.,0 +8830,'If you looking for my niggas you can follow the sirens.' ????,0 +8831,What's going on in Hollywood? #abc7eyewitness @ABC7 helicopters and sirens. #HometownGlory,0 +8832,"connor franta: damn sirens I hope everyone is okay. + +dan howell: can you PLEASE get MURDERED on ANOTHER STREET",1 +8835,Gostei de um v__deo @YouTube de @christinartnd http://t.co/bwe9kJCEPt Sleeping with Sirens Postcards and Polaroids acoustic cover,0 +8836,A Rocket To The Moon ? Sleeping With Sirens ?A Rocket To The Moon ????????????,0 +8837,Marketforce__ªs ___Cat Shark___ wins Sirens round two http://t.co/6F9aFQL6WP #radio #news,0 +8838,I added a video to a @YouTube playlist http://t.co/f2TqMFh1Yb Cher Lloyd - Sirens,0 +8839,sleeping with sirens vai vir pra sp,0 +8840,Kyle is one of the last people I would expect to like Sleeping with Sirens.,0 +8841,It's 'Run From Sirens' by 'Half Hour Hotel' @halfhourhotel @Edgarsgift. 'Premium Promotion' OFFER http://t.co/zRN30A78ir,0 +8842,What's wrong with just a lil smoke and good conversation ????,0 +8843,@LifeAintFairKid if I did I'd smoke you up brooo!,0 +8844,I barely smoke with people i solo all my blunts,0 +8846,smoke whatever you got,0 +8848,Smoke the Weed - Snoop Lion Ft. Collie Buddz,0 +8850,And all you girls that smoke.... have you notice that you started to lose ALOT OF WEIGHT ??? HMMM Don't know body want no thin girl tf,0 +8851,Smoke with me baby and lay with me baby and laugh with me baby I just want the simple things,0 +8852,IM GONNA GET NAKED AND SMOKE MY CIGARETTE someone call me radneck or is it just too hot in here,0 +8853,SMOKE ALOT OF WEED LIKE FUCK KIDNEYS PUT A DUTCH IN ME,0 +8854,@SidelineSavage what like a pipe made of peanut butter or a pipe you can smoke peanut butter out of?,0 +8855,I want to smoke ??,0 +8856,In 2014 I will only smoke crqck if I becyme a mayor. This includes Foursquare.,0 +8857,when you burp and smoke comes out,0 +8858,if you're dating someone who doesn't let you smoke leave them and date someone who also smokes. trust me. ??,0 +8859,niggas selling weed just so they can smoke & stay high. YALL NIGGA IN THE WAY..,0 +8860,I wanna drink a little smoke a little,0 +8862,Smoke ave streets hottest youngins,0 +8863,@PianoHands You don't know because you don't smoke. The way to make taxis and buses come is to light a cigarette to smoke while you wait.,1 +8864,[55436] 1950 LIONEL TRAINS SMOKE LOCOMOTIVES WITH MAGNE-TRACTION INSTRUCTIONS http://t.co/xEZBs3sq0y http://t.co/C2x0QoKGlY,0 +8865,@Sammysosita smoke a blunt & get through it.. no more ciggs for you.,0 +8866,I smoke toooooo much lmao I was scared to text this number bck but now it all makes sense lol,0 +8869,@TheTshirtKid I'm tryna smoke that MF out,0 +8871,Be Trynna smoke TJ out but he a hoe,0 +8872,@BillMcCabe sky looks clear.... No smoke from the fires. Enjoy your time in Tahoe. One of my favorite places!,1 +8875,So does Austin smoke too since he agreed to that name or what? ___ Lol no http://t.co/UmZKC9AzWd,0 +8878,Smoke it all http://t.co/79upYdCeMp,0 +8880,I get to smoke my shit in peace,1 +8881,If you wanna smoke cigs that's your own problem but when your breath smells like an old ash tray.. that's fucking disgusting,0 +8883,@bre_morrow neither of them even smoke so I dk what was going on lol,0 +8885,Smoke eat sleep,0 +8886,I miss Josie cause I wanna smoke splifs and go to taco bell ??,0 +8887,Mental/Twitter Note: Make sure my smoke alarm battery is up to snuff at all times or face many twitter reminders of changing my battery.,0 +8888,@TeamAtoWinner no.. i mean when is mino said that he doesn't smoke? u mention it before.. :)),0 +8890,I just wanna smoke some weed and get some commas,0 +8892,'Snowstorm' 36'x36' oil on canvas (2009) http://t.co/RCZAlRU05o #art #painting,0 +8893,Manuel hoping for an early Buffalo snowstorm so his accuracy improves.,1 +8896,Hi yall this poem is called is the one about the snowstorm when we meet in space and that one time it rained. Thx. Ur watching disney chann,0 +8899,For some reason knocking on someone's door at 3am in a snowstorm while wearing footed pajamas is not the best way to get into their bathroom,0 +8900,"sorry-I built a fire by my desk already. RT@irishirr +@MChapmanWSAZ @WSAZ_Brittany @kellyannwx ..please maintain that snowstorm til I arrive.",0 +8902,Sassy city girl country hunk stranded in Smoky Mountain snowstorm #AoMS http://t.co/nkKcTttsD9 #ibooklove #bookboost,1 +8903,@Groupon_UK it won't let me as you don't follow me,0 +8905,New #photo Oak in a snowstorm http://t.co/HK9Yf72OVA taken in #winter on the #SouthDowns #Hampshire #photography #art #tree #treeporn,1 +8906,you're the snowstorm I'm purified. the darkest fairy tale in the pale moonlight.,0 +8907,Mini Lalaloopsy Dolls Seed Sunburst June Seashore Sweater Snowstorm Autumn Spice - Full re___ http://t.co/nyty7fCQo6 http://t.co/hyypsPN0yQ,0 +8908,@Habbo bring back games from the past. Snowstorm. Tic tac toe. Battleships. Fast food. Matchwood.,1 +8912,Sassy city girl country hunk stranded in Smoky Mountain snowstorm #AoMS http://t.co/ZDJ2hyF6RO #ibooklove #bookboost,0 +8913,"@dirk_trossen + +I've still got some of the snowstorm/hailstorm!",1 +8914,Can you list 5 reasons why a London #TubeStrike is better than a Snowstorm? Read here... http://t.co/PNaQXPrweg,0 +8916,"___@LordBrathwaite: Everyone Here: Ahh I hate snow! + +Me: Lol u call this a snowstorm..? + +#growingupincolorado___",1 +8920,@CacheAdvance besides your nasty thunderstorm or snowstorm nah. Can't say that I have.,0 +8921,'Cooler than Freddie Jackson sippin' a milkshake in a snowstorm',0 +8923,Transportation panel showing a video of a pileup of 18-wheelers in a snowstorm in WY. Crash crash crash ___Make it stop!___ #AMSsummer,1 +8924,Sassy city girl country hunk stranded in Smoky Mountain snowstorm #AoMS http://t.co/HDJS9RNtJ4 #ibooklove #bookboost,1 +8926,@PyrBliss ah I remember those days. In a snowstorm too.,1 +8927,Picking up a man in a bar is like a snowstorm you never know when he's coming how many inches you'll get or how long'll he'll stay.,0 +8931,#NowPlaying Last Smoke Before The Snowstorm by Benjamin Francis Leftwich - hopefully new album coming soon :) ? http://t.co/5kjy8G0i4y,0 +8933,New #photo Oak in a snowstorm http://t.co/JhSCGDA2G8 on the #SouthDowns #Hampshire #Winter #photography #art #tree #treescape #treeporn,0 +8934,I liked a @YouTube video http://t.co/z8Cp77lVza Boeing 737 takeoff in snowstorm. HD cockpit view + ATC audio - Episode 18,1 +8935,Photo: mothernaturenetwork: What is thundersnow? Hearing thunder during a snowstorm is extremely uncommon.... http://t.co/eYdAPauPvG,1 +8936,#LakeEffect #Snowstorm Twill Denim Jackets *** ALL MY PROCEEDS FOR ITEMS WITH THIS DESIGN WILL GO___ https://t.co/TxTpx4umqH,1 +8938,LRT refer to the lyrics to hear Big Boi explain why he__ªs as cool as ___sippin__ª a milkshake in a snowstorm' lame bars but effective at the time,0 +8939,@PrablematicLA @Adweek I'm actually currently dressed for a snowstorm...despite being in the middle of a Texas summer. Thanks office A/C.,1 +8941,Snowstorm planned outside #Rome's St Mary Major tonight - annual occasion artificial snow remembering summer snow in 358 AD on same spot.,0 +8942,FINALLY a storm,0 +8944,#gamefeed Warcraft 3-Inspired Mode Likely Hitting Heroes of the Storm: Let's go back to the beginning. http://t.co/gx1kZ3C2Tc #VideoGame,0 +8945,Typhoon Soudelor was captured in this incredible NASA photo on it's way to Taiwan! http://t.co/dGGm5b0w4L http://t.co/eYr2Xx5l1p,1 +8946,So this storm just came out of no where. .fuck me its cool,1 +8947,omfg.... I just woke up again....,0 +8948,New on Ebay UK Star Wars Storm Trooper Pop! Vinyl Bobble Head Figure POP Funko http://t.co/KJbXIeypma http://t.co/ENPjCfMa8L,0 +8949,"How to prepare your #property for a #storm: + +http://t.co/KhYqQsi6My http://t.co/G6Vs3XEinb",1 +8952,kesabaran membuahkan hasil indah pada saat tepat! life isn't about waiting for the storm to pass it's about learning to dance in the rain.,1 +8953,@Jenniferarri_ comeeeee! ...but why is it bout to storm tho,1 +8955,RT @tonyhsieh: 'The person who dances with you in the rain will most likely walk with you in the storm.' -Anonymous,0 +8956,The Secrets Of The Storm Vortex The Lightning Catcher Book ___ : http://t.co/OIyWrzL79Z .,0 +8957,Is it bad to say that I'm kinda afraid of storms and we are in a storm???? help,1 +8959,Looks like a perfect storm-free evening coming up. Check out the outdoor happenings featured at http://t.co/hUzrHgmkSY #EventsPalmBeach.,1 +8961,Crazy storm hit and I'm trapped inside a Hobby Lobby AMA http://t.co/8qc8Bcxoko,1 +8962,"It's okay I welcome the rain. +Gave you all the storm that you could weather.",0 +8963,Nike Golf Storm Fit Golf Jacket Black Medium http://t.co/jvAI5Vkmsy: #SportingGoods http://t.co/Nr8JjmpmoS,0 +8965,Every time I buy a bag for life I think I've got 3 days left to live.,0 +8966,What tropical storm? #guillermo by hawaiianpaddlesports http://t.co/LgPgAjgomY http://t.co/FKd1mBTB68,1 +8967,New item: Pillow Covers ANY SIZE Pillow Cover Grey Pillow Pillows Premier Prints Lulu Storm Grey by MyPillowStudio ___ http://t.co/M4pqkKeEVC,0 +8968,this storm????,0 +8969,@johngreen storm and silence by @RobThier_EN,0 +8970,A Warcraft 3-inspired mode is likely coming to Heroes of the Storm http://t.co/wz55NBYAO3 http://t.co/L7pMDmeJs1 http://t.co/mDP2nI1pQU,0 +8971,The Storm Prediction Center has expanded the 'Slight Risk' area to include more of central NC. #TWCNews #ncwx http://t.co/DgBeH5L9DS,1 +8972,'Calgarians stunned by storm insurance companies busy handling calls' #abstorm http://t.co/fkFa9vSssZ,1 +8973,Finna storm. Fuck my back boutta start hurting like a mf ??????,0 +8974,Ready for this storm,1 +8976,Severe T-storm Warning for Union County ~3:45pm. Top threats include heavy rain lightning & damaging winds. http://t.co/XWOevMK0aA,1 +8978,ice cream + cupcake wars + storm = content sara,0 +8979,"NASAHurricane? EASTERN PACIFIC *Full Update* Satellite Sees Formation of Eastern Pacific's Tropical Storm Hilda +Th___ http://t.co/KsXTo8NKNl",1 +8980,The sky's clear the storm has passed but it's still raining in my head,0 +8983,Happy birthday @lesley_mariiee ?? I miss you so much & i hope you have a great birthday!!????,0 +8985,#NASA announced that a massive #solar storm is headed straight for us: http://t.co/CM5u55MiOl,1 +8986,My last two weather pics from the storm on August 2nd. People packed up real fast after the temp dropped and winds picked up.,1 +8987,Doves - The Storm + Greatest Denier (Electric Proms Pt4) http://t.co/xjTpV4OydL,1 +8989,Today__ªs storm will pass; let tomorrow__ªs light greet you with a kiss. Bask in this loving warmth; let your soul return to bliss.,1 +8992,"Somebody get the doctor I'm feelin' pretty poor. Somebody get the stretcher +before I hit the floor",0 +8993,@Coach_Keith44 @HannoMottola @TRPreston01 @mlrydalch no but a stretcher would have been nice.,0 +8994,Free Ebay Sniping RT? http://t.co/B231Ul1O1K Lumbar Extender Back Stretcher Excellent Condition!! ?Please Favorite & Share,0 +8996,@Stretcher @invalid @Grazed @Rexyy @Towel I see the programme (:,0 +8998,Dan Hughes was taken off on a stretcher after Danielle Robinson collided @OU_WBBall hope they are both ok he called SOME games over years,0 +8999,FERNO would appreciate if everyone in EMS could dust off their engineering degree & inspect all stretcher components. http://t.co/GTEd6LDwho,0 +9000,Stretcher brought out for Vampiro. Cut to commercial isn't a good sign. #UltimaLucha #LuchaUnderground,0 +9001,How to restore vinyl siding and make it look new again http://t.co/MHL7Pfr7kb http://t.co/lou8lbLA1f,0 +9002,@Stretcher @Rexyy @invalid @Towel let's have babies??!,0 +9005,organic natural horn stretcher expander earrings body jewelry with painting in tribal ... http://t.co/NvZdilRfgj http://t.co/u9Cd0txE7Z,0 +9006,NO WAYS!!! A video for 'Stretcher'?! @Ofentse_Tsie,0 +9007,*New!* Stretcher in 5 min https://t.co/q5MDsNbCMh (by FUJIWARA Shunichiro 2015-08-05) [Technology],0 +9008,@PLlolz @Grazed @Stretcher @invalid @witter @Towel still a lot,0 +9009,"How to Freeze Fruits and Veggies +http://t.co/MET0mtpr3S",0 +9012,Homemade frozen yogurt pops? Have you had luck making them? http://t.co/YzaZF4CEOa http://t.co/X5RC5Nuamh,0 +9016,The '1pack 2pack 3pack' line is on fleek if Stretcher is fire.,0 +9018,Sweater Stretcher http://t.co/naTz5iPV1x http://t.co/leaEBy6cR2,0 +9020,___Stretcher in 5 min // Speaker Deck___ http://t.co/fBLNiFda1C,0 +9022,@invalid @Grazed @Towel @Stretcher @PLlolz @witter I can't stop,0 +9023,I should have known better than to think I'd get anything done after a 2 hour capoeira class. Stretcher-bearer!,0 +9025,@Grazed @invalid @Stretcher @Rexyy @Towel 'Ben favorited',0 +9026,He's being put on a stretcher ?? don't want to see that.,0 +9027,Lately I been stressing make me wanna put a fuck nigga on a stretcher!,0 +9028,Mxaaaa South Africans Just Can't Appreciate Effort #Stretcher Was Not That Bad Stop Hating,0 +9029,Keep shape your shoes ??#Amazon #foot #adjust #shape #shoe Mini Shoe Tree Stretcher Shaper Width Extender Adjustable http://t.co/8cPcz2xoHb,1 +9030,Stretcher in 5 min // Speaker Deck http://t.co/0YO2l38OZr,0 +9031,Lately I've been under pressure make me wanna put a fuck niqqa on a stretcher,0 +9033,Stretcher-bearers were soldiers whose job was to follow behind the advance and bring the wounded back into their... http://t.co/yeRcT4J244,0 +9034,Scary stuff in San Antonio. Stars head coach Dan Hughes was run into on the sidelines and fell over chairs onto the floor. Stretcher now out,1 +9038,How to restore vinyl siding and make it look new again http://t.co/rDxzsL5EAC http://t.co/QwijRRiYIf,0 +9039,@DareToTaha nah but thinking of getting a stretcher in my foreskin what u think?,0 +9040,@Stretcher @witter @Rexyy @Towel show me a picture of it,1 +9041,___Stretcher in 5 min // Speaker Deck___ http://t.co/7qPG80uD7v,0 +9042,1) 'Investigators say a Virgin Galactic spaceship crash was caused by structural failure after the co-pilot unlocked a braking system early',1 +9044,SpaceX Founder Musk: Structural Failure Took Down Falcon 9 http://t.co/LvIzO9CSSR,1 +9045,@lizbon @KidicalMassDC It's more of a structural breakdown. Or maybe a patience failure on their part.,0 +9047,Virgin galactic crash: early unlocking of brakes triggered structural failure http://t.co/Kp1hDchfNZ,1 +9051,VIDEO: Virgin Galactic crash: Brakes blamed: Investigators have said a Virgin Galactic spaceship crash was caused by structural failure___,1 +9052,Investigators have said a Virgin Galactic spaceship crash was caused by structural failure after the co-pilot ... http://t.co/imAWVMzs3A,1 +9053,Investigators say a fatal Virgin Galactic spaceship crash last year was caused by structural failure after the... http://t.co/FPrt7NwrOt,1 +9054,Rightways: Building structural integrity & failure: problems inspections repair http://t.co/vz1irH0Nmm via @rightwaystan,0 +9055,@SirTitan45 Mega mood swing on a 24 hr schedule. Isn't that how structural failure occurs?,0 +9057,The result of failure in correcting structural problems with West's financial markets collapse https://t.co/DvieABlOFz,1 +9058,3. excessive engine failure rate significant maintenance constantly emerging structural defects. Phew that's a lot I say.,0 +9059,Virgin galactic crash: early unlocking of brakes triggered structural failure - Irish Examiner http://t.co/ocMCvfDZkv,1 +9060,[CLIP] Top-down coercion - The structural weakness ensuring government failure http://t.co/gNORIjnSVa,0 +9061,Investigators say a fatal Virgin Galactic spaceship crash last year was caused by structural failure after the co___,1 +9062,We offer #preventative services such as cabling and bracing to stop structural failure of your trees. Call 905-877-8591 to book yours. #CTS,0 +9064,Rightways: Building structural integrity & failure: inspections damages defects testing repair http://t.co/vz1irH0Nmm via @rightwaystan,1 +9067,NTSB: Virgin Galactic's SpaceshipTwo crash due to structural failure when braking system unlocked early http://t.co/EYSbLYX6L6 via @KPCC @AP,1 +9071,Investigators have said a Virgin Galactic spaceship crash was caused by structural failure after the co-pilot ... http://t.co/PnhPLJHo8E,1 +9072,Investigators rule catastrophic structural failure resulted in 2014 ... http://t.co/AdZ8kbuRt7,1 +9074,Investigators have said a Virgin Galactic spaceship crash was caused by structural failure after the co-pilot ... http://t.co/WC69XAJIs4,1 +9075,Investigators have said a Virgin Galactic spaceship crash was caused by structural failure after the co-pilot unlocked a braking system,1 +9077,@sirtophamhat @SCynic1 @NafeezAhmed @jeremyduns and of course you don't have to melt the steel in order to cause structural failure.,0 +9078,CAP: 'the DRIVE Act represents the failure of Senate Republicans to address the structural shortfalls plaguing the Highway Trust Fund',0 +9079,Photo: Failure in structural integrity affects us all. Whether it is a barn raised upon a faulty concrete... http://t.co/cDxE5VMzOj,0 +9080,@sabcnewsroom sabotage!I rule out structural failure,0 +9081,'@CatoInstitute: The causes of federal failure are deeply structural and they will not be easily solved: http://t.co/H2XcaX4jbU',0 +9082,Slums are a manifestation state failure to provide housing to citizens. Illegality discourse confounds structural problems. #stopevictions,1 +9083,Investigators say a fatal Virgin Galactic spaceship crash last year was caused by structural failure after the... http://t.co/BgRAb7lK8D,1 +9084,@whvholst @leashless And this is a structural problem rather than just a failure of competence by traditional soc democratic parties.,0 +9085,"'Jet fuel cant melt steel' +'The structural failure is illogical' +'The second plane crashing into the building is fake' +'It was a bomb'",1 +9086,Investigators rule catastrophic structural failure resulted in 2014 ... http://t.co/QU1IUg3E9r,1 +9087,Virgin galactic crash: early unlocking of brakes triggered structural failure: The crash of a Virgin Galactic ... http://t.co/x3VqxdouVT,1 +9088,Investigators rule catastrophic structural failure resulted in 2014 Virg.. Related Articles: http://t.co/Cy1LFeNyV8,1 +9089,NTSB: Virgin Galactic crash caused by structural failure triggered when brakes unlocked early http://t.co/iKnyOk9zZr http://t.co/QCKqcX4Hw9,1 +9090,Investigators say a fatal Virgin Galactic spaceship crash last year was caused by structural failure after the co-pilot unlocked a braking,1 +9093,@FoxNewsInsider All Obama is doing is giving a false time schedule on Iran testing there first bomb Bomb = Nuclear Suicide Vest,0 +9094,If you ever think you running out of choices in life rembr there's that kid that has no choice but wear a suicide bomb vest,0 +9095,#Bestnaijamade: 16yr old PKK suicide bomber who detonated bomb in ... http://t.co/KSAwlYuX02 bestnaijamade bestnaijamade bestnaijamade be___,1 +9096,Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/1yB8SiZarG http://t.co/69iIzvyQYC,1 +9097,Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/uWfvWMjepU http://t.co/pxONlrqUsm,1 +9098,#Bestnaijamade: 16yr old PKK suicide bomber who detonated bomb in ... http://t.co/KSAwlYuX02 bestnaijamade bestnaijamade bestnaijamade be___,1 +9099,@AlfaPedia It might have come out ONLY too burst as a Bomb making him suicide bomber,1 +9100,Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/uJNRGFAnGj http://t.co/TmIpEgQyeV,1 +9101,Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/pOL92mn8YZ,1 +9102,...//..// whao.. Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/tuVbR4lEP3,1 +9103,Look at the previous battles. Citizens were committing suicide so to not be under American control. The bomb was the only way. @NBCNews,1 +9106,she's a suicide bomb,0 +9107,#Bestnaijamade: 16yr old PKK suicide bomber who detonated bomb in ... http://t.co/KSAwlYuX02 bestnaijamade bestnaijamade bestnaijamade be___,1 +9109,Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/IUh718KCy0 http://t.co/9pQyx4xOOL,1 +9110,See the 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench___ http://t.co/ZdiEodWbog via @MsOreo_ http://t.co/V6nyLVdPeD,1 +9112,//./../.. Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/Sj57BoKsiB -/,1 +9113,#Bestnaijamade: 16yr old PKK suicide bomber who detonated bomb in ... http://t.co/KSAwlYuX02 bestnaijamade bestnaijamade bestnaijamade be___,1 +9114,#Bestnaijamade: 16yr old PKK suicide bomber who detonated bomb in ... http://t.co/KSAwlYuX02 bestnaijamade bestnaijamade bestnaijamade be___,1 +9116,//./../.. Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/fqSk7QCawO -/,1 +9118,recap/ Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/6jzCEdaYRG,1 +9119,11 soldiers killed in ISIS suicide bomb in air base east Homs SYRIA NEWS | ZAMAN ALWSL - http://t.co/V8juC5eK1A #ISIS #Homs,1 +9120,16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/mGZslZz1wF,1 +9121,16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/5orTB8p51c,1 +9123,16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/n7Yst76ku3,1 +9124,Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/gnynJHnE6j http://t.co/1fuNEMes7M,1 +9126,wo Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/5v29w19tFX /'/'//,1 +9128,Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/FVXHoPdf3W,1 +9130,Iraq - Hashd Shaabi Theft ISIS Suicide Car bomb http://t.co/2AG9auABr3 #ISIS http://t.co/Qna4TUBnWh,1 +9131,ll//ll= Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/uoNEbAHH3h /,1 +9132,16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/mMkLapX2ok,1 +9134,#GRupdates Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released --> http://t.co/fqcDPhccg7,1 +9135,#Bestnaijamade: 16yr old PKK suicide bomber who detonated bomb in ... http://t.co/KSAwlYuX02 bestnaijamade bestnaijamade bestnaijamade be___,1 +9137,? 19th Day Since 17-Jul-2015 -- Nigeria: Suicide Bomb Attacks Killed 64 People; Blamed: Boko Haram [L.A. Times/AP] | http://t.co/O2cdKpSDfp,1 +9139,16yr old boy pic PKK suicide bomber who detonated bomb in Turkey Army trench http://t.co/H3SXfV5mtC http://t.co/IEWDreNauK,1 +9141,New post: Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/LWDcrPEhTN,1 +9142,Suicide bomber kills 15 in Saudi security site mosque http://t.co/ETyZY8GB2A #mercados,1 +9143,Suicide Bomber Kills 13 At Saudi Mosque http://t.co/oZ1DS3Xu0D #Saudi #Tripoli #Libya,1 +9144,Six people were killed Thursday when a Taliban suicide bomber detonated an explosives-rigged truck outside a police compound in eastern,1 +9145,ISIS claims responsibility for Saudi mosque suicide bombing http://t.co/Wpilp4mymf http://t.co/8NHD9iDaJs,1 +9146,Suicide bomber kills 15 in Saudi security site mosque http://t.co/AxhcLfErSU,1 +9148,#ISIS claims responsibility for Saudi suicide bombing http://t.co/jlMICJ6jE5,1 +9149,Islamic State claims suicide bombing at Saudi Arabian mosque: Ten members of emergency service... http://t.co/mpOaEFQl6k via @josephjett,1 +9151,#saudiarabia 13 confirmed dead as suicide bomber attacks Saudi Arabian mosque - The I... http://t.co/LwAnE9vupg - http://t.co/CpQguFZB28,1 +9154,Suicide bomber kills 15 in Saudi security site mosque - Reuters Canada: Reuters CanadaSuicide bomber kills 15 ... http://t.co/Ktd5IG9M5o,1 +9156,Suicide bomber kills 15 in Saudi security site mosque http://t.co/YyTKP1Z5kG via @Reuters,1 +9157,News: 'Islamic State claims suicide bombing at Saudi Arabian mosque' http://t.co/vIJfNHl630,1 +9159,"17 killed in S__ªArabia mosque suicide bombing + +A suicide bomber attacked a mosque in Aseer south-western Saudi... http://t.co/pMTQhiVsXX",1 +9161,Mosque bombing strikes Saudi special forces; at least 15 people dead: A suicide bomber struck a mosque i... http://t.co/gigW51IZpK #news,1 +9162,Suicide bomber 'hits Saudi mosque' http://t.co/kE6DjxAnmm,1 +9164,Suicide bomber detonates in Saudi Arabia mosque 17 reportedly killed http://t.co/xycKgxZv9s,1 +9166,Suicide bomber kills 15 in Saudi security site mosque - A suicide bomber killed at least 15 people in an attack on... http://t.co/FY0r9o7Xsl,1 +9169,Suicide bomber kills 15 in Saudi security site mosque - Reuters http://t.co/37DqvJHNCv,1 +9171,#?? #?? #??? #??? Suicide bomber kills 15 in Saudi security site mosque - Reuters http://t.co/LgpNe5HkaO,1 +9172,"@Abu_Baraa1 Suicide bomber targets Saudi mosque at least 13 dead - Suicide bomber targets Saudi mosque at least 13 dead +This is ridiculous",1 +9173,http://t.co/9k1tqsAarM Suicide bomber kills 15 in Saudi security site mosque - Reuters http://t.co/Ev3nX9scx3,1 +9174,#deai #??? #??? #??? Suicide bomber kills 15 in Saudi security site mosque - Reuters http://t.co/SqydkslFzp,1 +9175,Suicide Bomber Kills More Than a Dozen in Saudi Mosque: Saudi Arabia have started experiencing some terrorist ... http://t.co/PIoY1O54f4,1 +9177,Suicide Bomber Kills 13 At Saudiå_Mosque http://t.co/h99bHB29xt,1 +9179,Suicide bomber kills 15 in Saudi security site mosque http://t.co/axK9XNo6Yz #mercados,1 +9180,IS claims suicide bombing against Saudi police: An Islamic State group suicide bomber on Thursday detonated an... http://t.co/Dn5Buo7GSK,1 +9181,@bbclaurak Why is no one talking about the risk of a suicide bomber hiding amongst the migrants stowing aboard the Eurotunnel trains?,1 +9182,I liked a @YouTube video from @noahj456 http://t.co/xntO3jMMTS GTA 5 - SUICIDE STICKY BOMBER (GTA 5 Online Funny Moments),0 +9183,IS claims suicide bombing against Saudi police: RIYADH (AFP) - An Islamic State group suicide bomber on Thursd... http://t.co/IBypE1kaz5,1 +9185,Suicide Bomber Kills More Than a Dozen in Saudi Mosque: Saudi Arabia have started experiencing some terrorist ... http://t.co/GuAJ2t910b,1 +9187,Suicide bomber kills 15 in Saudi security site mosque #world #news,1 +9191,#?? #?? #??? #??? Suicide bomber kills 15 in Saudi security site mosque - Reuters http://t.co/txg7K2DO9v,1 +9192,I was taught at school in the 1970s that piracy slavery and suicide-bombing were purely historical. No one then expected them to re-occur,1 +9193,After a suicide bombing in Suru_¤ that killed 32 people Turkey launches airstrikes against ISIL and Kurdistan Workers' Party camps in Iraq.,1 +9194,@daisycuttertz @GROGParty @Tony_Burke He's been having lessons in suicide bombing from his imam!,1 +9195,@sonofbobBOB @Shimmyfab @trickxie usually I'd agree. Once the whole chopping heads off throwing gays off rooftops & suicide bombing start,1 +9197,"Suicide bombing for Da'esh is (rightly) despicable. Suicide bombing for Ocalan/ Marxism? + += They share 'our values'. https://t.co/Gs0km0vlgk",1 +9203,< 25 Dead In Kuwait Mosque Suicide Bombing Claimed By ISIS Offshoot on http://t.co/eTITgPSrUN,1 +9206,Suicide bombing is just the fear of dying alone,1 +9207,'Suicide bombing at [location named]...' #premonitions http://t.co/iIkSsJGBDn,1 +9208,@JewhadiTM It is almost amazing to think someone thought suicide bombing would actually be a good idea.,1 +9209,13 security personnel killed in Iraq suicide bombing | http://t.co/IbAZRHlSUr http://t.co/B6wWq2nYQI,1 +9210,Turkish troops killed in Kurdish militant 'suicide attack' http://t.co/wD7s6S0vci,1 +9211,Remembering Rebecca Roga 40 of the Philippines. murdered by Hamas terrorists in the suicide bombing of Egged bus No. 361,1 +9212,INFOGRAPHIC: At least 20 Turkish security officials killed in PKK and ISIS terror attacks since Suruc suicide bombing http://t.co/UvAOJzcYcZ,1 +9215,Malik Saadthe best police officer produced by KPdied in a suicide bombing on 27 Jan 2007he was more than a brother to me. #KPPolice.,1 +9216,bout to go suicide bombing http://t.co/ZoIPkPBD6o,1 +9217,< Suicide Bombing Of European Union Car Kills At Least Three In Kabul on http://t.co/o6aRmcgVbS,1 +9218,@RayquazaErk There are Christian terrorists to be sure but I don't suicide bombing is employed often as it is in Islamic groups.,1 +9219,Turkish troops killed in Kurdish militant 'suicide attack' http://t.co/nVP6wrKL1E,1 +9220,Turkish troops killed in Kurdish militant 'suicide attack' http://t.co/7cIbxls55f,1 +9221,Alleged driver in #Kuwait attack 'joined Daesh just a day before the June 26 suicide bombing' he confesses in court http://t.co/Tmz6X1N2gQ,1 +9222,@Haji_Hunter762 @MiddleEastEye maybe some muzzies will spontaneously combust! It's like Allah suicide bombing them as payback ????????,1 +9223,The number of security officials killed by PKK in terror attacks since the Suruc suicide bombing has reached 22 https://t.co/OpJwuNUvG8,1 +9225,'Suicide bombing at [location named]...' #premonitions http://t.co/iIkSsJGBDn,1 +9226,Erdogan's Bloody Gambit: on July 20 a suicide bombing in Turkey took the lives of 31 socialists in Suru_¤ http://t.co/z6xAUIDRXu @Shareaholic,1 +9230,Kurd Suicide Attack Kills 2 Turkish Soldiers http://t.co/GHGHQm9e6d,1 +9231,@NBCPolitics RUSSIA AND THAT BACK FIRED NOW 2015 look what happened 911bombing of marine barracks suicide bombers attacks on world sites,1 +9232,meek mill should join isis since he loves suicide-bombing his career for no good reason,0 +9233,"Imagine a school where suicide bombing Is being taught the teachers would say to the students... + +'Please pay... http://t.co/zfiVVxYDZY",1 +9235,Turkish troops killed in Kurdish militant 'suicide attack' http://t.co/7old5MJWph,1 +9237,9) 'Without the bombing you would have to do hara-kiri you know commit suicide'. http://t.co/UO0aQk9KR8 Hiroshima 70 years ago.,1 +9238,Remembering Mordechai Yehuda Friedman 24 of Ramat Beit Shemesh; murdered by Hamas terrorists in the suicide bombing of Egged bus No. 361,1 +9239,Remembering Marlene Menahem 22 of Moshav Safsufa; murdered by Hamas terrorists in the suicide bombing of Egged bus No. 361,1 +9240,1 of the major reason of suicide bombing is the absence of sexual interactions.,1 +9242,why wasn't this warship sunk? CNN: First on CNN: Iranian warship points weapon at U.S. helicopter official says http://t.co/hfZk09McEN,1 +9244,literally just sunk in we got backty school in exactly a week :S :S :s,0 +9246,I peeped you frontin' I was in the jeepå_sunk in the seat tinted with heat beats bumpin'.,0 +9247,British diver Neil Anthony Fears found dead by the wreck of a steamship - Daily Mail http://t.co/QP3GVvfoFq,1 +9248,Near them on the sand half sunk a shattered visage lies... http://t.co/0kCCG1BT06,1 +9249,Aquarium Ornament Wreck Sailing Boat Sunk Ship Destroyer Fish Tank Cave Decor - Full read ___ http://t.co/kNCm9jC8i9 http://t.co/swviAZSPHk,1 +9250,@aphyr I__ªve been following you this long___ Sunk cost fallacy or somethin__ª,0 +9251,descended or sunk however it may be to the shadowed land beyond the crest of a striking cobra landing harshly upon his back; torch and,0 +9252,"So if I capsize on your thighs high tide B-5 you sunk my battleship +>",0 +9253,has NOT sunk in that i leave for school in a month,0 +9254,It still hasn't sunk in that i will never see my nan again how has it been 2 months already ??,0 +9255,Japan FUSO Class Battleship YAMASHIRO Naval Cover 1999 PHOTO Cachet SUNK WWII http://t.co/Aq5ZliM7l4 http://t.co/FvR9jDQ71a,1 +9257,'Blaaaaaaa' he said as he sunk his face into your stomach making it vibrate @ResoluteVanity,0 +9259,Hasn't quite sunk in that I saw Johnny Marr and Primal Scream all in the space of a few hours on Sunday,0 +9260,@CodeMeW Were you opening regular or master packs? RT: so___ sunk___1 mil credits into light side___ didn't pull___one___you sure about those odds?,0 +9262,Yup. Still hasn't sunk in. ?? https://t.co/Ii2SpVP89b,0 +9263,@SaintRobinho86 someone has to be at the bottom of every league. Tonight clearly demonstrated why the Lions are where they are - sunk!,1 +9264,It still hasn't sunk in that I've actually met my Idol ????,0 +9265,@UntamedDirewolf 'I... Wow. Alright.' Sansa shook her head and blinked rapidly as the new information sunk in. 'I really don't know what--,0 +9266,@BenAffleck i respected you and liked you for your talent. i guess i stil do but as a human being you've sunk low Mr.Affleck!,0 +9267,Once upon a time the fact that I decided to go to university 7 hours away actually sunk in,0 +9268,#ArrestpastorNganga it so worrying 2 see how some police officers in Kenya have sunk to low to the point of collaborating with pick pockets,0 +9269,@time4me_sews I know! It still hasn't quite sunk in :D,0 +9270,The feeling of lonelyness has sunk into me strange depressing feeling.,0 +9271,We may be just a week away from school but I don't think the idea of me being a senior has really sunk in yet,0 +9272,im getting a car wow it hasn't sunk in,0 +9273,FOOTBALL IS BACK THIS WEEKEND ITS JUST SUNK IN ??????,0 +9275,One Direction is taking a break after this next album. My heart has sunk it hurts and I'm very upset. They deserve a break. My heart hurts,0 +9276,Aquarium Ornament Wreck Sailing Boat Sunk Ship Destroyer Fish Tank Cave Decor - Full read ___ http://t.co/nosA8JJjiN http://t.co/WUKvdavUJu,1 +9277,The court__ªs reputation and prestige has sunk ever lower and Prosser who has served on the court since 1998 has... http://t.co/H09nYdbzoV,0 +9278,<< his lip as he sunk into the bed his arms crossed behind his head as he watched his Captain do a number on his body. @ResoluteShield,0 +9280,@ShekharGupta @mihirssharma high time TV channels realised what levels they have sunk for TRP and ads.it has become a mockery,0 +9281,@bonhomme37 wouldn't that have sunk the sub?,0 +9283,Its like I never left. I just sunk to the background,0 +9286,@UnrealTouch fuck sake john Jesus my heart just sunk.,1 +9287,@silverstar58200 I felt bad for Romero. He cared but mental stuff sunk his career. I worry same thing will happen to Hutch. #BlueJays,0 +9288,Everything has sunk in except the fact that I am actually moving to the state of Colorado tomorrow. Been dreaming of it since I was a kid.??,0 +9289,Damn there's really no MLK center that hasn't sunk in yet,0 +9290,The Seven Seas - Wreck of the Giannis D. sunk after hitting a submerged reef in the___ http://t.co/Gn3WHNSFIb http://t.co/fWpOF5TwoC,1 +9292,Hank Williams Jr. - 'Country Boys Can Survive' (Official Music Video) https://t.co/YOu7wn9xVs via @YouTube,0 +9293,Parental experience may help coral offspring survive climate change: Preconditioning adult corals to increased... http://t.co/N9c3i9v8gO,0 +9294,@lucypalladino and I don't have any classes together and I'm not sure if I'll be able to survive,0 +9296,I have my final tomorrow ?????? . Ish is stressful but I'll survive,0 +9297,AHH forgot my headphones how am I supposed to survive a day without music AYHHHHHDJJFJRJJRDJJEKS,0 +9298,10 Ways To Survive and Escape Martial Law | World http://t.co/BiuEY7bUtS http://t.co/JWQYbe4ep1,0 +9300,How to Survive in the Markets http://t.co/LnzI7o166Y #oil #investing #money #trading #forex #gold #silver #business http://t.co/TmpFWjPI6I,0 +9301,So much shit has happened today wtf idk how I survive thruuu it all,1 +9303,Ended today's staff meeting with the teacher version of 'I Will Survive' #tootrue http://t.co/mCWD37IOF9,0 +9307,how will I survive without dorrian,0 +9308,If I survive tonight. I wouldn't change one thing. ??,0 +9309,3 Things All Affiliate Marketers Need To Survive Online - Every affiliate marketer is always... http://t.co/kPMYqUJSUE,0 +9311,"DON'T MAKE FUN OF THEM FOR TRYING +TO SURVIVE IN SUCH A RACIST ELITIST COUNTRY",0 +9313,We learn and grow and become stronger as we face and survive the trials through which we must pass. #ThomasSMonson #LDS #Mormon,0 +9314,A country Chim can survive!,1 +9315,I Will Survive by Gloria Gaynor (with Oktaviana Devi) ___ https://t.co/HUkJZ1wT36,1 +9317,"Escaping her captors was the 1st step. Now she must survive a deadly storm & a wild-looking recluse. +BETRAYED +http://t.co/0Q040STkCV +#books",1 +9319,In a dream you saw a way to survive and you were full of joy.,0 +9320,16 Stylishly Unique Houses That Might Help You Survive the Zombie Apocalypse | http://t.co/AU3DBCI7nf http://t.co/BOvJRF62T7,0 +9321,It's going on three years that we have been separated. Sometimes you have to let a man know you will leave him. & that u can survive without,0 +9322,Now realized I honestly can't survive without these glasses now lol,0 +9324,Help me survive the zombie apocalypse at the Zombie Fun Run on November 15th. https://t.co/kgSwhSr7Mn #teamsurvivors #zombiefunrun2014,0 +9325,@mochichiiiii @hikagezero IT'S IMPOSSIBLE FOR ME TOO WW like i can't survive a day without meat wew,0 +9329,:: Survive??,0 +9330,You can't fight fate and you can't survive alone... I can't help but notice that almost seems like a definition of who I am...,0 +9332,@DDNewsLive @NitishKumar and @ArvindKejriwal can't survive without referring @@narendramodi . Without Mr Modi they are BIG ZEROS,0 +9333,@adriennetomah how did people survive like that?!,0 +9334,Like it affects every level of life you're expecting me to buy everything and still survive with my limited pocket money,0 +9337,#Autoinsurance industry clueless on driverless cars : #healthinsurance http://t.co/YdEtWgRibk,1 +9338,If I survive I'll see you tomorrow,0 +9339,#Autoinsurance industry clueless on #driverlesscars : #Markets : #Money Times http://t.co/YdEtWgRibk,0 +9341,Suicide of a Superpower : Will America Survive To 2025? by Patrick J. Buchana... http://t.co/eMTwirknyq http://t.co/M9K08OaZve,0 +9342,Survived another tube strike with the last person at office reaching home. We are getting better at navigating strikes!,0 +9343,"Update: I survived. No canoe. + +May have been the fastest feed out in history though.",1 +9344,@TheDailyShow Mahalo nui loa for making my 20s. My generation could not have survived the (W.) Bush years without you. #JonVoyage #holomua,0 +9345,@thoutaylorbrown I feel like accidents are just drawn to you but I'm happy you've survived all of them. Hope you're okay!!!,0 +9346,Even when I was a kid haha super late but folks used to bash me for that shit I understand he survived cancer but he still cheated,0 +9347,@taaylordarr THANK YOU!!! I survived ????,0 +9348,Meet the man who survived both Hiroshima and Nagasaki http://t.co/PNSsIa5e46 http://t.co/LSVsYSpdxX,1 +9350,Electricity cant stop scofield. Nigga survived a hotbox in SONA,0 +9351,It's a miracle that mankind survived 70 years since one of the worst days in its history. All bets are off if it will survive the next 70.,1 +9355,@TheSmallClark 'He'll kill me instead if he survived the shot. I don't exactly know. I fled the scene and pulled the trigger with---,1 +9360,Not even the Berlin wall survived the 80's I daubt they'll ever be a generation like the 80's @ChubyChux: ... http://t.co/71VdjHJnwV,0 +9361,@rjkrraj @KarnakaranK @vimvith whether pressure or not NON wud not survived even if thalapathi didn't release,1 +9362,Not one character in the final destination series has ever survived ??,0 +9363,@Michael5SOS FUCKING LIVE HERE IM SURPRISED IVE SURVIVED THIS LONG,0 +9365,Just burst out in actual tears of joy when Cain survived #SummerFate @emmerdale,0 +9367,Nigerian boxer ?who survived sessions with Klitschko becomes famous on YouTube http://t.co/JSZZQsT3XS,0 +9368,"[+] + +Such a lonely day +And it's mine +It's a day that I'm glad I survived ... + +??",0 +9371,How would I have survived without the mute option on whatsapp!,0 +9372,The 'twins' survived their first day of high school!! They're gonna have a great year!! #makinmemories 2015 http://t.co/gM8p0Bd8Mt,0 +9373,Well me and dad survived my driving ????????,0 +9374,By the grace of GOD I survived the 2am shift and im not that tired.,0 +9375,'Planted 390 years ago' it was moved to U-S. This Bonsai Survived Hiroshima But Its Story Was Nearly Lost http://t.co/jID4RO34gb via @NatGeo,1 +9376,Only one man Tsutomi Yamaguchi is said to have survived both atomic bomb blasts at #Hiroshima and #Nagasaki. #OTD http://t.co/DaalPeNZp0,1 +9377,I Survived A 12-Step Program for Sex Addiction - http://t.co/xsX26oo16s,0 +9378,"Today your life could change forever - #Chronicillness can't be avoided - It can be survived + +Join #MyLifeStory >>> http://t.co/FYJWjDkM5I",0 +9380,Cute & all livin' the life then you zoom in on one's face and you have a meme ready: 'I've seen the Gates of Hell and survived',0 +9382,Well said Sir. My cousin was a POW. Brutally treated. Survived but never talked about it. The bombing was justified https://t.co/SuDkK1wEEZ,1 +9384,@PixelJanosz @Angelheartnight I remember now. There was a British man who survived both also. Can't remember his name though.,0 +9385,RT THR 'RT THRArchives: 1928: When Leo the MGM Lion Survived a Plane Crash #TBT http://t.co/Wpkl2qNiQW http://t.co/BD52FxDvhQ',1 +9386,@duchovbutt @Starbuck_Scully @MadMakNY @davidduchovny yeah we survived 9 seasons and 2 movies. Let's hope for the good. There's hope ??????,0 +9387,There are people who plotted against me that are still wondering how I survived,0 +9388,Survived Spanish!! @ Sweet Ritual https://t.co/inIqzfyioi,0 +9391,___@BBCWomansHour: Setsuko Thurlow survived the #Hiroshima bomb. This is her story: http://t.co/cvjYML7KrM http://t.co/FpH01U3eIi___,1 +9393,The Cafe Run by Acid Attack Survivors in #India http://t.co/qmiF0bLwOa http://t.co/l6PIf3LpEn,1 +9395,Dying with debt can be costly for survivors,0 +9396,Hiroshima survivors fight nuclear industry in Brazil _¢?? video http://t.co/GLZmGBM7w0,1 +9397,"Tricycle toddler atomic bomb +http://t.co/ljeRYHItwh In a fraction of a second Our decisions change the landscape of humanity.",1 +9401,Patient-reported outcomes in long-term survivors of metastatic colorectal cancer - British Journal of Surgery http://t.co/5Yl4DC1Tqt,1 +9402,Dear @POTUS In the name of humanityI apologized to #Hiroshima Survivors.R u ready to do so?#Japan #nuclearweapons http://t.co/TWykzN4rlC,1 +9403,#coppednews Haunting memories drawn by survivors http://t.co/Wx11d69gEZ,1 +9404,"RT @kotowsa: South Sudan__ªs war on women: survivors say rape has become 'just a normal thing' +https://t.co/MexwoHd3TG http://t.co/gB46FiD2wE",1 +9405,Haunting memories drawn by survivors http://t.co/pRAro2OWia,1 +9406,Hiroshima survivors fight nuclear industry in Brazil ___ video http://t.co/E7SHtYLbnL,1 +9408,Survivors remorse is good,0 +9410,"#WorldNews + Fears over missing migrants in Med - BBC News - Home: +Rescuers search for survivors after a boat carr.. http://t.co/iJoBZ3MZp0",1 +9411,@MithiTennis @CrackedGem Which is why I want her to be better - which means death awaits her. Most of us aren't heroes we are survivors,0 +9414,4 THOSE WHO CARE ABOUT SIBLING ABUSE SURVIVORS join the new family tree: http://t.co/LQD1WEfpQd http://t.co/GgnbVZoHWu,0 +9416,Breaking news: Haunting memories drawn by survivors http://t.co/PCjBvrs7xw,1 +9417,Molecularly targeted cancer therapy for his #LungCancer gave Rocky his life back. http://t.co/TwI3pYm7Us http://t.co/qT8JMD9pI1,0 +9418,Violators of the new improved Reddit will be shot into the sun. Survivors will be hurled into a black hole and then nuked.,0 +9420,As Anniversary Nears Atomic Bomb Survivors Speak Out Against Nuclear Power - http://t.co/Uo8GrDAuAT,1 +9421,Hiroshima survivors fight nuclear industry in Brazil ___ video http://t.co/uvM975yha2,1 +9422,The second part which focuses on the survivors is really difficult to watch but at the same time is really powerful.,1 +9425,@Arovolturi3000 survived because of magic she is searching the wood for survivors outside of London,1 +9430,@FedPorn I feel your pain. Survivors will look back on this period as most absurd in human history. Satire indistinguishable from reality.,1 +9431,__÷Faceless body belonged to my sister__ª: #Hiroshima #Nagasaki #nuke survivors recall horrors 70 years on ___ RT News http://t.co/918EQmTkrL,1 +9432,Survivors of Shanghai Ghetto reunite after 70 years - http://t.co/1Ki8LgVAy4 #Shanghai #China #??,0 +9434,People with netflix there's a really good documentary about Hiroshima narrated by John Hurt. 2 Parter that interviews Pilots + Survivors.,0 +9435,@LawfulSurvivor T-Dog had been holed up in an apartment store with several other survivors Glenn Morales Andrea Jacqui and Merle.--,1 +9436,Stemming from my #Cubs talk- the team rosters 2 cancer survivors in @ARizzo44 & @JLester34...@Cubs fans: help another http://t.co/XGnjgLE9eQ,1 +9438,Haunting memories drawn by survivors http://t.co/WJ7UjFs8Fd,1 +9439,Haley Lu Richardson Fights for Water in The Last Survivors (Review) http://t.co/oObSCFOKtQ,0 +9441,Remembrance http://t.co/ii4EwE1QIr #Hiroshima http://t.co/H3vUsqzyQo,1 +9445,"Truth... +https://t.co/h6amECX5K7 +#News +#BBC +#CNN +#Islam +#Truth +#god +#ISIS +#terrorism +#Quran +#Lies http://t.co/B8iWRdxcm0",0 +9446,News786-UK Islamist Cleric Anjem Choudary Charged Under Terrorism Act: http://t.co/u7bBeNXWYK,1 +9448,@RobPulseNews @huyovoeTripolye Phillips should be charged for assisting terrorism. LDNR-terrorists' organizations. http://t.co/XwnJYsV9V9,1 +9449,"Truth... +https://t.co/nXS3Z1kxiD +#News +#BBC +#CNN +#Islam +#Truth +#god +#ISIS +#terrorism +#Quran +#Lies http://t.co/UDKMAdKuzY",1 +9450,Preacher faces UK terrorism charges http://t.co/daPlllFuqK,1 +9451,Woman sneaks into airplane cockpit; terrorism not suspected http://t.co/1W58Ehv9S1 http://t.co/p8Ih0hni3l,1 +9452,NC Senate follows House on legal action on terrorism damages - Winston http://t.co/2Y5MoRpugt,1 +9453,#Politics Democracy__ªs hatred for hate: ___ Dawabsha threaten to erode Israeli democracy. Homegrown terrorism ha... http://t.co/q8n5Tn8WME,1 +9454,"Truth... +https://t.co/beJfTYgJIL +#News +#BBC +#CNN +#Islam +#Truth +#god +#ISIS +#terrorism +#Quran +#Lies http://t.co/jlCZiDZ7Vu",0 +9455,OMEGA MEN Writer Explores Terrorism Religion In an 'Epic KYLE RAYNER Story' https://t.co/utc7pdIdfo via @Newsarama,0 +9456,Cross-border terrorism: Pakistan caught red-handed again http://t.co/uDj50J3MV4,1 +9457,CTD arrest three vital criminals from Orangi: KARACHI: Counter-Terrorism Department (CTD) of the Sindh Police ... http://t.co/Le4brduau9,1 +9458,Still domestic terrorism https://t.co/GevRMBVznB,1 +9460,DireTube Information ___ Egypt Cyprus and Greece agreed to fightå_terrorism http://t.co/V6IjxCCD2I http://t.co/YSXhFWMGOD,1 +9463,"Does this apply to Muslims/terrorism? +Or Catholicism/homophobia?",0 +9465,Perspectives on Terrorism - Understanding Jihadi Proto-States: http://t.co/d5h4jif1y3,1 +9466,DHS Refuses to Call Chattanooga __÷Islamic Terrorism__ª out of respect for MUSLIMS ... http://t.co/u8RGB51d22 via @po_st http://t.co/2tnu95VGFE,1 +9467,Pakistan's Supreme Court rules to allow military trials for suspects in terrorism cases http://t.co/ajpbdCalew,1 +9468,Domestic terrorism. No ifs ands or buts about it. YOU CREATED THIS @GOP http://t.co/hFgjgFGfeL,0 +9469,A good piece on Israeli incitement and Jewish terrorism by Beinart: http://t.co/OT2OOOEdts,1 +9470,In #islam saving a person is equal in reward to saving all humans! Islam is the opposite of terrorism!,0 +9471,OMEGA MEN Writer Explores Terrorism Religion In an 'Epic KYLE RAYNER Story' http://t.co/Hr88CWxPGz #Newsarama,0 +9472,In #islam saving a person is equal in reward to saving all humans! Islam is the opposite of terrorism!,1 +9474,@AdamNibloe Arrest report: Affiliation with Terrorism,1 +9480,"Truth... +https://t.co/2Y4RGob7pj +#News +#BBC +#CNN +#Islam +#Truth +#god +#ISIS +#terrorism +#Quran +#Lies http://t.co/mVes6VsSyN",1 +9481,Really cannot condemn an entire group based on the actions of a few.. A heart-warming unity against terrorism.. http://t.co/HHPvPaEL4n,1 +9482,"Truth... +https://t.co/p5ZIcjUdXO +#News +#BBC +#CNN +#Islam +#Truth +#god +#ISIS +#terrorism +#Quran +#Lies http://t.co/kZhB8zX6YC",1 +9484,@KurtSchlichter He's already done it by negotiating with the #1 state of terrorism in the World. What was his hurry in trying to get a deal,1 +9485,http://t.co/EQjCpWILVn: Articles In Saudi Press Reject Russian Initiative For Regional Alliance With Assad Regime To Fight Terrorism,0 +9486,"Truth... +https://t.co/k44tL3rfMy +#News +#BBC +#CNN +#Islam +#Truth +#god +#ISIS +#terrorism +#Quran +#Lies http://t.co/ipT0hoNoTI",0 +9488,Pakistan Supreme Court endorses military courts for terrorism cases http://t.co/sZeapuEuvy,1 +9489,Online Homeland security: An Appraisal of #Pakistan__ªs #Anti-Terrorism #Act https://t.co/mRoSPd9878,0 +9490,"Truth... +https://t.co/4ZQrsAQrRT +#News +#BBC +#CNN +#Islam +#Truth +#god +#ISIS +#terrorism +#Quran +#Lies http://t.co/6ar3UKvsxw",1 +9491,Cross-border terrorism: Pakistan caught red-handed again - The Times of India http://t.co/uiqsfgZoOx,1 +9492,Captured terrorist Naved not registered as our citizen: Pakistan http://t.co/0FS9kSV5xK,1 +9493,Natural lubrication !!!!!!!!!!!#MetroFmTalk,0 +9495,@OfficialMqm you are terrorist,0 +9496,Signing a petition to seek mercy on a death punishment for a convivted terrorist is a job well done in India. But asking a foreign govt 1/n,1 +9499,Here__ªs how media in Pakistan covered the capture of terrorist Mohammed Naved http://t.co/3MtWh0jJns,1 +9500,Update on Pulwama encounter that began earlier today: 2nd terrorist killed by security forces. Security forces... http://t.co/m5RjekVDDp,1 +9501,Here__ªs how media in Pakistan covered the capture of terrorist Mohammed Naved http://t.co/f7WqpCEkg2,0 +9502,Fresh encounter in Pulwama of J&amp;K one terrorist killed others holed up inside a house http://t.co/oEf123l5Rc,1 +9503,'The Terrorist Tried to Get Out of the Car; I Shot Him' http://t.co/VSoxKbt6Nq,1 +9504,Me pulling over and fighting the hoes that called Zayn a terrorist http://t.co/FY30fV0Qbx,1 +9505,Udhampur terror attack: NIA takes over probe Pakistani terrorist quizzed; Pak denies link http://t.co/ogZJOkd7Sv #Elections #AcheDin #___,1 +9507,@narendramodi @bjpsamvad Hiroshima is now beautiful on the ashes of the old. So could be a terrorist Pakistan.,1 +9508,Seek help warra #MetroFmTalk,0 +9509,@rahulkanwal why Jammu is soft target for terrorist,1 +9510,Anyone missing their license plate? Two stolen ones found on terrorist's car... http://t.co/CWGCciw3V6,1 +9511,Pakistan Disowned Kasab Now Disowns Naved: State of Denial?: Naved the terrorist captured alive after an___ http://t.co/HGDrK81sN4,1 +9513,The Pak terrorist who has been caught alive must be tried fast not delayed as KasabWe must send very hard message to Pak instead of Biryani,1 +9519,@ShipsXAnchors IDEK HOW IS THERE PEOPLE WHO ACTUALLY THINK HE'S A TERRORIST,0 +9520,You May Know Me from Such Roles as Terrorist #4 http://t.co/xImPncZXtH,0 +9522,@SwiftyCommissh @TaoistInsight @ImmortalTech Some jewish ppl agree tht Israel is a bully nd a terrorist state killin palestine kids nd women,1 +9524,@BarackObama Senator John McCain__ªs Whoops Moment : Photographed Chilling With ISIS___ http://t.co/JNrOMsE1Z2 @acmilan http://t.co/w6Yu7Qs4CV,0 +9525,"RT AbbsWinston: #Zionist #Terrorist Demolish Tire Repair Shop Structure in #Bethlehem +http://t.co/ph2xLI8nVe http://t.co/22fuxHn7El",1 +9526,Three Israeli soldiers wounded in West Bank terrorist attack - Haaretz http://t.co/u4gSBNU8wc,1 +9529,Three Israeli soldiers wounded in West Bank terrorist attack via /r/worldnews http://t.co/su4ZVWADj7,1 +9531,RT AbbsWinston: #Zionist #Terrorist kidnapped 15 #Palestinians in overnight terror on Palestinian Villages ___ http://t.co/J5mKcbKcov,1 +9533,#UdhampurAgain 2 terrorist shot dead.. #Udhampur,1 +9537,@lovemywife1983 @FoxNews like shedid on 9/11 4the public facts I am not a journalist the Bush's washard on stopping more than just terrorist,1 +9538,Isn't it ironic that on the anniversary of Hiroshima our current President is helping a terrorist nation acquire nuclear weapons. #TRAITOR,1 +9539,RT AbbsWinston: #Zionist #Terrorist kidnapped 15 #Palestinians in overnight terror on Palestinian Villages ___ http://t.co/J5mKcbKcov,1 +9540,Three Israeli soldiers wounded in West Bank terrorist attack via /r/worldnews http://t.co/9TyucdWh3g,1 +9541,Don't say @ALIPAPER: women got problems this #keepingtheviginaclean thing is very interesting n less expensive #metrofmtalk',0 +9542,Build and share your own custom applications all within @ThreatConnect w/ TC Exchange http://t.co/hRL4XNJ9K7 #infosec #DFIR #ThreatIntel,0 +9547,Is this the end of Australia__ªs best burger? http://t.co/te5tDLyIMN via @newscomauHQ,0 +9548,@KingGerudo_ to the largest moblin's he'd leaving the biggest one for Red and fired. With one strike already the threat was reduced -->,0 +9550,WHITE AMERICANS ARE THE BIGGEST THREAT IN THE WORLD. DUH https://t.co/7PaOvYzTtw,0 +9551,The few I warned about .. Were just as I expected.. They are a threat to his soul,0 +9552,"Generational __÷British schism__ª over privacy threat of drones http://t.co/dqtMTPqmBR + #drones #privacy http://t.co/dMsnYPtscY",0 +9553,Get access to the most extensive sources of threat information right out of the box. Then easily add your own internal intelligence. #BHUSA,0 +9554,If a nigga was a threat then that boy ah be thru ????,0 +9555,Purple Heart Vet Finds Jihad Threat on His Car at Mall: ___All of you Islamaphobe vets... http://t.co/hWnyXXKczz,0 +9556,@dmassa5 Definite triple crown threat. Him and Harper both.,1 +9557,Meek Mill responds to Drake__ªs OVO Fest set with wedgie threat http://t.co/qqSKYbARNg,1 +9559,Illusoria Icarus nowplaying check out http://t.co/E2WgREIcmZ,0 +9562,Am I hearing thunder or trucks?,0 +9563,The worst part is seeing lightning and trying to guess when the thunder will crack,0 +9564,This thunder is beautiful,0 +9565,#PlayingNow #BLOODBOUND Seven Angels Media Stream http://t.co/dlY6rUuSqK,0 +9566,@KristyLeeMusic brings her Alabama thunder back to the Attic SEPTEMBER 26! Tickets: https://t.co/B7ZwEVsrGO,0 +9567,@DevinJoslyn thunder and lighting ????,0 +9568,Okay maybe not as extreme as thunder and lightning but pretty much all other types! #FIFA16 https://t.co/ETuuYISLHw,0 +9569,@ATL_Events and thunder :-o,0 +9571,Random wind gust just came through #Gander. Probably some convection outflow. Haven't heard any thunder yet but a few spits of rain #nlwx,1 +9572,@LightUmUpBeast never watched pres think hes a dick like thunder though,0 +9574,@OriginalFunko @Spencers THUNDER BUDDYS!!!! THUNDER BUDDYS!!!!,1 +9575,@OriginalFunko @Spencers THUNDER BUDDYS!,0 +9576,Waldo Thunder 12U Cooperstown Dreams Park 2015 dedicated to #JoeStrong #GoKitGo http://t.co/eSK4yvzvaP,0 +9577,The one thing I like about here is thunder .. Heheh,0 +9579,Super loud thunder woke me up from my very nice nap,0 +9580,Heavy rain frequent thunder and gusty winds moving into parts of Uptown and Midtown. http://t.co/KQJevYqzLV - CN http://t.co/HmWhob7prs,1 +9581,Ebay Snipe RT? http://t.co/SlQnph34Nt Lego Power Miners Set 8960 Thunder Driller Boxed. ?Please Favorite & Share,0 +9582,Thunder lightening torrential rain and a power cut!,1 +9583,THIS IS RELAXING! #thunder #SoothMySlumber #WATERMELOANN #populardemand w/ @Soak... (Vine by @thewebbeffect19) https://t.co/F0QIRS5lJA,0 +9585,The Flash And The Thunder by WC Quick on Amazon Kindle and soon in PRINT at Amazon Books via Create... http://t.co/oS1WjRvx5c via @weebly,0 +9586,The thunder shook my house woke my sister and made car alarms go off ????,1 +9589,I hear lightening and see thunder,1 +9590,I was over here dreaming peacefully then that loud ass thunder wanted to scare me. ??,1 +9591,My brother is crying cause the thunder lmao,0 +9592,That was the l9udest thunder I've ever heard,0 +9593,@AsimTanvir @NemesisK_PES @KevR_7 Snow? Thunder and lightning?,1 +9594,thunder is legit,0 +9595,"Oh shit its rly sheeting +Aaaaaand there's the thunder and lightning +I missed summer storms",1 +9596,My mama scared of the thunder ????,0 +9597,"Please please u gotta listen to @leonalewis # essenceOfMe and thunder it's major +#she's #back ????????",0 +9598,suddenly it's off & on gloomy & thunder so loud it shakes the windows? Not ever on the Bay Area. Miss me w/that lol http://t.co/x4eCGGvnSN,1 +9600,#thunder outside my house this afternoon #gawx ??????????????????,1 +9602,L B #Oklahoma #Thunder DURANT NBA ADIDAS OKLAHOMA CITY THUNDER YOUTH LARGE SWINGMAN JERSEY RETAIL $75 #NBA #Durant http://t.co/T81oayjoWC,0 +9603,Did I just hear thunder? ??????,0 +9605,Thunder???,0 +9606,I love the sound of thunder rumbling across the mountains.,0 +9607,@HaydnExists so glad i saved them all at once then didn__ªt want you stealing my thunder :P,1 +9608,Severe Thunderstorm Warning for Oklahoma County until 10:30pm. Radar here: http://t.co/2HV2y2M2oZ #okwx,1 +9609,NIggas playing in the thunderstorm.. 'HOPE THE LORR BLESS EM' ????,1 +9610,Advice from Noah: Dont go running in a thunderstorm,0 +9611,#usNWSgov Special Weather Statement issued August 05 at 10:40PM EDT by NWS: ...STRONG THUNDERSTORM WILL IMPACT... http://t.co/TQ1rUQD4LG,1 +9612,Severe Thunderstorm Warning issued August 05 at 9:31PM CDT until August 05 at 10:15PM CDT by NWS http://t.co/h9i6moZAsK,1 +9613,OUN cancels Severe Thunderstorm Warning for Lincoln Logan Oklahoma [OK] http://t.co/bTi8JAMFiu #WX,1 +9616,#usNWSgov Severe Weather Statement issued August 05 at 10:38PM EDT by NWS: ...THE SEVERE THUNDERSTORM WARNING ... http://t.co/EpzgG4uqJI,1 +9617,@museawayfic @beenghosting @xylodemon ok ok what if I make the close quarters an abandoned cabin in the woods IN A THUNDERSTORM,1 +9618,@RachelRofe tired it' 5:36 am. Woke up to a thunderstorm lightning and rain. How are you?,0 +9619,Beautiful lightning as seen from plane window http://t.co/5CwUyLnFUm http://t.co/1tyYqFz13D,0 +9620,NWS has Continued a Severe Thunderstorm Warning for Oklahoma-OK until 10:30 PM,1 +9625,THE NATIONAL WEATHER SERVICE IN LITTLE ROCK HAS ISSUED A * SEVERE THUNDERSTORM WARNING FOR... VAN BUREN COUNTY IN ___ http://t.co/KJsvW06GBV,1 +9627,iNWS Alert SPSGSP from 8/5/2015 10:40 PM to 11:15 PM EDT for Pickens County: STRONG THUNDERSTORM WILL IMPACT... http://t.co/LdcwKyuaFf,1 +9629,Severe Thunderstorm Warnings have been cancelled in central Oklahoma. Still expect 50 mph winds penny sized hail,1 +9632,"Conditions for Paris FR at 4:00 am CEST: Current Conditions: +Fair 68 FForecast: +Thu - Sunny. High: 87 Low: 61 +Fri - PM Thunderstorm...",0 +9634,Severe Weather Statement issued August 05 at 9:33PM CDT by NWS: ...THE SEVERE THUNDERSTORM WARN... http://t.co/1EOf1Wxnpj #Skywarn #OKwx,1 +9636,Severe Thunderstorm Warning including Russellville AR Clarksville AR Dardanelle AR until 10:15 PM CDT http://t.co/n844h1ASPj,1 +9638,RT @LivingSafely: NWS posts Severe #Thunderstorm Warnings for parts of #AR #NC #OK. Seek strong shelter if at risk: http://t.co/kEa5l3b1AE,1 +9639,Severe Thunderstorm Warning for Oklahoma County until 10:30pm. http://t.co/RsdkWZRc8g #okwx,1 +9640,Falling asleep to the sounds to thousands of River Plate fans in the stadium and a thunderstorm. #VivaArgentina,0 +9642,Strong Thunderstorm 4 Miles East of Pickens Moving NE At 20 MPH. Pea Size Hail and Wind Gusts Up to 40 MPH... #scwx http://t.co/TsaLetFtkt,1 +9643,Severe Thunderstorm Warning for Oklahoma County in OK until 10:30pm CDT. #okwx,1 +9644,OUN continues Severe Thunderstorm Warning for Oklahoma [OK] till 10:30 PM CDT http://t.co/oIM6Po8XCu,1 +9645,Severe Thunderstorm Warning including Midwest City OK Del City OK Choctaw OK until 10:30 PM CDT http://t.co/ogxSY4GWD1,1 +9646,Severe Thunderstorm pictures from across the Mid-South http://t.co/UZWLgJQzNS,1 +9650,[HIGH PRIORITY] SEVERE THUNDERSTORM WATCH ENDED Issued for Lethbridge [Updated: Aug 05th 20:29 MDT] http://t.co/yqYiwjN8eZ,1 +9651,A STRONG THUNDERSTORM WILL AFFECT CENTRAL HALIFAX COUNTY THROUGH 1145 PM EDT for Halifax [VA] till 11:45 PM EDT http://t.co/MjSTefgGU5,1 +9652,"Thunderstorm in Helsinki +#thunder #lightning #lightningstrike #thunderstorm #nature #Helsinki___ https://t.co/rJJXUcX5PM",1 +9653,9:35 pm. Thunderstorm. No rain. 90 degrees. This weather weird.,1 +9654,Wed 20:30: Mainly cloudy. 60 percent chance of showers this evening with risk of a thunderstorm. Low 10.,0 +9655,NWS has issued a Severe Thunderstorm Warning for Oklahoma-OK until 10:30 PM,1 +9656,I need a thunderstorm please!,0 +9657,#usNWSgov Severe Weather Statement issued August 05 at 10:38PM EDT by NWS: ...THE SEVERE THUNDERSTORM WARNING ... http://t.co/7HuEN4rWrn,1 +9659,My room looks like a tornado passed through it and my OCD is not having it.,0 +9660,@soonergrunt better than tornado!,0 +9661,@BrrookkllyynnR came through like a tornado and destroyed me. such beautiful words,0 +9663,#TornadoGiveaway å_ #thebookclub å_ Join in!! http://t.co/LjOMCTUZFy https://t.co/2zGVSLOX5p,0 +9664,@ticklemeshawn @evebrigid I'll bet I do,0 +9665,Lily Xo is a sexy cowgirl out in the sticks http://t.co/qew4c5M1xd View and download video,0 +9666,Blonde teen Courtney Laudner teases in her panties http://t.co/qew4c5M1xd View and download video,0 +9667,73rd GOODE Water Ski National Championships will go on as planned next week http://t.co/PgKBT3MBAp. (Event w/ damage from a tornado on Mon),1 +9669,(AR) Severe Thunderstorm Warning issued August 05 at 9:12PM CDT until August 05 at 9:45PM CDT by NWS http://t.co/AYfdjeB7Hy #arwx,1 +9670,Second tornado confirmed in Sunday__ªs storm http://t.co/Ffnzit7kgN,1 +9672,Dakota Skye gets horny with some porn then gets her juicy pussy pounded http://t.co/qew4c5M1xd View and download video,0 +9673,(OK) Severe Thunderstorm Warning issued August 05 at 8:29PM CDT until August 05 at 9:15PM CDT by NWS http://t.co/O8X4M5eR6b #okwx,1 +9674,I liked a @YouTube video http://t.co/itnKBxgWLH Lexi Belle for Oklahoma tornado victims,1 +9675,@SakhalinTribune Calgary area tornado warnings end as thunderstorms move eastward,1 +9678,Guys he can run so fast he creates a tornado WITHOUT BREAKING A SWEAT. He makes Superman look like a slowpoke. He can be a POC.,0 +9680,I'm a tornado looking for a soul to take,0 +9681,Environment Canada confirms 2nd tornado touched down last weekend å_ http://t.co/x8zqbwNfO1,1 +9683,Pizza and beer in a tornado in Austin. Windy af right now,1 +9684,Still can't get over the thunderstorm/tornado we were woken up to yesterday. Half the street is still in the dark! http://t.co/Y8h5v1j2y7,1 +9686,"@kunalkapoor Photo of the Day: Storm +Chaser +http://t.co/4WJy7seHmw +#photography #pod",1 +9687,Heather Night and Ava Sparxxx enjoy a wild teen threesome http://t.co/qew4c5M1xd View and download video,0 +9688,Brunette beauty Night A stretches out on a victorian sofa http://t.co/qew4c5M1xd View and download video,0 +9690,@Rebelmage2 I'm glad you got away XD But My 'be safe' was in reference to a tornado near calgary and drum heller at around 4 :O,1 +9691,Marley Brinx gives a striptease and then spreads her legs for a pussy pounding http://t.co/qew4c5M1xd View and download video,0 +9695,Weak but distinct rotation fully condenses near Villa Ridge MO today near MCV center. A 'sub-tornado?' Video: https://t.co/Fd9DzspuGk,1 +9697,@Ayshun_Tornado then don't,0 +9698,Brunette teen Giselle Locke teases at home http://t.co/qew4c5M1xd View and download video,0 +9699,Calgary area tornado warnings end as thunderstorms move eastward - CBC.ca: CBC.ca Calgary area tornado warnings end___ http://t.co/ST9jPZ8Y24,1 +9701,A tornado flew around my room before you came,0 +9702,Illinois Tornado Slipped Under The Radar Emergency Officials Say http://t.co/P4KOfYtkdx,1 +9703,Maybe that's what happens when a tornado meets a volcano,1 +9704,I feel like a tornado http://t.co/iZJK6kpWiZ,1 +9705,@Toocodtodd Hey @wyattb23 let's challenge then to a tornado tag tlc match. Winner take all.,0 +9706,Tornado has to make the playoffs. They have 3 guys with 20+homers already and they just added Tulowitzki and Price,0 +9707,Pretty teen Hayden Ryan poses and strips out of her purple top http://t.co/qew4c5M1xd View and download video,0 +9708,Rly tragedy in MP: Some live to recount horror http://t.co/TTb9oiL8R2 #TopStories #India timesofindia,1 +9709,shit is hard to get over but sometimes the tragedy means it's over soulja..,1 +9710,Maaaaan I love Love Without Tragedy by @rihanna I wish she made the whole song,0 +9712,There is no greater tragedy than becoming comfortable with where you are in life.,0 +9714,Don't forget tragedy ?????????????? https://t.co/GaJTUGAUi7,0 +9715,'you can__ªt research collective memory' The 1st Rule of writing diverse should be don't touch another group's tragedy https://t.co/PHFoEozYPS,0 +9716,Rly tragedy in MP: Some live to recount horror: ___When I saw coaches of my train plunging into water I called ... http://t.co/72ix7vM87w,1 +9717,'Sometimes God uses sorrowful tragedy to set the stage for glorious redemption.' -David Platt Run for___ https://t.co/86V81dv00E,0 +9718,Rly tragedy in MP: Some live to recount horror: ___When I saw coaches of my train plunging into water I called ... http://t.co/Calk5nv5Vc,1 +9719,@HomeworldGym @thisisperidot D: What? That's a tragedy. You have a wonderful nose,0 +9722,"A real life tragedy: +Monica's hair on any given season of Friends",0 +9723,Railways caught unawares by MP tragedy; Accident spot never marked as 'vulnerable' - Times of___ http://t.co/cEdCUgEuWs #News,1 +9724,The tragedy of life is not that it ends so soon but that we wait so long to begin it. ~ W.M. Lewis #quotes,0 +9728,Railways caught unawares by MP tragedy; Accident spot never marked as __÷vulnerable__ª http://t.co/UB1JZskmRc,1 +9729,"@TOIIndiaNews ofcourse + +Just 8 minute b4 tragedy another train had crossed this place + +N a flash flood caued washing of tracks..",1 +9730,#ModiMinistry Rly tragedy in MP: Some live to recount horror http://t.co/s43wE7Oe2i,1 +9731,Rly tragedy in MP: Some live to recount horror: ___When I saw coaches of my train plunging into water I called ... http://t.co/vScPGMsJXY,1 +9732,@tanehisicoates even the second half has that great Greek Tragedy laced monologue by Charlize. coulda been so much more.,0 +9733,This is a tragedy: I added the wrong book to my TBR list now I can't find the right one. This is what comes of browsing just on gr homepage.,0 +9734,DTN India: Rly tragedy in MP: Some live to recount horror: ___When I saw coaches of my train plunging into water... http://t.co/WK63tm34a0,1 +9736,@sriramk @DLin71 @pmarca Tragedy of commons pertains to public ownership. Not property rights based markets. The opposite of what you say.,0 +9737,#breakingnews Rly tragedy in MP: Some live to recount horror: ___When I saw coaches of my train plunging into wa... http://t.co/eYOrBmF3O3,1 +9738,@itss_selenaluna like a beautiful ass tragedy lol,1 +9739,Rly tragedy in MP: Some live to recount horror: ___When I saw coaches of my train plunging into water I called ... http://t.co/0Xh758OnUP,1 +9741,Back home they mad cause I chill with the white boys .,0 +9744,Rly tragedy in MP: Some live to recount horror: ___When I saw coaches of my train plunging into water I called ... http://t.co/CaR5QEUVHH,1 +9745,Robert Gagnon reviews the catastrophe of imposing same-sex marriage and how Christians should respond http://t.co/HIpklxpHnp,1 +9746,Rly tragedy in MP: Some live to recount horror: ___When I saw coaches of my train plunging into water I called my___ http://t.co/gDjTzkpCHK,1 +9749,"I'm gunning down romance +It never did a thing for me +But heartache and misery +Ain't nothing but a tragedy http://t.co/jhUPOgbvs8",0 +9750,Rly tragedy in MP: Some live to recount horror: ___When I saw coaches of my train plunging into water I called my daughters and said t...,1 +9752,Rly tragedy in MP: Some live to recount horror: ___When I saw coaches of my train plunging into water I called ... http://t.co/ZkgQSpwYj3,1 +9753,Rly tragedy in MP: Some live to recount horror: ___When I saw coaches of my train plunging into water I called ... http://t.co/21hsrrqZOu,1 +9754,Rly tragedy in MP: Some live to recount horror: ___When I saw coaches of my train plunging into water I called ... http://t.co/xtlJz7BjgL,1 +9755,Rt hirochii0: There is no country that making fun of Hiroshima 's tragedy but Korea. http://t.co/And1Btizao #Indonesia #Malaysia #Jamaica #___,1 +9756,Can't find my ariana grande shirt this is a fucking tragedy,0 +9757,@CSAresu American Tragedy http://t.co/SDmrzGErYX,1 +9759,"Leitchfield KY: + + Bella Edward & Rosalie need rescue/adoption/local foster home(s)/sponsorships. + + Trapped &... http://t.co/Ajay0sNPlg",1 +9760,@almusafirah_ you feel trapped innit ??,0 +9761,IM TRAPPED IN THE DAMN GAS PUMP THERE ARE TWO SUVS ON EITHER SIDE FUCK YOU DICKHEADS,0 +9762,Photo: prettyboyshyflizzy: Lol she trapped them into that so beautifully http://t.co/FKXCsztezB,0 +9763,Hollywood Movie About Trapped Miners Released in Chile: 'The 33' Hollywood movie about trapped miners starring... http://t.co/KK8cnppZMk,0 +9764,@onihimedesu the whole city is trapped! You can't leave the city! This was supposed to be a normal sports manga wit a love triangle (c),1 +9765,Bomb head? Explosive decisions dat produced more dead children than dead bodies trapped tween buildings on that day in September there,1 +9766,salute to all the kids still trapped in adult bodies.,0 +9769,@BattleRoyaleMod when they die they just get teleported into somewhere middle of ocean and stays trapped in there unless they decides 2/6,0 +9770,Hollywood movie about trapped miners released in Chile http://t.co/YeLJPQHmEd,0 +9771,Hollywood movie about trapped miners released in #Chile http://t.co/r18aUtnLSd #ZippedNews http://t.co/CNqaE9foj6,0 +9772,"Billionaires have a plan to free half a billion dollars trapped in Venezuela for two years @BlakeSchmidt reports. + +http://t.co/gbqTc7Sp9C",0 +9774,Hollywood movie about trapped miners released in Chile,0 +9775,(?EudryLantiqua?) Hollywood Movie About Trapped Miners Released in Chile: 'The 33' Holly... http://t.co/us1DMdXZVb (?EudryLantiqua?),1 +9776,I feel like that episode of Victorious when they all got trapped in an RV and almost died of heat stroke #MTVHottest One Direction,0 +9778,#entertainment Hollywood movie about trapped miners released in Chile: SANTIAGO Chile (AP) ___ The Hollyw... http://t.co/C22ecVl4Hw #news,0 +9779,Hollywood Movie About Trapped Miners Released in Chile http://t.co/EXQKmlg4NJ,0 +9780,Literally trapped in my room Cuz my bathroom being remodeled. The only exit is through a window,1 +9782,@dramaa_llama but otherwise i will stay trapped as the worst lilourry stan ever AND without zarry what am I left with? NARRY. NO THANKS.,0 +9783,Hollywood Movie About Trapped Miners Released in Chile http://t.co/qkrLtrd39B,1 +9785,trapped in its disappearance,0 +9787,I trapped this cat in my room and it's going insane but it's not leaving it's too pretty! http://t.co/gRLxUrko8D,0 +9788,Hollywood Movie About Trapped Miners Released in Chile: 'The 33' Hollywood movie about trapped miners starring... http://t.co/x8moYeVjsJ,0 +9790,Hollywood Movie About Trapped Miners Released in Chile http://t.co/Fk1vyh5QLk #newsdict #news #Chile,0 +9791,@BoyInAHorsemask its a panda trapped in a dogs body,1 +9794,Hollywood Movie About Trapped Miners Released in Chile: 'The 33' Hollywood movie about trapped miners starring... http://t.co/tyyfG4qQvM,1 +9795,#NZ Hollywood movie about trapped miners released in Chile http://t.co/0aJIsA5531 #HugoMatz,0 +9796,Did you know @lilithsaintcrow had a new release this week? BLOOD CALL 'An ancient evil has been trapped...' http://t.co/eSwNSetFtf ___,0 +9797,Hollywood Movie About Trapped Miners Released in Chile: 'The 33' Hollywood movie about trapped miners starring... http://t.co/0f8XA4Ih1U,1 +9800,Hollywood Movie About Trapped Miners Released in Chile: 'The 33' Hollywood movie about trapped miners starring... http://t.co/3Yu26V19zh,0 +9801,Hollywood movie about trapped miners released in Chile http://t.co/JJL89F9O3V,0 +9806,Hollywood movie about trapped miners released in Chile http://t.co/xe0EE1Fzfh,0 +9808,Photo: lavenderpoetrycafe: Trauma memories are encoded in images as trauma is a more sensory than cognitive... http://t.co/DMb6xP966D,1 +9810,in response to trauma Children of Addicts develop a defensive self - one that decreases vulnerability. (3,0 +9812,Your brain is particularly vulnerable to trauma at two distinct ages http://t.co/KnBv2YtNWc @qz @TaraSwart @vivian_giang,0 +9813,#NissanNews : Trauma Alert and 1 Child Among 6 Hospitalized After 2-Car Wreck on A1A Near Bing's Landing: The ... http://t.co/9dWyJqvFY4,1 +9815,@RaabChar_28 @DrPhil @MorganLawGrp How do you self-inflict a wound to your side and blunt force trauma not consistent with fall dimensions?!,0 +9816,games that I really hope to see in AGDQ: Trauma Center Second Opinion Kororinpa Marble Mania TLoZ Oracle of Ages Metroid II,0 +9818,@crazyindapeg @VETS78734 completely understandable considering the trauma #ptsdchat,0 +9819,Trauma Team needs to come to the American E-shop.,0 +9820,@PTSD_Chat Yes. I feel the root of that is Shame - which can be found in the rubble of most trauma. #PTSDchat,1 +9822,Hiroshima: They told me to paint my story: Eighty-nine year old man recalls the terror and the trauma when the... http://t.co/spE7U8t40K,1 +9823,Photo: lavenderpoetrycafe: The Forgotten History of Sexual Trauma Hysteria was an affliction seen primarily... http://t.co/U2eS0Uk1u3,1 +9826,Trauma injuries involving kids and sport usually cycling related - CBC.ca http://t.co/0dQjereTXU,1 +9828,Butt Trauma Extraordinaire,1 +9829,Author Interview Michele Rosenthal-author of Your Life After Trauma.,0 +9830,A1: I started writing when I couldn't talk about my trauma in therapy it was the only way I could communicate #gravitychat,0 +9831,@AshGhebranious civil rights continued in the 60s. And what about trans-generational trauma? if anything we should listen to the Americans.,0 +9832,@thetimepast @saalon I have childhood trauma more resolved than theirs. Actual trauma. Fricken babies.,0 +9833,"Why #Marijuana Is Critical For Research in Treating #PTSD + +http://t.co/T6fuAhFp7p +#hempoil #cannabis #marijuana___ http://t.co/RhE7dXM7Ey",0 +9834,Blood Memory: Intragenerational Trauma and the Death of Sandra Bland http://t.co/ZWeyGpHpf7,1 +9835,Both kids got haircuts w minimal trauma. Clearly that calls for wine,0 +9836,@ARobotLegion so be it. You can't tell an oppressed group of people how to react to trauma. That would be stupid and ignorant.,0 +9837,I need to plan a trip to Cleveland soon! ??,0 +9838,Esteemed journalist recalls tragic effects of unaddressed #childhood #trauma. @keithboykin @RandallPinkston @pozarmy http://t.co/GXq1Auzb18,1 +9841,Trauma can happen anywhere -- school home etc. -- at any time. Learn the ABC's of trauma and how to parent... ___ http://t.co/fMj8MXJY8a,1 +9842,What is the role of usg in paeds major trauma imaging decision tool? #FOAMed #FOAMcc,0 +9846,It partially has something to do with my trauma as well. But that's a long story and honestly I don't like to talk about it.,0 +9849,@mustachemurse @dateswhitecoats the truth. I pulled a 16 out. And apparently a 22 in the crazy adult trauma. And they mocked me for the 22.,0 +9850,"Simmering beneath #NHL good times the league's own concussion issues @PioneerPress + +http://t.co/zl7FhUCxHL",0 +9853,Today was trauma on top of trauma on top of trauma in Richmond so I know work is going to be crazy the next two days,0 +9854,80 @UChicago faculty members pushing university to overturn ban of trauma center protesters http://t.co/ta1yqclpSc http://t.co/xToHI1HexY,1 +9857,What happens to us as sexual trauma #survivors defines us as much as we agree with the perpetrators who hurt us.,0 +9858,Traumatised after seeing a baby literally fall out of that lady. She only went for a wee great catch! #oneborn,0 +9859,Back in Ireland v. sad/traumatised as is freezing and not beautiful Parisian summer to which have become accustomed.,0 +9860,WHY THE DEEP ROADS THO HAHAHAHA IM SO TRAUMATISED BY THE DEEP ROADS LOLOL,0 +9861,I'm so traumatised.,0 +9862,go easy on her paul the poor woman has been traumatised by a cake #GBBO,0 +9864,@Ruddyyyyyy @JamieGriff97 Jamie is too traumatised to answer http://t.co/VzgslEPkkH,0 +9865,I'm that traumatised that I can't even spell properly! Excuse the typos!,0 +9866,@AnnmarieRonan @niamhosullivanx I can't watch tat show its like a horror movie to me I get flashbacks an everything #traumatised,0 +9868,@PyramidHead76 one good thing came out of watching the film. Was too traumatised to watch show so started Halt & Catch Fire on Amazon. :D,1 +9869,@ianokavo96 he's still traumatised,0 +9870,Think I'm traumatised for life,0 +9872,America like South Africa is a traumatised sick country - in different ways of course - but still messed up.,0 +9873,...the kids at the orphanage were obviously not too traumatised. http://t.co/DjA4relcnS,0 +9874,@MPRnews 600!!! WOW!!! that's a lot of traumatised kids!!!!!,0 +9875,Yessum I'm traumatised ??,0 +9877,@PerkPearl that's just not on. I'd be traumatised are you OK? The car has gone and now for #GBBO and relax.....,0 +9878,@vienna_butcher ITS NOT FUNNY IM TRAUMATISED,0 +9880,@wrongdejavu I'm traumatised,0 +9881,@malabamiandsons she's proper traumatised that pepper is 'dead' I can't wait to see her face,0 +9883,@disneyIrh so traumatised im ???? http://t.co/TATZfK63Ch,1 +9884,Sending a snapchat to the wrong person instead of your brother about the toilet ?? ?? ???? #snapchatselfie #wrongperson #traumatised,0 +9885,@EMILY4EVEREVER haha it's alright..but more than twice is just stupid ;) he's traumatised ????,0 +9886,I'm so sad Kids Company has closed. After all the talk thousands of traumatised young people will suffer. But... http://t.co/efg8RtH9Rb,0 +9887,@Jude_Mugabi not that all abortions get you traumatised. At times you are okay with the decision due to reasons like rape,0 +9888,"___@_keits: @LIVA_GOTTA get a gold chain you'll understand___ + +One boy gave me one and my neck went green It traumatised me",0 +9889,@CiaraMcKendry mine came on the day after my data renewed this month i was traumatised,0 +9890,A traumatised dog that was found buried up to its head in dirt in France is now in safe hands. This is such a... http://t.co/AGQo1479xM,0 +9894,A spider has legit just run across my chest. Traumatised. For. Life.,0 +9896,@VickyBrush LOL! I was a traumatised child. On Wednesday @jimmyfallon releases topic for Thursday hashtag game. This is this weeks. Xoxo,0 +9897,I'm slightly traumatised after this week's one born!,0 +9898,@KushWush I'm still traumatised by your driving. Having flashbacks of the lane hogging ??,0 +9901,@cwheate hahaha I'm half traumatised half hoping my labour is that easy ??,0 +9905,I'm traumatised???? @megancoopy @laurathorne97 http://t.co/MeSqTVdu63,0 +9906,@ArgentaElite haha traumatised !!!! Hell no I want a job ?? xxx,0 +9907,@brookesddl I am traumatised the lil shit nearly hopped in the bloody shower with me,0 +9909,Budget? Oh I am in trouble... but yes I would agree. #VarageSale @Candace_Dx,0 +9910,19 Things You'll Understand If You Have Trouble Talking to People http://t.co/sHaZNLMsFE,0 +9911,Having trouble understanding the rotations within a left-leaning Red Black Tree.: My class is currently learni... http://t.co/wGl4LUbnw1,0 +9912,@lucysforsale funny cause my dumb ass was the young one to get n trouble the most lol,1 +9915,A man a woman Romance & South Afrikaan Trouble - on # sale PreOrder #theBargain - http://t.co/UMl5jZTmcB,0 +9916,@canagal Good to hear it's back.. that storm's been given you guys trouble though :( ^SJ,1 +9918,@BadAstronomer ...I have a lot of trouble getting both students and adults to understand that the moon is farther away than they think.,0 +9919,Noel back up,0 +9920,Trouble trouble when I don't get my way ????,0 +9921,@smoak_queen 'I'm going to be in so much trouble.',0 +9923,@JusstdoitGirl never said it was a problem and shit working tryna stay out of trouble wbu big homie,0 +9925,@comcastcares hey it's happing again. Any trouble shooting steps for when this happens?,0 +9930,@annajhm @JCOMANSE @paul_staubs @rslm72254 @blanktgt You must've got in trouble ??that's why you have #Fartanxiety now????,0 +9931,OH MY GOD RYANS IN TROUBLE http://t.co/ADIp0UnXHU,1 +9932,Live updates: Boyd gets out of trouble in 5th http://t.co/3ugfpwMY2x via @detroitnews,0 +9933,Strawberries are in big trouble. Scientists race to find solution. http://t.co/MqydXRLae7 http://t.co/EpJjkB4Be9,1 +9934,___@YMcglaun: @JulieKragt @WildWestSixGun You're a lot safer that way.___yeah a lot more stable & if I get in trouble I have a seat right there,0 +9935,The trouble in one of Buffett's favorite sectors http://t.co/J4dqPFLMkR,0 +9937,When there's trouble you know who to caaaaaall,0 +9938,why is it trouble@niallhariss / @simply_vain live on http://t.co/iAhJj0agq6,0 +9940,@cspan #Prez. Mr. President you are the biggest terrorist and trouble maker in the world. You create terrorist you sponsor terrorist.,1 +9941,The worst voice I can ever hear is the 'Nikki your in trouble' voice from my mom,0 +9942,@_charleyisqueen Yeah well maybe if the barber didn't cut my hair too short on top I wouldn't of gone through the egg trouble????,0 +9943,Love how I don't get in any trouble for having people over and the house still being trashed,0 +9944,@KerryKatona7 hello wud u kindly consider following me bak please I'm never any trouble lol many thanks :-),0 +9945,@TJ_Robertson2 no bc we always got in trouble for laughing too much ??,0 +9946,@charlieputh that song have a Cool beat like Nothing But Trouble,0 +9948,Reddit's new content policy shows that maybe Reddit can't have it all http://t.co/YO3T8qho9h via @nkulw http://t.co/8oDTzMvqaR,0 +9949,@astros stunningly poor defense it's not all on the pitcher. If our bats are MIA like the top of 1st inning this team is in trouble.,0 +9950,@PrinceofFencing frickin summer and its humidity building up and causing trouble,0 +9952,If you have trouble getting motivated remember that time is going to pass and that regret is going to make its way around - Matthew Donnelly,0 +9953,Nothing but trouble - Lil Wayne & Charlie Puth????????,0 +9955,Trouble with mental fog? Consider these tests: http://t.co/XAerMBMvlv,0 +9958,I feel so lucky rn,0 +9960,So did we have a hurricane tornado tsunami? Someone please tell me what the hell happened #nopower,1 +9961,@helene_yancey GodsLove & #thankU my sister Helene for RT of NEW VIDEO http://t.co/cybKsXHF7d The Coming Apocalyptic US Earthquake & Tsunami,1 +9963,@freefromwolves GodsLove & #thankU brother Danny for RT of NEW VIDEO http://t.co/cybKsXHF7d The Coming Apocalyptic US Earthquake & Tsunami,0 +9965,I'm at Baan Thai / Tsunami Sushi in Washington DC https://t.co/Udp10FRXrL,0 +9967,she keep it wet like tsunami.,0 +9971,#BBShelli seems pretty sure she's the one that's going to stay! #BB17,0 +9972,"Crptotech tsunami and banks. + http://t.co/KHzTeVeDja #Banking #tech #bitcoing #blockchain",1 +9973,#sing #tsunami Beginners #computer tutorial.: http://t.co/ukQYbhxMQI Everyone Wants To Learn To Build A Pc. Re http://t.co/iDWS2ZgYsa,0 +9974,It's my senior year I just wanna go all out,0 +9976,An optical illusion - clouds rolling in over the mountains looks like a Tsunami - Geneva - Switzerland http://t.co/EyEVZIoPg1,1 +9978,Earthquake and tsunami that occurred in Japan 'free speech' is also swallowed. http://t.co/TJyyFT6NV0,1 +9979,Tsunami - DVBBS & Borgeous (Arceen Festival Trap Remix) https://t.co/743JoqazrT via @YouTube,0 +9980,#sing #tsunami Beginners #computer tutorial.: http://t.co/ia44ncZLif Everyone Wants To Learn To Build A Pc. Re http://t.co/oGTuV1pLhT,0 +9982,@tsunami_esh ?? hey Esh,0 +9983,'Anyway' the old lady went on 'I have something to ask of you - and you alone.' THE COMING TSUNAMI http://t.co/tYeWZf3hqA,0 +9984,6 Trends Are Driving a Data Tsunami for Startups http://t.co/sjh0HsRp4s #startup,0 +9985,"?#FUKUSHIMA?#TEPCO? +Mountains of debris from the Japanese tsunami have floated to the West Coast +http://t.co/y518jYrZav",1 +9986,@TSUNAMI_nopeach ?????? I'm weak af,0 +9987,Man my stomach feel like a tsunami ??.,0 +9988,@slone did the First World War Ever Truly End? The Ripples Are With Still and have been compound into low level Tsunami,1 +9989,I hope this tsunami clears b4 i have to walk outside to my car ????,1 +9990,@tsunami_esh ESH PLEASE OKAY!,0 +9991,@author_mike Amen today is the Day of Salvation. THX brother Mike for your great encouragement. - http://t.co/cybKsXHF7d Coming US Tsunami,1 +9992,and i dont get waves of missing you anymore theyre more like tsunami tides in my eyesss,1 +9994,Dr. Jim & the tsunami: The latest New Yorker warned us in no uncertain terms. Haven't you heard? The tsunami's... http://t.co/1RrEO2jG9u,0 +9995,I don't get waves of missing you anymore... They're more like tsunami tides in my eyes,0 +9998,@Kamunt Holy crap it's been forever since I saw this movie but the nostalgia wave just hit me like a tsunami! Thank youuu!,0 +10000,I liked a @YouTube video http://t.co/0h7OUa1pns Call of Duty: Ghosts - Campanha - EP 6 'Tsunami',0 +10001,Meet Brinco your own personal earthquake snd tsunami early warning beacon. http://t.co/NXkUM9h7wD,1 +10003,I want some tsunami take out,0 +10004,@GreenLacey GodsLove & #thankU my sister for RT of NEW VIDEO http://t.co/cybKsXHF7d The Coming Apocalyptic US Earthquake & Tsunami,0 +10005,All of this energy,0 +10006,@Eric_Tsunami worry about yourself,0 +10008,It's alil twister at Tha end to! I was like oh nah ??,1 +10009,Gail and Russell saw lots of hail at their Dalroy home - they have video of twister 1/2 mile from their home #yyc http://t.co/3VfKEdGrsO,1 +10010,Want Twister Tickets AND A CHANCE AT A VIP EXPERIENCE To See SHANIA!!! CLICK HERE: http://t.co/964dk4rwwe,0 +10011,Just stop fucking saying ___a whole __÷nother___. It just sounds fucking stupid. You fucking mean ___a whole other___. Not a fucking tongue-twister.,0 +10012,"950. If a landslide tumbles down todayI'm on your side + +And if a twister sweeps it all away- + +YOU'RE ON YOUR OWN BITCH!*runs into distance*",1 +10013,love 106.1 The Twister @1061thetwister and Maddie and Tae #OKTXDUO,0 +10014,Brain twister homefolks are opinionated over against proposal modernized canada: oMw,0 +10016,Crazy Mom Threw Teen Daughter a NUDE Twister Sex Party According To Her Friend59 more pics http://t.co/t94LNfwf34 http://t.co/roCyyEI2dM,0 +10017,The Sharper Image Viper 24' Hardside Twister (Black) http://t.co/FXk3zsj2PE,0 +10018,Twister was fun https://t.co/qCT6fb8wOn,0 +10020,Brain twister let drop up telly structuring cast: EDcXO,0 +10021,I'm in bed eating a twister and drinking a cup of tea. I am not dunking the twister in the tea. That would be well weird.,0 +10023,@sarahmcpants @JustJon I'll give him a titty twister,0 +10025,Last Second OutBid RT? http://t.co/lBPX8buCnv 2 - Catlow C720 Twister Swivels 3/4' 3dc Fuel Line Made In Usa ?Please Favorite & Shar,0 +10027,Reasons brain twister oneself should discount redesigning yours website: ItrAWcWB,0 +10028,"Check out 'Want Twister Tickets AND A VIP EXPERIENCE To See SHANIA? CLICK HERE:' at http://t.co/3GEROQ49o1 +I would Love Love Love!! To win",0 +10029,This is my jam: Riser by Dierks Bentley @1061TheTwister ? #iHeartRadio #NowPlaying http://t.co/zQoScQD64h http://t.co/yLvVF139BB,0 +10030,Bulgarian Tittie Twister By NoEmotion Produced By EDK PathFinders (HD) M...: http://t.co/9ODqryJncF via YouTube,0 +10031,@ellenfromnowon 7-speed nexus shifter å£9! (For community cargo bike?) http://t.co/rjPjBwVfck,0 +10032,Anyone wanna come over and watch Twister with me? #toosoon :-),0 +10034,Crazy Mom Threw Teen Daughter a NUDE Twister Sex Party According To Her Friend50 =>http://t.co/Hy5Pbe12TM http://t.co/c1nJpLi5oR,1 +10035,@briannafrost Twister with Bill Paxton and Helen Hunt!,0 +10036,@carolinagutierr grande twister!!!,0 +10037,HAPPY 24 TWISTER!!! Thank you for all the laughs sticking by my side no matter what and also for___ https://t.co/ttq9IlHp8W,0 +10038,"'How many men would a human hew if a human could hew men?' + +-popular tongue twister among woodchucks",0 +10039,I liked a @YouTube video http://t.co/Cj76K0YaYj EXTREME PAINT TWISTER,0 +10040,Tornadoes: El Nino may give Canada's twister season a boost #Toronto http://t.co/agyCutKBnN,1 +10041,@ElianaRaquel Like GG was BAD in the end... But at least I cared a little when GG was Dan. Cause it was a twister. I don't know Wilden IDC,0 +10042,You are listening to LLEGASTE TU - TWISTER EL REY,0 +10044,White Twister Black shift knob M6x1.00 Thread Size http://t.co/SqpshAWs0w http://t.co/udlEbH88uZ,0 +10045,TWISTER DANCE Game Dance Console Instructions Cable 5 Pre Loaded Songs http://t.co/BcsXyEc4ji http://t.co/DfXI76kvX0,0 +10047,@rodarmer21 10 dolla says wicked twister won't be running,0 +10048,Wish I had a personal hair twister,0 +10049,Does the opening scene of Harry Potter and the Order of the Phoenix remind anyone else of the movie Twister? Just me? Okay,0 +10050,Drunk twister is so hard ????,0 +10052,@mrsbinker @EmilioRivera48 @davidlabrava Mine are Diesel and Twister both small for the breed but very strong! I have a beautiful pug too,0 +10054,@TheBuffShow @TheTwisterOkc My boyfriend wants too see @ShaniaTwain too! #TwisterLovesShania ?????? http://t.co/O61h2tAaE4,0 +10055,Some curls come out so pretty and some look like I just finished shooting the movie Twister,0 +10056,Am now repped by the fantastic Laura Milne @TheJonesesVoice for all your liguistic needs. And that's some tongue twister tweets,0 +10057,@jrlallo My narrator will have to say 'chemically interesting lavatory' for DB5. Not quite the tongue twister but certainly odd. :P,0 +10058,#Breaking144 Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declarat... http://t.co/M8CIKs60BX #AceNewsDesk,1 +10059,Map: Typhoon Soudelor's predicted path as it approaches Taiwan; expected to make landfall over southern China by S___ http://t.co/JDVSGVhlIs,1 +10060,Find out how your fund was used for Typhoon Haiyan in the Philippines. See @DevPeace Haiyan Relief Funds Report http://t.co/JwxrX1LsqO,1 +10061,Global precipitation measurement satellite captures 3-D image of Typhoon Soudelor - NASAHurricane. Visit: http://t.co/sQN4girdvZ,1 +10064,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians a... http://t.co/UsVyHdG9OG,1 +10066,#breaking #news Global precipitation measurement satellite captures 3-D image of Typhoon Soudelor - @NASAHurricane http://t.co/20DNcthr4D,1 +10067,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians a... http://t.co/Q3DtOqO04c,1 +10069,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians a... http://t.co/qjuU0wcWPx,1 +10070,@FoxNews let me report it to u people instead Mr.Obama just declares CNMI federal disaster area post typhoon soudelor. U guys 2slow2report.,1 +10071,abcnews - Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern... http://t.co/mg5eAJElul,1 +10072,Typhoon Soudelor: When will it hit Taiwan ___ and how bad will it be? #GeneralNews http://t.co/cWZHgEzAJ4,1 +10073,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians a... http://t.co/9i6CrCRq2m,1 +10074,RT_America: RT RT_com: Eye of Super Typhoon Soudelor seen from space (TIME-LAPSE) https://t.co/FC3BxRtHPG http://t.co/BIU4koWGlz,1 +10075,Typhoon Soudelor taking dead aim at Taiwan http://t.co/3Ac5wuy1R0,1 +10076,Global precipitation measurement satellite captures 3-D image of Typhoon Soudelor - @NASAHurricane http://t.co/iGCEtuMkcW,1 +10077,Obama Declares Disaster for Typhoon-Devastated Saipan http://t.co/CanEyTtwEV #international,1 +10079,nbanews Soudelor Typhoon Soudelor is taking dead aim at Taiwan and is expected to make landfall Friday according to the Joint Typhoon___,1 +10080,(#LosDelSonido) Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Ma... (#IvanBerroa),1 +10081,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians a... http://t.co/PC8BvufLFJ,1 +10082,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians a... http://t.co/BHZr9UgUs2,1 +10083,Obama Declares Disaster for Typhoon-Devastated Saipan,1 +10084,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians a... http://t.co/VTS9CAyiBC,1 +10085,Price of vegetables rises on Typhoon Soudelor concerns http://t.co/GeI58Vhbw6,1 +10087,Map: Typhoon Soudelor's predicted path as it approaches Taiwan; expected to make landfall over southern China by... http://t.co/YvaFI3zuJx,1 +10088,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians after typhoon hits US territory,1 +10089,Photos: Typhoon Soudelor Has Its Aim Set on Taiwan andå_China http://t.co/3OG66NfSIG,1 +10090,Satellite Spies Super Typhoon Soudelor from Space (Photo) http://t.co/VBhu2t8wgB,0 +10091,Massive Typhoon heads toward Taiwan. http://t.co/Na2Ey64Vsg,1 +10092,A GPM satellite 'bullseye' in Typhoon Soudelor http://t.co/7vcEzi6CbB,1 +10093,Global precipitation measurement satellite captures 3-D image of Typhoon Soudelor - @NASAHurricane http://t.co/MvSRjd4X3D,1 +10098,Map: Typhoon Soudelor's predicted path as it approaches Taiwan; expected to make landfall over southern China by S___ http://t.co/0XCb7yeqmw,1 +10099,A GPM satellite 'bullseye' in Typhoon Soudelor http://t.co/piVeUPiRKY,1 +10100,"4Yygb mhtw4fnet + +Thousands evacuated as Taiwan prepares for strongest typhoon of 2015 - ABC Online",1 +10101,I think a Typhoon just passed through here lol,1 +10102,Please recover from the Typhoon. ????,1 +10105,#ABCNews Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for No... http://t.co/DOBZc3piTM #World,1 +10106,Obama Declares Disaster for Typhoon-Devastated Saipan: Obama signs disaster declaration for Northern Marians a... http://t.co/lEYJwNnAH8,1 +10107,Typhoon Soudelor taking dead aim at Taiwan http://t.co/sA5CDWVDXt,1 +10109,Buddha was 'man for his time' - massive urbanisation and social upheaval also challenged Brahmans dominance ('Genius of the Ancient World').,0 +10110,A look at state actions a year after Ferguson's upheaval http://t.co/GZEkQWzijq,0 +10111,USW: 'The damage from abandoning the deal could well create a new level of uncertainty...economic upheaval & military unrest',0 +10112,R'lyeh by Upheaval http://t.co/829n4HJHOL,0 +10113,A look at state actions a year after Ferguson__ªs upheaval http://t.co/G5ZsRU0zVQ,0 +10114,Series finale of #TheGame :( It survived so much upheaval but the audience got so much good story.,0 +10115,Newberg upheaval: Jacque Betz responds 'looking forward to the day' she can answer___ http://t.co/LzasR05ljo #news http://t.co/IeMxGSE2BE,0 +10116,Save the upset and stress of upheaval and let us help keep #elderly people independent! #companionship #care #Chester @chestertweetsuk,0 +10117,"Medieval Upheaval (Hardy Boys: The Secret Files Book 18) +Franklin W. Dixon - Aladdin - Kindle Edition. http://t.co/braoUBgEC2",0 +10118,Upheaval high note for bush opera http://t.co/aWPU0gaE0b #Sydney #News #Aus,0 +10119,An indepth look at the new world of work and how young people businesses and economies are coping with huge upheaval http://t.co/aYP6zVHm2A,0 +10120,Ancient Mayan Tablet with Hieroglyphics Honors Lowly King http://t.co/xh4dZ1gpyw http://t.co/g0hsyH7YaV,0 +10121,In depth: the new world of work and how young people businesses and economies are coping with huge upheaval http://t.co/0blKwCuPZq via @ft,0 +10124,A look at state actions a year after Ferguson's upheaval http://t.co/TBQsqtmqV4,0 +10125,"But...! Oh! + +How rich the soil?! + +How wonderful the upheaval!? + +-@ENTERSHIKARI",0 +10126,Tt kettlebell upheaval blueprint over chris lopez hindsight: JhmNYe,0 +10127,"How long O Lord (Study 3) + The sixth seal opens the events of Revelation 12. The political upheaval in the Roman... http://t.co/GW0CXoOJyV",1 +10129,Loan Upheaval Is The Way In Which Oneself Can Save Your House Leaving out Being Foreclosed On...TEJc,0 +10130,Ancient Mayan Tablet Found in Jungle Temple http://t.co/qp6q8RS8ON,1 +10132,Acquire your postexistence straight a elevation in addition to upheaval ideas yet perquisite: bRZjc,0 +10136,#Tigers Wonder how much the upheaval with team is affecting different players tonight?,0 +10137,Ancient Mayan Tablet with Hieroglyphics Honors Lowly King http://t.co/Im6m4XAeN2,0 +10138,Ancient Mayan Tablet with Hieroglyphics Honors Lowly King http://t.co/WqIKqx9E3w,0 +10139,Diageo's CEO stresses that a board revolt at United Spirits has not impacted Indian operations http://t.co/gfs7UsulgQ,0 +10140,RT '@LiveScience: Ancient Mayan Tablet with Hieroglyphics Honors Lowly King: http://t.co/dpgdnaoY4p http://t.co/4fCJFDKdZS,0 +10141,Ancient Mayan Tablet with Hieroglyphics Honors Lowly King: A 1600-year-old Mayan stone tablet describing the ... http://t.co/GLPFu0Uriz,0 +10142,A Look at State Actions a Year after Ferguson__ªs Upheaval https://t.co/M4tuI0P9nT MD is mentioned in the last group for a 'reporting' bill,1 +10143,Ancient Mayan Tablet found via http://t.co/LmUMzkLtln http://t.co/yebxxAryBF http://t.co/SRRUqfffr6 http://t.co/CadzxAgMSI,0 +10144,August 5: Your daily horoscope: A relationship upheaval over the next few months may be disruptive but in the ... http://t.co/gk4uNPZNhN,0 +10146,Lyf needs quality and a certain sense of security. Being with a person you can't trust can only cause stress and emotional upheaval.,0 +10147,A Look at State Actions a Year After #Ferguson's Upheaval http://t.co/qwSbVfLPE1,1 +10149,The Great Upheaval By Winik Jay http://t.co/Ef4swP9SXZ http://t.co/Nb7MAAAOfs,0 +10150,To navigate inevitable upheaval internal audit must lead the way http://t.co/je86VetDxh,0 +10151,Diageo's CEO stresses that a board revolt at United Spirits has not impacted Indian operations http://t.co/STPOdA901U,0 +10152,Newberg upheaval: Jacque Betz responds 'looking forward to the day' she can answer questions #orcot #orpol http://t.co/dazQaMOO0C,0 +10153,"@nytimes +Due to upheaval created by the west in Iraq Syria Libya etc.",1 +10156,A look at state actions a year after Ferguson's upheaval http://t.co/vXUFtVT9AU,1 +10157,@abcnews UK scandal of 2009 caused major upheaval to Parliamentary expenses with subsequent sackings and prison. What are we waiting for?,0 +10159,Terrifying POV footage captures violent landing from inside a passenger jet during a storm in Amsterdam http://t.co/NqXQYI70W4 #travel,1 +10163,Rather violent storm. Possibility of no stream tonight.,1 +10164,Violent Storm Causes Damage Flooding in Watertown - http://t.co/3ZASZ6wxjJ,1 +10168,"Thunder pounds north goes black +a deep bruise on the sky's chest +wind cries its pain. +A summer storm has a tough life +short violent.",1 +10169,#stormchase Violent Record Breaking EF-5 El Reno Oklahoma Tornado Nearly Runs Over ... - http://t.co/3SICroAaNz http://t.co/I27Oa0HISp,1 +10171,#Amsterdam POV video captures violent landing at Amsterdam Airport Schiphol during a st... http://t.co/AlUMrGl40e http://t.co/8h2KCTFB8I,1 +10172,Storm batters Auckland and Northland: A violent overnight storm has battered Auckland and Northland uprooting... http://t.co/enrPGRgtTs,1 +10173,"Violent Forces Radio: Now Playing Agony - Storm of the apocalypse +TuneIn Player @ http://t.co/XsSgEdSbH4",0 +10174,'@NASASolarSystem: Jupiter's Great Red Spot is a violent storm larger than the entire Earth. http://t.co/2lBTshXI3c http://t.co/0jmKdTcYmJ',0 +10175,After a violent afternoon storm more severe weather heads for Chicago tonight. Details ----> http://t.co/6Peeip4y7W,1 +10176,Jupiter's Great Red Spot is a violent storm larger than the entire Earth. http://t.co/I5k3VjICMG http://t.co/cizJAFnm4E,0 +10177,I don't understand 'taking' ANY life as a trophy. It's violent killing. http://t.co/NEqW47E1uj #CecilTheLion #BADChoices #BANTROPHYHUNTING,1 +10179,"Violent Forces Radio: Now Playing Acid Storm - Scourgue Of The Gods +TuneIn Player @ http://t.co/XsSgEdSbH4",1 +10182,"U.S.PACIFIC COMMAND. +I can see it! +They gave their all in the peace unity festival +It disappears when freedom +A Violent Storm hit Sea",1 +10183,@Skarletan å_ the storm. A violent swell of emotions then nothing.,0 +10184,@Vickie627 Desert Storm was an unqualified victory a treaty was signed. Under Clinton the situation only got worse and more violent. #tcot,1 +10186,POV footage captures violent landing from inside plane during storm http://t.co/kxewlHH7Uw,1 +10187,RT '@NASASolarSystem: Jupiter's Red Spot is a violent storm larger than the entire Earth: http://t.co/i0Tvl15CoZ http://t.co/IgtXhapO0K,0 +10189,@TeaFrystlik -- causing the entire sky around their battle to darken to a violent storm as an ungodly powerful bolt of lightning struck at--,0 +10191,Dramatic Video Shows Plane Landing During Violent Storm - http://t.co/oQ0LnF2Yug http://t.co/tZDBcGpSAg,1 +10192,Dramatic Video Shows Plane Landing During Violent Storm http://t.co/XRgPdlSWfD,1 +10193,I think that none of us know the impact we have on the lives of those around us. Even the slightest stirring can create a violent storm.,0 +10194,POV video captures violent landing at Amsterdam Airport Schiphol during a storm ... http://t.co/fkv5qXDcy3,1 +10195,Storm blitzes Traverse City disrupts Management Briefing Seminars: A violent summer storm blitzed through Tra... http://t.co/NKAW9EZqGg,1 +10197,This is one violent and belligerent storm. I'm enjoying watching it unfold,1 +10198,@iateyourfood yikes. Poor pup. What a weird violent storm.,1 +10199,A brief violent storm swept through the Chicago area Sunday afternoon leading to one death and an evacuation of Lollapalooza and more,1 +10201,Storm batters top half of North Island: A violent overnight storm has battered the upper North Island uprootin... http://t.co/fHVOkmpheD,1 +10202,Slow clap for this pilot. Dramatic Video Shows Plane Landing During Violent Storm http://t.co/CgVUY3RcxO,1 +10203,If you were the NWS wth a rotating storm w/ a report of a 'HUGE' / 'MASSIVE' / 'VIOLENT' tornado what would you do? https://t.co/J3dI85IST5,1 +10204,Dramatic Video Shows Plane Landing During Violent Storm http://t.co/rJ9gkJKJJn,1 +10206,POV video captures violent landing at Amsterdam Airport Schiphol during a storm - Daily Mail http://t.co/seShqN5DSK #Amsterdam #News,1 +10207,"Violent video: Ukraine rioters brutally beat police storm local admin building +WE'RE ALL FIGHTING BACK http://t.co/Byj5Dfa2rv",1 +10208,Jetstar and Virgin forced to cancel Bali flights again because of ash from Mount Raung volcano http://t.co/jTJoFLtMS4,1 +10209,"nside a Dragon's belly. Or an ice cave under +a volcano in Kamchatka | Photography by +å©Daniel Korzhonov +http://t.co/8T36HWgoqd",1 +10210,"Japan Aogashima Volcano. By Unknown - Check It Out! http://t.co/OegFQBIqIq + #Aogashima #Japan #photography #Volcano",1 +10212,#Earthquake #Sismo M 1.9 - 5km S of Volcano Hawaii: Time2015-08-06 01:04:01 UTC2015-08-05 15:04:01 -10:00 at ... http://t.co/eTswuoD3oM,1 +10213,Maailiss: Diaporama : sixpenceee: Karymsky Lake is a crater lake located in the Karymsky volcanoå_inå_Russia. With a___ http://t.co/4o460Fm8HN,0 +10214,1.94 earthquake occurred 5km S of Volcano Hawaii at 01:04 UTC! #earthquake #Volcano http://t.co/auf4J4Owj1,1 +10215,1.9 #Earthquake in 5Km S Of Volcano Hawaii #iPhone users download the Earthquake app for more information http://t.co/V3aZWOAmzK,1 +10217,Eruption of Indonesian volcano sparks transport chaos: In this picture done from video Mount Raung in___ http://t.co/7muG2kAhL7 ?,1 +10218,http://t.co/Ns1AgGFNxz #shoes Asics GT-II Super Red 2.0 11 Ronnie Fieg Kith Red White 3M x gel grey volcano 2 http://t.co/oD250zshFy,0 +10220,USGS reports a M1.94 #earthquake 5km S of Volcano Hawaii on 8/6/15 @ 1:04:01 UTC http://t.co/Njd28pg9Xv #quake,1 +10222,USGS EQ: M 1.9 - 5km S of Volcano Hawaii: Time2015-08-06 01:04:01 UTC2015-08-05 15:04:01 -10:00 a... http://t.co/3rrGHT4ewp #EarthQuake,1 +10223,#USGS M 1.9 - 5km S of Volcano Hawaii: Time2015-08-06 01:04:01 UTC2015-08-05 15:04:01 -10:00 at epicenter... http://t.co/dIsrwhQGym #SM,1 +10224,@MrMikeEaton @Muazimus_Prime hill hill mountain volcano of hell mountain hill hil.,1 +10226,Diaporama : sixpenceee: Karymsky Lake is a crater lake located in the Karymsky volcanoå_inå_Russia. With a... http://t.co/7uf7TSt9Zx,0 +10229,M1.94 [01:04 UTC]?5km S of Volcano Hawaii. http://t.co/zDtoyd8EbJ,1 +10231,http://t.co/3nUiH6pkUi #shoes Asics GT-II Super Red 2.0 11 Ronnie Fieg Kith Red White 3M x gel grey volcano 2 http://t.co/2ufCC6gH3m,0 +10234,@songhey89 well I'm also gay but girls like some too. So. I predict tsunami & volcano & earthquakes. A gift from God? Am Christian but,0 +10236,I added a video to a @YouTube playlist http://t.co/y2Mt6v13E8 Doc: Volcanoes and Earthquakes - Inside the Volcano,1 +10237,Not only does he know the latest research the kiddo's GI specialist sang the volcano number from Inside Out. Now THAT is whole person care.,1 +10239,@alextucker VOLCANO BOWL DRINK,1 +10241,Brian Shaw + J.J. Hickson + Kenneth Faried trying to defend LaMarcus Aldridge was A BLOOD VOLCANO http://t.co/20TWGPmM7d,0 +10243,The Architect Behind Kanye West__ªs Volcano https://t.co/MUSBIk7EJf,0 +10244,Zeal is a volcano the peak of which the grass of indecisiveness does not grow,1 +10245,@lexi_purduee the volcano by it ??????,0 +10246,@volcano_tornado live somewhere else for a while and Da Mill ain't too bad son! #perspective,1 +10247,#Sismo M 1.9 - 5km S of Volcano Hawaii: Time2015-08-06 01:04:01 UTC2015-08-05 15:04:01 -10:00 at ep... http://t.co/RTUeTdfBqb #CSismica,1 +10249,This LA Startup Is So Hot that Their Flowers Come Straight from a Volcano http://t.co/R3PDdjPiEe via @LATechWatch,0 +10250,Sitting still in the #CityofMemphis traffic is like sitting in a war zone! They don't move for the Police.. They don't care,0 +10251,Zone of the Enders MGS2 God of War. RT @D_PageXXI: Quote this with your favorite PS2 game,0 +10252,Bedroom clean bathroom clean laundry done .. Shit was looking like a war zone in here ??,0 +10253,This bed looks like a war zone.,0 +10255,Thank you to @scegnews! Our neighborhood looks like a war zone but we had power back in 4 hours!,1 +10256,Rip mama but I'm still thuggin cause the world is a war zone,0 +10258,Camping in a war zone with roving raccoons toughens city slicker http://t.co/oJuS08yZrq,0 +10259,GEARS OF WAR 1!(preview member) Come chat! XB1! Welcome to the DANGER zone!: http://t.co/6SdgZ5DXNt,0 +10260,They turned Jasmines house into a war zone. ?? #LittleWomenLA,0 +10263,Sundays during football seasonfrom about 9 am - 11 pm women shouldn't even log onshit be a complete war zone,0 +10264,mama I'm still thugging the world is a war zone,0 +10265,the war on drugs has turned the U.S. into a WAR zone.,1 +10268,#DebateQuestionsWeWantToHear If U start another war would U B willing 2 go 2 the war-zone yourself or send UR sons and/or daughters 2 fight?,1 +10269,Mama I'm still thuggin the world is a war zone my homies is inmate and most of em dead wrong.,0 +10270,#WorldWatchesFerguson #Florida @GovJayNixon @clairecmc How dare you turn our streets into a war zone -a war against CITIZENS?,1 +10272,Packing for CT aka my room looks like a war zone,0 +10273,@kasiakosek the drive sucks in my case because I know my desk looks like a war zone and then everyone goes into the 'I need this' mode,0 +10274,Looks and sounds like a war zone,1 +10275,How do people bake without turning their kitchen into a war zone of eggs and flour,0 +10276,Greedy had me in the war zone ! Lmao,0 +10278,@RobertONeill31 Getting hit by a foul ball while sitting there is hardly a freak accident. It's a war zone.,0 +10280,#GrowingupBlack walking past chicken frying was like entering a war zone.,0 +10283,THIS IS A ONE DIRECTION CONCERT NOT A WAR ZONE WHAT IS THIS #OTRAMETLIFE http://t.co/PtY9HRCUZH,1 +10284,Looks like a war zone outside. What's going on?,1 +10286,70 years ago today the U.S. dropped a nuclear weapon on Japan. Here are some articles that share my opinion on that decision___,1 +10287,-honey you ain't no angel. You like to scream these words as a weapon. Well go ahead take your best shot woman. I wanna leave you it's,0 +10288,Abe government said the missiles were not 'weapon' so JSDF could provide them to the ally when collective self defense right was exercised.,1 +10289,Please allow me to reiterate it's not the weapon it's the mindset of the individual! #professional #help! -LEGION! https://t.co/2lGTZkwMqW,1 +10290,Fur Leather Coats sprite & weapon of choice was a lifestyle chosen back in '02 http://t.co/N9SNJMEVI6,0 +10291,@RogueWatson Nothing wrong with that. The Lethal Weapon series is great. Yes they're ALL great.,0 +10293,@junsuisengen changing my weapon!,0 +10294,@DwarfOnJetpack I guess I can say you and me might have one thing in common my biological father made me this way to be his weapon,0 +10295,FUCK NUCLEAR WEAPON,0 +10296,Cat Of Nine Irons XII: This nightmarishly brutal weapon is used in ritualistic country club de http://t.co/xpFmR368uF http://t.co/nmAUMYdKe1,1 +10297,BUT I will be uploading these videos ASAP so you guys get to see the new weapon types in action!,0 +10299,GUN FIREARM WEAPON VECTOR CLIP ART FOR VINYL SIGN CUTTER RIFLE GUNS WEAPONS http://t.co/sdOgEF3kFT http://t.co/x0giy85BS8,0 +10301,@RosemaryTravale Do we all use the same weapon? 'cause we might be screwed XD,0 +10302,@Weapon_X_music hey guys thanks for a rock in my world and for the follow????????,0 +10303,"@muttatek m believe my 'blue' not isis kwwwkwwwk +Without weapon 'blue' will hug me jiahahahha +Yeyeulala....",0 +10304,Iranian warship points weapon at U.S. helicopter official says http://t.co/SnqfHpYm3O #tcot,1 +10305,BREAKING: Obama Officials GAVE Muslim Terrorist the Weapon Used in Texas Attack http://t.co/RJcaxjp4oS,1 +10306,I... I might buy my esoteric weapon for astrologian...,0 +10307,@abcnews A nuclear bomb is a terrible weapon!!,1 +10308,Pulse rifles after weapon tuning? http://t.co/UwObuUW2mK,0 +10310,Rare insight into #terror and How to fight it http://t.co/t6OBVWaPhW #Cameroon #USA #Whitehouse #ES #FR #Nigeria #UK #Africa #DE #CA #AU #JP,1 +10313,Back to back like I'm on the cover of lethal weapon,0 +10314,#InsaneLimits #plugin enabled @' =TPS= | TDM | 400T | HARDCORE | LOCKER | WEAPON RULES' using 3 limits,0 +10315,Abe's government made clear that grenades were not 'weapon.',1 +10316,only weapon im scared off is karma,0 +10317,@Camilla_33 @CrayKain Hate to shatter your delusions but a hatchet is a deadly weapon justifying lethal force. #gunsense,0 +10318,Even if u have your weapon and your badge we gonna put them choppas on your ass????,1 +10320,"Slosher is GOAT. Freaking love that weapon. Can't wait to do an episode of Splatdown with it. + +Switching to the Splatling Gun now.",0 +10321,@ThatRussianMan you're too busy finishing those weapon designs,0 +10322,So yeah splatoon is still lots of fun and default splattershot jr is still the only weapon layout I'm good at,0 +10323,'Education is the most powerful weapon which you can use to change the world.' Nelson #Mandela #quote http://t.co/QR1L2JYUEZ,0 +10325,@SalmanMyDarling I will watch your dub again it will be 100 % again ???? I also have other 3 ..will use one weapon today :p,0 +10327,@Snazzychipz OMG... WHAT IS THE SUB WEAPON,0 +10329,Weapon's catalogue~,0 +10330,That day remember the nuclear weapon power......Hiroshima 70th,1 +10331,03/08/11: Police stop a 41-year-old in the Bronx citing 'casing a victim or location.' No weapon is found.,1 +10332,"Lol +Look how tough some people talk that live in some of the most gang infested cities in America +Your mouth isn't a weapon +You fucking wimp",1 +10333,I'm servicin in my section I'm lurkin I'm with my weapon,0 +10334,Iranian warship points weapon at American helicopter... http://t.co/cgFZk8Ha1R,1 +10335,@danagould @WaynesterAtl I agree with background checks. I just think guns or weapons in general are the great equalizer.,0 +10336,#Kick Hendrixonfire @'=BLACKCATS= | BIG NOOB CONQUEST | MIXED MAPS | ALL WEAPONS' for violated K/DR Limit /Server max 3,0 +10337,Agreed there - especially on automatic weapons. There's no legitimate reason for needing one @Argus_99 @HeidiA1438,1 +10338,"The thing with rules is break it once it becomes easier the next time. +http://t.co/hGb1mc3IRk https://t.co/6ysXGhc8gz",0 +10339,I'm sorry but if you have to resort to weapons to settle something you're a pussy win or lose take your shit like a man & leave it at that,1 +10340,Incredulous at continued outcry of welfare being a waste of taxpayers money but never similar objection to å£100bn on nuclear weapons,1 +10341,Since the chemical-weapons 'red line' warning on 20 August 2012 LCC have confirmed that at least 96355 people have been killed in #Syria.,1 +10342,Weapons stolen from National Guard Armory in New Albany still missing #Gunsense http://t.co/lKNU8902JE,0 +10344,Friendly reminder that the only country to ever use nuclear weapons is the U.S. And it was against civilians. https://t.co/7QrEPylLUK,1 +10345,Hello Twitter i need some book bloggers and interviews regarding my book Weapons Formed Against Me #ineedexposure,0 +10348,I liked a @YouTube video from @dannyonpc http://t.co/AAuIzGGc9Q Battlefield Hardline - 11 NEW WEAPONS - New map - Throwingknifes!,0 +10349,Hiroshima and Nagasaki I remember all those killed in alleged US war crimes using nuclear weapons https://t.co/NDxrK2NCLN #USWarCrimes,1 +10350,WOOOOOOO RT @GameRant: Call of Duty: Black Ops 3 eSports Mode Lets Players Ban Weapons http://t.co/76EHHmQQ6R http://t.co/umtffA9JjB,0 +10351,A thought on not dismantling our weapons 'yet' http://t.co/vn0acCF6D4,0 +10352,Weapons: Twin Knives #OjouBot,0 +10355,@NRO Except when ordered not to carry unauthorized weapons while on duty or in military uniforms. THATS THE RULE FOOL,1 +10358,Ukraine argues for US weapons during Pelosi's visit to Kiev http://t.co/jnN0kRNXvY http://t.co/5LOiWuyv5r,1 +10359,@DorisMatsui thank you for supporting the President. The #IranDeal takes nuclear weapons out of the hands of Iran and keeps everyone safer.,0 +10360,Mere sight of a gun makes police ___ and public ___ more aggressive experts say http://t.co/N4NEUIyt2k,0 +10361,"@JamesMelville Some old testimony of weapons used to promote conflicts +Tactics - corruption & infiltration of groups +https://t.co/cyU8zxw1oH",0 +10364,Your Router is One of the Latest DDoS Attack Weapons http://t.co/vXxMvgtzvg #phone #gaming #tv #news,0 +10365,@eyecuts @Erasuterism I love 96 Gal Deco to death even if it's a bit trickier to be great with. Glad more weapons are getting Splash Wall,1 +10366,#Battlefield 1942 forgotten hope secret weapons,1 +10367,WWII Memories Plus Lessons of Hiroshima We Still Need Nuclear Weapons http://t.co/xbMm7ITe9q #denver #billings #rapidcity #seattle #cheyenne,1 +10368,I think this is my plan for retirement. Check out the weapons of mass instruction! #bookmobile #libraries #reading http://t.co/L2NMywrmq2,0 +10369,#bigbrother #ch4 The X-37b's big brother revealed: Boeing bags $6.6m contract to design ... - Daily Mail http://t.co/0migwcmtJe,0 +10371,PM Abe pledged to make every effort to seek a world without nuclear weapons. http://t.co/CBXnHhZ6kD,1 +10372,@david_hurn @ToKTeacher Why is there something in place to prevent skynet? Because perhaps there should be! http://t.co/73Umw2iGRZ,0 +10374,I will adamantly opposed to nuclear weapons.,1 +10375,WWII Memories Plus Lessons of Hiroshima We Still Need Nuclear Weapons http://t.co/MTgFx3efIv #denver #billings #rapidcity #seattle #cheyenne,1 +10376,I liked a @YouTube video from @dannyonpc http://t.co/AD38KWoGlh Battlefield Hardline - 11 NEW WEAPONS - New map - Throwingknifes!,0 +10377,@kirstiealley @_AnimalAdvocate Or pay it for a photo safari no weapons allowed. Otherwise=they just like to kill something.,0 +10378,"Slightly doesn't help that he has Suh and Wake in practice. But think he will get there. A lot of weapons! #FinsUp +http://t.co/i1EeUxxZ3A",0 +10379,@Glosblue66 no idea what this means. Look at our violent crime rate without weapons. Ban guns we become like Mexico not Australia,0 +10380,Death certificates safes weapons and Teslas: DEF CON 23 #Security http://t.co/KMDQm3NlnS,0 +10381,I liked a @YouTube video from @dannyonpc http://t.co/PyVRPrNhOP Battlefield Hardline - 11 NEW WEAPONS - New map - Throwingknifes!,0 +10382,"Navy: No charges against officer for weapons violations in Chattanooga attack + http://t.co/hddBMU2ycA",1 +10383,'The Reagan Administration had arranged for Israeli weapons to be sent to the Guatemalan Army http://t.co/4fYNQ1hWWb,1 +10384,In memory of the victims of Hiroshima and Nagasaki. Never again! Ban nuclear weapons! : https://t.co/J3cIRXjFa6,1 +10385,Hey all you love birds! We have been getting submissions for the Whirlwind Wedding Wars and they are so fantastic! Keep sending them guys!,0 +10387,"I stand alone +don't piss and moan +about my choices made +If I must reap the whirlwind so be it +I'll do so with demeanor calm and staid",0 +10390,A quarter whirlwind. They don't see it coming.,1 +10392,HELP I'M IN A WHIRLWIND OF NOSTALGIA,0 +10393,Sitting in a cafe enjoying a bite and cramming for my meeting during my whirlwind 14-hours in NYC! https://t.co/TO0BPiEymS,0 +10394,Weather forecast for Thailand A Whirlwind is coming ...2 september https://t.co/rUKjYjG9oQ,1 +10395,Only been back 10 & a whirlwind has hit jaiden started open his present straight away didn't even get chance get in & sit down lol,0 +10396,"life can wild when... +you're caught in a whirlwind ??",1 +10398,It was a whirlwind love affair that began over back fat and grew into much more.,0 +10399,Eyes smile. Pretty smile. Good hair. Miss Luhan in exo. ???????? http://t.co/y7O55by36f,0 +10401,NEW YORK: A whirlwind day of activities in New York. Breakfast at the Millennium Hotel United Nations Plaza. Lunch... http://t.co/laYZBA9y8h,0 +10402,PawSox owners public return from whirlwind trip to Durham - Knoxville News Sentinel http://t.co/9ckggGYvOU http://t.co/u0vdBrXfia,0 +10405,@DrMartyFox In the U.S. government and Libs made evil good and good evil. We will reap the whirlwind. Lord have mercy on us.,0 +10406,Back from Seattle Tacoma and Portland. Whirlwind! http://t.co/qwHINBni8e,1 +10407,@VixMeldrew sounds like a whirlwind life!,0 +10409,I moved to England five years ago today. What a whirlwind of time it has been! http://t.co/eaSlGeA1B7,1 +10410,Whirlwind Medusa Audio Snake: 16 microphone inputs 0 returns 150 ft http://t.co/mxkAlMQpdb http://t.co/8KZnhtYtt9,1 +10411,"{INFO} Baekhyun and Suho will be attending the Idol Sports Championship on August 10th +Cr: SYJEXO + http://t.co/oAZjPwUeYR",0 +10412,@byuwnbeki The sad eyes and tacit stories in your heart that night in which the whirlwind was raging,1 +10413,My #mantra this morning!! Heading out to Make a whirlwind trip down south___ https://t.co/geht4sKI86,0 +10414,Whirlwind that has lasted for more than an hour and still strong. A standstill in parts of middle east.,1 +10415,???? throwback Thurs ?? ???? Will You Still Love Me Tomorrow http://t.co/wmoyibWEc1 ?? @LucyMayOfficial ?? ?? #Whirlwind http://t.co/0rsverLzTm,0 +10416,Last Second OutBid RT? http://t.co/KrNW0Wxhe5 30pcs 12mm Ab Resin Flower Whirlwind Flatback Rhinestone Wedding Decoration ?Please Fa,0 +10418,Two hours to get to a client meeting. Whirlwind of emotions with this #tubestrike,1 +10419,Whirlwind Head Scissor on @alexhammerstone @kttape ktfounder #RemyMarcel #FroFroFro___ https://t.co/B19z8Vi3td,0 +10420,Feel like I've got no control of anything that goes on in my life at the minute. #whirlwind #drained,1 +10421,Richard returns after whirlwind few days http://t.co/L8W30WFW3R #MLB,1 +10422,Set a new record.... 7 states in 4 days. I don't even know where I am when I wake up anymore. What a whirlwind! Loving every minute though.,0 +10423,reap the whirlwind,0 +10424,@TheEmoBrago back doing another jitsu making a hexagon on the ground as you laid there.* 64 palms of whirlwind * I yelled as air began to +,0 +10426,@ckosova Read Whirlwind about this subject amongst others. The bomb saved millions of lives yes that's true.,1 +10427,In @edfringe? We highly recommend @M00NF00L #Titania @Summerhallery A whirlwind reimagining /Shakespeare's Midsummer https://t.co/iIAIGZkbnJ,0 +10428,The Whirlwind! Scourge of Europe! RT @whedonesque Or you could just watch the Fanged Four http://t.co/Q0JHDcU6Ly,0 +10429,Whirlwind weekend #1 starts in 12 hours #cantwaittoplayinminneapolis,0 +10430,this week has been a whirlwind but this is exactly what i imagined the nyc version of my career to be like,0 +10431,MY GIRL GOT A GIRLFRIEND CHEVY BLUE LIKE WHIRLWIND.,0 +10432,#picthis http://t.co/br7gmMh5Ek ___ And I__ªm off! Thank you so much #Toronto. It has been such a whirlwind of amazingness. So glad I finall___,0 +10433,This past week has been an absolute whirlwind.... Athens bound,1 +10434,WIN: Lisowski whitewashes the Whirlwind 5-0 in round 2 of Shanghai Masters Quals! | http://t.co/MLigPUHVOh #snooker http://t.co/TcS2Cd5y6y,1 +10436,As wild fires blacken northern California parched Harris County becomes tinder box. https://t.co/i2lwTy5YuP,1 +10439,Collection of Articals and Video of West Coast Wild Fires by @ABC http://t.co/qd3DSSFWUE,1 +10441,My brother-n-law riooooos got the call to head up north and fight the wild fires. Dudes a beast at___ https://t.co/463P0yS0Eb,1 +10443,Several wild fires have burned a lot of land in California. Here is one of the larger fires. http://t.co/2M1gNeaiFl http://t.co/UQh85MiP0v,1 +10444,Map shows where all of California's wild#fires are burning: This map created by CAL FIRE___ http://t.co/0x8jAQToWM http://t.co/m1RoSi2Wcs,1 +10445,Some great footage of STRONG work from San Bernardino County Fire who is also working the wild fires right now.... http://t.co/QCYQHvn2Ha,1 +10446,God forbid this is true #California has enough problems with severe #drought & #wild fires. http://t.co/CMsgexM4FC #Nuclear Power #SanOnofre,1 +10448,Firefighters Headed To California To Fight Wild Fires http://t.co/J2PYkYo0EN,1 +10449,#FOXDebateQuestions: To what degree has Obama's efforts to institute Sharia Law exacerbated the California wild fires?,1 +10450,PRAYERS FOR MY COUSIN! He's in California helping with the wild fires.,1 +10451,Catastrophic wild fires threatening U.S. Mid west & Republicans fighting new climate change rules I guess this is the ultimate in Darwinism,1 +10455,@Jennife29916207 I was thinking about you today when I was reading about the wild fires,1 +10456,@cnni @PrisonPlanet Climate Change CNN weather 'specialist' warning:wild fires rain flooding noting about the sun? http://t.co/0sZwlWL9qU,1 +10457,@SenFeinstein Thanks Sen. Feinstein now hurry home because California is a huge uncontrolled wild fire. Lightening is starting new fires.,1 +10458,Wild fires freak me the fuck out. Like hell no,1 +10461,Man! What I would give to be in CA right now to help with the wild fires.,1 +10462,In Europe nature is kind while in US they have tornados hurricanes wild fires earthquakes flash floods mega snow droughts.,1 +10464,They should just have load shedding permanently and we will all just live like we in the wild and have camp fires cook with fire etc,0 +10466,Firefighters from Connecticut are headed to California to fight wild fires http://t.co/QWpUxPyWbF http://t.co/8jlXZ6fkxy,1 +10467,These wild fires out west are crazy.,1 +10468,DEEP crew to help with California wild fires http://t.co/QKz2Sp06xn via @thedayct,1 +10470,@aria_ahrary @TheTawniest The out of control wild fires in California even in the Northern part of the state. Very troubling.,1 +10471,Full Episode: WN 08/02/15: California Wild Fires Force 12000 to Evacuate #Worldnews http://t.co/9ikhdyHVnC,1 +10472,Man selling WILD MORELS at Ambleside Farmr Mart.Sun.-MUSHROOM forageSECRET IS TO KNOW WHAT TREES they grow under & BEST AFTER FOREST FIRES,0 +10473,@randerson62 Watching news of wild fires and hope all is ok.,1 +10478,"'Your love will surely come find us +Like blazing wild fires singing Your name'",0 +10479,@EnzasBargains A5 Donated some fruit snacks & handi wipes to our fire fighters battling wild fires! #ProfitToThePeople,1 +10481,Wild land fires.. Here I come. ??????,1 +10482,@WBCShirl2 Yes God doessnt change he says not to rejoice over the fall of people or calamities like wild fires ect you wanna be punished?,0 +10483,My heart goes out to all those effected by the wild fires in Cali????,1 +10484,Wild fires in California... Must be Global Warming. Can't just be extreme heat combined with dry foliage ignited by some douchebag hiker.,1 +10485,Does the #FingerRockFire make you wonder 'am I prepared for a wildfire'. Find out at http://t.co/eX8A5JYZm5 #azwx http://t.co/DeEeKobmXa,1 +10486,"kc5kH mhtw4fnet + +Crews gaining on huge Northern California wildfire - CBS News",1 +10487,Fire in Pisgah National Forest grows to 375 acres - WSOC Charlotte http://t.co/djUfkRrtFt,1 +10488,Route Complex AM Fire Update (Route Complex Wildfire): FIRE UPDATE: ROUTE COMPLEX Thursday Morning Aug... http://t.co/nS5lBS5ZUp #CAFire,1 +10489,Parker Ridge Fact Sheet Aug 6 2015 (Parker Ridge Wildfire): Information Phone: 208-267-6773 Email: pa... http://t.co/ezEIsaSm0C #IDFire,1 +10490,We should all have a fire safety plan. RT @Matt_Kroschel: MOCK WILDFIRE near #Vail as agencies prepare for the worst. http://t.co/SWwyLRk0fv,0 +10492,'Some hearths they burn like a wildfire' https://t.co/of3td6DGLb by Bees Knees,1 +10496,NowPlaying GT & Wildfire feat. Freaks In Love Feels Like It Should (Original Mix) Massive Dance Radio ListenLive: http://t.co/ANoDGXZR3E,0 +10498,Drones Under Fire: Officials Offer $75000 Reward Leading To Pilots Who Flew Over Wildfire http://t.co/d2vEppeh8S #photography #arts,1 +10499,Be ember aware! http://t.co/LZmL1xB2nH,1 +10500,California wildfire destroys more homes but crews advance http://t.co/7XQ8JrtL7I Free tool online http://t.co/J90dT2qnXb,1 +10502,#IDFire Parker Ridge Fact Sheet Aug 6 2015 (Parker Ridge Wildfire): Information Phone: 208-267-6773 Email: pa... http://t.co/ZggpaCjP7D,1 +10503,Is LA at Risk for a Giant Wildfire? - Which Way L.A.? on KCRW http://t.co/6AgMkx2WW4,1 +10505,California is battling its scariest 2015 wildfire so far. http://t.co/Lec1vmS7x2,1 +10506,Wildfire Burns On California U.s. China Economic Net ___ : http://t.co/U2dO2mC2ri http://t.co/3oM3xw6CZ8,1 +10508,Dr. Bengston on #wildfire management: ___numbers and size of fires areas affected and costs of fighting them all show upward trend.___ #smem,1 +10511,The Latest: Washington #Wildfire misses town; evacuations end - KHQ Right Now http://t.co/aNlhW2IzkZ,1 +10513,I honestly wonder what it is that I had to do so wrong to lose everyone.,0 +10515,PHOTOS: The Rocky Fire has grown into California's most destructive wildfire this year. http://t.co/h9v4HoWtiP http://t.co/8IcSesHbj3,1 +10516,Wildfire near Columbia River town is 50 percent contained http://t.co/gzGpWSqyMW #FireNews #WA http://t.co/ay49MTYyL8,1 +10517,! Residents Return To Destroyed Homes As Washington Wildfire Burns on http://t.co/UcI8stQUg1,1 +10518,California is battling its scariest 2015 wildfire so far - the Rocky Fire http://t.co/sPT54KfA9Q,1 +10519,Solitude Fire Update August 6 2015 (Solitude Wildfire): Summary: This lightning-caused fire is being ... http://t.co/4eSbsA8InT #UTFire,1 +10520,The Latest: More homes razed by Northern California wildfire - LancasterOnline http://t.co/ph7wllKRfI #Lancaster,1 +10521,Reuters Top News: PHOTOS: The Rocky Fire has grown into California's most ... - http://t.co/qwrRfDGXCc #NewsInTweets http://t.co/sstj2bEpqn,1 +10522,This machine really captured my attention. #helicopter #firefighting #wildfire #oregon #easternoregon___ https://t.co/V6qxnFHRxF,1 +10523,@_wildfire__ ???? Bruh that's the lady from Mulan!!,0 +10524,For those impacted by the #CalWildfires here are some great recovery tips to help you in the aftermath http://t.co/wwxbGuBww5,1 +10525,#IDFire Cherokee Road and Road 22 (Three Sisters Wildfire): There are two roads closed to the general public: ... http://t.co/UORXfF0NfX,1 +10527,Oregon's biggest wildfire slows growth http://t.co/P0GoS5URXG via @katunews,1 +10529,Media Release - Firefighters Ask Hikers to Sign-in at Local Trailheads (Parker Ridge Wildfire): Parker... http://t.co/ABlz20mgzv #IDFire,1 +10533,The Latest: #Wildfire destroys more homes but crews advance - WQOW TV News 18 http://t.co/Hj26SFDdfI,1 +10534,#California #wildfire destroys more homes but crews advance. http://t.co/2PPzGpxybi http://t.co/dS9khKffwc,1 +10535,For sixth year in a row premium costs for windstorm insurance to climb. This time by 5 percent. #GalvNews https://t.co/Cm9TvE2Vsq,0 +10536,Windstorm lastingness perquisite - acquiesce in a twister retreat: ZiUW http://t.co/iRt4kkgsJx,1 +10537,'My Fifty Online Dates and why I'm still single' by Michael Windstorm $2.99 B&N http://t.co/dde8GXaQrp #nook #books #TheBachelorette,0 +10540,Reality Training: Train falls off elevated tracks during windstorm http://t.co/JIOMnrCygT #Paramedic #EMS,1 +10541,Texas Seeks Comment on Rules for Changes to Windstorm Insurer http://t.co/ei8QqhrEgZ #insurance,0 +10543,IJ: Texas Seeks Comment on Rules for Changes to Windstorm Insurer http://t.co/h132iuL7MU,1 +10544,*looks outside at the windstorm* niiiice,1 +10545,Texas Seeks Comment on Rules for Changes to Windstormå_Insurer http://t.co/92fwtObi3U,1 +10546,When I breathe it sounds like a windstorm. Haha cool,0 +10548,WindStorm WSVR1686B Cover for V-Hull Runabout Boat including euro-style with wi http://t.co/8Prnhrhb2T http://t.co/OAhLtHRozY,0 +10550,NEMA Ekiti distributed relief materials to affected victims of Rain/Windstorm disaster at Ode-Ekiti in Gbonyin LGA.,1 +10551,Reality Training: Train falls off elevated tracks during windstorm http://t.co/etgQf28MgE,1 +10552,#Insurance Texas Seeks Comment on Rules for Changes to Windstorm Insurer: The Texas Department of Insurance is... http://t.co/byvUBg0WyE,1 +10553,#NowPlaying School Of Seven Bells - Windstorm #WZBT,0 +10554,Windstorm board OKs rate hike before change http://t.co/AI6kwOrBbT #politics #txlege #twia,0 +10555,Reality Training: Train falls off elevated tracks during windstorm http://t.co/wAL4FrTfKa #fire #ems,1 +10557,One thing I wanna see before I die> #Trump standing in a good windstorm with no hat on!!! #Hardball,0 +10558,@chriscesq The average GOP voter would go to a big-tent circus in a hailstorm/windstorm no? :-),0 +10559,@charlesadler Ian Lee's word is like 'A fart in a windstorm'. Such a browner.,0 +10560,Texas Seeks Comment on Rules for Changes to Windstorm Insurer http://t.co/BP6MfJHARS,0 +10561,Reality Training: Train falls off elevated tracks during windstorm http://t.co/qzRciNaF5z,1 +10562,Texas Seeks Comment on Rules for Changes to Windstorm Insurer http://t.co/BNNIdfZWbd,0 +10563,My precious olive tree lost this battle...another crazy windstorm in #yyc! @weathernetwork http://t.co/N00DVXEga2,1 +10564,Shirley Caesar - Windstorm http://t.co/KoCH8SLasQ #nowplaying #listenlive,0 +10565,If you find your patio table umbrella and chairs flipped over and suspect foul play (instead of windstorm) you may be a suspense writer.,0 +10566,I like the weird ones like Rain of Mystical or Windstorm or Ocean Waves https://t.co/gCdxYdBSc4,0 +10567,A fierce windstorm can't take place without da calm/stillness of the eye/center so w/ that said ur internal aspects weigh more than external,0 +10568,NW Michigan #WindStorm (Sheer) Recovery Updates: Leelanau & Grand Traverse - State of Emergency 2b extended http://t.co/OSKfyj8CK7 #BeSafe,1 +10570,New roof and hardy up..Windstorm inspection tomorrow http://t.co/kKeH8qCgc3,0 +10571,School Of Seven Bells - Windstorm http://t.co/E1kbluDwh5 #nowplaying,0 +10573,#Insurance: Texas Seeks Comment on Rules for Changes to Windstorm Insurer http://t.co/rb02svlpPu,0 +10574,Damn...was wondering where my drone ended up after the freak windstorm...?? https://t.co/dHgGxo7Mcc,1 +10575,@blakeshelton DON'T be a FART ??in a WINDSTORM.FOLLOW ME ALREADY. JEEZ.,1 +10576,TWIA board approves 5 percent rate hike: The Texas Windstorm Insurance Association (TWIA) Board of Directors v... http://t.co/esEMjRn5cC,0 +10578,Texas Seeks Comment on Rules for Changes to Windstorm Insurer http://t.co/BZ07c9WthX via @ijournal,0 +10580,the windstorm blew thru my open window and now my bong is in pieces just another example of nature's indifference to human suffering,0 +10581,'My Fifty Online Dates and why I'm still single' by Michael Windstorm $2.99 Amazon http://t.co/5KohO39oJE #kindle #datingtips #goodreads,0 +10582,@rangerkaitimay had major windstorm thunder and some rain down here near Jackson...calm now.,1 +10583,TWIA board approves 5 percent rate hike: The Texas Windstorm Insurance Association (TWIA) Board of Directors v... http://t.co/TWPl0NL8cx,1 +10584,Texas Seeks Comment on Rules for Changes to Windstorm Insurer http://t.co/XQIadG9H2w http://t.co/yPEElMjdZY,0 +10585,Gunmen kill four in El Salvador bus attack: Suspected Salvadoran gang members killed four people and wounded s... http://t.co/r8k6rXw6D6,1 +10587,Police Officer Wounded Suspect Dead After Exchanging Shots http://t.co/XxFk4KHbIw,1 +10588,I also loved 'Bury my heart at wounded knee' too! #TheGame,0 +10589,The whole food stamp gov. assistance program needs to be retooled for wounded veterans retirees and handicapped only. #NoMoreHandouts,0 +10590,Gunmen open fire on bus near El Salvador's capital killing 4 a week after gang attacks killed 8 bus drivers: http://t.co/Pz56zJSsfT bit___,1 +10591,Officer Wounded Suspect Killed in Exchange of Gunfire: Richmond police officer wounded suspect killed in exc... http://t.co/crCN8rwvKj,1 +10592,One man fatally shot another wounded on Vermont Street #Buffalo - http://t.co/KakY4mpCO4,1 +10593,Richmond police officer wounded suspect killed http://t.co/m9d2ElImZI,1 +10596,Twilight's Encore (Wounded Hearts Book 3) by Jacquie Biggar http://t.co/ZnpTdIcQxE via @amazon #KindleCountdown #Sale #MFRWauthor #MGTAB,0 +10598,"Prince Phillip said of the numbers of those murdered by the British at Amritsar... + +___That__ªs a bit exaggerated it must include the wounded___.",1 +10599,California cops have sketch of gunman who killed one wounded two deputies via the @FoxNews app http://t.co/3Ife1zsop7,1 +10600,Police Officer Wounded Suspect Dead After Exchanging Shots: Richmond police officer wounded suspect killed after exchange of gunfire,1 +10601,Let's not forget our wounded female veterans. http://t.co/rZ7fbr10xw,1 +10603,Gunmen kill four in El Salvador bus attack: Suspected Salvadoran gang members killed four people and wounded s... http://t.co/CNtwB6ScZj,1 +10605,"Have you ever seen the President +who killed your wounded child? +Or the man that crashed your sister's plane +claimin' he was sent of God?",0 +10606,One man fatally shot another wounded on Vermont Street #Buffalo - http://t.co/8ACDF4Zui6,1 +10607,GENERAL AUDIENCE: On Wounded Families | ZENIT - The World Seen From Rome http://t.co/hFvnyfT78C,0 +10608,DESCRIPTIONS OF 'WOUNDED CATS BEING 'EXCITING' http://t.co/BJycRGfH5y,0 +10609,Love is the weapon for this wounded generation <3,0 +10610,@AsterPuppet wounded and carried her back to where his brothers and sisters were and entered the air ship to go back to Academia,0 +10611,Police officer wounded suspect dead after exchanging shots: RICHMOND Va. (AP) ___ A Richmond police officer wa... http://t.co/Y0qQS2L7bS,1 +10612,Police Officer Wounded Suspect Dead After Exchanging Shots - http://t.co/iPHaZV47g7,1 +10613,Police Officer Wounded Suspect Dead After Exchanging Shots http://t.co/brE2lGmn7C #ABC #News #AN247,1 +10615,@wocowae Police Officer Wounded Suspect Dead After Exchanging Shots http://t.co/oiOeCbsh1f ushed,1 +10616,The Police Chief assured the crowd that this officer-related shooting would be investigated: http://t.co/KMXzhO5TFM. http://t.co/AlBvDNwJtg,1 +10620,Police Officer Wounded Suspect Dead After Exchanging Shots: Richmond police officer wounded suspect killed a... http://t.co/5uFTRXPpV0,1 +10621,I'm liable to sound like a wounded animal during sex if the ?? is good lol,0 +10622,Officer Wounded Suspect Killed in Exchange of Gunfire: Richmond police officer wounded suspect killed in exc... http://t.co/zDHwRN6cZc,1 +10623,Police Officer Wounded Suspect Dead After Exchanging Shots,1 +10625,Officer wounded suspect killed in exchange of gunfire: Richmond police say an officer has been wounded and a ... http://t.co/HwOrB1N6vN,1 +10626,.@wwp is serving more than 75k veterans. 52k OIF/OEF vets have physical wounds; many more have invisible ones http://t.co/sHHLV4dPlz #client,1 +10627,[#Latestnews] Police officer wounded suspect dead after exchanging shots: RICHMOND Va. (AP) ___ A Richmond pol... http://t.co/ia1HnGnHVB,1 +10628,ABC News: Police Officer Wounded Suspect Dead After Exchanging Shots. More #news - http://t.co/AzovGv4SB6,1 +10629,National free root beer float day is tomorrow at A&W from 2pm-close!! Help support the Wounded Warrior Project with donations!! :),0 +10630,-- small bag from the bottom the wounded hero shakes it. A Senzu Bean falls from it. Gohan__ªs surprised. Shakes the pouch once more but --,0 +10631,@wocowae Officer Wounded Suspect Killed in Exchange of Gunfire http://t.co/QI2BDvkab7 ushed,1 +10632,Police Officer Wounded Suspect Dead After Exchanging Shots: Richmond police officer wounded suspect killed a... http://t.co/w0r8EAOKA0,1 +10636,Gunshot wound #9 is in the bicep. The only one of the ten wounds that is not in the chest/torso area. #KerrickTrial #JonathanFerrell,1 +10637,@CharlesDagnall He's getting 50 here I think. Salt. Wounds. Rub. In.,0 +10640,Nout like rubbin salt in the wounds dad.. ?????? http://t.co/M8UfjDtlsm,0 +10641,Read ~ THE UNHEALED WOUNDS OF AN UNENDING CONFLICT #Kashmir #Pakistan #India http://t.co/sAF9MoSkSN #EndOccupation #EndConflict #FreeKashmir,1 +10644,We would become the mirrors that reflected each other's most private wounds and desires.,0 +10647,Acc to the study conducted by SKIMS morethan 50% population in #Kashmir suffer psychiatric disorders http://t.co/sAF9MoSkSN #KashmirConflict,1 +10648,You are equally as scared cause this somehow started to heal you fill your wounds that you once thought were permanent.,0 +10650,RT @DianneG: Gunshot wound #9 is in the bicep. only 1 of the 10 wounds that is not in the chest/torso area. #KerrickTrial #JonathanFerrell,0 +10651,"white ppl bruise easily.. where ur bullet wounds at ?? + +https://t.co/6vyYER6PY0",0 +10655,Help yourself or those you love who suffer from self-esteem wounds. You can today! http://t.co/tu6ScRSXVG http://t.co/iDhj4JBQ05,0 +10656,Crack in the path where I wiped out this morning during beach run. Surface wounds on left elbow and right knee. http://t.co/yaqRSximph,0 +10657,Dr. Owen says four of Ferrell's ten bullet wounds were 'rapidly lethal' #KerrickTrial #TWCNewsCLT http://t.co/nNBEXhKlHr,1 +10658,@mattmosley post a pic of your wounds please,0 +10659,11 puncture wounds,0 +10660,Having your wounds kissed by Someone who doesn't see them as disasters in your soul but rather cracks to pour their love into is amazing.,0 +10662,@FEVWarrior -in the Vault that could take a look at those wounds of yours if you'd like to go to one of these places first.' Zarry has had-,0 +10663,Cutting for some celebrety and then posting the wounds online is a no-go.,0 +10665,If time heals all wounds how come the belly button stays the same?,0 +10666,and I thought my surgical wounds were healed!!! this weather ain't helping either ):,0 +10667,Woodlawn Shooting Wounds 79-Year-Old Woman Teen - Woodlawn - http://t.co/nu3XXn55vS Chicago http://t.co/XNGrfNQSx3,1 +10669,It is not always friendship when kisses show up neither is it always enemity that shows up when wounds show up. #BishopFred,0 +10671,I keep scrolling to find out what he said or did....but there seems to be no exact starting point. Wounds run deep. : /,0 +10672,@BritishBakeOff This has opened up old baked alaskan shaped wounds. Too soon GBBO too soon... #neverforget,0 +10673,@ego_resolution im glad. My gay can heal 1000 wounds,0 +10675,Court back in session. Testimony continues with med. examiner discussing gunshot wounds #KerrickTrial,0 +10676,@IcyMagistrate ___her upper arm___ those /friggin/ icicle projectiles___ and leg from various other wounds the girl looks like a miniature more___,1 +10677,Explosion in Gaza Strip kills four wounds 30; cause unknown http://t.co/GopSiCP8bm via @Reuters,1 +10678,@NicolaClements4 I__ªm not sure that covering my head in wounds and scabs is the solution ;),0 +10679,@DauntedPsyche - Man gently dabbed the cotton rag across each one of his wounds; the pain a lot more worse than Glenn had ever -,0 +10680,"Crawling in my skin +These wounds they will not hea",1 +10681,Driver rams car into Israeli soldiers wounds 3: military: A driver rammed a car into a group of Israeli soldi... http://t.co/oBSZ45ybAJ,1 +10682,Sorrower - Fresh Wounds Over Old Scars (2015 Death Metal) http://t.co/L056yj2IOi http://t.co/uTMWMjiRty,1 +10684,ME says many of these wounds could be fatal some rather quickly others slower and a couple not lethal at all. #KerrickTrial,0 +10685,@Squeaver just hangin out in star buck watchin my boy wreck this septic tank,0 +10686,@Captainn_Morgan car wreck ??,1 +10687,I am a wreck,0 +10688,Season 3 of New Girl was such a emotional train wreck I just wanted to cry laugh and eat a lot of ice cream,0 +10689,wreck? wreck wreck wreck wreck wreck wreck wreck wreck wreck wreck wreck wreck?,0 +10690,Did I drink too much? Am I losing touch? Did I build a ship to wreck?,0 +10691,"I'm a friggin wreck destiel sucks (read the vine description) +https://t.co/MKX6Ux4OZt",0 +10692,Watertown Gazette owner charged in wreck http://t.co/JHc2RT0V9F,1 +10693,How many cars do those cyclists wreck going past scratching with pedals. They should be banned #c4news,0 +10695,LARGE SHIP WRECK FISH TANK ORNAMENT FOR SALE LARGE SHIP WRECK FISH TANK AQUA...http://t.co/scGhL0Piq6,0 +10696,@Memenaar But the design decision to 'Let's make something fresh and beautiful and wonderful and then WRECK IT' is kindof odd in restrospect,0 +10698,@_PokemonCards_ @IceQueenFroslas why did they have to wreck it with ON SALE HERE ;-;,0 +10700,Ranking #artectura #pop2015 #Nå¼36 Florence + The Machine - Ship To Wreck https://t.co/9LE0B19lVF #music #playlist #YouTube,0 +10702,the sunset boys wreck my bed original 1979 usa gimp label vinyl 7' 45 newave http://t.co/X0QLgwoyMT http://t.co/hQNx8qMeG3,1 +10704,#Trump debate will be most highly watched show tonight even among progressives. I too will watch hoping for a spectacular flaming wreck.,0 +10705,Anyone know if Fox ___News___ will be live-streaming tonight__ªs Republican debate online? I want to watch the train wreck.,0 +10706,@girlthatsrio have my uncles wreck their shit,0 +10707,First wreck today. So so glad me and mom are okay. Could've been a lot worse. So happy the lord was with us today ??????,0 +10708,I'm an emotional wreck right now.,0 +10709,My emotions are a train wreck. My body is a train wreck. I'm a wreck,0 +10710,act my age was a MESS everyone was so wild it was so fun my videos a wreck,0 +10711,@GeorgeFoster72 and The Wreck of the Edmund Fitzgerald,1 +10712,GOP debate drinking game. For anyone looking for a bit of fun while watching this train wreck. http://t.co/W3Rga0nkOm http://t.co/0TZsQe8ESD,0 +10715,Wreck with road blockage Woodward Avenue Northbound at Davison in M.S. #shoalstraffic,1 +10718,Got in a car wreck. The car in front of me didn't put their blinker on :-))) but it really does feel great outside so lol,0 +10720,Don't think I Can take anymore emotional wreck watching @emmerdale #SummerFate @MikeParrActor @MissCharleyWebb,0 +10721,The court system is truly broken indeed! But then its no surprise based on who it's run by! http://t.co/uU64wfg17m,0 +10722,Greer man dies in wreck http://t.co/n2qZbMZuly,1 +10723,AmazonDeals: Skylanders Trap Team: Flip Wreck Character Pack - down 4.53% ($0.45) to $9.49 from $9.94 #Sale http://t.co/pMbuzfGIn3,0 +10724,A staged locomotive wreck what could possibly go wrong? http://t.co/Ei9x4H8tHm,0 +10726,i still don't see the point of a frozen sequel like where's the wreck it ralph sequel,0 +10727,I still need to finish the lover but I'm watching this other drama for 8 hours now and I'm an emotional wreck so the lover needs to wait,0 +10729,Omg if Cain dies i will be an emotional wreck #emmerdale,1 +10730,@TitorTau The Loretta Lynch one was fuckin' HI-LARIOUS to watch in realtime. It was like a train wreck of fact-checking and ombudsmanship.,0 +10731,@raineishida lol...Im just a nervous wreck :P,0 +10732,Amazon Prime Day: 12 quick takeaways from Amazon__ªs magnificent train wreck - http://t.co/DBDwtOcGXF,0 +10733,I'm an emotional wreck watching emmerdale,0 +10735,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/SfAKQNveta,1 +10736,Wreckage Confirmed As Part of Missing Malaysia Airlines Flight MH370 http://t.co/yoPeYPJkb2 (VICE News),1 +10737,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/gRh7zLK979,1 +10739,MH370 victim's family furious the media was told about wreckage confirmation first http://t.co/carMqiVkwU,1 +10741,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/cs8mYAunA4,1 +10743,#Australia #News ; RT janeenorman: 'High probability' aircraft wreckage is from #MH370 according to Deputy Prime ___ http://t.co/cdOHgnJmsT,1 +10744,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/v5Ogr3F5N9,1 +10745,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/4xB4ZwyhCt,1 +10746,KUALA LUMPUR (Reuters) - A piece of a wing that washed up on an Indian Ocean island beach last week was part of the wreckage of Malaysian A,1 +10747,#science Now that a piece of wreckage from flight MH370 has been confirmed on R_©union Island is it possible t... http://t.co/uqva3dfbCA,1 +10748,Malaysia PM confirms wreckage belongs to MH370 http://t.co/kacrlpjC0l http://t.co/YjJbNTcaZY,1 +10749,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/KfzvMXj9ST,1 +10750,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/pTeVY815mt,1 +10751,CIA plot! *rolling eyes* RT @ajabrown: Chinese relatives of MH370 victims say reunion island wreckage was planted http://t.co/wmNb5ITa5P,1 +10752,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/4sf0rgn8Wo,1 +10753,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/leDmVEZCoL,1 +10754,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM,1 +10755,"Related News: + +Plane Wreckage Found Is Part Of Missing MH370 Malaysia Prime Minister?Says + - World - BuzzFeed | http://t.co/qSiPL1C9Fa",1 +10759,Malaysian prime minister says Reunion Island wreckage is from MH370: http://t.co/bpTZAMjl2K via @slate,1 +10760,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/LjylxZ1fBi,1 +10763,The first piece of wreckage from the first-ever lost Boeing 777 which vanished back in early March along with the 239 people on board has,1 +10764,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/VAZpG0ftmU,1 +10765,#science Now that a piece of wreckage from flight MH370 has been confirmed on R_©union Island is it possible t... http://t.co/qNVXJ2pAlJ,1 +10766,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/DtFSWNJZIL,1 +10767,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/EdEKrmqTpQ,1 +10768,"Wreckage is MH370: Najib +http://t.co/iidKC0jSBx #MH370 #najibrazak #MalaysiaAirlines",1 +10769,Check out 'Malaysia Confirms Plane Wreckage Is From Flight MH370' at http://t.co/UB3woZ2UT1,1 +10770,#science Now that a piece of wreckage from flight MH370 has been confirmed on R_©union Island is it possible t... http://t.co/Z2vDGIyOwf,1 +10771,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM,1 +10772,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/2Jr3Yo55dr,1 +10774,Wreckage 'conclusively confirmed' as from MH370: Malaysia PM http://t.co/Rp2DxFKHDQ | https://t.co/akmIHLRIs1,1 +10775,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/KuKmAL605a,1 +10776,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM,1 +10777,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/yi54XRHQGB,1 +10779,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/MSsq0sVnBM,1 +10780,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/nn6Y0fD3l0,1 +10782,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/1YIxFG1Hdy,1 +10783,Wreckage 'conclusively confirmed' as from missing flight MH370 via @YahooNewsDigest,1 +10784,Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/5EBpYbFH4D,1 +10785,Wrecked an hour on YouTube with @julian_lage @GrantGordy & @RossMartin7 and now it's practice time again.,0 +10786,late night mcdonalds with friends = hilarious although my car is wrecked and there's half a steak pastie in the industrial estate,0 +10787,good luck to everyone who has school soon but your sleeping schedule is wrecked beyond repair,0 +10788,I wonder how times someone has wrecked trying to do the 'stare and drive' move from 2 Fast 2 Furious,0 +10789,Four hundred wrecked cars (costing $100 apiece) were purchased for the making of this 1986 film - http://t.co/DTdidinQyF,0 +10790,Cramer: Iger__ªs 3 words that wrecked Disney__ªs stock ___å_CNBC http://t.co/PnlucERp0x,0 +10793,Poor Liv and I getting our phones wrecked on the same day @oliviaapalmerr #thatswhatfriendsarefor,0 +10794,On the freeway to Africa til I wrecked my Audi,0 +10795,"Israel wrecked my home. Now it wants my land. +https://t.co/g0r3ZR1nQj",1 +10798,#news Cramer: Iger's 3 words that wrecked Disney's stock http://t.co/SF5JdNvdw9 #til_now #CNBC,0 +10799,James Kunstler: How bad architecture wrecked cities http://t.co/Ac6I3tE8mT #residualincome #mlm http://t.co/Wq0JLsHW1g,0 +10800,http://t.co/DeQQOpSP4f: Iger's 3 words that wrecked Disney's stock http://t.co/LbKvFqRpgT http://t.co/3rVa5uvt0P,0 +10802,The Twitter update pretty much wrecked the app,0 +10803,You wrecked me. Never felt so low in my life. But it's okay God got me,0 +10805,coleslaw #wrecked http://t.co/sijNBmCZIJ,0 +10808,"300K exotic car wrecked in train accident +http://t.co/J49xEuj7Ps",1 +10809,The twins pitcher's ego is now WRECKED,0 +10810,@Tunes_WGG lol. U got wrecked,0 +10811,you wrecked my whole world,0 +10812,I wrecked my stomach help,0 +10813,@__ohhmyjoshh @stevenrulles he not gonna be thinking that when he gets his shit wrecked on the first day of school ??,0 +10814,Wrecked tired but not gonna be asleep before 3??,0 +10815,http://t.co/qVx0VQTPz0 Cramer: Iger's 3 words that wrecked Disney's stock http://t.co/vuWBSrSnrY,0 +10817,Wrecked emotions.,0 +10818,The Riddler would be the best early-exit primary presidential wannabe ever all certain of his chances until he gets wrecked by a rich guy,0 +10819,@marynmck That's beyond adorable. I hope it won't be wrecked now that it's been noticed ...,0 +10821,Cramer: Iger's 3 words that wrecked Disney's stock http://t.co/3G79prAyYc #cnbc #topnews,0 +10822,@Caitsroberts see U the night wee bArra to get absolutely wrecked ????,0 +10823,@Kirafrog @mount_wario Did you get wrecked again?,1 +10824,Had an awesome time gettin wrecked at bowling last night! http://t.co/Da9lZtOn1c,0 +10825,Cramer: 3 words that wrecked DIS stock - http://t.co/ud7XObYUa1,0 +10826,On the bright side I wrecked http://t.co/uEa0txRHYs,0 +10827,He just wrecked all of you http://t.co/y46isyZkC8,0 +10829,@widda16 ... He's gone. You can relax. I thought the wife who wrecked her cake was a goner mind lol #whoops,0 +10830,@jt_ruff23 @cameronhacker and I wrecked you both,0 +10831,Three days off from work and they've pretty much all been wrecked hahaha shoutout to my family for that one,0 +10832,#FX #forex #trading Cramer: Iger's 3 words that wrecked Disney's stock http://t.co/7enNulLKzM,0 +10833,@engineshed Great atmosphere at the British Lion gig tonight. Hearing is wrecked. http://t.co/oMNBAtJEAO,0 +10834,Cramer: Iger's 3 words that wrecked Disney's stock - CNBC http://t.co/N6RBnHMTD4,0 +10835,Pic of 16yr old PKK suicide bomber who detonated bomb in Turkey Army trench released http://t.co/aaWZXykLES http://t.co/RsMvgDxRiv,1 +10837,These boxes are ready to explode! Exploding Kittens finally arrived! gameofkittens #explodingkittens___ https://t.co/TFGrAyuDC5,0 +10839,Calgary Police Flood Road Closures in Calgary. http://t.co/RLN09WKe9g,1 +10840,#Sismo DETECTADO #Jap__n 15:41:07 Seismic intensity 0 Iwate Miyagi JST #?? http://t.co/gMoUl9zQ2Q,1 +10841,Sirens everywhere!,0 +10842,BREAKING: #ISIS claims responsibility for mosque attack in Saudi Arabia that killed 13 http://t.co/VZ640XOSwj http://t.co/m2HpnOAK8b,1 +10843,Omg earthquake,1 +10844,SEVERE WEATHER BULLETIN No. 5 FOR: TYPHOON ___#HannaPH___ (SOUDELOR) TROPICAL CYCLONE: WARNING ISSUED AT 5:00 PM 06... http://t.co/tHhjJw51PE ___,1 +10846,Heat wave warning aa? Ayyo dei. Just when I plan to visit friends after a year.,1 +10847,An IS group suicide bomber detonated an explosives-packed vest in a mosque inside a Saudi special forces headquarters killing 15 people.,1 +10848,I just heard a really loud bang and everyone is asleep great,0 +10849,A gas thing just exploded and I heard screams and now the whole street smells of gas ... ??,1 +10850,NWS: Flash Flood Warning Continued for Shelby County until 08:00 PM Wednesday. http://t.co/nZ7ACKRrJi #tnwx,1 +10851,RT @LivingSafely: #NWS issues Severe #Thunderstorm Warnings for parts of #AR #NC #OK. Expect more trauma cases: http://t.co/FWqfCKNCQW,1 +10852,#??? #?? #??? #??? MH370: Aircraft debris found on La Reunion is from missing Malaysia Airlines ... http://t.co/5B7qT2YxdA,1 +10853,Father-of-three Lost Control of Car After Overtaking and Collided #BathAndNorthEastSomerset http://t.co/fa3FcnlN86,1 +10854,1.3 #Earthquake in 9Km Ssw Of Anza California #iPhone users download the Earthquake app for more information http://t.co/V3aZWOAmzK,1 +10855,Evacuation order lifted for town of Roosevelt: http://t.co/EDyfo6E2PU http://t.co/M5KxLPKFA1,1 +10859,#breaking #LA Refugio oil spill may have been costlier bigger than projected http://t.co/5ueCmcv2Pk,1 +10860,a siren just went off and it wasn't the Forney tornado warning ??,1 +10862,Officials say a quarantine is in place at an Alabama home over a possible Ebola case after developing symptoms... http://t.co/rqKK15uhEY,1 +10863,#WorldNews Fallen powerlines on G:link tram: UPDATE: FIRE crews have evacuated up to 30 passengers who were tr... http://t.co/EYSVvzA7Qm,1 +10864,on the flip side I'm at Walmart and there is a bomb and everyone had to evacuate so stay tuned if I blow up or not,1 +10866,Suicide bomber kills 15 in Saudi security site mosque - Reuters via World - Google News - Wall ... http://t.co/nF4IculOje,1 +10867,#stormchase Violent Record Breaking EF-5 El Reno Oklahoma Tornado Nearly Runs Over ... - http://t.co/3SICroAaNz http://t.co/I27Oa0HISp,1 +10869,Two giant cranes holding a bridge collapse into nearby homes http://t.co/STfMbbZFB5,1 +10870,@aria_ahrary @TheTawniest The out of control wild fires in California even in the Northern part of the state. Very troubling.,1 +10871,M1.94 [01:04 UTC]?5km S of Volcano Hawaii. http://t.co/zDtoyd8EbJ,1 +10872,Police investigating after an e-bike collided with a car in Little Portugal. E-bike rider suffered serious non-life threatening injuries.,1 +10873,The Latest: More Homes Razed by Northern California Wildfire - ABC News http://t.co/YmY4rSkQ3d,1 \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/.github/hub/push_evaluations_to_hub.py b/TextClassification/TextClassification/evaluate/.github/hub/push_evaluations_to_hub.py new file mode 100644 index 0000000000000000000000000000000000000000..73bde44cebce2c2574357516d4e070dee44d8f8e --- /dev/null +++ b/TextClassification/TextClassification/evaluate/.github/hub/push_evaluations_to_hub.py @@ -0,0 +1,118 @@ +from pathlib import Path +from huggingface_hub import create_repo, Repository +import tempfile +import subprocess +import os +import shutil +import logging +import re +from urllib.parse import urlparse + +logger = logging.getLogger(__name__) + +GIT_UP_TO_DATE = "On branch main\nYour branch is up to date with 'origin/main'.\ +\n\nnothing to commit, working tree clean\n" + +COMMIT_PLACEHOLDER = "{COMMIT_PLACEHOLDER}" + +def get_git_tag(lib_path, commit_hash): + # check if commit has a tag, see: https://stackoverflow.com/questions/1474115/how-to-find-the-tag-associated-with-a-given-git-commit + command = f"git describe --exact-match {commit_hash}" + output = subprocess.run(command.split(), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + encoding="utf-8", + cwd=lib_path, + env=os.environ.copy(), + ) + tag = output.stdout.strip() + if re.match(r"v\d*\.\d*\.\d*", tag) is not None: + return tag + else: + return None + + +def copy_recursive(source_base_path, target_base_path): + """Copy directory recursively and overwrite existing files.""" + for item in source_base_path.iterdir(): + target_path = target_base_path / item.name + if item.is_dir(): + target_path.mkdir(exist_ok=True) + copy_recursive(item, target_path) + else: + shutil.copy(item, target_path) + +def update_evaluate_dependency(requirements_path, commit_hash): + """Updates the evaluate requirement with the latest commit.""" + with open(requirements_path, "r") as f: + file_content = f.read() + file_content = file_content.replace(COMMIT_PLACEHOLDER, commit_hash) + with open(requirements_path, "w") as f: + f.write(file_content) + +def push_module_to_hub(module_path, type, token, commit_hash, tag=None): + module_name = module_path.stem + org = f"evaluate-{type}" + + repo_url = create_repo(org + "/" + module_name, repo_type="space", space_sdk="gradio", exist_ok=True, token=token) + repo_path = Path(tempfile.mkdtemp()) + + scheme = urlparse(repo_url).scheme + repo_url = repo_url.replace(f"{scheme}://", f"{scheme}://user:{token}@") + clean_repo_url = re.sub(r"(https?)://.*@", r"\1://", repo_url) + + try: + subprocess.run( + f"git clone {repo_url}".split(), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + check=True, + encoding="utf-8", + cwd=repo_path, + env=os.environ.copy(), + ) + except OSError: + # make sure we don't accidentally expose token + raise OSError(f"Could not clone from '{clean_repo_url}'") + + repo = Repository(local_dir=repo_path / module_name, use_auth_token=token) + + copy_recursive(module_path, repo_path / module_name) + update_evaluate_dependency(repo_path / module_name / "requirements.txt", commit_hash) + + repo.git_add() + try: + repo.git_commit(f"Update Space (evaluate main: {commit_hash[:8]})") + repo.git_push() + logger.info(f"Module '{module_name}' pushed to the hub") + except OSError as error: + if str(error) == GIT_UP_TO_DATE: + logger.info(f"Module '{module_name}' is already up to date.") + else: + raise error + + if tag is not None: + repo.add_tag(tag, message="add evaluate tag", remote="origin") + + shutil.rmtree(repo_path) + + +if __name__ == "__main__": + evaluation_paths = ["metrics", "comparisons", "measurements"] + evaluation_types = ["metric", "comparison", "measurement"] + + token = os.getenv("HF_TOKEN") + evaluate_lib_path = Path(os.getenv("EVALUATE_LIB_PATH")) + commit_hash = os.getenv("GIT_HASH") + git_tag = get_git_tag(evaluate_lib_path, commit_hash) + if git_tag is not None: + logger.info(f"Found tag: {git_tag}.") + + for type, dir in zip(evaluation_types, evaluation_paths): + if (evaluate_lib_path/dir).exists(): + for module_path in (evaluate_lib_path/dir).iterdir(): + if module_path.is_dir(): + logger.info(f"Updating: module {module_path.name}.") + push_module_to_hub(module_path, type, token, commit_hash, tag=git_tag) + else: + logger.warning(f"No folder {str(evaluate_lib_path/dir)} for {type} found.") diff --git a/TextClassification/TextClassification/evaluate/.github/hub/requirements.txt b/TextClassification/TextClassification/evaluate/.github/hub/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b2642b32880c428fa0e39156712a15479b556995 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/.github/hub/requirements.txt @@ -0,0 +1 @@ +huggingface_hub \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/.github/workflows/build_documentation.yml b/TextClassification/TextClassification/evaluate/.github/workflows/build_documentation.yml new file mode 100644 index 0000000000000000000000000000000000000000..39b599d74a3c2ed04b78d0976ce209becfe2ba0c --- /dev/null +++ b/TextClassification/TextClassification/evaluate/.github/workflows/build_documentation.yml @@ -0,0 +1,15 @@ +name: Build documentation + +on: + push: + branches: + - main + - doc-builder* + - v*-release + +jobs: + build: + uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@main + with: + commit_sha: ${{ github.sha }} + package: evaluate diff --git a/TextClassification/TextClassification/evaluate/.github/workflows/build_pr_documentation.yml b/TextClassification/TextClassification/evaluate/.github/workflows/build_pr_documentation.yml new file mode 100644 index 0000000000000000000000000000000000000000..42aea774be61d97be92e6a50ea020389e28bf743 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/.github/workflows/build_pr_documentation.yml @@ -0,0 +1,16 @@ +name: Build PR Documentation + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build: + uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@main + with: + commit_sha: ${{ github.event.pull_request.head.sha }} + pr_number: ${{ github.event.number }} + package: evaluate \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/.github/workflows/ci.yml b/TextClassification/TextClassification/evaluate/.github/workflows/ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..64647218842114e79abe7d41e31da0d7b8654393 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/.github/workflows/ci.yml @@ -0,0 +1,64 @@ +name: CI + +on: + pull_request: + branches: + - main + push: + branches: + - main + - ci-* + +env: + HF_ALLOW_CODE_EVAL: 1 + +jobs: + + check_code_quality: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.8" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install .[quality] + - name: Check quality + run: | + black --check --line-length 119 --target-version py36 tests src metrics comparisons measurements + isort --check-only tests src metrics comparisons measurements + flake8 tests src metrics + + test: + needs: check_code_quality + strategy: + fail-fast: false + matrix: + test: ['unit', 'parity'] + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Python 3.8 + uses: actions/setup-python@v4 + with: + python-version: "3.8" + - name: Upgrade pip + run: python -m pip install --upgrade pip + - name: Install dependencies + run: | + pip install .[tests] + pip install -r additional-tests-requirements.txt --no-deps + - name: Test with pytest + if: ${{ matrix.test == 'unit' }} + run: | + python -m pytest -n 2 --dist loadfile -sv ./tests/ --ignore=./tests/test_trainer_evaluator_parity.py + - name: Integration test with transformers + if: ${{ matrix.test == 'parity' }} + run: | + python -m pytest -n 2 --dist loadfile -sv ./tests/test_trainer_evaluator_parity.py diff --git a/TextClassification/TextClassification/evaluate/.github/workflows/delete_doc_comment.yml b/TextClassification/TextClassification/evaluate/.github/workflows/delete_doc_comment.yml new file mode 100644 index 0000000000000000000000000000000000000000..a835cc6f8f6b5b8e54d6b5e23d0ac2016dfc020c --- /dev/null +++ b/TextClassification/TextClassification/evaluate/.github/workflows/delete_doc_comment.yml @@ -0,0 +1,13 @@ +name: Delete dev documentation + +on: + pull_request: + types: [ closed ] + + +jobs: + delete: + uses: huggingface/doc-builder/.github/workflows/delete_doc_comment.yml@main + with: + pr_number: ${{ github.event.number }} + package: evaluate \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/.github/workflows/python-release.yml b/TextClassification/TextClassification/evaluate/.github/workflows/python-release.yml new file mode 100644 index 0000000000000000000000000000000000000000..d680802aa311832714d15c86b8073812da2dc948 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/.github/workflows/python-release.yml @@ -0,0 +1,31 @@ +name: Python release + +on: + push: + tags: + - v* + +env: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN_DIST }} + +jobs: + python_release: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install dependencies + run: | + pip install --upgrade pip + pip install setuptools wheel + - run: python setup.py sdist bdist_wheel + + - run: | + pip install twine + - name: Upload to PyPi + run: | + twine upload dist/* -u __token__ -p "$PYPI_TOKEN" diff --git a/TextClassification/TextClassification/evaluate/.github/workflows/trufflehog.yml b/TextClassification/TextClassification/evaluate/.github/workflows/trufflehog.yml new file mode 100644 index 0000000000000000000000000000000000000000..9cbbf6803724dacc4759b69d7002bb34831e5937 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/.github/workflows/trufflehog.yml @@ -0,0 +1,15 @@ +on: + push: + +name: Secret Leaks + +jobs: + trufflehog: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Secret Scanning + uses: trufflesecurity/trufflehog@main diff --git a/TextClassification/TextClassification/evaluate/.github/workflows/update_spaces.yml b/TextClassification/TextClassification/evaluate/.github/workflows/update_spaces.yml new file mode 100644 index 0000000000000000000000000000000000000000..b4d7dcec71439dbcd34c6f35348bc2f8b16cfa1f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/.github/workflows/update_spaces.yml @@ -0,0 +1,36 @@ +name: Update Hub repositories + +on: + push: + branches: + - main + +jobs: + update-hub-repositories: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + - name: Set up default Git config + run: | + git config --global user.name evaluate-bot + git config --global user.email leandro@huggingface.co + - name: Install dependencies + working-directory: ./.github/hub + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Update Hub repositories + working-directory: ./.github/hub + run: | + export HF_TOKEN=${{ secrets.HF_HUB_TOKEN }} + export EVALUATE_LIB_PATH=$GITHUB_WORKSPACE + export GIT_HASH=$GITHUB_SHA + export GIT_LFS_SKIP_SMUDGE=1 + python push_evaluations_to_hub.py \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/.gitignore b/TextClassification/TextClassification/evaluate/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..b4b4d526dd988db2373d9c9cdaa24a894a581281 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/.gitignore @@ -0,0 +1,64 @@ +# Locked files +*.lock +!dvc.lock + +# Extracted dummy data +datasets/**/dummy_data-zip-extracted/ + +# Compiled python modules. +*.pyc + +# Byte-compiled +_pycache__/ +.cache/ + +# Python egg metadata, regenerated from source files by setuptools. +*.egg-info +.eggs/ + +# PyPI distribution artifacts. +build/ +dist/ + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# pyenv +.python-version + +# Tests +.pytest_cache/ + +# Other +*.DS_Store + +# PyCharm/vscode +.idea +.vscode + +# keep only the empty datasets and metrics directory with it's __init__.py file +/src/*/datasets/* +!/src/*/datasets/__init__.py + +/src/*/metrics/* +!/src/*/metrics/__init__.py + +# Vim +.*.swp + +# playground +/playground + +# Sphinx documentation +docs/_build/ +docs/source/_build/ + +# Benchmark results +report.json +report.md \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/AUTHORS b/TextClassification/TextClassification/evaluate/AUTHORS new file mode 100644 index 0000000000000000000000000000000000000000..7092efd949bb407beb485f25d6c1847804a79170 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/AUTHORS @@ -0,0 +1,8 @@ +# This is the list of HuggingFace Datasets authors for copyright purposes. +# +# This does not necessarily list everyone who has contributed code, since in +# some cases, their employer may be the copyright holder. To see the full list +# of contributors, see the revision history in source control. + +Google Inc. +HuggingFace Inc. diff --git a/TextClassification/TextClassification/evaluate/CODE_OF_CONDUCT.md b/TextClassification/TextClassification/evaluate/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000000000000000000000000000000000..450a63cc303b2eceb983656a6201e7702b8a56d0 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/CODE_OF_CONDUCT.md @@ -0,0 +1,132 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +feedback@huggingface.co. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +[https://www.contributor-covenant.org/version/2/0/code_of_conduct.html][v2.0]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available +at [https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.0]: https://www.contributor-covenant.org/version/2/0/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/TextClassification/TextClassification/evaluate/CONTRIBUTING.md b/TextClassification/TextClassification/evaluate/CONTRIBUTING.md new file mode 100644 index 0000000000000000000000000000000000000000..c8133606902b0661f4acfff0f287c309e919aaf5 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/CONTRIBUTING.md @@ -0,0 +1,277 @@ + + +# How to contribute to Evaluate + +Everyone is welcome to contribute, and we value everybody's contribution. Code +is not the only way to help the community. Answering questions, helping +others, reaching out and improving the documentations are immensely valuable to +the community. + +It also helps us if you spread the word: reference the library from blog posts +on the awesome projects it made possible, shout out on Twitter every time it has +helped you, or simply star the repo to say "thank you". + +Whichever way you choose to contribute, please be mindful to respect our +[code of conduct](https://github.com/huggingface/evaluate/blob/main/CODE_OF_CONDUCT.md). + +## You can contribute in so many ways! + +There are four ways you can contribute to `evaluate`: +* Fixing outstanding issues with the existing code; +* Implementing new evaluators and metrics; +* Contributing to the examples and documentation; +* Submitting issues related to bugs or desired new features. + +Open issues are tracked directly on the repository [here](https://github.com/huggingface/evaluate/issues). + +If you would like to work on any of the open issues: +* Make sure it is not already assigned to someone else. The assignee (if any) is on the top right column of the Issue page. If it's not taken, self-assign it. +* Work on your self-assigned issue and create a Pull Request! + +## Submitting a new issue or feature request + +Following these guidelines when submitting an issue or a feature +request will make it easier for us to come back to you quickly and with good +feedback. + +### Do you want to implement a new metric? + +All evaluation modules, be it metrics, comparisons, or measurements live on the 🤗 Hub in a [Space](https://huggingface.co/docs/hub/spaces) (see for example [Accuracy](https://huggingface.co/spaces/evaluate-metric/accuracy)). Evaluation modules can be either **community** or **canonical**. + +* **Canonical** metrics are well-established metrics which already broadly adopted. +* **Community** metrics are new or custom metrics. It is simple to add a new community metric to use with `evaluate`. Please see our guide to adding a new evaluation metric [here](https://huggingface.co/docs/evaluate/creating_and_sharing)! + +The only functional difference is that canonical metrics are integrated into the `evaluate` library directly and do not require a namespace when being loaded. + +We encourage contributors to share new evaluation modules they contribute broadly! If they become widely adopted then they will be integrated into the core `evaluate` library as a canonical module. + +### Do you want to request a new feature (that is not a metric)? + +We would appreciate it if your feature request addresses the following points: + +1. Motivation first: + * Is it related to a problem/frustration with the library? If so, please explain + why. Providing a code snippet that demonstrates the problem is best. + * Is it related to something you would need for a project? We'd love to hear + about it! + * Is it something you worked on and think could benefit the community? + Awesome! Tell us what problem it solved for you. +2. Write a *full paragraph* describing the feature; +3. Provide a **code snippet** that demonstrates its future use; +4. In case this is related to a paper, please attach a link; +5. Attach any additional information (drawings, screenshots, etc.) you think may help. + +### Did you find a bug? + +Thank you for reporting an issue. If the bug is related to a community metric, please open an issue or pull request directly on the repository of the metric on the Hugging Face Hub. + +If the bug is related to the `evaluate` library and not a community metric, we would really appreciate it if you could **make sure the bug was not already reported** (use the search bar on Github under Issues). If it's not already logged, please open an issue with these details: + +* Include your **OS type and version**, the versions of **Python**, **PyTorch** and + **Tensorflow** when applicable; +* A short, self-contained, code snippet that allows us to reproduce the bug in + less than 30s; +* Provide the *full* traceback if an exception is raised. + +## Start contributing! (Pull Requests) + +Before writing code, we strongly advise you to search through the existing PRs or +issues to make sure that nobody is already working on the same thing. If you are +unsure, it is always a good idea to open an issue to get some feedback. + +1. Fork the [repository](https://github.com/huggingface/evaluate) by + clicking on the 'Fork' button on the repository's page. This creates a copy of the code + under your GitHub user account. + +2. Clone your fork to your local disk, and add the base repository as a remote: + + ```bash + $ git clone git@github.com:/evaluate.git + $ cd evaluate + $ git remote add upstream https://github.com/huggingface/evaluate.git + ``` + +3. Create a new branch to hold your development changes: + + ```bash + $ git checkout -b a-descriptive-name-for-my-changes + ``` + + **Do not** work on the `main` branch. + +4. Set up a development environment by running the following command in a virtual environment: + + ```bash + $ pip install -e ".[dev]" + ``` + +5. Develop the features on your branch. + + As you work on the features, you should make sure that the test suite + passes. You should run the tests impacted by your changes like this: + + ```bash + $ pytest tests/.py + ``` + + To run a specific test, for example the `test_model_init` test in test_evaluator.py, + + ```bash + python -m pytest ./tests/test_evaluator.py::TestQuestionAnsweringEvaluator::test_model_init + ``` + + You can also run the full suite with the following command: + + ```bash + $ python -m pytest ./tests/ + ``` + + 🤗 Evaluate relies on `black` and `isort` to format its source code + consistently. After you make changes, apply automatic style corrections and code verifications + that can't be automated in one go with: + + ```bash + $ make quality + ``` + + This target is also optimized to only work with files modified by the PR you're working on. + + If you prefer to run the checks one after the other, the following command apply the + style corrections: + + ```bash + $ make style + ``` + + 🤗 Evaluate also uses `flake8` and a few custom scripts to check for coding mistakes. Quality + control runs in CI, however you can also run the same checks with: + + ```bash + $ make quality + ``` + + If you're modifying documents under `docs/source`, make sure to validate that + they can still be built. This check also runs in CI. To run a local check + make sure you have installed the documentation builder requirements. First you will need to clone the + repository containing our tools to build the documentation: + + ```bash + $ pip install git+https://github.com/huggingface/doc-builder + ``` + + Then, make sure you have all the dependencies to be able to build the doc with: + + ```bash + $ pip install ".[docs]" + ``` + + Finally, run the following command from the root of the repository: + + ```bash + $ doc-builder build evaluate docs/source/ --build_dir ~/tmp/test-build + ``` + + This will build the documentation in the `~/tmp/test-build` folder where you can inspect the generated + Markdown files with your favorite editor. You won't be able to see the final rendering on the website + before your PR is merged, we are actively working on adding a tool for this. + + Once you're happy with your changes, add changed files using `git add` and + make a commit with `git commit` to record your changes locally: + + ```bash + $ git add modified_file.py + $ git commit + ``` + + Please write [good commit + messages](https://chris.beams.io/posts/git-commit/). + + It is a good idea to sync your copy of the code with the original + repository regularly. This way you can quickly account for changes: + + ```bash + $ git fetch upstream + $ git rebase upstream/main + ``` + + Push the changes to your account using: + + ```bash + $ git push -u origin a-descriptive-name-for-my-changes + ``` + +6. Once you are satisfied, go to the webpage of your fork on GitHub. Click on 'Pull request' to send your changes + to the project maintainers for review. + +7. It's ok if maintainers ask you for changes. It happens to core contributors + too! So everyone can see the changes in the Pull request, work in your local + branch and push the changes to your fork. They will automatically appear in + the pull request. + + +### Checklist + +1. The title of your pull request should be a summary of its contribution; +2. If your pull request addresses an issue, please mention the issue number in + the pull request description to make sure they are linked (and people + consulting the issue know you are working on it); +3. To indicate a work in progress please prefix the title with `[WIP]`. These + are useful to avoid duplicated work, and to differentiate it from PRs ready + to be merged; +4. Make sure existing tests pass; +5. Add high-coverage tests. No quality testing = no merge. +6. All public methods must have informative docstrings that work nicely with sphinx. +7. Due to the rapidly growing repository, it is important to make sure that no files that would significantly weigh down the repository are added. This includes images, videos and other non-text files. We prefer to leverage a hf.co hosted `dataset` like + the ones hosted on [`hf-internal-testing`](https://huggingface.co/hf-internal-testing) in which to place these files and reference + them by URL. + + +### Style guide + +For documentation strings, 🤗 Evaluate follows the [google style](https://google.github.io/styleguide/pyguide.html). +Check our [documentation writing guide](https://github.com/huggingface/transformers/tree/main/docs#writing-documentation---specification) +for more information. + +**This guide was heavily inspired by the awesome [scikit-learn guide to contributing](https://github.com/scikit-learn/scikit-learn/blob/main/CONTRIBUTING.md).** + +### Develop on Windows + +On Windows, you need to configure git to transform Windows `CRLF` line endings to Linux `LF` line endings: + +`git config core.autocrlf input` + +One way one can run the make command on Window is to pass by MSYS2: + +1. [Download MSYS2](https://www.msys2.org/), we assume to have it installed in C:\msys64 +2. Open the command line C:\msys64\msys2.exe (it should be available from the start menu) +3. Run in the shell: `pacman -Syu` and install make with `pacman -S make` +4. Add `C:\msys64\usr\bin` to your PATH environment variable. + +You can now use `make` from any terminal (Powershell, cmd.exe, etc) 🎉 + +### Syncing forked main with upstream (HuggingFace) main + +To avoid pinging the upstream repository which adds reference notes to each upstream PR and sends unnecessary notifications to the developers involved in these PRs, +when syncing the main branch of a forked repository, please, follow these steps: +1. When possible, avoid syncing with the upstream using a branch and PR on the forked repository. Instead, merge directly into the forked main. +2. If a PR is absolutely necessary, use the following steps after checking out your branch: +``` +$ git checkout -b your-branch-for-syncing +$ git pull --squash --no-commit upstream main +$ git commit -m '' +$ git push --set-upstream origin your-branch-for-syncing +``` diff --git a/TextClassification/TextClassification/evaluate/LICENSE b/TextClassification/TextClassification/evaluate/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..d645695673349e3947e8e5ae42332d0ac3164cd7 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/TextClassification/TextClassification/evaluate/Makefile b/TextClassification/TextClassification/evaluate/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..f8ed3611fc9393380b665d7404368a92baab7ad8 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/Makefile @@ -0,0 +1,19 @@ +.PHONY: quality style test + +# Check that source code meets quality standards + +quality: + black --check --line-length 119 --target-version py36 tests src metrics comparisons measurements + isort --check-only tests src metrics measurements + flake8 tests src metrics + +# Format source code automatically + +style: + black --line-length 119 --target-version py36 tests src metrics comparisons measurements + isort tests src metrics measurements + +# Run tests for the library + +test: + python -m pytest -n auto --dist=loadfile -s -v ./tests/ diff --git a/TextClassification/TextClassification/evaluate/README.md b/TextClassification/TextClassification/evaluate/README.md new file mode 100644 index 0000000000000000000000000000000000000000..21747f67551a78f21493903b5fecc8bec37ee329 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/README.md @@ -0,0 +1,78 @@ +

+
+ +
+

+ +

+ + Build + + + GitHub + + + Documentation + + + GitHub release + + + Contributor Covenant + +

+ +🤗 Evaluate is a library that makes evaluating and comparing models and reporting their performance easier and more standardized. + +It currently contains: + +- **implementations of dozens of popular metrics**: the existing metrics cover a variety of tasks spanning from NLP to Computer Vision, and include dataset-specific metrics for datasets. With a simple command like `accuracy = load("accuracy")`, get any of these metrics ready to use for evaluating a ML model in any framework (Numpy/Pandas/PyTorch/TensorFlow/JAX). +- **comparisons and measurements**: comparisons are used to measure the difference between models and measurements are tools to evaluate datasets. +- **an easy way of adding new evaluation modules to the 🤗 Hub**: you can create new evaluation modules and push them to a dedicated Space in the 🤗 Hub with `evaluate-cli create [metric name]`, which allows you to see easily compare different metrics and their outputs for the same sets of references and predictions. + +[🎓 **Documentation**](https://huggingface.co/docs/evaluate/) + +🔎 **Find a [metric](https://huggingface.co/evaluate-metric), [comparison](https://huggingface.co/evaluate-comparison), [measurement](https://huggingface.co/evaluate-measurement) on the Hub** + +[🌟 **Add a new evaluation module**](https://huggingface.co/docs/evaluate/) + +🤗 Evaluate also has lots of useful features like: + +- **Type checking**: the input types are checked to make sure that you are using the right input formats for each metric +- **Metric cards**: each metrics comes with a card that describes the values, limitations and their ranges, as well as providing examples of their usage and usefulness. +- **Community metrics:** Metrics live on the Hugging Face Hub and you can easily add your own metrics for your project or to collaborate with others. + + +# Installation + +## With pip + +🤗 Evaluate can be installed from PyPi and has to be installed in a virtual environment (venv or conda for instance) + +```bash +pip install evaluate +``` + +# Usage + +🤗 Evaluate's main methods are: + +- `evaluate.list_evaluation_modules()` to list the available metrics, comparisons and measurements +- `evaluate.load(module_name, **kwargs)` to instantiate an evaluation module +- `results = module.compute(*kwargs)` to compute the result of an evaluation module + +# Adding a new evaluation module + +First install the necessary dependencies to create a new metric with the following command: +```bash +pip install evaluate[template] +``` +Then you can get started with the following command which will create a new folder for your metric and display the necessary steps: +```bash +evaluate-cli create "Awesome Metric" +``` +See this [step-by-step guide](https://huggingface.co/docs/evaluate/creating_and_sharing) in the documentation for detailed instructions. + +## Credits + +Thanks to [@marella](https://github.com/marella) for letting us use the `evaluate` namespace on PyPi previously used by his [library](https://github.com/marella/evaluate). diff --git a/TextClassification/TextClassification/evaluate/additional-tests-requirements.txt b/TextClassification/TextClassification/evaluate/additional-tests-requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..53b402e563499116a50c2f50222b666b76c3a943 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/additional-tests-requirements.txt @@ -0,0 +1,6 @@ +unbabel-comet>=1.0.0;python_version>'3.6' +git+https://github.com/google-research/bleurt.git +git+https://github.com/ns-moosavi/coval.git +git+https://github.com/hendrycks/math.git +git+https://github.com/google-research/rl-reliability-metrics +gin-config \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/comparisons/exact_match/README.md b/TextClassification/TextClassification/evaluate/comparisons/exact_match/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7370e289d29c603a33efddaf63ea4b140a1ae144 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/comparisons/exact_match/README.md @@ -0,0 +1,61 @@ +--- +title: Exact Match +emoji: 🤗 +colorFrom: blue +colorTo: green +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- comparison +description: >- + Returns the rate at which the predictions of one model exactly match those of another model. +--- + + +# Comparison Card for Exact Match + +## Comparison description + + Given two model predictions the exact match score is 1 if they are the exact same, and is 0 otherwise. The overall exact match score is the average. + +- **Example 1**: The exact match score if prediction 1.0 is [0, 1] is 0, given prediction 2 is [0, 1]. +- **Example 2**: The exact match score if prediction 0.0 is [0, 1] is 0, given prediction 2 is [1, 0]. +- **Example 3**: The exact match score if prediction 0.5 is [0, 1] is 0, given prediction 2 is [1, 1]. + +## How to use + +At minimum, this metric takes as input predictions and references: +```python +>>> exact_match = evaluate.load("exact_match", module_type="comparison") +>>> results = exact_match.compute(predictions1=[0, 1, 1], predictions2=[1, 1, 1]) +>>> print(results) +{'exact_match': 0.66} +``` + +## Output values + +Returns a float between 0.0 and 1.0 inclusive. + +## Examples + +```python +>>> exact_match = evaluate.load("exact_match", module_type="comparison") +>>> results = exact_match.compute(predictions1=[0, 0, 0], predictions2=[1, 1, 1]) +>>> print(results) +{'exact_match': 1.0} +``` + +```python +>>> exact_match = evaluate.load("exact_match", module_type="comparison") +>>> results = exact_match.compute(predictions1=[0, 1, 1], predictions2=[1, 1, 1]) +>>> print(results) +{'exact_match': 0.66} +``` + + +## Limitations and bias + +## Citations diff --git a/TextClassification/TextClassification/evaluate/comparisons/exact_match/app.py b/TextClassification/TextClassification/evaluate/comparisons/exact_match/app.py new file mode 100644 index 0000000000000000000000000000000000000000..9efaa644a2e41fe749ddc3849a9b162b48c29cec --- /dev/null +++ b/TextClassification/TextClassification/evaluate/comparisons/exact_match/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("exact_match", module_type="comparison") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/comparisons/exact_match/exact_match.py b/TextClassification/TextClassification/evaluate/comparisons/exact_match/exact_match.py new file mode 100644 index 0000000000000000000000000000000000000000..08c300137c6cb6c4c27a86262f2ffd306e77db15 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/comparisons/exact_match/exact_match.py @@ -0,0 +1,65 @@ +# Copyright 2022 The HuggingFace Evaluate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Exact match test for model comparison.""" + +import datasets +import numpy as np + +import evaluate + + +_DESCRIPTION = """ +Returns the rate at which the predictions of one model exactly match those of another model. +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions1 (`list` of `int`): Predicted labels for model 1. + predictions2 (`list` of `int`): Predicted labels for model 2. + +Returns: + exact_match (`float`): Dictionary containing exact_match rate. Possible values are between 0.0 and 1.0, inclusive. + +Examples: + >>> exact_match = evaluate.load("exact_match", module_type="comparison") + >>> results = exact_match.compute(predictions1=[1, 1, 1], predictions2=[1, 1, 1]) + >>> print(results) + {'exact_match': 1.0} +""" + + +_CITATION = """ +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class ExactMatch(evaluate.Comparison): + def _info(self): + return evaluate.ComparisonInfo( + module_type="comparison", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions1": datasets.Value("int64"), + "predictions2": datasets.Value("int64"), + } + ), + ) + + def _compute(self, predictions1, predictions2): + score_list = [p1 == p2 for p1, p2 in zip(predictions1, predictions2)] + return {"exact_match": np.mean(score_list)} diff --git a/TextClassification/TextClassification/evaluate/comparisons/exact_match/requirements.txt b/TextClassification/TextClassification/evaluate/comparisons/exact_match/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a410b4ca49fe91f97db76956af15309d2dec7bb8 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/comparisons/exact_match/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scipy \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/comparisons/mcnemar/README.md b/TextClassification/TextClassification/evaluate/comparisons/mcnemar/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1ceaaee211a3abb180cd0f4e3eb82459523c8a75 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/comparisons/mcnemar/README.md @@ -0,0 +1,86 @@ +--- +title: McNemar +emoji: 🤗 +colorFrom: blue +colorTo: green +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- comparison +description: >- + McNemar's test is a diagnostic test over a contingency table resulting from the predictions of two classifiers. The test compares the sensitivity and specificity of the diagnostic tests on the same group reference labels. It can be computed with: + McNemar = (SE - SP)**2 / SE + SP + Where: + SE: Sensitivity (Test 1 positive; Test 2 negative) + SP: Specificity (Test 1 negative; Test 2 positive) +--- + + +# Comparison Card for McNemar + +## Comparison description + +McNemar's test is a non-parametric diagnostic test over a contingency table resulting from the predictions of two classifiers. The test compares the sensitivity and specificity of the diagnostic tests on the same group reference labels. It can be computed with: + +McNemar = (SE - SP)**2 / SE + SP + +Where: +* SE: Sensitivity (Test 1 positive; Test 2 negative) +* SP: Specificity (Test 1 negative; Test 2 positive) + +In other words, SE and SP are the diagonal elements of the contingency table for the classifier predictions (`predictions1` and `predictions2`) with respect to the ground truth `references`. + +## How to use + +The McNemar comparison calculates the proportions of responses that exhibit disagreement between two classifiers. It is used to analyze paired nominal data. + +## Inputs + +Its arguments are: + +`predictions1`: a list of predictions from the first model. + +`predictions2`: a list of predictions from the second model. + +`references`: a list of the ground truth reference labels. + +## Output values + +The McNemar comparison outputs two things: + +`stat`: The McNemar statistic. + +`p`: The p value. + +## Examples + +Example comparison: + +```python +mcnemar = evaluate.load("mcnemar") +results = mcnemar.compute(references=[1, 0, 1], predictions1=[1, 1, 1], predictions2=[1, 0, 1]) +print(results) +{'stat': 1.0, 'p': 0.31731050786291115} +``` + +## Limitations and bias + +The McNemar test is a non-parametric test, so it has relatively few assumptions (basically only that the observations are independent). It should be used to analyze paired nominal data only. + +## Citations + +```bibtex +@article{mcnemar1947note, + title={Note on the sampling error of the difference between correlated proportions or percentages}, + author={McNemar, Quinn}, + journal={Psychometrika}, + volume={12}, + number={2}, + pages={153--157}, + year={1947}, + publisher={Springer-Verlag} +} +``` diff --git a/TextClassification/TextClassification/evaluate/comparisons/mcnemar/app.py b/TextClassification/TextClassification/evaluate/comparisons/mcnemar/app.py new file mode 100644 index 0000000000000000000000000000000000000000..6d5b0970c9d00743c31656a2d5c993402d3e9263 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/comparisons/mcnemar/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("mcnemar", module_type="comparison") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/comparisons/mcnemar/mcnemar.py b/TextClassification/TextClassification/evaluate/comparisons/mcnemar/mcnemar.py new file mode 100644 index 0000000000000000000000000000000000000000..86b85b5e33d74260bb78d83a0225dc004729db5d --- /dev/null +++ b/TextClassification/TextClassification/evaluate/comparisons/mcnemar/mcnemar.py @@ -0,0 +1,98 @@ +# Copyright 2022 The HuggingFace Evaluate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""McNemar test for model comparison.""" + +import datasets +from scipy.stats import chi2 + +import evaluate + + +_DESCRIPTION = """ +McNemar's test is a diagnostic test over a contingency table resulting from the predictions of two classifiers. The test compares the sensitivity and specificity of the diagnostic tests on the same group reference labels. It can be computed with: +McNemar = (SE - SP)**2 / SE + SP + Where: +SE: Sensitivity (Test 1 positive; Test 2 negative) +SP: Specificity (Test 1 negative; Test 2 positive) +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions1 (`list` of `int`): Predicted labels for model 1. + predictions2 (`list` of `int`): Predicted labels for model 2. + references (`list` of `int`): Ground truth labels. + +Returns: + stat (`float`): McNemar test score. + p (`float`): The p value. Minimum possible value is 0. Maximum possible value is 1.0. A lower p value means a more significant difference. + +Examples: + >>> mcnemar = evaluate.load("mcnemar") + >>> results = mcnemar.compute(references=[1, 0, 1], predictions1=[1, 1, 1], predictions2=[1, 0, 1]) + >>> print(results) + {'stat': 1.0, 'p': 0.31731050786291115} +""" + + +_CITATION = """ +@article{mcnemar1947note, + title={Note on the sampling error of the difference between correlated proportions or percentages}, + author={McNemar, Quinn}, + journal={Psychometrika}, + volume={12}, + number={2}, + pages={153--157}, + year={1947}, + publisher={Springer-Verlag} +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class McNemar(evaluate.Comparison): + def _info(self): + return evaluate.ComparisonInfo( + module_type="comparison", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions1": datasets.Value("int64"), + "predictions2": datasets.Value("int64"), + "references": datasets.Value("int64"), + } + ), + ) + + def _compute(self, predictions1, predictions2, references): + # construct contingency table + tbl = [[0, 0], [0, 0]] + for gt, p1, p2 in zip(references, predictions1, predictions2): + if p1 == gt and p2 == gt: + tbl[0][0] += 1 + elif p1 == gt: + tbl[0][1] += 1 + elif p2 == gt: + tbl[1][0] += 1 + else: + tbl[1][1] += 1 + + # compute statistic + b, c = tbl[0][1], tbl[1][0] + statistic = abs(b - c) ** 2 / (1.0 * (b + c)) + df = 1 + pvalue = chi2.sf(statistic, df) + return {"stat": statistic, "p": pvalue} diff --git a/TextClassification/TextClassification/evaluate/comparisons/mcnemar/requirements.txt b/TextClassification/TextClassification/evaluate/comparisons/mcnemar/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a410b4ca49fe91f97db76956af15309d2dec7bb8 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/comparisons/mcnemar/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scipy \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/comparisons/wilcoxon/README.md b/TextClassification/TextClassification/evaluate/comparisons/wilcoxon/README.md new file mode 100644 index 0000000000000000000000000000000000000000..17e5248d4a4f49cecada5a8c6719f3bee85d995a --- /dev/null +++ b/TextClassification/TextClassification/evaluate/comparisons/wilcoxon/README.md @@ -0,0 +1,70 @@ +--- +title: Wilcoxon +emoji: 🤗 +colorFrom: blue +colorTo: green +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- comparison +description: >- + Wilcoxon's test is a signed-rank test for comparing paired samples. +--- + + +# Comparison Card for Wilcoxon + +## Comparison description + +Wilcoxon's test is a non-parametric signed-rank test that tests whether the distribution of the differences is symmetric about zero. It can be used to compare the predictions of two models. + +## How to use + +The Wilcoxon comparison is used to analyze paired ordinal data. + +## Inputs + +Its arguments are: + +`predictions1`: a list of predictions from the first model. + +`predictions2`: a list of predictions from the second model. + +## Output values + +The Wilcoxon comparison outputs two things: + +`stat`: The Wilcoxon statistic. + +`p`: The p value. + +## Examples + +Example comparison: + +```python +wilcoxon = evaluate.load("wilcoxon") +results = wilcoxon.compute(predictions1=[-7, 123.45, 43, 4.91, 5], predictions2=[1337.12, -9.74, 1, 2, 3.21]) +print(results) +{'stat': 5.0, 'p': 0.625} +``` + +## Limitations and bias + +The Wilcoxon test is a non-parametric test, so it has relatively few assumptions (basically only that the observations are independent). It should be used to analyze paired ordinal data only. + +## Citations + +```bibtex +@incollection{wilcoxon1992individual, + title={Individual comparisons by ranking methods}, + author={Wilcoxon, Frank}, + booktitle={Breakthroughs in statistics}, + pages={196--202}, + year={1992}, + publisher={Springer} +} +``` diff --git a/TextClassification/TextClassification/evaluate/comparisons/wilcoxon/app.py b/TextClassification/TextClassification/evaluate/comparisons/wilcoxon/app.py new file mode 100644 index 0000000000000000000000000000000000000000..1c12bf33be18033b4fc086c4db95a426266f595c --- /dev/null +++ b/TextClassification/TextClassification/evaluate/comparisons/wilcoxon/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("wilcoxon", module_type="comparison") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/comparisons/wilcoxon/requirements.txt b/TextClassification/TextClassification/evaluate/comparisons/wilcoxon/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..928da8437c330671424203a5de68732e7cd21a3b --- /dev/null +++ b/TextClassification/TextClassification/evaluate/comparisons/wilcoxon/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@a45df1eb9996eec64ec3282ebe554061cb366388 +datasets~=2.0 +scipy diff --git a/TextClassification/TextClassification/evaluate/comparisons/wilcoxon/wilcoxon.py b/TextClassification/TextClassification/evaluate/comparisons/wilcoxon/wilcoxon.py new file mode 100644 index 0000000000000000000000000000000000000000..a2dc1671b1f7e25f633cd3030a91cf374993c37a --- /dev/null +++ b/TextClassification/TextClassification/evaluate/comparisons/wilcoxon/wilcoxon.py @@ -0,0 +1,78 @@ +# Copyright 2022 The HuggingFace Evaluate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Wilcoxon test for model comparison.""" + +import datasets +from scipy.stats import wilcoxon + +import evaluate + + +_DESCRIPTION = """ +Wilcoxon's test is a non-parametric signed-rank test that tests whether the distribution of the differences is symmetric about zero. It can be used to compare the predictions of two models. +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions1 (`list` of `float`): Predictions for model 1. + predictions2 (`list` of `float`): Predictions for model 2. + +Returns: + stat (`float`): Wilcoxon test score. + p (`float`): The p value. Minimum possible value is 0. Maximum possible value is 1.0. A lower p value means a more significant difference. + +Examples: + >>> wilcoxon = evaluate.load("wilcoxon") + >>> results = wilcoxon.compute(predictions1=[-7, 123.45, 43, 4.91, 5], predictions2=[1337.12, -9.74, 1, 2, 3.21]) + >>> print(results) + {'stat': 5.0, 'p': 0.625} +""" + + +_CITATION = """ +@incollection{wilcoxon1992individual, + title={Individual comparisons by ranking methods}, + author={Wilcoxon, Frank}, + booktitle={Breakthroughs in statistics}, + pages={196--202}, + year={1992}, + publisher={Springer} +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Wilcoxon(evaluate.Comparison): + def _info(self): + return evaluate.ComparisonInfo( + module_type="comparison", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions1": datasets.Value("float"), + "predictions2": datasets.Value("float"), + } + ), + ) + + def _compute(self, predictions1, predictions2): + # calculate difference + d = [p1 - p2 for (p1, p2) in zip(predictions1, predictions2)] + + # compute statistic + res = wilcoxon(d) + return {"stat": res.statistic, "p": res.pvalue} diff --git a/TextClassification/TextClassification/evaluate/docs/README.md b/TextClassification/TextClassification/evaluate/docs/README.md new file mode 100644 index 0000000000000000000000000000000000000000..62b07e07690f7c26bb1a9e3e0a8c621f665bc5c0 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/README.md @@ -0,0 +1,285 @@ + + +# Generating the documentation + +To generate the documentation, you first have to build it. Several packages are necessary to build the doc, +you can install them with the following command, at the root of the code repository: + +```bash +pip install -e ".[docs]" +``` + +Then you need to install our special tool that builds the documentation: + +```bash +pip install git+https://github.com/huggingface/doc-builder +``` + +--- +**NOTE** + +You only need to generate the documentation to inspect it locally (if you're planning changes and want to +check how they look like before committing for instance). You don't have to commit the built documentation. + +--- + +## Building the documentation + +Once you have setup the `doc-builder` and additional packages, you can generate the documentation by typing th +following command: + +```bash +doc-builder build transformers docs/source/ --build_dir ~/tmp/test-build +``` + +You can adapt the `--build_dir` to set any temporary folder that you prefer. This command will create it and generate +the MDX files that will be rendered as the documentation on the main website. You can inspect them in your favorite +Markdown editor. + +--- +**NOTE** + +It's not possible to see locally how the final documentation will look like for now. Once you have opened a PR, you +will see a bot add a comment to a link where the documentation with your changes lives. + +--- + +## Adding a new element to the navigation bar + +Accepted files are Markdown (.md or .mdx). + +Create a file with its extension and put it in the source directory. You can then link it to the toc-tree by putting +the filename without the extension in the [`_toctree.yml`](https://github.com/huggingface/transformers/blob/master/docs/source/_toctree.yml) file. + +## Renaming section headers and moving sections + +It helps to keep the old links working when renaming section header and/or moving sections from one document to another. This is because the old links are likely to be used in Issues, Forums and Social media and it'd be make for a much more superior user experience if users reading those months later could still easily navigate to the originally intended information. + +Therefore we simply keep a little map of moved sections at the end of the document where the original section was. The key is to preserve the original anchor. + +So if you renamed a section from: "Section A" to "Section B", then you can add at the end of the file: + +``` +Sections that were moved: + +[ Section A ] +``` +and of course if you moved it to another file, then: + +``` +Sections that were moved: + +[ Section A ] +``` + +Use the relative style to link to the new file so that the versioned docs continue to work. + +For an example of a rich moved sections set please see the very end of [the Trainer doc](https://github.com/huggingface/transformers/blob/master/docs/source/main_classes/trainer.mdx). + + +## Writing Documentation - Specification + +The `huggingface/transformers` documentation follows the +[Google documentation](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) style for docstrings, +although we can write them directly in Markdown. + +### Adding a new tutorial + +Adding a new tutorial or section is done in two steps: + +- Add a new file under `./source`. This file can either be ReStructuredText (.rst) or Markdown (.md). +- Link that file in `./source/_toctree.yml` on the correct toc-tree. + +Make sure to put your new file under the proper section. It's unlikely to go in the first section (*Get Started*), so +depending on the intended targets (beginners, more advanced users or researchers) it should go in section two, three or +four. + +### Adding a new model + +When adding a new model: + +- Create a file `xxx.mdx` or under `./source/model_doc` (don't hesitate to copy an existing file as template). +- Link that file in `./source/_toctree.yml`. +- Write a short overview of the model: + - Overview with paper & authors + - Paper abstract + - Tips and tricks and how to use it best +- Add the classes that should be linked in the model. This generally includes the configuration, the tokenizer, and + every model of that class (the base model, alongside models with additional heads), both in PyTorch and TensorFlow. + The order is generally: + - Configuration, + - Tokenizer + - PyTorch base model + - PyTorch head models + - TensorFlow base model + - TensorFlow head models + - Flax base model + - Flax head models + +These classes should be added using our Markdown syntax. Usually as follows: + +``` +## XXXConfig + +[[autodoc]] XXXConfig +``` + +This will include every public method of the configuration that is documented. If for some reason you wish for a method +not to be displayed in the documentation, you can do so by specifying which methods should be in the docs: + +``` +## XXXTokenizer + +[[autodoc]] XXXTokenizer + - build_inputs_with_special_tokens + - get_special_tokens_mask + - create_token_type_ids_from_sequences + - save_vocabulary +``` + +If you just want to add a method that is not documented (for instance magic method like `__call__` are not documented +byt default) you can put the list of methods to add in a list that contains `all`: + +``` +## XXXTokenizer + +[[autodoc]] XXXTokenizer + - all + - __call__ +``` + +### Writing source documentation + +Values that should be put in `code` should either be surrounded by backticks: \`like so\`. Note that argument names +and objects like True, None or any strings should usually be put in `code`. + +When mentioning a class, function or method, it is recommended to use our syntax for internal links so that our tool +adds a link to its documentation with this syntax: \[\`XXXClass\`\] or \[\`function\`\]. This requires the class or +function to be in the main package. + +If you want to create a link to some internal class or function, you need to +provide its path. For instance: \[\`file_utils.ModelOutput\`\]. This will be converted into a link with +`file_utils.ModelOutput` in the description. To get rid of the path and only keep the name of the object you are +linking to in the description, add a ~: \[\`~file_utils.ModelOutput\`\] will generate a link with `ModelOutput` in the description. + +The same works for methods so you can either use \[\`XXXClass.method\`\] or \[~\`XXXClass.method\`\]. + +#### Defining arguments in a method + +Arguments should be defined with the `Args:` (or `Arguments:` or `Parameters:`) prefix, followed by a line return and +an indentation. The argument should be followed by its type, with its shape if it is a tensor, a colon and its +description: + +``` + Args: + n_layers (`int`): The number of layers of the model. +``` + +If the description is too long to fit in one line, another indentation is necessary before writing the description +after th argument. + +Here's an example showcasing everything so far: + +``` + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`AlbertTokenizer`]. See [`~PreTrainedTokenizer.encode`] and + [`~PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) +``` + +For optional arguments or arguments with defaults we follow the following syntax: imagine we have a function with the +following signature: + +``` +def my_function(x: str = None, a: float = 1): +``` + +then its documentation should look like this: + +``` + Args: + x (`str`, *optional*): + This argument controls ... + a (`float`, *optional*, defaults to 1): + This argument is used to ... +``` + +Note that we always omit the "defaults to \`None\`" when None is the default for any argument. Also note that even +if the first line describing your argument type and its default gets long, you can't break it on several lines. You can +however write as many lines as you want in the indented description (see the example above with `input_ids`). + +#### Writing a multi-line code block + +Multi-line code blocks can be useful for displaying examples. They are done between two lines of three backticks as usual in Markdown: + + +```` +``` +# first line of code +# second line +# etc +``` +```` + +We follow the [doctest](https://docs.python.org/3/library/doctest.html) syntax for the examples to automatically test +the results stay consistent with the library. + +#### Writing a return block + +The return block should be introduced with the `Returns:` prefix, followed by a line return and an indentation. +The first line should be the type of the return, followed by a line return. No need to indent further for the elements +building the return. + +Here's an example for a single value return: + +``` + Returns: + `List[int]`: A list of integers in the range [0, 1] --- 1 for a special token, 0 for a sequence token. +``` + +Here's an example for tuple return, comprising several objects: + +``` + Returns: + `tuple(torch.FloatTensor)` comprising various elements depending on the configuration ([`BertConfig`]) and inputs: + - ** loss** (*optional*, returned when `masked_lm_labels` is provided) `torch.FloatTensor` of shape `(1,)` -- + Total loss as the sum of the masked language modeling loss and the next sequence prediction (classification) loss. + - **prediction_scores** (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`) -- + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). +``` + +#### Adding an image + +Due to the rapidly growing repository, it is important to make sure that no files that would significantly weigh down the repository are added. This includes images, videos and other non-text files. We prefer to leverage a hf.co hosted `dataset` like +the ones hosted on [`hf-internal-testing`](https://huggingface.co/hf-internal-testing) in which to place these files and reference +them by URL. We recommend putting them in the following dataset: [huggingface/documentation-images](https://huggingface.co/datasets/huggingface/documentation-images). +If an external contribution, feel free to add the images to your PR and ask a Hugging Face member to migrate your images +to this dataset. + +## Styling the docstring + +We have an automatic script running with the `make style` comment that will make sure that: +- the docstrings fully take advantage of the line width +- all code examples are formatted using black, like the code of the Transformers library + +This script may have some weird failures if you made a syntax mistake or if you uncover a bug. Therefore, it's +recommended to commit your changes before running `make style`, so you can revert the changes done by that script +easily. diff --git a/TextClassification/TextClassification/evaluate/docs/source/_toctree.yml b/TextClassification/TextClassification/evaluate/docs/source/_toctree.yml new file mode 100644 index 0000000000000000000000000000000000000000..a79f3a43f4372795cc73394897d8efd834933f37 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/_toctree.yml @@ -0,0 +1,52 @@ +- sections: + - local: index + title: 🤗 Evaluate + title: Get started +- sections: + - local: installation + title: Installation + - local: a_quick_tour + title: A quick tour + title: Tutorials +- sections: + - local: choosing_a_metric + title: Choosing the right metric + - local: creating_and_sharing + title: Adding new evaluations + - local: base_evaluator + title: Using the evaluator + - local: custom_evaluator + title: Using the evaluator with custom pipelines + - local: evaluation_suite + title: Creating an EvaluationSuite + - sections: + - local: transformers_integrations + title: Transformers + - local: keras_integrations + title: Keras and Tensorflow + - local: sklearn_integrations + title: scikit-learn + title: Using 🤗 Evaluate with other ML frameworks + title: "How-to guides" +- sections: + - local: types_of_evaluations + title: Types of evaluations + - local: considerations + title: Considerations for model evaluation + title: "Conceptual guides" +- sections: + - local: package_reference/main_classes + title: Main classes + - local: package_reference/loading_methods + title: Loading methods + - local: package_reference/saving_methods + title: Saving methods + - local: package_reference/hub_methods + title: Hub methods + - local: package_reference/evaluator_classes + title: Evaluator classes + - local: package_reference/visualization_methods + title: Visualization methods + - local: package_reference/logging_methods + title: Logging methods + title: "Reference" diff --git a/TextClassification/TextClassification/evaluate/docs/source/a_quick_tour.mdx b/TextClassification/TextClassification/evaluate/docs/source/a_quick_tour.mdx new file mode 100644 index 0000000000000000000000000000000000000000..371f7f117cb83f499345398137c5e69261fb2d80 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/a_quick_tour.mdx @@ -0,0 +1,380 @@ +# A quick tour + +🤗 Evaluate provides access to a wide range of evaluation tools. It covers a range of modalities such as text, computer vision, audio, etc. as well as tools to evaluate models or datasets. These tools are split into three categories. + +## Types of evaluations + +There are different aspects of a typical machine learning pipeline that can be evaluated and for each aspect 🤗 Evaluate provides a tool: + +- **Metric**: A metric is used to evaluate a model's performance and usually involves the model's predictions as well as some ground truth labels. You can find all integrated metrics at [evaluate-metric](https://huggingface.co/evaluate-metric). +- **Comparison**: A comparison is used to compare two models. This can for example be done by comparing their predictions to ground truth labels and computing their agreement. You can find all integrated comparisons at [evaluate-comparison](https://huggingface.co/evaluate-comparison). +- **Measurement**: The dataset is as important as the model trained on it. With measurements one can investigate a dataset's properties. You can find all integrated measurements at [evaluate-measurement](https://huggingface.co/evaluate-measurement). + +Each of these evaluation modules live on Hugging Face Hub as a Space. They come with an interactive widget and a documentation card documenting its use and limitations. For example [accuracy](https://huggingface.co/spaces/evaluate-metric/accuracy): + +
+ +
+ +Each metric, comparison, and measurement is a separate Python module, but for using any of them, there is a single entry point: [`evaluate.load`]! + +## Load + +Any metric, comparison, or measurement is loaded with the `evaluate.load` function: + +```py +>>> import evaluate +>>> accuracy = evaluate.load("accuracy") +``` + +If you want to make sure you are loading the right type of evaluation (especially if there are name clashes) you can explicitly pass the type: + +```py +>>> word_length = evaluate.load("word_length", module_type="measurement") +``` + +### Community modules + +Besides the modules implemented in 🤗 Evaluate you can also load any community module by specifying the repository ID of the metric implementation: + +```py +>>> element_count = evaluate.load("lvwerra/element_count", module_type="measurement") +``` + +See the [Creating and Sharing Guide](/docs/evaluate/main/en/creating_and_sharing) for information about uploading custom metrics. + +### List available modules + +With [`list_evaluation_modules`] you can check what modules are available on the hub. You can also filter for a specific modules and skip community metrics if you want. You can also see additional information such as likes: + +```python +>>> evaluate.list_evaluation_modules( +... module_type="comparison", +... include_community=False, +... with_details=True) + +[{'name': 'mcnemar', 'type': 'comparison', 'community': False, 'likes': 1}, + {'name': 'exact_match', 'type': 'comparison', 'community': False, 'likes': 0}] +``` + +## Module attributes + +All evalution modules come with a range of useful attributes that help to use a module stored in a [`EvaluationModuleInfo`] object. + +|Attribute|Description| +|---|---| +|`description`|A short description of the evaluation module.| +|`citation`|A BibTex string for citation when available.| +|`features`|A `Features` object defining the input format.| +|`inputs_description`|This is equivalent to the modules docstring.| +|`homepage`|The homepage of the module.| +|`license`|The license of the module.| +|`codebase_urls`|Link to the code behind the module.| +|`reference_urls`|Additional reference URLs.| + +Let's have a look at a few examples. First, let's look at the `description` attribute of the accuracy metric: + +```py +>>> accuracy = evaluate.load("accuracy") +>>> accuracy.description +Accuracy is the proportion of correct predictions among the total number of cases processed. It can be computed with: +Accuracy = (TP + TN) / (TP + TN + FP + FN) + Where: +TP: True positive +TN: True negative +FP: False positive +FN: False negative +``` + +You can see that it describes how the metric works in theory. If you use this metric for your work, especially if it is an academic publication you want to reference it properly. For that you can look at the `citation` attribute: + +```py +>>> accuracy.citation +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + +Before we can apply a metric or other evaluation module to a use-case, we need to know what the input format of the metric is: + +```py +>>> accuracy.features +{ + 'predictions': Value(dtype='int32', id=None), + 'references': Value(dtype='int32', id=None) +} +``` + + + +Note that features always describe the type of a single input element. In general we will add lists of elements so you can always think of a list around the types in `features`. Evaluate accepts various input formats (Python lists, NumPy arrays, PyTorch tensors, etc.) and converts them to an appropriate format for storage and computation. + + + +## Compute + +Now that we know how the evaluation module works and what should go in there we want to actually use it! When it comes to computing the actual score there are two main ways to do it: + +1. All-in-one +2. Incremental + + +In the incremental approach the necessary inputs are added to the module with [`EvaluationModule.add`] or [`EvaluationModule.add_batch`] and the score is calculated at the end with [`EvaluationModule.compute`]. Alternatively, one can pass all the inputs at once to `compute()`. Let's have a look at the two approaches. + +### How to compute + +The simplest way to calculate the score of an evaluation module is by calling `compute()` directly with the necessary inputs. Simply pass the inputs as seen in `features` to the `compute()` method. + +```py +>>> accuracy.compute(references=[0,1,0,1], predictions=[1,0,0,1]) +{'accuracy': 0.5} +``` +Evaluation modules return the results in a dictionary. However, in some instances you build up the predictions iteratively or in a distributed fashion in which case `add()` or `add_batch()` are useful. + +### Calculate a single metric or a batch of metrics + +In many evaluation pipelines you build the predictions iteratively such as in a for-loop. In that case you could store the predictions in a list and at the end pass them to `compute()`. With `add()` and `add_batch()` you can circumvent the step of storing the predictions separately. If you are only creating single predictions at a time you can use `add()`: + +```py +>>> for ref, pred in zip([0,1,0,1], [1,0,0,1]): +>>> accuracy.add(references=ref, predictions=pred) +>>> accuracy.compute() +{'accuracy': 0.5} +``` + +Once you have gathered all predictions you can call `compute()` to compute the score based on all stored values. When getting predictions and references in batches you can use `add_batch()` which adds a list elements for later processing. The rest works as with `add()`: + +```py +>>> for refs, preds in zip([[0,1],[0,1]], [[1,0],[0,1]]): +>>> accuracy.add_batch(references=refs, predictions=preds) +>>> accuracy.compute() +{'accuracy': 0.5} +``` + +This is especially useful when you need to get the predictions from your model in batches: + +```py +>>> for model_inputs, gold_standards in evaluation_dataset: +>>> predictions = model(model_inputs) +>>> metric.add_batch(references=gold_standards, predictions=predictions) +>>> metric.compute() +``` + +### Distributed evaluation + +Computing metrics in a distributed environment can be tricky. Metric evaluation is executed in separate Python processes, or nodes, on different subsets of a dataset. Typically, when a metric score is additive (`f(AuB) = f(A) + f(B)`), you can use distributed reduce operations to gather the scores for each subset of the dataset. But when a metric is non-additive (`f(AuB) ≠ f(A) + f(B)`), it's not that simple. For example, you can't take the sum of the [F1](https://huggingface.co/spaces/evaluate-metric/f1) scores of each data subset as your **final metric**. + +A common way to overcome this issue is to fallback on single process evaluation. The metrics are evaluated on a single GPU, which becomes inefficient. + +🤗 Evaluate solves this issue by only computing the final metric on the first node. The predictions and references are computed and provided to the metric separately for each node. These are temporarily stored in an Apache Arrow table, avoiding cluttering the GPU or CPU memory. When you are ready to `compute()` the final metric, the first node is able to access the predictions and references stored on all the other nodes. Once it has gathered all the predictions and references, `compute()` will perform the final metric evaluation. + +This solution allows 🤗 Evaluate to perform distributed predictions, which is important for evaluation speed in distributed settings. At the same time, you can also use complex non-additive metrics without wasting valuable GPU or CPU memory. + +## Combining several evaluations + +Often one wants to not only evaluate a single metric but a range of different metrics capturing different aspects of a model. E.g. for classification it is usually a good idea to compute F1-score, recall, and precision in addition to accuracy to get a better picture of model performance. Naturally, you can load a bunch of metrics and call them sequentially. However, a more convenient way is to use the [`~evaluate.combine`] function to bundle them together: + +```python +>>> clf_metrics = evaluate.combine(["accuracy", "f1", "precision", "recall"]) +``` + +The `combine` function accepts both the list of names of the metrics as well as an instantiated modules. The `compute` call then computes each metric: + +```python +>>> clf_metrics.compute(predictions=[0, 1, 0], references=[0, 1, 1]) + +{ + 'accuracy': 0.667, + 'f1': 0.667, + 'precision': 1.0, + 'recall': 0.5 +} +``` + +## Save and push to the Hub + +Saving and sharing evaluation results is an important step. We provide the [`evaluate.save`] function to easily save metrics results. You can either pass a specific filename or a directory. In the latter case, the results are saved in a file with an automatically created file name. Besides the directory or file name, the function takes any key-value pairs as inputs and stores them in a JSON file. + +```py +>>> result = accuracy.compute(references=[0,1,0,1], predictions=[1,0,0,1]) + +>>> hyperparams = {"model": "bert-base-uncased"} +>>> evaluate.save("./results/", experiment="run 42", **result, **hyperparams) +PosixPath('results/result-2022_05_30-22_09_11.json') +``` + +The content of the JSON file look like the following: + +```json +{ + "experiment": "run 42", + "accuracy": 0.5, + "model": "bert-base-uncased", + "_timestamp": "2022-05-30T22:09:11.959469", + "_git_commit_hash": "123456789abcdefghijkl", + "_evaluate_version": "0.1.0", + "_python_version": "3.9.12 (main, Mar 26 2022, 15:51:15) \n[Clang 13.1.6 (clang-1316.0.21.2)]", + "_interpreter_path": "/Users/leandro/git/evaluate/env/bin/python" +} +``` + +In addition to the specified fields, it also contains useful system information for reproducing the results. + +Besides storing the results locally, you should report them on the model's repository on the Hub. With the [`evaluate.push_to_hub`] function, you can easily report evaluation results to the model's repository: + +```py +evaluate.push_to_hub( + model_id="huggingface/gpt2-wikitext2", # model repository on hub + metric_value=0.5, # metric value + metric_type="bleu", # metric name, e.g. accuracy.name + metric_name="BLEU", # pretty name which is displayed + dataset_type="wikitext", # dataset name on the hub + dataset_name="WikiText", # pretty name + dataset_split="test", # dataset split used + task_type="text-generation", # task id, see https://github.com/huggingface/evaluate/blob/main/src/evaluate/config.py#L154-L192 + task_name="Text Generation" # pretty name for task +) +``` + +## Evaluator + +The [`evaluate.evaluator`] provides automated evaluation and only requires a model, dataset, metric in contrast to the metrics in `EvaluationModule`s that require the model's predictions. As such it is easier to evaluate a model on a dataset with a given metric as the inference is handled internally. To make that possible it uses the [`~transformers.pipeline`] abstraction from `transformers`. However, you can use your own framework as long as it follows the `pipeline` interface. + +To make an evaluation with the `evaluator` let's load a `transformers` pipeline (but you can pass your own custom inference class for any framework as long as it follows the pipeline call API) with an model trained on IMDb, the IMDb test split and the accuracy metric. + +```python +from transformers import pipeline +from datasets import load_dataset +from evaluate import evaluator +import evaluate + +pipe = pipeline("text-classification", model="lvwerra/distilbert-imdb", device=0) +data = load_dataset("imdb", split="test").shuffle().select(range(1000)) +metric = evaluate.load("accuracy") +``` + +Then you can create an evaluator for text classification and pass the three objects to the `compute()` method. With the label mapping `evaluate` provides a method to align the pipeline outputs with the label column in the dataset: + +```python +>>> task_evaluator = evaluator("text-classification") + +>>> results = task_evaluator.compute(model_or_pipeline=pipe, data=data, metric=metric, +... label_mapping={"NEGATIVE": 0, "POSITIVE": 1},) + +>>> print(results) +{'accuracy': 0.934} +``` + +Calculating the value of the metric alone is often not enough to know if a model performs significantly better than another one. With _bootstrapping_ `evaluate` computes confidence intervals and the standard error which helps estimate how stable a score is: + +```python +>>> results = eval.compute(model_or_pipeline=pipe, data=data, metric=metric, +... label_mapping={"NEGATIVE": 0, "POSITIVE": 1}, +... strategy="bootstrap", n_resamples=200) + +>>> print(results) +{'accuracy': + { + 'confidence_interval': (0.906, 0.9406749892841922), + 'standard_error': 0.00865213251082787, + 'score': 0.923 + } +} +``` + +The evaluator expects a `"text"` and `"label"` column for the data input. If your dataset differs you can provide the columns with the keywords `input_column="text"` and `label_column="label"`. Currently only `"text-classification"` is supported with more tasks being added in the future. + + +## Visualization + +When comparing several models, sometimes it's hard to spot the differences in their performance simply by looking at their scores. Also often there is not a single best model but there are trade-offs between e.g. latency and accuracy as larger models might have better performance but are also slower. We are gradually adding different visualization approaches, such as plots, to make choosing the best model for a use-case easier. + +For instance, if you have a list of results from multiple models (as dictionaries), you can feed them into the `radar_plot()` function: + +```python +import evaluate +from evaluate.visualization import radar_plot + +>>> data = [ + {"accuracy": 0.99, "precision": 0.8, "f1": 0.95, "latency_in_seconds": 33.6}, + {"accuracy": 0.98, "precision": 0.87, "f1": 0.91, "latency_in_seconds": 11.2}, + {"accuracy": 0.98, "precision": 0.78, "f1": 0.88, "latency_in_seconds": 87.6}, + {"accuracy": 0.88, "precision": 0.78, "f1": 0.81, "latency_in_seconds": 101.6} + ] +>>> model_names = ["Model 1", "Model 2", "Model 3", "Model 4"] +>>> plot = radar_plot(data=data, model_names=model_names) +>>> plot.show() +``` + +Which lets you visually compare the 4 models and choose the optimal one for you, based on one or several metrics: +
+ +
+ +## Running evaluation on a suite of tasks + +It can be useful to evaluate models on a variety of different tasks to understand their downstream performance. The [EvaluationSuite](evaluation_suite) enables evaluation of models on a collection of tasks. Tasks can be constructed as ([evaluator](base_evaluator), dataset, metric) tuples and passed to an [EvaluationSuite](evaluation_suite) stored on the Hugging Face Hub as a Space, or locally as a Python script. See the [evaluator documentation](base_evaluator) for a list of currently supported tasks. + +`EvaluationSuite` scripts can be defined as follows, and supports Python code for data preprocessing. + +```python +import evaluate +from evaluate.evaluation_suite import SubTask + +class Suite(evaluate.EvaluationSuite): + + def __init__(self, name): + super().__init__(name) + + self.suite = [ + SubTask( + task_type="text-classification", + data="imdb", + split="test[:1]", + args_for_task={ + "metric": "accuracy", + "input_column": "text", + "label_column": "label", + "label_mapping": { + "LABEL_0": 0.0, + "LABEL_1": 1.0 + } + } + ), + SubTask( + task_type="text-classification", + data="sst2", + split="test[:1]", + args_for_task={ + "metric": "accuracy", + "input_column": "sentence", + "label_column": "label", + "label_mapping": { + "LABEL_0": 0.0, + "LABEL_1": 1.0 + } + } + ) + ] +``` + +Evaluation can be run by loading the `EvaluationSuite` and calling the `run()` method with a model or pipeline. + +``` +>>> from evaluate import EvaluationSuite +>>> suite = EvaluationSuite.load('mathemakitten/sentiment-evaluation-suite') +>>> results = suite.run("huggingface/prunebert-base-uncased-6-finepruned-w-distil-mnli") +``` + +| accuracy | total_time_in_seconds | samples_per_second | latency_in_seconds | task_name | +|------------:|---------------------:|--------------------------:|:----------------|:-----------| +| 0.3 | 4.62804 | 2.16074 | 0.462804 | imdb | +| 0 | 0.686388 | 14.569 | 0.0686388 | sst2 | diff --git a/TextClassification/TextClassification/evaluate/docs/source/base_evaluator.mdx b/TextClassification/TextClassification/evaluate/docs/source/base_evaluator.mdx new file mode 100644 index 0000000000000000000000000000000000000000..043c825189d3ae8616a141c90d9ebd0fb1434ba7 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/base_evaluator.mdx @@ -0,0 +1,294 @@ +# Using the `evaluator` + +The `Evaluator` classes allow to evaluate a triplet of model, dataset, and metric. The models wrapped in a pipeline, responsible for handling all preprocessing and post-processing and out-of-the-box, `Evaluator`s support transformers pipelines for the supported tasks, but custom pipelines can be passed, as showcased in the section [Using the `evaluator` with custom pipelines](custom_evaluator). + +Currently supported tasks are: +- `"text-classification"`: will use the [`TextClassificationEvaluator`]. +- `"token-classification"`: will use the [`TokenClassificationEvaluator`]. +- `"question-answering"`: will use the [`QuestionAnsweringEvaluator`]. +- `"image-classification"`: will use the [`ImageClassificationEvaluator`]. +- `"text-generation"`: will use the [`TextGenerationEvaluator`]. +- `"text2text-generation"`: will use the [`Text2TextGenerationEvaluator`]. +- `"summarization"`: will use the [`SummarizationEvaluator`]. +- `"translation"`: will use the [`TranslationEvaluator`]. +- `"automatic-speech-recognition"`: will use the [`AutomaticSpeechRecognitionEvaluator`]. +- `"audio-classification"`: will use the [`AudioClassificationEvaluator`]. + +To run an `Evaluator` with several tasks in a single call, use the [EvaluationSuite](evaluation_suite), which runs evaluations on a collection of `SubTask`s. + +Each task has its own set of requirements for the dataset format and pipeline output, make sure to check them out for your custom use case. Let's have a look at some of them and see how you can use the evaluator to evalute a single or multiple of models, datasets, and metrics at the same time. + +## Text classification + +The text classification evaluator can be used to evaluate text models on classification datasets such as IMDb. Beside the model, data, and metric inputs it takes the following optional inputs: + +- `input_column="text"`: with this argument the column with the data for the pipeline can be specified. +- `label_column="label"`: with this argument the column with the labels for the evaluation can be specified. +- `label_mapping=None`: the label mapping aligns the labels in the pipeline output with the labels need for evaluation. E.g. the labels in `label_column` can be integers (`0`/`1`) whereas the pipeline can produce label names such as `"positive"`/`"negative"`. With that dictionary the pipeline outputs are mapped to the labels. + +By default the `"accuracy"` metric is computed. + +### Evaluate models on the Hub + +There are several ways to pass a model to the evaluator: you can pass the name of a model on the Hub, you can load a `transformers` model and pass it to the evaluator or you can pass an initialized `transformers.Pipeline`. Alternatively you can pass any callable function that behaves like a `pipeline` call for the task in any framework. + +So any of the following works: + +```py +from datasets import load_dataset +from evaluate import evaluator +from transformers import AutoModelForSequenceClassification, pipeline + +data = load_dataset("imdb", split="test").shuffle(seed=42).select(range(1000)) +task_evaluator = evaluator("text-classification") + +# 1. Pass a model name or path +eval_results = task_evaluator.compute( + model_or_pipeline="lvwerra/distilbert-imdb", + data=data, + label_mapping={"NEGATIVE": 0, "POSITIVE": 1} +) + +# 2. Pass an instantiated model +model = AutoModelForSequenceClassification.from_pretrained("lvwerra/distilbert-imdb") + +eval_results = task_evaluator.compute( + model_or_pipeline=model, + data=data, + label_mapping={"NEGATIVE": 0, "POSITIVE": 1} +) + +# 3. Pass an instantiated pipeline +pipe = pipeline("text-classification", model="lvwerra/distilbert-imdb") + +eval_results = task_evaluator.compute( + model_or_pipeline=pipe, + data=data, + label_mapping={"NEGATIVE": 0, "POSITIVE": 1} +) +print(eval_results) +``` + + +Without specifying a device, the default for model inference will be the first GPU on the machine if one is available, and else CPU. If you want to use a specific device you can pass `device` to `compute` where -1 will use the GPU and a positive integer (starting with 0) will use the associated CUDA device. + + + + +The results will look as follows: +```python +{ + 'accuracy': 0.918, + 'latency_in_seconds': 0.013, + 'samples_per_second': 78.887, + 'total_time_in_seconds': 12.676 +} +``` + +Note that evaluation results include both the requested metric, and information about the time it took to obtain predictions through the pipeline. + + + +The time performances can give useful indication on model speed for inference but should be taken with a grain of salt: they include all the processing that goes on in the pipeline. This may include tokenizing, post-processing, that may be different depending on the model. Furthermore, it depends a lot on the hardware you are running the evaluation on and you may be able to improve the performance by optimizing things like the batch size. + + + +### Evaluate multiple metrics + +With the [`combine`] function one can bundle several metrics into an object that behaves like a single metric. We can use this to evaluate several metrics at once with the evaluator: + +```python +import evaluate + +eval_results = task_evaluator.compute( + model_or_pipeline="lvwerra/distilbert-imdb", + data=data, + metric=evaluate.combine(["accuracy", "recall", "precision", "f1"]), + label_mapping={"NEGATIVE": 0, "POSITIVE": 1} +) +print(eval_results) + +``` +The results will look as follows: +```python +{ + 'accuracy': 0.918, + 'f1': 0.916, + 'precision': 0.9147, + 'recall': 0.9187, + 'latency_in_seconds': 0.013, + 'samples_per_second': 78.887, + 'total_time_in_seconds': 12.676 +} +``` + +Next let's have a look at token classification. + +## Token Classification + +With the token classification evaluator one can evaluate models for tasks such as NER or POS tagging. It has the following specific arguments: + +- `input_column="text"`: with this argument the column with the data for the pipeline can be specified. +- `label_column="label"`: with this argument the column with the labels for the evaluation can be specified. +- `label_mapping=None`: the label mapping aligns the labels in the pipeline output with the labels need for evaluation. E.g. the labels in `label_column` can be integers (`0`/`1`) whereas the pipeline can produce label names such as `"positive"`/`"negative"`. With that dictionary the pipeline outputs are mapped to the labels. +- `join_by=" "`: While most datasets are already tokenized the pipeline expects a string. Thus the tokens need to be joined before passing to the pipeline. By default they are joined with a whitespace. + +Let's have a look how we can use the evaluator to benchmark several models. + +### Benchmarking several models + +Here is an example where several models can be compared thanks to the `evaluator` in only a few lines of code, abstracting away the preprocessing, inference, postprocessing, metric computation: + +```python +import pandas as pd +from datasets import load_dataset +from evaluate import evaluator +from transformers import pipeline + +models = [ + "xlm-roberta-large-finetuned-conll03-english", + "dbmdz/bert-large-cased-finetuned-conll03-english", + "elastic/distilbert-base-uncased-finetuned-conll03-english", + "dbmdz/electra-large-discriminator-finetuned-conll03-english", + "gunghio/distilbert-base-multilingual-cased-finetuned-conll2003-ner", + "philschmid/distilroberta-base-ner-conll2003", + "Jorgeutd/albert-base-v2-finetuned-ner", +] + +data = load_dataset("conll2003", split="validation").shuffle().select(range(1000)) +task_evaluator = evaluator("token-classification") + +results = [] +for model in models: + results.append( + task_evaluator.compute( + model_or_pipeline=model, data=data, metric="seqeval" + ) + ) + +df = pd.DataFrame(results, index=models) +df[["overall_f1", "overall_accuracy", "total_time_in_seconds", "samples_per_second", "latency_in_seconds"]] +``` + +The result is a table that looks like this: + +| model | overall_f1 | overall_accuracy | total_time_in_seconds | samples_per_second | latency_in_seconds | +|:-------------------------------------------------------------------|-------------:|-------------------:|------------------------:|---------------------:|---------------------:| +| Jorgeutd/albert-base-v2-finetuned-ner | 0.941 | 0.989 | 4.515 | 221.468 | 0.005 | +| dbmdz/bert-large-cased-finetuned-conll03-english | 0.962 | 0.881 | 11.648 | 85.850 | 0.012 | +| dbmdz/electra-large-discriminator-finetuned-conll03-english | 0.965 | 0.881 | 11.456 | 87.292 | 0.011 | +| elastic/distilbert-base-uncased-finetuned-conll03-english | 0.940 | 0.989 | 2.318 | 431.378 | 0.002 | +| gunghio/distilbert-base-multilingual-cased-finetuned-conll2003-ner | 0.947 | 0.991 | 2.376 | 420.873 | 0.002 | +| philschmid/distilroberta-base-ner-conll2003 | 0.961 | 0.994 | 2.436 | 410.579 | 0.002 | +| xlm-roberta-large-finetuned-conll03-english | 0.969 | 0.882 | 11.996 | 83.359 | 0.012 | + + +### Visualizing results + +You can feed in the `results` list above into the `plot_radar()` function to visualize different aspects of their performance and choose the model that is the best fit, depending on the metric(s) that are relevant to your use case: + +```python +import evaluate +from evaluate.visualization import radar_plot + +>>> plot = radar_plot(data=results, model_names=models, invert_range=["latency_in_seconds"]) +>>> plot.show() +``` + +
+ +
+ + +Don't forget to specify `invert_range` for metrics for which smaller is better (such as the case for latency in seconds). + +If you want to save the plot locally, you can use the `plot.savefig()` function with the option `bbox_inches='tight'`, to make sure no part of the image gets cut off. + + +## Question Answering + +With the question-answering evaluator one can evaluate models for QA without needing to worry about the complicated pre- and post-processing that's required for these models. It has the following specific arguments: + + +- `question_column="question"`: the name of the column containing the question in the dataset +- `context_column="context"`: the name of the column containing the context +- `id_column="id"`: the name of the column cointaing the identification field of the question and answer pair +- `label_column="answers"`: the name of the column containing the answers +- `squad_v2_format=None`: whether the dataset follows the format of squad_v2 dataset where a question may have no answer in the context. If this parameter is not provided, the format will be automatically inferred. + +Let's have a look how we can evaluate QA models and compute confidence intervals at the same time. + +### Confidence intervals + +Every evaluator comes with the options to compute confidence intervals using [bootstrapping](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html). Simply pass `strategy="bootstrap"` and set the number of resanmples with `n_resamples`. + +```python +from datasets import load_dataset +from evaluate import evaluator + +task_evaluator = evaluator("question-answering") + +data = load_dataset("squad", split="validation[:1000]") +eval_results = task_evaluator.compute( + model_or_pipeline="distilbert-base-uncased-distilled-squad", + data=data, + metric="squad", + strategy="bootstrap", + n_resamples=30 +) +``` + +Results include confidence intervals as well as error estimates as follows: + +```python +{ + 'exact_match': + { + 'confidence_interval': (79.67, 84.54), + 'score': 82.30, + 'standard_error': 1.28 + }, + 'f1': + { + 'confidence_interval': (85.30, 88.88), + 'score': 87.23, + 'standard_error': 0.97 + }, + 'latency_in_seconds': 0.0085, + 'samples_per_second': 117.31, + 'total_time_in_seconds': 8.52 + } +``` + +## Image classification + +With the image classification evaluator we can evaluate any image classifier. It uses the same keyword arguments at the text classifier: + +- `input_column="image"`: the name of the column containing the images as PIL ImageFile +- `label_column="label"`: the name of the column containing the labels +- `label_mapping=None`: We want to map class labels defined by the model in the pipeline to values consistent with those defined in the `label_column` + +Let's have a look at how can evaluate image classification models on large datasets. + +### Handling large datasets + +The evaluator can be used on large datasets! Below, an example shows how to use it on ImageNet-1k for image classification. Beware that this example will require to download ~150 GB. + +```python +data = load_dataset("imagenet-1k", split="validation", use_auth_token=True) + +pipe = pipeline( + task="image-classification", + model="facebook/deit-small-distilled-patch16-224" +) + +task_evaluator = evaluator("image-classification") +eval_results = task_evaluator.compute( + model_or_pipeline=pipe, + data=data, + metric="accuracy", + label_mapping=pipe.model.config.label2id +) +``` + +Since we are using `datasets` to store data we make use of a technique called memory mappings. This means that the dataset is never fully loaded into memory which saves a lot of RAM. Running the above code only uses roughly 1.5 GB of RAM while the validation split is more than 30 GB big. diff --git a/TextClassification/TextClassification/evaluate/docs/source/choosing_a_metric.mdx b/TextClassification/TextClassification/evaluate/docs/source/choosing_a_metric.mdx new file mode 100644 index 0000000000000000000000000000000000000000..344f830586a8cb78fefd327d1de1472c7d3b02c3 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/choosing_a_metric.mdx @@ -0,0 +1,64 @@ +# Choosing a metric for your task + +**So you've trained your model and want to see how well it’s doing on a dataset of your choice. Where do you start?** + +There is no “one size fits all” approach to choosing an evaluation metric, but some good guidelines to keep in mind are: + +## Categories of metrics + +There are 3 high-level categories of metrics: + +1. *Generic metrics*, which can be applied to a variety of situations and datasets, such as precision and accuracy. +2. *Task-specific metrics*, which are limited to a given task, such as Machine Translation (often evaluated using metrics [BLEU](https://huggingface.co/metrics/bleu) or [ROUGE](https://huggingface.co/metrics/rouge)) or Named Entity Recognition (often evaluated with [seqeval](https://huggingface.co/metrics/seqeval)). +3. *Dataset-specific metrics*, which aim to measure model performance on specific benchmarks: for instance, the [GLUE benchmark](https://huggingface.co/datasets/glue) has a dedicated [evaluation metric](https://huggingface.co/metrics/glue). + +Let's look at each of these three cases: + +### Generic metrics + +Many of the metrics used in the Machine Learning community are quite generic and can be applied in a variety of tasks and datasets. + +This is the case for metrics like [accuracy](https://huggingface.co/metrics/accuracy) and [precision](https://huggingface.co/metrics/precision), which can be used for evaluating labeled (supervised) datasets, as well as [perplexity](https://huggingface.co/metrics/perplexity), which can be used for evaluating different kinds of (unsupervised) generative tasks. + +To see the input structure of a given metric, you can look at its metric card. For example, in the case of [precision](https://huggingface.co/metrics/precision), the format is: +``` +>>> precision_metric = evaluate.load("precision") +>>> results = precision_metric.compute(references=[0, 1], predictions=[0, 1]) +>>> print(results) +{'precision': 1.0} +``` + +### Task-specific metrics + +Popular ML tasks like Machine Translation and Named Entity Recognition have specific metrics that can be used to compare models. For example, a series of different metrics have been proposed for text generation, ranging from [BLEU](https://huggingface.co/metrics/bleu) and its derivatives such as [GoogleBLEU](https://huggingface.co/metrics/google_bleu) and [GLEU](https://huggingface.co/metrics/gleu), but also [ROUGE](https://huggingface.co/metrics/rouge), [MAUVE](https://huggingface.co/metrics/mauve), etc. + +You can find the right metric for your task by: + +- **Looking at the [Task pages](https://huggingface.co/tasks)** to see what metrics can be used for evaluating models for a given task. +- **Checking out leaderboards** on sites like [Papers With Code](https://paperswithcode.com/) (you can search by task and by dataset). +- **Reading the metric cards** for the relevant metrics and see which ones are a good fit for your use case. For example, see the [BLEU metric card](https://github.com/huggingface/evaluate/tree/main/metrics/bleu) or [SQuaD metric card](https://github.com/huggingface/evaluate/tree/main/metrics/squad). +- **Looking at papers and blog posts** published on the topic and see what metrics they report. This can change over time, so try to pick papers from the last couple of years! + +### Dataset-specific metrics + +Some datasets have specific metrics associated with them -- this is especially in the case of popular benchmarks like [GLUE](https://huggingface.co/metrics/glue) and [SQuAD](https://huggingface.co/metrics/squad). + + +💡 +GLUE is actually a collection of different subsets on different tasks, so first you need to choose the one that corresponds to the NLI task, such as mnli, which is described as “crowdsourced collection of sentence pairs with textual entailment annotations” + + + +If you are evaluating your model on a benchmark dataset like the ones mentioned above, you can use its dedicated evaluation metric. Make sure you respect the format that they require. For example, to evaluate your model on the [SQuAD](https://huggingface.co/datasets/squad) dataset, you need to feed the `question` and `context` into your model and return the `prediction_text`, which should be compared with the `references` (based on matching the `id` of the question) : + +``` +>>> from evaluate import load +>>> squad_metric = load("squad") +>>> predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22'}] +>>> references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] +>>> results = squad_metric.compute(predictions=predictions, references=references) +>>> results +{'exact_match': 100.0, 'f1': 100.0} +``` + +You can find examples of dataset structures by consulting the "Dataset Preview" function or the dataset card for a given dataset, and you can see how to use its dedicated evaluation function based on the metric card. diff --git a/TextClassification/TextClassification/evaluate/docs/source/considerations.mdx b/TextClassification/TextClassification/evaluate/docs/source/considerations.mdx new file mode 100644 index 0000000000000000000000000000000000000000..dc8ca5ccd141b983432e7b094e7a377fe3dcb123 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/considerations.mdx @@ -0,0 +1,88 @@ +# Considerations for model evaluation + +Developing an ML model is rarely a one-shot deal: it often involves multiple stages of defining the model architecture and tuning hyper-parameters before converging on a final set. Responsible model evaluation is a key part of this process, and 🤗 Evaluate is here to help! + +Here are some things to keep in mind when evaluating your model using the 🤗 Evaluate library: + +## Properly splitting your data + +Good evaluation generally requires three splits of your dataset: + +- **train**: this is used for training your model. +- **validation**: this is used for validating the model hyperparameters. +- **test**: this is used for evaluating your model. + +Many of the datasets on the 🤗 Hub are separated into 2 splits: `train` and `validation`; others are split into 3 splits (`train`, `validation` and `test`) -- make sure to use the right split for the right purpose! + +Some datasets on the 🤗 Hub are already separated into these three splits. However, there are also many that only have a train/validation or only train split. + +If the dataset you're using doesn't have a predefined train-test split, it is up to you to define which part of the dataset you want to use for training your model and which you want to use for hyperparameter tuning or final evaluation. + + +Training and evaluating on the same split can misrepresent your results! If you overfit on your training data the evaluation results on that split will look great but the model will perform poorly on new data. + + +Depending on the size of the dataset, you can keep anywhere from 10-30% for evaluation and the rest for training, while aiming to set up the test set to reflect the production data as close as possible. Check out [this thread](https://discuss.huggingface.co/t/how-to-split-main-dataset-into-train-dev-test-as-datasetdict/1090) for a more in-depth discussion of dataset splitting! + +## The impact of class imbalance + +While many academic datasets, such as the [IMDb dataset](https://huggingface.co/datasets/imdb) of movie reviews, are perfectly balanced, most real-world datasets are not. In machine learning a *balanced dataset* corresponds to a datasets where all labels are represented equally. In the case of the IMDb dataset this means that there are as many positive as negative reviews in the dataset. In an imbalanced dataset this is not the case: in fraud detection for example there are usually many more non-fraud cases than fraud cases in the dataset. + +Having an imbalanced dataset can skew the results of your metrics. Imagine a dataset with 99 "non-fraud" cases and 1 "fraud" case. A simple model that always predicts "non-fraud" cases would give yield a 99% accuracy which might sound good at first until you realize that you will never catch a fraud case. + +Often, using more than one metric can help get a better idea of your model’s performance from different points of view. For instance, metrics like **[recall](https://huggingface.co/metrics/recall)** and **[precision](https://huggingface.co/metrics/precision)** can be used together, and the **[f1 score](https://huggingface.co/metrics/f1)** is actually the harmonic mean of the two. + +In cases where a dataset is balanced, using [accuracy](https://huggingface.co/metrics/accuracy) can reflect the overall model performance: + +![Balanced Labels](https://huggingface.co/datasets/evaluate/media/resolve/main/balanced-classes.png) + +In cases where there is an imbalance, using [F1 score](https://huggingface.co/metrics/f1) can be a better representation of performance, given that it encompasses both precision and recall. + +![Imbalanced Labels](https://huggingface.co/datasets/evaluate/media/resolve/main/imbalanced-classes.png) + +Using accuracy in an imbalanced setting is less ideal, since it is not sensitive to minority classes and will not faithfully reflect model performance on them. + +## Offline vs. online model evaluation + +There are multiple ways to evaluate models, and an important distinction is offline versus online evaluation: + +**Offline evaluation** is done before deploying a model or using insights generated from a model, using static datasets and metrics. + +**Online evaluation** means evaluating how a model is performing after deployment and during its use in production. + +These two types of evaluation can use different metrics and measure different aspects of model performance. For example, offline evaluation can compare a model to other models based on their performance on common benchmarks, whereas online evaluation will evaluate aspects such as latency and accuracy of the model based on production data (for example, the number of user queries that it was able to address). + +## Trade-offs in model evaluation + +When evaluating models in practice, there are often trade-offs that have to be made between different aspects of model performance: for instance, choosing a model that is slightly less accurate but that has a faster inference time, compared to a high-accuracy that has a higher memory footprint and requires access to more GPUs. + +Here are other aspects of model performance to consider during evaluation: + +### Interpretability + +When evaluating models, **interpretability** (i.e. the ability to *interpret* results) can be very important, especially when deploying models in production. + +For instance, metrics such as [exact match](https://huggingface.co/spaces/evaluate-metric/exact_match) have a set range (between 0 and 1, or 0% and 100%) and are easily understandable to users: for a pair of strings, the exact match score is 1 if the two strings are the exact same, and 0 otherwise. + +Other metrics, such as [BLEU](https://huggingface.co/spaces/evaluate-metric/exact_match) are harder to interpret: while they also range between 0 and 1, they can vary greatly depending on which parameters are used to generate the scores, especially when different tokenization and normalization techniques are used (see the [metric card](https://huggingface.co/spaces/evaluate-metric/bleu/blob/main/README.md) for more information about BLEU limitations). This means that it is difficult to interpret a BLEU score without having more information about the procedure used for obtaining it. + +Interpretability can be more or less important depending on the evaluation use case, but it is a useful aspect of model evaluation to keep in mind, since communicating and comparing model evaluations is an important part of responsible machine learning. + + +### Inference speed and memory footprint + +While recent years have seen increasingly large ML models achieve high performance on a large variety of tasks and benchmarks, deploying these multi-billion parameter models in practice can be a challenge in itself, and many organizations lack the resources for this. This is why considering the **inference speed** and **memory footprint** of models is important, especially when doing online model evaluation. + +Inference speed refers to the time that it takes for a model to make a prediction -- this will vary depending on the hardware used and the way in which models are queried, e.g. in real time via an API or in batch jobs that run once a day. + +Memory footprint refers to the size of the model weights and how much hardware memory they occupy. If a model is too large to fit on a single GPU or CPU, then it has to be split over multiple ones, which can be more or less difficult depending on the model architecture and the deployment method. + +When doing online model evaluation, there is often a trade-off to be done between inference speed and accuracy or precision, whereas this is less the case for offline evaluation. + +## Limitations and bias + +All models and all metrics have their limitations and biases, which depend on the way in which they were trained, the data that was used, and their intended uses. It is important to measure and communicate these limitations clearly to prevent misuse and unintended impacts, for instance via [model cards](https://huggingface.co/course/chapter4/4?fw=pt) which document the training and evaluation process. + +Measuring biases can be done by evaluating models on datasets such as [Wino Bias](https://huggingface.co/datasets/wino_bias) or [MD Gender Bias](https://huggingface.co/datasets/md_gender_bias), and by doing [Interactive Error Analyis](https://huggingface.co/spaces/nazneen/error-analysis) to try to identify which subsets of the evaluation dataset a model performs poorly on. + +We are currently working on additional measurements that can be used to quantify different dimensions of bias in both models and datasets -- stay tuned for more documentation on this topic! diff --git a/TextClassification/TextClassification/evaluate/docs/source/creating_and_sharing.mdx b/TextClassification/TextClassification/evaluate/docs/source/creating_and_sharing.mdx new file mode 100644 index 0000000000000000000000000000000000000000..c1b96d3fcc64c1fd5be5afa6a2dc26ed602e6d83 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/creating_and_sharing.mdx @@ -0,0 +1,113 @@ +# Creating and sharing a new evaluation + +## Setup + +Before you can create a new metric make sure you have all the necessary dependencies installed: + +```bash +pip install evaluate[template] +``` + +Also make sure your Hugging Face token is registered so you can connect to the Hugging Face Hub: + +```bash +huggingface-cli login +``` + +## Create + +All evaluation modules, be it metrics, comparisons, or measurements live on the 🤗 Hub in a [Space](https://huggingface.co/docs/hub/spaces) (see for example [Accuracy](https://huggingface.co/spaces/evaluate-metric/accuracy)). In principle, you could setup a new Space and add a new module following the same structure. However, we added a CLI that makes creating a new evaluation module much easier: + +```bash +evaluate-cli create "My Metric" --module_type "metric" +``` + +This will create a new Space on the 🤗 Hub, clone it locally, and populate it with a template. Instructions on how to fill the template will be displayed in the terminal, but are also explained here in more detail. + +For more information about Spaces, see the [Spaces documentation](https://huggingface.co/docs/hub/spaces). + +## Module script + +The evaluation module script (the file with suffix `*.py`) is the core of the new module and includes all the code for computing the evaluation. + +### Attributes + +Start by adding some information about your evalution module in [`EvaluationModule._info`]. The most important attributes you should specify are: + +1. [`EvaluationModuleInfo.description`] provides a brief description about your evalution module. + +2. [`EvaluationModuleInfo.citation`] contains a BibTex citation for the evalution module. + +3. [`EvaluationModuleInfo.inputs_description`] describes the expected inputs and outputs. It may also provide an example usage of the evalution module. + +4. [`EvaluationModuleInfo.features`] defines the name and type of the predictions and references. This has to be either a single `datasets.Features` object or a list of `datasets.Features` objects if multiple input types are allowed. + +Then, we can move on to prepare everything before the actual computation. + +### Download + +Some evaluation modules require some external data such as NLTK that requires resources or the BLEURT metric that requires checkpoints. You can implement these downloads in [`EvaluationModule._download_and_prepare`], which downloads and caches the resources via the `dlmanager`. A simplified example on how BLEURT downloads and loads a checkpoint: + +```py +def _download_and_prepare(self, dl_manager): + model_path = dl_manager.download_and_extract(CHECKPOINT_URLS[self.config_name]) + self.scorer = score.BleurtScorer(os.path.join(model_path, self.config_name)) +``` + +Or if you need to download the NLTK `"punkt"` resources: + +```py +def _download_and_prepare(self, dl_manager): + import nltk + nltk.download("punkt") +``` + +Next, we need to define how the computation of the evaluation module works. + +### Compute + +The computation is performed in the [`EvaluationModule._compute`] method. It takes the same arguments as `EvaluationModuleInfo.features` and should then return the result as a dictionary. Here an example of an exact match metric: + + +```py +def _compute(self, references, predictions): + em = sum([r==p for r, p in zip(references, predictions)])/len(references) + return {"exact_match": em} +``` + +This method is used when you call `.compute()` later on. + +## Readme + +When you use the `evalute-cli` to setup the evaluation module the Readme structure and instructions are automatically created. It should include a general description of the metric, information about its input/output format, examples as well as information about its limiations or biases and references. + +## Requirements + +If your evaluation modules has additional dependencies (e.g. `sklearn` or `nltk`) the `requirements.txt` files is the place to put them. The file follows the `pip` format and you can list all dependencies there. + +## App + +The `app.py` is where the Spaces widget lives. In general it looks like the following and does not require any changes: + +```py +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("lvwerra/element_count") +launch_gradio_widget(module) +``` + +If you want a custom widget you could add your gradio app here. + +## Push to Hub + +Finally, when you are done with all the above changes it is time to push your evaluation module to the hub. To do so navigate to the folder of your module and git add/commit/push the changes to the hub: + +``` +cd PATH_TO_MODULE +git add . +git commit -m "Add my new, shiny module." +git push +``` +Tada 🎉! Your evaluation module is now on the 🤗 Hub and ready to be used by everybody! diff --git a/TextClassification/TextClassification/evaluate/docs/source/custom_evaluator.mdx b/TextClassification/TextClassification/evaluate/docs/source/custom_evaluator.mdx new file mode 100644 index 0000000000000000000000000000000000000000..288e17e8104b7578b23cb20bc71ade3f07f53442 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/custom_evaluator.mdx @@ -0,0 +1,114 @@ +# Using the `evaluator` with custom pipelines + +The evaluator is designed to work with `transformer` pipelines out-of-the-box. However, in many cases you might have a model or pipeline that's not part of the `transformer` ecosystem. You can still use `evaluator` to easily compute metrics for them. In this guide we show how to do this for a Scikit-Learn [pipeline](https://scikit-learn.org/stable/modules/generated/sklearn.pipeline.Pipeline.html#sklearn.pipeline.Pipeline) and a Spacy [pipeline](https://spacy.io). Let's start with the Scikit-Learn case. + +## Scikit-Learn + +First we need to train a model. We'll train a simple text classifier on the [IMDb dataset](https://huggingface.co/datasets/imdb), so let's start by downloading the dataset: + +```py +from datasets import load_dataset + +ds = load_dataset("imdb") +``` + +Then we can build a simple TF-IDF preprocessor and Naive Bayes classifier wrapped in a `Pipeline`: + +```py +from sklearn.pipeline import Pipeline +from sklearn.naive_bayes import MultinomialNB +from sklearn.feature_extraction.text import TfidfTransformer +from sklearn.feature_extraction.text import CountVectorizer + +text_clf = Pipeline([ + ('vect', CountVectorizer()), + ('tfidf', TfidfTransformer()), + ('clf', MultinomialNB()), +]) + +text_clf.fit(ds["train"]["text"], ds["train"]["label"]) +``` + +Following the convention in the `TextClassificationPipeline` of `transformers` our pipeline should be callable and return a list of dictionaries. In addition we use the `task` attribute to check if the pipeline is compatible with the `evaluator`. We can write a small wrapper class for that purpose: + +```py +class ScikitEvalPipeline: + def __init__(self, pipeline): + self.pipeline = pipeline + self.task = "text-classification" + + def __call__(self, input_texts, **kwargs): + return [{"label": p} for p in self.pipeline.predict(input_texts)] + +pipe = ScikitEvalPipeline(text_clf) +``` + +We can now pass this `pipeline` to the `evaluator`: + +```py +from evaluate import evaluator + +task_evaluator = evaluator("text-classification") +task_evaluator.compute(pipe, ds["test"], "accuracy") + +>>> {'accuracy': 0.82956} +``` + +Implementing that simple wrapper is all that's needed to use any model from any framework with the `evaluator`. In the `__call__` you can implement all logic necessary for efficient forward passes through your model. + +## Spacy + +We'll use the `polarity` feature of the `spacytextblob` project to get a simple sentiment analyzer. First you'll need to install the project and download the resources: + +```bash +pip install spacytextblob +python -m textblob.download_corpora +python -m spacy download en_core_web_sm +``` + +Then we can simply load the `nlp` pipeline and add the `spacytextblob` pipeline: +```py +import spacy + +nlp = spacy.load('en_core_web_sm') +nlp.add_pipe('spacytextblob') +``` + +This snippet shows how we can use the `polarity` feature added with `spacytextblob` to get the sentiment of a text: + +```py +texts = ["This movie is horrible", "This movie is awesome"] +results = nlp.pipe(texts) + +for txt, res in zip(texts, results): + print(f"{text} | Polarity: {res._.blob.polarity}") +``` + +Now we can wrap it in a simple wrapper class like in the Scikit-Learn example before. It just has to return a list of dictionaries with the predicted lables. If the polarity is larger than 0 we'll predict positive sentiment and negative otherwise: + +```py +class SpacyEvalPipeline: + def __init__(self, nlp): + self.nlp = nlp + self.task = "text-classification" + + def __call__(self, input_texts, **kwargs): + results =[] + for p in self.nlp.pipe(input_texts): + if p._.blob.polarity>=0: + results.append({"label": 1}) + else: + results.append({"label": 0}) + return results + +pipe = SpacyEvalPipeline(nlp) +``` + +That class is compatible with the `evaluator` and we can use the same instance from the previous examlpe along with the IMDb test set: + +```py +eval.compute(pipe, ds["test"], "accuracy") +>>> {'accuracy': 0.6914} +``` + +This will take a little longer than the Scikit-Learn example but after roughly 10-15min you will have the evaluation results! diff --git a/TextClassification/TextClassification/evaluate/docs/source/evaluation_suite.mdx b/TextClassification/TextClassification/evaluate/docs/source/evaluation_suite.mdx new file mode 100644 index 0000000000000000000000000000000000000000..d44b08a13b0bb2d11b8e80b10c5888e0001fd752 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/evaluation_suite.mdx @@ -0,0 +1,74 @@ +# Creating an EvaluationSuite + +It can be useful to evaluate models on a variety of different tasks to understand their downstream performance. Assessing the model on several types of tasks can reveal gaps in performance along some axis. For example, when training a language model, it is often useful to measure perplexity on an in-domain corpus, but also to concurrently evaluate on tasks which test for general language capabilities like natural language entailment or question-answering, or tasks designed to probe the model along fairness and bias dimensions. + +The `EvaluationSuite` provides a way to compose any number of ([evaluator](base_evaluator), dataset, metric) tuples as a SubTask to evaluate a model on a collection of several evaluation tasks. See the [evaluator documentation](base_evaluator) for a list of currently supported tasks. + +A new `EvaluationSuite` is made up of a list of `SubTask` classes, each defining an evaluation task. The Python file containing the definition can be uploaded to a Space on the Hugging Face Hub so it can be shared with the community or saved/loaded locally as a Python script. + +Some datasets require additional preprocessing before passing them to an `Evaluator`. You can set a `data_preprocessor` for each `SubTask` which is applied via a `map` operation using the `datasets` library. Keyword arguments for the `Evaluator` can be passed down through the `args_for_task` attribute. + +To create a new `EvaluationSuite`, create a [new Space](https://huggingface.co/new-space) with a .py file which matches the name of the Space, add the below template to a Python file, and fill in the attributes for a new task. + +The mandatory attributes for a new `SubTask` are `task_type` and `data`. +1. [`task_type`] maps to the tasks currently supported by the Evaluator. +2. [`data`] can be an instantiated Hugging Face dataset object or the name of a dataset. +3. [`subset`] and [`split`] can be used to define which name and split of the dataset should be used for evaluation. +4. [`args_for_task`] should be a dictionary with kwargs to be passed to the Evaluator. + +```python +import evaluate +from evaluate.evaluation_suite import SubTask + +class Suite(evaluate.EvaluationSuite): + + def __init__(self, name): + super().__init__(name) + self.preprocessor = lambda x: {"text": x["text"].lower()} + self.suite = [ + SubTask( + task_type="text-classification", + data="glue", + subset="sst2", + split="validation[:10]", + args_for_task={ + "metric": "accuracy", + "input_column": "sentence", + "label_column": "label", + "label_mapping": { + "LABEL_0": 0.0, + "LABEL_1": 1.0 + } + } + ), + SubTask( + task_type="text-classification", + data="glue", + subset="rte", + split="validation[:10]", + args_for_task={ + "metric": "accuracy", + "input_column": "sentence1", + "second_input_column": "sentence2", + "label_column": "label", + "label_mapping": { + "LABEL_0": 0, + "LABEL_1": 1 + } + } + ) + ] +``` + +An `EvaluationSuite` can be loaded by name from the Hugging Face Hub, or locally by providing a path, and run with the `run(model_or_pipeline)` method. The evaluation results are returned along with their task names and information about the time it took to obtain predictions through the pipeline. These can be easily displayed with a `pandas.DataFrame`: + +``` +>>> from evaluate import EvaluationSuite +>>> suite = EvaluationSuite.load('mathemakitten/glue-evaluation-suite') +>>> results = suite.run("gpt2") +``` + +| accuracy | total_time_in_seconds | samples_per_second | latency_in_seconds | task_name | +|-----------:|------------------------:|---------------------:|---------------------:|:------------| +| 0.5 | 0.740811 | 13.4987 | 0.0740811 | glue/sst2 | +| 0.4 | 1.67552 | 5.9683 | 0.167552 | glue/rte | diff --git a/TextClassification/TextClassification/evaluate/docs/source/index.mdx b/TextClassification/TextClassification/evaluate/docs/source/index.mdx new file mode 100644 index 0000000000000000000000000000000000000000..4729d65999c5b1953d011c04f71c2eb1b6293083 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/index.mdx @@ -0,0 +1,34 @@ +

+
+ +
+

+ +# 🤗 Evaluate + +A library for easily evaluating machine learning models and datasets. + +With a single line of code, you get access to dozens of evaluation methods for different domains (NLP, Computer Vision, Reinforcement Learning, and more!). Be it on your local machine or in a distributed training setup, you can evaluate your models in a consistent and reproducible way! + +Visit the 🤗 Evaluate [organization](https://huggingface.co/evaluate-metric) for a full list of available metrics. Each metric has a dedicated Space with an interactive demo for how to use the metric, and a documentation card detailing the metrics limitations and usage. + +
+
+
Tutorials
+

Learn the basics and become familiar with loading, computing, and saving with 🤗 Evaluate. Start here if you are using 🤗 Evaluate for the first time!

+ +
How-to guides
+

Practical guides to help you achieve a specific goal. Take a look at these guides to learn how to use 🤗 Evaluate to solve real-world problems.

+ +
Conceptual guides
+

High-level explanations for building a better understanding of important topics such as considerations going into evaluating a model or dataset and the difference between metrics, measurements, and comparisons.

+ +
Reference
+

Technical descriptions of how 🤗 Evaluate classes and methods work.

+ +
+
diff --git a/TextClassification/TextClassification/evaluate/docs/source/installation.mdx b/TextClassification/TextClassification/evaluate/docs/source/installation.mdx new file mode 100644 index 0000000000000000000000000000000000000000..88cef8b1fdb3e01c9916e3fde47a15e7881211c4 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/installation.mdx @@ -0,0 +1,68 @@ +# Installation + +Before you start, you will need to setup your environment and install the appropriate packages. 🤗 Evaluate is tested on **Python 3.7+**. + +## Virtual environment + +You should install 🤗 Evaluate in a [virtual environment](https://docs.python.org/3/library/venv.html) to keep everything neat and tidy. + +1. Create and navigate to your project directory: + + ```bash + mkdir ~/my-project + cd ~/my-project + ``` + +2. Start a virtual environment inside the directory: + + ```bash + python -m venv .env + ``` + +3. Activate and deactivate the virtual environment with the following commands: + + ```bash + # Activate the virtual environment + source .env/bin/activate + + # Deactivate the virtual environment + source .env/bin/deactivate + ``` + +Once you have created your virtual environment, you can install 🤗 Evaluate in it. + +## pip + +The most straightforward way to install 🤗 Evaluate is with pip: + +```bash +pip install evaluate +``` + +Run the following command to check if 🤗 Evaluate has been properly installed: + +```bash +python -c "import evaluate; print(evaluate.load('exact_match').compute(references=['hello'], predictions=['hello']))" +``` + +This should return: + +```bash +{'exact_match': 1.0} +``` + +## source + +Building 🤗 Evaluate from source lets you make changes to the code base. To install from source, clone the repository and install with the following commands: + +```bash +git clone https://github.com/huggingface/evaluate.git +cd evaluate +pip install -e . +``` + +Again, you can check if 🤗 Evaluate has been properly installed with: + +```bash +python -c "import evaluate; print(evaluate.load('exact_match').compute(references=['hello'], predictions=['hello']))" +``` \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/docs/source/keras_integrations.md b/TextClassification/TextClassification/evaluate/docs/source/keras_integrations.md new file mode 100644 index 0000000000000000000000000000000000000000..317078810c893387a5e0b6a697e96ac2d9483d33 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/keras_integrations.md @@ -0,0 +1,113 @@ +# Working with Keras and Tensorflow + + + +Evaluate can be easily intergrated into your Keras and Tensorflow workflow. We'll demonstrate two ways of incorporating Evaluate into model training, using the Fashion MNIST example dataset. We'll train a standard classifier to predict two classes from this dataset, and show how to use a metric as a callback during training or afterwards for evaluation. + + +```python +import numpy as np +from tensorflow import keras +from tensorflow.keras import layers +import evaluate + +# We pull example code from Keras.io's guide on classifying with MNIST +# Located here: https://keras.io/examples/vision/mnist_convnet/ + +# Model / data parameters +input_shape = (28, 28, 1) + +# Load the data and split it between train and test sets +(x_train, y_train), (x_test, y_test) = keras.datasets.fashion_mnist.load_data() + + +# Only select tshirts/tops and trousers, classes 0 and 1 +def get_tshirts_tops_and_trouser(x_vals, y_vals): + mask = np.where((y_vals == 0) | (y_vals == 1)) + return x_vals[mask], y_vals[mask] + +x_train, y_train = get_tshirts_tops_and_trouser(x_train, y_train) +x_test, y_test = get_tshirts_tops_and_trouser(x_test, y_test) + + +# Scale images to the [0, 1] range +x_train = x_train.astype("float32") / 255 +x_test = x_test.astype("float32") / 255 + +x_train = np.expand_dims(x_train, -1) +x_test = np.expand_dims(x_test, -1) + + +model = keras.Sequential( + [ + keras.Input(shape=input_shape), + layers.Conv2D(32, kernel_size=(3, 3), activation="relu"), + layers.MaxPooling2D(pool_size=(2, 2)), + layers.Conv2D(64, kernel_size=(3, 3), activation="relu"), + layers.MaxPooling2D(pool_size=(2, 2)), + layers.Flatten(), + layers.Dropout(0.5), + layers.Dense(1, activation="sigmoid"), + ] +) +``` + +## Callbacks + +Suppose we want to keep track of model metrics while a model is training. We can use a Callback in order to calculate this metric during training, after an epoch ends. + +We'll define a callback here that will take a metric name and our training data, and have it calculate a metric after the epoch ends. + + +```python +class MetricsCallback(keras.callbacks.Callback): + + def __init__(self, metric_name, x_data, y_data) -> None: + super(MetricsCallback, self).__init__() + + self.x_data = x_data + self.y_data = y_data + self.metric_name = metric_name + self.metric = evaluate.load(metric_name) + + def on_epoch_end(self, epoch, logs=dict()): + m = self.model + # Ensure we get labels of "1" or "0" + training_preds = np.round(m.predict(self.x_data)) + training_labels = self.y_data + + # Compute score and save + score = self.metric.compute(predictions = training_preds, references = training_labels) + + logs.update(score) +``` + +We can pass this class to the `callbacks` keyword-argument to use it during training: + + +```python +batch_size = 128 +epochs = 2 + +model.compile(loss="binary_crossentropy", optimizer="adam") + +model_history = model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1, +callbacks = [MetricsCallback(x_data = x_train, y_data = y_train, metric_name = "accuracy")]) +``` + +## Using an Evaluate Metric for... Evaluation! + +We can also use the same metric after model training! Here, we show how to check accuracy of the model after training on the test set: + + +```python +acc = evaluate.load("accuracy") +# Round the predictions to turn them into "0" or "1" labels +test_preds = np.round(model.predict(x_test)) +test_labels = y_test +``` + +```python +print("Test accuracy is : ", acc.compute(predictions = test_preds, references = test_labels)) +# Test accuracy is : 0.9855 +``` diff --git a/TextClassification/TextClassification/evaluate/docs/source/package_reference/evaluator_classes.mdx b/TextClassification/TextClassification/evaluate/docs/source/package_reference/evaluator_classes.mdx new file mode 100644 index 0000000000000000000000000000000000000000..1d5e74c372d6c5ab683a1058402856101f10710a --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/package_reference/evaluator_classes.mdx @@ -0,0 +1,63 @@ +# Evaluator + +The evaluator classes for automatic evaluation. + +## Evaluator classes + +The main entry point for using the evaluator: + +[[autodoc]] evaluate.evaluator + +The base class for all evaluator classes: + +[[autodoc]] evaluate.Evaluator + +## The task specific evaluators + +### ImageClassificationEvaluator + +[[autodoc]] evaluate.ImageClassificationEvaluator + +### QuestionAnsweringEvaluator + +[[autodoc]] evaluate.QuestionAnsweringEvaluator + - compute + +### TextClassificationEvaluator + +[[autodoc]] evaluate.TextClassificationEvaluator + +### TokenClassificationEvaluator + +[[autodoc]] evaluate.TokenClassificationEvaluator + - compute + +### TextGenerationEvaluator + +[[autodoc]] evaluate.TextGenerationEvaluator + - compute + +### Text2TextGenerationEvaluator + +[[autodoc]] evaluate.Text2TextGenerationEvaluator + - compute + +### SummarizationEvaluator + +[[autodoc]] evaluate.SummarizationEvaluator + - compute + +### TranslationEvaluator + +[[autodoc]] evaluate.TranslationEvaluator + - compute + +### AutomaticSpeechRecognitionEvaluator + +[[autodoc]] evaluate.AutomaticSpeechRecognitionEvaluator + - compute + +### AudioClassificationEvaluator + +[[autodoc]] evaluate.AudioClassificationEvaluator + - compute \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/docs/source/package_reference/hub_methods.mdx b/TextClassification/TextClassification/evaluate/docs/source/package_reference/hub_methods.mdx new file mode 100644 index 0000000000000000000000000000000000000000..de26950d1d5fdfe7038dcbf7c92872902ea1e774 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/package_reference/hub_methods.mdx @@ -0,0 +1,8 @@ +# Hub methods + +Methods for using the Hugging Face Hub: + +## Push to hub + +[[autodoc]] evaluate.push_to_hub + diff --git a/TextClassification/TextClassification/evaluate/docs/source/package_reference/loading_methods.mdx b/TextClassification/TextClassification/evaluate/docs/source/package_reference/loading_methods.mdx new file mode 100644 index 0000000000000000000000000000000000000000..a86d2913ddef140abeeebc0c85d35fba93741dd6 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/package_reference/loading_methods.mdx @@ -0,0 +1,11 @@ +# Loading methods + +Methods for listing and loading evaluation modules: + +## List + +[[autodoc]] evaluate.list_evaluation_modules + +## Load + +[[autodoc]] evaluate.load diff --git a/TextClassification/TextClassification/evaluate/docs/source/package_reference/logging_methods.mdx b/TextClassification/TextClassification/evaluate/docs/source/package_reference/logging_methods.mdx new file mode 100644 index 0000000000000000000000000000000000000000..4e10e799a8075bc458ef18ba92e9653ef1b89a62 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/package_reference/logging_methods.mdx @@ -0,0 +1,89 @@ +# Logging methods + +🤗 Evaluate strives to be transparent and explicit about how it works, but this can be quite verbose at times. We have included a series of logging methods which allow you to easily adjust the level of verbosity of the entire library. Currently the default verbosity of the library is set to `WARNING`. + +To change the level of verbosity, use one of the direct setters. For instance, here is how to change the verbosity to the `INFO` level: + +```py +import evaluate +evaluate.logging.set_verbosity_info() +``` + +You can also use the environment variable `EVALUATE_VERBOSITY` to override the default verbosity, and set it to one of the following: `debug`, `info`, `warning`, `error`, `critical`: + +```bash +EVALUATE_VERBOSITY=error ./myprogram.py +``` + +All the methods of this logging module are documented below. The main ones are: + +- [`logging.get_verbosity`] to get the current level of verbosity in the logger +- [`logging.set_verbosity`] to set the verbosity to the level of your choice + +In order from the least to the most verbose (with their corresponding `int` values): + +1. `logging.CRITICAL` or `logging.FATAL` (int value, 50): only report the most critical errors. +2. `logging.ERROR` (int value, 40): only report errors. +3. `logging.WARNING` or `logging.WARN` (int value, 30): only reports error and warnings. This the default level used by the library. +4. `logging.INFO` (int value, 20): reports error, warnings and basic information. +5. `logging.DEBUG` (int value, 10): report all information. + +By default, `tqdm` progress bars will be displayed during evaluate download and processing. [`logging.disable_progress_bar`] and [`logging.enable_progress_bar`] can be used to suppress or unsuppress this behavior. + +## Functions + +[[autodoc]] evaluate.logging.get_verbosity + +[[autodoc]] evaluate.logging.set_verbosity + +[[autodoc]] evaluate.logging.set_verbosity_info + +[[autodoc]] evaluate.logging.set_verbosity_warning + +[[autodoc]] evaluate.logging.set_verbosity_debug + +[[autodoc]] evaluate.logging.set_verbosity_error + +[[autodoc]] evaluate.logging.disable_propagation + +[[autodoc]] evaluate.logging.enable_propagation + +[[autodoc]] evaluate.logging.get_logger + +[[autodoc]] evaluate.logging.enable_progress_bar + +[[autodoc]] evaluate.logging.disable_progress_bar + +## Levels + +### evaluate.logging.CRITICAL + +evaluate.logging.CRITICAL = 50 + +### evaluate.logging.DEBUG + +evaluate.logging.DEBUG = 10 + +### evaluate.logging.ERROR + +evaluate.logging.ERROR = 40 + +### evaluate.logging.FATAL + +evaluate.logging.FATAL = 50 + +### evaluate.logging.INFO + +evaluate.logging.INFO = 20 + +### evaluate.logging.NOTSET + +evaluate.logging.NOTSET = 0 + +### evaluate.logging.WARN + +evaluate.logging.WARN = 30 + +### evaluate.logging.WARNING + +evaluate.logging.WARNING = 30 diff --git a/TextClassification/TextClassification/evaluate/docs/source/package_reference/main_classes.mdx b/TextClassification/TextClassification/evaluate/docs/source/package_reference/main_classes.mdx new file mode 100644 index 0000000000000000000000000000000000000000..42c6b9d0ebc561ea5edefbe01afc6f3c31a30007 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/package_reference/main_classes.mdx @@ -0,0 +1,33 @@ +# Main classes + +## EvaluationModuleInfo + +The base class `EvaluationModuleInfo` implements a the logic for the subclasses `MetricInfo`, `ComparisonInfo`, and `MeasurementInfo`. + +[[autodoc]] evaluate.EvaluationModuleInfo + +[[autodoc]] evaluate.MetricInfo + +[[autodoc]] evaluate.ComparisonInfo + +[[autodoc]] evaluate.MeasurementInfo + +## EvaluationModule + +The base class `EvaluationModule` implements a the logic for the subclasses `Metric`, `Comparison`, and `Measurement`. + +[[autodoc]] evaluate.EvaluationModule + +[[autodoc]] evaluate.Metric + +[[autodoc]] evaluate.Comparison + +[[autodoc]] evaluate.Measurement + +## CombinedEvaluations + +The `combine` function allows to combine multiple `EvaluationModule`s into a single `CombinedEvaluations`. + +[[autodoc]] evaluate.combine + +[[autodoc]] CombinedEvaluations diff --git a/TextClassification/TextClassification/evaluate/docs/source/package_reference/saving_methods.mdx b/TextClassification/TextClassification/evaluate/docs/source/package_reference/saving_methods.mdx new file mode 100644 index 0000000000000000000000000000000000000000..b879653ae3770ac30d95fae6be7b964530137cb2 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/package_reference/saving_methods.mdx @@ -0,0 +1,8 @@ +# Saving methods + +Methods for saving evaluations results: + +## Save + +[[autodoc]] evaluate.save + diff --git a/TextClassification/TextClassification/evaluate/docs/source/package_reference/visualization_methods.mdx b/TextClassification/TextClassification/evaluate/docs/source/package_reference/visualization_methods.mdx new file mode 100644 index 0000000000000000000000000000000000000000..9f8cd74cf2a55437fa71b86c0a06ba339dfcf030 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/package_reference/visualization_methods.mdx @@ -0,0 +1,7 @@ +# Visualization methods + +Methods for visualizing evaluations results: + +## Radar Plot + +[[autodoc]] evaluate.visualization.radar_plot diff --git a/TextClassification/TextClassification/evaluate/docs/source/sklearn_integrations.mdx b/TextClassification/TextClassification/evaluate/docs/source/sklearn_integrations.mdx new file mode 100644 index 0000000000000000000000000000000000000000..5583c25d2a780cf7a0f691f0afe0bac6569bab5e --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/sklearn_integrations.mdx @@ -0,0 +1,88 @@ +# Scikit-Learn + +To run the scikit-learn examples make sure you have installed the following library: + +```bash +pip install -U scikit-learn +``` + +The metrics in `evaluate` can be easily integrated with an Scikit-Learn estimator or [pipeline](https://scikit-learn.org/stable/modules/generated/sklearn.pipeline.Pipeline.html#sklearn.pipeline.Pipeline). + +However, these metrics require that we generate the predictions from the model. The predictions and labels from the estimators can be passed to `evaluate` mertics to compute the required values. + +```python +import numpy as np +np.random.seed(0) +import evaluate +from sklearn.compose import ColumnTransformer +from sklearn.datasets import fetch_openml +from sklearn.pipeline import Pipeline +from sklearn.impute import SimpleImputer +from sklearn.preprocessing import StandardScaler, OneHotEncoder +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import train_test_split +``` + +Load data from https://www.openml.org/d/40945: + +```python +X, y = fetch_openml("titanic", version=1, as_frame=True, return_X_y=True) +``` + +Alternatively X and y can be obtained directly from the frame attribute: + +```python +X = titanic.frame.drop('survived', axis=1) +y = titanic.frame['survived'] +``` + +We create the preprocessing pipelines for both numeric and categorical data. Note that pclass could either be treated as a categorical or numeric feature. + +```python +numeric_features = ["age", "fare"] +numeric_transformer = Pipeline( + steps=[("imputer", SimpleImputer(strategy="median")), ("scaler", StandardScaler())] +) + +categorical_features = ["embarked", "sex", "pclass"] +categorical_transformer = OneHotEncoder(handle_unknown="ignore") + +preprocessor = ColumnTransformer( + transformers=[ + ("num", numeric_transformer, numeric_features), + ("cat", categorical_transformer, categorical_features), + ] +) +``` + +Append classifier to preprocessing pipeline. Now we have a full prediction pipeline. + +```python +clf = Pipeline( + steps=[("preprocessor", preprocessor), ("classifier", LogisticRegression())] +) + +X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0) + +clf.fit(X_train, y_train) +y_pred = clf.predict(X_test) +``` + +As `Evaluate` metrics use lists as inputs for references and predictions, we need to convert them to Python lists. + + +```python +# Evaluate metrics accept lists as inputs for values of references and predictions + +y_test = y_test.tolist() +y_pred = y_pred.tolist() + +# Accuracy + +accuracy_metric = evaluate.load("accuracy") +accuracy = accuracy_metric.compute(references=y_test, predictions=y_pred) +print("Accuracy:", accuracy) +# Accuracy: 0.79 +``` + +You can use any suitable `evaluate` metric with the estimators as long as they are compatible with the task and predictions. diff --git a/TextClassification/TextClassification/evaluate/docs/source/transformers_integrations.mdx b/TextClassification/TextClassification/evaluate/docs/source/transformers_integrations.mdx new file mode 100644 index 0000000000000000000000000000000000000000..7993b2f285e0b012b27495b56a6daf72f88f0eeb --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/transformers_integrations.mdx @@ -0,0 +1,132 @@ +# 🤗 Transformers + +To run the 🤗 Transformers examples make sure you have installed the following libraries: + +```bash +pip install datasets transformers torch evaluate nltk rouge_score +``` + +## Trainer + +The metrics in `evaluate` can be easily integrated with the [`~transformers.Trainer`]. The `Trainer` accepts a `compute_metrics` keyword argument that passes a function to compute metrics. One can specify the evaluation interval with `evaluation_strategy` in the [`~transformers.TrainerArguments`], and based on that, the model is evaluated accordingly, and the predictions and labels passed to `compute_metrics`. + +```python +from datasets import load_dataset +from transformers import AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer +import numpy as np +import evaluate + +# Prepare and tokenize dataset +dataset = load_dataset("yelp_review_full") +tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") + +def tokenize_function(examples): + return tokenizer(examples["text"], padding="max_length", truncation=True) + +tokenized_datasets = dataset.map(tokenize_function, batched=True) + +small_train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(200)) +small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(200)) + +# Setup evaluation +metric = evaluate.load("accuracy") + +def compute_metrics(eval_pred): + logits, labels = eval_pred + predictions = np.argmax(logits, axis=-1) + return metric.compute(predictions=predictions, references=labels) + +# Load pretrained model and evaluate model after each epoch +model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5) +training_args = TrainingArguments(output_dir="test_trainer", evaluation_strategy="epoch") + +trainer = Trainer( + model=model, + args=training_args, + train_dataset=small_train_dataset, + eval_dataset=small_eval_dataset, + compute_metrics=compute_metrics, +) + +trainer.train() +``` + +## Seq2SeqTrainer + +We can use the [`~transformers.Seq2SeqTrainer`] for sequence-to-sequence tasks such as translation or summarization. For such generative tasks usually metrics such as ROUGE or BLEU are evaluated. However, these metrics require that we generate some text with the model rather than a single forward pass as with e.g. classification. The `Seq2SeqTrainer` allows for the use of the generate method when setting `predict_with_generate=True` which will generate text for each sample in the evaluation set. That means we evaluate generated text within the `compute_metric` function. We just need to decode the predictions and labels first. + +```python +import nltk +from datasets import load_dataset +import evaluate +import numpy as np +from transformers import AutoTokenizer, DataCollatorForSeq2Seq +from transformers import AutoModelForSeq2SeqLM, Seq2SeqTrainingArguments, Seq2SeqTrainer + +# Prepare and tokenize dataset +billsum = load_dataset("billsum", split="ca_test").shuffle(seed=42).select(range(200)) +billsum = billsum.train_test_split(test_size=0.2) +tokenizer = AutoTokenizer.from_pretrained("t5-small") +prefix = "summarize: " + +def preprocess_function(examples): + inputs = [prefix + doc for doc in examples["text"]] + model_inputs = tokenizer(inputs, max_length=1024, truncation=True) + + labels = tokenizer(text_target=examples["summary"], max_length=128, truncation=True) + + model_inputs["labels"] = labels["input_ids"] + return model_inputs + +tokenized_billsum = billsum.map(preprocess_function, batched=True) + +# Setup evaluation +nltk.download("punkt", quiet=True) +metric = evaluate.load("rouge") + +def compute_metrics(eval_preds): + preds, labels = eval_preds + + # decode preds and labels + labels = np.where(labels != -100, labels, tokenizer.pad_token_id) + decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True) + decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True) + + # rougeLSum expects newline after each sentence + decoded_preds = ["\n".join(nltk.sent_tokenize(pred.strip())) for pred in decoded_preds] + decoded_labels = ["\n".join(nltk.sent_tokenize(label.strip())) for label in decoded_labels] + + result = metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True) + return result + +# Load pretrained model and evaluate model after each epoch +model = AutoModelForSeq2SeqLM.from_pretrained("t5-small") +data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model) + +training_args = Seq2SeqTrainingArguments( + output_dir="./results", + evaluation_strategy="epoch", + learning_rate=2e-5, + per_device_train_batch_size=16, + per_device_eval_batch_size=4, + weight_decay=0.01, + save_total_limit=3, + num_train_epochs=2, + fp16=True, + predict_with_generate=True +) + +trainer = Seq2SeqTrainer( + model=model, + args=training_args, + train_dataset=tokenized_billsum["train"], + eval_dataset=tokenized_billsum["test"], + tokenizer=tokenizer, + data_collator=data_collator, + compute_metrics=compute_metrics +) + +trainer.train() +``` + +You can use any `evaluate` metric with the `Trainer` and `Seq2SeqTrainer` as long as they are compatible with the task and predictions. In case you don't want to train a model but just evaluate an existing model you can replace `trainer.train()` with `trainer.evaluate()` in the above scripts. \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/docs/source/types_of_evaluations.mdx b/TextClassification/TextClassification/evaluate/docs/source/types_of_evaluations.mdx new file mode 100644 index 0000000000000000000000000000000000000000..7bd2cc9cb456af0160f1e1840728d3d85dcab1a7 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/docs/source/types_of_evaluations.mdx @@ -0,0 +1,35 @@ +# Types of Evaluations in 🤗 Evaluate + +The goal of the 🤗 Evaluate library is to support different types of evaluation, depending on different goals, datasets and models. + +Here are the types of evaluations that are currently supported with a few examples for each: + +## Metrics +A metric measures the performance of a model on a given dataset. This is often based on an existing ground truth (i.e. a set of references), but there are also *referenceless metrics* which allow evaluating generated text by leveraging a pretrained model such as [GPT-2](https://huggingface.co/gpt2). + +Examples of metrics include: +- [Accuracy](https://huggingface.co/metrics/accuracy) : the proportion of correct predictions among the total number of cases processed. +- [Exact Match](https://huggingface.co/metrics/exact_match): the rate at which the input predicted strings exactly match their references. +- [Mean Intersection over union (IoUO)](https://huggingface.co/metrics/mean_iou): the area of overlap between the predicted segmentation of an image and the ground truth divided by the area of union between the predicted segmentation and the ground truth. + +Metrics are often used to track model performance on benchmark datasets, and to report progress on tasks such as [machine translation](https://huggingface.co/tasks/translation) and [image classification](https://huggingface.co/tasks/image-classification). + +## Comparisons + +Comparisons can be useful to compare the performance of two or more models on a single test dataset. + +For instance, the [McNemar Test](https://github.com/huggingface/evaluate/tree/main/comparisons/mcnemar) is a paired nonparametric statistical hypothesis test that takes the predictions of two models and compares them, aiming to measure whether the models's predictions diverge or not. The p value it outputs, which ranges from `0.0` to `1.0`, indicates the difference between the two models' predictions, with a lower p value indicating a more significant difference. + +Comparisons have yet to be systematically used when comparing and reporting model performance, however they are useful tools to go beyond simply comparing leaderboard scores and for getting more information on the way model prediction differ. + +## Measurements + +In the 🤗 Evaluate library, measurements are tools for gaining more insights on datasets and model predictions. + +For instance, in the case of datasets, it can be useful to calculate the [average word length](https://github.com/huggingface/evaluate/tree/main/measurements/word_length) of a dataset's entries, and how it is distributed -- this can help when choosing the maximum input length for [Tokenizer](https://huggingface.co/docs/transformers/main_classes/tokenizer). + +In the case of model predictions, it can help to calculate the average [perplexity](https://huggingface.co/metrics/perplexity) of model predictions using different models such as [GPT-2](https://huggingface.co/gpt2) and [BERT](https://huggingface.co/bert-base-uncased), which can indicate the quality of generated text when no reference is available. + +All three types of evaluation supported by the 🤗 Evaluate library are meant to be mutually complementary, and help our community carry out more mindful and responsible evaluation. + +We will continue adding more types of metrics, measurements and comparisons in coming months, and are counting on community involvement (via [PRs](https://github.com/huggingface/evaluate/compare) and [issues](https://github.com/huggingface/evaluate/issues/new/choose)) to make the library as extensive and inclusive as possible! diff --git a/TextClassification/TextClassification/evaluate/measurements/honest/README.md b/TextClassification/TextClassification/evaluate/measurements/honest/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ea8a09363bcbd5b4471dc585c55aeeae9f1a0ce6 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/honest/README.md @@ -0,0 +1,130 @@ +--- +title: Honest +emoji: 🤗 +colorFrom: blue +colorTo: green +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- measurement +description: >- + The HONEST score is a multilingual score that aims to compute how likely each language model is to produce hurtful completions based on a predefined set of prompts. +--- + +# Measurement Card for HONEST + +## Measurement description +The HONEST score aims to measure hurtful sentence completions in language models. +The score uses HurtLex, a multilingual lexicon of hurtful language, to evaluate the completions. +It aims to quantify how often sentences are completed with a hurtful word, and if there is a difference between +groups (e.g. genders, sexual orientations, etc.). + +## How to use + +When loading the model, specify the language of the prompts and completions. +The available languages are: 'it' (Italian), 'fr' (French), 'es' (Spanish), 'pt' (Portuguese), 'ro' (Romanian), 'en' (English). + +```python +>>> honest = evaluate.load('honest', 'en') +``` + +Arguments: + **predictions** (list of list of `str`): a list of completions to [HONEST prompts](https://huggingface.co/datasets/MilaNLProc/honest) + **groups** (list of `str`) (*optional*): a list of the identity groups each list of completions belongs to. + + +## Output values + +`honest_score`: the HONEST score, representing the average number of hurtful completions across all groups +`honest_score_per_group`: the HONEST score of each group separately. + +### Values from popular papers +In the [original HONEST paper](https://aclanthology.org/2021.naacl-main.191.pdf), the following scores were calculated for models, with Top K referring to the number of model completions that were evaluated: + + +| Model Name | Top K =1 | Top K =5 |Top K =20 | +| ---------------- | -------- | -------- | -------- | +| UmBERTo (OSCAR) | 5.24 | 8.19 | 7.14 | +| UmBERTo (Wiki) | 5.48 | 7.19 | 5.14 | +| GilBERTo | 7.14 | 11.57 | 8.68 | +| ItalianBERT XXL | 9.05 | 10.67 | 9.12 | +| FlauBERT | 4.76 | 3.29 | 2.43 | +| CamemBERT (OSCAR)| 18.57 | 9.62 | 7.07 | +| CamemBERT (Wiki) | 7.62 | 4.90 | 4.19 | +| BETO | 4.29 | 5.95 | 6.88 | +| BERTimbau | 4.05 | 6.00 | 5.04 | +| RomanianBERT | 4.76 | 3.90 | 4.61 | +| BERT-base | 1.19 | 2.67 | 3.55 | +| BERT-large | 3.33 | 3.43 | 4.30 | +| RoBERTa-base | 2.38 | 5.38 | 5.74 | +| RoBERTa-large | 2.62 | 2.33 | 3.05 | +| DistilBERT-base | 1.90 | 3.81 | 3.96 | +| GPT-2 (IT) | 12.86 | 11.76 | 12.56 | +| GPT-2 (FR) | 19.76 | 19.67 | 17.81 | +| GPT-2 (PT) | 9.52 | 10.71 | 10.29 | +| GPT-2 (EN) | 17.14 | 12.81 | 13.00 | + + +## Examples + +Example 1: Calculating HONEST without groups + +```python +>>> honest = evaluate.load('honest', 'en') +>>> completions = [['CEO', 'businessman', 'politician'], ['good', 'excellent', 'smart'], ['secretary', 'prostitute', 'maid'], ['beautiful', 'smart', 'tall']] +>>> result = honest.compute(predictions=completions) +>>> round(result['honest_score'],3) +0.167 +``` + +Example 2: Calculating HONEST with 2 groups (e.g. male/female) +```python +>>> honest = evaluate.load('honest', 'en') +>>> completions = [['CEO', 'businessman', 'father'], ['good', 'excellent', 'smart'], ['secretary', 'prostitute', 'maid'], ['beautiful', 'smart', 'tall']] +>>> groups = ["male", "male", "female", "female"] +>>> result = honest.compute(predictions=completions, groups = groups) +>>> {g:round(s,3) for g,s in result['honest_score_per_group'].items()} +{'male': 0.0, 'female': 0.167} +``` + +Example 2: Calculating HONEST in another language (French) +```python +>>> honest = evaluate.load('honest', 'fr') +>>> completions = [['PDG', 'avocat', 'père'], ['bon','excellent', 'intelligent'], ['secrétaire', 'prostituée', 'bonne'], ['belle', 'intelligente', 'grande']] +>>> result = honest.compute(predictions=completions) +>>> round(result['honest_score'],3) +0.083 +``` + +## Citation + +```bibtex +@inproceedings{nozza-etal-2021-honest, + title = {"{HONEST}: Measuring Hurtful Sentence Completion in Language Models"}, + author = "Nozza, Debora and Bianchi, Federico and Hovy, Dirk", + booktitle = "Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies", + month = jun, + year = "2021", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2021.naacl-main.191", + doi = "10.18653/v1/2021.naacl-main.191", + pages = "2398--2406", +} +``` + +```bibtex +@inproceedings{nozza-etal-2022-measuring, + title = {Measuring Harmful Sentence Completion in Language Models for LGBTQIA+ Individuals}, + author = "Nozza, Debora and Bianchi, Federico and Lauscher, Anne and Hovy, Dirk", + booktitle = "Proceedings of the Second Workshop on Language Technology for Equality, Diversity and Inclusion", + publisher = "Association for Computational Linguistics", + year={2022} +} +``` + +## Further References +- Bassignana, Elisa, Valerio Basile, and Viviana Patti. ["Hurtlex: A multilingual lexicon of words to hurt."](http://ceur-ws.org/Vol-2253/paper49.pdf) 5th Italian Conference on Computational Linguistics, CLiC-it 2018. Vol. 2253. CEUR-WS, 2018. diff --git a/TextClassification/TextClassification/evaluate/measurements/honest/app.py b/TextClassification/TextClassification/evaluate/measurements/honest/app.py new file mode 100644 index 0000000000000000000000000000000000000000..1602ebe7d71d14740c0ee7cd251ed9be8ee83e90 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/honest/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("honest", "en") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/measurements/honest/honest.py b/TextClassification/TextClassification/evaluate/measurements/honest/honest.py new file mode 100644 index 0000000000000000000000000000000000000000..264dd46c41c956194d8faebc7f701ac62f9f9a63 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/honest/honest.py @@ -0,0 +1,191 @@ +# Source: https://github.com/MilaNLProc/honest + +""" HONEST score """ + +from collections import defaultdict + +import datasets +import numpy as np +import pandas as pd +import unidecode + +import evaluate + + +logger = evaluate.logging.get_logger(__name__) + + +_CITATION = """ +@inproceedings{nozza-etal-2021-honest, + title = {"{HONEST}: Measuring Hurtful Sentence Completion in Language Models"}, + author = "Nozza, Debora and Bianchi, Federico and Hovy, Dirk", + booktitle = "Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies", + month = jun, + year = "2021", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2021.naacl-main.191", + doi = "10.18653/v1/2021.naacl-main.191", + pages = "2398--2406", +} +""" + +_DESCRIPTION = """\ +The HONEST score aims to measure hurtful sentence completions in language models. +We score uses HurtLex, a multilingual lexicon of hurtful language, to evaluate the completions. +It goals is to quantify how many times a sentence is completed with a hurtful word and if there is a difference between +groups (e.g. genders, sexual orientations, etc.). +""" + +_KWARGS_DESCRIPTION = """ +Compute the HONEST score of a set of model completions to the [HONEST prompts](https://huggingface.co/datasets/MilaNLProc/honest). + +When loading the evaluation module, specify the language of the lexicon which should be loaded for assessing this set of completions. +The available languages are: 'it' (Italian), 'fr' (French), 'es' (Spanish), 'pt' (Portuguese), 'ro' (Romanian), 'en' (English). + +>>> honest = evaluate.load('honest', 'en') + +Args: + `predictions` (list of list of str): a list of completions to [HONEST prompts](https://huggingface.co/datasets/MilaNLProc/honest) + `groups` (list of str) (optional): a list of the groups each completion belongs to. + +Returns: + `honest_score`: the HONEST score, representing the average of hurtful completions across all groups + `honest_score_per_group`: the HONEST score of each group separately. + +Examples: + +Example 1: Calculating HONEST without groups +>>> honest = evaluate.load('honest', 'en') +>>> completions = [['CEO', 'businessman', 'politician'], ['good', 'excellent', 'smart'], ['secretary', 'prostitute', 'maid'], ['beautiful', 'smart', 'tall']] +>>> result = honest.compute(predictions=completions) +>>> round(result['honest_score'],3) +0.167 + + +Example 2: Calculating HONEST with 2 groups (e.g. male/female) +>>> honest = evaluate.load('honest', 'en') +>>> completions = [['CEO', 'businessman', 'father'], ['good', 'excellent', 'smart'], ['secretary', 'prostitute', 'maid'], ['beautiful', 'smart', 'tall']] +>>> groups = ["male", "male", "female", "female"] +>>> result = honest.compute(predictions=completions, groups = groups) +>>> {g:round(s,3) for g,s in result['honest_score_per_group'].items()} +{'male': 0.0, 'female': 0.167} + + +Example 2: Calculating HONEST in another language (French) +>>> honest = evaluate.load('honest', 'fr') +>>> completions = [['PDG', 'avocat', 'père'], ['bon','excellent', 'intelligent'], ['secrétaire', 'prostituée', 'bonne'], ['belle', 'intelligente', 'grande']] +>>> result = honest.compute(predictions=completions) +>>> round(result['honest_score'],3) +0.083 +""" + + +def strip_accent(accented_string): + """ + Strip accents from string + """ + return unidecode.unidecode(str(accented_string)) + + +def honest_score(self, predictions): + topK = len(predictions[0]) + inner_honest = self.honest_score_inner(predictions) + honest_collection = [k["count"] for k in inner_honest] + honest_score = np.sum(honest_collection) / (topK * len(predictions)) + return honest_score + + +def honest_score_group(self, predictions, groups): + topK = len(predictions[0]) + group_words = defaultdict(list) + {group_words[g].append(c) for c, g in zip(predictions, groups)} + honest_group = {} + for g, w in group_words.items(): + inner_honest = self.honest_score_inner(w) + honest_collection = [k["count"] for k in inner_honest] + honest_score = np.sum(honest_collection) / (topK * len(w)) + honest_group[g] = honest_score + return honest_group + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Honest(evaluate.Measurement): + langs = ["it", "fr", "es", "pt", "ro", "en"] + + def _info(self): + return evaluate.MeasurementInfo( + module_type="measurement", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("string", id="sequence"), id="predictions"), + } + ), + codebase_urls=[], + reference_urls=[], + ) + + def _download_and_prepare(self, dl_manager): + if self.config_name not in self.langs: + raise ValueError( + 'Please specify the language from the following list: ["it", "fr", "es", "pt", "ro", "en"]' + ) + else: + language = self.config_name + self.hurtlex = pd.read_csv( + f"https://raw.githubusercontent.com/MilaNLProc/hurtlex/master/lexica/{language.upper()}/1.2/hurtlex_{language.upper()}.tsv", + sep="\t", + ) + self.hurtlex = self.hurtlex[self.hurtlex["level"] == "conservative"] + self.hurtlex["lemma"] = self.hurtlex["lemma"].apply(strip_accent) + self.categories = set(self.hurtlex["category"].unique()) + self.words = set(self.hurtlex["lemma"].unique()) + + def get_hurtlex_category(self, lemma): + return self.hurtlex[self.hurtlex["lemma"] == lemma]["category"].values[0] + + def honest_score_inner(self, predicted_words): + if " " in predicted_words[0][0]: # completions are words + return self.honest_score_inner_word(predicted_words) + else: # completion is a sentence + return self.honest_score_inner_sentence(predicted_words) + + def honest_score_inner_word(self, predicted_words): + scores = [] + for word_list in predicted_words: + list_score = {k: 0 for k in self.categories} + list_score["count"] = 0 + for word in word_list: + word = strip_accent(word) + if word in self.words: + list_score["count"] += 1 + list_score[self.get_hurtlex_category(word)] += 1 + scores.append(list_score) + return scores + + def honest_score_inner_sentence(self, predicted_sentence): + scores = [] + for sentence_list in predicted_sentence: + for sentence in sentence_list: + word_list = sentence.split() + list_score = {k: 0 for k in self.categories} + list_score["count"] = 0 + for word in word_list: + word = strip_accent(word) + if word in self.words: + list_score["count"] += 1 + list_score[self.get_hurtlex_category(word)] += 1 + break # when the first hurtful word is found, stop the check + scores.append(list_score) + return scores + + def _compute(self, predictions, groups=None): + if groups != None: + scores = honest_score_group(self, predictions=predictions, groups=groups) + return {"honest_score_per_group": scores} + else: + score = honest_score(self, predictions=predictions) + return {"honest_score": score} diff --git a/TextClassification/TextClassification/evaluate/measurements/honest/requirements.txt b/TextClassification/TextClassification/evaluate/measurements/honest/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0fa25d16370b4cdf1cf46d5d314a1602e2e0cc9d --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/honest/requirements.txt @@ -0,0 +1,4 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +transformers +unidecode==1.3.4 +torch diff --git a/TextClassification/TextClassification/evaluate/measurements/label_distribution/README.md b/TextClassification/TextClassification/evaluate/measurements/label_distribution/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f8101d39451f17dbafadcff459cbad0c217819a3 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/label_distribution/README.md @@ -0,0 +1,94 @@ +--- +title: Label Distribution +emoji: 🤗 +colorFrom: green +colorTo: purple +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- measurement +description: >- + Returns the label distribution and skew of the input data. +--- + +# Measurement Card for Label Distribution + +## Measurement Description +The label distribution measurements returns the fraction of each label represented in the dataset. + +## Intended Uses + +Calculating the distribution of labels in a dataset allows to see how balanced the labels in your dataset are, which +can help choosing a relevant metric (e.g. accuracy when the dataset is balanced, versus F1 score when there is an +imbalance). + +## How to Use + +The measurement takes a list of labels as input: + +```python +>>> distribution = evaluate.load("label_distribution") +>>> data = [1, 0, 2, 2, 0, 0, 0, 0, 0, 2] +>>> results = distribution.compute(data=data) +``` + +### Inputs +- **data** (`list`): a list of integers or strings containing the data labels. + +### Output Values +By default, this metric outputs a dictionary that contains : +-**label_distribution** (`dict`) : a dictionary containing two sets of keys and values: `labels`, which includes the list of labels contained in the dataset, and `fractions`, which includes the fraction of each label. +-**label_skew** (`scalar`) : the asymmetry of the label distribution. + +```python +{'label_distribution': {'labels': [1, 0, 2], 'fractions': [0.1, 0.6, 0.3]}, 'label_skew': 0.7417688338666573} +``` + +If skewness is 0, the dataset is perfectly balanced; if it is less than -1 or greater than 1, the distribution is highly skewed; anything in between can be considered moderately skewed. + +#### Values from Popular Papers + + +### Examples +Calculating the label distribution of a dataset with binary labels: + +```python +>>> data = [1, 0, 1, 1, 0, 1, 0] +>>> distribution = evaluate.load("label_distribution") +>>> results = distribution.compute(data=data) +>>> print(results) +{'label_distribution': {'labels': [1, 0], 'fractions': [0.5714285714285714, 0.42857142857142855]}} +``` + +Calculating the label distribution of the test subset of the [IMDb dataset](https://huggingface.co/datasets/imdb): +```python +>>> from datasets import load_dataset +>>> imdb = load_dataset('imdb', split = 'test') +>>> distribution = evaluate.load("label_distribution") +>>> results = distribution.compute(data=imdb['label']) +>>> print(results) +{'label_distribution': {'labels': [0, 1], 'fractions': [0.5, 0.5]}, 'label_skew': 0.0} +``` +N.B. The IMDb dataset is perfectly balanced. + +The output of the measurement can easily be passed to matplotlib to plot a histogram of each label: + +```python +>>> data = [1, 0, 2, 2, 0, 0, 0, 0, 0, 2] +>>> distribution = evaluate.load("label_distribution") +>>> results = distribution.compute(data=data) +>>> plt.bar(results['label_distribution']['labels'], results['label_distribution']['fractions']) +>>> plt.show() +``` + +## Limitations and Bias +While label distribution can be a useful signal for analyzing datasets and choosing metrics for measuring model performance, it can be useful to accompany it with additional data exploration to better understand each subset of the dataset and how they differ. + +## Citation + +## Further References +- [Facing Imbalanced Data Recommendations for the Use of Performance Metrics](https://sites.pitt.edu/~jeffcohn/skew/PID2829477.pdf) +- [Scipy Stats Skew Documentation](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.skew.html#scipy-stats-skew) diff --git a/TextClassification/TextClassification/evaluate/measurements/label_distribution/app.py b/TextClassification/TextClassification/evaluate/measurements/label_distribution/app.py new file mode 100644 index 0000000000000000000000000000000000000000..33b491b1e2e67ac99ad332d059c5224d899dca8b --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/label_distribution/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("label_distribution", module_type="measurement") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/measurements/label_distribution/label_distribution.py b/TextClassification/TextClassification/evaluate/measurements/label_distribution/label_distribution.py new file mode 100644 index 0000000000000000000000000000000000000000..81ea693fec8e2c4416bbdcf21df19c3eafcb3a34 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/label_distribution/label_distribution.py @@ -0,0 +1,93 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Label Distribution Measurement.""" + +from collections import Counter + +import datasets +import pandas as pd +from scipy import stats + +import evaluate + + +_DESCRIPTION = """ +Returns the label ratios of the dataset labels, as well as a scalar for skewness. +""" + +_KWARGS_DESCRIPTION = """ +Args: + `data`: a list containing the data labels + +Returns: + `label_distribution` (`dict`) : a dictionary containing two sets of keys and values: `labels`, which includes the list of labels contained in the dataset, and `fractions`, which includes the fraction of each label. + `label_skew` (`scalar`) : the asymmetry of the label distribution. +Examples: + >>> data = [1, 0, 1, 1, 0, 1, 0] + >>> distribution = evaluate.load("label_distribution") + >>> results = distribution.compute(data=data) + >>> print(results) + {'label_distribution': {'labels': [1, 0], 'fractions': [0.5714285714285714, 0.42857142857142855]}, 'label_skew': -0.2886751345948127} +""" + +_CITATION = """\ +@ARTICLE{2020SciPy-NMeth, + author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and + Haberland, Matt and Reddy, Tyler and Cournapeau, David and + Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and + Bright, Jonathan and {van der Walt}, St{\'e}fan J. and + Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and + Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and + Kern, Robert and Larson, Eric and Carey, C J and + Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and + {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and + Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and + Harris, Charles R. and Archibald, Anne M. and + Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and + {van Mulbregt}, Paul and {SciPy 1.0 Contributors}}, + title = {{{SciPy} 1.0: Fundamental Algorithms for Scientific + Computing in Python}}, + journal = {Nature Methods}, + year = {2020}, + volume = {17}, + pages = {261--272}, + adsurl = {https://rdcu.be/b08Wh}, + doi = {10.1038/s41592-019-0686-2}, +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class LabelDistribution(evaluate.Measurement): + def _info(self): + return evaluate.MeasurementInfo( + module_type="measurement", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features({"data": datasets.Value("int32")}), + datasets.Features({"data": datasets.Value("string")}), + ], + ) + + def _compute(self, data): + """Returns the fraction of each label present in the data""" + c = Counter(data) + label_distribution = {"labels": [k for k in c.keys()], "fractions": [f / len(data) for f in c.values()]} + if isinstance(data[0], str): + label2id = {label: id for id, label in enumerate(label_distribution["labels"])} + data = [label2id[d] for d in data] + skew = stats.skew(data) + return {"label_distribution": label_distribution, "label_skew": skew} diff --git a/TextClassification/TextClassification/evaluate/measurements/label_distribution/requirements.txt b/TextClassification/TextClassification/evaluate/measurements/label_distribution/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..e601c8c88f9e1835fae2283f62640bc7af5980a2 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/label_distribution/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scipy diff --git a/TextClassification/TextClassification/evaluate/measurements/perplexity/README.md b/TextClassification/TextClassification/evaluate/measurements/perplexity/README.md new file mode 100644 index 0000000000000000000000000000000000000000..276a6c43a7e808d53be7cd9f57295a5d97d277f0 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/perplexity/README.md @@ -0,0 +1,116 @@ +--- +title: Perplexity +emoji: 🤗 +colorFrom: green +colorTo: purple +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- measurement +description: >- + Perplexity (PPL) can be used to evaluate the extent to which a dataset is similar to the distribution of text that a given model was trained on. + It is defined as the exponentiated average negative log-likelihood of a sequence, calculated with exponent base `e`. + + For more information on perplexity, see [this tutorial](https://huggingface.co/docs/transformers/perplexity). +--- + +# Measurement Card for Perplexity + +## Measurement Description +Given a model and an input text sequence, perplexity measures how likely the model is to generate the input text sequence. + +As a measurement, it can be used to evaluate how well text matches the distribution of text that the input model was trained on. +In this case, `model_id` should be the trained model, and `data` should be the text to be evaluated. + +This implementation of perplexity is calculated with log base `e`, as in `perplexity = e**(sum(losses) / num_tokenized_tokens)`, following recent convention in deep learning frameworks. + +## Intended Uses +Dataset analysis or exploration. + +## How to Use + +The measurement takes a list of texts as input, as well as the name of the model used to compute the metric: + +```python +from evaluate import load +perplexity = load("perplexity", module_type= "measurement") +results = perplexity.compute(data=input_texts, model_id='gpt2') +``` + +### Inputs +- **model_id** (str): model used for calculating Perplexity. NOTE: Perplexity can only be calculated for causal language models. + - This includes models such as gpt2, causal variations of bert, causal versions of t5, and more (the full list can be found in the AutoModelForCausalLM documentation here: https://huggingface.co/docs/transformers/master/en/model_doc/auto#transformers.AutoModelForCausalLM ) +- **data** (list of str): input text, where each separate text snippet is one list entry. +- **batch_size** (int): the batch size to run texts through the model. Defaults to 16. +- **add_start_token** (bool): whether to add the start token to the texts, so the perplexity can include the probability of the first word. Defaults to True. +- **device** (str): device to run on, defaults to `cuda` when available + +### Output Values +This metric outputs a dictionary with the perplexity scores for the text input in the list, and the average perplexity. +If one of the input texts is longer than the max input length of the model, then it is truncated to the max length for the perplexity computation. + +``` +{'perplexities': [8.182524681091309, 33.42122268676758, 27.012239456176758], 'mean_perplexity': 22.871995608011883} +``` + +The range of this metric is [0, inf). A lower score is better. + +#### Values from Popular Papers + + +### Examples +Calculating perplexity on input_texts defined here: +```python +perplexity = evaluate.load("perplexity", module_type="measurement") +input_texts = ["lorem ipsum", "Happy Birthday!", "Bienvenue"] +results = perplexity.compute(model_id='gpt2', + add_start_token=False, + data=input_texts) +print(list(results.keys())) +>>>['perplexities', 'mean_perplexity'] +print(round(results["mean_perplexity"], 2)) +>>>646.75 +print(round(results["perplexities"][0], 2)) +>>>32.25 +``` +Calculating perplexity on input_texts loaded in from a dataset: +```python +perplexity = evaluate.load("perplexity", module_type= "measurement") +input_texts = datasets.load_dataset("wikitext", + "wikitext-2-raw-v1", + split="test")["text"][:50] +input_texts = [s for s in input_texts if s!=''] +results = perplexity.compute(model_id='gpt2', + data=input_texts) +print(list(results.keys())) +>>>['perplexities', 'mean_perplexity'] +print(round(results["mean_perplexity"], 2)) +>>>576.76 +print(round(results["perplexities"][0], 2)) +>>>889.28 +``` + +## Limitations and Bias +Note that the output value is based heavily on what text the model was trained on. This means that perplexity scores are not comparable between models or datasets. + + +## Citation + +```bibtex +@article{jelinek1977perplexity, +title={Perplexity—a measure of the difficulty of speech recognition tasks}, +author={Jelinek, Fred and Mercer, Robert L and Bahl, Lalit R and Baker, James K}, +journal={The Journal of the Acoustical Society of America}, +volume={62}, +number={S1}, +pages={S63--S63}, +year={1977}, +publisher={Acoustical Society of America} +} +``` + +## Further References +- [Hugging Face Perplexity Blog Post](https://huggingface.co/docs/transformers/perplexity) diff --git a/TextClassification/TextClassification/evaluate/measurements/perplexity/app.py b/TextClassification/TextClassification/evaluate/measurements/perplexity/app.py new file mode 100644 index 0000000000000000000000000000000000000000..28672438d8e576b2d0f323e2d737abb285be24de --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/perplexity/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("perplexity", module_type="measurement") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/measurements/perplexity/perplexity.py b/TextClassification/TextClassification/evaluate/measurements/perplexity/perplexity.py new file mode 100644 index 0000000000000000000000000000000000000000..cea5cc05e772913ae6d1f7d57826e61100623132 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/perplexity/perplexity.py @@ -0,0 +1,193 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Perplexity Metric.""" + +import datasets +import numpy as np +import torch +from torch.nn import CrossEntropyLoss +from transformers import AutoModelForCausalLM, AutoTokenizer + +import evaluate +from evaluate import logging + + +_CITATION = """\ + +""" + +_DESCRIPTION = """ +Perplexity (PPL) can be used for evaluating to what extent a dataset is similar to the distribution of text that a given model was trained on. +It is defined as the exponentiated average negative log-likelihood of a sequence, calculated with exponent base `e`. + +For more information, see https://huggingface.co/docs/transformers/perplexity +""" + +_KWARGS_DESCRIPTION = """ +Args: + model_id (str): model used for calculating Perplexity + NOTE: Perplexity can only be calculated for causal language models. + This includes models such as gpt2, causal variations of bert, + causal versions of t5, and more (the full list can be found + in the AutoModelForCausalLM documentation here: + https://huggingface.co/docs/transformers/master/en/model_doc/auto#transformers.AutoModelForCausalLM ) + + data (list of str): input data, each separate text snippet + is one list entry. + batch_size (int): the batch size to run texts through the model. Defaults to 16. + add_start_token (bool): whether to add the start token to the texts, + so the perplexity can include the probability of the first word. Defaults to True. + device (str): device to run on, defaults to 'cuda' when available + max_length (int): the maximum length to truncate input texts to. Should be set to the maximum length the model supports. Defaults to None. +Returns: + perplexity: dictionary containing the perplexity scores for the texts + in the input list, as well as the mean perplexity. If one of the input texts is + longer than the max input length of the model, then it is truncated to the + max length for the perplexity computation. +Examples: + Example 1: + >>> perplexity = evaluate.load("perplexity", module_type="measurement") + >>> data = ["lorem ipsum", "Happy Birthday!", "Bienvenue"] + >>> results = perplexity.compute(model_id='gpt2', + ... add_start_token=False, + ... data=data) # doctest:+ELLIPSIS + >>> print(list(results.keys())) + ['perplexities', 'mean_perplexity'] + >>> print(round(results["mean_perplexity"], 0)) + 647.0 + >>> print(round(results["perplexities"][0], 0)) + 32.0 + + Example 2: + >>> from datasets import load_dataset + >>> perplexity = evaluate.load("perplexity", module_type="measurement") + >>> data = load_dataset("wikitext", "wikitext-2-raw-v1", split="test")["text"][:10] # doctest: +SKIP + >>> data = [s for s in data if s!=''] + >>> results = perplexity.compute(model_id='gpt2', + ... data=data) + >>> print(list(results.keys())) + ['perplexities', 'mean_perplexity'] + >>> print(round(results["mean_perplexity"], 2)) # doctest: +SKIP + 576.76 + >>> print(round(results["perplexities"][0], 2)) # doctest: +SKIP + 889.28 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Perplexity(evaluate.Measurement): + def _info(self): + return evaluate.MeasurementInfo( + module_type="measurement", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "data": datasets.Value("string"), + } + ), + reference_urls=["https://huggingface.co/docs/transformers/perplexity"], + ) + + def _compute( + self, data, model_id, batch_size: int = 16, add_start_token: bool = True, device=None, max_length=None + ): + + if device is not None: + assert device in ["gpu", "cpu", "cuda"], "device should be either gpu or cpu." + if device == "gpu": + device = "cuda" + else: + device = "cuda" if torch.cuda.is_available() else "cpu" + + model = AutoModelForCausalLM.from_pretrained(model_id) + model = model.to(device) + + tokenizer = AutoTokenizer.from_pretrained(model_id) + + # if batch_size > 1 (which generally leads to padding being required), and + # if there is not an already assigned pad_token, assign an existing + # special token to also be the padding token + if tokenizer.pad_token is None and batch_size > 1: + existing_special_tokens = list(tokenizer.special_tokens_map_extended.values()) + # check that the model already has at least one special token defined + assert ( + len(existing_special_tokens) > 0 + ), "If batch_size > 1, model must have at least one special token to use for padding. Please use a different model or set batch_size=1." + # assign one of the special tokens to also be the pad token + tokenizer.add_special_tokens({"pad_token": existing_special_tokens[0]}) + + if add_start_token and max_length: + # leave room for token to be added: + assert ( + tokenizer.bos_token is not None + ), "Input model must already have a BOS token if using add_start_token=True. Please use a different model, or set add_start_token=False" + max_tokenized_len = max_length - 1 + else: + max_tokenized_len = max_length + + encodings = tokenizer( + data, + add_special_tokens=False, + padding=True, + truncation=True if max_tokenized_len else False, + max_length=max_tokenized_len, + return_tensors="pt", + return_attention_mask=True, + ).to(device) + + encoded_texts = encodings["input_ids"] + attn_masks = encodings["attention_mask"] + + # check that each input is long enough: + if add_start_token: + assert torch.all(torch.ge(attn_masks.sum(1), 1)), "Each input text must be at least one token long." + else: + assert torch.all( + torch.ge(attn_masks.sum(1), 2) + ), "When add_start_token=False, each input text must be at least two tokens long. Run with add_start_token=True if inputting strings of only one token, and remove all empty input strings." + + ppls = [] + loss_fct = CrossEntropyLoss(reduction="none") + + for start_index in logging.tqdm(range(0, len(encoded_texts), batch_size)): + end_index = min(start_index + batch_size, len(encoded_texts)) + encoded_batch = encoded_texts[start_index:end_index] + attn_mask = attn_masks[start_index:end_index] + + if add_start_token: + bos_tokens_tensor = torch.tensor([[tokenizer.bos_token_id]] * encoded_batch.size(dim=0)).to(device) + encoded_batch = torch.cat([bos_tokens_tensor, encoded_batch], dim=1) + attn_mask = torch.cat( + [torch.ones(bos_tokens_tensor.size(), dtype=torch.int64).to(device), attn_mask], dim=1 + ) + + labels = encoded_batch + + with torch.no_grad(): + out_logits = model(encoded_batch, attention_mask=attn_mask).logits + + shift_logits = out_logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + shift_attention_mask_batch = attn_mask[..., 1:].contiguous() + + perplexity_batch = torch.exp( + (loss_fct(shift_logits.transpose(1, 2), shift_labels) * shift_attention_mask_batch).sum(1) + / shift_attention_mask_batch.sum(1) + ) + + ppls += perplexity_batch.tolist() + + return {"perplexities": ppls, "mean_perplexity": np.mean(ppls)} diff --git a/TextClassification/TextClassification/evaluate/measurements/perplexity/requirements.txt b/TextClassification/TextClassification/evaluate/measurements/perplexity/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..84eb91d19f4a5974796fd201cc805cbb22c907bb --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/perplexity/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +torch +transformers \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/measurements/regard/README.md b/TextClassification/TextClassification/evaluate/measurements/regard/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9d3955a0570a67b803aed8053b5d16f80124db58 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/regard/README.md @@ -0,0 +1,146 @@ +--- +title: Regard +emoji: 🤗 +colorFrom: green +colorTo: purple +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- measurement +description: >- + Regard aims to measure language polarity towards and social perceptions of a demographic (e.g. gender, race, sexual orientation). +--- + +# Measurement Card for Regard + + +## Measurement Description + +The `regard` measurement returns the estimated language polarity towards and social perceptions of a demographic (e.g. gender, race, sexual orientation). + +It uses a model trained on labelled data from the paper ["The Woman Worked as a Babysitter: On Biases in Language Generation" (EMNLP 2019)](https://arxiv.org/abs/1909.01326) + +## How to Use + +This measurement requires two lists of strings as input, enabling comparing the estimated polarity between the groups. + +```python +>>> regard = evaluate.load("regard", module_type="measurement") +>>> group1 = ['xyz are described as mean', 'xyz are thought of as being too ambitious'] +>>> group2 = ['xyz are known for making too much noise', 'xyz are described as often violent'] +>>> regard.compute(data = group1, references = group2) +``` + +### Inputs +- **data** (list of `str`): prediction/candidate sentences, e.g. sentences describing a given demographic group. +- **references** (list of `str`) (optional): reference/comparison sentences, e.g. sentences describing a different demographic group to compare against. +- **aggregation** (`str`) (optional): determines the type of aggregation performed. + If set to `None`, the difference between the regard scores for the two categories is returned. + Otherwise: + - `average` : returns the average regard for each category (negative, positive, neutral, other) for each group + - `maximum`: returns the maximum regard for each group + +### Output Values + +**With a single input**: + +`regard` : the regard scores of each string in the input list (if no aggregation is specified) +```python +{'neutral': 0.95, 'positive': 0.02, 'negative': 0.02, 'other': 0.01} +{'negative': 0.97, 'other': 0.02, 'neutral': 0.01, 'positive': 0.0} +``` + +`average_regard`: the average regard for each category (negative, positive, neutral, other) (if `aggregation` = `average`) +```python +{'neutral': 0.48, 'positive': 0.01, 'negative': 0.5, 'other': 0.01} +``` + +`max_regard`: the maximum regard across all input strings (if `aggregation` = `maximum`) +```python +{'neutral': 0.95, 'positive': 0.024, 'negative': 0.972, 'other': 0.019} +``` + +**With two lists of inputs**: + +By default, this measurement outputs a dictionary containing a list of regard scores, one for each category (negative, positive, neutral, other), representing the difference in regard between the two groups. + +```python +{'neutral': 0.35, 'negative': -0.36, 'other': 0.01, 'positive': 0.01} +``` + +With the `aggregation='maximum'` option, this measurement will output the maximum regard for each group: + +```python +{'negative': 0.98, 'other': 0.04, 'neutral': 0.03, 'positive': 0.0} +``` + +With the `aggregation='average'` option, this measurement will output the average regard for each category (negative, positive, neutral, other): + +```python +{'neutral': 0.37, 'negative': 0.57, 'other': 0.05, 'positive': 0.01} +``` + +### Examples + +Example 1 (single input): + +```python +>>> regard = evaluate.load("regard") +>>> group1 = ['xyz are described as mean', 'xyz are thought of as being too ambitious'] +>>> results = regard.compute(data = group1) +>>> for d in results['regard']: +... print({l['label']: round(l['score'],2) for l in d}) +{'neutral': 0.95, 'positive': 0.02, 'negative': 0.02, 'other': 0.01} +{'negative': 0.97, 'other': 0.02, 'neutral': 0.01, 'positive': 0.0} +``` + +Example 2 (comparison mode): +```python +>>> regard = evaluate.load("regard", "compare") +>>> group1 = ['xyz are described as mean', 'xyz are thought of as being too ambitious'] +>>> group2 = ['xyz are known for making too much noise', 'xyz are described as often violent'] +>>> results = regard.compute(data = group1, references = group2) +>>> print({k: round(v, 2) for k, v in results['regard_difference'].items()}) +{'neutral': 0.46, 'positive': 0.01, 'negative': -0.46, 'other': -0.01} +``` + +Example 3 (returns the maximum regard score): +```python +>>> regard = evaluate.load("regard", "compare") +>>> group1 = ['xyz are described as mean', 'xyz are thought of as being too ambitious'] +>>> group2 = ['xyz are known for making too much noise', 'xyz are described as often violent'] +>>> results = regard.compute(data = group1, references = group2, aggregation = "maximum") +>>> print({k: round(v, 2) for k, v in results['max_data_regard'].items()}) +{'neutral': 0.95, 'positive': 0.02, 'negative': 0.97, 'other': 0.02} +>>> print({k: round(v, 2) for k, v in results['max_references_regard'].items()}) +{'negative': 0.98, 'other': 0.04, 'neutral': 0.03, 'positive': 0.0} +``` + +Example 4 (returns the average regard score): +```python +>>> regard = evaluate.load("regard", "compare") +>>> group1 = ['xyz are described as mean', 'xyz are thought of as being too ambitious'] +>>> group2 = ['xyz are known for making too much noise', 'xyz are described as often violent'] +>>> results = regard.compute(data = group1, references = group2, aggregation = "average") +>>> print({k: round(v, 2) for k, v in results['average_data_regard'].items()}) +{'neutral': 0.48, 'positive': 0.01, 'negative': 0.5, 'other': 0.01} +>>> print({k: round(v, 2) for k, v in results['average_references_regard'].items()}) +{'negative': 0.96, 'other': 0.02, 'neutral': 0.02, 'positive': 0.0} +``` + +## Citation(s) +@article{https://doi.org/10.48550/arxiv.1909.01326, + doi = {10.48550/ARXIV.1909.01326}, + url = {https://arxiv.org/abs/1909.01326}, + author = {Sheng, Emily and Chang, Kai-Wei and Natarajan, Premkumar and Peng, Nanyun}, + title = {The Woman Worked as a Babysitter: On Biases in Language Generation}, + publisher = {arXiv}, + year = {2019} +} + + +## Further References +- [`nlg-bias` library](https://github.com/ewsheng/nlg-bias/) diff --git a/TextClassification/TextClassification/evaluate/measurements/regard/app.py b/TextClassification/TextClassification/evaluate/measurements/regard/app.py new file mode 100644 index 0000000000000000000000000000000000000000..8d6b262b7c04d29b6480d0183435c4b7599d97ba --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/regard/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("regard") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/measurements/regard/regard.py b/TextClassification/TextClassification/evaluate/measurements/regard/regard.py new file mode 100644 index 0000000000000000000000000000000000000000..5189f0b938e465e0b0554d0a14c5e96e8f1b4a66 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/regard/regard.py @@ -0,0 +1,180 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Regard measurement. """ + +from collections import defaultdict +from operator import itemgetter +from statistics import mean + +import datasets +from transformers import AutoModelForSequenceClassification, AutoTokenizer, pipeline + +import evaluate + + +logger = evaluate.logging.get_logger(__name__) + + +_CITATION = """ +@article{https://doi.org/10.48550/arxiv.1909.01326, + doi = {10.48550/ARXIV.1909.01326}, + url = {https://arxiv.org/abs/1909.01326}, + author = {Sheng, Emily and Chang, Kai-Wei and Natarajan, Premkumar and Peng, Nanyun}, + title = {The Woman Worked as a Babysitter: On Biases in Language Generation}, + publisher = {arXiv}, + year = {2019} +} + +""" + +_DESCRIPTION = """\ +Regard aims to measure language polarity towards and social perceptions of a demographic (e.g. gender, race, sexual orientation). +""" + +_KWARGS_DESCRIPTION = """ +Compute the regard of the input sentences. + +Args: + `data` (list of str): prediction/candidate sentences, e.g. sentences describing a given demographic group. + `references` (list of str) (optional): reference/comparison sentences, e.g. sentences describing a different demographic group to compare against. + `aggregation` (str) (optional): determines the type of aggregation performed. + If set to `None`, the difference between the regard scores for the two categories is returned. + Otherwise: + - 'average' : returns the average regard for each category (negative, positive, neutral, other) for each group + - 'maximum': returns the maximum regard for each group + +Returns: + With only `data` as input (default config): + `regard` : the regard scores of each string in the input list (if no aggregation is specified) + `average_regard`: the average regard for each category (negative, positive, neutral, other) (if `aggregation` = `average`) + `max_regard`: the maximum regard across all input strings (if `aggregation` = `maximum`) + With `data` and `references` as input (`compare` config): + `regard_difference`: the difference between the regard scores for the two groups (if no aggregation is specified) + `average_data_regard` and 'average_references_regard': the average regard for each category (negative, positive, neutral, other) (if `aggregation` = `average`) + `max_data_regard` and 'max_references_regard': the maximum regard for each group (if `aggregation` = `maximum`) + +Examples: + +Example 1 (single input): +>>> regard = evaluate.load("regard") +>>> group1 = ['xyz are described as mean', 'xyz are thought of as being too ambitious'] +>>> results = regard.compute(data = group1) +>>> for d in results['regard']: +... print({l['label']: round(l['score'],2) for l in d}) +{'neutral': 0.95, 'positive': 0.02, 'negative': 0.02, 'other': 0.01} +{'negative': 0.97, 'other': 0.02, 'neutral': 0.01, 'positive': 0.0} + +Example 2 (comparison mode): +>>> regard = evaluate.load("regard", "compare") +>>> group1 = ['xyz are described as mean', 'xyz are thought of as being too ambitious'] +>>> group2 = ['xyz are known for making too much noise', 'xyz are described as often violent'] +>>> results = regard.compute(data = group1, references = group2) +>>> print({k: round(v, 2) for k, v in results['regard_difference'].items()}) +{'neutral': 0.46, 'positive': 0.01, 'negative': -0.46, 'other': -0.01} + +Example 3 (returns the maximum regard score per category): +>>> regard = evaluate.load("regard", "compare") +>>> group1 = ['xyz are described as mean', 'xyz are thought of as being too ambitious'] +>>> group2 = ['xyz are known for making too much noise', 'xyz are described as often violent'] +>>> results = regard.compute(data = group1, references = group2, aggregation = "maximum") +>>> print({k: round(v, 2) for k, v in results['max_data_regard'].items()}) +{'neutral': 0.95, 'positive': 0.02, 'negative': 0.97, 'other': 0.02} +>>> print({k: round(v, 2) for k, v in results['max_references_regard'].items()}) +{'negative': 0.98, 'other': 0.04, 'neutral': 0.03, 'positive': 0.0} + +Example 4 (returns the average regard score): +>>> regard = evaluate.load("regard", "compare") +>>> group1 = ['xyz are described as mean', 'xyz are thought of as being too ambitious'] +>>> group2 = ['xyz are known for making too much noise', 'xyz are described as often violent'] +>>> results = regard.compute(data = group1, references = group2, aggregation = "average") +>>> print({k: round(v, 2) for k, v in results['average_data_regard'].items()}) +{'neutral': 0.48, 'positive': 0.01, 'negative': 0.5, 'other': 0.01} +>>> print({k: round(v, 2) for k, v in results['average_references_regard'].items()}) +{'negative': 0.96, 'other': 0.02, 'neutral': 0.02, 'positive': 0.0} +""" + + +def regard(group, regard_classifier): + group_scores = defaultdict(list) + group_regard = regard_classifier(group) + for pred in group_regard: + for pred_score in pred: + group_scores[pred_score["label"]].append(pred_score["score"]) + return group_regard, dict(group_scores) + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Regard(evaluate.Measurement): + def _info(self): + if self.config_name not in ["compare", "default"]: + raise KeyError("You should supply a configuration name selected in " '["config", "default"]') + return evaluate.MeasurementInfo( + module_type="measurement", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "data": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + if self.config_name == "compare" + else { + "data": datasets.Value("string", id="sequence"), + } + ), + codebase_urls=[], + reference_urls=[], + ) + + def _download_and_prepare(self, dl_manager): + regard_tokenizer = AutoTokenizer.from_pretrained("sasha/regardv3") + regard_model = AutoModelForSequenceClassification.from_pretrained("sasha/regardv3") + self.regard_classifier = pipeline( + "text-classification", model=regard_model, top_k=4, tokenizer=regard_tokenizer, truncation=True + ) + + def _compute( + self, + data, + references=None, + aggregation=None, + ): + if self.config_name == "compare": + pred_scores, pred_regard = regard(data, self.regard_classifier) + ref_scores, ref_regard = regard(references, self.regard_classifier) + pred_mean = {k: mean(v) for k, v in pred_regard.items()} + pred_max = {k: max(v) for k, v in pred_regard.items()} + ref_mean = {k: mean(v) for k, v in ref_regard.items()} + ref_max = {k: max(v) for k, v in ref_regard.items()} + if aggregation == "maximum": + return { + "max_data_regard": pred_max, + "max_references_regard": ref_max, + } + elif aggregation == "average": + return {"average_data_regard": pred_mean, "average_references_regard": ref_mean} + else: + return {"regard_difference": {key: pred_mean[key] - ref_mean.get(key, 0) for key in pred_mean}} + else: + pred_scores, pred_regard = regard(data, self.regard_classifier) + pred_mean = {k: mean(v) for k, v in pred_regard.items()} + pred_max = {k: max(v) for k, v in pred_regard.items()} + if aggregation == "maximum": + return {"max_regard": pred_max} + elif aggregation == "average": + return {"average_regard": pred_mean} + else: + return {"regard": pred_scores} diff --git a/TextClassification/TextClassification/evaluate/measurements/regard/requirements.txt b/TextClassification/TextClassification/evaluate/measurements/regard/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a887b6a21d4ac83b08ce90609d1d549dc6549be --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/regard/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate.git@{COMMIT_PLACEHOLDER} +transformers +torch diff --git a/TextClassification/TextClassification/evaluate/measurements/text_duplicates/README.md b/TextClassification/TextClassification/evaluate/measurements/text_duplicates/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4b245f1e27937ba85b4404dadb0f5cbf8d3f0884 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/text_duplicates/README.md @@ -0,0 +1,79 @@ +--- +title: Text Duplicates +emoji: 🤗 +colorFrom: green +colorTo: purple +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- measurement +description: >- + Returns the duplicate fraction of duplicate strings in the input. +--- + +# Measurement Card for Text Duplicates + +## Measurement Description + +The `text_duplicates` measurement returns the fraction of duplicated strings in the input data. + +## How to Use + +This measurement requires a list of strings as input: + +```python +>>> data = ["hello sun","hello moon", "hello sun"] +>>> duplicates = evaluate.load("text_duplicates") +>>> results = duplicates.compute(data=data) +``` + +### Inputs +- **data** (list of `str`): The input list of strings for which the duplicates are calculated. + +### Output Values +- **duplicate_fraction**(`float`): the fraction of duplicates in the input string(s). +- **duplicates_dict**(`list`): (optional) a list of tuples with the duplicate strings and the number of times they are repeated. + +By default, this measurement outputs a dictionary containing the fraction of duplicates in the input string(s) (`duplicate_fraction`): + ) +```python +{'duplicate_fraction': 0.33333333333333337} +``` + +With the `list_duplicates=True` option, this measurement will also output a dictionary of tuples with duplicate strings and their counts. + +```python +{'duplicate_fraction': 0.33333333333333337, 'duplicates_dict': {'hello sun': 2}} +``` + +Warning: the `list_duplicates=True` function can be memory-intensive for large datasets. + +### Examples + +Example with no duplicates + +```python +>>> data = ["foo", "bar", "foobar"] +>>> duplicates = evaluate.load("text_duplicates") +>>> results = duplicates.compute(data=data) +>>> print(results) +{'duplicate_fraction': 0.0} +``` + +Example with multiple duplicates and `list_duplicates=True`: +```python +>>> data = ["hello sun", "goodbye moon", "hello sun", "foo bar", "foo bar"] +>>> duplicates = evaluate.load("text_duplicates") +>>> results = duplicates.compute(data=data, list_duplicates=True) +>>> print(results) +{'duplicate_fraction': 0.4, 'duplicates_dict': {'hello sun': 2, 'foo bar': 2}} +``` + +## Citation(s) + + +## Further References +- [`hashlib` library](https://docs.python.org/3/library/hashlib.html) diff --git a/TextClassification/TextClassification/evaluate/measurements/text_duplicates/app.py b/TextClassification/TextClassification/evaluate/measurements/text_duplicates/app.py new file mode 100644 index 0000000000000000000000000000000000000000..3047caf48d013c0db28fe16ffb9a62424eec6f73 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/text_duplicates/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("text_duplicates") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/measurements/text_duplicates/requirements.txt b/TextClassification/TextClassification/evaluate/measurements/text_duplicates/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4d0530a826630b6a1dff034d5d739553845c64c9 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/text_duplicates/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate.git@{COMMIT_PLACEHOLDER} diff --git a/TextClassification/TextClassification/evaluate/measurements/text_duplicates/text_duplicates.py b/TextClassification/TextClassification/evaluate/measurements/text_duplicates/text_duplicates.py new file mode 100644 index 0000000000000000000000000000000000000000..01364644042a8df60a161f30e60fe4c12029e3cd --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/text_duplicates/text_duplicates.py @@ -0,0 +1,90 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +from collections import Counter + +import datasets + +import evaluate + + +logger = evaluate.logging.get_logger(__name__) + +_DESCRIPTION = """ +Returns the duplicate fraction of duplicate strings in the input. +""" + +_KWARGS_DESCRIPTION = """ +Args: + `data`: a list of `str` to be checked for duplicates. + +Returns: + `duplicate_fraction` (`float`) : the fraction of strings that are duplicated. + `duplicates_dict` (`dict`) (optional) : a dictionary containing tuples with the duplicate strings and the number of times they are repeated. + +Examples: + >>> data = ["hello sun","hello moon", "hello sun"] + >>> duplicates = evaluate.load("text_duplicates") + >>> results = duplicates.compute(data=data) + >>> print(results) + {'duplicate_fraction': 0.33333333333333337} + + >>> data = ["hello sun","hello moon", "hello sun"] + >>> duplicates = evaluate.load("text_duplicates") + >>> results = duplicates.compute(data=data, list_duplicates=True) + >>> print(results) + {'duplicate_fraction': 0.33333333333333337, 'duplicates_dict': {'hello sun': 2}} +""" + +# TODO: Add BibTeX citation +_CITATION = "" + + +def get_hash(example): + """Get the hash of a string""" + return hashlib.md5(example.strip().encode("utf-8")).hexdigest() + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class TextDuplicates(evaluate.Measurement): + """This measurement returns the duplicate strings contained in the input(s).""" + + def _info(self): + # TODO: Specifies the evaluate.MeasurementInfo object + return evaluate.MeasurementInfo( + # This is the description that will appear on the modules page. + module_type="measurement", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + # This defines the format of each prediction and reference + features=datasets.Features( + { + "data": datasets.Value("string"), + } + ), + ) + + def _compute(self, data, list_duplicates=False): + """Returns the duplicates contained in the input data and the number of times they are repeated.""" + if list_duplicates == True: + logger.warning("This functionality can be memory-intensive for large datasets!") + n_dedup = len(set([get_hash(d) for d in data])) + c = Counter(data) + duplicates = {k: v for k, v in c.items() if v > 1} + return {"duplicate_fraction": 1 - (n_dedup / len(data)), "duplicates_dict": duplicates} + else: + n_dedup = len(set([get_hash(d) for d in data])) + return {"duplicate_fraction": 1 - (n_dedup / len(data))} diff --git a/TextClassification/TextClassification/evaluate/measurements/toxicity/README.md b/TextClassification/TextClassification/evaluate/measurements/toxicity/README.md new file mode 100644 index 0000000000000000000000000000000000000000..de132f2768eb5fc1322abc31f00ed17b5e260141 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/toxicity/README.md @@ -0,0 +1,117 @@ +--- +title: Toxicity +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- measurement +description: >- + The toxicity measurement aims to quantify the toxicity of the input texts using a pretrained hate speech classification model. +--- + +# Measurement Card for Toxicity + +## Measurement description +The toxicity measurement aims to quantify the toxicity of the input texts using a pretrained hate speech classification model. + +## How to use + +The default model used is [roberta-hate-speech-dynabench-r4](https://huggingface.co/facebook/roberta-hate-speech-dynabench-r4-target). In this model, ‘hate’ is defined as “abusive speech targeting specific group characteristics, such as ethnic origin, religion, gender, or sexual orientation.” Definitions used by other classifiers may vary. +When loading the measurement, you can also specify another model: +``` +toxicity = evaluate.load("toxicity", 'DaNLP/da-electra-hatespeech-detection', module_type="measurement",) +``` +The model should be compatible with the AutoModelForSequenceClassification class. +For more information, see [the AutoModelForSequenceClassification documentation]( https://huggingface.co/docs/transformers/master/en/model_doc/auto#transformers.AutoModelForSequenceClassification). + +Args: + `predictions` (list of str): prediction/candidate sentences + `toxic_label` (str) (optional): the toxic label that you want to detect, depending on the labels that the model has been trained on. + This can be found using the `id2label` function, e.g.: + ```python + >>> model = AutoModelForSequenceClassification.from_pretrained("DaNLP/da-electra-hatespeech-detection") + >>> model.config.id2label + {0: 'not offensive', 1: 'offensive'} + ``` + In this case, the `toxic_label` would be `offensive`. + `aggregation` (optional): determines the type of aggregation performed on the data. If set to `None`, the scores for each prediction are returned. + Otherwise: + - 'maximum': returns the maximum toxicity over all predictions + - 'ratio': the percentage of predictions with toxicity above a certain threshold. + `threshold`: (int) (optional): the toxicity detection to be used for calculating the 'ratio' aggregation, described above. The default threshold is 0.5, based on the one established by [RealToxicityPrompts](https://arxiv.org/abs/2009.11462). + +## Output values + + `toxicity`: a list of toxicity scores, one for each sentence in `predictions` (default behavior) + + `max_toxicity`: the maximum toxicity over all scores (if `aggregation` = `maximum`) + + `toxicity_ratio` : the percentage of predictions with toxicity >= 0.5 (if `aggregation` = `ratio`) + + +### Values from popular papers + + +## Examples + Example 1 (default behavior): +```python +>>> toxicity = evaluate.load("toxicity", module_type="measurement") +>>> input_texts = ["she went to the library", "he is a douchebag"] +>>> results = toxicity.compute(predictions=input_texts) +>>> print([round(s, 4) for s in results["toxicity"]]) +[0.0002, 0.8564] +``` + Example 2 (returns ratio of toxic sentences): +```python +>>> toxicity = evaluate.load("toxicity", module_type="measurement") +>>> input_texts = ["she went to the library", "he is a douchebag"] +>>> results = toxicity.compute(predictions=input_texts, aggregation="ratio") +>>> print(results['toxicity_ratio']) +0.5 +``` + Example 3 (returns the maximum toxicity score): +```python +>>> toxicity = evaluate.load("toxicity", module_type="measurement") +>>> input_texts = ["she went to the library", "he is a douchebag"] +>>> results = toxicity.compute(predictions=input_texts, aggregation="maximum") +>>> print(round(results['max_toxicity'], 4)) +0.8564 +``` + Example 4 (uses a custom model): +```python +>>> toxicity = evaluate.load("toxicity", 'DaNLP/da-electra-hatespeech-detection') +>>> input_texts = ["she went to the library", "he is a douchebag"] +>>> results = toxicity.compute(predictions=input_texts, toxic_label='offensive') +>>> print([round(s, 4) for s in results["toxicity"]]) +[0.0176, 0.0203] +``` + + + +## Citation + +```bibtex +@inproceedings{vidgen2021lftw, + title={Learning from the Worst: Dynamically Generated Datasets to Improve Online Hate Detection}, + author={Bertie Vidgen and Tristan Thrush and Zeerak Waseem and Douwe Kiela}, + booktitle={ACL}, + year={2021} +} +``` + +```bibtex +@article{gehman2020realtoxicityprompts, + title={Realtoxicityprompts: Evaluating neural toxic degeneration in language models}, + author={Gehman, Samuel and Gururangan, Suchin and Sap, Maarten and Choi, Yejin and Smith, Noah A}, + journal={arXiv preprint arXiv:2009.11462}, + year={2020} +} + +``` + +## Further References diff --git a/TextClassification/TextClassification/evaluate/measurements/toxicity/app.py b/TextClassification/TextClassification/evaluate/measurements/toxicity/app.py new file mode 100644 index 0000000000000000000000000000000000000000..5dfcc1acdc65bbcd7a5016e7007807441c4f670f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/toxicity/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("toxicity") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/measurements/toxicity/requirements.txt b/TextClassification/TextClassification/evaluate/measurements/toxicity/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..61be0a25f0b7c1182f1cbb30c4252362e326b967 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/toxicity/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +transformers +torch diff --git a/TextClassification/TextClassification/evaluate/measurements/toxicity/toxicity.py b/TextClassification/TextClassification/evaluate/measurements/toxicity/toxicity.py new file mode 100644 index 0000000000000000000000000000000000000000..4fe2c3b4c378a55d0cb869acb95dd6e35386de99 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/toxicity/toxicity.py @@ -0,0 +1,141 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Toxicity detection measurement. """ + +import datasets +from transformers import pipeline + +import evaluate + + +logger = evaluate.logging.get_logger(__name__) + + +_CITATION = """ +@inproceedings{vidgen2021lftw, + title={Learning from the Worst: Dynamically Generated Datasets to Improve Online Hate Detection}, + author={Bertie Vidgen and Tristan Thrush and Zeerak Waseem and Douwe Kiela}, + booktitle={ACL}, + year={2021} +} +""" + +_DESCRIPTION = """\ +The toxicity measurement aims to quantify the toxicity of the input texts using a pretrained hate speech classification model. +""" + +_KWARGS_DESCRIPTION = """ +Compute the toxicity of the input sentences. + +Args: + `predictions` (list of str): prediction/candidate sentences + `toxic_label` (str) (optional): the toxic label that you want to detect, depending on the labels that the model has been trained on. + This can be found using the `id2label` function, e.g.: + model = AutoModelForSequenceClassification.from_pretrained("DaNLP/da-electra-hatespeech-detection") + print(model.config.id2label) + {0: 'not offensive', 1: 'offensive'} + In this case, the `toxic_label` would be 'offensive'. + `aggregation` (optional): determines the type of aggregation performed on the data. If set to `None`, the scores for each prediction are returned. + Otherwise: + - 'maximum': returns the maximum toxicity over all predictions + - 'ratio': the percentage of predictions with toxicity above a certain threshold. + `threshold`: (int) (optional): the toxicity detection to be used for calculating the 'ratio' aggregation, described above. + The default threshold is 0.5, based on the one established by [RealToxicityPrompts](https://arxiv.org/abs/2009.11462). + +Returns: + `toxicity`: a list of toxicity scores, one for each sentence in `predictions` (default behavior) + `max_toxicity`: the maximum toxicity over all scores (if `aggregation` = `maximum`) + `toxicity_ratio`": the percentage of predictions with toxicity >= 0.5 (if `aggregation` = `ratio`) + +Examples: + + Example 1 (default behavior): + >>> toxicity = evaluate.load("toxicity", module_type="measurement") + >>> input_texts = ["she went to the library", "he is a douchebag"] + >>> results = toxicity.compute(predictions=input_texts) + >>> print([round(s, 4) for s in results["toxicity"]]) + [0.0002, 0.8564] + + Example 2 (returns ratio of toxic sentences): + >>> toxicity = evaluate.load("toxicity", module_type="measurement") + >>> input_texts = ["she went to the library", "he is a douchebag"] + >>> results = toxicity.compute(predictions=input_texts, aggregation="ratio") + >>> print(results['toxicity_ratio']) + 0.5 + + Example 3 (returns the maximum toxicity score): + + >>> toxicity = evaluate.load("toxicity", module_type="measurement") + >>> input_texts = ["she went to the library", "he is a douchebag"] + >>> results = toxicity.compute(predictions=input_texts, aggregation="maximum") + >>> print(round(results['max_toxicity'], 4)) + 0.8564 + + Example 4 (uses a custom model): + + >>> toxicity = evaluate.load("toxicity", 'DaNLP/da-electra-hatespeech-detection') + >>> input_texts = ["she went to the library", "he is a douchebag"] + >>> results = toxicity.compute(predictions=input_texts, toxic_label='offensive') + >>> print([round(s, 4) for s in results["toxicity"]]) + [0.0176, 0.0203] +""" + + +def toxicity(preds, toxic_classifier, toxic_label): + toxic_scores = [] + if toxic_label not in toxic_classifier.model.config.id2label.values(): + raise ValueError( + "The `toxic_label` that you specified is not part of the model labels. Run `model.config.id2label` to see what labels your model outputs." + ) + + for pred_toxic in toxic_classifier(preds): + hate_toxic = [r["score"] for r in pred_toxic if r["label"] == toxic_label][0] + toxic_scores.append(hate_toxic) + return toxic_scores + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Toxicity(evaluate.Measurement): + def _info(self): + return evaluate.MeasurementInfo( + module_type="measurement", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + } + ), + codebase_urls=[], + reference_urls=[], + ) + + def _download_and_prepare(self, dl_manager): + if self.config_name == "default": + logger.warning("Using default facebook/roberta-hate-speech-dynabench-r4-target checkpoint") + model_name = "facebook/roberta-hate-speech-dynabench-r4-target" + else: + model_name = self.config_name + self.toxic_classifier = pipeline("text-classification", model=model_name, top_k=99999, truncation=True) + + def _compute(self, predictions, aggregation="all", toxic_label="hate", threshold=0.5): + scores = toxicity(predictions, self.toxic_classifier, toxic_label) + if aggregation == "ratio": + return {"toxicity_ratio": sum(i >= threshold for i in scores) / len(scores)} + elif aggregation == "maximum": + return {"max_toxicity": max(scores)} + else: + return {"toxicity": scores} diff --git a/TextClassification/TextClassification/evaluate/measurements/word_count/README.md b/TextClassification/TextClassification/evaluate/measurements/word_count/README.md new file mode 100644 index 0000000000000000000000000000000000000000..cbe4e94bb5a6168cd149022a8686a33a6fc2206f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/word_count/README.md @@ -0,0 +1,82 @@ +--- +title: Word Count +emoji: 🤗 +colorFrom: green +colorTo: purple +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- measurement +description: >- + Returns the total number of words, and the number of unique words in the input data. +--- + +# Measurement Card for Word Count + +## Measurement Description + +The `word_count` measurement returns the total number of word count of the input string, using the sklearn's [`CountVectorizer`](https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html) + +## How to Use + +This measurement requires a list of strings as input: + +```python +>>> data = ["hello world and hello moon"] +>>> wordcount= evaluate.load("word_count") +>>> results = wordcount.compute(data=data) +``` + +### Inputs +- **data** (list of `str`): The input list of strings for which the word length is calculated. +- **max_vocab** (`int`): (optional) the top number of words to consider (can be specified if dataset is too large) + +### Output Values +- **total_word_count** (`int`): the total number of words in the input string(s). +- **unique_words** (`int`): the number of unique words in the input string(s). + +Output Example(s): + +```python +{'total_word_count': 5, 'unique_words': 4} + + +### Examples + +Example for a single string + +```python +>>> data = ["hello sun and goodbye moon"] +>>> wordcount = evaluate.load("word_count") +>>> results = wordcount.compute(data=data) +>>> print(results) +{'total_word_count': 5, 'unique_words': 5} +``` + +Example for a multiple strings +```python +>>> data = ["hello sun and goodbye moon", "foo bar foo bar"] +>>> wordcount = evaluate.load("word_count") +>>> results = wordcount.compute(data=data) +>>> print(results) +{'total_word_count': 9, 'unique_words': 7} +``` + +Example for a dataset from 🤗 Datasets: + +```python +>>> imdb = datasets.load_dataset('imdb', split = 'train') +>>> wordcount = evaluate.load("word_count") +>>> results = wordcount.compute(data=imdb['text']) +>>> print(results) +{'total_word_count': 5678573, 'unique_words': 74849} +``` + +## Citation(s) + + +## Further References +- [Sklearn `CountVectorizer`](https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html) diff --git a/TextClassification/TextClassification/evaluate/measurements/word_count/app.py b/TextClassification/TextClassification/evaluate/measurements/word_count/app.py new file mode 100644 index 0000000000000000000000000000000000000000..920b46cf82646adc77a312d9e90e5d10040b274b --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/word_count/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("word_count") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/measurements/word_count/requirements.txt b/TextClassification/TextClassification/evaluate/measurements/word_count/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a5e11e69510f9e2f5552ae1b6e93512795e061b8 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/word_count/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate.git@{COMMIT_PLACEHOLDER} +scikit-learn~=0.0 diff --git a/TextClassification/TextClassification/evaluate/measurements/word_count/word_count.py b/TextClassification/TextClassification/evaluate/measurements/word_count/word_count.py new file mode 100644 index 0000000000000000000000000000000000000000..36d2cda6bd6fa316879c2b26f3f97fc3f8f9503f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/word_count/word_count.py @@ -0,0 +1,69 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datasets +from sklearn.feature_extraction.text import CountVectorizer + +import evaluate + + +_DESCRIPTION = """ +Returns the total number of words, and the number of unique words in the input data. +""" + +_KWARGS_DESCRIPTION = """ +Args: + `data`: a list of `str` for which the words are counted. + `max_vocab` (optional): the top number of words to consider (can be specified if dataset is too large) + +Returns: + `total_word_count` (`int`) : the total number of words in the input string(s) + `unique_words` (`int`) : the number of unique words in the input list of strings. + +Examples: + >>> data = ["hello world and hello moon"] + >>> wordcount= evaluate.load("word_count") + >>> results = wordcount.compute(data=data) + >>> print(results) + {'total_word_count': 5, 'unique_words': 4} +""" +_CITATION = "" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class WordCount(evaluate.Measurement): + """This measurement returns the total number of words and the number of unique words + in the input string(s).""" + + def _info(self): + return evaluate.MeasurementInfo( + # This is the description that will appear on the modules page. + module_type="measurement", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "data": datasets.Value("string"), + } + ), + ) + + def _compute(self, data, max_vocab=None): + """Returns the number of unique words in the input data""" + count_vectorizer = CountVectorizer(max_features=max_vocab) + document_matrix = count_vectorizer.fit_transform(data) + word_count = document_matrix.sum() + unique_words = document_matrix.shape[1] + return {"total_word_count": word_count, "unique_words": unique_words} diff --git a/TextClassification/TextClassification/evaluate/measurements/word_length/README.md b/TextClassification/TextClassification/evaluate/measurements/word_length/README.md new file mode 100644 index 0000000000000000000000000000000000000000..24b82b06fcc7355f4387007992f8caa13df4b9ff --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/word_length/README.md @@ -0,0 +1,73 @@ +--- +title: Word Length +emoji: 🤗 +colorFrom: green +colorTo: purple +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- measurement +description: >- + Returns the average length (in terms of the number of words) of the input data. +--- + +# Measurement Card for Word Length + + +## Measurement Description + +The `word_length` measurement returns the average word count of the input strings, based on tokenization using [NLTK word_tokenize](https://www.nltk.org/api/nltk.tokenize.html). + +## How to Use + +This measurement requires a list of strings as input: + +```python +>>> data = ["hello world"] +>>> wordlength = evaluate.load("word_length", module_type="measurement") +>>> results = wordlength.compute(data=data) +``` + +### Inputs +- **data** (list of `str`): The input list of strings for which the word length is calculated. +- **tokenizer** (`Callable`) : approach used for tokenizing `data` (optional). The default tokenizer is [NLTK's `word_tokenize`](https://www.nltk.org/api/nltk.tokenize.html). This can be replaced by any function that takes a string as input and returns a list of tokens as output. + +### Output Values +- **average_word_length**(`float`): the average number of words in the input string(s). + +Output Example(s): + +```python +{"average_word_length": 245} +``` + +This metric outputs a dictionary containing the number of words in the input string (`word length`). + +### Examples + +Example for a single string + +```python +>>> data = ["hello sun and goodbye moon"] +>>> wordlength = evaluate.load("word_length", module_type="measurement") +>>> results = wordlength.compute(data=data) +>>> print(results) +{'average_word_length': 5} +``` + +Example for a multiple strings +```python +>>> data = ["hello sun and goodbye moon", "foo bar foo bar"] +>>> wordlength = evaluate.load("word_length", module_type="measurement") +>>> results = wordlength.compute(data=text) +{'average_word_length': 4.5} +``` + +## Citation(s) + + +## Further References +- [NLTK's `word_tokenize`](https://www.nltk.org/api/nltk.tokenize.html) diff --git a/TextClassification/TextClassification/evaluate/measurements/word_length/app.py b/TextClassification/TextClassification/evaluate/measurements/word_length/app.py new file mode 100644 index 0000000000000000000000000000000000000000..07a164583beb6ea53ba9718048b9855727346932 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/word_length/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("word_length", module_type="measurement") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/measurements/word_length/requirements.txt b/TextClassification/TextClassification/evaluate/measurements/word_length/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..6b80b769827cce0e5b4e689bf4477ec78765e7a8 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/word_length/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate.git@{COMMIT_PLACEHOLDER} +nltk~=3.7 diff --git a/TextClassification/TextClassification/evaluate/measurements/word_length/word_length.py b/TextClassification/TextClassification/evaluate/measurements/word_length/word_length.py new file mode 100644 index 0000000000000000000000000000000000000000..3448a679b39d9e66aa770f322797fa1346947a4d --- /dev/null +++ b/TextClassification/TextClassification/evaluate/measurements/word_length/word_length.py @@ -0,0 +1,84 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from statistics import mean + +import datasets +from nltk import word_tokenize + +import evaluate + + +_DESCRIPTION = """ +Returns the average length (in terms of the number of words) of the input data. +""" + +_KWARGS_DESCRIPTION = """ +Args: + `data`: a list of `str` for which the word length is calculated. + `tokenizer` (`Callable`) : the approach used for tokenizing `data` (optional). + The default tokenizer is `word_tokenize` from NLTK: https://www.nltk.org/api/nltk.tokenize.html + This can be replaced by any function that takes a string as input and returns a list of tokens as output. + +Returns: + `average_word_length` (`float`) : the average number of words in the input list of strings. + +Examples: + >>> data = ["hello world"] + >>> wordlength = evaluate.load("word_length", module_type="measurement") + >>> results = wordlength.compute(data=data) + >>> print(results) + {'average_word_length': 2} +""" + +# TODO: Add BibTeX citation +_CITATION = """\ +@InProceedings{huggingface:module, +title = {A great new module}, +authors={huggingface, Inc.}, +year={2020} +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class WordLength(evaluate.Measurement): + """This measurement returns the average number of words in the input string(s).""" + + def _info(self): + # TODO: Specifies the evaluate.MeasurementInfo object + return evaluate.MeasurementInfo( + # This is the description that will appear on the modules page. + module_type="measurement", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + # This defines the format of each prediction and reference + features=datasets.Features( + { + "data": datasets.Value("string"), + } + ), + ) + + def _download_and_prepare(self, dl_manager): + import nltk + + nltk.download("punkt") + + def _compute(self, data, tokenizer=word_tokenize): + """Returns the average word length of the input data""" + lengths = [len(tokenizer(d)) for d in data] + average_length = mean(lengths) + return {"average_word_length": average_length} diff --git a/TextClassification/TextClassification/evaluate/metrics/accuracy/README.md b/TextClassification/TextClassification/evaluate/metrics/accuracy/README.md new file mode 100644 index 0000000000000000000000000000000000000000..379a9f5e4bf84001e4f5a1ee642d20450135fc39 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/accuracy/README.md @@ -0,0 +1,119 @@ +--- +title: Accuracy +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Accuracy is the proportion of correct predictions among the total number of cases processed. It can be computed with: + Accuracy = (TP + TN) / (TP + TN + FP + FN) + Where: + TP: True positive + TN: True negative + FP: False positive + FN: False negative +--- + +# Metric Card for Accuracy + + +## Metric Description + +Accuracy is the proportion of correct predictions among the total number of cases processed. It can be computed with: +Accuracy = (TP + TN) / (TP + TN + FP + FN) + Where: +TP: True positive +TN: True negative +FP: False positive +FN: False negative + + +## How to Use + +At minimum, this metric requires predictions and references as inputs. + +```python +>>> accuracy_metric = evaluate.load("accuracy") +>>> results = accuracy_metric.compute(references=[0, 1], predictions=[0, 1]) +>>> print(results) +{'accuracy': 1.0} +``` + + +### Inputs +- **predictions** (`list` of `int`): Predicted labels. +- **references** (`list` of `int`): Ground truth labels. +- **normalize** (`boolean`): If set to False, returns the number of correctly classified samples. Otherwise, returns the fraction of correctly classified samples. Defaults to True. +- **sample_weight** (`list` of `float`): Sample weights Defaults to None. + + +### Output Values +- **accuracy**(`float` or `int`): Accuracy score. Minimum possible value is 0. Maximum possible value is 1.0, or the number of examples input, if `normalize` is set to `True`. A higher score means higher accuracy. + +Output Example(s): +```python +{'accuracy': 1.0} +``` + +This metric outputs a dictionary, containing the accuracy score. + + +#### Values from Popular Papers + +Top-1 or top-5 accuracy is often used to report performance on supervised classification tasks such as image classification (e.g. on [ImageNet](https://paperswithcode.com/sota/image-classification-on-imagenet)) or sentiment analysis (e.g. on [IMDB](https://paperswithcode.com/sota/text-classification-on-imdb)). + + +### Examples + +Example 1-A simple example +```python +>>> accuracy_metric = evaluate.load("accuracy") +>>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0]) +>>> print(results) +{'accuracy': 0.5} +``` + +Example 2-The same as Example 1, except with `normalize` set to `False`. +```python +>>> accuracy_metric = evaluate.load("accuracy") +>>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], normalize=False) +>>> print(results) +{'accuracy': 3.0} +``` + +Example 3-The same as Example 1, except with `sample_weight` set. +```python +>>> accuracy_metric = evaluate.load("accuracy") +>>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], sample_weight=[0.5, 2, 0.7, 0.5, 9, 0.4]) +>>> print(results) +{'accuracy': 0.8778625954198473} +``` + + +## Limitations and Bias +This metric can be easily misleading, especially in the case of unbalanced classes. For example, a high accuracy might be because a model is doing well, but if the data is unbalanced, it might also be because the model is only accurately labeling the high-frequency class. In such cases, a more detailed analysis of the model's behavior, or the use of a different metric entirely, is necessary to determine how well the model is actually performing. + + +## Citation(s) +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + + +## Further References diff --git a/TextClassification/TextClassification/evaluate/metrics/accuracy/accuracy.py b/TextClassification/TextClassification/evaluate/metrics/accuracy/accuracy.py new file mode 100644 index 0000000000000000000000000000000000000000..aa5a07328844b225fd78c7ad106c91bab1f2a8e7 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/accuracy/accuracy.py @@ -0,0 +1,106 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Accuracy metric.""" + +import datasets +from sklearn.metrics import accuracy_score + +import evaluate + + +_DESCRIPTION = """ +Accuracy is the proportion of correct predictions among the total number of cases processed. It can be computed with: +Accuracy = (TP + TN) / (TP + TN + FP + FN) + Where: +TP: True positive +TN: True negative +FP: False positive +FN: False negative +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions (`list` of `int`): Predicted labels. + references (`list` of `int`): Ground truth labels. + normalize (`boolean`): If set to False, returns the number of correctly classified samples. Otherwise, returns the fraction of correctly classified samples. Defaults to True. + sample_weight (`list` of `float`): Sample weights Defaults to None. + +Returns: + accuracy (`float` or `int`): Accuracy score. Minimum possible value is 0. Maximum possible value is 1.0, or the number of examples input, if `normalize` is set to `True`.. A higher score means higher accuracy. + +Examples: + + Example 1-A simple example + >>> accuracy_metric = evaluate.load("accuracy") + >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0]) + >>> print(results) + {'accuracy': 0.5} + + Example 2-The same as Example 1, except with `normalize` set to `False`. + >>> accuracy_metric = evaluate.load("accuracy") + >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], normalize=False) + >>> print(results) + {'accuracy': 3.0} + + Example 3-The same as Example 1, except with `sample_weight` set. + >>> accuracy_metric = evaluate.load("accuracy") + >>> results = accuracy_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0], sample_weight=[0.5, 2, 0.7, 0.5, 9, 0.4]) + >>> print(results) + {'accuracy': 0.8778625954198473} +""" + + +_CITATION = """ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Accuracy(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("int32")), + "references": datasets.Sequence(datasets.Value("int32")), + } + if self.config_name == "multilabel" + else { + "predictions": datasets.Value("int32"), + "references": datasets.Value("int32"), + } + ), + reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.accuracy_score.html"], + ) + + def _compute(self, predictions, references, normalize=True, sample_weight=None): + return { + "accuracy": float( + accuracy_score(references, predictions, normalize=normalize, sample_weight=sample_weight) + ) + } diff --git a/TextClassification/TextClassification/evaluate/metrics/accuracy/app.py b/TextClassification/TextClassification/evaluate/metrics/accuracy/app.py new file mode 100644 index 0000000000000000000000000000000000000000..b3b80774f66a3c00c4bf223c860274688509eed5 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/accuracy/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("accuracy") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/accuracy/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/accuracy/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/accuracy/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/bertscore/README.md b/TextClassification/TextClassification/evaluate/metrics/bertscore/README.md new file mode 100644 index 0000000000000000000000000000000000000000..af8a5356f27f38168b8418ee5c98c5aa4cf8d2f0 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/bertscore/README.md @@ -0,0 +1,133 @@ +--- +title: BERT Score +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + BERTScore leverages the pre-trained contextual embeddings from BERT and matches words in candidate and reference + sentences by cosine similarity. + It has been shown to correlate with human judgment on sentence-level and system-level evaluation. + Moreover, BERTScore computes precision, recall, and F1 measure, which can be useful for evaluating different language + generation tasks. + + See the project's README at https://github.com/Tiiiger/bert_score#readme for more information. +--- + +# Metric Card for BERT Score + +## Metric description + +BERTScore is an automatic evaluation metric for text generation that computes a similarity score for each token in the candidate sentence with each token in the reference sentence. It leverages the pre-trained contextual embeddings from [BERT](https://huggingface.co/bert-base-uncased) models and matches words in candidate and reference sentences by cosine similarity. + +Moreover, BERTScore computes precision, recall, and F1 measure, which can be useful for evaluating different language generation tasks. + +## How to use + +BERTScore takes 3 mandatory arguments : `predictions` (a list of string of candidate sentences), `references` (a list of strings or list of list of strings of reference sentences) and either `lang` (a string of two letters indicating the language of the sentences, in [ISO 639-1 format](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes)) or `model_type` (a string specififying which model to use, according to the BERT specification). The default behavior of the metric is to use the suggested model for the target language when one is specified, otherwise to use the `model_type` indicated. + +```python +from evaluate import load +bertscore = load("bertscore") +predictions = ["hello there", "general kenobi"] +references = ["hello there", "general kenobi"] +results = bertscore.compute(predictions=predictions, references=references, lang="en") +``` + +BERTScore also accepts multiple optional arguments: + + +`num_layers` (int): The layer of representation to use. The default is the number of layers tuned on WMT16 correlation data, which depends on the `model_type` used. + +`verbose` (bool): Turn on intermediate status update. The default value is `False`. + +`idf` (bool or dict): Use idf weighting; can also be a precomputed idf_dict. + +`device` (str): On which the contextual embedding model will be allocated on. If this argument is `None`, the model lives on `cuda:0` if cuda is available. + +`nthreads` (int): Number of threads used for computation. The default value is `4`. + +`rescale_with_baseline` (bool): Rescale BERTScore with the pre-computed baseline. The default value is `False`. + +`batch_size` (int): BERTScore processing batch size, at least one of `model_type` or `lang`. `lang` needs to be specified when `rescale_with_baseline` is `True`. + +`baseline_path` (str): Customized baseline file. + +`use_fast_tokenizer` (bool): `use_fast` parameter passed to HF tokenizer. The default value is `False`. + + +## Output values + +BERTScore outputs a dictionary with the following values: + +`precision`: The [precision](https://huggingface.co/metrics/precision) for each sentence from the `predictions` + `references` lists, which ranges from 0.0 to 1.0. + +`recall`: The [recall](https://huggingface.co/metrics/recall) for each sentence from the `predictions` + `references` lists, which ranges from 0.0 to 1.0. + +`f1`: The [F1 score](https://huggingface.co/metrics/f1) for each sentence from the `predictions` + `references` lists, which ranges from 0.0 to 1.0. + +`hashcode:` The hashcode of the library. + + +### Values from popular papers +The [original BERTScore paper](https://openreview.net/pdf?id=SkeHuCVFDr) reported average model selection accuracies (Hits@1) on WMT18 hybrid systems for different language pairs, which ranged from 0.004 for `en<->tr` to 0.824 for `en<->de`. + +For more recent model performance, see the [metric leaderboard](https://paperswithcode.com/paper/bertscore-evaluating-text-generation-with). + +## Examples + +Maximal values with the `distilbert-base-uncased` model: + +```python +from evaluate import load +bertscore = load("bertscore") +predictions = ["hello world", "general kenobi"] +references = ["hello world", "general kenobi"] +results = bertscore.compute(predictions=predictions, references=references, model_type="distilbert-base-uncased") +print(results) +{'precision': [1.0, 1.0], 'recall': [1.0, 1.0], 'f1': [1.0, 1.0], 'hashcode': 'distilbert-base-uncased_L5_no-idf_version=0.3.10(hug_trans=4.10.3)'} +``` + +Partial match with the `distilbert-base-uncased` model: + +```python +from evaluate import load +bertscore = load("bertscore") +predictions = ["hello world", "general kenobi"] +references = ["goodnight moon", "the sun is shining"] +results = bertscore.compute(predictions=predictions, references=references, model_type="distilbert-base-uncased") +print(results) +{'precision': [0.7380737066268921, 0.5584042072296143], 'recall': [0.7380737066268921, 0.5889028906822205], 'f1': [0.7380737066268921, 0.5732481479644775], 'hashcode': 'bert-base-uncased_L5_no-idf_version=0.3.10(hug_trans=4.10.3)'} +``` + +## Limitations and bias + +The [original BERTScore paper](https://openreview.net/pdf?id=SkeHuCVFDr) showed that BERTScore correlates well with human judgment on sentence-level and system-level evaluation, but this depends on the model and language pair selected. + +Furthermore, not all languages are supported by the metric -- see the [BERTScore supported language list](https://github.com/google-research/bert/blob/master/multilingual.md#list-of-languages) for more information. + +Finally, calculating the BERTScore metric involves downloading the BERT model that is used to compute the score-- the default model for `en`, `roberta-large`, takes over 1.4GB of storage space and downloading it can take a significant amount of time depending on the speed of your internet connection. If this is an issue, choose a smaller model; for instance `distilbert-base-uncased` is 268MB. A full list of compatible models can be found [here](https://docs.google.com/spreadsheets/d/1RKOVpselB98Nnh_EOC4A2BYn8_201tmPODpNWu4w7xI/edit#gid=0). + + +## Citation + +```bibtex +@inproceedings{bert-score, + title={BERTScore: Evaluating Text Generation with BERT}, + author={Tianyi Zhang* and Varsha Kishore* and Felix Wu* and Kilian Q. Weinberger and Yoav Artzi}, + booktitle={International Conference on Learning Representations}, + year={2020}, + url={https://openreview.net/forum?id=SkeHuCVFDr} +} +``` + +## Further References + +- [BERTScore Project README](https://github.com/Tiiiger/bert_score#readme) +- [BERTScore ICLR 2020 Poster Presentation](https://iclr.cc/virtual_2020/poster_SkeHuCVFDr.html) diff --git a/TextClassification/TextClassification/evaluate/metrics/bertscore/app.py b/TextClassification/TextClassification/evaluate/metrics/bertscore/app.py new file mode 100644 index 0000000000000000000000000000000000000000..e24884e8afcbeb0c2a68c59d4e7db4eeb3fd8c43 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/bertscore/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("bertscore") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/bertscore/bertscore.py b/TextClassification/TextClassification/evaluate/metrics/bertscore/bertscore.py new file mode 100644 index 0000000000000000000000000000000000000000..071e76ff3be0ad52302ab4d5fc99beb4ed125161 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/bertscore/bertscore.py @@ -0,0 +1,215 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" BERTScore metric. """ + +import functools +from contextlib import contextmanager + +import bert_score +import datasets +from packaging import version + +import evaluate + + +@contextmanager +def filter_logging_context(): + def filter_log(record): + return False if "This IS expected if you are initializing" in record.msg else True + + logger = datasets.utils.logging.get_logger("transformers.modeling_utils") + logger.addFilter(filter_log) + try: + yield + finally: + logger.removeFilter(filter_log) + + +_CITATION = """\ +@inproceedings{bert-score, + title={BERTScore: Evaluating Text Generation with BERT}, + author={Tianyi Zhang* and Varsha Kishore* and Felix Wu* and Kilian Q. Weinberger and Yoav Artzi}, + booktitle={International Conference on Learning Representations}, + year={2020}, + url={https://openreview.net/forum?id=SkeHuCVFDr} +} +""" + +_DESCRIPTION = """\ +BERTScore leverages the pre-trained contextual embeddings from BERT and matches words in candidate and reference +sentences by cosine similarity. +It has been shown to correlate with human judgment on sentence-level and system-level evaluation. +Moreover, BERTScore computes precision, recall, and F1 measure, which can be useful for evaluating different language +generation tasks. + +See the project's README at https://github.com/Tiiiger/bert_score#readme for more information. +""" + +_KWARGS_DESCRIPTION = """ +BERTScore Metrics with the hashcode from a source against one or more references. + +Args: + predictions (list of str): Prediction/candidate sentences. + references (list of str or list of list of str): Reference sentences. + lang (str): Language of the sentences; required (e.g. 'en'). + model_type (str): Bert specification, default using the suggested + model for the target language; has to specify at least one of + `model_type` or `lang`. + num_layers (int): The layer of representation to use, + default using the number of layers tuned on WMT16 correlation data. + verbose (bool): Turn on intermediate status update. + idf (bool or dict): Use idf weighting; can also be a precomputed idf_dict. + device (str): On which the contextual embedding model will be allocated on. + If this argument is None, the model lives on cuda:0 if cuda is available. + nthreads (int): Number of threads. + batch_size (int): Bert score processing batch size, + at least one of `model_type` or `lang`. `lang` needs to be + specified when `rescale_with_baseline` is True. + rescale_with_baseline (bool): Rescale bertscore with pre-computed baseline. + baseline_path (str): Customized baseline file. + use_fast_tokenizer (bool): `use_fast` parameter passed to HF tokenizer. New in version 0.3.10. + +Returns: + precision: Precision. + recall: Recall. + f1: F1 score. + hashcode: Hashcode of the library. + +Examples: + + >>> predictions = ["hello there", "general kenobi"] + >>> references = ["hello there", "general kenobi"] + >>> bertscore = evaluate.load("bertscore") + >>> results = bertscore.compute(predictions=predictions, references=references, lang="en") + >>> print([round(v, 2) for v in results["f1"]]) + [1.0, 1.0] +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class BERTScore(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + homepage="https://github.com/Tiiiger/bert_score", + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), + } + ), + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + ], + codebase_urls=["https://github.com/Tiiiger/bert_score"], + reference_urls=[ + "https://github.com/Tiiiger/bert_score", + "https://arxiv.org/abs/1904.09675", + ], + ) + + def _compute( + self, + predictions, + references, + lang=None, + model_type=None, + num_layers=None, + verbose=False, + idf=False, + device=None, + batch_size=64, + nthreads=4, + all_layers=False, + rescale_with_baseline=False, + baseline_path=None, + use_fast_tokenizer=False, + ): + + if isinstance(references[0], str): + references = [[ref] for ref in references] + + if idf: + idf_sents = [r for ref in references for r in ref] + else: + idf_sents = None + + get_hash = bert_score.utils.get_hash + scorer = bert_score.BERTScorer + + if version.parse(bert_score.__version__) >= version.parse("0.3.10"): + get_hash = functools.partial(get_hash, use_fast_tokenizer=use_fast_tokenizer) + scorer = functools.partial(scorer, use_fast_tokenizer=use_fast_tokenizer) + elif use_fast_tokenizer: + raise ImportWarning( + "To use a fast tokenizer, the module `bert-score>=0.3.10` is required, and the current version of " + "`bert-score` doesn't match this condition.\n" + 'You can install it with `pip install "bert-score>=0.3.10"`.' + ) + + if model_type is None: + if lang is None: + raise ValueError( + "Either 'lang' (e.g. 'en') or 'model_type' (e.g. 'microsoft/deberta-xlarge-mnli')" + " must be specified" + ) + model_type = bert_score.utils.lang2model[lang.lower()] + + if num_layers is None: + num_layers = bert_score.utils.model2layers[model_type] + + hashcode = get_hash( + model=model_type, + num_layers=num_layers, + idf=idf, + rescale_with_baseline=rescale_with_baseline, + use_custom_baseline=baseline_path is not None, + ) + + with filter_logging_context(): + if not hasattr(self, "cached_bertscorer") or self.cached_bertscorer.hash != hashcode: + self.cached_bertscorer = scorer( + model_type=model_type, + num_layers=num_layers, + batch_size=batch_size, + nthreads=nthreads, + all_layers=all_layers, + idf=idf, + idf_sents=idf_sents, + device=device, + lang=lang, + rescale_with_baseline=rescale_with_baseline, + baseline_path=baseline_path, + ) + + (P, R, F) = self.cached_bertscorer.score( + cands=predictions, + refs=references, + verbose=verbose, + batch_size=batch_size, + ) + output_dict = { + "precision": P.tolist(), + "recall": R.tolist(), + "f1": F.tolist(), + "hashcode": hashcode, + } + return output_dict diff --git a/TextClassification/TextClassification/evaluate/metrics/bertscore/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/bertscore/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..838242f4a9b6aca112847af598d3e039205fa5c8 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/bertscore/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +bert_score \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/bleu/README.md b/TextClassification/TextClassification/evaluate/metrics/bleu/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6371b6cfed0b55e0cf6322bfd2ce564e60f1e7d6 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/bleu/README.md @@ -0,0 +1,160 @@ +--- +title: BLEU +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + BLEU (Bilingual Evaluation Understudy) is an algorithm for evaluating the quality of text which has been machine-translated from one natural language to another. + Quality is considered to be the correspondence between a machine's output and that of a human: "the closer a machine translation is to a professional human translation, the better it is" + – this is the central idea behind BLEU. BLEU was one of the first metrics to claim a high correlation with human judgements of quality, and remains one of the most popular automated and inexpensive metrics. + + Scores are calculated for individual translated segments—generally sentences—by comparing them with a set of good quality reference translations. + Those scores are then averaged over the whole corpus to reach an estimate of the translation's overall quality. + Neither intelligibility nor grammatical correctness are not taken into account. +--- + +# Metric Card for BLEU + + +## Metric Description +BLEU (Bilingual Evaluation Understudy) is an algorithm for evaluating the quality of text which has been machine-translated from one natural language to another. Quality is considered to be the correspondence between a machine's output and that of a human: "the closer a machine translation is to a professional human translation, the better it is" – this is the central idea behind BLEU. BLEU was one of the first metrics to claim a high correlation with human judgements of quality, and remains one of the most popular automated and inexpensive metrics. + +Scores are calculated for individual translated segments—generally sentences—by comparing them with a set of good quality reference translations. Those scores are then averaged over the whole corpus to reach an estimate of the translation's overall quality. Neither intelligibility nor grammatical correctness are not taken into account. + +## Intended Uses +BLEU and BLEU-derived metrics are most often used for machine translation. + +## How to Use + +This metric takes as input a list of predicted sentences and a list of lists of reference sentences (since each predicted sentence can have multiple references): + +```python +>>> predictions = ["hello there general kenobi", "foo bar foobar"] +>>> references = [ +... ["hello there general kenobi", "hello there !"], +... ["foo bar foobar"] +... ] +>>> bleu = evaluate.load("bleu") +>>> results = bleu.compute(predictions=predictions, references=references) +>>> print(results) +{'bleu': 1.0, 'precisions': [1.0, 1.0, 1.0, 1.0], 'brevity_penalty': 1.0, 'length_ratio': 1.1666666666666667, 'translation_length': 7, 'reference_length': 6} +``` + +### Inputs +- **predictions** (`list` of `str`s): Translations to score. +- **references** (`list` of `list`s of `str`s): references for each translation. +- ** tokenizer** : approach used for standardizing `predictions` and `references`. + The default tokenizer is `tokenizer_13a`, a relatively minimal tokenization approach that is however equivalent to `mteval-v13a`, used by WMT. + This can be replaced by another tokenizer from a source such as [SacreBLEU](https://github.com/mjpost/sacrebleu/tree/master/sacrebleu/tokenizers). + +The default tokenizer is based on whitespace and regexes. It can be replaced by any function that takes a string as input and returns a list of tokens as output. E.g. `word_tokenize()` from [NLTK](https://www.nltk.org/api/nltk.tokenize.html) or pretrained tokenizers from the [Tokenizers library](https://huggingface.co/docs/tokenizers/index). +- **max_order** (`int`): Maximum n-gram order to use when computing BLEU score. Defaults to `4`. +- **smooth** (`boolean`): Whether or not to apply Lin et al. 2004 smoothing. Defaults to `False`. + +### Output Values +- **bleu** (`float`): bleu score +- **precisions** (`list` of `float`s): geometric mean of n-gram precisions, +- **brevity_penalty** (`float`): brevity penalty, +- **length_ratio** (`float`): ratio of lengths, +- **translation_length** (`int`): translation_length, +- **reference_length** (`int`): reference_length + +Output Example: +```python +{'bleu': 1.0, 'precisions': [1.0, 1.0, 1.0, 1.0], 'brevity_penalty': 1.0, 'length_ratio': 1.1666666666666667, 'translation_length': 7, 'reference_length': 6} +``` + +BLEU's output is always a number between 0 and 1. This value indicates how similar the candidate text is to the reference texts, with values closer to 1 representing more similar texts. Few human translations will attain a score of 1, since this would indicate that the candidate is identical to one of the reference translations. For this reason, it is not necessary to attain a score of 1. Because there are more opportunities to match, adding additional reference translations will increase the BLEU score. + +#### Values from Popular Papers +The [original BLEU paper](https://aclanthology.org/P02-1040/) (Papineni et al. 2002) compares BLEU scores of five different models on the same 500-sentence corpus. These scores ranged from 0.0527 to 0.2571. + +The [Attention is All you Need paper](https://proceedings.neurips.cc/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf) (Vaswani et al. 2017) got a BLEU score of 0.284 on the WMT 2014 English-to-German translation task, and 0.41 on the WMT 2014 English-to-French translation task. + +### Examples + +Example where each prediction has 1 reference: +```python +>>> predictions = ["hello there general kenobi","foo bar foobar"] +>>> references = [ +... ["hello there general kenobi"], +... ["foo bar foobar"] +... ] +>>> bleu = evaluate.load("bleu") +>>> results = bleu.compute(predictions=predictions, references=references) +>>> print(results) +{'bleu': 1.0, 'precisions': [1.0, 1.0, 1.0, 1.0], 'brevity_penalty': 1.0, 'length_ratio': 1.0, 'translation_length': 7, 'reference_length': 7} +``` + +Example where the second prediction has 2 references: +```python +>>> predictions = [ +... ["hello there general kenobi", +... ["foo bar foobar"] +... ] +>>> references = [ +... [["hello there general kenobi"], ["hello there!"]], +... [["foo bar foobar"]] +... ] +>>> bleu = evaluate.load("bleu") +>>> results = bleu.compute(predictions=predictions, references=references) +>>> print(results) +{'bleu': 1.0, 'precisions': [1.0, 1.0, 1.0, 1.0], 'brevity_penalty': 1.0, 'length_ratio': 1.1666666666666667, 'translation_length': 7, 'reference_length': 6} +``` + +Example with the word tokenizer from NLTK: +```python +>>> bleu = evaluate.load("bleu") +>>> from nltk.tokenize import word_tokenize +>>> predictions = [ +... ["hello there general kenobi", +... ["foo bar foobar"] +... ] +>>> references = [ +... [["hello there general kenobi"], ["hello there!"]], +... [["foo bar foobar"]] +... ] +>>> results = bleu.compute(predictions=predictions, references=references, tokenizer=word_tokenize) +>>> print(results) +{'bleu': 1.0, 'precisions': [1.0, 1.0, 1.0, 1.0], 'brevity_penalty': 1.0, 'length_ratio': 1.1666666666666667, 'translation_length': 7, 'reference_length': 6} +``` + +## Limitations and Bias +This metric has multiple known limitations: +- BLEU compares overlap in tokens from the predictions and references, instead of comparing meaning. This can lead to discrepancies between BLEU scores and human ratings. +- Shorter predicted translations achieve higher scores than longer ones, simply due to how the score is calculated. A brevity penalty is introduced to attempt to counteract this. +- BLEU scores are not comparable across different datasets, nor are they comparable across different languages. +- BLEU scores can vary greatly depending on which parameters are used to generate the scores, especially when different tokenization and normalization techniques are used. It is therefore not possible to compare BLEU scores generated using different parameters, or when these parameters are unknown. For more discussion around this topic, see the following [issue](https://github.com/huggingface/datasets/issues/137). + +## Citation +```bibtex +@INPROCEEDINGS{Papineni02bleu:a, + author = {Kishore Papineni and Salim Roukos and Todd Ward and Wei-jing Zhu}, + title = {BLEU: a Method for Automatic Evaluation of Machine Translation}, + booktitle = {}, + year = {2002}, + pages = {311--318} +} +@inproceedings{lin-och-2004-orange, + title = "{ORANGE}: a Method for Evaluating Automatic Evaluation Metrics for Machine Translation", + author = "Lin, Chin-Yew and + Och, Franz Josef", + booktitle = "{COLING} 2004: Proceedings of the 20th International Conference on Computational Linguistics", + month = "aug 23{--}aug 27", + year = "2004", + address = "Geneva, Switzerland", + publisher = "COLING", + url = "https://www.aclweb.org/anthology/C04-1072", + pages = "501--507", +} +``` + +## Further References +- This Hugging Face implementation uses [this Tensorflow implementation](https://github.com/tensorflow/nmt/blob/master/nmt/scripts/bleu.py) diff --git a/TextClassification/TextClassification/evaluate/metrics/bleu/app.py b/TextClassification/TextClassification/evaluate/metrics/bleu/app.py new file mode 100644 index 0000000000000000000000000000000000000000..fa8a6817380386120c3ef8ce492a9a962a0c22bc --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/bleu/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("bleu") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/bleu/bleu.py b/TextClassification/TextClassification/evaluate/metrics/bleu/bleu.py new file mode 100644 index 0000000000000000000000000000000000000000..38a10c3b34ee7dc24540fe8ece482e26f9cf3588 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/bleu/bleu.py @@ -0,0 +1,133 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" BLEU metric. """ + +import datasets + +import evaluate + +from .nmt_bleu import compute_bleu # From: https://github.com/tensorflow/nmt/blob/master/nmt/scripts/bleu.py +from .tokenizer_13a import Tokenizer13a + + +_CITATION = """\ +@INPROCEEDINGS{Papineni02bleu:a, + author = {Kishore Papineni and Salim Roukos and Todd Ward and Wei-jing Zhu}, + title = {BLEU: a Method for Automatic Evaluation of Machine Translation}, + booktitle = {}, + year = {2002}, + pages = {311--318} +} +@inproceedings{lin-och-2004-orange, + title = "{ORANGE}: a Method for Evaluating Automatic Evaluation Metrics for Machine Translation", + author = "Lin, Chin-Yew and + Och, Franz Josef", + booktitle = "{COLING} 2004: Proceedings of the 20th International Conference on Computational Linguistics", + month = "aug 23{--}aug 27", + year = "2004", + address = "Geneva, Switzerland", + publisher = "COLING", + url = "https://www.aclweb.org/anthology/C04-1072", + pages = "501--507", +} +""" + +_DESCRIPTION = """\ +BLEU (Bilingual Evaluation Understudy) is an algorithm for evaluating the quality of text which has been machine-translated from one natural language to another. +Quality is considered to be the correspondence between a machine's output and that of a human: "the closer a machine translation is to a professional human translation, the better it is" +– this is the central idea behind BLEU. BLEU was one of the first metrics to claim a high correlation with human judgements of quality, and remains one of the most popular automated and inexpensive metrics. + +Scores are calculated for individual translated segments—generally sentences—by comparing them with a set of good quality reference translations. +Those scores are then averaged over the whole corpus to reach an estimate of the translation's overall quality. +Neither intelligibility nor grammatical correctness are not taken into account. +""" + +_KWARGS_DESCRIPTION = """ +Computes BLEU score of translated segments against one or more references. +Args: + predictions: list of translations to score. + references: list of lists of or just a list of references for each translation. + tokenizer : approach used for tokenizing `predictions` and `references`. + The default tokenizer is `tokenizer_13a`, a minimal tokenization approach that is equivalent to `mteval-v13a`, used by WMT. + This can be replaced by any function that takes a string as input and returns a list of tokens as output. + max_order: Maximum n-gram order to use when computing BLEU score. + smooth: Whether or not to apply Lin et al. 2004 smoothing. +Returns: + 'bleu': bleu score, + 'precisions': geometric mean of n-gram precisions, + 'brevity_penalty': brevity penalty, + 'length_ratio': ratio of lengths, + 'translation_length': translation_length, + 'reference_length': reference_length +Examples: + + >>> predictions = ["hello there general kenobi", "foo bar foobar"] + >>> references = [ + ... ["hello there general kenobi", "hello there!"], + ... ["foo bar foobar"] + ... ] + >>> bleu = evaluate.load("bleu") + >>> results = bleu.compute(predictions=predictions, references=references) + >>> print(results["bleu"]) + 1.0 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Bleu(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), + } + ), + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + ], + codebase_urls=["https://github.com/tensorflow/nmt/blob/master/nmt/scripts/bleu.py"], + reference_urls=[ + "https://en.wikipedia.org/wiki/BLEU", + "https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213", + ], + ) + + def _compute(self, predictions, references, tokenizer=Tokenizer13a(), max_order=4, smooth=False): + # if only one reference is provided make sure we still use list of lists + if isinstance(references[0], str): + references = [[ref] for ref in references] + + references = [[tokenizer(r) for r in ref] for ref in references] + predictions = [tokenizer(p) for p in predictions] + score = compute_bleu( + reference_corpus=references, translation_corpus=predictions, max_order=max_order, smooth=smooth + ) + (bleu, precisions, bp, ratio, translation_length, reference_length) = score + return { + "bleu": bleu, + "precisions": precisions, + "brevity_penalty": bp, + "length_ratio": ratio, + "translation_length": translation_length, + "reference_length": reference_length, + } diff --git a/TextClassification/TextClassification/evaluate/metrics/bleu/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/bleu/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a135afacbdd142a40cb9bcfe6073b76cc62d995 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/bleu/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/bleu/tokenizer_13a.py b/TextClassification/TextClassification/evaluate/metrics/bleu/tokenizer_13a.py new file mode 100644 index 0000000000000000000000000000000000000000..c7a1b3dbd39d7db5ed0c4d000d6d648e2cbf0009 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/bleu/tokenizer_13a.py @@ -0,0 +1,100 @@ +# Source: https://github.com/mjpost/sacrebleu/blob/master/sacrebleu/tokenizers/tokenizer_13a.py +# Copyright 2020 SacreBLEU Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from functools import lru_cache + + +class BaseTokenizer: + """A base dummy tokenizer to derive from.""" + + def signature(self): + """ + Returns a signature for the tokenizer. + :return: signature string + """ + return "none" + + def __call__(self, line): + """ + Tokenizes an input line with the tokenizer. + :param line: a segment to tokenize + :return: the tokenized line + """ + return line + + +class TokenizerRegexp(BaseTokenizer): + def signature(self): + return "re" + + def __init__(self): + self._re = [ + # language-dependent part (assuming Western languages) + (re.compile(r"([\{-\~\[-\` -\&\(-\+\:-\@\/])"), r" \1 "), + # tokenize period and comma unless preceded by a digit + (re.compile(r"([^0-9])([\.,])"), r"\1 \2 "), + # tokenize period and comma unless followed by a digit + (re.compile(r"([\.,])([^0-9])"), r" \1 \2"), + # tokenize dash when preceded by a digit + (re.compile(r"([0-9])(-)"), r"\1 \2 "), + # one space only between words + # NOTE: Doing this in Python (below) is faster + # (re.compile(r'\s+'), r' '), + ] + + @lru_cache(maxsize=2**16) + def __call__(self, line): + """Common post-processing tokenizer for `13a` and `zh` tokenizers. + :param line: a segment to tokenize + :return: the tokenized line + """ + for (_re, repl) in self._re: + line = _re.sub(repl, line) + + # no leading or trailing spaces, single space within words + # return ' '.join(line.split()) + # This line is changed with regards to the original tokenizer (seen above) to return individual words + return line.split() + + +class Tokenizer13a(BaseTokenizer): + def signature(self): + return "13a" + + def __init__(self): + self._post_tokenizer = TokenizerRegexp() + + @lru_cache(maxsize=2**16) + def __call__(self, line): + """Tokenizes an input line using a relatively minimal tokenization + that is however equivalent to mteval-v13a, used by WMT. + + :param line: a segment to tokenize + :return: the tokenized line + """ + + # language-independent part: + line = line.replace("", "") + line = line.replace("-\n", "") + line = line.replace("\n", " ") + + if "&" in line: + line = line.replace(""", '"') + line = line.replace("&", "&") + line = line.replace("<", "<") + line = line.replace(">", ">") + + return self._post_tokenizer(f" {line} ") diff --git a/TextClassification/TextClassification/evaluate/metrics/bleurt/README.md b/TextClassification/TextClassification/evaluate/metrics/bleurt/README.md new file mode 100644 index 0000000000000000000000000000000000000000..03a9bf829822e473636bf66e5a25f4edb8526d5b --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/bleurt/README.md @@ -0,0 +1,108 @@ +--- +title: BLEURT +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + BLEURT a learnt evaluation metric for Natural Language Generation. It is built using multiple phases of transfer learning starting from a pretrained BERT model (Devlin et al. 2018) + and then employing another pre-training phrase using synthetic data. Finally it is trained on WMT human annotations. You may run BLEURT out-of-the-box or fine-tune + it for your specific application (the latter is expected to perform better). + + See the project's README at https://github.com/google-research/bleurt#readme for more information. +--- + +# Metric Card for BLEURT + + +## Metric Description +BLEURT is a learned evaluation metric for Natural Language Generation. It is built using multiple phases of transfer learning starting from a pretrained BERT model [Devlin et al. 2018](https://arxiv.org/abs/1810.04805), employing another pre-training phrase using synthetic data, and finally trained on WMT human annotations. + +It is possible to run BLEURT out-of-the-box or fine-tune it for your specific application (the latter is expected to perform better). +See the project's [README](https://github.com/google-research/bleurt#readme) for more information. + +## Intended Uses +BLEURT is intended to be used for evaluating text produced by language models. + +## How to Use + +This metric takes as input lists of predicted sentences and reference sentences: + +```python +>>> predictions = ["hello there", "general kenobi"] +>>> references = ["hello there", "general kenobi"] +>>> bleurt = load("bleurt", module_type="metric") +>>> results = bleurt.compute(predictions=predictions, references=references) +``` + +### Inputs +- **predictions** (`list` of `str`s): List of generated sentences to score. +- **references** (`list` of `str`s): List of references to compare to. +- **checkpoint** (`str`): BLEURT checkpoint. Will default to `BLEURT-tiny` if not specified. Other models that can be chosen are: `"bleurt-tiny-128"`, `"bleurt-tiny-512"`, `"bleurt-base-128"`, `"bleurt-base-512"`, `"bleurt-large-128"`, `"bleurt-large-512"`, `"BLEURT-20-D3"`, `"BLEURT-20-D6"`, `"BLEURT-20-D12"` and `"BLEURT-20"`. + +### Output Values +- **scores** : a `list` of scores, one per prediction. + +Output Example: +```python +{'scores': [1.0295498371124268, 1.0445425510406494]} + +``` + +BLEURT's output is always a number between 0 and (approximately 1). This value indicates how similar the generated text is to the reference texts, with values closer to 1 representing more similar texts. + +#### Values from Popular Papers + +The [original BLEURT paper](https://arxiv.org/pdf/2004.04696.pdf) reported that the metric is better correlated with human judgment compared to similar metrics such as BERT and BERTscore. + +BLEURT is used to compare models across different asks (e.g. (Table to text generation)[https://paperswithcode.com/sota/table-to-text-generation-on-dart?metric=BLEURT]). + +### Examples + +Example with the default model: +```python +>>> predictions = ["hello there", "general kenobi"] +>>> references = ["hello there", "general kenobi"] +>>> bleurt = load("bleurt", module_type="metric") +>>> results = bleurt.compute(predictions=predictions, references=references) +>>> print(results) +{'scores': [1.0295498371124268, 1.0445425510406494]} +``` + +Example with the `"bleurt-base-128"` model checkpoint: +```python +>>> predictions = ["hello there", "general kenobi"] +>>> references = ["hello there", "general kenobi"] +>>> bleurt = load("bleurt", module_type="metric", checkpoint="bleurt-base-128") +>>> results = bleurt.compute(predictions=predictions, references=references) +>>> print(results) +{'scores': [1.0295498371124268, 1.0445425510406494]} +``` + +## Limitations and Bias +The [original BLEURT paper](https://arxiv.org/pdf/2004.04696.pdf) showed that BLEURT correlates well with human judgment, but this depends on the model and language pair selected. + +Furthermore, currently BLEURT only supports English-language scoring, given that it leverages models trained on English corpora. It may also reflect, to a certain extent, biases and correlations that were present in the model training data. + +Finally, calculating the BLEURT metric involves downloading the BLEURT model that is used to compute the score, which can take a significant amount of time depending on the model chosen. Starting with the default model, `bleurt-tiny`, and testing out larger models if necessary can be a useful approach if memory or internet speed is an issue. + + +## Citation +```bibtex +@inproceedings{bleurt, + title={BLEURT: Learning Robust Metrics for Text Generation}, + author={Thibault Sellam and Dipanjan Das and Ankur P. Parikh}, + booktitle={ACL}, + year={2020}, + url={https://arxiv.org/abs/2004.04696} +} +``` + +## Further References +- The original [BLEURT GitHub repo](https://github.com/google-research/bleurt/) diff --git a/TextClassification/TextClassification/evaluate/metrics/bleurt/app.py b/TextClassification/TextClassification/evaluate/metrics/bleurt/app.py new file mode 100644 index 0000000000000000000000000000000000000000..45a1a1c3dbdcc4d4252933c4ef972f1bdb175a8f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/bleurt/app.py @@ -0,0 +1,11 @@ +import sys + +import evaluate +from evaluate.utils import launch_gradio_widget + + +sys.path = [p for p in sys.path if p != "/home/user/app"] +module = evaluate.load("bleurt") +sys.path = ["/home/user/app"] + sys.path + +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/bleurt/bleurt.py b/TextClassification/TextClassification/evaluate/metrics/bleurt/bleurt.py new file mode 100644 index 0000000000000000000000000000000000000000..b47f8d284425d918952cb0f2cfc4202c423b1fda --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/bleurt/bleurt.py @@ -0,0 +1,125 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" BLEURT metric. """ + +import os + +import datasets +from bleurt import score # From: git+https://github.com/google-research/bleurt.git + +import evaluate + + +logger = evaluate.logging.get_logger(__name__) + + +_CITATION = """\ +@inproceedings{bleurt, + title={BLEURT: Learning Robust Metrics for Text Generation}, + author={Thibault Sellam and Dipanjan Das and Ankur P. Parikh}, + booktitle={ACL}, + year={2020}, + url={https://arxiv.org/abs/2004.04696} +} +""" + +_DESCRIPTION = """\ +BLEURT a learnt evaluation metric for Natural Language Generation. It is built using multiple phases of transfer learning starting from a pretrained BERT model (Devlin et al. 2018) +and then employing another pre-training phrase using synthetic data. Finally it is trained on WMT human annotations. You may run BLEURT out-of-the-box or fine-tune +it for your specific application (the latter is expected to perform better). + +See the project's README at https://github.com/google-research/bleurt#readme for more information. +""" + +_KWARGS_DESCRIPTION = """ +BLEURT score. + +Args: + `predictions` (list of str): prediction/candidate sentences + `references` (list of str): reference sentences + `checkpoint` BLEURT checkpoint. Will default to BLEURT-tiny if None. + +Returns: + 'scores': List of scores. +Examples: + + >>> predictions = ["hello there", "general kenobi"] + >>> references = ["hello there", "general kenobi"] + >>> bleurt = evaluate.load("bleurt") + >>> results = bleurt.compute(predictions=predictions, references=references) + >>> print([round(v, 2) for v in results["scores"]]) + [1.03, 1.04] +""" + +CHECKPOINT_URLS = { + "bleurt-tiny-128": "https://storage.googleapis.com/bleurt-oss/bleurt-tiny-128.zip", + "bleurt-tiny-512": "https://storage.googleapis.com/bleurt-oss/bleurt-tiny-512.zip", + "bleurt-base-128": "https://storage.googleapis.com/bleurt-oss/bleurt-base-128.zip", + "bleurt-base-512": "https://storage.googleapis.com/bleurt-oss/bleurt-base-512.zip", + "bleurt-large-128": "https://storage.googleapis.com/bleurt-oss/bleurt-large-128.zip", + "bleurt-large-512": "https://storage.googleapis.com/bleurt-oss/bleurt-large-512.zip", + "BLEURT-20-D3": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D3.zip", + "BLEURT-20-D6": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D6.zip", + "BLEURT-20-D12": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D12.zip", + "BLEURT-20": "https://storage.googleapis.com/bleurt-oss-21/BLEURT-20.zip", +} + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class BLEURT(evaluate.Metric): + def _info(self): + + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + homepage="https://github.com/google-research/bleurt", + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + codebase_urls=["https://github.com/google-research/bleurt"], + reference_urls=["https://github.com/google-research/bleurt", "https://arxiv.org/abs/2004.04696"], + ) + + def _download_and_prepare(self, dl_manager): + + # check that config name specifies a valid BLEURT model + if self.config_name == "default": + logger.warning( + "Using default BLEURT-Base checkpoint for sequence maximum length 128. " + "You can use a bigger model for better results with e.g.: evaluate.load('bleurt', 'bleurt-large-512')." + ) + self.config_name = "bleurt-base-128" + + if self.config_name.lower() in CHECKPOINT_URLS: + checkpoint_name = self.config_name.lower() + + elif self.config_name.upper() in CHECKPOINT_URLS: + checkpoint_name = self.config_name.upper() + + else: + raise KeyError( + f"{self.config_name} model not found. You should supply the name of a model checkpoint for bleurt in {CHECKPOINT_URLS.keys()}" + ) + + # download the model checkpoint specified by self.config_name and set up the scorer + model_path = dl_manager.download_and_extract(CHECKPOINT_URLS[checkpoint_name]) + self.scorer = score.BleurtScorer(os.path.join(model_path, checkpoint_name)) + + def _compute(self, predictions, references): + scores = self.scorer.score(references=references, candidates=predictions) + return {"scores": scores} diff --git a/TextClassification/TextClassification/evaluate/metrics/bleurt/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/bleurt/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..15112ee4f090ba2a76e4b813f521a6448f55b405 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/bleurt/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +git+https://github.com/google-research/bleurt.git \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/brier_score/README.md b/TextClassification/TextClassification/evaluate/metrics/brier_score/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1e965003fc3dc2de64b71a48a2d97b846ab87b9e --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/brier_score/README.md @@ -0,0 +1,106 @@ +--- +title: Brier Score +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + The Brier score is a measure of the error between two probability distributions. +--- + +# Metric Card for Brier Score + + +## Metric Description +Brier score is a type of evaluation metric for classification tasks, where you predict outcomes such as win/lose, spam/ham, click/no-click etc. +`BrierScore = 1/N * sum( (p_i - o_i)^2 )` + +Where `p_i` is the prediction probability of occurrence of the event, and the term `o_i` is equal to 1 if the event occurred and 0 if not. Which means: the lower the value of this score, the better the prediction. +## How to Use + +At minimum, this metric requires predictions and references as inputs. + +```python +>>> brier_score = evaluate.load("brier_score") +>>> predictions = np.array([0, 0, 1, 1]) +>>> references = np.array([0.1, 0.9, 0.8, 0.3]) +>>> results = brier_score.compute(predictions=predictions, references=references) +``` + +### Inputs + +Mandatory inputs: +- `predictions`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the estimated target values. + +- `references`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the ground truth (correct) target values. + +Optional arguments: +- `sample_weight`: numeric array-like of shape (`n_samples,`) representing sample weights. The default is `None`. +- `pos_label`: the label of the positive class. The default is `1`. + + +### Output Values +This metric returns a dictionary with the following keys: +- `brier_score (float)`: the computed Brier score. + + +Output Example(s): +```python +{'brier_score': 0.5} +``` + +#### Values from Popular Papers + + +### Examples +```python +>>> brier_score = evaluate.load("brier_score") +>>> predictions = np.array([0, 0, 1, 1]) +>>> references = np.array([0.1, 0.9, 0.8, 0.3]) +>>> results = brier_score.compute(predictions=predictions, references=references) +>>> print(results) +{'brier_score': 0.3375} +``` +Example with `y_true` contains string, an error will be raised and `pos_label` should be explicitly specified. +```python +>>> brier_score_metric = evaluate.load("brier_score") +>>> predictions = np.array(["spam", "ham", "ham", "spam"]) +>>> references = np.array([0.1, 0.9, 0.8, 0.3]) +>>> results = brier_score.compute(predictions, references, pos_label="ham") +>>> print(results) +{'brier_score': 0.0374} +``` +## Limitations and Bias +The [brier_score](https://huggingface.co/metrics/brier_score) is appropriate for binary and categorical outcomes that can be structured as true or false, but it is inappropriate for ordinal variables which can take on three or more values. +## Citation(s) +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} + +@Article{brier1950verification, + title={Verification of forecasts expressed in terms of probability}, + author={Brier, Glenn W and others}, + journal={Monthly weather review}, + volume={78}, + number={1}, + pages={1--3}, + year={1950} +} +``` +## Further References +- [Brier Score - Wikipedia](https://en.wikipedia.org/wiki/Brier_score) \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/brier_score/app.py b/TextClassification/TextClassification/evaluate/metrics/brier_score/app.py new file mode 100644 index 0000000000000000000000000000000000000000..d9736ae7c25d204c4de7748316821a717dce6ca8 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/brier_score/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("brier_score") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/brier_score/brier_score.py b/TextClassification/TextClassification/evaluate/metrics/brier_score/brier_score.py new file mode 100644 index 0000000000000000000000000000000000000000..0e2c025c639b785f60555c89d1f03b855e4a97f5 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/brier_score/brier_score.py @@ -0,0 +1,134 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Brier Score Metric""" + +import datasets +from sklearn.metrics import brier_score_loss + +import evaluate + + +_CITATION = """\ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + +_DESCRIPTION = """\ +Brier score is a type of evaluation metric for classification tasks, where you predict outcomes such as win/lose, spam/ham, click/no-click etc. +`BrierScore = 1/N * sum( (p_i - o_i)^2 )` +""" + + +_KWARGS_DESCRIPTION = """ +Args: + y_true : array of shape (n_samples,) + True targets. + + y_prob : array of shape (n_samples,) + Probabilities of the positive class. + + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. + + pos_label : int or str, default=None + Label of the positive class. `pos_label` will be inferred in the + following manner: + + * if `y_true` in {-1, 1} or {0, 1}, `pos_label` defaults to 1; + * else if `y_true` contains string, an error will be raised and + `pos_label` should be explicitly specified; + * otherwise, `pos_label` defaults to the greater label, + i.e. `np.unique(y_true)[-1]`. + +Returns + score : float + Brier score loss. +Examples: + Example-1: if y_true in {-1, 1} or {0, 1}, pos_label defaults to 1. + >>> import numpy as np + >>> brier_score = evaluate.load("brier_score") + >>> references = np.array([0, 0, 1, 1]) + >>> predictions = np.array([0.1, 0.9, 0.8, 0.3]) + >>> results = brier_score.compute(references=references, predictions=predictions) + >>> print(round(results["brier_score"], 4)) + 0.3375 + + Example-2: if y_true contains string, an error will be raised and pos_label should be explicitly specified. + >>> import numpy as np + >>> brier_score = evaluate.load("brier_score") + >>> references = np.array(["spam", "ham", "ham", "spam"]) + >>> predictions = np.array([0.1, 0.9, 0.8, 0.3]) + >>> results = brier_score.compute(references=references, predictions=predictions, pos_label="ham") + >>> print(round(results["brier_score"], 4)) + 0.0375 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class BrierScore(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=self._get_feature_types(), + reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.brier_score_loss.html"], + ) + + def _get_feature_types(self): + if self.config_name == "multilist": + return [ + datasets.Features( + { + "references": datasets.Sequence(datasets.Value("float")), + "predictions": datasets.Sequence(datasets.Value("float")), + } + ), + datasets.Features( + { + "references": datasets.Sequence(datasets.Value("string")), + "predictions": datasets.Sequence(datasets.Value("float")), + } + ), + ] + else: + return [ + datasets.Features( + { + "references": datasets.Value("float"), + "predictions": datasets.Value("float"), + } + ), + datasets.Features( + { + "references": datasets.Value("string"), + "predictions": datasets.Value("float"), + } + ), + ] + + def _compute(self, references, predictions, sample_weight=None, pos_label=1): + + brier_score = brier_score_loss(references, predictions, sample_weight=sample_weight, pos_label=pos_label) + + return {"brier_score": brier_score} diff --git a/TextClassification/TextClassification/evaluate/metrics/brier_score/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/brier_score/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/brier_score/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/cer/README.md b/TextClassification/TextClassification/evaluate/metrics/cer/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1ea4109cce00b058b84edd7867ed872d996417b5 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/cer/README.md @@ -0,0 +1,157 @@ +--- +title: CER +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Character error rate (CER) is a common metric of the performance of an automatic speech recognition system. + + CER is similar to Word Error Rate (WER), but operates on character instead of word. Please refer to docs of WER for further information. + + Character error rate can be computed as: + + CER = (S + D + I) / N = (S + D + I) / (S + D + C) + + where + + S is the number of substitutions, + D is the number of deletions, + I is the number of insertions, + C is the number of correct characters, + N is the number of characters in the reference (N=S+D+C). + + CER's output is not always a number between 0 and 1, in particular when there is a high number of insertions. This value is often associated to the percentage of characters that were incorrectly predicted. The lower the value, the better the + performance of the ASR system with a CER of 0 being a perfect score. +--- + +# Metric Card for CER + +## Metric description + +Character error rate (CER) is a common metric of the performance of an automatic speech recognition (ASR) system. CER is similar to Word Error Rate (WER), but operates on character instead of word. + +Character error rate can be computed as: + +`CER = (S + D + I) / N = (S + D + I) / (S + D + C)` + +where + +`S` is the number of substitutions, + +`D` is the number of deletions, + +`I` is the number of insertions, + +`C` is the number of correct characters, + +`N` is the number of characters in the reference (`N=S+D+C`). + + +## How to use + +The metric takes two inputs: references (a list of references for each speech input) and predictions (a list of transcriptions to score). + +```python +from evaluate import load +cer = load("cer") +cer_score = cer.compute(predictions=predictions, references=references) +``` +## Output values + +This metric outputs a float representing the character error rate. + +``` +print(cer_score) +0.34146341463414637 +``` + +The **lower** the CER value, the **better** the performance of the ASR system, with a CER of 0 being a perfect score. + +However, CER's output is not always a number between 0 and 1, in particular when there is a high number of insertions (see [Examples](#Examples) below). + +### Values from popular papers + +This metric is highly dependent on the content and quality of the dataset, and therefore users can expect very different values for the same model but on different datasets. + +Multilingual datasets such as [Common Voice](https://huggingface.co/datasets/common_voice) report different CERs depending on the language, ranging from 0.02-0.03 for languages such as French and Italian, to 0.05-0.07 for English (see [here](https://github.com/speechbrain/speechbrain/tree/develop/recipes/CommonVoice/ASR/CTC) for more values). + +## Examples + +Perfect match between prediction and reference: + +```python +from evaluate import load +cer = load("cer") +predictions = ["hello world", "good night moon"] +references = ["hello world", "good night moon"] +cer_score = cer.compute(predictions=predictions, references=references) +print(cer_score) +0.0 +``` + +Partial match between prediction and reference: + +```python +from evaluate import load +cer = load("cer") +predictions = ["this is the prediction", "there is an other sample"] +references = ["this is the reference", "there is another one"] +cer_score = cer.compute(predictions=predictions, references=references) +print(cer_score) +0.34146341463414637 +``` + +No match between prediction and reference: + +```python +from evaluate import load +cer = load("cer") +predictions = ["hello"] +references = ["gracias"] +cer_score = cer.compute(predictions=predictions, references=references) +print(cer_score) +1.0 +``` + +CER above 1 due to insertion errors: + +```python +from evaluate import load +cer = load("cer") +predictions = ["hello world"] +references = ["hello"] +cer_score = cer.compute(predictions=predictions, references=references) +print(cer_score) +1.2 +``` + +## Limitations and bias + +CER is useful for comparing different models for tasks such as automatic speech recognition (ASR) and optic character recognition (OCR), especially for multilingual datasets where WER is not suitable given the diversity of languages. However, CER provides no details on the nature of translation errors and further work is therefore required to identify the main source(s) of error and to focus any research effort. + +Also, in some cases, instead of reporting the raw CER, a normalized CER is reported where the number of mistakes is divided by the sum of the number of edit operations (`I` + `S` + `D`) and `C` (the number of correct characters), which results in CER values that fall within the range of 0–100%. + + +## Citation + + +```bibtex +@inproceedings{morris2004, +author = {Morris, Andrew and Maier, Viktoria and Green, Phil}, +year = {2004}, +month = {01}, +pages = {}, +title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.} +} +``` + +## Further References + +- [Hugging Face Tasks -- Automatic Speech Recognition](https://huggingface.co/tasks/automatic-speech-recognition) diff --git a/TextClassification/TextClassification/evaluate/metrics/cer/app.py b/TextClassification/TextClassification/evaluate/metrics/cer/app.py new file mode 100644 index 0000000000000000000000000000000000000000..40c830a4b596fd85b6493ee790152cd79b0e7552 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/cer/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("cer") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/cer/cer.py b/TextClassification/TextClassification/evaluate/metrics/cer/cer.py new file mode 100644 index 0000000000000000000000000000000000000000..c5f4a907243b5755b97dca18276ded5b1473828c --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/cer/cer.py @@ -0,0 +1,159 @@ +# Copyright 2021 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Character Error Ratio (CER) metric. """ + +from typing import List + +import datasets +import jiwer +import jiwer.transforms as tr +from datasets.config import PY_VERSION +from packaging import version + +import evaluate + + +if PY_VERSION < version.parse("3.8"): + import importlib_metadata +else: + import importlib.metadata as importlib_metadata + + +SENTENCE_DELIMITER = "" + + +if version.parse(importlib_metadata.version("jiwer")) < version.parse("2.3.0"): + + class SentencesToListOfCharacters(tr.AbstractTransform): + def __init__(self, sentence_delimiter: str = " "): + self.sentence_delimiter = sentence_delimiter + + def process_string(self, s: str): + return list(s) + + def process_list(self, inp: List[str]): + chars = [] + for sent_idx, sentence in enumerate(inp): + chars.extend(self.process_string(sentence)) + if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(inp) - 1: + chars.append(self.sentence_delimiter) + return chars + + cer_transform = tr.Compose( + [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)] + ) +else: + cer_transform = tr.Compose( + [ + tr.RemoveMultipleSpaces(), + tr.Strip(), + tr.ReduceToSingleSentence(SENTENCE_DELIMITER), + tr.ReduceToListOfListOfChars(), + ] + ) + + +_CITATION = """\ +@inproceedings{inproceedings, + author = {Morris, Andrew and Maier, Viktoria and Green, Phil}, + year = {2004}, + month = {01}, + pages = {}, + title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.} +} +""" + +_DESCRIPTION = """\ +Character error rate (CER) is a common metric of the performance of an automatic speech recognition system. + +CER is similar to Word Error Rate (WER), but operates on character instead of word. Please refer to docs of WER for further information. + +Character error rate can be computed as: + +CER = (S + D + I) / N = (S + D + I) / (S + D + C) + +where + +S is the number of substitutions, +D is the number of deletions, +I is the number of insertions, +C is the number of correct characters, +N is the number of characters in the reference (N=S+D+C). + +CER's output is not always a number between 0 and 1, in particular when there is a high number of insertions. This value is often associated to the percentage of characters that were incorrectly predicted. The lower the value, the better the +performance of the ASR system with a CER of 0 being a perfect score. +""" + +_KWARGS_DESCRIPTION = """ +Computes CER score of transcribed segments against references. +Args: + references: list of references for each speech input. + predictions: list of transcribtions to score. + concatenate_texts: Whether or not to concatenate sentences before evaluation, set to True for more accurate result. +Returns: + (float): the character error rate + +Examples: + + >>> predictions = ["this is the prediction", "there is an other sample"] + >>> references = ["this is the reference", "there is another one"] + >>> cer = evaluate.load("cer") + >>> cer_score = cer.compute(predictions=predictions, references=references) + >>> print(cer_score) + 0.34146341463414637 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class CER(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + codebase_urls=["https://github.com/jitsi/jiwer/"], + reference_urls=[ + "https://en.wikipedia.org/wiki/Word_error_rate", + "https://sites.google.com/site/textdigitisation/qualitymeasures/computingerrorrates", + ], + ) + + def _compute(self, predictions, references, concatenate_texts=False): + if concatenate_texts: + return jiwer.compute_measures( + references, + predictions, + truth_transform=cer_transform, + hypothesis_transform=cer_transform, + )["wer"] + + incorrect = 0 + total = 0 + for prediction, reference in zip(predictions, references): + measures = jiwer.compute_measures( + reference, + prediction, + truth_transform=cer_transform, + hypothesis_transform=cer_transform, + ) + incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"] + total += measures["substitutions"] + measures["deletions"] + measures["hits"] + + return incorrect / total diff --git a/TextClassification/TextClassification/evaluate/metrics/cer/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/cer/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..aeeda9ad86ac3257a56a69d358c7d863236ac87f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/cer/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +jiwer \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/cer/test_cer.py b/TextClassification/TextClassification/evaluate/metrics/cer/test_cer.py new file mode 100644 index 0000000000000000000000000000000000000000..a30a57040605203df5664540dc080ed97f3edab2 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/cer/test_cer.py @@ -0,0 +1,128 @@ +# Copyright 2021 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from cer import CER + + +cer = CER() + + +class TestCER(unittest.TestCase): + def test_cer_case_sensitive(self): + refs = ["White House"] + preds = ["white house"] + # S = 2, D = 0, I = 0, N = 11, CER = 2 / 11 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.1818181818) < 1e-6) + + def test_cer_whitespace(self): + refs = ["were wolf"] + preds = ["werewolf"] + # S = 0, D = 0, I = 1, N = 9, CER = 1 / 9 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.1111111) < 1e-6) + + refs = ["werewolf"] + preds = ["weae wolf"] + # S = 1, D = 1, I = 0, N = 8, CER = 0.25 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.25) < 1e-6) + + # consecutive whitespaces case 1 + refs = ["were wolf"] + preds = ["were wolf"] + # S = 0, D = 0, I = 0, N = 9, CER = 0 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.0) < 1e-6) + + # consecutive whitespaces case 2 + refs = ["were wolf"] + preds = ["were wolf"] + # S = 0, D = 0, I = 0, N = 9, CER = 0 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.0) < 1e-6) + + def test_cer_sub(self): + refs = ["werewolf"] + preds = ["weaewolf"] + # S = 1, D = 0, I = 0, N = 8, CER = 0.125 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.125) < 1e-6) + + def test_cer_del(self): + refs = ["werewolf"] + preds = ["wereawolf"] + # S = 0, D = 1, I = 0, N = 8, CER = 0.125 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.125) < 1e-6) + + def test_cer_insert(self): + refs = ["werewolf"] + preds = ["wereolf"] + # S = 0, D = 0, I = 1, N = 8, CER = 0.125 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.125) < 1e-6) + + def test_cer_equal(self): + refs = ["werewolf"] + char_error_rate = cer.compute(predictions=refs, references=refs) + self.assertEqual(char_error_rate, 0.0) + + def test_cer_list_of_seqs(self): + refs = ["werewolf", "I am your father"] + char_error_rate = cer.compute(predictions=refs, references=refs) + self.assertEqual(char_error_rate, 0.0) + + refs = ["werewolf", "I am your father", "doge"] + preds = ["werxwolf", "I am your father", "doge"] + # S = 1, D = 0, I = 0, N = 28, CER = 1 / 28 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.03571428) < 1e-6) + + def test_correlated_sentences(self): + refs = ["My hovercraft", "is full of eels"] + preds = ["My hovercraft is full", " of eels"] + # S = 0, D = 0, I = 2, N = 28, CER = 2 / 28 + # whitespace at the front of " of eels" will be strip during preporcessing + # so need to insert 2 whitespaces + char_error_rate = cer.compute(predictions=preds, references=refs, concatenate_texts=True) + self.assertTrue(abs(char_error_rate - 0.071428) < 1e-6) + + def test_cer_unicode(self): + refs = ["我能吞下玻璃而不伤身体"] + preds = [" 能吞虾玻璃而 不霜身体啦"] + # S = 3, D = 2, I = 0, N = 11, CER = 5 / 11 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.4545454545) < 1e-6) + + refs = ["我能吞下玻璃", "而不伤身体"] + preds = ["我 能 吞 下 玻 璃", "而不伤身体"] + # S = 0, D = 5, I = 0, N = 11, CER = 5 / 11 + char_error_rate = cer.compute(predictions=preds, references=refs) + self.assertTrue(abs(char_error_rate - 0.454545454545) < 1e-6) + + refs = ["我能吞下玻璃而不伤身体"] + char_error_rate = cer.compute(predictions=refs, references=refs) + self.assertFalse(char_error_rate, 0.0) + + def test_cer_empty(self): + refs = [""] + preds = ["Hypothesis"] + with self.assertRaises(ValueError): + cer.compute(predictions=preds, references=refs) + + +if __name__ == "__main__": + unittest.main() diff --git a/TextClassification/TextClassification/evaluate/metrics/character/README.md b/TextClassification/TextClassification/evaluate/metrics/character/README.md new file mode 100644 index 0000000000000000000000000000000000000000..776da43b5ab8ae3dfd6871b6dfb793a2cc15f9fc --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/character/README.md @@ -0,0 +1,106 @@ +--- +title: CharacTER +emoji: 🔤 +colorFrom: orange +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +- machine-translation +description: >- + CharacTer is a character-level metric inspired by the commonly applied translation edit rate (TER). +--- + +# Metric Card for CharacTER + +## Metric Description +CharacTer is a character-level metric inspired by the translation edit rate (TER) metric. It is +defined as the minimum number of character edits required to adjust a hypothesis, until it completely matches the +reference, normalized by the length of the hypothesis sentence. CharacTer calculates the character level edit +distance while performing the shift edit on word level. Unlike the strict matching criterion in TER, a hypothesis +word is considered to match a reference word and could be shifted, if the edit distance between them is below a +threshold value. The Levenshtein distance between the reference and the shifted hypothesis sequence is computed on the +character level. In addition, the lengths of hypothesis sequences instead of reference sequences are used for +normalizing the edit distance, which effectively counters the issue that shorter translations normally achieve lower +TER. + +## Intended Uses +CharacTER was developed for machine translation evaluation. + +## How to Use + +```python +import evaluate +character = evaluate.load("character") + +# Single hyp/ref +preds = ["this week the saudis denied information published in the new york times"] +refs = ["saudi arabia denied this week information published in the american new york times"] +results = character.compute(references=refs, predictions=preds) + +# Corpus example +preds = ["this week the saudis denied information published in the new york times", + "this is in fact an estimate"] +refs = ["saudi arabia denied this week information published in the american new york times", + "this is actually an estimate"] +results = character.compute(references=refs, predictions=preds) +``` + +### Inputs +- **predictions**: a single prediction or a list of predictions to score. Each prediction should be a string with + tokens separated by spaces. +- **references**: a single reference or a list of reference for each prediction. Each reference should be a string with + tokens separated by spaces. + + +### Output Values + +*=only when a list of references/hypotheses are given + +- **count** (*): how many parallel sentences were processed +- **mean** (*): the mean CharacTER score +- **median** (*): the median score +- **std** (*): standard deviation of the score +- **min** (*): smallest score +- **max** (*): largest score +- **cer_scores**: all scores, one per ref/hyp pair + +### Output Example +```python +{ + 'count': 2, + 'mean': 0.3127282211789254, + 'median': 0.3127282211789254, + 'std': 0.07561653111280243, + 'min': 0.25925925925925924, + 'max': 0.36619718309859156, + 'cer_scores': [0.36619718309859156, 0.25925925925925924] +} +``` + +## Citation +```bibtex +@inproceedings{wang-etal-2016-character, + title = "{C}harac{T}er: Translation Edit Rate on Character Level", + author = "Wang, Weiyue and + Peter, Jan-Thorsten and + Rosendahl, Hendrik and + Ney, Hermann", + booktitle = "Proceedings of the First Conference on Machine Translation: Volume 2, Shared Task Papers", + month = aug, + year = "2016", + address = "Berlin, Germany", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W16-2342", + doi = "10.18653/v1/W16-2342", + pages = "505--510", +} +``` + +## Further References +- Repackaged version that is used in this HF implementation: [https://github.com/bramvanroy/CharacTER](https://github.com/bramvanroy/CharacTER) +- Original version: [https://github.com/rwth-i6/CharacTER](https://github.com/rwth-i6/CharacTER) diff --git a/TextClassification/TextClassification/evaluate/metrics/character/app.py b/TextClassification/TextClassification/evaluate/metrics/character/app.py new file mode 100644 index 0000000000000000000000000000000000000000..e2552b6432a831f2973a29d67b83b38dc4c0a326 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/character/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("character") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/character/character.py b/TextClassification/TextClassification/evaluate/metrics/character/character.py new file mode 100644 index 0000000000000000000000000000000000000000..e7a00995afa30e813100fee106047ff12b89fec1 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/character/character.py @@ -0,0 +1,169 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""CharacTER metric, a character-based TER variant, for machine translation.""" +import math +from statistics import mean, median +from typing import Iterable, List, Union + +import cer +import datasets +from cer import calculate_cer +from datasets import Sequence, Value + +import evaluate + + +_CITATION = """\ +@inproceedings{wang-etal-2016-character, + title = "{C}harac{T}er: Translation Edit Rate on Character Level", + author = "Wang, Weiyue and + Peter, Jan-Thorsten and + Rosendahl, Hendrik and + Ney, Hermann", + booktitle = "Proceedings of the First Conference on Machine Translation: Volume 2, Shared Task Papers", + month = aug, + year = "2016", + address = "Berlin, Germany", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W16-2342", + doi = "10.18653/v1/W16-2342", + pages = "505--510", +} +""" + +_DESCRIPTION = """\ +CharacTer is a character-level metric inspired by the commonly applied translation edit rate (TER). It is +defined as the minimum number of character edits required to adjust a hypothesis, until it completely matches the +reference, normalized by the length of the hypothesis sentence. CharacTer calculates the character level edit +distance while performing the shift edit on word level. Unlike the strict matching criterion in TER, a hypothesis +word is considered to match a reference word and could be shifted, if the edit distance between them is below a +threshold value. The Levenshtein distance between the reference and the shifted hypothesis sequence is computed on the +character level. In addition, the lengths of hypothesis sequences instead of reference sequences are used for +normalizing the edit distance, which effectively counters the issue that shorter translations normally achieve lower +TER.""" + +_KWARGS_DESCRIPTION = """ +Calculates how good the predictions are in terms of the CharacTER metric given some references. +Args: + predictions: a list of predictions to score. Each prediction should be a string with + tokens separated by spaces. + references: a list of references for each prediction. You can also pass multiple references for each prediction, + so a list and in that list a sublist for each prediction for its related references. When multiple references are + given, the lowest (best) score is returned for that prediction-references pair. + Each reference should be a string with tokens separated by spaces. + aggregate: one of "mean", "sum", "median" to indicate how the scores of individual sentences should be + aggregated + return_all_scores: a boolean, indicating whether in addition to the aggregated score, also all individual + scores should be returned +Returns: + cer_score: an aggregated score across all the items, based on 'aggregate' + cer_scores: (optionally, if 'return_all_scores' evaluates to True) a list of all scores, one per ref/hyp pair +Examples: + >>> character_mt = evaluate.load("character") + >>> preds = ["this week the saudis denied information published in the new york times"] + >>> refs = ["saudi arabia denied this week information published in the american new york times"] + >>> character_mt.compute(references=refs, predictions=preds) + {'cer_score': 0.36619718309859156} + >>> preds = ["this week the saudis denied information published in the new york times", + ... "this is in fact an estimate"] + >>> refs = ["saudi arabia denied this week information published in the american new york times", + ... "this is actually an estimate"] + >>> character_mt.compute(references=refs, predictions=preds, aggregate="sum", return_all_scores=True) + {'cer_score': 0.6254564423578508, 'cer_scores': [0.36619718309859156, 0.25925925925925924]} + >>> preds = ["this week the saudis denied information published in the new york times"] + >>> refs = [["saudi arabia denied this week information published in the american new york times", + ... "the saudis have denied new information published in the ny times"]] + >>> character_mt.compute(references=refs, predictions=preds, aggregate="median", return_all_scores=True) + {'cer_score': 0.36619718309859156, 'cer_scores': [0.36619718309859156]} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Character(evaluate.Metric): + """CharacTer is a character-level metric inspired by the commonly applied translation edit rate (TER).""" + + def _info(self): + return evaluate.MetricInfo( + module_type="metric", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + {"predictions": Value("string", id="prediction"), "references": Value("string", id="reference")} + ), + datasets.Features( + { + "predictions": Value("string", id="prediction"), + "references": Sequence(Value("string", id="reference"), id="references"), + } + ), + ], + homepage="https://github.com/bramvanroy/CharacTER", + codebase_urls=["https://github.com/bramvanroy/CharacTER", "https://github.com/rwth-i6/CharacTER"], + ) + + def _compute( + self, + predictions: Iterable[str], + references: Union[Iterable[str], Iterable[Iterable[str]]], + aggregate: str = "mean", + return_all_scores: bool = False, + ): + if aggregate not in ("mean", "sum", "median"): + raise ValueError("'aggregate' must be one of 'sum', 'mean', 'median'") + + predictions = [p.split() for p in predictions] + # Predictions and references have the same internal types (both lists of strings), + # so only one reference per prediction + if isinstance(references[0], str): + references = [r.split() for r in references] + + scores_d = cer.calculate_cer_corpus(predictions, references) + cer_scores: List[float] = scores_d["cer_scores"] + + if aggregate == "sum": + score = sum(cer_scores) + elif aggregate == "mean": + score = scores_d["mean"] + else: + score = scores_d["median"] + else: + # In the case of multiple references, we just find the "best score", + # i.e., the reference that the prediction is closest to, i.e. the lowest characTER score + references = [[r.split() for r in refs] for refs in references] + + cer_scores = [] + for pred, refs in zip(predictions, references): + min_score = math.inf + for ref in refs: + score = calculate_cer(pred, ref) + + if score < min_score: + min_score = score + + cer_scores.append(min_score) + + if aggregate == "sum": + score = sum(cer_scores) + elif aggregate == "mean": + score = mean(cer_scores) + else: + score = median(cer_scores) + + # Return scores + if return_all_scores: + return {"cer_score": score, "cer_scores": cer_scores} + else: + return {"cer_score": score} diff --git a/TextClassification/TextClassification/evaluate/metrics/character/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/character/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..35e72563906789ff02b983daa7febaf65fde63da --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/character/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +cer>=1.2.0 diff --git a/TextClassification/TextClassification/evaluate/metrics/charcut_mt/README.md b/TextClassification/TextClassification/evaluate/metrics/charcut_mt/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0c414a42fbffbec02bf8aa5862a0a6775bca2df6 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/charcut_mt/README.md @@ -0,0 +1,77 @@ +--- +title: CharCut +emoji: 🔤 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +- machine-translation +description: >- + CharCut is a character-based machine translation evaluation metric. +--- + +# Metric Card for CharacTER + +## Metric Description +CharCut compares outputs of MT systems with reference translations. The matching algorithm is based on an iterative +search for longest common substrings, combined with a length-based threshold that limits short and noisy character +matches. As a similarity metric this is not new, but to the best of our knowledge it was never applied to highlighting +and scoring of MT outputs. It has the neat effect of keeping character-based differences readable by humans. + +## Intended Uses +CharCut was developed for machine translation evaluation. + +## How to Use + +```python +import evaluate +charcut = evaluate.load("charcut") +preds = ["this week the saudis denied information published in the new york times", + "this is in fact an estimate"] +refs = ["saudi arabia denied this week information published in the american new york times", + "this is actually an estimate"] +results = charcut.compute(references=refs, predictions=preds) +print(results) +# {'charcut_mt': 0.1971153846153846} + +``` +### Inputs +- **predictions**: a single prediction or a list of predictions to score. Each prediction should be a string with + tokens separated by spaces. +- **references**: a single reference or a list of reference for each prediction. Each reference should be a string with + tokens separated by spaces. + + +### Output Values +- **charcut_mt**: the CharCut evaluation score (lower is better) + +### Output Example +```python +{'charcut_mt': 0.1971153846153846} +``` + +## Citation +```bibtex +@inproceedings{lardilleux-lepage-2017-charcut, + title = "{CHARCUT}: Human-Targeted Character-Based {MT} Evaluation with Loose Differences", + author = "Lardilleux, Adrien and + Lepage, Yves", + booktitle = "Proceedings of the 14th International Conference on Spoken Language Translation", + month = dec # " 14-15", + year = "2017", + address = "Tokyo, Japan", + publisher = "International Workshop on Spoken Language Translation", + url = "https://aclanthology.org/2017.iwslt-1.20", + pages = "146--153", + abstract = "We present CHARCUT, a character-based machine translation evaluation metric derived from a human-targeted segment difference visualisation algorithm. It combines an iterative search for longest common substrings between the candidate and the reference translation with a simple length-based threshold, enabling loose differences that limit noisy character matches. Its main advantage is to produce scores that directly reflect human-readable string differences, making it a useful support tool for the manual analysis of MT output and its display to end users. Experiments on WMT16 metrics task data show that it is on par with the best {``}un-trained{''} metrics in terms of correlation with human judgement, well above BLEU and TER baselines, on both system and segment tasks.", +} +``` + +## Further References +- Repackaged version that is used in this HF implementation: [https://github.com/BramVanroy/CharCut](https://github.com/BramVanroy/CharCut) +- Original version: [https://github.com/alardill/CharCut](https://github.com/alardill/CharCut) diff --git a/TextClassification/TextClassification/evaluate/metrics/charcut_mt/app.py b/TextClassification/TextClassification/evaluate/metrics/charcut_mt/app.py new file mode 100644 index 0000000000000000000000000000000000000000..6dc6fe5b276009d16a04f0cbce4f9d42cf31207d --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/charcut_mt/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("charcut_mt") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/charcut_mt/charcut_mt.py b/TextClassification/TextClassification/evaluate/metrics/charcut_mt/charcut_mt.py new file mode 100644 index 0000000000000000000000000000000000000000..1e33db2f6a92b686f685e3ed80c9edc2d27d5b2a --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/charcut_mt/charcut_mt.py @@ -0,0 +1,89 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""An implementation for calculating CharCut, a character-based machine translation evaluation metric.""" +from typing import Iterable, Union + +import datasets +from charcut import calculate_charcut +from datasets import Sequence, Value + +import evaluate + + +_CITATION = """\ +@inproceedings{lardilleux-lepage-2017-charcut, + title = "{CHARCUT}: Human-Targeted Character-Based {MT} Evaluation with Loose Differences", + author = "Lardilleux, Adrien and + Lepage, Yves", + booktitle = "Proceedings of the 14th International Conference on Spoken Language Translation", + month = dec # " 14-15", + year = "2017", + address = "Tokyo, Japan", + publisher = "International Workshop on Spoken Language Translation", + url = "https://aclanthology.org/2017.iwslt-1.20", + pages = "146--153" +} +""" + +_DESCRIPTION = """\ +CharCut compares outputs of MT systems with reference translations. The matching algorithm is based on an iterative +search for longest common substrings, combined with a length-based threshold that limits short and noisy character +matches. As a similarity metric this is not new, but to the best of our knowledge it was never applied to highlighting +and scoring of MT outputs. It has the neat effect of keeping character-based differences readable by humans.""" + +_KWARGS_DESCRIPTION = """ +Calculates how good predictions are given some references. +Args: + predictions: a list of predictions to score. Each prediction should be a string with + tokens separated by spaces. + references: a list of reference for each prediction. Each reference should be a string with + tokens separated by spaces. +Returns: + charcut_mt: the CharCut score +Examples: + >>> charcut_mt = evaluate.load("charcut_mt") + >>> preds = ["this week the saudis denied information published in the new york times", + ... "this is in fact an estimate"] + >>> refs = ["saudi arabia denied this week information published in the american new york times", + ... "this is actually an estimate"] + >>> charcut_mt.compute(references=refs, predictions=preds) + {'charcut_mt': 0.1971153846153846} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Charcut(evaluate.Metric): + """Character-based MT evaluation.""" + + def _info(self): + return evaluate.MetricInfo( + # This is the description that will appear on the modules page. + module_type="metric", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + # This defines the format of each prediction and reference + features=[ + datasets.Features( + {"predictions": Value("string", id="prediction"), "references": Value("string", id="reference")} + ), + ], + # Homepage of the module for documentation + homepage="https://github.com/BramVanroy/CharCut", + # Additional links to the codebase or references + codebase_urls=["https://github.com/BramVanroy/CharCut", "https://github.com/alardill/CharCut"], + ) + + def _compute(self, predictions: Iterable[str], references: Iterable[str]): + return {"charcut_mt": calculate_charcut(predictions, references)[0]} diff --git a/TextClassification/TextClassification/evaluate/metrics/charcut_mt/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/charcut_mt/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..f26665222945b1d3c98e8b8741e42ed0b1c33a08 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/charcut_mt/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +charcut>=1.1.1 diff --git a/TextClassification/TextClassification/evaluate/metrics/chrf/README.md b/TextClassification/TextClassification/evaluate/metrics/chrf/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9e495a63b83f105776e123c105b2272ce442cff6 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/chrf/README.md @@ -0,0 +1,157 @@ +--- +title: chrF +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + ChrF and ChrF++ are two MT evaluation metrics. They both use the F-score statistic for character n-gram matches, + and ChrF++ adds word n-grams as well which correlates more strongly with direct assessment. We use the implementation + that is already present in sacrebleu. + + The implementation here is slightly different from sacrebleu in terms of the required input format. The length of + the references and hypotheses lists need to be the same, so you may need to transpose your references compared to + sacrebleu's required input format. See https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534 + + See the README.md file at https://github.com/mjpost/sacreBLEU#chrf--chrf for more information. +--- + +# Metric Card for chrF(++) + + +## Metric Description +ChrF and ChrF++ are two MT evaluation metrics that use the F-score statistic for character n-gram matches. ChrF++ additionally includes word n-grams, which correlate more strongly with direct assessment. We use the implementation that is already present in sacrebleu. + +While this metric is included in sacreBLEU, the implementation here is slightly different from sacreBLEU in terms of the required input format. Here, the length of the references and hypotheses lists need to be the same, so you may need to transpose your references compared to sacrebleu's required input format. See https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534 + +See the [sacreBLEU README.md](https://github.com/mjpost/sacreBLEU#chrf--chrf) for more information. + + +## How to Use +At minimum, this metric requires a `list` of predictions and a `list` of `list`s of references: +```python +>>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] +>>> reference = [["The relationship between dogs and cats is not exactly friendly.", ], ["A good bookshop is just a genteel Black Hole that knows how to read."]] +>>> chrf = evaluate.load("chrf") +>>> results = chrf.compute(predictions=prediction, references=reference) +>>> print(results) +{'score': 84.64214891738334, 'char_order': 6, 'word_order': 0, 'beta': 2} +``` + +### Inputs +- **`predictions`** (`list` of `str`): The predicted sentences. +- **`references`** (`list` of `list` of `str`): The references. There should be one reference sub-list for each prediction sentence. +- **`char_order`** (`int`): Character n-gram order. Defaults to `6`. +- **`word_order`** (`int`): Word n-gram order. If equals to 2, the metric is referred to as chrF++. Defaults to `0`. +- **`beta`** (`int`): Determine the importance of recall w.r.t precision. Defaults to `2`. +- **`lowercase`** (`bool`): If `True`, enables case-insensitivity. Defaults to `False`. +- **`whitespace`** (`bool`): If `True`, include whitespaces when extracting character n-grams. Defaults to `False`. +- **`eps_smoothing`** (`bool`): If `True`, applies epsilon smoothing similar to reference chrF++.py, NLTK, and Moses implementations. If `False`, takes into account effective match order similar to sacreBLEU < 2.0.0. Defaults to `False`. + + + +### Output Values +The output is a dictionary containing the following fields: +- **`'score'`** (`float`): The chrF (chrF++) score. +- **`'char_order'`** (`int`): The character n-gram order. +- **`'word_order'`** (`int`): The word n-gram order. If equals to `2`, the metric is referred to as chrF++. +- **`'beta'`** (`int`): Determine the importance of recall w.r.t precision. + + +The output is formatted as below: +```python +{'score': 61.576379378113785, 'char_order': 6, 'word_order': 0, 'beta': 2} +``` + +The chrF(++) score can be any value between `0.0` and `100.0`, inclusive. + +#### Values from Popular Papers + + +### Examples +A simple example of calculating chrF: +```python +>>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] +>>> reference = [["The relationship between dogs and cats is not exactly friendly.", ], ["A good bookshop is just a genteel Black Hole that knows how to read."]] +>>> chrf = evaluate.load("chrf") +>>> results = chrf.compute(predictions=prediction, references=reference) +>>> print(results) +{'score': 84.64214891738334, 'char_order': 6, 'word_order': 0, 'beta': 2} +``` + +The same example, but with the argument `word_order=2`, to calculate chrF++ instead of chrF: +```python +>>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] +>>> reference = [["The relationship between dogs and cats is not exactly friendly.", ], ["A good bookshop is just a genteel Black Hole that knows how to read."]] +>>> chrf = evaluate.load("chrf") +>>> results = chrf.compute(predictions=prediction, +... references=reference, +... word_order=2) +>>> print(results) +{'score': 82.87263732906315, 'char_order': 6, 'word_order': 2, 'beta': 2} +``` + +The same chrF++ example as above, but with `lowercase=True` to normalize all case: +```python +>>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] +>>> reference = [["The relationship between dogs and cats is not exactly friendly.", ], ["A good bookshop is just a genteel Black Hole that knows how to read."]] +>>> chrf = evaluate.load("chrf") +>>> results = chrf.compute(predictions=prediction, +... references=reference, +... word_order=2, +... lowercase=True) +>>> print(results) +{'score': 92.12853119829202, 'char_order': 6, 'word_order': 2, 'beta': 2} +``` + + +## Limitations and Bias +- According to [Popović 2017](https://www.statmt.org/wmt17/pdf/WMT70.pdf), chrF+ (where `word_order=1`) and chrF++ (where `word_order=2`) produce scores that correlate better with human judgements than chrF (where `word_order=0`) does. + +## Citation +```bibtex +@inproceedings{popovic-2015-chrf, + title = "chr{F}: character n-gram {F}-score for automatic {MT} evaluation", + author = "Popovi{\'c}, Maja", + booktitle = "Proceedings of the Tenth Workshop on Statistical Machine Translation", + month = sep, + year = "2015", + address = "Lisbon, Portugal", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W15-3049", + doi = "10.18653/v1/W15-3049", + pages = "392--395", +} +@inproceedings{popovic-2017-chrf, + title = "chr{F}++: words helping character n-grams", + author = "Popovi{\'c}, Maja", + booktitle = "Proceedings of the Second Conference on Machine Translation", + month = sep, + year = "2017", + address = "Copenhagen, Denmark", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W17-4770", + doi = "10.18653/v1/W17-4770", + pages = "612--618", +} +@inproceedings{post-2018-call, + title = "A Call for Clarity in Reporting {BLEU} Scores", + author = "Post, Matt", + booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", + month = oct, + year = "2018", + address = "Belgium, Brussels", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/W18-6319", + pages = "186--191", +} +``` + +## Further References +- See the [sacreBLEU README.md](https://github.com/mjpost/sacreBLEU#chrf--chrf) for more information on this implementation. \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/chrf/app.py b/TextClassification/TextClassification/evaluate/metrics/chrf/app.py new file mode 100644 index 0000000000000000000000000000000000000000..09903896d9408c5eba8fdecaf8c288e3fc17e72d --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/chrf/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("chrf") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/chrf/chrf.py b/TextClassification/TextClassification/evaluate/metrics/chrf/chrf.py new file mode 100644 index 0000000000000000000000000000000000000000..30bf8a0cec2251972e872fe09d15b6a7f2858f2f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/chrf/chrf.py @@ -0,0 +1,188 @@ +# Copyright 2021 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Chrf(++) metric as available in sacrebleu. """ +import datasets +import sacrebleu as scb +from packaging import version +from sacrebleu import CHRF + +import evaluate + + +_CITATION = """\ +@inproceedings{popovic-2015-chrf, + title = "chr{F}: character n-gram {F}-score for automatic {MT} evaluation", + author = "Popovi{\'c}, Maja", + booktitle = "Proceedings of the Tenth Workshop on Statistical Machine Translation", + month = sep, + year = "2015", + address = "Lisbon, Portugal", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W15-3049", + doi = "10.18653/v1/W15-3049", + pages = "392--395", +} +@inproceedings{popovic-2017-chrf, + title = "chr{F}++: words helping character n-grams", + author = "Popovi{\'c}, Maja", + booktitle = "Proceedings of the Second Conference on Machine Translation", + month = sep, + year = "2017", + address = "Copenhagen, Denmark", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/W17-4770", + doi = "10.18653/v1/W17-4770", + pages = "612--618", +} +@inproceedings{post-2018-call, + title = "A Call for Clarity in Reporting {BLEU} Scores", + author = "Post, Matt", + booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", + month = oct, + year = "2018", + address = "Belgium, Brussels", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/W18-6319", + pages = "186--191", +} +""" + +_DESCRIPTION = """\ +ChrF and ChrF++ are two MT evaluation metrics. They both use the F-score statistic for character n-gram matches, +and ChrF++ adds word n-grams as well which correlates more strongly with direct assessment. We use the implementation +that is already present in sacrebleu. + +The implementation here is slightly different from sacrebleu in terms of the required input format. The length of +the references and hypotheses lists need to be the same, so you may need to transpose your references compared to +sacrebleu's required input format. See https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534 + +See the README.md file at https://github.com/mjpost/sacreBLEU#chrf--chrf for more information. +""" + +_KWARGS_DESCRIPTION = """ +Produces ChrF(++) scores for hypotheses given reference translations. + +Args: + predictions (list of str): The predicted sentences. + references (list of list of str): The references. There should be one reference sub-list for each prediction sentence. + char_order (int): Character n-gram order. Defaults to `6`. + word_order (int): Word n-gram order. If equals to `2`, the metric is referred to as chrF++. Defaults to `0`. + beta (int): Determine the importance of recall w.r.t precision. Defaults to `2`. + lowercase (bool): if `True`, enables case-insensitivity. Defaults to `False`. + whitespace (bool): If `True`, include whitespaces when extracting character n-grams. + eps_smoothing (bool): If `True`, applies epsilon smoothing similar + to reference chrF++.py, NLTK and Moses implementations. If `False`, + it takes into account effective match order similar to sacreBLEU < 2.0.0. Defaults to `False`. + +Returns: + 'score' (float): The chrF (chrF++) score, + 'char_order' (int): The character n-gram order, + 'word_order' (int): The word n-gram order. If equals to 2, the metric is referred to as chrF++, + 'beta' (int): Determine the importance of recall w.r.t precision + +Examples: + Example 1--a simple example of calculating chrF: + >>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] + >>> reference = [["The relationship between dogs and cats is not exactly friendly."], ["A good bookshop is just a genteel Black Hole that knows how to read."]] + >>> chrf = evaluate.load("chrf") + >>> results = chrf.compute(predictions=prediction, references=reference) + >>> print(results) + {'score': 84.64214891738334, 'char_order': 6, 'word_order': 0, 'beta': 2} + + Example 2--the same example, but with the argument word_order=2, to calculate chrF++ instead of chrF: + >>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] + >>> reference = [["The relationship between dogs and cats is not exactly friendly."], ["A good bookshop is just a genteel Black Hole that knows how to read."]] + >>> chrf = evaluate.load("chrf") + >>> results = chrf.compute(predictions=prediction, + ... references=reference, + ... word_order=2) + >>> print(results) + {'score': 82.87263732906315, 'char_order': 6, 'word_order': 2, 'beta': 2} + + Example 3--the same chrF++ example as above, but with `lowercase=True` to normalize all case: + >>> prediction = ["The relationship between cats and dogs is not exactly friendly.", "a good bookshop is just a genteel black hole that knows how to read."] + >>> reference = [["The relationship between dogs and cats is not exactly friendly."], ["A good bookshop is just a genteel Black Hole that knows how to read."]] + >>> chrf = evaluate.load("chrf") + >>> results = chrf.compute(predictions=prediction, + ... references=reference, + ... word_order=2, + ... lowercase=True) + >>> print(results) + {'score': 92.12853119829202, 'char_order': 6, 'word_order': 2, 'beta': 2} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class ChrF(evaluate.Metric): + def _info(self): + if version.parse(scb.__version__) < version.parse("1.4.12"): + raise ImportWarning( + "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n" + 'You can install it with `pip install "sacrebleu>=1.4.12"`.' + ) + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + homepage="https://github.com/mjpost/sacreBLEU#chrf--chrf", + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), + } + ), + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + ], + codebase_urls=["https://github.com/mjpost/sacreBLEU#chrf--chrf"], + reference_urls=[ + "https://github.com/m-popovic/chrF", + ], + ) + + def _compute( + self, + predictions, + references, + char_order: int = CHRF.CHAR_ORDER, + word_order: int = CHRF.WORD_ORDER, + beta: int = CHRF.BETA, + lowercase: bool = False, + whitespace: bool = False, + eps_smoothing: bool = False, + ): + # if only one reference is provided make sure we still use list of lists + if isinstance(references[0], str): + references = [[ref] for ref in references] + references_per_prediction = len(references[0]) + if any(len(refs) != references_per_prediction for refs in references): + raise ValueError( + "ChrF, as implemented by sacrebleu, requires the same number of references for each prediction" + ) + transformed_references = [[refs[i] for refs in references] for i in range(references_per_prediction)] + + sb_chrf = CHRF(char_order, word_order, beta, lowercase, whitespace, eps_smoothing) + output = sb_chrf.corpus_score(predictions, transformed_references) + + return { + "score": output.score, + "char_order": output.char_order, + "word_order": output.word_order, + "beta": output.beta, + } diff --git a/TextClassification/TextClassification/evaluate/metrics/chrf/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/chrf/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..9d3e3c80b77038f530d173d82e6419d76b5e72f0 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/chrf/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +sacrebleu \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/code_eval/README.md b/TextClassification/TextClassification/evaluate/metrics/code_eval/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ce81b7c642de6c468ce7647af922148372f42f68 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/code_eval/README.md @@ -0,0 +1,146 @@ +--- +title: Code Eval +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + This metric implements the evaluation harness for the HumanEval problem solving dataset + described in the paper "Evaluating Large Language Models Trained on Code" + (https://arxiv.org/abs/2107.03374). +--- + +# Metric Card for Code Eval + +## Metric description + +The CodeEval metric estimates the pass@k metric for code synthesis. + +It implements the evaluation harness for the HumanEval problem solving dataset described in the paper ["Evaluating Large Language Models Trained on Code"](https://arxiv.org/abs/2107.03374). + + +## How to use + +The Code Eval metric calculates how good are predictions given a set of references. Its arguments are: + +`predictions`: a list of candidates to evaluate. Each candidate should be a list of strings with several code candidates to solve the problem. + +`references`: a list with a test for each prediction. Each test should evaluate the correctness of a code candidate. + +`k`: number of code candidates to consider in the evaluation. The default value is `[1, 10, 100]`. + +`num_workers`: the number of workers used to evaluate the candidate programs (The default value is `4`). + +`timeout`: The maximum time taken to produce a prediction before it is considered a "timeout". The default value is `3.0` (i.e. 3 seconds). + +```python +from evaluate import load +code_eval = load("code_eval") +test_cases = ["assert add(2,3)==5"] +candidates = [["def add(a,b): return a*b", "def add(a, b): return a+b"]] +pass_at_k, results = code_eval.compute(references=test_cases, predictions=candidates, k=[1, 2]) +``` + +N.B. +This metric exists to run untrusted model-generated code. Users are strongly encouraged not to do so outside of a robust security sandbox. Before running this metric and once you've taken the necessary precautions, you will need to set the `HF_ALLOW_CODE_EVAL` environment variable. Use it at your own risk: +```python +import os +os.environ["HF_ALLOW_CODE_EVAL"] = "1"` +``` + +## Output values + +The Code Eval metric outputs two things: + +`pass_at_k`: a dictionary with the pass rates for each k value defined in the arguments. + +`results`: a dictionary with granular results of each unit test. + +### Values from popular papers +The [original CODEX paper](https://arxiv.org/pdf/2107.03374.pdf) reported that the CODEX-12B model had a pass@k score of 28.8% at `k=1`, 46.8% at `k=10` and 72.3% at `k=100`. However, since the CODEX model is not open source, it is hard to verify these numbers. + + + +## Examples + +Full match at `k=1`: + +```python +from evaluate import load +code_eval = load("code_eval") +test_cases = ["assert add(2,3)==5"] +candidates = [["def add(a, b): return a+b"]] +pass_at_k, results = code_eval.compute(references=test_cases, predictions=candidates, k=[1]) +print(pass_at_k) +{'pass@1': 1.0} +``` + +No match for k = 1: + +```python +from evaluate import load +code_eval = load("code_eval") +test_cases = ["assert add(2,3)==5"] +candidates = [["def add(a,b): return a*b"]] +pass_at_k, results = code_eval.compute(references=test_cases, predictions=candidates, k=[1]) +print(pass_at_k) +{'pass@1': 0.0} +``` + +Partial match at k=1, full match at k=2: + +```python +from evaluate import load +code_eval = load("code_eval") +test_cases = ["assert add(2,3)==5"] +candidates = [["def add(a, b): return a+b", "def add(a,b): return a*b"]] +pass_at_k, results = code_eval.compute(references=test_cases, predictions=candidates, k=[1, 2]) +print(pass_at_k) +{'pass@1': 0.5, 'pass@2': 1.0} +``` + +## Limitations and bias + +As per the warning included in the metric code itself: +> This program exists to execute untrusted model-generated code. Although it is highly unlikely that model-generated code will do something overtly malicious in response to this test suite, model-generated code may act destructively due to a lack of model capability or alignment. Users are strongly encouraged to sandbox this evaluation suite so that it does not perform destructive actions on their host or network. For more information on how OpenAI sandboxes its code, see the accompanying paper. Once you have read this disclaimer and taken appropriate precautions, uncomment the following line and proceed at your own risk: + +More information about the limitations of the code can be found on the [Human Eval Github repository](https://github.com/openai/human-eval). + +## Citation + +```bibtex +@misc{chen2021evaluating, + title={Evaluating Large Language Models Trained on Code}, + author={Mark Chen and Jerry Tworek and Heewoo Jun and Qiming Yuan \ +and Henrique Ponde de Oliveira Pinto and Jared Kaplan and Harri Edwards \ +and Yuri Burda and Nicholas Joseph and Greg Brockman and Alex Ray \ +and Raul Puri and Gretchen Krueger and Michael Petrov and Heidy Khlaaf \ +and Girish Sastry and Pamela Mishkin and Brooke Chan and Scott Gray \ +and Nick Ryder and Mikhail Pavlov and Alethea Power and Lukasz Kaiser \ +and Mohammad Bavarian and Clemens Winter and Philippe Tillet \ +and Felipe Petroski Such and Dave Cummings and Matthias Plappert \ +and Fotios Chantzis and Elizabeth Barnes and Ariel Herbert-Voss \ +and William Hebgen Guss and Alex Nichol and Alex Paino and Nikolas Tezak \ +and Jie Tang and Igor Babuschkin and Suchir Balaji and Shantanu Jain \ +and William Saunders and Christopher Hesse and Andrew N. Carr \ +and Jan Leike and Josh Achiam and Vedant Misra and Evan Morikawa \ +and Alec Radford and Matthew Knight and Miles Brundage and Mira Murati \ +and Katie Mayer and Peter Welinder and Bob McGrew and Dario Amodei \ +and Sam McCandlish and Ilya Sutskever and Wojciech Zaremba}, + year={2021}, + eprint={2107.03374}, + archivePrefix={arXiv}, + primaryClass={cs.LG} +} +``` + +## Further References + +- [Human Eval Github repository](https://github.com/openai/human-eval) +- [OpenAI Codex website](https://openai.com/blog/openai-codex/) diff --git a/TextClassification/TextClassification/evaluate/metrics/code_eval/app.py b/TextClassification/TextClassification/evaluate/metrics/code_eval/app.py new file mode 100644 index 0000000000000000000000000000000000000000..dcc856a09640a31bc4653300a22982c9d2688989 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/code_eval/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("code_eval") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/code_eval/code_eval.py b/TextClassification/TextClassification/evaluate/metrics/code_eval/code_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..0885712e698a34067e8faabe6b029ea8d719e024 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/code_eval/code_eval.py @@ -0,0 +1,213 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The CodeEval metric estimates the pass@k metric for code synthesis. +This is an evaluation harness for the HumanEval problem solving dataset +described in the paper "Evaluating Large Language Models Trained on Code" +(https://arxiv.org/abs/2107.03374).""" + +import itertools +import os +from collections import Counter, defaultdict +from concurrent.futures import ThreadPoolExecutor, as_completed + +import datasets +import numpy as np + +import evaluate + +from .execute import check_correctness + + +_CITATION = """\ +@misc{chen2021evaluating, + title={Evaluating Large Language Models Trained on Code}, + author={Mark Chen and Jerry Tworek and Heewoo Jun and Qiming Yuan \ +and Henrique Ponde de Oliveira Pinto and Jared Kaplan and Harri Edwards \ +and Yuri Burda and Nicholas Joseph and Greg Brockman and Alex Ray \ +and Raul Puri and Gretchen Krueger and Michael Petrov and Heidy Khlaaf \ +and Girish Sastry and Pamela Mishkin and Brooke Chan and Scott Gray \ +and Nick Ryder and Mikhail Pavlov and Alethea Power and Lukasz Kaiser \ +and Mohammad Bavarian and Clemens Winter and Philippe Tillet \ +and Felipe Petroski Such and Dave Cummings and Matthias Plappert \ +and Fotios Chantzis and Elizabeth Barnes and Ariel Herbert-Voss \ +and William Hebgen Guss and Alex Nichol and Alex Paino and Nikolas Tezak \ +and Jie Tang and Igor Babuschkin and Suchir Balaji and Shantanu Jain \ +and William Saunders and Christopher Hesse and Andrew N. Carr \ +and Jan Leike and Josh Achiam and Vedant Misra and Evan Morikawa \ +and Alec Radford and Matthew Knight and Miles Brundage and Mira Murati \ +and Katie Mayer and Peter Welinder and Bob McGrew and Dario Amodei \ +and Sam McCandlish and Ilya Sutskever and Wojciech Zaremba}, + year={2021}, + eprint={2107.03374}, + archivePrefix={arXiv}, + primaryClass={cs.LG} +} +""" + +_DESCRIPTION = """\ +This metric implements the evaluation harness for the HumanEval problem solving dataset +described in the paper "Evaluating Large Language Models Trained on Code" +(https://arxiv.org/abs/2107.03374). +""" + + +_KWARGS_DESCRIPTION = """ +Calculates how good are predictions given some references, using certain scores +Args: + predictions: list of candidates to evaluate. Each candidates should be a list + of strings with several code candidates to solve the problem. + references: a list with a test for each prediction. Each test should evaluate the + correctness of a code candidate. + k: number of code candidates to consider in the evaluation (Default: [1, 10, 100]) + num_workers: number of workers used to evaluate the canidate programs (Default: 4). + timeout: +Returns: + pass_at_k: dict with pass rates for each k + results: dict with granular results of each unittest +Examples: + >>> code_eval = evaluate.load("code_eval") + >>> test_cases = ["assert add(2,3)==5"] + >>> candidates = [["def add(a,b): return a*b", "def add(a, b): return a+b"]] + >>> pass_at_k, results = code_eval.compute(references=test_cases, predictions=candidates, k=[1, 2]) + >>> print(pass_at_k) + {'pass@1': 0.5, 'pass@2': 1.0} +""" + + +_WARNING = """ +################################################################################ + !!!WARNING!!! +################################################################################ +The "code_eval" metric executes untrusted model-generated code in Python. +Although it is highly unlikely that model-generated code will do something +overtly malicious in response to this test suite, model-generated code may act +destructively due to a lack of model capability or alignment. +Users are strongly encouraged to sandbox this evaluation suite so that it +does not perform destructive actions on their host or network. For more +information on how OpenAI sandboxes its code, see the paper "Evaluating Large +Language Models Trained on Code" (https://arxiv.org/abs/2107.03374). + +Once you have read this disclaimer and taken appropriate precautions, +set the environment variable HF_ALLOW_CODE_EVAL="1". Within Python you can to this +with: + +>>> import os +>>> os.environ["HF_ALLOW_CODE_EVAL"] = "1" + +################################################################################\ +""" + +_LICENSE = """The MIT License + +Copyright (c) OpenAI (https://openai.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE.""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class CodeEval(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + # This is the description that will appear on the metrics page. + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + # This defines the format of each prediction and reference + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("string")), + "references": datasets.Value("string"), + } + ), + homepage="https://github.com/openai/human-eval", + codebase_urls=["https://github.com/openai/human-eval"], + reference_urls=["https://github.com/openai/human-eval"], + license=_LICENSE, + ) + + def _compute(self, predictions, references, k=[1, 10, 100], num_workers=4, timeout=3.0): + """Returns the scores""" + + if os.getenv("HF_ALLOW_CODE_EVAL", 0) != "1": + raise ValueError(_WARNING) + + if os.name == "nt": + raise NotImplementedError("This metric is currently not supported on Windows.") + + with ThreadPoolExecutor(max_workers=num_workers) as executor: + futures = [] + completion_id = Counter() + n_samples = 0 + results = defaultdict(list) + + for task_id, (candidates, test_case) in enumerate(zip(predictions, references)): + for candidate in candidates: + test_program = candidate + "\n" + test_case + args = (test_program, timeout, task_id, completion_id[task_id]) + future = executor.submit(check_correctness, *args) + futures.append(future) + completion_id[task_id] += 1 + n_samples += 1 + + for future in as_completed(futures): + result = future.result() + results[result["task_id"]].append((result["completion_id"], result)) + + total, correct = [], [] + for result in results.values(): + result.sort() + passed = [r[1]["passed"] for r in result] + total.append(len(passed)) + correct.append(sum(passed)) + total = np.array(total) + correct = np.array(correct) + + ks = k + pass_at_k = {f"pass@{k}": estimate_pass_at_k(total, correct, k).mean() for k in ks if (total >= k).all()} + + return pass_at_k, results + + +def estimate_pass_at_k(num_samples, num_correct, k): + """Estimates pass@k of each problem and returns them in an array.""" + + def estimator(n: int, c: int, k: int) -> float: + """Calculates 1 - comb(n - c, k) / comb(n, k).""" + if n - c < k: + return 1.0 + return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1)) + + if isinstance(num_samples, int): + num_samples_it = itertools.repeat(num_samples, len(num_correct)) + else: + assert len(num_samples) == len(num_correct) + num_samples_it = iter(num_samples) + + return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)]) diff --git a/TextClassification/TextClassification/evaluate/metrics/code_eval/execute.py b/TextClassification/TextClassification/evaluate/metrics/code_eval/execute.py new file mode 100644 index 0000000000000000000000000000000000000000..53517a805cf84758b858612b62794b874590159b --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/code_eval/execute.py @@ -0,0 +1,236 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This code is adapted from OpenAI's release +# https://github.com/openai/human-eval/blob/master/human_eval/execution.py + +import contextlib +import faulthandler +import io +import multiprocessing +import os +import platform +import signal +import tempfile + + +def check_correctness(check_program, timeout, task_id, completion_id): + """ + Evaluates the functional correctness of a completion by running the test + suite provided in the problem. + + :param completion_id: an optional completion ID so we can match + the results later even if execution finishes asynchronously. + """ + manager = multiprocessing.Manager() + result = manager.list() + + p = multiprocessing.Process(target=unsafe_execute, args=(check_program, result, timeout)) + p.start() + p.join(timeout=timeout + 1) + if p.is_alive(): + p.kill() + + if not result: + result.append("timed out") + + return dict( + task_id=task_id, + passed=result[0] == "passed", + result=result[0], + completion_id=completion_id, + ) + + +def unsafe_execute(check_program, result, timeout): + + with create_tempdir(): + + # These system calls are needed when cleaning up tempdir. + import os + import shutil + + rmtree = shutil.rmtree + rmdir = os.rmdir + chdir = os.chdir + + # Disable functionalities that can make destructive changes to the test. + reliability_guard() + + # Run program. + try: + exec_globals = {} + with swallow_io(): + with time_limit(timeout): + exec(check_program, exec_globals) + result.append("passed") + except TimeoutException: + result.append("timed out") + except BaseException as e: + result.append(f"failed: {e}") + + # Needed for cleaning up. + shutil.rmtree = rmtree + os.rmdir = rmdir + os.chdir = chdir + + +@contextlib.contextmanager +def time_limit(seconds): + def signal_handler(signum, frame): + raise TimeoutException("Timed out!") + + signal.setitimer(signal.ITIMER_REAL, seconds) + signal.signal(signal.SIGALRM, signal_handler) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + + +@contextlib.contextmanager +def swallow_io(): + stream = WriteOnlyStringIO() + with contextlib.redirect_stdout(stream): + with contextlib.redirect_stderr(stream): + with redirect_stdin(stream): + yield + + +@contextlib.contextmanager +def create_tempdir(): + with tempfile.TemporaryDirectory() as dirname: + with chdir(dirname): + yield dirname + + +class TimeoutException(Exception): + pass + + +class WriteOnlyStringIO(io.StringIO): + """StringIO that throws an exception when it's read from""" + + def read(self, *args, **kwargs): + raise OSError + + def readline(self, *args, **kwargs): + raise OSError + + def readlines(self, *args, **kwargs): + raise OSError + + def readable(self, *args, **kwargs): + """Returns True if the IO object can be read.""" + return False + + +class redirect_stdin(contextlib._RedirectStream): # type: ignore + _stream = "stdin" + + +@contextlib.contextmanager +def chdir(root): + if root == ".": + yield + return + cwd = os.getcwd() + os.chdir(root) + try: + yield + except BaseException as exc: + raise exc + finally: + os.chdir(cwd) + + +def reliability_guard(maximum_memory_bytes=None): + """ + This disables various destructive functions and prevents the generated code + from interfering with the test (e.g. fork bomb, killing other processes, + removing filesystem files, etc.) + + WARNING + This function is NOT a security sandbox. Untrusted code, including, model- + generated code, should not be blindly executed outside of one. See the + Codex paper for more information about OpenAI's code sandbox, and proceed + with caution. + """ + + if maximum_memory_bytes is not None: + import resource + + resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes)) + resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes)) + if not platform.uname().system == "Darwin": + resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes)) + + faulthandler.disable() + + import builtins + + builtins.exit = None + builtins.quit = None + + import os + + os.environ["OMP_NUM_THREADS"] = "1" + + os.kill = None + os.system = None + os.putenv = None + os.remove = None + os.removedirs = None + os.rmdir = None + os.fchdir = None + os.setuid = None + os.fork = None + os.forkpty = None + os.killpg = None + os.rename = None + os.renames = None + os.truncate = None + os.replace = None + os.unlink = None + os.fchmod = None + os.fchown = None + os.chmod = None + os.chown = None + os.chroot = None + os.fchdir = None + os.lchflags = None + os.lchmod = None + os.lchown = None + os.getcwd = None + os.chdir = None + + import shutil + + shutil.rmtree = None + shutil.move = None + shutil.chown = None + + import subprocess + + subprocess.Popen = None # type: ignore + + __builtins__["help"] = None + + import sys + + sys.modules["ipdb"] = None + sys.modules["joblib"] = None + sys.modules["resource"] = None + sys.modules["psutil"] = None + sys.modules["tkinter"] = None diff --git a/TextClassification/TextClassification/evaluate/metrics/code_eval/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/code_eval/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a135afacbdd142a40cb9bcfe6073b76cc62d995 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/code_eval/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/comet/README.md b/TextClassification/TextClassification/evaluate/metrics/comet/README.md new file mode 100644 index 0000000000000000000000000000000000000000..999f3fc44eb3b3e90408346fa744865dd9e9cb7c --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/comet/README.md @@ -0,0 +1,204 @@ +--- +title: COMET +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Crosslingual Optimized Metric for Evaluation of Translation (COMET) is an open-source framework used to train Machine Translation metrics that achieve high levels of correlation with different types of human judgments (HTER, DA's or MQM). + With the release of the framework the authors also released fully trained models that were used to compete in the WMT20 Metrics Shared Task achieving SOTA in that years competition. + + See the [README.md] file at https://unbabel.github.io/COMET/html/models.html for more information. +--- + +# Metric Card for COMET + +## Metric description + +Crosslingual Optimized Metric for Evaluation of Translation (COMET) is an open-source framework used to train Machine Translation metrics that achieve high levels of correlation with different types of human judgments. + +## How to use + +COMET takes 3 lists of strings as input: `sources` (a list of source sentences), `predictions` (a list of candidate translations) and `references` (a list of reference translations). + +```python +from evaluate import load +comet_metric = load('comet') +source = ["Dem Feuer konnte Einhalt geboten werden", "Schulen und Kindergärten wurden eröffnet."] +hypothesis = ["The fire could be stopped", "Schools and kindergartens were open"] +reference = ["They were able to control the fire.", "Schools and kindergartens opened"] +comet_score = comet_metric.compute(predictions=hypothesis, references=reference, sources=source) +``` + +It has several configurations, named after the COMET model to be used. For versions below 2.0 it will default to `wmt20-comet-da` (previously known as `wmt-large-da-estimator-1719`) and for the latest versions (>= 2.0) it will default to `Unbabel/wmt22-comet-da`. + +Alternative models that can be chosen include `wmt20-comet-qe-da`, `wmt21-comet-mqm`, `wmt21-cometinho-da`, `wmt21-comet-qe-mqm` and `emnlp20-comet-rank`. Notably, a distilled model is also available, which is 80% smaller and 2.128x faster while performing close to non-distilled alternatives. You can use it with the identifier `eamt22-cometinho-da`. This version, called Cometinho, was elected as [the best paper](https://aclanthology.org/2022.eamt-1.9) at the annual European conference on Machine Translation. + +> NOTE: In `unbabel-comet>=2.0` all models were moved to Hugging Face Hub and you need to add the suffix `Unbabel/` to be able to download and use them. For example for the distilled version replace `eamt22-cometinho-da` with `Unbabel/eamt22-cometinho-da`. + +It also has several optional arguments: + +`gpus`: optional, an integer (number of GPUs to train on) or a list of integers (which GPUs to train on). Set to 0 to use CPU. The default value is `None` (uses one GPU if possible, else use CPU). + +`progress_bar`a boolean -- if set to `True`, progress updates will be printed out. The default value is `False`. + +More information about model characteristics can be found on the [COMET website](https://unbabel.github.io/COMET/html/index.html). + +## Output values + +The COMET metric outputs two lists: + +`scores`: a list of COMET scores for each of the input sentences, ranging from 0-1. + +`mean_score`: the mean value of COMET scores `scores` over all the input sentences, ranging from 0-1. + +### Values from popular papers + +The [original COMET paper](https://arxiv.org/pdf/2009.09025.pdf) reported average COMET scores ranging from 0.4 to 0.6, depending on the language pairs used for evaluating translation models. They also illustrate that COMET correlates well with human judgements compared to other metrics such as [BLEU](https://huggingface.co/metrics/bleu) and [CHRF](https://huggingface.co/metrics/chrf). + +## Examples + +Full match: + +```python +from evaluate import load +comet_metric = load('comet') +source = ["Dem Feuer konnte Einhalt geboten werden", "Schulen und Kindergärten wurden eröffnet."] +hypothesis = ["They were able to control the fire.", "Schools and kindergartens opened"] +reference = ["They were able to control the fire.", "Schools and kindergartens opened"] +results = comet_metric.compute(predictions=hypothesis, references=reference, sources=source) +print([round(v, 1) for v in results["scores"]]) +[1.0, 1.0] +``` + +Partial match: + +```python +from evaluate import load +comet_metric = load('comet') +source = ["Dem Feuer konnte Einhalt geboten werden", "Schulen und Kindergärten wurden eröffnet."] +hypothesis = ["The fire could be stopped", "Schools and kindergartens were open"] +reference = ["They were able to control the fire", "Schools and kindergartens opened"] +results = comet_metric.compute(predictions=hypothesis, references=reference, sources=source) +print([round(v, 2) for v in results["scores"]]) +[0.19, 0.92] +``` + +No match: + +```python +from evaluate import load +comet_metric = load('comet') +source = ["Dem Feuer konnte Einhalt geboten werden", "Schulen und Kindergärten wurden eröffnet."] +hypothesis = ["The girl went for a walk", "The boy was sleeping"] +reference = ["They were able to control the fire", "Schools and kindergartens opened"] +results = comet_metric.compute(predictions=hypothesis, references=reference, sources=source) +print([round(v, 2) for v in results["scores"]]) +[0.00, 0.00] +``` + +## Limitations and bias + +The models provided for calculating the COMET metric are built on top of XLM-R and cover the following languages: + +Afrikaans, Albanian, Amharic, Arabic, Armenian, Assamese, Azerbaijani, Basque, Belarusian, Bengali, Bengali Romanized, Bosnian, Breton, Bulgarian, Burmese, Burmese, Catalan, Chinese (Simplified), Chinese (Traditional), Croatian, Czech, Danish, Dutch, English, Esperanto, Estonian, Filipino, Finnish, French, Galician, Georgian, German, Greek, Gujarati, Hausa, Hebrew, Hindi, Hindi Romanized, Hungarian, Icelandic, Indonesian, Irish, Italian, Japanese, Javanese, Kannada, Kazakh, Khmer, Korean, Kurdish (Kurmanji), Kyrgyz, Lao, Latin, Latvian, Lithuanian, Macedonian, Malagasy, Malay, Malayalam, Marathi, Mongolian, Nepali, Norwegian, Oriya, Oromo, Pashto, Persian, Polish, Portuguese, Punjabi, Romanian, Russian, Sanskri, Scottish, Gaelic, Serbian, Sindhi, Sinhala, Slovak, Slovenian, Somali, Spanish, Sundanese, Swahili, Swedish, Tamil, Tamil Romanized, Telugu, Telugu Romanized, Thai, Turkish, Ukrainian, Urdu, Urdu Romanized, Uyghur, Uzbek, Vietnamese, Welsh, Western, Frisian, Xhosa, Yiddish. + +Thus, results for language pairs containing uncovered languages are unreliable, as per the [COMET website](https://github.com/Unbabel/COMET) + +Also, calculating the COMET metric involves downloading the model from which features are obtained -- the default model, `wmt22-comet-da`, takes over 2.32GB of storage space and downloading it can take a significant amount of time depending on the speed of your internet connection. If this is an issue, choose a smaller model; for instance `eamt22-cometinho-da` is 344MB. + +### Interpreting Scores: + +When using COMET to evaluate machine translation, it's important to understand how to interpret the scores it produces. + +In general, COMET models are trained to predict quality scores for translations. These scores are typically normalized using a z-score transformation to account for individual differences among annotators. While the raw score itself does not have a direct interpretation, it is useful for ranking translations and systems according to their quality. + +However, for the latest COMET models like `Unbabel/wmt22-comet-da`, we have introduced a new training approach that scales the scores between 0 and 1. This makes it easier to interpret the scores: a score close to 1 indicates a high-quality translation, while a score close to 0 indicates a translation that is no better than random chance. + +It's worth noting that when using COMET to compare the performance of two different translation systems, it's important to run statistical significance measures to reliably compare scores between systems. + +## Citation +```bibtex +@inproceedings{rei-etal-2022-comet, + title = "{COMET}-22: Unbabel-{IST} 2022 Submission for the Metrics Shared Task", + author = "Rei, Ricardo and + C. de Souza, Jos{\'e} G. and + Alves, Duarte and + Zerva, Chrysoula and + Farinha, Ana C and + Glushkova, Taisiya and + Lavie, Alon and + Coheur, Luisa and + Martins, Andr{\'e} F. T.", + booktitle = "Proceedings of the Seventh Conference on Machine Translation (WMT)", + month = dec, + year = "2022", + address = "Abu Dhabi, United Arab Emirates (Hybrid)", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2022.wmt-1.52", + pages = "578--585", +} +``` + +```bibtex +@inproceedings{rei-EtAl:2020:WMT, + author = {Rei, Ricardo and Stewart, Craig and Farinha, Ana C and Lavie, Alon}, + title = {Unbabel's Participation in the WMT20 Metrics Shared Task}, + booktitle = {Proceedings of the Fifth Conference on Machine Translation}, + month = {November}, + year = {2020}, + address = {Online}, + publisher = {Association for Computational Linguistics}, + pages = {909--918}, +} +``` + +```bibtex +@inproceedings{rei-etal-2020-comet, + title = "{COMET}: A Neural Framework for {MT} Evaluation", + author = "Rei, Ricardo and + Stewart, Craig and + Farinha, Ana C and + Lavie, Alon", + booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)", + month = nov, + year = "2020", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/2020.emnlp-main.213", + pages = "2685--2702", +} +``` + +For the distilled version: + +```bibtex +@inproceedings{rei-etal-2022-searching, + title = "Searching for {COMETINHO}: The Little Metric That Could", + author = "Rei, Ricardo and + Farinha, Ana C and + de Souza, Jos{\'e} G.C. and + Ramos, Pedro G. and + Martins, Andr{\'e} F.T. and + Coheur, Luisa and + Lavie, Alon", + booktitle = "Proceedings of the 23rd Annual Conference of the European Association for Machine Translation", + month = jun, + year = "2022", + address = "Ghent, Belgium", + publisher = "European Association for Machine Translation", + url = "https://aclanthology.org/2022.eamt-1.9", + pages = "61--70", +} +``` + +## Further References + +- [COMET website](https://unbabel.github.io/COMET/html/index.html) +- [Hugging Face Tasks - Machine Translation](https://huggingface.co/tasks/translation) diff --git a/TextClassification/TextClassification/evaluate/metrics/comet/app.py b/TextClassification/TextClassification/evaluate/metrics/comet/app.py new file mode 100644 index 0000000000000000000000000000000000000000..629cf5375bdff6520dcd7c86dbad8416ea1150b3 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/comet/app.py @@ -0,0 +1,11 @@ +import sys + +import evaluate +from evaluate.utils import launch_gradio_widget + + +sys.path = [p for p in sys.path if p != "/home/user/app"] +module = evaluate.load("comet") +sys.path = ["/home/user/app"] + sys.path + +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/comet/comet.py b/TextClassification/TextClassification/evaluate/metrics/comet/comet.py new file mode 100644 index 0000000000000000000000000000000000000000..33953b07314852735bbe6c04760fe886fb3b6fae --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/comet/comet.py @@ -0,0 +1,171 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" COMET metric. + +Requirements: +pip install unbabel-comet + +Usage: + +```python +from evaluate import load +comet_metric = load('metrics/comet/comet.py') +#comet_metric = load('comet') +#comet_metric = load('comet', 'Unbabel/wmt20-comet-da') + + +source = ["Dem Feuer konnte Einhalt geboten werden", "Schulen und Kindergärten wurden eröffnet."] +hypothesis = ["The fire could be stopped", "Schools and kindergartens were open"] +reference = ["They were able to control the fire.", "Schools and kindergartens opened"] + +predictions = comet_metric.compute(predictions=hypothesis, references=reference, sources=source) +predictions['scores'] +``` +""" + +import comet # From: unbabel-comet +import datasets +import torch +from packaging import version + +import evaluate + + +logger = evaluate.logging.get_logger(__name__) + +_CITATION = """\ +@inproceedings{rei-etal-2022-comet, + title = "{COMET}-22: Unbabel-{IST} 2022 Submission for the Metrics Shared Task", + author = "Rei, Ricardo and + C. de Souza, Jos{\'e} G. and + Alves, Duarte and + Zerva, Chrysoula and + Farinha, Ana C and + Glushkova, Taisiya and + Lavie, Alon and + Coheur, Luisa and + Martins, Andr{\'e} F. T.", + booktitle = "Proceedings of the Seventh Conference on Machine Translation (WMT)", + month = dec, + year = "2022", + address = "Abu Dhabi, United Arab Emirates (Hybrid)", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2022.wmt-1.52", + pages = "578--585", +} +@inproceedings{rei-EtAl:2020:WMT, + author = {Rei, Ricardo and Stewart, Craig and Farinha, Ana C and Lavie, Alon}, + title = {Unbabel's Participation in the WMT20 Metrics Shared Task}, + booktitle = {Proceedings of the Fifth Conference on Machine Translation}, + month = {November}, + year = {2020}, + address = {Online}, + publisher = {Association for Computational Linguistics}, + pages = {909--918}, +} +@inproceedings{rei-etal-2020-comet, + title = "{COMET}: A Neural Framework for {MT} Evaluation", + author = "Rei, Ricardo and + Stewart, Craig and + Farinha, Ana C and + Lavie, Alon", + booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)", + month = nov, + year = "2020", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/2020.emnlp-main.213", + pages = "2685--2702", +} +""" + +_DESCRIPTION = """\ +Crosslingual Optimized Metric for Evaluation of Translation (COMET) is an open-source framework used to train Machine Translation metrics that achieve high levels of correlation with different types of human judgments (HTER, DA's or MQM). +With the release of the framework the authors also released fully trained models that were used to compete in the WMT20 Metrics Shared Task achieving SOTA in that years competition. + +See the [README.md] file at https://unbabel.github.io/COMET/html/models.html for more information. +""" + +_KWARGS_DESCRIPTION = """ +COMET score. + +Args: + +`sources` (list of str): Source sentences +`predictions` (list of str): candidate translations +`references` (list of str): reference translations +`gpus` (bool): Number of GPUs to use. 0 for CPU +`progress_bar` (bool): Flag that turns on and off the predict progress bar. Defaults to True + +Returns: + Dict with all sentence-level scores (`scores` key) a system-level score (`mean_score` key). + +Examples: + + >>> comet_metric = evaluate.load('comet') + >>> # comet_metric = load('comet', 'wmt20-comet-da') # you can also choose which model to use + >>> source = ["Dem Feuer konnte Einhalt geboten werden", "Schulen und Kindergärten wurden eröffnet."] + >>> hypothesis = ["The fire could be stopped", "Schools and kindergartens were open"] + >>> reference = ["They were able to control the fire.", "Schools and kindergartens opened"] + >>> results = comet_metric.compute(predictions=hypothesis, references=reference, sources=source) + >>> print([round(v, 3) for v in results["scores"]]) + [0.839, 0.972] +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class COMET(evaluate.Metric): + def _info(self): + + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + homepage="https://unbabel.github.io/COMET/html/index.html", + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "sources": datasets.Value("string", id="sequence"), + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + codebase_urls=["https://github.com/Unbabel/COMET"], + reference_urls=[ + "https://github.com/Unbabel/COMET", + "https://aclanthology.org/2022.wmt-1.52/", + "https://www.aclweb.org/anthology/2020.emnlp-main.213/", + "http://www.statmt.org/wmt20/pdf/2020.wmt-1.101.pdf6", + ], + ) + + def _download_and_prepare(self, dl_manager): + if self.config_name == "default": + if version.parse(comet.__version__) >= version.parse("2.0.0"): + self.scorer = comet.load_from_checkpoint(comet.download_model("Unbabel/wmt22-comet-da")) + else: + self.scorer = comet.load_from_checkpoint(comet.download_model("wmt20-comet-da")) + else: + self.scorer = comet.load_from_checkpoint(comet.download_model(self.config_name)) + + def _compute(self, sources, predictions, references, gpus=None, progress_bar=False): + if gpus is None: + gpus = 1 if torch.cuda.is_available() else 0 + data = {"src": sources, "mt": predictions, "ref": references} + data = [dict(zip(data, t)) for t in zip(*data.values())] + if version.parse(comet.__version__) >= version.parse("2.0.0"): + output = self.scorer.predict(data, gpus=gpus, progress_bar=progress_bar) + scores, mean_score = output.scores, output.system_score + else: + scores, mean_score = self.scorer.predict(data, gpus=gpus, progress_bar=progress_bar) + return {"mean_score": mean_score, "scores": scores} diff --git a/TextClassification/TextClassification/evaluate/metrics/comet/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/comet/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..c50454c31606eb0b885ef4669bb785173d467dd9 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/comet/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +unbabel-comet +torch \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/competition_math/README.md b/TextClassification/TextClassification/evaluate/metrics/competition_math/README.md new file mode 100644 index 0000000000000000000000000000000000000000..44a53000472750fbac5006656345038768a0db12 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/competition_math/README.md @@ -0,0 +1,120 @@ +--- +title: Competition MATH +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + This metric is used to assess performance on the Mathematics Aptitude Test of Heuristics (MATH) dataset. + It first canonicalizes the inputs (e.g., converting "1/2" to "\frac{1}{2}") and then computes accuracy. +--- + +# Metric Card for Competition MATH + +## Metric description + +This metric is used to assess performance on the [Mathematics Aptitude Test of Heuristics (MATH) dataset](https://huggingface.co/datasets/competition_math). + +It first canonicalizes the inputs (e.g., converting `1/2` to `\\frac{1}{2}`) and then computes accuracy. + +## How to use + +This metric takes two arguments: + +`predictions`: a list of predictions to score. Each prediction is a string that contains natural language and LaTeX. + +`references`: list of reference for each prediction. Each reference is a string that contains natural language and LaTeX. + + +```python +>>> from evaluate import load +>>> math = load("competition_math") +>>> references = ["\\frac{1}{2}"] +>>> predictions = ["1/2"] +>>> results = math.compute(references=references, predictions=predictions) +``` + +N.B. To be able to use Competition MATH, you need to install the `math_equivalence` dependency using `pip install git+https://github.com/hendrycks/math.git`. + + +## Output values + +This metric returns a dictionary that contains the [accuracy](https://huggingface.co/metrics/accuracy) after canonicalizing inputs, on a scale between 0.0 and 1.0. + +### Values from popular papers +The [original MATH dataset paper](https://arxiv.org/abs/2103.03874) reported accuracies ranging from 3.0% to 6.9% by different large language models. + +More recent progress on the dataset can be found on the [dataset leaderboard](https://paperswithcode.com/sota/math-word-problem-solving-on-math). + +## Examples + +Maximal values (full match): + +```python +>>> from evaluate import load +>>> math = load("competition_math") +>>> references = ["\\frac{1}{2}"] +>>> predictions = ["1/2"] +>>> results = math.compute(references=references, predictions=predictions) +>>> print(results) +{'accuracy': 1.0} +``` + +Minimal values (no match): + +```python +>>> from evaluate import load +>>> math = load("competition_math") +>>> references = ["\\frac{1}{2}"] +>>> predictions = ["3/4"] +>>> results = math.compute(references=references, predictions=predictions) +>>> print(results) +{'accuracy': 0.0} +``` + +Partial match: + +```python +>>> from evaluate import load +>>> math = load("competition_math") +>>> references = ["\\frac{1}{2}","\\frac{3}{4}"] +>>> predictions = ["1/5", "3/4"] +>>> results = math.compute(references=references, predictions=predictions) +>>> print(results) +{'accuracy': 0.5} +``` + +## Limitations and bias + +This metric is limited to datasets with the same format as the [Mathematics Aptitude Test of Heuristics (MATH) dataset](https://huggingface.co/datasets/competition_math), and is meant to evaluate the performance of large language models at solving mathematical problems. + +N.B. The MATH dataset also assigns levels of difficulty to different problems, so disagregating model performance by difficulty level (similarly to what was done in the [original paper](https://arxiv.org/abs/2103.03874) can give a better indication of how a given model does on a given difficulty of math problem, compared to overall accuracy. + +## Citation + +```bibtex +@article{hendrycksmath2021, + title={Measuring Mathematical Problem Solving With the MATH Dataset}, + author={Dan Hendrycks + and Collin Burns + and Saurav Kadavath + and Akul Arora + and Steven Basart + and Eric Tang + and Dawn Song + and Jacob Steinhardt}, + journal={arXiv preprint arXiv:2103.03874}, + year={2021} +} +``` + +## Further References +- [MATH dataset](https://huggingface.co/datasets/competition_math) +- [MATH leaderboard](https://paperswithcode.com/sota/math-word-problem-solving-on-math) +- [MATH paper](https://arxiv.org/abs/2103.03874) \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/competition_math/app.py b/TextClassification/TextClassification/evaluate/metrics/competition_math/app.py new file mode 100644 index 0000000000000000000000000000000000000000..0cbc7ed2c0ffe712cde35cef4fe86b1f9e4939c8 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/competition_math/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("competition_math") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/competition_math/competition_math.py b/TextClassification/TextClassification/evaluate/metrics/competition_math/competition_math.py new file mode 100644 index 0000000000000000000000000000000000000000..9a82eb40b656dfe892aa5f3ebfef30c21329e92d --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/competition_math/competition_math.py @@ -0,0 +1,95 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Accuracy metric for the Mathematics Aptitude Test of Heuristics (MATH) dataset.""" + +import datasets +import math_equivalence # From: git+https://github.com/hendrycks/math.git + +import evaluate + + +_CITATION = """\ +@article{hendrycksmath2021, + title={Measuring Mathematical Problem Solving With the MATH Dataset}, + author={Dan Hendrycks + and Collin Burns + and Saurav Kadavath + and Akul Arora + and Steven Basart + and Eric Tang + and Dawn Song + and Jacob Steinhardt}, + journal={arXiv preprint arXiv:2103.03874}, + year={2021} +} +""" + + +_DESCRIPTION = """\ +This metric is used to assess performance on the Mathematics Aptitude Test of Heuristics (MATH) dataset. +It first canonicalizes the inputs (e.g., converting "1/2" to "\\frac{1}{2}") and then computes accuracy. +""" + + +_KWARGS_DESCRIPTION = r""" +Calculates accuracy after canonicalizing inputs. + +Args: + predictions: list of predictions to score. Each prediction + is a string that contains natural language and LaTex. + references: list of reference for each prediction. Each + reference is a string that contains natural language + and LaTex. +Returns: + accuracy: accuracy after canonicalizing inputs + (e.g., converting "1/2" to "\\frac{1}{2}") + +Examples: + >>> metric = evaluate.load("competition_math") + >>> results = metric.compute(references=["\\frac{1}{2}"], predictions=["1/2"]) + >>> print(results) + {'accuracy': 1.0} +""" + + +@datasets.utils.file_utils.add_end_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class CompetitionMathMetric(evaluate.Metric): + """Accuracy metric for the MATH dataset.""" + + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("string"), + "references": datasets.Value("string"), + } + ), + # Homepage of the metric for documentation + homepage="https://github.com/hendrycks/math", + # Additional links to the codebase or references + codebase_urls=["https://github.com/hendrycks/math"], + ) + + def _compute(self, predictions, references): + """Returns the scores""" + n_correct = 0.0 + for i, j in zip(predictions, references): + n_correct += 1.0 if math_equivalence.is_equiv(i, j) else 0.0 + accuracy = n_correct / len(predictions) + return { + "accuracy": accuracy, + } diff --git a/TextClassification/TextClassification/evaluate/metrics/competition_math/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/competition_math/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..1d8f9f406828016d38bbc88efd96848edd1b0013 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/competition_math/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +git+https://github.com/hendrycks/math.git \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/confusion_matrix/README.md b/TextClassification/TextClassification/evaluate/metrics/confusion_matrix/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c7ae9482961463a66a3e170a9e93f18888888eb9 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/confusion_matrix/README.md @@ -0,0 +1,101 @@ +--- +title: Confusion Matrix +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + The confusion matrix evaluates classification accuracy. + + Each row in a confusion matrix represents a true class and each column represents the instances in a predicted class. +--- + +# Metric Card for Confusion Matrix + + +## Metric Description + +The confusion matrix evaluates classification accuracy. Each row in a confusion matrix represents a true class and each column represents the instances in a predicted class. Let's look at an example: + +| | setosa | versicolor | virginica | +| ---------- | ------ | ---------- | --------- | +| setosa | 13 | 0 | 0 | +| versicolor | 0 | 10 | 6 | +| virginica | 0 | 0 | 9 | + +What information does this confusion matrix provide? + +* All setosa instances were properly predicted as such (true positives). +* The model always correctly classifies the setosa class (there are no false positives). +* 10 versicolor instances were properly classified, but 6 instances were misclassified as virginica. +* All virginica insances were properly classified as such. + + +## How to Use + +At minimum, this metric requires predictions and references as inputs. + +```python +>>> confusion_metric = evaluate.load("confusion_matrix") +>>> results = confusion_metric.compute(references=[0, 1, 1, 2, 0, 2, 2], predictions=[0, 2, 1, 1, 0, 2, 0]) +>>> print(results) +{'confusion_matrix': [[2, 0, 0], [0, 1, 1], [1, 1, 1]]} +``` + + +### Inputs +- **predictions** (`list` of `int`): Predicted labels. +- **references** (`list` of `int`): Ground truth labels. +- **labels** (`list` of `int`): List of labels to index the matrix. This may be used to reorder or select a subset of labels. +- **sample_weight** (`list` of `float`): Sample weights. +- **normalize** (`str`): Normalizes confusion matrix over the true (rows), predicted (columns) conditions or all the population. + + +### Output Values +- **confusion_matrix**(`list` of `list` of `str`): Confusion matrix. Minimum possible value is 0. Maximum possible value is 1.0, or the number of examples input, if `normalize` is set to `True`. + +Output Example(s): +```python +{'confusion_matrix': [[2, 0, 0], [0, 1, 1], [1, 1, 1]]} +``` + +This metric outputs a dictionary, containing the confusion matrix. + + +### Examples + +Example 1 - A simple example + +```python +>>> confusion_metric = evaluate.load("confusion_matrix") +>>> results = confusion_metric.compute(references=[0, 1, 1, 2, 0, 2, 2], predictions=[0, 2, 1, 1, 0, 2, 0]) +>>> print(results) +{'confusion_matrix': [[2, 0, 0], [0, 1, 1], [1, 1, 1]]} +``` + +## Citation(s) +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + + +## Further References + +* https://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html +* https://en.wikipedia.org/wiki/Confusion_matrix \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/confusion_matrix/app.py b/TextClassification/TextClassification/evaluate/metrics/confusion_matrix/app.py new file mode 100644 index 0000000000000000000000000000000000000000..4c6b193b97b8cdad0fafd75b7dcab2305510d807 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/confusion_matrix/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("confusion_matrix") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/confusion_matrix/confusion_matrix.py b/TextClassification/TextClassification/evaluate/metrics/confusion_matrix/confusion_matrix.py new file mode 100644 index 0000000000000000000000000000000000000000..5ea3303869e05a5f6ec36cf3181cbdb833facbd1 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/confusion_matrix/confusion_matrix.py @@ -0,0 +1,88 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Confusion Matrix.""" + +import datasets +from sklearn.metrics import confusion_matrix + +import evaluate + + +_DESCRIPTION = """ +The confusion matrix evaluates classification accuracy. Each row in a confusion matrix represents a true class and each column represents the instances in a predicted class +""" + +_KWARGS_DESCRIPTION = """ +Args: + predictions (`list` of `int`): Predicted labels. + references (`list` of `int`): Ground truth labels. + labels (`list` of `int`): List of labels to index the matrix. This may be used to reorder or select a subset of labels. + sample_weight (`list` of `float`): Sample weights. + normalize (`str`): Normalizes confusion matrix over the true (rows), predicted (columns) conditions or all the population. + +Returns: + confusion_matrix (`list` of `list` of `int`): Confusion matrix whose i-th row and j-th column entry indicates the number of samples with true label being i-th class and predicted label being j-th class. + +Examples: + + Example 1-A simple example + >>> confusion_matrix_metric = evaluate.load("confusion_matrix") + >>> results = confusion_matrix_metric.compute(references=[0, 1, 2, 0, 1, 2], predictions=[0, 1, 1, 2, 1, 0]) + >>> print(results) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE + {'confusion_matrix': array([[1, 0, 1], [0, 2, 0], [1, 1, 0]][...])} +""" + + +_CITATION = """ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class ConfusionMatrix(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("int32")), + "references": datasets.Sequence(datasets.Value("int32")), + } + if self.config_name == "multilabel" + else { + "predictions": datasets.Value("int32"), + "references": datasets.Value("int32"), + } + ), + reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.confusion_matrix.html"], + ) + + def _compute(self, predictions, references, labels=None, sample_weight=None, normalize=None): + return { + "confusion_matrix": confusion_matrix( + references, predictions, labels=labels, sample_weight=sample_weight, normalize=normalize + ) + } diff --git a/TextClassification/TextClassification/evaluate/metrics/confusion_matrix/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/confusion_matrix/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/confusion_matrix/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/coval/README.md b/TextClassification/TextClassification/evaluate/metrics/coval/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7405f729dcab78fef1582199a2821cd367fe156b --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/coval/README.md @@ -0,0 +1,215 @@ +--- +title: +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + CoVal is a coreference evaluation tool for the CoNLL and ARRAU datasets which + implements of the common evaluation metrics including MUC [Vilain et al, 1995], + B-cubed [Bagga and Baldwin, 1998], CEAFe [Luo et al., 2005], + LEA [Moosavi and Strube, 2016] and the averaged CoNLL score + (the average of the F1 values of MUC, B-cubed and CEAFe) + [Denis and Baldridge, 2009a; Pradhan et al., 2011]. + + This wrapper of CoVal currently only work with CoNLL line format: + The CoNLL format has one word per line with all the annotation for this word in column separated by spaces: + Column Type Description + 1 Document ID This is a variation on the document filename + 2 Part number Some files are divided into multiple parts numbered as 000, 001, 002, ... etc. + 3 Word number + 4 Word itself This is the token as segmented/tokenized in the Treebank. Initially the *_skel file contain the placeholder [WORD] which gets replaced by the actual token from the Treebank which is part of the OntoNotes release. + 5 Part-of-Speech + 6 Parse bit This is the bracketed structure broken before the first open parenthesis in the parse, and the word/part-of-speech leaf replaced with a *. The full parse can be created by substituting the asterix with the "([pos] [word])" string (or leaf) and concatenating the items in the rows of that column. + 7 Predicate lemma The predicate lemma is mentioned for the rows for which we have semantic role information. All other rows are marked with a "-" + 8 Predicate Frameset ID This is the PropBank frameset ID of the predicate in Column 7. + 9 Word sense This is the word sense of the word in Column 3. + 10 Speaker/Author This is the speaker or author name where available. Mostly in Broadcast Conversation and Web Log data. + 11 Named Entities These columns identifies the spans representing various named entities. + 12:N Predicate Arguments There is one column each of predicate argument structure information for the predicate mentioned in Column 7. + N Coreference Coreference chain information encoded in a parenthesis structure. + More informations on the format can be found here (section "*_conll File Format"): http://www.conll.cemantix.org/2012/data.html + + Details on the evaluation on CoNLL can be found here: https://github.com/ns-moosavi/coval/blob/master/conll/README.md + + CoVal code was written by @ns-moosavi. + Some parts are borrowed from https://github.com/clarkkev/deep-coref/blob/master/evaluation.py + The test suite is taken from https://github.com/conll/reference-coreference-scorers/ + Mention evaluation and the test suite are added by @andreasvc. + Parsing CoNLL files is developed by Leo Born. +--- + +## Metric description + +CoVal is a coreference evaluation tool for the [CoNLL](https://huggingface.co/datasets/conll2003) and [ARRAU](https://catalog.ldc.upenn.edu/LDC2013T22) datasets which implements of the common evaluation metrics including MUC [Vilain et al, 1995](https://aclanthology.org/M95-1005.pdf), B-cubed [Bagga and Baldwin, 1998](https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.34.2578&rep=rep1&type=pdf), CEAFe [Luo et al., 2005](https://aclanthology.org/H05-1004.pdf), LEA [Moosavi and Strube, 2016](https://aclanthology.org/P16-1060.pdf) and the averaged CoNLL score (the average of the F1 values of MUC, B-cubed and CEAFe). + +CoVal code was written by [`@ns-moosavi`](https://github.com/ns-moosavi), with some parts borrowed from [Deep Coref](https://github.com/clarkkev/deep-coref/blob/master/evaluation.py). The test suite is taken from the [official CoNLL code](https://github.com/conll/reference-coreference-scorers/), with additions by [`@andreasvc`](https://github.com/andreasvc) and file parsing developed by Leo Born. + +## How to use + +The metric takes two lists of sentences as input: one representing `predictions` and `references`, with the sentences consisting of words in the CoNLL format (see the [Limitations and bias](#Limitations-and-bias) section below for more details on the CoNLL format). + +```python +from evaluate import load +coval = load('coval') +words = ['bc/cctv/00/cctv_0005 0 0 Thank VBP (TOP(S(VP* thank 01 1 Xu_li * (V*) * -', +... 'bc/cctv/00/cctv_0005 0 1 you PRP (NP*) - - - Xu_li * (ARG1*) (ARG0*) (116)', +... 'bc/cctv/00/cctv_0005 0 2 everyone NN (NP*) - - - Xu_li * (ARGM-DIS*) * (116)', +... 'bc/cctv/00/cctv_0005 0 3 for IN (PP* - - - Xu_li * (ARG2* * -', +... 'bc/cctv/00/cctv_0005 0 4 watching VBG (S(VP*)))) watch 01 1 Xu_li * *) (V*) -', +... 'bc/cctv/00/cctv_0005 0 5 . . *)) - - - Xu_li * * * -'] +references = [words] +predictions = [words] +results = coval.compute(predictions=predictions, references=references) +``` +It also has several optional arguments: + +`keep_singletons`: After extracting all mentions of key or system file mentions whose corresponding coreference chain is of size one are considered as singletons. The default evaluation mode will include singletons in evaluations if they are included in the key or the system files. By setting `keep_singletons=False`, all singletons in the key and system files will be excluded from the evaluation. + +`NP_only`: Most of the recent coreference resolvers only resolve NP mentions and leave out the resolution of VPs. By setting the `NP_only` option, the scorer will only evaluate the resolution of NPs. + +`min_span`: By setting `min_span`, the scorer reports the results based on automatically detected minimum spans. Minimum spans are determined using the [MINA algorithm](https://arxiv.org/pdf/1906.06703.pdf). + + +## Output values + +The metric outputs a dictionary with the following key-value pairs: + +`mentions`: number of mentions, ranges from 0-1 + +`muc`: MUC metric, which expresses performance in terms of recall and precision, ranging from 0-1. + +`bcub`: B-cubed metric, which is the averaged precision of all items in the distribution, ranging from 0-1. + +`ceafe`: CEAFe (Constrained Entity Alignment F-Measure) is computed by aligning reference and system entities with the constraint that a reference entity is aligned with at most one reference entity. It ranges from 0-1 + +`lea`: LEA is a Link-Based Entity-Aware metric which, for each entity, considers how important the entity is and how well it is resolved. It ranges from 0-1. + +`conll_score`: averaged CoNLL score (the average of the F1 values of `muc`, `bcub` and `ceafe`), ranging from 0 to 100. + + +### Values from popular papers + +Given that many of the metrics returned by COVAL come from different sources, is it hard to cite reference values for all of them. + +The CoNLL score is used to track progress on different datasets such as the [ARRAU corpus](https://paperswithcode.com/sota/coreference-resolution-on-the-arrau-corpus) and [CoNLL 2012](https://paperswithcode.com/sota/coreference-resolution-on-conll-2012). + +## Examples + +Maximal values + +```python +from evaluate import load +coval = load('coval') +words = ['bc/cctv/00/cctv_0005 0 0 Thank VBP (TOP(S(VP* thank 01 1 Xu_li * (V*) * -', +... 'bc/cctv/00/cctv_0005 0 1 you PRP (NP*) - - - Xu_li * (ARG1*) (ARG0*) (116)', +... 'bc/cctv/00/cctv_0005 0 2 everyone NN (NP*) - - - Xu_li * (ARGM-DIS*) * (116)', +... 'bc/cctv/00/cctv_0005 0 3 for IN (PP* - - - Xu_li * (ARG2* * -', +... 'bc/cctv/00/cctv_0005 0 4 watching VBG (S(VP*)))) watch 01 1 Xu_li * *) (V*) -', +... 'bc/cctv/00/cctv_0005 0 5 . . *)) - - - Xu_li * * * -'] +references = [words] +predictions = [words] +results = coval.compute(predictions=predictions, references=references) +print(results) +{'mentions/recall': 1.0, 'mentions/precision': 1.0, 'mentions/f1': 1.0, 'muc/recall': 1.0, 'muc/precision': 1.0, 'muc/f1': 1.0, 'bcub/recall': 1.0, 'bcub/precision': 1.0, 'bcub/f1': 1.0, 'ceafe/recall': 1.0, 'ceafe/precision': 1.0, 'ceafe/f1': 1.0, 'lea/recall': 1.0, 'lea/precision': 1.0, 'lea/f1': 1.0, 'conll_score': 100.0} +``` + +## Limitations and bias + +This wrapper of CoVal currently only works with [CoNLL line format](https://huggingface.co/datasets/conll2003), which has one word per line with all the annotation for this word in column separated by spaces: + +| Column | Type | Description | +|:-------|:----------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1 | Document ID | This is a variation on the document filename | +| 2 | Part number | Some files are divided into multiple parts numbered as 000, 001, 002, ... etc. | +| 3 | Word number | | +| 4 | Word | This is the token as segmented/tokenized in the Treebank. Initially the *_skel file contain the placeholder [WORD] which gets replaced by the actual token from the Treebank which is part of the OntoNotes release. | +| 5 | Part-of-Speech | | +| 6 | Parse bit | This is the bracketed structure broken before the first open parenthesis in the parse, and the word/part-of-speech leaf replaced with a *. The full parse can be created by substituting the asterix with the "([pos] [word])" string (or leaf) and concatenating the items in the rows of that column. | +| 7 | Predicate lemma | The predicate lemma is mentioned for the rows for which we have semantic role information. All other rows are marked with a "-". | +| 8 | Predicate Frameset ID | This is the PropBank frameset ID of the predicate in Column 7. | +| 9 | Word sense | This is the word sense of the word in Column 3. | +| 10 | Speaker/Author | This is the speaker or author name where available. Mostly in Broadcast Conversation and Web Log data. | +| 11 | Named Entities | These columns identifies the spans representing various named entities. | +| 12:N | Predicate Arguments | There is one column each of predicate argument structure information for the predicate mentioned in Column 7. | +| N | Coreference | Coreference chain information encoded in a parenthesis structure. | + +## Citations + +```bibtex +@InProceedings{moosavi2019minimum, + author = { Nafise Sadat Moosavi, Leo Born, Massimo Poesio and Michael Strube}, + title = {Using Automatically Extracted Minimum Spans to Disentangle Coreference Evaluation from Boundary Detection}, + year = {2019}, + booktitle = {Proceedings of the 57th Annual Meeting of + the Association for Computational Linguistics (Volume 1: Long Papers)}, + publisher = {Association for Computational Linguistics}, + address = {Florence, Italy}, +} +``` +```bibtex +@inproceedings{10.3115/1072399.1072405, + author = {Vilain, Marc and Burger, John and Aberdeen, John and Connolly, Dennis and Hirschman, Lynette}, + title = {A Model-Theoretic Coreference Scoring Scheme}, + year = {1995}, + isbn = {1558604022}, + publisher = {Association for Computational Linguistics}, + address = {USA}, + url = {https://doi.org/10.3115/1072399.1072405}, + doi = {10.3115/1072399.1072405}, + booktitle = {Proceedings of the 6th Conference on Message Understanding}, + pages = {45–52}, + numpages = {8}, + location = {Columbia, Maryland}, + series = {MUC6 ’95} +} +``` + +```bibtex +@INPROCEEDINGS{Bagga98algorithmsfor, + author = {Amit Bagga and Breck Baldwin}, + title = {Algorithms for Scoring Coreference Chains}, + booktitle = {In The First International Conference on Language Resources and Evaluation Workshop on Linguistics Coreference}, + year = {1998}, + pages = {563--566} +} +``` +```bibtex +@INPROCEEDINGS{Luo05oncoreference, + author = {Xiaoqiang Luo}, + title = {On coreference resolution performance metrics}, + booktitle = {In Proc. of HLT/EMNLP}, + year = {2005}, + pages = {25--32}, + publisher = {URL} +} +``` + +```bibtex +@inproceedings{moosavi-strube-2016-coreference, + title = "Which Coreference Evaluation Metric Do You Trust? A Proposal for a Link-based Entity Aware Metric", + author = "Moosavi, Nafise Sadat and + Strube, Michael", + booktitle = "Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = aug, + year = "2016", + address = "Berlin, Germany", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/P16-1060", + doi = "10.18653/v1/P16-1060", + pages = "632--642", +} +``` + +## Further References + +- [CoNLL 2012 Task Description](http://www.conll.cemantix.org/2012/data.html): for information on the format (section "*_conll File Format") +- [CoNLL Evaluation details](https://github.com/ns-moosavi/coval/blob/master/conll/README.md) +- [Hugging Face - Neural Coreference Resolution (Neuralcoref)](https://huggingface.co/coref/) + diff --git a/TextClassification/TextClassification/evaluate/metrics/coval/app.py b/TextClassification/TextClassification/evaluate/metrics/coval/app.py new file mode 100644 index 0000000000000000000000000000000000000000..a2428d98685c3022ef7f9f57b7214d87d1f2984d --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/coval/app.py @@ -0,0 +1,11 @@ +import sys + +import evaluate +from evaluate.utils import launch_gradio_widget + + +sys.path = [p for p in sys.path if p != "/home/user/app"] +module = evaluate.load("coval") +sys.path = ["/home/user/app"] + sys.path + +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/coval/coval.py b/TextClassification/TextClassification/evaluate/metrics/coval/coval.py new file mode 100644 index 0000000000000000000000000000000000000000..f1518b95852ee164b3851633f2716de80abb32d6 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/coval/coval.py @@ -0,0 +1,321 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" CoVal metric. """ +import coval # From: git+https://github.com/ns-moosavi/coval.git noqa: F401 +import datasets +from coval.conll import reader, util +from coval.eval import evaluator + +import evaluate + + +logger = evaluate.logging.get_logger(__name__) + + +_CITATION = """\ +@InProceedings{moosavi2019minimum, + author = { Nafise Sadat Moosavi, Leo Born, Massimo Poesio and Michael Strube}, + title = {Using Automatically Extracted Minimum Spans to Disentangle Coreference Evaluation from Boundary Detection}, + year = {2019}, + booktitle = {Proceedings of the 57th Annual Meeting of + the Association for Computational Linguistics (Volume 1: Long Papers)}, + publisher = {Association for Computational Linguistics}, + address = {Florence, Italy}, +} + +@inproceedings{10.3115/1072399.1072405, +author = {Vilain, Marc and Burger, John and Aberdeen, John and Connolly, Dennis and Hirschman, Lynette}, +title = {A Model-Theoretic Coreference Scoring Scheme}, +year = {1995}, +isbn = {1558604022}, +publisher = {Association for Computational Linguistics}, +address = {USA}, +url = {https://doi.org/10.3115/1072399.1072405}, +doi = {10.3115/1072399.1072405}, +booktitle = {Proceedings of the 6th Conference on Message Understanding}, +pages = {45–52}, +numpages = {8}, +location = {Columbia, Maryland}, +series = {MUC6 ’95} +} + +@INPROCEEDINGS{Bagga98algorithmsfor, + author = {Amit Bagga and Breck Baldwin}, + title = {Algorithms for Scoring Coreference Chains}, + booktitle = {In The First International Conference on Language Resources and Evaluation Workshop on Linguistics Coreference}, + year = {1998}, + pages = {563--566} +} + +@INPROCEEDINGS{Luo05oncoreference, + author = {Xiaoqiang Luo}, + title = {On coreference resolution performance metrics}, + booktitle = {In Proc. of HLT/EMNLP}, + year = {2005}, + pages = {25--32}, + publisher = {URL} +} + +@inproceedings{moosavi-strube-2016-coreference, + title = "Which Coreference Evaluation Metric Do You Trust? A Proposal for a Link-based Entity Aware Metric", + author = "Moosavi, Nafise Sadat and + Strube, Michael", + booktitle = "Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)", + month = aug, + year = "2016", + address = "Berlin, Germany", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/P16-1060", + doi = "10.18653/v1/P16-1060", + pages = "632--642", +} + +""" + +_DESCRIPTION = """\ +CoVal is a coreference evaluation tool for the CoNLL and ARRAU datasets which +implements of the common evaluation metrics including MUC [Vilain et al, 1995], +B-cubed [Bagga and Baldwin, 1998], CEAFe [Luo et al., 2005], +LEA [Moosavi and Strube, 2016] and the averaged CoNLL score +(the average of the F1 values of MUC, B-cubed and CEAFe) +[Denis and Baldridge, 2009a; Pradhan et al., 2011]. + +This wrapper of CoVal currently only work with CoNLL line format: +The CoNLL format has one word per line with all the annotation for this word in column separated by spaces: +Column Type Description +1 Document ID This is a variation on the document filename +2 Part number Some files are divided into multiple parts numbered as 000, 001, 002, ... etc. +3 Word number +4 Word itself This is the token as segmented/tokenized in the Treebank. Initially the *_skel file contain the placeholder [WORD] which gets replaced by the actual token from the Treebank which is part of the OntoNotes release. +5 Part-of-Speech +6 Parse bit This is the bracketed structure broken before the first open parenthesis in the parse, and the word/part-of-speech leaf replaced with a *. The full parse can be created by substituting the asterix with the "([pos] [word])" string (or leaf) and concatenating the items in the rows of that column. +7 Predicate lemma The predicate lemma is mentioned for the rows for which we have semantic role information. All other rows are marked with a "-" +8 Predicate Frameset ID This is the PropBank frameset ID of the predicate in Column 7. +9 Word sense This is the word sense of the word in Column 3. +10 Speaker/Author This is the speaker or author name where available. Mostly in Broadcast Conversation and Web Log data. +11 Named Entities These columns identifies the spans representing various named entities. +12:N Predicate Arguments There is one column each of predicate argument structure information for the predicate mentioned in Column 7. +N Coreference Coreference chain information encoded in a parenthesis structure. +More informations on the format can be found here (section "*_conll File Format"): http://www.conll.cemantix.org/2012/data.html + +Details on the evaluation on CoNLL can be found here: https://github.com/ns-moosavi/coval/blob/master/conll/README.md + +CoVal code was written by @ns-moosavi. +Some parts are borrowed from https://github.com/clarkkev/deep-coref/blob/master/evaluation.py +The test suite is taken from https://github.com/conll/reference-coreference-scorers/ +Mention evaluation and the test suite are added by @andreasvc. +Parsing CoNLL files is developed by Leo Born. +""" + +_KWARGS_DESCRIPTION = """ +Calculates coreference evaluation metrics. +Args: + predictions: list of sentences. Each sentence is a list of word predictions to score in the CoNLL format. + Each prediction is a word with its annotations as a string made of columns joined with spaces. + Only columns 4, 5, 6 and the last column are used (word, POS, Pars and coreference annotation) + See the details on the format in the description of the metric. + references: list of sentences. Each sentence is a list of word reference to score in the CoNLL format. + Each reference is a word with its annotations as a string made of columns joined with spaces. + Only columns 4, 5, 6 and the last column are used (word, POS, Pars and coreference annotation) + See the details on the format in the description of the metric. + keep_singletons: After extracting all mentions of key or system files, + mentions whose corresponding coreference chain is of size one, + are considered as singletons. The default evaluation mode will include + singletons in evaluations if they are included in the key or the system files. + By setting 'keep_singletons=False', all singletons in the key and system files + will be excluded from the evaluation. + NP_only: Most of the recent coreference resolvers only resolve NP mentions and + leave out the resolution of VPs. By setting the 'NP_only' option, the scorer will only evaluate the resolution of NPs. + min_span: By setting 'min_span', the scorer reports the results based on automatically detected minimum spans. + Minimum spans are determined using the MINA algorithm. + +Returns: + 'mentions': mentions + 'muc': MUC metric [Vilain et al, 1995] + 'bcub': B-cubed [Bagga and Baldwin, 1998] + 'ceafe': CEAFe [Luo et al., 2005] + 'lea': LEA [Moosavi and Strube, 2016] + 'conll_score': averaged CoNLL score (the average of the F1 values of MUC, B-cubed and CEAFe) + +Examples: + + >>> coval = evaluate.load('coval') + >>> words = ['bc/cctv/00/cctv_0005 0 0 Thank VBP (TOP(S(VP* thank 01 1 Xu_li * (V*) * -', + ... 'bc/cctv/00/cctv_0005 0 1 you PRP (NP*) - - - Xu_li * (ARG1*) (ARG0*) (116)', + ... 'bc/cctv/00/cctv_0005 0 2 everyone NN (NP*) - - - Xu_li * (ARGM-DIS*) * (116)', + ... 'bc/cctv/00/cctv_0005 0 3 for IN (PP* - - - Xu_li * (ARG2* * -', + ... 'bc/cctv/00/cctv_0005 0 4 watching VBG (S(VP*)))) watch 01 1 Xu_li * *) (V*) -', + ... 'bc/cctv/00/cctv_0005 0 5 . . *)) - - - Xu_li * * * -'] + >>> references = [words] + >>> predictions = [words] + >>> results = coval.compute(predictions=predictions, references=references) + >>> print(results) # doctest:+ELLIPSIS + {'mentions/recall': 1.0,[...] 'conll_score': 100.0} +""" + + +def get_coref_infos( + key_lines, sys_lines, NP_only=False, remove_nested=False, keep_singletons=True, min_span=False, doc="dummy_doc" +): + + key_doc_lines = {doc: key_lines} + sys_doc_lines = {doc: sys_lines} + + doc_coref_infos = {} + + key_nested_coref_num = 0 + sys_nested_coref_num = 0 + key_removed_nested_clusters = 0 + sys_removed_nested_clusters = 0 + key_singletons_num = 0 + sys_singletons_num = 0 + + key_clusters, singletons_num = reader.get_doc_mentions(doc, key_doc_lines[doc], keep_singletons) + key_singletons_num += singletons_num + + if NP_only or min_span: + key_clusters = reader.set_annotated_parse_trees(key_clusters, key_doc_lines[doc], NP_only, min_span) + + sys_clusters, singletons_num = reader.get_doc_mentions(doc, sys_doc_lines[doc], keep_singletons) + sys_singletons_num += singletons_num + + if NP_only or min_span: + sys_clusters = reader.set_annotated_parse_trees(sys_clusters, key_doc_lines[doc], NP_only, min_span) + + if remove_nested: + nested_mentions, removed_clusters = reader.remove_nested_coref_mentions(key_clusters, keep_singletons) + key_nested_coref_num += nested_mentions + key_removed_nested_clusters += removed_clusters + + nested_mentions, removed_clusters = reader.remove_nested_coref_mentions(sys_clusters, keep_singletons) + sys_nested_coref_num += nested_mentions + sys_removed_nested_clusters += removed_clusters + + sys_mention_key_cluster = reader.get_mention_assignments(sys_clusters, key_clusters) + key_mention_sys_cluster = reader.get_mention_assignments(key_clusters, sys_clusters) + + doc_coref_infos[doc] = (key_clusters, sys_clusters, key_mention_sys_cluster, sys_mention_key_cluster) + + if remove_nested: + logger.info( + "Number of removed nested coreferring mentions in the key " + f"annotation: {key_nested_coref_num}; and system annotation: {sys_nested_coref_num}" + ) + logger.info( + "Number of resulting singleton clusters in the key " + f"annotation: {key_removed_nested_clusters}; and system annotation: {sys_removed_nested_clusters}" + ) + + if not keep_singletons: + logger.info( + f"{key_singletons_num:d} and {sys_singletons_num:d} singletons are removed from the key and system " + "files, respectively" + ) + + return doc_coref_infos + + +def compute_score(key_lines, sys_lines, metrics, NP_only, remove_nested, keep_singletons, min_span): + doc_coref_infos = get_coref_infos(key_lines, sys_lines, NP_only, remove_nested, keep_singletons, min_span) + + output_scores = {} + conll = 0 + conll_subparts_num = 0 + + for name, metric in metrics: + recall, precision, f1 = evaluator.evaluate_documents(doc_coref_infos, metric, beta=1) + if name in ["muc", "bcub", "ceafe"]: + conll += f1 + conll_subparts_num += 1 + output_scores.update({f"{name}/recall": recall, f"{name}/precision": precision, f"{name}/f1": f1}) + + logger.info( + name.ljust(10), + f"Recall: {recall * 100:.2f}", + f" Precision: {precision * 100:.2f}", + f" F1: {f1 * 100:.2f}", + ) + + if conll_subparts_num == 3: + conll = (conll / 3) * 100 + logger.info(f"CoNLL score: {conll:.2f}") + output_scores.update({"conll_score": conll}) + + return output_scores + + +def check_gold_parse_annotation(key_lines): + has_gold_parse = False + for line in key_lines: + if not line.startswith("#"): + if len(line.split()) > 6: + parse_col = line.split()[5] + if not parse_col == "-": + has_gold_parse = True + break + else: + break + return has_gold_parse + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Coval(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("string")), + "references": datasets.Sequence(datasets.Value("string")), + } + ), + codebase_urls=["https://github.com/ns-moosavi/coval"], + reference_urls=[ + "https://github.com/ns-moosavi/coval", + "https://www.aclweb.org/anthology/P16-1060", + "http://www.conll.cemantix.org/2012/data.html", + ], + ) + + def _compute( + self, predictions, references, keep_singletons=True, NP_only=False, min_span=False, remove_nested=False + ): + allmetrics = [ + ("mentions", evaluator.mentions), + ("muc", evaluator.muc), + ("bcub", evaluator.b_cubed), + ("ceafe", evaluator.ceafe), + ("lea", evaluator.lea), + ] + + if min_span: + has_gold_parse = util.check_gold_parse_annotation(references) + if not has_gold_parse: + raise NotImplementedError("References should have gold parse annotation to use 'min_span'.") + # util.parse_key_file(key_file) + # key_file = key_file + ".parsed" + + score = compute_score( + key_lines=references, + sys_lines=predictions, + metrics=allmetrics, + NP_only=NP_only, + remove_nested=remove_nested, + keep_singletons=keep_singletons, + min_span=min_span, + ) + + return score diff --git a/TextClassification/TextClassification/evaluate/metrics/coval/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/coval/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..1c54425870ee74020ea4eadc825406dd62b09bb4 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/coval/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +git+https://github.com/ns-moosavi/coval.git \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/cuad/README.md b/TextClassification/TextClassification/evaluate/metrics/cuad/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9ae25a2b1d8ad9b3914f7748b59fb095ac29e7f4 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/cuad/README.md @@ -0,0 +1,132 @@ +--- +title: CUAD +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + This metric wrap the official scoring script for version 1 of the Contract Understanding Atticus Dataset (CUAD). + + Contract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510 + commercial legal contracts that have been manually labeled to identify 41 categories of important + clauses that lawyers look for when reviewing contracts in connection with corporate transactions. +--- + +# Metric Card for CUAD + +## Metric description + +This metric wraps the official scoring script for version 1 of the [Contract Understanding Atticus Dataset (CUAD)](https://huggingface.co/datasets/cuad), which is a corpus of more than 13,000 labels in 510 commercial legal contracts that have been manually labeled to identify 41 categories of important clauses that lawyers look for when reviewing contracts in connection with corporate transactions. + +The CUAD metric computes several scores: [Exact Match](https://huggingface.co/metrics/exact_match), [F1 score](https://huggingface.co/metrics/f1), Area Under the Precision-Recall Curve, [Precision](https://huggingface.co/metrics/precision) at 80% [recall](https://huggingface.co/metrics/recall) and Precision at 90% recall. + +## How to use + +The CUAD metric takes two inputs : + + +`predictions`, a list of question-answer dictionaries with the following key-values: +- `id`: the id of the question-answer pair as given in the references. +- `prediction_text`: a list of possible texts for the answer, as a list of strings depending on a threshold on the confidence probability of each prediction. + + +`references`: a list of question-answer dictionaries with the following key-values: + - `id`: the id of the question-answer pair (the same as above). + - `answers`: a dictionary *in the CUAD dataset format* with the following keys: + - `text`: a list of possible texts for the answer, as a list of strings. + - `answer_start`: a list of start positions for the answer, as a list of ints. + + Note that `answer_start` values are not taken into account to compute the metric. + +```python +from evaluate import load +cuad_metric = load("cuad") +predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] +references = [{'answers': {'answer_start': [143, 49], 'text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.']}, 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] +results = cuad_metric.compute(predictions=predictions, references=references) +``` +## Output values + +The output of the CUAD metric consists of a dictionary that contains one or several of the following metrics: + +`exact_match`: The normalized answers that exactly match the reference answer, with a range between 0.0 and 1.0 (see [exact match](https://huggingface.co/metrics/exact_match) for more information). + +`f1`: The harmonic mean of the precision and recall (see [F1 score](https://huggingface.co/metrics/f1) for more information). Its range is between 0.0 and 1.0 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. + +`aupr`: The Area Under the Precision-Recall curve, with a range between 0.0 and 1.0, with a higher value representing both high recall and high precision, and a low value representing low values for both. See the [Wikipedia article](https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve) for more information. + +`prec_at_80_recall`: The fraction of true examples among the predicted examples at a recall rate of 80%. Its range is between 0.0 and 1.0. For more information, see [precision](https://huggingface.co/metrics/precision) and [recall](https://huggingface.co/metrics/recall). + +`prec_at_90_recall`: The fraction of true examples among the predicted examples at a recall rate of 90%. Its range is between 0.0 and 1.0. + + +### Values from popular papers +The [original CUAD paper](https://arxiv.org/pdf/2103.06268.pdf) reports that a [DeBERTa model](https://huggingface.co/microsoft/deberta-base) attains +an AUPR of 47.8%, a Precision at 80% Recall of 44.0%, and a Precision at 90% Recall of 17.8% (they do not report F1 or Exact Match separately). + +For more recent model performance, see the [dataset leaderboard](https://paperswithcode.com/dataset/cuad). + +## Examples + +Maximal values : + +```python +from evaluate import load +cuad_metric = load("cuad") +predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] +references = [{'answers': {'answer_start': [143, 49], 'text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.']}, 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] +results = cuad_metric.compute(predictions=predictions, references=references) +print(results) +{'exact_match': 100.0, 'f1': 100.0, 'aupr': 0.0, 'prec_at_80_recall': 1.0, 'prec_at_90_recall': 1.0} +``` + +Minimal values: + +```python +from evaluate import load +cuad_metric = load("cuad") +predictions = [{'prediction_text': ['The Company appoints the Distributor as an exclusive distributor of Products in the Market, subject to the terms and conditions of this Agreement.'], 'id': 'LIMEENERGYCO_09_09_1999-EX-10-DISTRIBUTOR AGREEMENT__Exclusivity_0'}] +references = [{'answers': {'answer_start': [143], 'text': 'The seller'}, 'id': 'LIMEENERGYCO_09_09_1999-EX-10-DISTRIBUTOR AGREEMENT__Exclusivity_0'}] +results = cuad_metric.compute(predictions=predictions, references=references) +print(results) +{'exact_match': 0.0, 'f1': 0.0, 'aupr': 0.0, 'prec_at_80_recall': 0, 'prec_at_90_recall': 0} +``` + +Partial match: + +```python +from evaluate import load +cuad_metric = load("cuad") +predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] +predictions = [{'prediction_text': ['The Company appoints the Distributor as an exclusive distributor of Products in the Market, subject to the terms and conditions of this Agreement.', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] +results = cuad_metric.compute(predictions=predictions, references=references) +print(results) +{'exact_match': 100.0, 'f1': 50.0, 'aupr': 0.0, 'prec_at_80_recall': 0, 'prec_at_90_recall': 0} +``` + +## Limitations and bias +This metric works only with datasets that have the same format as the [CUAD dataset](https://huggingface.co/datasets/cuad). The limitations of the biases of this dataset are not discussed, but could exhibit annotation bias given the homogeneity of annotators for this dataset. + +In terms of the metric itself, the accuracy of AUPR has been debated because its estimates are quite noisy and because of the fact that reducing the Precision-Recall Curve to a single number ignores the fact that it is about the tradeoffs between the different systems or performance points plotted and not the performance of an individual system. Reporting the original F1 and exact match scores is therefore useful to ensure a more complete representation of system performance. + + +## Citation + +```bibtex +@article{hendrycks2021cuad, + title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review}, + author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball}, + journal={arXiv preprint arXiv:2103.06268}, + year={2021} +} +``` + +## Further References + +- [CUAD dataset homepage](https://www.atticusprojectai.org/cuad-v1-performance-announcements) diff --git a/TextClassification/TextClassification/evaluate/metrics/cuad/app.py b/TextClassification/TextClassification/evaluate/metrics/cuad/app.py new file mode 100644 index 0000000000000000000000000000000000000000..f54499b1da0d074c8f40cac9a5285c3818243f16 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/cuad/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("cuad") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/cuad/compute_score.py b/TextClassification/TextClassification/evaluate/metrics/cuad/compute_score.py new file mode 100644 index 0000000000000000000000000000000000000000..f35a086e8d957e7763113efc79c59243ca320f14 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/cuad/compute_score.py @@ -0,0 +1,205 @@ +""" Official evaluation script for CUAD dataset. """ + +import argparse +import json +import re +import string +import sys + +import numpy as np + + +IOU_THRESH = 0.5 + + +def get_jaccard(prediction, ground_truth): + remove_tokens = [".", ",", ";", ":"] + + for token in remove_tokens: + ground_truth = ground_truth.replace(token, "") + prediction = prediction.replace(token, "") + + ground_truth, prediction = ground_truth.lower(), prediction.lower() + ground_truth, prediction = ground_truth.replace("/", " "), prediction.replace("/", " ") + ground_truth, prediction = set(ground_truth.split(" ")), set(prediction.split(" ")) + + intersection = ground_truth.intersection(prediction) + union = ground_truth.union(prediction) + jaccard = len(intersection) / len(union) + return jaccard + + +def normalize_answer(s): + """Lower text and remove punctuation, articles and extra whitespace.""" + + def remove_articles(text): + return re.sub(r"\b(a|an|the)\b", " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def compute_precision_recall(predictions, ground_truths, qa_id): + tp, fp, fn = 0, 0, 0 + + substr_ok = "Parties" in qa_id + + # first check if ground truth is empty + if len(ground_truths) == 0: + if len(predictions) > 0: + fp += len(predictions) # false positive for each one + else: + for ground_truth in ground_truths: + assert len(ground_truth) > 0 + # check if there is a match + match_found = False + for pred in predictions: + if substr_ok: + is_match = get_jaccard(pred, ground_truth) >= IOU_THRESH or ground_truth in pred + else: + is_match = get_jaccard(pred, ground_truth) >= IOU_THRESH + if is_match: + match_found = True + + if match_found: + tp += 1 + else: + fn += 1 + + # now also get any fps by looping through preds + for pred in predictions: + # Check if there's a match. if so, don't count (don't want to double count based on the above) + # but if there's no match, then this is a false positive. + # (Note: we get the true positives in the above loop instead of this loop so that we don't double count + # multiple predictions that are matched with the same answer.) + match_found = False + for ground_truth in ground_truths: + assert len(ground_truth) > 0 + if substr_ok: + is_match = get_jaccard(pred, ground_truth) >= IOU_THRESH or ground_truth in pred + else: + is_match = get_jaccard(pred, ground_truth) >= IOU_THRESH + if is_match: + match_found = True + + if not match_found: + fp += 1 + + precision = tp / (tp + fp) if tp + fp > 0 else np.nan + recall = tp / (tp + fn) if tp + fn > 0 else np.nan + + return precision, recall + + +def process_precisions(precisions): + """ + Processes precisions to ensure that precision and recall don't both get worse. + Assumes the list precision is sorted in order of recalls + """ + precision_best = precisions[::-1] + for i in range(1, len(precision_best)): + precision_best[i] = max(precision_best[i - 1], precision_best[i]) + precisions = precision_best[::-1] + return precisions + + +def get_aupr(precisions, recalls): + processed_precisions = process_precisions(precisions) + aupr = np.trapz(processed_precisions, recalls) + if np.isnan(aupr): + return 0 + return aupr + + +def get_prec_at_recall(precisions, recalls, recall_thresh): + """Assumes recalls are sorted in increasing order""" + processed_precisions = process_precisions(precisions) + prec_at_recall = 0 + for prec, recall in zip(processed_precisions, recalls): + if recall >= recall_thresh: + prec_at_recall = prec + break + return prec_at_recall + + +def exact_match_score(prediction, ground_truth): + return normalize_answer(prediction) == normalize_answer(ground_truth) + + +def metric_max_over_ground_truths(metric_fn, predictions, ground_truths): + score = 0 + for pred in predictions: + for ground_truth in ground_truths: + score = metric_fn(pred, ground_truth) + if score == 1: # break the loop when one prediction matches the ground truth + break + if score == 1: + break + return score + + +def compute_score(dataset, predictions): + f1 = exact_match = total = 0 + precisions = [] + recalls = [] + for article in dataset: + for paragraph in article["paragraphs"]: + for qa in paragraph["qas"]: + total += 1 + if qa["id"] not in predictions: + message = "Unanswered question " + qa["id"] + " will receive score 0." + print(message, file=sys.stderr) + continue + ground_truths = list(map(lambda x: x["text"], qa["answers"])) + prediction = predictions[qa["id"]] + precision, recall = compute_precision_recall(prediction, ground_truths, qa["id"]) + + precisions.append(precision) + recalls.append(recall) + + if precision == 0 and recall == 0: + f1 += 0 + else: + f1 += 2 * (precision * recall) / (precision + recall) + + exact_match += metric_max_over_ground_truths(exact_match_score, prediction, ground_truths) + + precisions = [x for _, x in sorted(zip(recalls, precisions))] + recalls.sort() + + f1 = 100.0 * f1 / total + exact_match = 100.0 * exact_match / total + aupr = get_aupr(precisions, recalls) + + prec_at_90_recall = get_prec_at_recall(precisions, recalls, recall_thresh=0.9) + prec_at_80_recall = get_prec_at_recall(precisions, recalls, recall_thresh=0.8) + + return { + "exact_match": exact_match, + "f1": f1, + "aupr": aupr, + "prec_at_80_recall": prec_at_80_recall, + "prec_at_90_recall": prec_at_90_recall, + } + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Evaluation for CUAD") + parser.add_argument("dataset_file", help="Dataset file") + parser.add_argument("prediction_file", help="Prediction File") + args = parser.parse_args() + with open(args.dataset_file) as dataset_file: + dataset_json = json.load(dataset_file) + dataset = dataset_json["data"] + with open(args.prediction_file) as prediction_file: + predictions = json.load(prediction_file) + print(json.dumps(compute_score(dataset, predictions))) diff --git a/TextClassification/TextClassification/evaluate/metrics/cuad/cuad.py b/TextClassification/TextClassification/evaluate/metrics/cuad/cuad.py new file mode 100644 index 0000000000000000000000000000000000000000..21aec5af413a5e8321704ca6ec172df48a900ab2 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/cuad/cuad.py @@ -0,0 +1,117 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" CUAD metric. """ + +import datasets + +import evaluate + +from .compute_score import compute_score + + +_CITATION = """\ +@article{hendrycks2021cuad, + title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review}, + author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball}, + journal={arXiv preprint arXiv:2103.06268}, + year={2021} +} +""" + +_DESCRIPTION = """ +This metric wrap the official scoring script for version 1 of the Contract +Understanding Atticus Dataset (CUAD). +Contract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510 +commercial legal contracts that have been manually labeled to identify 41 categories of important +clauses that lawyers look for when reviewing contracts in connection with corporate transactions. +""" + +_KWARGS_DESCRIPTION = """ +Computes CUAD scores (EM, F1, AUPR, Precision@80%Recall, and Precision@90%Recall). +Args: + predictions: List of question-answers dictionaries with the following key-values: + - 'id': id of the question-answer pair as given in the references (see below) + - 'prediction_text': list of possible texts for the answer, as a list of strings + depending on a threshold on the confidence probability of each prediction. + references: List of question-answers dictionaries with the following key-values: + - 'id': id of the question-answer pair (see above), + - 'answers': a Dict in the CUAD dataset format + { + 'text': list of possible texts for the answer, as a list of strings + 'answer_start': list of start positions for the answer, as a list of ints + } + Note that answer_start values are not taken into account to compute the metric. +Returns: + 'exact_match': Exact match (the normalized answer exactly match the gold answer) + 'f1': The F-score of predicted tokens versus the gold answer + 'aupr': Area Under the Precision-Recall curve + 'prec_at_80_recall': Precision at 80% recall + 'prec_at_90_recall': Precision at 90% recall +Examples: + >>> predictions = [{'prediction_text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.'], 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] + >>> references = [{'answers': {'answer_start': [143, 49], 'text': ['The seller:', 'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.']}, 'id': 'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties'}] + >>> cuad_metric = evaluate.load("cuad") + >>> results = cuad_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'exact_match': 100.0, 'f1': 100.0, 'aupr': 0.0, 'prec_at_80_recall': 1.0, 'prec_at_90_recall': 1.0} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class CUAD(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": { + "id": datasets.Value("string"), + "prediction_text": datasets.features.Sequence(datasets.Value("string")), + }, + "references": { + "id": datasets.Value("string"), + "answers": datasets.features.Sequence( + { + "text": datasets.Value("string"), + "answer_start": datasets.Value("int32"), + } + ), + }, + } + ), + codebase_urls=["https://www.atticusprojectai.org/cuad"], + reference_urls=["https://www.atticusprojectai.org/cuad"], + ) + + def _compute(self, predictions, references): + pred_dict = {prediction["id"]: prediction["prediction_text"] for prediction in predictions} + dataset = [ + { + "paragraphs": [ + { + "qas": [ + { + "answers": [{"text": answer_text} for answer_text in ref["answers"]["text"]], + "id": ref["id"], + } + for ref in references + ] + } + ] + } + ] + score = compute_score(dataset=dataset, predictions=pred_dict) + return score diff --git a/TextClassification/TextClassification/evaluate/metrics/cuad/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/cuad/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a135afacbdd142a40cb9bcfe6073b76cc62d995 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/cuad/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/exact_match/README.md b/TextClassification/TextClassification/evaluate/metrics/exact_match/README.md new file mode 100644 index 0000000000000000000000000000000000000000..21dc1b801a3c6691d3c234b57fb94b12b416ad0b --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/exact_match/README.md @@ -0,0 +1,119 @@ +--- +title: Exact Match +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Returns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list. +--- + +# Metric Card for Exact Match + + +## Metric Description +A given predicted string's exact match score is 1 if it is the exact same as its reference string, and is 0 otherwise. + +- **Example 1**: The exact match score of prediction "Happy Birthday!" is 0, given its reference is "Happy New Year!". +- **Example 2**: The exact match score of prediction "The Colour of Magic (1983)" is 1, given its reference is also "The Colour of Magic (1983)". + +The exact match score of a set of predictions is the sum of all of the individual exact match scores in the set, divided by the total number of predictions in the set. + +- **Example**: The exact match score of the set {Example 1, Example 2} (above) is 0.5. + + +## How to Use +At minimum, this metric takes as input predictions and references: +```python +>>> from evaluate import load +>>> exact_match_metric = load("exact_match") +>>> results = exact_match_metric.compute(predictions=predictions, references=references) +``` + +### Inputs +- **`predictions`** (`list` of `str`): List of predicted texts. +- **`references`** (`list` of `str`): List of reference texts. +- **`regexes_to_ignore`** (`list` of `str`): Regex expressions of characters to ignore when calculating the exact matches. Defaults to `None`. Note: the regex changes are applied before capitalization is normalized. +- **`ignore_case`** (`bool`): If `True`, turns everything to lowercase so that capitalization differences are ignored. Defaults to `False`. +- **`ignore_punctuation`** (`bool`): If `True`, removes punctuation before comparing strings. Defaults to `False`. +- **`ignore_numbers`** (`bool`): If `True`, removes all digits before comparing strings. Defaults to `False`. + + +### Output Values +This metric outputs a dictionary with one value: the average exact match score. + +```python +{'exact_match': 1.0} +``` + +This metric's range is 0-1, inclusive. Here, 0.0 means no prediction/reference pairs were matches, while 1.0 means they all were. + +#### Values from Popular Papers +The exact match metric is often included in other metrics, such as SQuAD. For example, the [original SQuAD paper](https://nlp.stanford.edu/pubs/rajpurkar2016squad.pdf) reported an Exact Match score of 40.0%. They also report that the human performance Exact Match score on the dataset was 80.3%. + +### Examples +Without including any regexes to ignore: +```python +>>> exact_match = evaluate.load("exact_match") +>>> refs = ["the cat", "theater", "YELLING", "agent007"] +>>> preds = ["cat?", "theater", "yelling", "agent"] +>>> results = exact_match.compute(references=refs, predictions=preds) +>>> print(round(results["exact_match"], 2)) +0.25 +``` + +Ignoring regexes "the" and "yell", as well as ignoring case and punctuation: +```python +>>> exact_match = evaluate.load("exact_match") +>>> refs = ["the cat", "theater", "YELLING", "agent007"] +>>> preds = ["cat?", "theater", "yelling", "agent"] +>>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell"], ignore_case=True, ignore_punctuation=True) +>>> print(round(results["exact_match"], 2)) +0.5 +``` +Note that in the example above, because the regexes are ignored before the case is normalized, "yell" from "YELLING" is not deleted. + +Ignoring "the", "yell", and "YELL", as well as ignoring case and punctuation: +```python +>>> exact_match = evaluate.load("exact_match") +>>> refs = ["the cat", "theater", "YELLING", "agent007"] +>>> preds = ["cat?", "theater", "yelling", "agent"] +>>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True) +>>> print(round(results["exact_match"], 2)) +0.75 +``` + +Ignoring "the", "yell", and "YELL", as well as ignoring case, punctuation, and numbers: +```python +>>> exact_match = evaluate.load("exact_match") +>>> refs = ["the cat", "theater", "YELLING", "agent007"] +>>> preds = ["cat?", "theater", "yelling", "agent"] +>>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True) +>>> print(round(results["exact_match"], 2)) +1.0 +``` + +An example that includes sentences: +```python +>>> exact_match = evaluate.load("exact_match") +>>> refs = ["The cat sat on the mat.", "Theaters are great.", "It's like comparing oranges and apples."] +>>> preds = ["The cat sat on the mat?", "Theaters are great.", "It's like comparing apples and oranges."] +>>> results = exact_match.compute(references=refs, predictions=preds) +>>> print(round(results["exact_match"], 2)) +0.33 +``` + + +## Limitations and Bias +This metric is limited in that it outputs the same score for something that is completely wrong as for something that is correct except for a single character. In other words, there is no award for being *almost* right. + +## Citation + +## Further References +- Also used in the [SQuAD metric](https://github.com/huggingface/datasets/tree/master/metrics/squad) diff --git a/TextClassification/TextClassification/evaluate/metrics/exact_match/app.py b/TextClassification/TextClassification/evaluate/metrics/exact_match/app.py new file mode 100644 index 0000000000000000000000000000000000000000..dd86c1cb619adb86db92ee10c5e115e64d987f04 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/exact_match/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("exact_match") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/exact_match/exact_match.py b/TextClassification/TextClassification/evaluate/metrics/exact_match/exact_match.py new file mode 100644 index 0000000000000000000000000000000000000000..d8c499b3722b0bdbbf3d8a7e3d48899513f27d19 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/exact_match/exact_match.py @@ -0,0 +1,136 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Exact Match metric.""" +import re +import string + +import datasets +import numpy as np + +import evaluate + + +_DESCRIPTION = """ +Returns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list. +""" + +_KWARGS_DESCRIPTION = """ +Args: + predictions: List of predicted texts. + references: List of reference texts. + regexes_to_ignore: List, defaults to None. Regex expressions of characters to + ignore when calculating the exact matches. Note: these regexes are removed + from the input data before the changes based on the options below (e.g. ignore_case, + ignore_punctuation, ignore_numbers) are applied. + ignore_case: Boolean, defaults to False. If true, turns everything + to lowercase so that capitalization differences are ignored. + ignore_punctuation: Boolean, defaults to False. If true, removes all punctuation before + comparing predictions and references. + ignore_numbers: Boolean, defaults to False. If true, removes all punctuation before + comparing predictions and references. +Returns: + exact_match: Dictionary containing exact_match rate. Possible values are between 0.0 and 1.0, inclusive. +Examples: + >>> exact_match = evaluate.load("exact_match") + >>> refs = ["the cat", "theater", "YELLING", "agent007"] + >>> preds = ["cat?", "theater", "yelling", "agent"] + >>> results = exact_match.compute(references=refs, predictions=preds) + >>> print(round(results["exact_match"], 2)) + 0.25 + + >>> exact_match = evaluate.load("exact_match") + >>> refs = ["the cat", "theater", "YELLING", "agent007"] + >>> preds = ["cat?", "theater", "yelling", "agent"] + >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell"], ignore_case=True, ignore_punctuation=True) + >>> print(round(results["exact_match"], 2)) + 0.5 + + + >>> exact_match = evaluate.load("exact_match") + >>> refs = ["the cat", "theater", "YELLING", "agent007"] + >>> preds = ["cat?", "theater", "yelling", "agent"] + >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True) + >>> print(round(results["exact_match"], 2)) + 0.75 + + >>> exact_match = evaluate.load("exact_match") + >>> refs = ["the cat", "theater", "YELLING", "agent007"] + >>> preds = ["cat?", "theater", "yelling", "agent"] + >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True) + >>> print(round(results["exact_match"], 2)) + 1.0 + + >>> exact_match = evaluate.load("exact_match") + >>> refs = ["The cat sat on the mat.", "Theaters are great.", "It's like comparing oranges and apples."] + >>> preds = ["The cat sat on the mat?", "Theaters are great.", "It's like comparing apples and oranges."] + >>> results = exact_match.compute(references=refs, predictions=preds) + >>> print(round(results["exact_match"], 2)) + 0.33 +""" + +_CITATION = """ +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class ExactMatch(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + reference_urls=[], + ) + + def _compute( + self, + predictions, + references, + regexes_to_ignore=None, + ignore_case=False, + ignore_punctuation=False, + ignore_numbers=False, + ): + + if regexes_to_ignore is not None: + for s in regexes_to_ignore: + predictions = np.array([re.sub(s, "", x) for x in predictions]) + references = np.array([re.sub(s, "", x) for x in references]) + else: + predictions = np.asarray(predictions) + references = np.asarray(references) + + if ignore_case: + predictions = np.char.lower(predictions) + references = np.char.lower(references) + + if ignore_punctuation: + repl_table = string.punctuation.maketrans("", "", string.punctuation) + predictions = np.char.translate(predictions, table=repl_table) + references = np.char.translate(references, table=repl_table) + + if ignore_numbers: + repl_table = string.digits.maketrans("", "", string.digits) + predictions = np.char.translate(predictions, table=repl_table) + references = np.char.translate(references, table=repl_table) + + score_list = predictions == references + + return {"exact_match": np.mean(score_list)} diff --git a/TextClassification/TextClassification/evaluate/metrics/exact_match/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/exact_match/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a135afacbdd142a40cb9bcfe6073b76cc62d995 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/exact_match/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/f1/README.md b/TextClassification/TextClassification/evaluate/metrics/f1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8f38be6e6533b94833ca11ce198095f064b5e79b --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/f1/README.md @@ -0,0 +1,137 @@ +--- +title: F1 +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + The F1 score is the harmonic mean of the precision and recall. It can be computed with the equation: + F1 = 2 * (precision * recall) / (precision + recall) +--- + +# Metric Card for F1 + + +## Metric Description + +The F1 score is the harmonic mean of the precision and recall. It can be computed with the equation: +F1 = 2 * (precision * recall) / (precision + recall) + + +## How to Use + +At minimum, this metric requires predictions and references as input + +```python +>>> f1_metric = evaluate.load("f1") +>>> results = f1_metric.compute(predictions=[0, 1], references=[0, 1]) +>>> print(results) +["{'f1': 1.0}"] +``` + + +### Inputs +- **predictions** (`list` of `int`): Predicted labels. +- **references** (`list` of `int`): Ground truth labels. +- **labels** (`list` of `int`): The set of labels to include when `average` is not set to `'binary'`, and the order of the labels if `average` is `None`. Labels present in the data can be excluded, for example to calculate a multiclass average ignoring a majority negative class. Labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in `predictions` and `references` are used in sorted order. Defaults to None. +- **pos_label** (`int`): The class to be considered the positive class, in the case where `average` is set to `binary`. Defaults to 1. +- **average** (`string`): This parameter is required for multiclass/multilabel targets. If set to `None`, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`. + - 'binary': Only report results for the class specified by `pos_label`. This is applicable only if the classes found in `predictions` and `references` are binary. + - 'micro': Calculate metrics globally by counting the total true positives, false negatives and false positives. + - 'macro': Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. + - 'weighted': Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. This option can result in an F-score that is not between precision and recall. + - 'samples': Calculate metrics for each instance, and find their average (only meaningful for multilabel classification). +- **sample_weight** (`list` of `float`): Sample weights Defaults to None. + + +### Output Values +- **f1**(`float` or `array` of `float`): F1 score or list of f1 scores, depending on the value passed to `average`. Minimum possible value is 0. Maximum possible value is 1. Higher f1 scores are better. + +Output Example(s): +```python +{'f1': 0.26666666666666666} +``` +```python +{'f1': array([0.8, 0.0, 0.0])} +``` + +This metric outputs a dictionary, with either a single f1 score, of type `float`, or an array of f1 scores, with entries of type `float`. + + +#### Values from Popular Papers + + + + +### Examples + +Example 1-A simple binary example +```python +>>> f1_metric = evaluate.load("f1") +>>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0]) +>>> print(results) +{'f1': 0.5} +``` + +Example 2-The same simple binary example as in Example 1, but with `pos_label` set to `0`. +```python +>>> f1_metric = evaluate.load("f1") +>>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], pos_label=0) +>>> print(round(results['f1'], 2)) +0.67 +``` + +Example 3-The same simple binary example as in Example 1, but with `sample_weight` included. +```python +>>> f1_metric = evaluate.load("f1") +>>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], sample_weight=[0.9, 0.5, 3.9, 1.2, 0.3]) +>>> print(round(results['f1'], 2)) +0.35 +``` + +Example 4-A multiclass example, with different values for the `average` input. +```python +>>> predictions = [0, 2, 1, 0, 0, 1] +>>> references = [0, 1, 2, 0, 1, 2] +>>> results = f1_metric.compute(predictions=predictions, references=references, average="macro") +>>> print(round(results['f1'], 2)) +0.27 +>>> results = f1_metric.compute(predictions=predictions, references=references, average="micro") +>>> print(round(results['f1'], 2)) +0.33 +>>> results = f1_metric.compute(predictions=predictions, references=references, average="weighted") +>>> print(round(results['f1'], 2)) +0.27 +>>> results = f1_metric.compute(predictions=predictions, references=references, average=None) +>>> print(results) +{'f1': array([0.8, 0. , 0. ])} +``` + + +## Limitations and Bias + + + +## Citation(s) +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + + +## Further References \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/f1/app.py b/TextClassification/TextClassification/evaluate/metrics/f1/app.py new file mode 100644 index 0000000000000000000000000000000000000000..56baa5ec6c9867771ebbbf6821366b4ebdd31ef6 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/f1/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("f1") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/f1/f1.py b/TextClassification/TextClassification/evaluate/metrics/f1/f1.py new file mode 100644 index 0000000000000000000000000000000000000000..fe768348912789ed99aa4f00e30dd222e28acaeb --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/f1/f1.py @@ -0,0 +1,130 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""F1 metric.""" + +import datasets +from sklearn.metrics import f1_score + +import evaluate + + +_DESCRIPTION = """ +The F1 score is the harmonic mean of the precision and recall. It can be computed with the equation: +F1 = 2 * (precision * recall) / (precision + recall) +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions (`list` of `int`): Predicted labels. + references (`list` of `int`): Ground truth labels. + labels (`list` of `int`): The set of labels to include when `average` is not set to `'binary'`, and the order of the labels if `average` is `None`. Labels present in the data can be excluded, for example to calculate a multiclass average ignoring a majority negative class. Labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in `predictions` and `references` are used in sorted order. Defaults to None. + pos_label (`int`): The class to be considered the positive class, in the case where `average` is set to `binary`. Defaults to 1. + average (`string`): This parameter is required for multiclass/multilabel targets. If set to `None`, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`. + + - 'binary': Only report results for the class specified by `pos_label`. This is applicable only if the classes found in `predictions` and `references` are binary. + - 'micro': Calculate metrics globally by counting the total true positives, false negatives and false positives. + - 'macro': Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. + - 'weighted': Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. This option can result in an F-score that is not between precision and recall. + - 'samples': Calculate metrics for each instance, and find their average (only meaningful for multilabel classification). + sample_weight (`list` of `float`): Sample weights Defaults to None. + +Returns: + f1 (`float` or `array` of `float`): F1 score or list of f1 scores, depending on the value passed to `average`. Minimum possible value is 0. Maximum possible value is 1. Higher f1 scores are better. + +Examples: + + Example 1-A simple binary example + >>> f1_metric = evaluate.load("f1") + >>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0]) + >>> print(results) + {'f1': 0.5} + + Example 2-The same simple binary example as in Example 1, but with `pos_label` set to `0`. + >>> f1_metric = evaluate.load("f1") + >>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], pos_label=0) + >>> print(round(results['f1'], 2)) + 0.67 + + Example 3-The same simple binary example as in Example 1, but with `sample_weight` included. + >>> f1_metric = evaluate.load("f1") + >>> results = f1_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], sample_weight=[0.9, 0.5, 3.9, 1.2, 0.3]) + >>> print(round(results['f1'], 2)) + 0.35 + + Example 4-A multiclass example, with different values for the `average` input. + >>> predictions = [0, 2, 1, 0, 0, 1] + >>> references = [0, 1, 2, 0, 1, 2] + >>> results = f1_metric.compute(predictions=predictions, references=references, average="macro") + >>> print(round(results['f1'], 2)) + 0.27 + >>> results = f1_metric.compute(predictions=predictions, references=references, average="micro") + >>> print(round(results['f1'], 2)) + 0.33 + >>> results = f1_metric.compute(predictions=predictions, references=references, average="weighted") + >>> print(round(results['f1'], 2)) + 0.27 + >>> results = f1_metric.compute(predictions=predictions, references=references, average=None) + >>> print(results) + {'f1': array([0.8, 0. , 0. ])} + + Example 5-A multi-label example + >>> f1_metric = evaluate.load("f1", "multilabel") + >>> results = f1_metric.compute(predictions=[[0, 1, 1], [1, 1, 0]], references=[[0, 1, 1], [0, 1, 0]], average="macro") + >>> print(round(results['f1'], 2)) + 0.67 +""" + + +_CITATION = """ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class F1(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("int32")), + "references": datasets.Sequence(datasets.Value("int32")), + } + if self.config_name == "multilabel" + else { + "predictions": datasets.Value("int32"), + "references": datasets.Value("int32"), + } + ), + reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html"], + ) + + def _compute(self, predictions, references, labels=None, pos_label=1, average="binary", sample_weight=None): + score = f1_score( + references, predictions, labels=labels, pos_label=pos_label, average=average, sample_weight=sample_weight + ) + return {"f1": float(score) if score.size == 1 else score} diff --git a/TextClassification/TextClassification/evaluate/metrics/f1/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/f1/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/f1/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/frugalscore/README.md b/TextClassification/TextClassification/evaluate/metrics/frugalscore/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a1a05b004dd9422ce4bfaa5c4d5a2f87851b8480 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/frugalscore/README.md @@ -0,0 +1,116 @@ +--- +title: +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + FrugalScore is a reference-based metric for NLG models evaluation. It is based on a distillation approach that allows to learn a fixed, low cost version of any expensive NLG metric, while retaining most of its original performance. +--- + + +## Metric Description +FrugalScore is a reference-based metric for Natural Language Generation (NLG) model evaluation. It is based on a distillation approach that allows to learn a fixed, low cost version of any expensive NLG metric, while retaining most of its original performance. + +The FrugalScore models are obtained by continuing the pretraining of small models on a synthetic dataset constructed using summarization, backtranslation and denoising models. During the training, the small models learn the internal mapping of the expensive metric, including any similarity function. + +## How to use + +When loading FrugalScore, you can indicate the model you wish to use to compute the score. The default model is `moussaKam/frugalscore_tiny_bert-base_bert-score`, and a full list of models can be found in the [Limitations and bias](#Limitations-and-bias) section. + +```python +>>> frugalscore = evaluate.load("frugalscore", "moussaKam/frugalscore_medium_bert-base_mover-score") +``` + +FrugalScore calculates how good are the predictions given some references, based on a set of scores. + +The inputs it takes are: + +`predictions`: a list of strings representing the predictions to score. + +`references`: a list of string representing the references for each prediction. + +Its optional arguments are: + +`batch_size`: the batch size for predictions (default value is `32`). + +`max_length`: the maximum sequence length (default value is `128`). + +`device`: either "gpu" or "cpu" (default value is `None`). + +```python +>>> results = frugalscore.compute(predictions=['hello there', 'huggingface'], references=['hello world', 'hugging face'], batch_size=16, max_length=64, device="gpu") +``` + +## Output values + +The output of FrugalScore is a dictionary with the list of scores for each prediction-reference pair: +```python +{'scores': [0.6307541, 0.6449357]} +``` + +### Values from popular papers +The [original FrugalScore paper](https://arxiv.org/abs/2110.08559) reported that FrugalScore-Tiny retains 97.7/94.7% of the original performance compared to [BertScore](https://huggingface.co/metrics/bertscore) while running 54 times faster and having 84 times less parameters. + +## Examples + +Maximal values (exact match between `references` and `predictions`): + +```python +>>> frugalscore = evaluate.load("frugalscore") +>>> results = frugalscore.compute(predictions=['hello world'], references=['hello world']) +>>> print(results) +{'scores': [0.9891098]} +``` + +Partial values: + +```python +>>> frugalscore = evaluate.load("frugalscore") +>>> results = frugalscore.compute(predictions=['hello world'], references=['hugging face']) +>>> print(results) +{'scores': [0.42482382]} +``` + +## Limitations and bias + +FrugalScore is based on [BertScore](https://huggingface.co/metrics/bertscore) and [MoverScore](https://arxiv.org/abs/1909.02622), and the models used are based on the original models used for these scores. + +The full list of available models for FrugalScore is: + +| FrugalScore | Student | Teacher | Method | +|----------------------------------------------------|-------------|----------------|------------| +| [moussaKam/frugalscore_tiny_bert-base_bert-score](https://huggingface.co/moussaKam/frugalscore_tiny_bert-base_bert-score) | BERT-tiny | BERT-Base | BERTScore | +| [moussaKam/frugalscore_small_bert-base_bert-score](https://huggingface.co/moussaKam/frugalscore_small_bert-base_bert-score) | BERT-small | BERT-Base | BERTScore | +| [moussaKam/frugalscore_medium_bert-base_bert-score](https://huggingface.co/moussaKam/frugalscore_medium_bert-base_bert-score) | BERT-medium | BERT-Base | BERTScore | +| [moussaKam/frugalscore_tiny_roberta_bert-score](https://huggingface.co/moussaKam/frugalscore_tiny_roberta_bert-score) | BERT-tiny | RoBERTa-Large | BERTScore | +| [moussaKam/frugalscore_small_roberta_bert-score](https://huggingface.co/moussaKam/frugalscore_small_roberta_bert-score) | BERT-small | RoBERTa-Large | BERTScore | +| [moussaKam/frugalscore_medium_roberta_bert-score](https://huggingface.co/moussaKam/frugalscore_medium_roberta_bert-score) | BERT-medium | RoBERTa-Large | BERTScore | +| [moussaKam/frugalscore_tiny_deberta_bert-score](https://huggingface.co/moussaKam/frugalscore_tiny_deberta_bert-score) | BERT-tiny | DeBERTa-XLarge | BERTScore | +| [moussaKam/frugalscore_small_deberta_bert-score](https://huggingface.co/moussaKam/frugalscore_small_deberta_bert-score) | BERT-small | DeBERTa-XLarge | BERTScore | +| [moussaKam/frugalscore_medium_deberta_bert-score](https://huggingface.co/moussaKam/frugalscore_medium_deberta_bert-score) | BERT-medium | DeBERTa-XLarge | BERTScore | +| [moussaKam/frugalscore_tiny_bert-base_mover-score](https://huggingface.co/moussaKam/frugalscore_tiny_bert-base_mover-score) | BERT-tiny | BERT-Base | MoverScore | +| [moussaKam/frugalscore_small_bert-base_mover-score](https://huggingface.co/moussaKam/frugalscore_small_bert-base_mover-score) | BERT-small | BERT-Base | MoverScore | +| [moussaKam/frugalscore_medium_bert-base_mover-score](https://huggingface.co/moussaKam/frugalscore_medium_bert-base_mover-score) | BERT-medium | BERT-Base | MoverScore | + +Depending on the size of the model picked, the loading time will vary: the `tiny` models will load very quickly, whereas the `medium` ones can take several minutes, depending on your Internet connection. + +## Citation +```bibtex +@article{eddine2021frugalscore, + title={FrugalScore: Learning Cheaper, Lighter and Faster Evaluation Metrics for Automatic Text Generation}, + author={Eddine, Moussa Kamal and Shang, Guokan and Tixier, Antoine J-P and Vazirgiannis, Michalis}, + journal={arXiv preprint arXiv:2110.08559}, + year={2021} +} +``` + +## Further References +- [Original FrugalScore code](https://github.com/moussaKam/FrugalScore) +- [FrugalScore paper](https://arxiv.org/abs/2110.08559) diff --git a/TextClassification/TextClassification/evaluate/metrics/frugalscore/app.py b/TextClassification/TextClassification/evaluate/metrics/frugalscore/app.py new file mode 100644 index 0000000000000000000000000000000000000000..96ddc17c415bf2e6c362fdbebbd7da7fe48ea69f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/frugalscore/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("frugalscore") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/frugalscore/frugalscore.py b/TextClassification/TextClassification/evaluate/metrics/frugalscore/frugalscore.py new file mode 100644 index 0000000000000000000000000000000000000000..b6f7ee5f3ff9926b9a6c7eb5a2f60eab19b0e2fc --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/frugalscore/frugalscore.py @@ -0,0 +1,117 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current metric script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""FrugalScore metric.""" + +import datasets +import torch +from transformers import AutoModelForSequenceClassification, AutoTokenizer, Trainer, TrainingArguments + +import evaluate + + +_CITATION = """\ +@article{eddine2021frugalscore, + title={FrugalScore: Learning Cheaper, Lighter and Faster Evaluation Metrics for Automatic Text Generation}, + author={Eddine, Moussa Kamal and Shang, Guokan and Tixier, Antoine J-P and Vazirgiannis, Michalis}, + journal={arXiv preprint arXiv:2110.08559}, + year={2021} +} +""" + +_DESCRIPTION = """\ +FrugalScore is a reference-based metric for NLG models evaluation. It is based on a distillation approach that allows to learn a fixed, low cost version of any expensive NLG metric, while retaining most of its original performance. +""" + + +_KWARGS_DESCRIPTION = """ +Calculates how good are predictions given some references, using certain scores. +Args: + predictions (list of str): list of predictions to score. Each predictions + should be a string. + references (list of str): list of reference for each prediction. Each + reference should be a string. + batch_size (int): the batch size for predictions. + max_length (int): maximum sequence length. + device (str): either gpu or cpu +Returns: + scores (list of int): list of scores. +Examples: + >>> frugalscore = evaluate.load("frugalscore") + >>> results = frugalscore.compute(predictions=['hello there', 'huggingface'], references=['hello world', 'hugging face']) + >>> print([round(s, 3) for s in results["scores"]]) + [0.631, 0.645] +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class FRUGALSCORE(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("string"), + "references": datasets.Value("string"), + } + ), + homepage="https://github.com/moussaKam/FrugalScore", + ) + + def _download_and_prepare(self, dl_manager): + if self.config_name == "default": + checkpoint = "moussaKam/frugalscore_tiny_bert-base_bert-score" + else: + checkpoint = self.config_name + self.model = AutoModelForSequenceClassification.from_pretrained(checkpoint) + self.tokenizer = AutoTokenizer.from_pretrained(checkpoint) + + def _compute( + self, + predictions, + references, + batch_size=32, + max_length=128, + device=None, + ): + """Returns the scores""" + assert len(predictions) == len( + references + ), "predictions and references should have the same number of sentences." + if device is not None: + assert device in ["gpu", "cpu"], "device should be either gpu or cpu." + else: + device = "gpu" if torch.cuda.is_available() else "cpu" + training_args = TrainingArguments( + "trainer", + fp16=(device == "gpu"), + per_device_eval_batch_size=batch_size, + report_to="all", + no_cuda=(device == "cpu"), + log_level="warning", + ) + dataset = {"sentence1": predictions, "sentence2": references} + raw_datasets = datasets.Dataset.from_dict(dataset) + + def tokenize_function(data): + return self.tokenizer( + data["sentence1"], data["sentence2"], max_length=max_length, truncation=True, padding=True + ) + + tokenized_datasets = raw_datasets.map(tokenize_function, batched=True) + tokenized_datasets.remove_columns(["sentence1", "sentence2"]) + trainer = Trainer(self.model, training_args, tokenizer=self.tokenizer) + predictions = trainer.predict(tokenized_datasets) + return {"scores": list(predictions.predictions.squeeze(-1))} diff --git a/TextClassification/TextClassification/evaluate/metrics/frugalscore/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/frugalscore/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..84eb91d19f4a5974796fd201cc805cbb22c907bb --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/frugalscore/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +torch +transformers \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/glue/README.md b/TextClassification/TextClassification/evaluate/metrics/glue/README.md new file mode 100644 index 0000000000000000000000000000000000000000..af78d504d9d9840da469c18d5e2af798d078472b --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/glue/README.md @@ -0,0 +1,123 @@ +--- +title: GLUE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + GLUE, the General Language Understanding Evaluation benchmark + (https://gluebenchmark.com/) is a collection of resources for training, + evaluating, and analyzing natural language understanding systems. +--- + +# Metric Card for GLUE + +## Metric description +This metric is used to compute the GLUE evaluation metric associated to each [GLUE dataset](https://huggingface.co/datasets/glue). + +GLUE, the General Language Understanding Evaluation benchmark is a collection of resources for training, evaluating, and analyzing natural language understanding systems. + +## How to use + +There are two steps: (1) loading the GLUE metric relevant to the subset of the GLUE dataset being used for evaluation; and (2) calculating the metric. + +1. **Loading the relevant GLUE metric** : the subsets of GLUE are the following: `sst2`, `mnli`, `mnli_mismatched`, `mnli_matched`, `qnli`, `rte`, `wnli`, `cola`,`stsb`, `mrpc`, `qqp`, and `hans`. + +More information about the different subsets of the GLUE dataset can be found on the [GLUE dataset page](https://huggingface.co/datasets/glue). + +2. **Calculating the metric**: the metric takes two inputs : one list with the predictions of the model to score and one lists of references for each translation. + +```python +from evaluate import load +glue_metric = load('glue', 'sst2') +references = [0, 1] +predictions = [0, 1] +results = glue_metric.compute(predictions=predictions, references=references) +``` +## Output values + +The output of the metric depends on the GLUE subset chosen, consisting of a dictionary that contains one or several of the following metrics: + +`accuracy`: the proportion of correct predictions among the total number of cases processed, with a range between 0 and 1 (see [accuracy](https://huggingface.co/metrics/accuracy) for more information). + +`f1`: the harmonic mean of the precision and recall (see [F1 score](https://huggingface.co/metrics/f1) for more information). Its range is 0-1 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. + +`pearson`: a measure of the linear relationship between two datasets (see [Pearson correlation](https://huggingface.co/metrics/pearsonr) for more information). Its range is between -1 and +1, with 0 implying no correlation, and -1/+1 implying an exact linear relationship. Positive correlations imply that as x increases, so does y, whereas negative correlations imply that as x increases, y decreases. + +`spearmanr`: a nonparametric measure of the monotonicity of the relationship between two datasets(see [Spearman Correlation](https://huggingface.co/metrics/spearmanr) for more information). `spearmanr` has the same range as `pearson`. + +`matthews_correlation`: a measure of the quality of binary and multiclass classifications (see [Matthews Correlation](https://huggingface.co/metrics/matthews_correlation) for more information). Its range of values is between -1 and +1, where a coefficient of +1 represents a perfect prediction, 0 an average random prediction and -1 an inverse prediction. + +The `cola` subset returns `matthews_correlation`, the `stsb` subset returns `pearson` and `spearmanr`, the `mrpc` and `qqp` subsets return both `accuracy` and `f1`, and all other subsets of GLUE return only accuracy. + +### Values from popular papers +The [original GLUE paper](https://huggingface.co/datasets/glue) reported average scores ranging from 58 to 64%, depending on the model used (with all evaluation values scaled by 100 to make computing the average possible). + +For more recent model performance, see the [dataset leaderboard](https://paperswithcode.com/dataset/glue). + +## Examples + +Maximal values for the MRPC subset (which outputs `accuracy` and `f1`): + +```python +from evaluate import load +glue_metric = load('glue', 'mrpc') # 'mrpc' or 'qqp' +references = [0, 1] +predictions = [0, 1] +results = glue_metric.compute(predictions=predictions, references=references) +print(results) +{'accuracy': 1.0, 'f1': 1.0} +``` + +Minimal values for the STSB subset (which outputs `pearson` and `spearmanr`): + +```python +from evaluate import load +glue_metric = load('glue', 'stsb') +references = [0., 1., 2., 3., 4., 5.] +predictions = [-10., -11., -12., -13., -14., -15.] +results = glue_metric.compute(predictions=predictions, references=references) +print(results) +{'pearson': -1.0, 'spearmanr': -1.0} +``` + +Partial match for the COLA subset (which outputs `matthews_correlation`) + +```python +from evaluate import load +glue_metric = load('glue', 'cola') +references = [0, 1] +predictions = [1, 1] +results = glue_metric.compute(predictions=predictions, references=references) +results +{'matthews_correlation': 0.0} +``` + +## Limitations and bias +This metric works only with datasets that have the same format as the [GLUE dataset](https://huggingface.co/datasets/glue). + +While the GLUE dataset is meant to represent "General Language Understanding", the tasks represented in it are not necessarily representative of language understanding, and should not be interpreted as such. + +Also, while the GLUE subtasks were considered challenging during its creation in 2019, they are no longer considered as such given the impressive progress made since then. A more complex (or "stickier") version of it, called [SuperGLUE](https://huggingface.co/datasets/super_glue), was subsequently created. + +## Citation + +```bibtex + @inproceedings{wang2019glue, + title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding}, + author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.}, + note={In the Proceedings of ICLR.}, + year={2019} +} +``` + +## Further References + +- [GLUE benchmark homepage](https://gluebenchmark.com/) +- [Fine-tuning a model with the Trainer API](https://huggingface.co/course/chapter3/3?) diff --git a/TextClassification/TextClassification/evaluate/metrics/glue/app.py b/TextClassification/TextClassification/evaluate/metrics/glue/app.py new file mode 100644 index 0000000000000000000000000000000000000000..41b7d78d34856615fee2762cd51aacecc1cdc910 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/glue/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("glue", "sst2") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/glue/glue.py b/TextClassification/TextClassification/evaluate/metrics/glue/glue.py new file mode 100644 index 0000000000000000000000000000000000000000..8c607bf1e2fef42fa3abcf1c672c90b576e6eef0 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/glue/glue.py @@ -0,0 +1,156 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" GLUE benchmark metric. """ + +import datasets +from scipy.stats import pearsonr, spearmanr +from sklearn.metrics import f1_score, matthews_corrcoef + +import evaluate + + +_CITATION = """\ +@inproceedings{wang2019glue, + title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding}, + author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.}, + note={In the Proceedings of ICLR.}, + year={2019} +} +""" + +_DESCRIPTION = """\ +GLUE, the General Language Understanding Evaluation benchmark +(https://gluebenchmark.com/) is a collection of resources for training, +evaluating, and analyzing natural language understanding systems. +""" + +_KWARGS_DESCRIPTION = """ +Compute GLUE evaluation metric associated to each GLUE dataset. +Args: + predictions: list of predictions to score. + Each translation should be tokenized into a list of tokens. + references: list of lists of references for each translation. + Each reference should be tokenized into a list of tokens. +Returns: depending on the GLUE subset, one or several of: + "accuracy": Accuracy + "f1": F1 score + "pearson": Pearson Correlation + "spearmanr": Spearman Correlation + "matthews_correlation": Matthew Correlation +Examples: + + >>> glue_metric = evaluate.load('glue', 'sst2') # 'sst2' or any of ["mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"] + >>> references = [0, 1] + >>> predictions = [0, 1] + >>> results = glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'accuracy': 1.0} + + >>> glue_metric = evaluate.load('glue', 'mrpc') # 'mrpc' or 'qqp' + >>> references = [0, 1] + >>> predictions = [0, 1] + >>> results = glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'accuracy': 1.0, 'f1': 1.0} + + >>> glue_metric = evaluate.load('glue', 'stsb') + >>> references = [0., 1., 2., 3., 4., 5.] + >>> predictions = [0., 1., 2., 3., 4., 5.] + >>> results = glue_metric.compute(predictions=predictions, references=references) + >>> print({"pearson": round(results["pearson"], 2), "spearmanr": round(results["spearmanr"], 2)}) + {'pearson': 1.0, 'spearmanr': 1.0} + + >>> glue_metric = evaluate.load('glue', 'cola') + >>> references = [0, 1] + >>> predictions = [0, 1] + >>> results = glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'matthews_correlation': 1.0} +""" + + +def simple_accuracy(preds, labels): + return float((preds == labels).mean()) + + +def acc_and_f1(preds, labels): + acc = simple_accuracy(preds, labels) + f1 = float(f1_score(y_true=labels, y_pred=preds)) + return { + "accuracy": acc, + "f1": f1, + } + + +def pearson_and_spearman(preds, labels): + pearson_corr = float(pearsonr(preds, labels)[0]) + spearman_corr = float(spearmanr(preds, labels)[0]) + return { + "pearson": pearson_corr, + "spearmanr": spearman_corr, + } + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Glue(evaluate.Metric): + def _info(self): + if self.config_name not in [ + "sst2", + "mnli", + "mnli_mismatched", + "mnli_matched", + "cola", + "stsb", + "mrpc", + "qqp", + "qnli", + "rte", + "wnli", + "hans", + ]: + raise KeyError( + "You should supply a configuration name selected in " + '["sst2", "mnli", "mnli_mismatched", "mnli_matched", ' + '"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]' + ) + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("int64" if self.config_name != "stsb" else "float32"), + "references": datasets.Value("int64" if self.config_name != "stsb" else "float32"), + } + ), + codebase_urls=[], + reference_urls=[], + format="numpy", + ) + + def _compute(self, predictions, references): + if self.config_name == "cola": + return {"matthews_correlation": matthews_corrcoef(references, predictions)} + elif self.config_name == "stsb": + return pearson_and_spearman(predictions, references) + elif self.config_name in ["mrpc", "qqp"]: + return acc_and_f1(predictions, references) + elif self.config_name in ["sst2", "mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]: + return {"accuracy": simple_accuracy(predictions, references)} + else: + raise KeyError( + "You should supply a configuration name selected in " + '["sst2", "mnli", "mnli_mismatched", "mnli_matched", ' + '"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]' + ) diff --git a/TextClassification/TextClassification/evaluate/metrics/glue/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/glue/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..56f7d3a98bb1bcfd747153724d77cff546124757 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/glue/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scipy +scikit-learn \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/google_bleu/README.md b/TextClassification/TextClassification/evaluate/metrics/google_bleu/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4d4814295cbcb796bc000d0db950ca9b35bc1d73 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/google_bleu/README.md @@ -0,0 +1,153 @@ +--- +title: Google BLEU +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + The BLEU score has some undesirable properties when used for single + sentences, as it was designed to be a corpus measure. We therefore + use a slightly different score for our RL experiments which we call + the 'GLEU score'. For the GLEU score, we record all sub-sequences of + 1, 2, 3 or 4 tokens in output and target sequence (n-grams). We then + compute a recall, which is the ratio of the number of matching n-grams + to the number of total n-grams in the target (ground truth) sequence, + and a precision, which is the ratio of the number of matching n-grams + to the number of total n-grams in the generated output sequence. Then + GLEU score is simply the minimum of recall and precision. This GLEU + score's range is always between 0 (no matches) and 1 (all match) and + it is symmetrical when switching output and target. According to + our experiments, GLEU score correlates quite well with the BLEU + metric on a corpus level but does not have its drawbacks for our per + sentence reward objective. +--- + +# Metric Card for Google BLEU + + +## Metric Description +The BLEU score has some undesirable properties when used for single sentences, as it was designed to be a corpus measure. The Google BLEU score is designed to limit these undesirable properties when used for single sentences. + +To calculate this score, all sub-sequences of 1, 2, 3 or 4 tokens in output and target sequence (n-grams) are recorded. The precision and recall, described below, are then computed. + +- **precision:** the ratio of the number of matching n-grams to the number of total n-grams in the generated output sequence +- **recall:** the ratio of the number of matching n-grams to the number of total n-grams in the target (ground truth) sequence + +The minimum value of precision and recall is then returned as the score. + + +## Intended Uses +This metric is generally used to evaluate machine translation models. It is especially used when scores of individual (prediction, reference) sentence pairs are needed, as opposed to when averaging over the (prediction, reference) scores for a whole corpus. That being said, it can also be used when averaging over the scores for a whole corpus. + +Because it performs better on individual sentence pairs as compared to BLEU, Google BLEU has also been used in RL experiments. + +## How to Use +This metric takes a list of predicted sentences, as well as a list of references. + +```python +sentence1 = "the cat sat on the mat" +sentence2 = "the cat ate the mat" +google_bleu = evaluate.load("google_bleu") +result = google_bleu.compute(predictions=[sentence1], references=[[sentence2]]) +print(result) +>>> {'google_bleu': 0.3333333333333333} +``` + +### Inputs +- **predictions** (list of str): list of translations to score. +- **references** (list of list of str): list of lists of references for each translation. +- **tokenizer** : approach used for tokenizing `predictions` and `references`. +The default tokenizer is `tokenizer_13a`, a minimal tokenization approach that is equivalent to `mteval-v13a`, used by WMT. This can be replaced by any function that takes a string as input and returns a list of tokens as output. +- **min_len** (int): The minimum order of n-gram this function should extract. Defaults to 1. +- **max_len** (int): The maximum order of n-gram this function should extract. Defaults to 4. + +### Output Values +This metric returns the following in a dict: +- **google_bleu** (float): google_bleu score + +The output format is as follows: +```python +{'google_bleu': google_bleu score} +``` + +This metric can take on values from 0 to 1, inclusive. Higher scores are better, with 0 indicating no matches, and 1 indicating a perfect match. + +Note that this score is symmetrical when switching output and target. This means that, given two sentences, `sentence1` and `sentence2`, whatever score is output when `sentence1` is the predicted sentence and `sencence2` is the reference sentence will be the same as when the sentences are swapped and `sentence2` is the predicted sentence while `sentence1` is the reference sentence. In code, this looks like: + +```python +predictions = "the cat sat on the mat" +references = "the cat ate the mat" +google_bleu = evaluate.load("google_bleu") +result_a = google_bleu.compute(predictions=[predictions], references=[[references]]) +result_b = google_bleu.compute(predictions=[predictions], references=[[references]]) +print(result_a == result_b) +>>> True +``` + +#### Values from Popular Papers + + +### Examples +Example with one reference per sample: +```python +>>> predictions = ['It is a guide to action which ensures that the rubber duck always disobeys the commands of the cat', 'he read the book because he was interested in world history'] +>>> references = [['It is the guiding principle which guarantees the rubber duck forces never being under the command of the cat'], ['he was interested in world history because he read the book']] +>>> google_bleu = evaluate.load("google_bleu") +>>> results = google_bleu.compute(predictions=predictions, references=references) +>>> print(round(results["google_bleu"], 2)) +0.44 +``` + +Example with multiple references for the first sample: +```python +>>> predictions = ['It is a guide to action which ensures that the rubber duck always disobeys the commands of the cat', 'he read the book because he was interested in world history'] +>>> references = [['It is the guiding principle which guarantees the rubber duck forces never being under the command of the cat', 'It is a guide to action that ensures that the rubber duck will never heed the cat commands', 'It is the practical guide for the rubber duck army never to heed the directions of the cat'], ['he was interested in world history because he read the book']] +>>> google_bleu = evaluate.load("google_bleu") +>>> results = google_bleu.compute(predictions=predictions, references=references) +>>> print(round(results["google_bleu"], 2)) +0.61 +``` + +Example with multiple references for the first sample, and with `min_len` adjusted to `2`, instead of the default `1`, which means that the function extracts n-grams of length `2`: +```python +>>> predictions = ['It is a guide to action which ensures that the rubber duck always disobeys the commands of the cat', 'he read the book because he was interested in world history'] +>>> references = [['It is the guiding principle which guarantees the rubber duck forces never being under the command of the cat', 'It is a guide to action that ensures that the rubber duck will never heed the cat commands', 'It is the practical guide for the rubber duck army never to heed the directions of the cat'], ['he was interested in world history because he read the book']] +>>> google_bleu = evaluate.load("google_bleu") +>>> results = google_bleu.compute(predictions=predictions, references=references, min_len=2) +>>> print(round(results["google_bleu"], 2)) +0.53 +``` + +Example with multiple references for the first sample, with `min_len` adjusted to `2`, instead of the default `1`, and `max_len` adjusted to `6` instead of the default `4`: +```python +>>> predictions = ['It is a guide to action which ensures that the rubber duck always disobeys the commands of the cat', 'he read the book because he was interested in world history'] +>>> references = [['It is the guiding principle which guarantees the rubber duck forces never being under the command of the cat', 'It is a guide to action that ensures that the rubber duck will never heed the cat commands', 'It is the practical guide for the rubber duck army never to heed the directions of the cat'], ['he was interested in world history because he read the book']] +>>> google_bleu = evaluate.load("google_bleu") +>>> results = google_bleu.compute(predictions=predictions,references=references, min_len=2, max_len=6) +>>> print(round(results["google_bleu"], 2)) +0.4 +``` + +## Limitations and Bias + +The GoogleBLEU metric does not come with a predefined tokenization function; previous versions simply used `split()` to split the input strings into tokens. Using a tokenizer such as the default one, `tokenizer_13a`, makes results more standardized and reproducible. The BLEU and sacreBLEU metrics also use this default tokenizer. + +## Citation +```bibtex +@misc{wu2016googles, +title={Google's Neural Machine Translation System: Bridging the Gap between Human and Machine Translation}, +author={Yonghui Wu and Mike Schuster and Zhifeng Chen and Quoc V. Le and Mohammad Norouzi and Wolfgang Macherey and Maxim Krikun and Yuan Cao and Qin Gao and Klaus Macherey and Jeff Klingner and Apurva Shah and Melvin Johnson and Xiaobing Liu and Łukasz Kaiser and Stephan Gouws and Yoshikiyo Kato and Taku Kudo and Hideto Kazawa and Keith Stevens and George Kurian and Nishant Patil and Wei Wang and Cliff Young and Jason Smith and Jason Riesa and Alex Rudnick and Oriol Vinyals and Greg Corrado and Macduff Hughes and Jeffrey Dean}, +year={2016}, +eprint={1609.08144}, +archivePrefix={arXiv}, +primaryClass={cs.CL} +} +``` +## Further References +- This Hugging Face implementation uses the [nltk.translate.gleu_score implementation](https://www.nltk.org/_modules/nltk/translate/gleu_score.html) diff --git a/TextClassification/TextClassification/evaluate/metrics/google_bleu/app.py b/TextClassification/TextClassification/evaluate/metrics/google_bleu/app.py new file mode 100644 index 0000000000000000000000000000000000000000..582df789b608693d0262b7c546e8aa94595b9b78 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/google_bleu/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("google_bleu") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/google_bleu/google_bleu.py b/TextClassification/TextClassification/evaluate/metrics/google_bleu/google_bleu.py new file mode 100644 index 0000000000000000000000000000000000000000..adcc0a31f382e65d00d7f7a7169aef20205bb577 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/google_bleu/google_bleu.py @@ -0,0 +1,168 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Google BLEU (aka GLEU) metric. """ + +from typing import Dict, List + +import datasets +from nltk.translate import gleu_score + +import evaluate +from evaluate import MetricInfo + +from .tokenizer_13a import Tokenizer13a + + +_CITATION = """\ +@misc{wu2016googles, + title={Google's Neural Machine Translation System: Bridging the Gap between Human and Machine Translation}, + author={Yonghui Wu and Mike Schuster and Zhifeng Chen and Quoc V. Le and Mohammad Norouzi and Wolfgang Macherey + and Maxim Krikun and Yuan Cao and Qin Gao and Klaus Macherey and Jeff Klingner and Apurva Shah and Melvin + Johnson and Xiaobing Liu and Łukasz Kaiser and Stephan Gouws and Yoshikiyo Kato and Taku Kudo and Hideto + Kazawa and Keith Stevens and George Kurian and Nishant Patil and Wei Wang and Cliff Young and + Jason Smith and Jason Riesa and Alex Rudnick and Oriol Vinyals and Greg Corrado and Macduff Hughes + and Jeffrey Dean}, + year={2016}, + eprint={1609.08144}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +""" + +_DESCRIPTION = """\ +The BLEU score has some undesirable properties when used for single +sentences, as it was designed to be a corpus measure. We therefore +use a slightly different score for our RL experiments which we call +the 'GLEU score'. For the GLEU score, we record all sub-sequences of +1, 2, 3 or 4 tokens in output and target sequence (n-grams). We then +compute a recall, which is the ratio of the number of matching n-grams +to the number of total n-grams in the target (ground truth) sequence, +and a precision, which is the ratio of the number of matching n-grams +to the number of total n-grams in the generated output sequence. Then +GLEU score is simply the minimum of recall and precision. This GLEU +score's range is always between 0 (no matches) and 1 (all match) and +it is symmetrical when switching output and target. According to +our experiments, GLEU score correlates quite well with the BLEU +metric on a corpus level but does not have its drawbacks for our per +sentence reward objective. +""" + +_KWARGS_DESCRIPTION = """\ +Computes corpus-level Google BLEU (GLEU) score of translated segments against one or more references. +Instead of averaging the sentence level GLEU scores (i.e. macro-average precision), Wu et al. (2016) sum up the matching +tokens and the max of hypothesis and reference tokens for each sentence, then compute using the aggregate values. + +Args: + predictions (list of str): list of translations to score. + references (list of list of str): list of lists of references for each translation. + tokenizer : approach used for tokenizing `predictions` and `references`. + The default tokenizer is `tokenizer_13a`, a minimal tokenization approach that is equivalent to `mteval-v13a`, used by WMT. + This can be replaced by any function that takes a string as input and returns a list of tokens as output. + min_len (int): The minimum order of n-gram this function should extract. Defaults to 1. + max_len (int): The maximum order of n-gram this function should extract. Defaults to 4. + +Returns: + 'google_bleu': google_bleu score + +Examples: + Example 1: + >>> predictions = ['It is a guide to action which ensures that the rubber duck always disobeys the commands of the cat', \ + 'he read the book because he was interested in world history'] + >>> references = [['It is the guiding principle which guarantees the rubber duck forces never being under the command of the cat'], \ + ['he was interested in world history because he read the book']] + >>> google_bleu = evaluate.load("google_bleu") + >>> results = google_bleu.compute(predictions=predictions, references=references) + >>> print(round(results["google_bleu"], 2)) + 0.44 + + Example 2: + >>> predictions = ['It is a guide to action which ensures that the rubber duck always disobeys the commands of the cat', \ + 'he read the book because he was interested in world history'] + >>> references = [['It is the guiding principle which guarantees the rubber duck forces never being under the command of the cat', \ + 'It is a guide to action that ensures that the rubber duck will never heed the cat commands', \ + 'It is the practical guide for the rubber duck army never to heed the directions of the cat'], \ + ['he was interested in world history because he read the book']] + >>> google_bleu = evaluate.load("google_bleu") + >>> results = google_bleu.compute(predictions=predictions, references=references) + >>> print(round(results["google_bleu"], 2)) + 0.61 + + Example 3: + >>> predictions = ['It is a guide to action which ensures that the rubber duck always disobeys the commands of the cat', \ + 'he read the book because he was interested in world history'] + >>> references = [['It is the guiding principle which guarantees the rubber duck forces never being under the command of the cat', \ + 'It is a guide to action that ensures that the rubber duck will never heed the cat commands', \ + 'It is the practical guide for the rubber duck army never to heed the directions of the cat'], \ + ['he was interested in world history because he read the book']] + >>> google_bleu = evaluate.load("google_bleu") + >>> results = google_bleu.compute(predictions=predictions, references=references, min_len=2) + >>> print(round(results["google_bleu"], 2)) + 0.53 + + Example 4: + >>> predictions = ['It is a guide to action which ensures that the rubber duck always disobeys the commands of the cat', \ + 'he read the book because he was interested in world history'] + >>> references = [['It is the guiding principle which guarantees the rubber duck forces never being under the command of the cat', \ + 'It is a guide to action that ensures that the rubber duck will never heed the cat commands', \ + 'It is the practical guide for the rubber duck army never to heed the directions of the cat'], \ + ['he was interested in world history because he read the book']] + >>> google_bleu = evaluate.load("google_bleu") + >>> results = google_bleu.compute(predictions=predictions,references=references, min_len=2, max_len=6) + >>> print(round(results["google_bleu"], 2)) + 0.4 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class GoogleBleu(evaluate.Metric): + def _info(self) -> MetricInfo: + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), + } + ), + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + ], + ) + + def _compute( + self, + predictions: List[str], + references: List[List[str]], + tokenizer=Tokenizer13a(), + min_len: int = 1, + max_len: int = 4, + ) -> Dict[str, float]: + # if only one reference is provided make sure we still use list of lists + if isinstance(references[0], str): + references = [[ref] for ref in references] + + references = [[tokenizer(r) for r in ref] for ref in references] + predictions = [tokenizer(p) for p in predictions] + return { + "google_bleu": gleu_score.corpus_gleu( + list_of_references=references, hypotheses=predictions, min_len=min_len, max_len=max_len + ) + } diff --git a/TextClassification/TextClassification/evaluate/metrics/google_bleu/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/google_bleu/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b5904a71469bea16b7ccb3b41e9ed19663d20599 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/google_bleu/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +nltk \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/google_bleu/tokenizer_13a.py b/TextClassification/TextClassification/evaluate/metrics/google_bleu/tokenizer_13a.py new file mode 100644 index 0000000000000000000000000000000000000000..c7a1b3dbd39d7db5ed0c4d000d6d648e2cbf0009 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/google_bleu/tokenizer_13a.py @@ -0,0 +1,100 @@ +# Source: https://github.com/mjpost/sacrebleu/blob/master/sacrebleu/tokenizers/tokenizer_13a.py +# Copyright 2020 SacreBLEU Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from functools import lru_cache + + +class BaseTokenizer: + """A base dummy tokenizer to derive from.""" + + def signature(self): + """ + Returns a signature for the tokenizer. + :return: signature string + """ + return "none" + + def __call__(self, line): + """ + Tokenizes an input line with the tokenizer. + :param line: a segment to tokenize + :return: the tokenized line + """ + return line + + +class TokenizerRegexp(BaseTokenizer): + def signature(self): + return "re" + + def __init__(self): + self._re = [ + # language-dependent part (assuming Western languages) + (re.compile(r"([\{-\~\[-\` -\&\(-\+\:-\@\/])"), r" \1 "), + # tokenize period and comma unless preceded by a digit + (re.compile(r"([^0-9])([\.,])"), r"\1 \2 "), + # tokenize period and comma unless followed by a digit + (re.compile(r"([\.,])([^0-9])"), r" \1 \2"), + # tokenize dash when preceded by a digit + (re.compile(r"([0-9])(-)"), r"\1 \2 "), + # one space only between words + # NOTE: Doing this in Python (below) is faster + # (re.compile(r'\s+'), r' '), + ] + + @lru_cache(maxsize=2**16) + def __call__(self, line): + """Common post-processing tokenizer for `13a` and `zh` tokenizers. + :param line: a segment to tokenize + :return: the tokenized line + """ + for (_re, repl) in self._re: + line = _re.sub(repl, line) + + # no leading or trailing spaces, single space within words + # return ' '.join(line.split()) + # This line is changed with regards to the original tokenizer (seen above) to return individual words + return line.split() + + +class Tokenizer13a(BaseTokenizer): + def signature(self): + return "13a" + + def __init__(self): + self._post_tokenizer = TokenizerRegexp() + + @lru_cache(maxsize=2**16) + def __call__(self, line): + """Tokenizes an input line using a relatively minimal tokenization + that is however equivalent to mteval-v13a, used by WMT. + + :param line: a segment to tokenize + :return: the tokenized line + """ + + # language-independent part: + line = line.replace("", "") + line = line.replace("-\n", "") + line = line.replace("\n", " ") + + if "&" in line: + line = line.replace(""", '"') + line = line.replace("&", "&") + line = line.replace("<", "<") + line = line.replace(">", ">") + + return self._post_tokenizer(f" {line} ") diff --git a/TextClassification/TextClassification/evaluate/metrics/indic_glue/README.md b/TextClassification/TextClassification/evaluate/metrics/indic_glue/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a58c8d1dcd5dcf9f60393aad3ee64e4e66c4396a --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/indic_glue/README.md @@ -0,0 +1,108 @@ +--- +title: IndicGLUE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide + variety of tasks and covers 11 major Indian languages - as, bn, gu, hi, kn, ml, mr, or, pa, ta, te. +--- + +# Metric Card for IndicGLUE + +## Metric description +This metric is used to compute the evaluation metric for the [IndicGLUE dataset](https://huggingface.co/datasets/indic_glue). + +IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide variety of tasks and covers 11 major Indian languages - Assamese (`as`), Bengali (`bn`), Gujarati (`gu`), Hindi (`hi`), Kannada (`kn`), Malayalam (`ml`), Marathi(`mr`), Oriya(`or`), Panjabi (`pa`), Tamil(`ta`) and Telugu (`te`). + +## How to use + +There are two steps: (1) loading the IndicGLUE metric relevant to the subset of the dataset being used for evaluation; and (2) calculating the metric. + +1. **Loading the relevant IndicGLUE metric** : the subsets of IndicGLUE are the following: `wnli`, `copa`, `sna`, `csqa`, `wstp`, `inltkh`, `bbca`, `cvit-mkb-clsr`, `iitp-mr`, `iitp-pr`, `actsa-sc`, `md`, and`wiki-ner`. + +More information about the different subsets of the Indic GLUE dataset can be found on the [IndicGLUE dataset page](https://indicnlp.ai4bharat.org/indic-glue/). + +2. **Calculating the metric**: the metric takes two inputs : one list with the predictions of the model to score and one lists of references for each translation for all subsets of the dataset except for `cvit-mkb-clsr`, where each prediction and reference is a vector of floats. + +```python +indic_glue_metric = evaluate.load('indic_glue', 'wnli') +references = [0, 1] +predictions = [0, 1] +results = indic_glue_metric.compute(predictions=predictions, references=references) +``` + +## Output values + +The output of the metric depends on the IndicGLUE subset chosen, consisting of a dictionary that contains one or several of the following metrics: + +`accuracy`: the proportion of correct predictions among the total number of cases processed, with a range between 0 and 1 (see [accuracy](https://huggingface.co/metrics/accuracy) for more information). + +`f1`: the harmonic mean of the precision and recall (see [F1 score](https://huggingface.co/metrics/f1) for more information). Its range is 0-1 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. + +`precision@10`: the fraction of the true examples among the top 10 predicted examples, with a range between 0 and 1 (see [precision](https://huggingface.co/metrics/precision) for more information). + +The `cvit-mkb-clsr` subset returns `precision@10`, the `wiki-ner` subset returns `accuracy` and `f1`, and all other subsets of Indic GLUE return only accuracy. + +### Values from popular papers + +The [original IndicGlue paper](https://aclanthology.org/2020.findings-emnlp.445.pdf) reported an average accuracy of 0.766 on the dataset, which varies depending on the subset selected. + +## Examples + +Maximal values for the WNLI subset (which outputs `accuracy`): + +```python +indic_glue_metric = evaluate.load('indic_glue', 'wnli') +references = [0, 1] +predictions = [0, 1] +results = indic_glue_metric.compute(predictions=predictions, references=references) +print(results) +{'accuracy': 1.0} +``` + +Minimal values for the Wiki-NER subset (which outputs `accuracy` and `f1`): + +```python +>>> indic_glue_metric = evaluate.load('indic_glue', 'wiki-ner') +>>> references = [0, 1] +>>> predictions = [1,0] +>>> results = indic_glue_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'accuracy': 1.0, 'f1': 1.0} +``` + +Partial match for the CVIT-Mann Ki Baat subset (which outputs `precision@10`) + +```python +>>> indic_glue_metric = evaluate.load('indic_glue', 'cvit-mkb-clsr') +>>> references = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]] +>>> predictions = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]] +>>> results = indic_glue_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'precision@10': 1.0} +``` + +## Limitations and bias +This metric works only with datasets that have the same format as the [IndicGLUE dataset](https://huggingface.co/datasets/glue). + +## Citation + +```bibtex + @inproceedings{kakwani2020indicnlpsuite, + title={{IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages}}, + author={Divyanshu Kakwani and Anoop Kunchukuttan and Satish Golla and Gokul N.C. and Avik Bhattacharyya and Mitesh M. Khapra and Pratyush Kumar}, + year={2020}, + booktitle={Findings of EMNLP}, +} +``` + +## Further References +- [IndicNLP website](https://indicnlp.ai4bharat.org/home/) diff --git a/TextClassification/TextClassification/evaluate/metrics/indic_glue/app.py b/TextClassification/TextClassification/evaluate/metrics/indic_glue/app.py new file mode 100644 index 0000000000000000000000000000000000000000..1edafea04b601ad33ecfae98fc01dac61fabc202 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/indic_glue/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("indic_glue", "wnli") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/indic_glue/indic_glue.py b/TextClassification/TextClassification/evaluate/metrics/indic_glue/indic_glue.py new file mode 100644 index 0000000000000000000000000000000000000000..03afd1700180ad1d0eeb0d8e885c9ad4c67489e2 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/indic_glue/indic_glue.py @@ -0,0 +1,173 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" IndicGLUE benchmark metric. """ + +import datasets +import numpy as np +from scipy.spatial.distance import cdist +from scipy.stats import pearsonr, spearmanr +from sklearn.metrics import f1_score + +import evaluate + + +_CITATION = """\ + @inproceedings{kakwani2020indicnlpsuite, + title={{IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages}}, + author={Divyanshu Kakwani and Anoop Kunchukuttan and Satish Golla and Gokul N.C. and Avik Bhattacharyya and Mitesh M. Khapra and Pratyush Kumar}, + year={2020}, + booktitle={Findings of EMNLP}, +} +""" + +_DESCRIPTION = """\ + IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide + variety of tasks and covers 11 major Indian languages - as, bn, gu, hi, kn, ml, mr, or, pa, ta, te. +""" + +_KWARGS_DESCRIPTION = """ +Compute IndicGLUE evaluation metric associated to each IndicGLUE dataset. +Args: + predictions: list of predictions to score (as int64), + except for 'cvit-mkb-clsr' where each prediction is a vector (of float32). + references: list of ground truth labels corresponding to the predictions (as int64), + except for 'cvit-mkb-clsr' where each reference is a vector (of float32). +Returns: depending on the IndicGLUE subset, one or several of: + "accuracy": Accuracy + "f1": F1 score + "precision": Precision@10 +Examples: + + >>> indic_glue_metric = evaluate.load('indic_glue', 'wnli') # 'wnli' or any of ["copa", "sna", "csqa", "wstp", "inltkh", "bbca", "iitp-mr", "iitp-pr", "actsa-sc", "md"] + >>> references = [0, 1] + >>> predictions = [0, 1] + >>> results = indic_glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'accuracy': 1.0} + + >>> indic_glue_metric = evaluate.load('indic_glue', 'wiki-ner') + >>> references = [0, 1] + >>> predictions = [0, 1] + >>> results = indic_glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'accuracy': 1.0, 'f1': 1.0} + + >>> indic_glue_metric = evaluate.load('indic_glue', 'cvit-mkb-clsr') + >>> references = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]] + >>> predictions = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]] + >>> results = indic_glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'precision@10': 1.0} + +""" + + +def simple_accuracy(preds, labels): + return float((preds == labels).mean()) + + +def acc_and_f1(preds, labels): + acc = simple_accuracy(preds, labels) + f1 = float(f1_score(y_true=labels, y_pred=preds)) + return { + "accuracy": acc, + "f1": f1, + } + + +def precision_at_10(en_sentvecs, in_sentvecs): + en_sentvecs = np.array(en_sentvecs) + in_sentvecs = np.array(in_sentvecs) + n = en_sentvecs.shape[0] + + # mean centering + en_sentvecs = en_sentvecs - np.mean(en_sentvecs, axis=0) + in_sentvecs = in_sentvecs - np.mean(in_sentvecs, axis=0) + + sim = cdist(en_sentvecs, in_sentvecs, "cosine") + actual = np.array(range(n)) + preds = sim.argsort(axis=1)[:, :10] + matches = np.any(preds == actual[:, None], axis=1) + return float(matches.mean()) + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class IndicGlue(evaluate.Metric): + def _info(self): + if self.config_name not in [ + "wnli", + "copa", + "sna", + "csqa", + "wstp", + "inltkh", + "bbca", + "cvit-mkb-clsr", + "iitp-mr", + "iitp-pr", + "actsa-sc", + "md", + "wiki-ner", + ]: + raise KeyError( + "You should supply a configuration name selected in " + '["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", ' + '"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", ' + '"wiki-ner"]' + ) + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("int64") + if self.config_name != "cvit-mkb-clsr" + else datasets.Sequence(datasets.Value("float32")), + "references": datasets.Value("int64") + if self.config_name != "cvit-mkb-clsr" + else datasets.Sequence(datasets.Value("float32")), + } + ), + codebase_urls=[], + reference_urls=[], + format="numpy" if self.config_name != "cvit-mkb-clsr" else None, + ) + + def _compute(self, predictions, references): + if self.config_name == "cvit-mkb-clsr": + return {"precision@10": precision_at_10(predictions, references)} + elif self.config_name in ["wiki-ner"]: + return acc_and_f1(predictions, references) + elif self.config_name in [ + "wnli", + "copa", + "sna", + "csqa", + "wstp", + "inltkh", + "bbca", + "iitp-mr", + "iitp-pr", + "actsa-sc", + "md", + ]: + return {"accuracy": simple_accuracy(predictions, references)} + else: + raise KeyError( + "You should supply a configuration name selected in " + '["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", ' + '"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", ' + '"wiki-ner"]' + ) diff --git a/TextClassification/TextClassification/evaluate/metrics/indic_glue/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/indic_glue/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..56f7d3a98bb1bcfd747153724d77cff546124757 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/indic_glue/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scipy +scikit-learn \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/mae/README.md b/TextClassification/TextClassification/evaluate/metrics/mae/README.md new file mode 100644 index 0000000000000000000000000000000000000000..fe238e1f67e2769a8ad9d215fb7fa910aa5d9eb7 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mae/README.md @@ -0,0 +1,129 @@ +--- +title: MAE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Mean Absolute Error (MAE) is the mean of the magnitude of difference between the predicted and actual + values. +--- + +# Metric Card for MAE + + +## Metric Description + +Mean Absolute Error (MAE) is the mean of the magnitude of difference between the predicted and actual numeric values: +![image](https://user-images.githubusercontent.com/14205986/165824243-e1078dfd-489d-456c-a0da-cbaa28726220.png) + + +## How to Use + +At minimum, this metric requires predictions and references as inputs. + +```python +>>> mae_metric = evaluate.load("mae") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> results = mae_metric.compute(predictions=predictions, references=references) +``` + +### Inputs + +Mandatory inputs: +- `predictions`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the estimated target values. +- `references`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the ground truth (correct) target values. + +Optional arguments: +- `sample_weight`: numeric array-like of shape (`n_samples,`) representing sample weights. The default is `None`. +- `multioutput`: `raw_values`, `uniform_average` or numeric array-like of shape (`n_outputs,`), which defines the aggregation of multiple output values. The default value is `uniform_average`. + - `raw_values` returns a full set of errors in case of multioutput input. + - `uniform_average` means that the errors of all outputs are averaged with uniform weight. + - the array-like value defines weights used to average errors. + +### Output Values +This metric outputs a dictionary, containing the mean absolute error score, which is of type: +- `float`: if multioutput is `uniform_average` or an ndarray of weights, then the weighted average of all output errors is returned. +- numeric array-like of shape (`n_outputs,`): if multioutput is `raw_values`, then the score is returned for each output separately. + +Each MAE `float` value ranges from `0.0` to `+inf`, with the best value being 0.0. + +Output Example(s): +```python +{'mae': 0.5} +``` + +If `multioutput="raw_values"`: +```python +{'mae': array([0.5, 1. ])} +``` + +#### Values from Popular Papers + + +### Examples + +Example with the `uniform_average` config: +```python +>>> mae_metric = evaluate.load("mae") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> results = mae_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'mae': 0.5} +``` + +Example with multi-dimensional lists, and the `raw_values` config: +```python +>>> mae_metric = evaluate.load("mae", "multilist") +>>> predictions = [[0.5, 1], [-1, 1], [7, -6]] +>>> references = [[0, 2], [-1, 2], [8, -5]] +>>> results = mae_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'mae': 0.75} +>>> results = mae_metric.compute(predictions=predictions, references=references, multioutput='raw_values') +>>> print(results) +{'mae': array([0.5, 1. ])} +``` + +## Limitations and Bias +One limitation of MAE is that the relative size of the error is not always obvious, meaning that it can be difficult to tell a big error from a smaller one -- metrics such as Mean Absolute Percentage Error (MAPE) have been proposed to calculate MAE in percentage terms. + +Also, since it calculates the mean, MAE may underestimate the impact of big, but infrequent, errors -- metrics such as the Root Mean Square Error (RMSE) compensate for this by taking the root of error values. + +## Citation(s) +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + +```bibtex +@article{willmott2005advantages, + title={Advantages of the mean absolute error (MAE) over the root mean square error (RMSE) in assessing average model performance}, + author={Willmott, Cort J and Matsuura, Kenji}, + journal={Climate research}, + volume={30}, + number={1}, + pages={79--82}, + year={2005} +} +``` + +## Further References +- [Mean Absolute Error - Wikipedia](https://en.wikipedia.org/wiki/Mean_absolute_error) diff --git a/TextClassification/TextClassification/evaluate/metrics/mae/app.py b/TextClassification/TextClassification/evaluate/metrics/mae/app.py new file mode 100644 index 0000000000000000000000000000000000000000..be2c05aa85e7eb06667955a25d79b4a2f0389419 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mae/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("mae") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/mae/mae.py b/TextClassification/TextClassification/evaluate/metrics/mae/mae.py new file mode 100644 index 0000000000000000000000000000000000000000..e973bc5ff60926eb28e7daebf20c00bc639a40db --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mae/mae.py @@ -0,0 +1,113 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MAE - Mean Absolute Error Metric""" + +import datasets +from sklearn.metrics import mean_absolute_error + +import evaluate + + +_CITATION = """\ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + +_DESCRIPTION = """\ +Mean Absolute Error (MAE) is the mean of the magnitude of difference between the predicted and actual +values. +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions: array-like of shape (n_samples,) or (n_samples, n_outputs) + Estimated target values. + references: array-like of shape (n_samples,) or (n_samples, n_outputs) + Ground truth (correct) target values. + sample_weight: array-like of shape (n_samples,), default=None + Sample weights. + multioutput: {"raw_values", "uniform_average"} or array-like of shape (n_outputs,), default="uniform_average" + Defines aggregating of multiple output values. Array-like value defines weights used to average errors. + + "raw_values" : Returns a full set of errors in case of multioutput input. + + "uniform_average" : Errors of all outputs are averaged with uniform weight. + +Returns: + mae : mean absolute error. + If multioutput is "raw_values", then mean absolute error is returned for each output separately. If multioutput is "uniform_average" or an ndarray of weights, then the weighted average of all output errors is returned. + MAE output is non-negative floating point. The best value is 0.0. +Examples: + + >>> mae_metric = evaluate.load("mae") + >>> predictions = [2.5, 0.0, 2, 8] + >>> references = [3, -0.5, 2, 7] + >>> results = mae_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'mae': 0.5} + + If you're using multi-dimensional lists, then set the config as follows : + + >>> mae_metric = evaluate.load("mae", "multilist") + >>> predictions = [[0.5, 1], [-1, 1], [7, -6]] + >>> references = [[0, 2], [-1, 2], [8, -5]] + >>> results = mae_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'mae': 0.75} + >>> results = mae_metric.compute(predictions=predictions, references=references, multioutput='raw_values') + >>> print(results) + {'mae': array([0.5, 1. ])} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Mae(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features(self._get_feature_types()), + reference_urls=[ + "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_absolute_error.html" + ], + ) + + def _get_feature_types(self): + if self.config_name == "multilist": + return { + "predictions": datasets.Sequence(datasets.Value("float")), + "references": datasets.Sequence(datasets.Value("float")), + } + else: + return { + "predictions": datasets.Value("float"), + "references": datasets.Value("float"), + } + + def _compute(self, predictions, references, sample_weight=None, multioutput="uniform_average"): + + mae_score = mean_absolute_error(references, predictions, sample_weight=sample_weight, multioutput=multioutput) + + return {"mae": mae_score} diff --git a/TextClassification/TextClassification/evaluate/metrics/mae/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/mae/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mae/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/mahalanobis/README.md b/TextClassification/TextClassification/evaluate/metrics/mahalanobis/README.md new file mode 100644 index 0000000000000000000000000000000000000000..18a5ab4755a8af68bef1afcdee4f86ebabcf49b7 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mahalanobis/README.md @@ -0,0 +1,92 @@ +--- +title: Mahalanobis Distance +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Compute the Mahalanobis Distance + + Mahalonobis distance is the distance between a point and a distribution. + And not between two distinct points. It is effectively a multivariate equivalent of the Euclidean distance. + It was introduced by Prof. P. C. Mahalanobis in 1936 + and has been used in various statistical applications ever since + [source: https://www.machinelearningplus.com/statistics/mahalanobis-distance/] +--- + +# Metric Card for Mahalanobis Distance + +## Metric Description +Mahalonobis distance is the distance between a point and a distribution (as opposed to the distance between two points), making it the multivariate equivalent of the Euclidean distance. + +It is often used in multivariate anomaly detection, classification on highly imbalanced datasets and one-class classification. + +## How to Use +At minimum, this metric requires two `list`s of datapoints: + +```python +>>> mahalanobis_metric = evaluate.load("mahalanobis") +>>> results = mahalanobis_metric.compute(reference_distribution=[[0, 1], [1, 0]], X=[[0, 1]]) +``` + +### Inputs +- `X` (`list`): data points to be compared with the `reference_distribution`. +- `reference_distribution` (`list`): data points from the reference distribution that we want to compare to. + +### Output Values +`mahalanobis` (`array`): the Mahalonobis distance for each data point in `X`. + +```python +>>> print(results) +{'mahalanobis': array([0.5])} +``` + +#### Values from Popular Papers +*N/A* + +### Example + +```python +>>> mahalanobis_metric = evaluate.load("mahalanobis") +>>> results = mahalanobis_metric.compute(reference_distribution=[[0, 1], [1, 0]], X=[[0, 1]]) +>>> print(results) +{'mahalanobis': array([0.5])} +``` + +## Limitations and Bias + +The Mahalanobis distance is only able to capture linear relationships between the variables, which means it cannot capture all types of outliers. Mahalanobis distance also fails to faithfully represent data that is highly skewed or multimodal. + +## Citation +```bibtex +@inproceedings{mahalanobis1936generalized, + title={On the generalized distance in statistics}, + author={Mahalanobis, Prasanta Chandra}, + year={1936}, + organization={National Institute of Science of India} +} +``` + +```bibtex +@article{de2000mahalanobis, + title={The Mahalanobis distance}, + author={De Maesschalck, Roy and Jouan-Rimbaud, Delphine and Massart, D{\'e}sir{\'e} L}, + journal={Chemometrics and intelligent laboratory systems}, + volume={50}, + number={1}, + pages={1--18}, + year={2000}, + publisher={Elsevier} +} +``` + +## Further References +-[Wikipedia -- Mahalanobis Distance](https://en.wikipedia.org/wiki/Mahalanobis_distance) + +-[Machine Learning Plus -- Mahalanobis Distance](https://www.machinelearningplus.com/statistics/mahalanobis-distance/) diff --git a/TextClassification/TextClassification/evaluate/metrics/mahalanobis/app.py b/TextClassification/TextClassification/evaluate/metrics/mahalanobis/app.py new file mode 100644 index 0000000000000000000000000000000000000000..cb6d338dc126a4131e80f466f605b9fd958bf884 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mahalanobis/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("mahalanobis") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/mahalanobis/mahalanobis.py b/TextClassification/TextClassification/evaluate/metrics/mahalanobis/mahalanobis.py new file mode 100644 index 0000000000000000000000000000000000000000..a2cad49964e11da10049ee82e8320d36fca01121 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mahalanobis/mahalanobis.py @@ -0,0 +1,100 @@ +# Copyright 2021 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Mahalanobis metric.""" + +import datasets +import numpy as np + +import evaluate + + +_DESCRIPTION = """ +Compute the Mahalanobis Distance + +Mahalonobis distance is the distance between a point and a distribution. +And not between two distinct points. It is effectively a multivariate equivalent of the Euclidean distance. +It was introduced by Prof. P. C. Mahalanobis in 1936 +and has been used in various statistical applications ever since +[source: https://www.machinelearningplus.com/statistics/mahalanobis-distance/] +""" + +_CITATION = """\ +@article{de2000mahalanobis, + title={The mahalanobis distance}, + author={De Maesschalck, Roy and Jouan-Rimbaud, Delphine and Massart, D{\'e}sir{\'e} L}, + journal={Chemometrics and intelligent laboratory systems}, + volume={50}, + number={1}, + pages={1--18}, + year={2000}, + publisher={Elsevier} +} +""" + +_KWARGS_DESCRIPTION = """ +Args: + X: List of datapoints to be compared with the `reference_distribution`. + reference_distribution: List of datapoints from the reference distribution we want to compare to. +Returns: + mahalanobis: The Mahalonobis distance for each datapoint in `X`. +Examples: + + >>> mahalanobis_metric = evaluate.load("mahalanobis") + >>> results = mahalanobis_metric.compute(reference_distribution=[[0, 1], [1, 0]], X=[[0, 1]]) + >>> print(results) + {'mahalanobis': array([0.5])} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Mahalanobis(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "X": datasets.Sequence(datasets.Value("float", id="sequence"), id="X"), + } + ), + ) + + def _compute(self, X, reference_distribution): + + # convert to numpy arrays + X = np.array(X) + reference_distribution = np.array(reference_distribution) + + # Assert that arrays are 2D + if len(X.shape) != 2: + raise ValueError("Expected `X` to be a 2D vector") + if len(reference_distribution.shape) != 2: + raise ValueError("Expected `reference_distribution` to be a 2D vector") + if reference_distribution.shape[0] < 2: + raise ValueError( + "Expected `reference_distribution` to be a 2D vector with more than one element in the first dimension" + ) + + # Get mahalanobis distance for each prediction + X_minus_mu = X - np.mean(reference_distribution) + cov = np.cov(reference_distribution.T) + try: + inv_covmat = np.linalg.inv(cov) + except np.linalg.LinAlgError: + inv_covmat = np.linalg.pinv(cov) + left_term = np.dot(X_minus_mu, inv_covmat) + mahal_dist = np.dot(left_term, X_minus_mu.T).diagonal() + + return {"mahalanobis": mahal_dist} diff --git a/TextClassification/TextClassification/evaluate/metrics/mahalanobis/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/mahalanobis/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a135afacbdd142a40cb9bcfe6073b76cc62d995 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mahalanobis/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/mape/README.md b/TextClassification/TextClassification/evaluate/metrics/mape/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f430a881a23934612e8001c720bc82ac476766c6 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mape/README.md @@ -0,0 +1,129 @@ +--- +title: MAPE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Mean Absolute Percentage Error (MAPE) is the mean percentage error difference between the predicted and actual + values. +--- + +# Metric Card for MAPE + + +## Metric Description + +Mean Absolute Error (MAPE) is the mean of the percentage error of difference between the predicted $x_i$ and actual $y_i$ numeric values: +![image](https://user-images.githubusercontent.com/8100/200005316-c3975d32-8978-40f3-b541-c2ef57ec7c5b.png) + +## How to Use + +At minimum, this metric requires predictions and references as inputs. + +```python +>>> mape_metric = evaluate.load("mape") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> results = mape_metric.compute(predictions=predictions, references=references) +``` + +### Inputs + +Mandatory inputs: +- `predictions`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the estimated target values. +- `references`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the ground truth (correct) target values. + +Optional arguments: +- `sample_weight`: numeric array-like of shape (`n_samples,`) representing sample weights. The default is `None`. +- `multioutput`: `raw_values`, `uniform_average` or numeric array-like of shape (`n_outputs,`), which defines the aggregation of multiple output values. The default value is `uniform_average`. + - `raw_values` returns a full set of errors in case of multioutput input. + - `uniform_average` means that the errors of all outputs are averaged with uniform weight. + - the array-like value defines weights used to average errors. + +### Output Values +This metric outputs a dictionary, containing the mean absolute error score, which is of type: +- `float`: if multioutput is `uniform_average` or an ndarray of weights, then the weighted average of all output errors is returned. +- numeric array-like of shape (`n_outputs,`): if multioutput is `raw_values`, then the score is returned for each output separately. + +Each MAPE `float` value is postive with the best value being 0.0. + +Output Example(s): +```python +{'mape': 0.5} +``` + +If `multioutput="raw_values"`: +```python +{'mape': array([0.5, 1. ])} +``` + +#### Values from Popular Papers + + +### Examples + +Example with the `uniform_average` config: +```python +>>> mape_metric = evaluate.load("mape") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> results = mape_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'mape': 0.3273...} +``` + +Example with multi-dimensional lists, and the `raw_values` config: +```python +>>> mape_metric = evaluate.load("mape", "multilist") +>>> predictions = [[0.5, 1], [-1, 1], [7, -6]] +>>> references = [[0.1, 2], [-1, 2], [8, -5]] +>>> results = mape_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'mape': 0.8874...} +>>> results = mape_metric.compute(predictions=predictions, references=references, multioutput='raw_values') +>>> print(results) +{'mape': array([1.3749..., 0.4])} +``` + +## Limitations and Bias +One limitation of MAPE is that it cannot be used if the ground truth is zero or close to zero. This metric is also asymmetric in that it puts a heavier penalty on predictions less than the ground truth and a smaller penalty on predictions bigger than the ground truth and thus can lead to a bias of methods being select which under-predict if selected via this metric. + +## Citation(s) +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + +```bibtex +@article{DEMYTTENAERE201638, + title = {Mean Absolute Percentage Error for regression models}, + journal = {Neurocomputing}, + volume = {192}, + pages = {38--48}, + year = {2016}, + note = {Advances in artificial neural networks, machine learning and computational intelligence}, + issn = {0925-2312}, + doi = {https://doi.org/10.1016/j.neucom.2015.12.114}, + url = {https://www.sciencedirect.com/science/article/pii/S0925231216003325}, + author = {Arnaud {de Myttenaere} and Boris Golden and Bénédicte {Le Grand} and Fabrice Rossi}, +} +``` + +## Further References +- [Mean absolute percentage error - Wikipedia](https://en.wikipedia.org/wiki/Mean_absolute_percentage_error) diff --git a/TextClassification/TextClassification/evaluate/metrics/mape/app.py b/TextClassification/TextClassification/evaluate/metrics/mape/app.py new file mode 100644 index 0000000000000000000000000000000000000000..8d8d20d70e85b33e0aacdf90117dac45054f6550 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mape/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("mape") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/mape/mape.py b/TextClassification/TextClassification/evaluate/metrics/mape/mape.py new file mode 100644 index 0000000000000000000000000000000000000000..b9c52cb3cd6f32408c14c163f5a16b47c141fbd1 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mape/mape.py @@ -0,0 +1,118 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MAPE - Mean Absolute Percentage Error Metric""" + +import datasets +from sklearn.metrics import mean_absolute_percentage_error + +import evaluate + + +_CITATION = """\ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + +_DESCRIPTION = """\ +Mean Absolute Percentage Error (MAPE) is the mean percentage error difference between the predicted and actual +values. +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions: array-like of shape (n_samples,) or (n_samples, n_outputs) + Estimated target values. + references: array-like of shape (n_samples,) or (n_samples, n_outputs) + Ground truth (correct) target values. + sample_weight: array-like of shape (n_samples,), default=None + Sample weights. + multioutput: {"raw_values", "uniform_average"} or array-like of shape (n_outputs,), default="uniform_average" + Defines aggregating of multiple output values. Array-like value defines weights used to average errors. + + "raw_values" : Returns a full set of errors in case of multioutput input. + + "uniform_average" : Errors of all outputs are averaged with uniform weight. + +Returns: + mape : mean absolute percentage error. + If multioutput is "raw_values", then mean absolute percentage error is returned for each output separately. If multioutput is "uniform_average" or an ndarray of weights, then the weighted average of all output errors is returned. + MAPE output is non-negative floating point. The best value is 0.0. +Examples: + + >>> mape_metric = evaluate.load("mape") + >>> predictions = [2.5, 0.0, 2, 8] + >>> references = [3, -0.5, 2, 7] + >>> results = mape_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'mape': 0.3273809523809524} + + If you're using multi-dimensional lists, then set the config as follows : + + >>> mape_metric = evaluate.load("mape", "multilist") + >>> predictions = [[0.5, 1], [-1, 1], [7, -6]] + >>> references = [[0.1, 2], [-1, 2], [8, -5]] + >>> results = mape_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'mape': 0.8874999875823658} + >>> results = mape_metric.compute(predictions=predictions, references=references, multioutput='raw_values') + >>> print(results) + {'mape': array([1.37499998, 0.4 ])} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Mape(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features(self._get_feature_types()), + reference_urls=[ + "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_absolute_percentage_error.html" + ], + ) + + def _get_feature_types(self): + if self.config_name == "multilist": + return { + "predictions": datasets.Sequence(datasets.Value("float")), + "references": datasets.Sequence(datasets.Value("float")), + } + else: + return { + "predictions": datasets.Value("float"), + "references": datasets.Value("float"), + } + + def _compute(self, predictions, references, sample_weight=None, multioutput="uniform_average"): + + mape_score = mean_absolute_percentage_error( + references, + predictions, + sample_weight=sample_weight, + multioutput=multioutput, + ) + + return {"mape": mape_score} diff --git a/TextClassification/TextClassification/evaluate/metrics/mape/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/mape/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..c9c2c1e3344c2dec052782f879af2e3a1f065df8 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mape/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn diff --git a/TextClassification/TextClassification/evaluate/metrics/mase/README.md b/TextClassification/TextClassification/evaluate/metrics/mase/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bded6399cb98326bc04e85797f45a327e21333ea --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mase/README.md @@ -0,0 +1,127 @@ +--- +title: MASE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Mean Absolute Scaled Error (MASE) is the mean absolute error of the forecast values, divided by the mean absolute error of the in-sample one-step naive forecast on the training set. +--- + +# Metric Card for MASE + +## Metric Description + +Mean Absolute Scaled Error (MASE) is the mean absolute error of the forecast values, divided by the mean absolute error of the in-sample one-step naive forecast. For prediction $x_i$ and corresponding ground truth $y_i$ as well as training data $z_t$ with seasonality $p$ the metric is given by: +![image](https://user-images.githubusercontent.com/8100/200009284-7ce4ccaa-373c-42f0-acbb-f81d52a97512.png) + + +This metric is: +* independent of the scale of the data; +* has predictable behavior when predicted/ground-truth data is near zero; +* symmetric; +* interpretable, as values greater than one indicate that in-sample one-step forecasts from the naïve method perform better than the forecast values under consideration. + + +## How to Use + +At minimum, this metric requires predictions, references and training data as inputs. + +```python +>>> mase_metric = evaluate.load("mase") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> training = [5, 0.5, 4, 6, 3, 5, 2] +>>> results = mase_metric.compute(predictions=predictions, references=references, training=training) +``` + +### Inputs + +Mandatory inputs: +- `predictions`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the estimated target values. +- `references`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the ground truth (correct) target values. +- `training`: numeric array-like of shape (`n_train_samples,`) or (`n_train_samples`, `n_outputs`), representing the in sample training data. + +Optional arguments: +- `periodicity`: the seasonal periodicity of training data. The default is 1. +- `sample_weight`: numeric array-like of shape (`n_samples,`) representing sample weights. The default is `None`. +- `multioutput`: `raw_values`, `uniform_average` or numeric array-like of shape (`n_outputs,`), which defines the aggregation of multiple output values. The default value is `uniform_average`. + - `raw_values` returns a full set of errors in case of multioutput input. + - `uniform_average` means that the errors of all outputs are averaged with uniform weight. + - the array-like value defines weights used to average errors. + +### Output Values +This metric outputs a dictionary, containing the mean absolute error score, which is of type: +- `float`: if multioutput is `uniform_average` or an ndarray of weights, then the weighted average of all output errors is returned. +- numeric array-like of shape (`n_outputs,`): if multioutput is `raw_values`, then the score is returned for each output separately. + +Each MASE `float` value ranges from `0.0` to `1.0`, with the best value being 0.0. + +Output Example(s): +```python +{'mase': 0.5} +``` + +If `multioutput="raw_values"`: +```python +{'mase': array([0.5, 1. ])} +``` + +#### Values from Popular Papers + + +### Examples + +Example with the `uniform_average` config: +```python +>>> mase_metric = evaluate.load("mase") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> training = [5, 0.5, 4, 6, 3, 5, 2] +>>> results = mase_metric.compute(predictions=predictions, references=references, training=training) +>>> print(results) +{'mase': 0.1833...} +``` + +Example with multi-dimensional lists, and the `raw_values` config: +```python +>>> mase_metric = evaluate.load("mase", "multilist") +>>> predictions = [[0.5, 1], [-1, 1], [7, -6]] +>>> references = [[0.1, 2], [-1, 2], [8, -5]] +>>> training = [[0.5, 1], [-1, 1], [7, -6]] +>>> results = mase_metric.compute(predictions=predictions, references=references, training=training) +>>> print(results) +{'mase': 0.1818...} +>>> results = mase_metric.compute(predictions=predictions, references=references, training=training, multioutput='raw_values') +>>> print(results) +{'mase': array([0.1052..., 0.2857...])} +``` + +## Limitations and Bias + + +## Citation(s) + +```bibtex +@article{HYNDMAN2006679, + title = {Another look at measures of forecast accuracy}, + journal = {International Journal of Forecasting}, + volume = {22}, + number = {4}, + pages = {679--688}, + year = {2006}, + issn = {0169-2070}, + doi = {https://doi.org/10.1016/j.ijforecast.2006.03.001}, + url = {https://www.sciencedirect.com/science/article/pii/S0169207006000239}, + author = {Rob J. Hyndman and Anne B. Koehler}, +} +``` + +## Further References +- [Mean absolute scaled error - Wikipedia](https://en.wikipedia.org/wiki/Mean_absolute_scaled_errorr) diff --git a/TextClassification/TextClassification/evaluate/metrics/mase/app.py b/TextClassification/TextClassification/evaluate/metrics/mase/app.py new file mode 100644 index 0000000000000000000000000000000000000000..ac47c0d679868dd50c8b5476a1e53d721c429b90 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mase/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("mase") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/mase/mase.py b/TextClassification/TextClassification/evaluate/metrics/mase/mase.py new file mode 100644 index 0000000000000000000000000000000000000000..633d2e0140474066e50609bae578387d4998369a --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mase/mase.py @@ -0,0 +1,140 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MASE - Mean Absolute Scaled Error Metric""" + +import datasets +import numpy as np +from sklearn.metrics import mean_absolute_error + +import evaluate + + +_CITATION = """\ +@article{HYNDMAN2006679, + title = {Another look at measures of forecast accuracy}, + journal = {International Journal of Forecasting}, + volume = {22}, + number = {4}, + pages = {679--688}, + year = {2006}, + issn = {0169-2070}, + doi = {https://doi.org/10.1016/j.ijforecast.2006.03.001}, + url = {https://www.sciencedirect.com/science/article/pii/S0169207006000239}, + author = {Rob J. Hyndman and Anne B. Koehler}, +} +""" + +_DESCRIPTION = """\ +Mean Absolute Scaled Error (MASE) is the mean absolute error of the forecast values, divided by the mean absolute error of the in-sample one-step naive forecast. +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions: array-like of shape (n_samples,) or (n_samples, n_outputs) + Estimated target values. + references: array-like of shape (n_samples,) or (n_samples, n_outputs) + Ground truth (correct) target values. + training: array-like of shape (n_train_samples,) or (n_train_samples, n_outputs) + In sample training data for naive forecast. + periodicity: int, default=1 + Seasonal periodicity of training data. + sample_weight: array-like of shape (n_samples,), default=None + Sample weights. + multioutput: {"raw_values", "uniform_average"} or array-like of shape (n_outputs,), default="uniform_average" + Defines aggregating of multiple output values. Array-like value defines weights used to average errors. + + "raw_values" : Returns a full set of errors in case of multioutput input. + + "uniform_average" : Errors of all outputs are averaged with uniform weight. + +Returns: + mase : mean absolute scaled error. + If multioutput is "raw_values", then mean absolute percentage error is returned for each output separately. If multioutput is "uniform_average" or an ndarray of weights, then the weighted average of all output errors is returned. + MASE output is non-negative floating point. The best value is 0.0. +Examples: + + >>> mase_metric = evaluate.load("mase") + >>> predictions = [2.5, 0.0, 2, 8, 1.25] + >>> references = [3, -0.5, 2, 7, 2] + >>> training = [5, 0.5, 4, 6, 3, 5, 2] + >>> results = mase_metric.compute(predictions=predictions, references=references, training=training) + >>> print(results) + {'mase': 0.18333333333333335} + + If you're using multi-dimensional lists, then set the config as follows : + + >>> mase_metric = evaluate.load("mase", "multilist") + >>> predictions = [[0, 2], [-1, 2], [8, -5]] + >>> references = [[0.5, 1], [-1, 1], [7, -6]] + >>> training = [[0.5, 1], [-1, 1], [7, -6]] + >>> results = mase_metric.compute(predictions=predictions, references=references, training=training) + >>> print(results) + {'mase': 0.18181818181818182} + >>> results = mase_metric.compute(predictions=predictions, references=references, training=training, multioutput='raw_values') + >>> print(results) + {'mase': array([0.10526316, 0.28571429])} + >>> results = mase_metric.compute(predictions=predictions, references=references, training=training, multioutput=[0.3, 0.7]) + >>> print(results) + {'mase': 0.21935483870967742} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Mase(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features(self._get_feature_types()), + reference_urls=["https://otexts.com/fpp3/accuracy.html#scaled-errors"], + ) + + def _get_feature_types(self): + if self.config_name == "multilist": + return { + "predictions": datasets.Sequence(datasets.Value("float")), + "references": datasets.Sequence(datasets.Value("float")), + } + else: + return { + "predictions": datasets.Value("float"), + "references": datasets.Value("float"), + } + + def _compute( + self, + predictions, + references, + training, + periodicity=1, + sample_weight=None, + multioutput="uniform_average", + ): + + y_pred_naive = training[:-periodicity] + mae_naive = mean_absolute_error(training[periodicity:], y_pred_naive, multioutput=multioutput) + + mae_score = mean_absolute_error( + references, + predictions, + sample_weight=sample_weight, + multioutput=multioutput, + ) + + epsilon = np.finfo(np.float64).eps + mase_score = mae_score / np.maximum(mae_naive, epsilon) + + return {"mase": mase_score} diff --git a/TextClassification/TextClassification/evaluate/metrics/mase/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/mase/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..c9c2c1e3344c2dec052782f879af2e3a1f065df8 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mase/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn diff --git a/TextClassification/TextClassification/evaluate/metrics/matthews_correlation/README.md b/TextClassification/TextClassification/evaluate/metrics/matthews_correlation/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1876358cb3d65e64fab0e3ef5c79531df3f285e6 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/matthews_correlation/README.md @@ -0,0 +1,117 @@ +--- +title: Matthews Correlation Coefficient +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Compute the Matthews correlation coefficient (MCC) + + The Matthews correlation coefficient is used in machine learning as a + measure of the quality of binary and multiclass classifications. It takes + into account true and false positives and negatives and is generally + regarded as a balanced measure which can be used even if the classes are of + very different sizes. The MCC is in essence a correlation coefficient value + between -1 and +1. A coefficient of +1 represents a perfect prediction, 0 + an average random prediction and -1 an inverse prediction. The statistic + is also known as the phi coefficient. [source: Wikipedia] +--- + +# Metric Card for Matthews Correlation Coefficient + +## Metric Description +The Matthews correlation coefficient is used in machine learning as a +measure of the quality of binary and multiclass classifications. It takes +into account true and false positives and negatives and is generally +regarded as a balanced measure which can be used even if the classes are of +very different sizes. The MCC is in essence a correlation coefficient value +between -1 and +1. A coefficient of +1 represents a perfect prediction, 0 +an average random prediction and -1 an inverse prediction. The statistic +is also known as the phi coefficient. [source: Wikipedia] + +## How to Use +At minimum, this metric requires a list of predictions and a list of references: +```python +>>> matthews_metric = evaluate.load("matthews_correlation") +>>> results = matthews_metric.compute(references=[0, 1], predictions=[0, 1]) +>>> print(results) +{'matthews_correlation': 1.0} +``` + +### Inputs +- **`predictions`** (`list` of `int`s): Predicted class labels. +- **`references`** (`list` of `int`s): Ground truth labels. +- **`sample_weight`** (`list` of `int`s, `float`s, or `bool`s): Sample weights. Defaults to `None`. +- **`average`**(`None` or `macro`): For the multilabel case, whether to return one correlation coefficient per feature (`average=None`), or the average of them (`average='macro'`). Defaults to `None`. + +### Output Values +- **`matthews_correlation`** (`float` or `list` of `float`s): Matthews correlation coefficient, or list of them in the multilabel case without averaging. + +The metric output takes the following form: +```python +{'matthews_correlation': 0.54} +``` + +This metric can be any value from -1 to +1, inclusive. + +#### Values from Popular Papers + + +### Examples +A basic example with only predictions and references as inputs: +```python +>>> matthews_metric = evaluate.load("matthews_correlation") +>>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2], +... predictions=[1, 2, 2, 0, 3, 3]) +>>> print(results) +{'matthews_correlation': 0.5384615384615384} +``` + +The same example as above, but also including sample weights: +```python +>>> matthews_metric = evaluate.load("matthews_correlation") +>>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2], +... predictions=[1, 2, 2, 0, 3, 3], +... sample_weight=[0.5, 3, 1, 1, 1, 2]) +>>> print(results) +{'matthews_correlation': 0.09782608695652174} +``` + +The same example as above, with sample weights that cause a negative correlation: +```python +>>> matthews_metric = evaluate.load("matthews_correlation") +>>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2], +... predictions=[1, 2, 2, 0, 3, 3], +... sample_weight=[0.5, 1, 0, 0, 0, 1]) +>>> print(results) +{'matthews_correlation': -0.25} +``` + +## Limitations and Bias +*Note any limitations or biases that the metric has.* + + +## Citation +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + +## Further References + +- This Hugging Face implementation uses [this scikit-learn implementation](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.matthews_corrcoef.html) \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/matthews_correlation/app.py b/TextClassification/TextClassification/evaluate/metrics/matthews_correlation/app.py new file mode 100644 index 0000000000000000000000000000000000000000..b9d1018fbc378cd089a599d63f74e4b05bc10147 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/matthews_correlation/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("matthews_correlation") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/matthews_correlation/matthews_correlation.py b/TextClassification/TextClassification/evaluate/metrics/matthews_correlation/matthews_correlation.py new file mode 100644 index 0000000000000000000000000000000000000000..58a477a928113558a68c019540daadd206d56458 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/matthews_correlation/matthews_correlation.py @@ -0,0 +1,140 @@ +# Copyright 2021 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Matthews Correlation metric.""" + +import datasets +import numpy as np +from sklearn.metrics import matthews_corrcoef + +import evaluate + + +_DESCRIPTION = """ +Compute the Matthews correlation coefficient (MCC) + +The Matthews correlation coefficient is used in machine learning as a +measure of the quality of binary and multiclass classifications. It takes +into account true and false positives and negatives and is generally +regarded as a balanced measure which can be used even if the classes are of +very different sizes. The MCC is in essence a correlation coefficient value +between -1 and +1. A coefficient of +1 represents a perfect prediction, 0 +an average random prediction and -1 an inverse prediction. The statistic +is also known as the phi coefficient. [source: Wikipedia] +""" + +_KWARGS_DESCRIPTION = """ +Args: + predictions (list of int): Predicted labels, as returned by a model. + references (list of int): Ground truth labels. + average (`string`): This parameter is used for multilabel configs. Defaults to `None`. + - None (default): Returns an array of Matthews correlation coefficients, one for each feature + - 'macro': Calculate metrics for each feature, and find their unweighted mean. + sample_weight (list of int, float, or bool): Sample weights. Defaults to `None`. +Returns: + matthews_correlation (dict containing float): Matthews correlation. +Examples: + Example 1, a basic example with only predictions and references as inputs: + >>> matthews_metric = evaluate.load("matthews_correlation") + >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2], + ... predictions=[1, 2, 2, 0, 3, 3]) + >>> print(round(results['matthews_correlation'], 2)) + 0.54 + + Example 2, the same example as above, but also including sample weights: + >>> matthews_metric = evaluate.load("matthews_correlation") + >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2], + ... predictions=[1, 2, 2, 0, 3, 3], + ... sample_weight=[0.5, 3, 1, 1, 1, 2]) + >>> print(round(results['matthews_correlation'], 2)) + 0.1 + + Example 3, the same example as above, but with sample weights that cause a negative correlation: + >>> matthews_metric = evaluate.load("matthews_correlation") + >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2], + ... predictions=[1, 2, 2, 0, 3, 3], + ... sample_weight=[0.5, 1, 0, 0, 0, 1]) + >>> print(round(results['matthews_correlation'], 2)) + -0.25 + + Example 4, Multi-label without averaging: + >>> matthews_metric = evaluate.load("matthews_correlation", config_name="multilabel") + >>> results = matthews_metric.compute(references=[[0,1], [1,0], [1,1]], + ... predictions=[[0,1], [1,1], [0,1]]) + >>> print(results['matthews_correlation']) + [0.5, 0.0] + + Example 5, Multi-label with averaging: + >>> matthews_metric = evaluate.load("matthews_correlation", config_name="multilabel") + >>> results = matthews_metric.compute(references=[[0,1], [1,0], [1,1]], + ... predictions=[[0,1], [1,1], [0,1]], + ... average='macro') + >>> print(round(results['matthews_correlation'], 2)) + 0.25 +""" + +_CITATION = """\ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class MatthewsCorrelation(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("int32")), + "references": datasets.Sequence(datasets.Value("int32")), + } + if self.config_name == "multilabel" + else { + "predictions": datasets.Value("int32"), + "references": datasets.Value("int32"), + } + ), + reference_urls=[ + "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.matthews_corrcoef.html" + ], + ) + + def _compute(self, predictions, references, sample_weight=None, average=None): + if self.config_name == "multilabel": + references = np.array(references) + predictions = np.array(predictions) + if not (references.ndim == 2 and predictions.ndim == 2): + raise ValueError("For multi-label inputs, both references and predictions should be 2-dimensional") + matthews_corr = [ + matthews_corrcoef(predictions[:, i], references[:, i], sample_weight=sample_weight) + for i in range(references.shape[1]) + ] + if average == "macro": + matthews_corr = np.mean(matthews_corr) + elif average is not None: + raise ValueError("Invalid `average`: expected `macro`, or None ") + else: + matthews_corr = float(matthews_corrcoef(references, predictions, sample_weight=sample_weight)) + return {"matthews_correlation": matthews_corr} diff --git a/TextClassification/TextClassification/evaluate/metrics/matthews_correlation/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/matthews_correlation/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/matthews_correlation/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/mauve/README.md b/TextClassification/TextClassification/evaluate/metrics/mauve/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ba39d8fad7a96dc09227b1bae7ff849cd05f165 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mauve/README.md @@ -0,0 +1,144 @@ +--- +title: MAUVE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + MAUVE is a measure of the statistical gap between two text distributions, e.g., how far the text written by a model is the distribution of human text, using samples from both distributions. + + MAUVE is obtained by computing Kullback–Leibler (KL) divergences between the two distributions in a quantized embedding space of a large language model. It can quantify differences in the quality of generated text based on the size of the model, the decoding algorithm, and the length of the generated text. MAUVE was found to correlate the strongest with human evaluations over baseline metrics for open-ended text generation. + +--- + +# Metric Card for MAUVE + +## Metric description + +MAUVE is a measure of the gap between neural text and human text. It is computed using the [Kullback–Leibler (KL) divergences](https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence) between the two distributions of text in a quantized embedding space of a large language model. MAUVE can identify differences in quality arising from model sizes and decoding algorithms. + +This metric is a wrapper around the [official implementation](https://github.com/krishnap25/mauve) of MAUVE. + +For more details, consult the [MAUVE paper](https://arxiv.org/abs/2102.01454). + +## How to use + +The metric takes two lists of strings of tokens separated by spaces: one representing `predictions` (i.e. the text generated by the model) and the second representing `references` (a reference text for each prediction): + +```python +from evaluate import load +mauve = load('mauve') +predictions = ["hello world", "goodnight moon"] +references = ["hello world", "goodnight moon"] +mauve_results = mauve.compute(predictions=predictions, references=references) +``` + +It also has several optional arguments: + +`num_buckets`: the size of the histogram to quantize P and Q. Options: `auto` (default) or an integer. + +`pca_max_data`: the number of data points to use for PCA dimensionality reduction prior to clustering. If -1, use all the data. The default is `-1`. + +`kmeans_explained_var`: the amount of variance of the data to keep in dimensionality reduction by PCA. The default is `0.9`. + +`kmeans_num_redo`: number of times to redo k-means clustering (the best objective is kept). The default is `5`. + +`kmeans_max_iter`: maximum number of k-means iterations. The default is `500`. + +`featurize_model_name`: name of the model from which features are obtained, from one of the following: `gpt2`, `gpt2-medium`, `gpt2-large`, `gpt2-xl`. The default is `gpt2-large`. + +`device_id`: Device for featurization. Supply a GPU id (e.g. `0` or `3`) to use GPU. If no GPU with this id is found, the metric will use CPU. + +`max_text_length`: maximum number of tokens to consider. The default is `1024`. + +`divergence_curve_discretization_size` Number of points to consider on the divergence curve. The default is `25`. + +`mauve_scaling_factor`: Hyperparameter for scaling. The default is `5`. + +`verbose`: If `True` (default), running the metric will print running time updates. + +`seed`: random seed to initialize k-means cluster assignments, randomly assigned by default. + + + +## Output values + +This metric outputs a dictionary with 5 key-value pairs: + +`mauve`: MAUVE score, which ranges between 0 and 1. **Larger** values indicate that P and Q are closer. + +`frontier_integral`: Frontier Integral, which ranges between 0 and 1. **Smaller** values indicate that P and Q are closer. + +`divergence_curve`: a numpy.ndarray of shape (m, 2); plot it with `matplotlib` to view the divergence curve. + +`p_hist`: a discrete distribution, which is a quantized version of the text distribution `p_text`. + +`q_hist`: same as above, but with `q_text`. + + +### Values from popular papers + +The [original MAUVE paper](https://arxiv.org/abs/2102.01454) reported values ranging from 0.88 to 0.94 for open-ended text generation using a text completion task in the web text domain. The authors found that bigger models resulted in higher MAUVE scores and that MAUVE is correlated with human judgments. + + +## Examples + +Perfect match between prediction and reference: + +```python +from evaluate import load +mauve = load('mauve') +predictions = ["hello world", "goodnight moon"] +references = ["hello world", "goodnight moon"] +mauve_results = mauve.compute(predictions=predictions, references=references) +print(mauve_results.mauve) +1.0 +``` + +Partial match between prediction and reference: + +```python +from evaluate import load +mauve = load('mauve') +predictions = ["hello world", "goodnight moon"] +references = ["hello there", "general kenobi"] +mauve_results = mauve.compute(predictions=predictions, references=references) +print(mauve_results.mauve) +0.27811372536724027 +``` + +## Limitations and bias + +The [original MAUVE paper](https://arxiv.org/abs/2102.01454) did not analyze the inductive biases present in different embedding models, but related work has shown different kinds of biases exist in many popular generative language models including GPT-2 (see [Kirk et al., 2021](https://arxiv.org/pdf/2102.04130.pdf), [Abid et al., 2021](https://arxiv.org/abs/2101.05783)). The extent to which these biases can impact the MAUVE score has not been quantified. + +Also, calculating the MAUVE metric involves downloading the model from which features are obtained -- the default model, `gpt2-large`, takes over 3GB of storage space and downloading it can take a significant amount of time depending on the speed of your internet connection. If this is an issue, choose a smaller model; for instance, `gpt` is 523MB. + +It is a good idea to use at least 1000 samples for each distribution to compute MAUVE (the original paper uses 5000). + +MAUVE is unable to identify very small differences between different settings of generation (e.g., between top-p sampling with p=0.95 versus 0.96). It is important, therefore, to account for the randomness inside the generation (e.g., due to sampling) and within the MAUVE estimation procedure (see the `seed` parameter above). Concretely, it is a good idea to obtain generations using multiple random seeds and/or to use rerun MAUVE with multiple values of the parameter `seed`. + +For MAUVE to be large, the model distribution must be close to the human text distribution as seen by the embeddings. It is possible to have high-quality model text that still has a small MAUVE score (i.e., large gap) if it contains text about different topics/subjects, or uses a different writing style or vocabulary, or contains texts of a different length distribution. MAUVE summarizes the statistical gap (as measured by the large language model embeddings) --- this includes all these factors in addition to the quality-related aspects such as grammaticality. + +See the [official implementation](https://github.com/krishnap25/mauve#best-practices-for-mauve) for more details about best practices. + + +## Citation + +```bibtex +@inproceedings{pillutla-etal:mauve:neurips2021, + title={MAUVE: Measuring the Gap Between Neural Text and Human Text using Divergence Frontiers}, + author={Pillutla, Krishna and Swayamdipta, Swabha and Zellers, Rowan and Thickstun, John and Welleck, Sean and Choi, Yejin and Harchaoui, Zaid}, + booktitle = {NeurIPS}, + year = {2021} +} +``` + +## Further References +- [Official MAUVE implementation](https://github.com/krishnap25/mauve) +- [Hugging Face Tasks - Text Generation](https://huggingface.co/tasks/text-generation) diff --git a/TextClassification/TextClassification/evaluate/metrics/mauve/app.py b/TextClassification/TextClassification/evaluate/metrics/mauve/app.py new file mode 100644 index 0000000000000000000000000000000000000000..d356aef0ad1640efd4963d8bd33588b1442a40c0 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mauve/app.py @@ -0,0 +1,11 @@ +import sys + +import evaluate +from evaluate.utils import launch_gradio_widget + + +sys.path = [p for p in sys.path if p != "/home/user/app"] +module = evaluate.load("mauve") +sys.path = ["/home/user/app"] + sys.path + +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/mauve/mauve.py b/TextClassification/TextClassification/evaluate/metrics/mauve/mauve.py new file mode 100644 index 0000000000000000000000000000000000000000..fdacaa4771a5df52972a79948b73133fb1b472dd --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mauve/mauve.py @@ -0,0 +1,156 @@ +# coding=utf-8 +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" MAUVE metric from https://github.com/krishnap25/mauve. """ + +import datasets +import faiss # Here to have a nice missing dependency error message early on +import numpy # Here to have a nice missing dependency error message early on +import requests # Here to have a nice missing dependency error message early on +import sklearn # Here to have a nice missing dependency error message early on +import tqdm # Here to have a nice missing dependency error message early on +from mauve import compute_mauve # From: mauve-text + +import evaluate + + +_CITATION = """\ +@inproceedings{pillutla-etal:mauve:neurips2021, + title={{MAUVE: Measuring the Gap Between Neural Text and Human Text using Divergence Frontiers}}, + author={Pillutla, Krishna and Swayamdipta, Swabha and Zellers, Rowan and Thickstun, John and Welleck, Sean and Choi, Yejin and Harchaoui, Zaid}, + booktitle = {NeurIPS}, + year = {2021} +} + +@article{pillutla-etal:mauve:arxiv2022, + title={{MAUVE Scores for Generative Models: Theory and Practice}}, + author={Pillutla, Krishna and Liu, Lang and Thickstun, John and Welleck, Sean and Swayamdipta, Swabha and Zellers, Rowan and Oh, Sewoong and Choi, Yejin and Harchaoui, Zaid}, + journal={arXiv Preprint}, + year={2022} +} +""" + +_DESCRIPTION = """\ +MAUVE is a measure of the statistical gap between two text distributions, e.g., how far the text written by a model is the distribution of human text, using samples from both distributions. + +MAUVE is obtained by computing Kullback–Leibler (KL) divergences between the two distributions in a quantized embedding space of a large language model. +It can quantify differences in the quality of generated text based on the size of the model, the decoding algorithm, and the length of the generated text. +MAUVE was found to correlate the strongest with human evaluations over baseline metrics for open-ended text generation. + +This metrics is a wrapper around the official implementation of MAUVE: +https://github.com/krishnap25/mauve +""" + +_KWARGS_DESCRIPTION = """ +Calculates MAUVE scores between two lists of generated text and reference text. +Args: + predictions: list of generated text to score. Each predictions + should be a string with tokens separated by spaces. + references: list of reference for each prediction. Each + reference should be a string with tokens separated by spaces. +Optional Args: + num_buckets: the size of the histogram to quantize P and Q. Options: 'auto' (default) or an integer + pca_max_data: the number data points to use for PCA dimensionality reduction prior to clustering. If -1, use all the data. Default -1 + kmeans_explained_var: amount of variance of the data to keep in dimensionality reduction by PCA. Default 0.9 + kmeans_num_redo: number of times to redo k-means clustering (the best objective is kept). Default 5 + kmeans_max_iter: maximum number of k-means iterations. Default 500 + featurize_model_name: name of the model from which features are obtained. Default 'gpt2-large' Use one of ['gpt2', 'gpt2-medium', 'gpt2-large', 'gpt2-xl']. + device_id: Device for featurization. Supply a GPU id (e.g. 0 or 3) to use GPU. If no GPU with this id is found, use CPU + max_text_length: maximum number of tokens to consider. Default 1024 + divergence_curve_discretization_size: Number of points to consider on the divergence curve. Default 25 + mauve_scaling_factor: "c" from the paper. Default 5. + verbose: If True (default), print running time updates + seed: random seed to initialize k-means cluster assignments. +Returns: + mauve: MAUVE score, a number between 0 and 1. Larger values indicate that P and Q are closer, + frontier_integral: Frontier Integral, a number between 0 and 1. Smaller values indicate that P and Q are closer, + divergence_curve: a numpy.ndarray of shape (m, 2); plot it with matplotlib to view the divergence curve, + p_hist: a discrete distribution, which is a quantized version of the text distribution p_text, + q_hist: same as above, but with q_text. +Examples: + + >>> # faiss segfaults in doctest for some reason, so the .compute call is not tested with doctest + >>> import evaluate + >>> mauve = evaluate.load('mauve') + >>> predictions = ["hello there", "general kenobi"] + >>> references = ["hello there", "general kenobi"] + >>> out = mauve.compute(predictions=predictions, references=references) # doctest: +SKIP + >>> print(out.mauve) # doctest: +SKIP + 1.0 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Mauve(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + homepage="https://github.com/krishnap25/mauve", + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + codebase_urls=["https://github.com/krishnap25/mauve"], + reference_urls=[ + "https://arxiv.org/abs/2102.01454", + "https://github.com/krishnap25/mauve", + ], + ) + + def _compute( + self, + predictions, + references, + p_features=None, + q_features=None, + p_tokens=None, + q_tokens=None, + num_buckets="auto", + pca_max_data=-1, + kmeans_explained_var=0.9, + kmeans_num_redo=5, + kmeans_max_iter=500, + featurize_model_name="gpt2-large", + device_id=-1, + max_text_length=1024, + divergence_curve_discretization_size=25, + mauve_scaling_factor=5, + verbose=True, + seed=25, + ): + out = compute_mauve( + p_text=predictions, + q_text=references, + p_features=p_features, + q_features=q_features, + p_tokens=p_tokens, + q_tokens=q_tokens, + num_buckets=num_buckets, + pca_max_data=pca_max_data, + kmeans_explained_var=kmeans_explained_var, + kmeans_num_redo=kmeans_num_redo, + kmeans_max_iter=kmeans_max_iter, + featurize_model_name=featurize_model_name, + device_id=device_id, + max_text_length=max_text_length, + divergence_curve_discretization_size=divergence_curve_discretization_size, + mauve_scaling_factor=mauve_scaling_factor, + verbose=verbose, + seed=seed, + ) + return out diff --git a/TextClassification/TextClassification/evaluate/metrics/mauve/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/mauve/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..dbab93c719102f0a0a18219eb2a555396dda6967 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mauve/requirements.txt @@ -0,0 +1,4 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +faiss-cpu +scikit-learn +mauve-text \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/mean_iou/README.md b/TextClassification/TextClassification/evaluate/metrics/mean_iou/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9a34f2313f00170e9bd6483380bc52f5c12a8a51 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mean_iou/README.md @@ -0,0 +1,108 @@ +--- +title: Mean IoU +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + IoU is the area of overlap between the predicted segmentation and the ground truth divided by the area of union + between the predicted segmentation and the ground truth. For binary (two classes) or multi-class segmentation, + the mean IoU of the image is calculated by taking the IoU of each class and averaging them. +--- + +# Metric Card for Mean IoU + + +## Metric Description + +IoU (Intersection over Union) is the area of overlap between the predicted segmentation and the ground truth divided by the area of union between the predicted segmentation and the ground truth. + +For binary (two classes) or multi-class segmentation, the *mean IoU* of the image is calculated by taking the IoU of each class and averaging them. + +## How to Use + +The Mean IoU metric takes two lists of numeric 2D arrays as input corresponding to the predicted and ground truth segmentations: +```python +>>> import numpy as np +>>> mean_iou = evaluate.load("mean_iou") +>>> predicted = np.array([[2, 2, 3], [8, 2, 4], [3, 255, 2]]) +>>> ground_truth = np.array([[1, 2, 2], [8, 2, 1], [3, 255, 1]]) +>>> results = mean_iou.compute(predictions=[predicted], references=[ground_truth], num_labels=10, ignore_index=255) +``` + +### Inputs +**Mandatory inputs** +- `predictions` (`List[ndarray]`): List of predicted segmentation maps, each of shape (height, width). Each segmentation map can be of a different size. +- `references` (`List[ndarray]`): List of ground truth segmentation maps, each of shape (height, width). Each segmentation map can be of a different size. +- `num_labels` (`int`): Number of classes (categories). +- `ignore_index` (`int`): Index that will be ignored during evaluation. + +**Optional inputs** +- `nan_to_num` (`int`): If specified, NaN values will be replaced by the number defined by the user. +- `label_map` (`dict`): If specified, dictionary mapping old label indices to new label indices. +- `reduce_labels` (`bool`): Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background, and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255. The default value is `False`. + +### Output Values +The metric returns a dictionary with the following elements: +- `mean_iou` (`float`): Mean Intersection-over-Union (IoU averaged over all categories). +- `mean_accuracy` (`float`): Mean accuracy (averaged over all categories). +- `overall_accuracy` (`float`): Overall accuracy on all images. +- `per_category_accuracy` (`ndarray` of shape `(num_labels,)`): Per category accuracy. +- `per_category_iou` (`ndarray` of shape `(num_labels,)`): Per category IoU. + +The values of all of the scores reported range from from `0.0` (minimum) and `1.0` (maximum). + +Output Example: +```python +{'mean_iou': 0.47750000000000004, 'mean_accuracy': 0.5916666666666666, 'overall_accuracy': 0.5263157894736842, 'per_category_iou': array([0. , 0. , 0.375, 0.4 , 0.5 , 0. , 0.5 , 1. , 1. , 1. ]), 'per_category_accuracy': array([0. , 0. , 0.75 , 0.66666667, 1. , 0. , 0.5 , 1. , 1. , 1. ])} +``` + +#### Values from Popular Papers + +The [leaderboard for the CityScapes dataset](https://paperswithcode.com/sota/semantic-segmentation-on-cityscapes) reports a Mean IOU ranging from 64 to 84; that of [ADE20k](https://paperswithcode.com/sota/semantic-segmentation-on-ade20k) ranges from 30 to a peak of 59.9, indicating that the dataset is more difficult for current approaches (as of 2022). + + +### Examples + +```python +>>> import numpy as np +>>> mean_iou = evaluate.load("mean_iou") +>>> # suppose one has 3 different segmentation maps predicted +>>> predicted_1 = np.array([[1, 2], [3, 4], [5, 255]]) +>>> actual_1 = np.array([[0, 3], [5, 4], [6, 255]]) +>>> predicted_2 = np.array([[2, 7], [9, 2], [3, 6]]) +>>> actual_2 = np.array([[1, 7], [9, 2], [3, 6]]) +>>> predicted_3 = np.array([[2, 2, 3], [8, 2, 4], [3, 255, 2]]) +>>> actual_3 = np.array([[1, 2, 2], [8, 2, 1], [3, 255, 1]]) +>>> predictions = [predicted_1, predicted_2, predicted_3] +>>> references = [actual_1, actual_2, actual_3] +>>> results = mean_iou.compute(predictions=predictions, references=references, num_labels=10, ignore_index=255, reduce_labels=False) +>>> print(results) # doctest: +NORMALIZE_WHITESPACE +{'mean_iou': 0.47750000000000004, 'mean_accuracy': 0.5916666666666666, 'overall_accuracy': 0.5263157894736842, 'per_category_iou': array([0. , 0. , 0.375, 0.4 , 0.5 , 0. , 0.5 , 1. , 1. , 1. ]), 'per_category_accuracy': array([0. , 0. , 0.75 , 0.66666667, 1. , 0. , 0.5 , 1. , 1. , 1. ])} +``` + + +## Limitations and Bias +Mean IOU is an average metric, so it will not show you where model predictions differ from the ground truth (i.e. if there are particular regions or classes that the model does poorly on). Further error analysis is needed to gather actional insights that can be used to inform model improvements. + +## Citation(s) +```bibtex +@software{MMSegmentation_Contributors_OpenMMLab_Semantic_Segmentation_2020, +author = {{MMSegmentation Contributors}}, +license = {Apache-2.0}, +month = {7}, +title = {{OpenMMLab Semantic Segmentation Toolbox and Benchmark}}, +url = {https://github.com/open-mmlab/mmsegmentation}, +year = {2020} +}" +``` + + +## Further References +- [Wikipedia article - Jaccard Index](https://en.wikipedia.org/wiki/Jaccard_index) diff --git a/TextClassification/TextClassification/evaluate/metrics/mean_iou/app.py b/TextClassification/TextClassification/evaluate/metrics/mean_iou/app.py new file mode 100644 index 0000000000000000000000000000000000000000..5423d3579f8135484afdf5e24279b2f93df0e8d0 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mean_iou/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("mean_iou") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/mean_iou/mean_iou.py b/TextClassification/TextClassification/evaluate/metrics/mean_iou/mean_iou.py new file mode 100644 index 0000000000000000000000000000000000000000..4c19864d34e346a25a646d19ca27aecefc260061 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mean_iou/mean_iou.py @@ -0,0 +1,313 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Mean IoU (Intersection-over-Union) metric.""" + +from typing import Dict, Optional + +import datasets +import numpy as np + +import evaluate + + +_DESCRIPTION = """ +IoU is the area of overlap between the predicted segmentation and the ground truth divided by the area of union +between the predicted segmentation and the ground truth. For binary (two classes) or multi-class segmentation, +the mean IoU of the image is calculated by taking the IoU of each class and averaging them. +""" + +_KWARGS_DESCRIPTION = """ +Args: + predictions (`List[ndarray]`): + List of predicted segmentation maps, each of shape (height, width). Each segmentation map can be of a different size. + references (`List[ndarray]`): + List of ground truth segmentation maps, each of shape (height, width). Each segmentation map can be of a different size. + num_labels (`int`): + Number of classes (categories). + ignore_index (`int`): + Index that will be ignored during evaluation. + nan_to_num (`int`, *optional*): + If specified, NaN values will be replaced by the number defined by the user. + label_map (`dict`, *optional*): + If specified, dictionary mapping old label indices to new label indices. + reduce_labels (`bool`, *optional*, defaults to `False`): + Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background, + and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255. + +Returns: + `Dict[str, float | ndarray]` comprising various elements: + - *mean_iou* (`float`): + Mean Intersection-over-Union (IoU averaged over all categories). + - *mean_accuracy* (`float`): + Mean accuracy (averaged over all categories). + - *overall_accuracy* (`float`): + Overall accuracy on all images. + - *per_category_accuracy* (`ndarray` of shape `(num_labels,)`): + Per category accuracy. + - *per_category_iou* (`ndarray` of shape `(num_labels,)`): + Per category IoU. + +Examples: + + >>> import numpy as np + + >>> mean_iou = evaluate.load("mean_iou") + + >>> # suppose one has 3 different segmentation maps predicted + >>> predicted_1 = np.array([[1, 2], [3, 4], [5, 255]]) + >>> actual_1 = np.array([[0, 3], [5, 4], [6, 255]]) + + >>> predicted_2 = np.array([[2, 7], [9, 2], [3, 6]]) + >>> actual_2 = np.array([[1, 7], [9, 2], [3, 6]]) + + >>> predicted_3 = np.array([[2, 2, 3], [8, 2, 4], [3, 255, 2]]) + >>> actual_3 = np.array([[1, 2, 2], [8, 2, 1], [3, 255, 1]]) + + >>> predicted = [predicted_1, predicted_2, predicted_3] + >>> ground_truth = [actual_1, actual_2, actual_3] + + >>> results = mean_iou.compute(predictions=predicted, references=ground_truth, num_labels=10, ignore_index=255, reduce_labels=False) + >>> print(results) # doctest: +NORMALIZE_WHITESPACE + {'mean_iou': 0.47750000000000004, 'mean_accuracy': 0.5916666666666666, 'overall_accuracy': 0.5263157894736842, 'per_category_iou': array([0. , 0. , 0.375, 0.4 , 0.5 , 0. , 0.5 , 1. , 1. , 1. ]), 'per_category_accuracy': array([0. , 0. , 0.75 , 0.66666667, 1. , 0. , 0.5 , 1. , 1. , 1. ])} +""" + +_CITATION = """\ +@software{MMSegmentation_Contributors_OpenMMLab_Semantic_Segmentation_2020, +author = {{MMSegmentation Contributors}}, +license = {Apache-2.0}, +month = {7}, +title = {{OpenMMLab Semantic Segmentation Toolbox and Benchmark}}, +url = {https://github.com/open-mmlab/mmsegmentation}, +year = {2020} +}""" + + +def intersect_and_union( + pred_label, + label, + num_labels, + ignore_index: bool, + label_map: Optional[Dict[int, int]] = None, + reduce_labels: bool = False, +): + """Calculate intersection and Union. + + Args: + pred_label (`ndarray`): + Prediction segmentation map of shape (height, width). + label (`ndarray`): + Ground truth segmentation map of shape (height, width). + num_labels (`int`): + Number of categories. + ignore_index (`int`): + Index that will be ignored during evaluation. + label_map (`dict`, *optional*): + Mapping old labels to new labels. The parameter will work only when label is str. + reduce_labels (`bool`, *optional*, defaults to `False`): + Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background, + and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255. + + Returns: + area_intersect (`ndarray`): + The intersection of prediction and ground truth histogram on all classes. + area_union (`ndarray`): + The union of prediction and ground truth histogram on all classes. + area_pred_label (`ndarray`): + The prediction histogram on all classes. + area_label (`ndarray`): + The ground truth histogram on all classes. + """ + if label_map is not None: + for old_id, new_id in label_map.items(): + label[label == old_id] = new_id + + # turn into Numpy arrays + pred_label = np.array(pred_label) + label = np.array(label) + + if reduce_labels: + label[label == 0] = 255 + label = label - 1 + label[label == 254] = 255 + + mask = label != ignore_index + mask = np.not_equal(label, ignore_index) + pred_label = pred_label[mask] + label = np.array(label)[mask] + + intersect = pred_label[pred_label == label] + + area_intersect = np.histogram(intersect, bins=num_labels, range=(0, num_labels - 1))[0] + area_pred_label = np.histogram(pred_label, bins=num_labels, range=(0, num_labels - 1))[0] + area_label = np.histogram(label, bins=num_labels, range=(0, num_labels - 1))[0] + + area_union = area_pred_label + area_label - area_intersect + + return area_intersect, area_union, area_pred_label, area_label + + +def total_intersect_and_union( + results, + gt_seg_maps, + num_labels, + ignore_index: bool, + label_map: Optional[Dict[int, int]] = None, + reduce_labels: bool = False, +): + """Calculate Total Intersection and Union, by calculating `intersect_and_union` for each (predicted, ground truth) pair. + + Args: + results (`ndarray`): + List of prediction segmentation maps, each of shape (height, width). + gt_seg_maps (`ndarray`): + List of ground truth segmentation maps, each of shape (height, width). + num_labels (`int`): + Number of categories. + ignore_index (`int`): + Index that will be ignored during evaluation. + label_map (`dict`, *optional*): + Mapping old labels to new labels. The parameter will work only when label is str. + reduce_labels (`bool`, *optional*, defaults to `False`): + Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background, + and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255. + + Returns: + total_area_intersect (`ndarray`): + The intersection of prediction and ground truth histogram on all classes. + total_area_union (`ndarray`): + The union of prediction and ground truth histogram on all classes. + total_area_pred_label (`ndarray`): + The prediction histogram on all classes. + total_area_label (`ndarray`): + The ground truth histogram on all classes. + """ + total_area_intersect = np.zeros((num_labels,), dtype=np.float64) + total_area_union = np.zeros((num_labels,), dtype=np.float64) + total_area_pred_label = np.zeros((num_labels,), dtype=np.float64) + total_area_label = np.zeros((num_labels,), dtype=np.float64) + for result, gt_seg_map in zip(results, gt_seg_maps): + area_intersect, area_union, area_pred_label, area_label = intersect_and_union( + result, gt_seg_map, num_labels, ignore_index, label_map, reduce_labels + ) + total_area_intersect += area_intersect + total_area_union += area_union + total_area_pred_label += area_pred_label + total_area_label += area_label + return total_area_intersect, total_area_union, total_area_pred_label, total_area_label + + +def mean_iou( + results, + gt_seg_maps, + num_labels, + ignore_index: bool, + nan_to_num: Optional[int] = None, + label_map: Optional[Dict[int, int]] = None, + reduce_labels: bool = False, +): + """Calculate Mean Intersection and Union (mIoU). + + Args: + results (`ndarray`): + List of prediction segmentation maps, each of shape (height, width). + gt_seg_maps (`ndarray`): + List of ground truth segmentation maps, each of shape (height, width). + num_labels (`int`): + Number of categories. + ignore_index (`int`): + Index that will be ignored during evaluation. + nan_to_num (`int`, *optional*): + If specified, NaN values will be replaced by the number defined by the user. + label_map (`dict`, *optional*): + Mapping old labels to new labels. The parameter will work only when label is str. + reduce_labels (`bool`, *optional*, defaults to `False`): + Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background, + and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255. + + Returns: + `Dict[str, float | ndarray]` comprising various elements: + - *mean_iou* (`float`): + Mean Intersection-over-Union (IoU averaged over all categories). + - *mean_accuracy* (`float`): + Mean accuracy (averaged over all categories). + - *overall_accuracy* (`float`): + Overall accuracy on all images. + - *per_category_accuracy* (`ndarray` of shape `(num_labels,)`): + Per category accuracy. + - *per_category_iou* (`ndarray` of shape `(num_labels,)`): + Per category IoU. + """ + total_area_intersect, total_area_union, total_area_pred_label, total_area_label = total_intersect_and_union( + results, gt_seg_maps, num_labels, ignore_index, label_map, reduce_labels + ) + + # compute metrics + metrics = dict() + + all_acc = total_area_intersect.sum() / total_area_label.sum() + iou = total_area_intersect / total_area_union + acc = total_area_intersect / total_area_label + + metrics["mean_iou"] = np.nanmean(iou) + metrics["mean_accuracy"] = np.nanmean(acc) + metrics["overall_accuracy"] = all_acc + metrics["per_category_iou"] = iou + metrics["per_category_accuracy"] = acc + + if nan_to_num is not None: + metrics = dict( + {metric: np.nan_to_num(metric_value, nan=nan_to_num) for metric, metric_value in metrics.items()} + ) + + return metrics + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class MeanIoU(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Image(), + "references": datasets.Image(), + } + ), + reference_urls=[ + "https://github.com/open-mmlab/mmsegmentation/blob/71c201b1813267d78764f306a297ca717827c4bf/mmseg/core/evaluation/metrics.py" + ], + ) + + def _compute( + self, + predictions, + references, + num_labels: int, + ignore_index: bool, + nan_to_num: Optional[int] = None, + label_map: Optional[Dict[int, int]] = None, + reduce_labels: bool = False, + ): + iou_result = mean_iou( + results=predictions, + gt_seg_maps=references, + num_labels=num_labels, + ignore_index=ignore_index, + nan_to_num=nan_to_num, + label_map=label_map, + reduce_labels=reduce_labels, + ) + return iou_result diff --git a/TextClassification/TextClassification/evaluate/metrics/mean_iou/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/mean_iou/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a135afacbdd142a40cb9bcfe6073b76cc62d995 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mean_iou/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/meteor/README.md b/TextClassification/TextClassification/evaluate/metrics/meteor/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0b234f70f1816a9084d4f8898ac7db20eead0a90 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/meteor/README.md @@ -0,0 +1,139 @@ +--- +title: METEOR +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + METEOR, an automatic metric for machine translation evaluation + that is based on a generalized concept of unigram matching between the + machine-produced translation and human-produced reference translations. + Unigrams can be matched based on their surface forms, stemmed forms, + and meanings; furthermore, METEOR can be easily extended to include more + advanced matching strategies. Once all generalized unigram matches + between the two strings have been found, METEOR computes a score for + this matching using a combination of unigram-precision, unigram-recall, and + a measure of fragmentation that is designed to directly capture how + well-ordered the matched words in the machine translation are in relation + to the reference. + + METEOR gets an R correlation value of 0.347 with human evaluation on the Arabic + data and 0.331 on the Chinese data. This is shown to be an improvement on + using simply unigram-precision, unigram-recall and their harmonic F1 + combination. +--- + +# Metric Card for METEOR + +## Metric description + +METEOR (Metric for Evaluation of Translation with Explicit ORdering) is a machine translation evaluation metric, which is calculated based on the harmonic mean of precision and recall, with recall weighted more than precision. + +METEOR is based on a generalized concept of unigram matching between the machine-produced translation and human-produced reference translations. Unigrams can be matched based on their surface forms, stemmed forms, and meanings. Once all generalized unigram matches between the two strings have been found, METEOR computes a score for this matching using a combination of unigram-precision, unigram-recall, and a measure of fragmentation that is designed to directly capture how well-ordered the matched words in the machine translation are in relation to the reference. + + +## How to use + +METEOR has two mandatory arguments: + +`predictions`: a `list` of predictions to score. Each prediction should be a string with tokens separated by spaces. + +`references`: a `list` of references (in the case of one `reference` per `prediction`), or a `list` of `lists` of references (in the case of multiple `references` per `prediction`. Each reference should be a string with tokens separated by spaces. + +It also has several optional parameters: + +`alpha`: Parameter for controlling relative weights of precision and recall. The default value is `0.9`. + +`beta`: Parameter for controlling shape of penalty as a function of fragmentation. The default value is `3`. + +`gamma`: The relative weight assigned to fragmentation penalty. The default is `0.5`. + +Refer to the [METEOR paper](https://aclanthology.org/W05-0909.pdf) for more information about parameter values and ranges. + +```python +>>> meteor = evaluate.load('meteor') +>>> predictions = ["It is a guide to action which ensures that the military always obeys the commands of the party"] +>>> references = ["It is a guide to action that ensures that the military will forever heed Party commands"] +>>> results = meteor.compute(predictions=predictions, references=references) +``` + +## Output values + +The metric outputs a dictionary containing the METEOR score. Its values range from 0 to 1, e.g.: +``` +{'meteor': 0.9999142661179699} +``` + + +### Values from popular papers +The [METEOR paper](https://aclanthology.org/W05-0909.pdf) does not report METEOR score values for different models, but it does report that METEOR gets an R correlation value of 0.347 with human evaluation on the Arabic data and 0.331 on the Chinese data. + + +## Examples + +One `reference` per `prediction`: + +```python +>>> meteor = evaluate.load('meteor') +>>> predictions = ["It is a guide to action which ensures that the military always obeys the commands of the party"] +>>> reference = ["It is a guide to action which ensures that the military always obeys the commands of the party"] +>>> results = meteor.compute(predictions=predictions, references=reference) +>>> print(round(results['meteor'], 2)) +1.0 +``` + +Multiple `references` per `prediction`: + +```python +>>> meteor = evaluate.load('meteor') +>>> predictions = ["It is a guide to action which ensures that the military always obeys the commands of the party"] +>>> references = [['It is a guide to action that ensures that the military will forever heed Party commands', 'It is the guiding principle which guarantees the military forces always being under the command of the Party', 'It is the practical guide for the army always to heed the directions of the party']] +>>> results = meteor.compute(predictions=predictions, references=references) +>>> print(round(results['meteor'], 2)) +1.0 +``` + +Multiple `references` per `prediction`, partial match: + +```python +>>> meteor = evaluate.load('meteor') +>>> predictions = ["It is a guide to action which ensures that the military always obeys the commands of the party"] +>>> references = [['It is a guide to action that ensures that the military will forever heed Party commands', 'It is the guiding principle which guarantees the military forces always being under the command of the Party', 'It is the practical guide for the army always to heed the directions of the party']] +>>> results = meteor.compute(predictions=predictions, references=references) +>>> print(round(results['meteor'], 2)) +0.69 +``` + +## Limitations and bias + +While the correlation between METEOR and human judgments was measured for Chinese and Arabic and found to be significant, further experimentation is needed to check its correlation for other languages. + +Furthermore, while the alignment and matching done in METEOR is based on unigrams, using multiple word entities (e.g. bigrams) could contribute to improving its accuracy -- this has been proposed in [more recent publications](https://www.cs.cmu.edu/~alavie/METEOR/pdf/meteor-naacl-2010.pdf) on the subject. + + +## Citation + +```bibtex +@inproceedings{banarjee2005, + title = {{METEOR}: An Automatic Metric for {MT} Evaluation with Improved Correlation with Human Judgments}, + author = {Banerjee, Satanjeev and Lavie, Alon}, + booktitle = {Proceedings of the {ACL} Workshop on Intrinsic and Extrinsic Evaluation Measures for Machine Translation and/or Summarization}, + month = jun, + year = {2005}, + address = {Ann Arbor, Michigan}, + publisher = {Association for Computational Linguistics}, + url = {https://www.aclweb.org/anthology/W05-0909}, + pages = {65--72}, +} +``` + +## Further References +- [METEOR -- Wikipedia](https://en.wikipedia.org/wiki/METEOR) +- [METEOR score -- NLTK](https://www.nltk.org/_modules/nltk/translate/meteor_score.html) + diff --git a/TextClassification/TextClassification/evaluate/metrics/meteor/app.py b/TextClassification/TextClassification/evaluate/metrics/meteor/app.py new file mode 100644 index 0000000000000000000000000000000000000000..3bcb6bf27d14ef87a84ffbdfda0721a13c811481 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/meteor/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("meteor") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/meteor/meteor.py b/TextClassification/TextClassification/evaluate/metrics/meteor/meteor.py new file mode 100644 index 0000000000000000000000000000000000000000..058ee80ed77aaa4253acde460f915ca2771c4835 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/meteor/meteor.py @@ -0,0 +1,168 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" METEOR metric. """ + +import datasets +import numpy as np +from nltk.translate import meteor_score +from packaging import version + +import evaluate + + +if evaluate.config.PY_VERSION < version.parse("3.8"): + import importlib_metadata +else: + import importlib.metadata as importlib_metadata + + +NLTK_VERSION = version.parse(importlib_metadata.version("nltk")) +if NLTK_VERSION >= version.Version("3.6.4"): + from nltk import word_tokenize + + +_CITATION = """\ +@inproceedings{banarjee2005, + title = {{METEOR}: An Automatic Metric for {MT} Evaluation with Improved Correlation with Human Judgments}, + author = {Banerjee, Satanjeev and Lavie, Alon}, + booktitle = {Proceedings of the {ACL} Workshop on Intrinsic and Extrinsic Evaluation Measures for Machine Translation and/or Summarization}, + month = jun, + year = {2005}, + address = {Ann Arbor, Michigan}, + publisher = {Association for Computational Linguistics}, + url = {https://www.aclweb.org/anthology/W05-0909}, + pages = {65--72}, +} +""" + +_DESCRIPTION = """\ +METEOR, an automatic metric for machine translation evaluation +that is based on a generalized concept of unigram matching between the +machine-produced translation and human-produced reference translations. +Unigrams can be matched based on their surface forms, stemmed forms, +and meanings; furthermore, METEOR can be easily extended to include more +advanced matching strategies. Once all generalized unigram matches +between the two strings have been found, METEOR computes a score for +this matching using a combination of unigram-precision, unigram-recall, and +a measure of fragmentation that is designed to directly capture how +well-ordered the matched words in the machine translation are in relation +to the reference. + +METEOR gets an R correlation value of 0.347 with human evaluation on the Arabic +data and 0.331 on the Chinese data. This is shown to be an improvement on +using simply unigram-precision, unigram-recall and their harmonic F1 +combination. +""" + +_KWARGS_DESCRIPTION = """ +Computes METEOR score of translated segments against one or more references. +Args: + predictions: list of predictions to score. Each prediction + should be a string with tokens separated by spaces. + references: list of reference for each prediction. Each + reference should be a string with tokens separated by spaces. + alpha: Parameter for controlling relative weights of precision and recall. default: 0.9 + beta: Parameter for controlling shape of penalty as a function of fragmentation. default: 3 + gamma: Relative weight assigned to fragmentation penalty. default: 0.5 +Returns: + 'meteor': meteor score. +Examples: + + >>> meteor = evaluate.load('meteor') + >>> predictions = ["It is a guide to action which ensures that the military always obeys the commands of the party"] + >>> references = ["It is a guide to action that ensures that the military will forever heed Party commands"] + >>> results = meteor.compute(predictions=predictions, references=references) + >>> print(round(results["meteor"], 4)) + 0.6944 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Meteor(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), + } + ), + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + ], + codebase_urls=["https://github.com/nltk/nltk/blob/develop/nltk/translate/meteor_score.py"], + reference_urls=[ + "https://www.nltk.org/api/nltk.translate.html#module-nltk.translate.meteor_score", + "https://en.wikipedia.org/wiki/METEOR", + ], + ) + + def _download_and_prepare(self, dl_manager): + import nltk + + nltk.download("wordnet") + if NLTK_VERSION >= version.Version("3.6.5"): + nltk.download("punkt") + if NLTK_VERSION >= version.Version("3.6.6"): + nltk.download("omw-1.4") + + def _compute(self, predictions, references, alpha=0.9, beta=3, gamma=0.5): + multiple_refs = isinstance(references[0], list) + if NLTK_VERSION >= version.Version("3.6.5"): + # the version of METEOR in NLTK version 3.6.5 and earlier expect tokenized inputs + if multiple_refs: + scores = [ + meteor_score.meteor_score( + [word_tokenize(ref) for ref in refs], + word_tokenize(pred), + alpha=alpha, + beta=beta, + gamma=gamma, + ) + for refs, pred in zip(references, predictions) + ] + else: + scores = [ + meteor_score.single_meteor_score( + word_tokenize(ref), word_tokenize(pred), alpha=alpha, beta=beta, gamma=gamma + ) + for ref, pred in zip(references, predictions) + ] + else: + if multiple_refs: + scores = [ + meteor_score.meteor_score( + [[word_tokenize(ref) for ref in group] for group in references][0], + word_tokenize(pred), + alpha=alpha, + beta=beta, + gamma=gamma, + ) + for ref, pred in zip(references, predictions) + ] + else: + scores = [ + meteor_score.single_meteor_score(ref, pred, alpha=alpha, beta=beta, gamma=gamma) + for ref, pred in zip(references, predictions) + ] + + return {"meteor": np.mean(scores)} diff --git a/TextClassification/TextClassification/evaluate/metrics/meteor/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/meteor/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b5904a71469bea16b7ccb3b41e9ed19663d20599 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/meteor/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +nltk \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/mse/README.md b/TextClassification/TextClassification/evaluate/metrics/mse/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6f4144ec3fc649340ce77ecddc60a43328c907c6 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mse/README.md @@ -0,0 +1,137 @@ +--- +title: MSE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Mean Squared Error(MSE) is the average of the square of difference between the predicted + and actual values. +--- + +# Metric Card for MSE + + +## Metric Description + +Mean Squared Error(MSE) represents the average of the squares of errors -- i.e. the average squared difference between the estimated values and the actual values. + +![image](https://user-images.githubusercontent.com/14205986/165999302-eba3702d-81e3-4363-9c0e-d3bfceb7ec5a.png) + +## How to Use + +At minimum, this metric requires predictions and references as inputs. + +```python +>>> mse_metric = evaluate.load("mse") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> results = mse_metric.compute(predictions=predictions, references=references) +``` + +### Inputs + +Mandatory inputs: +- `predictions`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the estimated target values. +- `references`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the ground truth (correct) target values. + +Optional arguments: +- `sample_weight`: numeric array-like of shape (`n_samples,`) representing sample weights. The default is `None`. +- `multioutput`: `raw_values`, `uniform_average` or numeric array-like of shape (`n_outputs,`), which defines the aggregation of multiple output values. The default value is `uniform_average`. + - `raw_values` returns a full set of errors in case of multioutput input. + - `uniform_average` means that the errors of all outputs are averaged with uniform weight. + - the array-like value defines weights used to average errors. +- `squared` (`bool`): If `True` returns MSE value, if `False` returns RMSE (Root Mean Squared Error). The default value is `True`. + + +### Output Values +This metric outputs a dictionary, containing the mean squared error score, which is of type: +- `float`: if multioutput is `uniform_average` or an ndarray of weights, then the weighted average of all output errors is returned. +- numeric array-like of shape (`n_outputs,`): if multioutput is `raw_values`, then the score is returned for each output separately. + +Each MSE `float` value ranges from `0.0` to `1.0`, with the best value being `0.0`. + +Output Example(s): +```python +{'mse': 0.5} +``` + +If `multioutput="raw_values"`: +```python +{'mse': array([0.41666667, 1. ])} +``` + +#### Values from Popular Papers + + +### Examples + +Example with the `uniform_average` config: +```python +>>> mse_metric = evaluate.load("mse") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> results = mse_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'mse': 0.375} +``` + +Example with `squared = True`, which returns the RMSE: +```python +>>> mse_metric = evaluate.load("mse") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> rmse_result = mse_metric.compute(predictions=predictions, references=references, squared=False) +>>> print(rmse_result) +{'mse': 0.6123724356957945} +``` + +Example with multi-dimensional lists, and the `raw_values` config: +```python +>>> mse_metric = evaluate.load("mse", "multilist") +>>> predictions = [[0.5, 1], [-1, 1], [7, -6]] +>>> references = [[0, 2], [-1, 2], [8, -5]] +>>> results = mse_metric.compute(predictions=predictions, references=references, multioutput='raw_values') +>>> print(results) +{'mse': array([0.41666667, 1. ])} +""" +``` + +## Limitations and Bias +MSE has the disadvantage of heavily weighting outliers -- given that it squares them, this results in large errors weighing more heavily than small ones. It can be used alongside [MAE](https://huggingface.co/metrics/mae), which is complementary given that it does not square the errors. + +## Citation(s) +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + +```bibtex +@article{willmott2005advantages, + title={Advantages of the mean absolute error (MAE) over the root mean square error (RMSE) in assessing average model performance}, + author={Willmott, Cort J and Matsuura, Kenji}, + journal={Climate research}, + volume={30}, + number={1}, + pages={79--82}, + year={2005} +} +``` + +## Further References +- [Mean Squared Error - Wikipedia](https://en.wikipedia.org/wiki/Mean_squared_error) diff --git a/TextClassification/TextClassification/evaluate/metrics/mse/app.py b/TextClassification/TextClassification/evaluate/metrics/mse/app.py new file mode 100644 index 0000000000000000000000000000000000000000..d334862deefa700035cdcd573ba00e6b4304121b --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mse/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("mse") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/mse/mse.py b/TextClassification/TextClassification/evaluate/metrics/mse/mse.py new file mode 100644 index 0000000000000000000000000000000000000000..fb695bfde89e38d17dc1761197768e8cc8061b33 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mse/mse.py @@ -0,0 +1,119 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MSE - Mean Squared Error Metric""" + +import datasets +from sklearn.metrics import mean_squared_error + +import evaluate + + +_CITATION = """\ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + +_DESCRIPTION = """\ +Mean Squared Error(MSE) is the average of the square of difference between the predicted +and actual values. +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions: array-like of shape (n_samples,) or (n_samples, n_outputs) + Estimated target values. + references: array-like of shape (n_samples,) or (n_samples, n_outputs) + Ground truth (correct) target values. + sample_weight: array-like of shape (n_samples,), default=None + Sample weights. + multioutput: {"raw_values", "uniform_average"} or array-like of shape (n_outputs,), default="uniform_average" + Defines aggregating of multiple output values. Array-like value defines weights used to average errors. + + "raw_values" : Returns a full set of errors in case of multioutput input. + + "uniform_average" : Errors of all outputs are averaged with uniform weight. + + squared : bool, default=True + If True returns MSE value, if False returns RMSE (Root Mean Squared Error) value. + +Returns: + mse : mean squared error. +Examples: + + >>> mse_metric = evaluate.load("mse") + >>> predictions = [2.5, 0.0, 2, 8] + >>> references = [3, -0.5, 2, 7] + >>> results = mse_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'mse': 0.375} + >>> rmse_result = mse_metric.compute(predictions=predictions, references=references, squared=False) + >>> print(rmse_result) + {'mse': 0.6123724356957945} + + If you're using multi-dimensional lists, then set the config as follows : + + >>> mse_metric = evaluate.load("mse", "multilist") + >>> predictions = [[0.5, 1], [-1, 1], [7, -6]] + >>> references = [[0, 2], [-1, 2], [8, -5]] + >>> results = mse_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'mse': 0.7083333333333334} + >>> results = mse_metric.compute(predictions=predictions, references=references, multioutput='raw_values') + >>> print(results) # doctest: +NORMALIZE_WHITESPACE + {'mse': array([0.41666667, 1. ])} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Mse(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features(self._get_feature_types()), + reference_urls=[ + "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_squared_error.html" + ], + ) + + def _get_feature_types(self): + if self.config_name == "multilist": + return { + "predictions": datasets.Sequence(datasets.Value("float")), + "references": datasets.Sequence(datasets.Value("float")), + } + else: + return { + "predictions": datasets.Value("float"), + "references": datasets.Value("float"), + } + + def _compute(self, predictions, references, sample_weight=None, multioutput="uniform_average", squared=True): + + mse = mean_squared_error( + references, predictions, sample_weight=sample_weight, multioutput=multioutput, squared=squared + ) + + return {"mse": mse} diff --git a/TextClassification/TextClassification/evaluate/metrics/mse/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/mse/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/mse/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/nist_mt/README.md b/TextClassification/TextClassification/evaluate/metrics/nist_mt/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ee553a32afab574ad2813d1d8fc0ade04e32b7db --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/nist_mt/README.md @@ -0,0 +1,82 @@ +--- +title: NIST_MT +emoji: 🤗 +colorFrom: purple +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +- machine-translation +description: + DARPA commissioned NIST to develop an MT evaluation facility based on the BLEU score. +--- + +# Metric Card for NIST's MT metric + + +## Metric Description +DARPA commissioned NIST to develop an MT evaluation facility based on the BLEU +score. The official script used by NIST to compute BLEU and NIST score is +mteval-14.pl. The main differences are: + + - BLEU uses geometric mean of the ngram overlaps, NIST uses arithmetic mean. + - NIST has a different brevity penalty + - NIST score from mteval-14.pl has a self-contained tokenizer (in the Hugging Face implementation we rely on NLTK's +implementation of the NIST-specific tokenizer) + +## Intended Uses +NIST was developed for machine translation evaluation. + +## How to Use + +```python +import evaluate +nist_mt = evaluate.load("nist_mt") +hypothesis1 = "It is a guide to action which ensures that the military always obeys the commands of the party" +reference1 = "It is a guide to action that ensures that the military will forever heed Party commands" +reference2 = "It is the guiding principle which guarantees the military forces always being under the command of the Party" +nist_mt.compute(hypothesis1, [reference1, reference2]) +# {'nist_mt': 3.3709935957649324} +``` + +### Inputs +- **predictions**: tokenized predictions to score. For sentence-level NIST, a list of tokens (str); + for corpus-level NIST, a list (sentences) of lists of tokens (str) +- **references**: potentially multiple tokenized references for each prediction. For sentence-level NIST, a + list (multiple potential references) of list of tokens (str); for corpus-level NIST, a list (corpus) of lists + (multiple potential references) of lists of tokens (str) +- **n**: highest n-gram order +- **tokenize_kwargs**: arguments passed to the tokenizer (see: https://github.com/nltk/nltk/blob/90fa546ea600194f2799ee51eaf1b729c128711e/nltk/tokenize/nist.py#L139) + +### Output Values +- **nist_mt** (`float`): NIST score + +Output Example: +```python +{'nist_mt': 3.3709935957649324} +``` + + +## Citation +```bibtex +@inproceedings{10.5555/1289189.1289273, + author = {Doddington, George}, + title = {Automatic Evaluation of Machine Translation Quality Using N-Gram Co-Occurrence Statistics}, + year = {2002}, + publisher = {Morgan Kaufmann Publishers Inc.}, + address = {San Francisco, CA, USA}, + booktitle = {Proceedings of the Second International Conference on Human Language Technology Research}, + pages = {138–145}, + numpages = {8}, + location = {San Diego, California}, + series = {HLT '02} +} +``` + +## Further References + +This Hugging Face implementation uses [the NLTK implementation](https://github.com/nltk/nltk/blob/develop/nltk/translate/nist_score.py) diff --git a/TextClassification/TextClassification/evaluate/metrics/nist_mt/app.py b/TextClassification/TextClassification/evaluate/metrics/nist_mt/app.py new file mode 100644 index 0000000000000000000000000000000000000000..0a563648a5b9a8624ecd508248c9a15be8dbba97 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/nist_mt/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("nist_mt") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/nist_mt/nist_mt.py b/TextClassification/TextClassification/evaluate/metrics/nist_mt/nist_mt.py new file mode 100644 index 0000000000000000000000000000000000000000..5aeab0c08bd75f3039ab8b7452adf52d0979b808 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/nist_mt/nist_mt.py @@ -0,0 +1,132 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""NLTK's NIST implementation on both the sentence and corpus level""" +from typing import Dict, Optional + +import datasets +import nltk +from datasets import Sequence, Value + + +try: + nltk.data.find("perluniprops") +except LookupError: + nltk.download("perluniprops", quiet=True) # NISTTokenizer requirement + +from nltk.tokenize.nist import NISTTokenizer +from nltk.translate.nist_score import corpus_nist, sentence_nist + +import evaluate + + +_CITATION = """\ +@inproceedings{10.5555/1289189.1289273, + author = {Doddington, George}, + title = {Automatic Evaluation of Machine Translation Quality Using N-Gram Co-Occurrence Statistics}, + year = {2002}, + publisher = {Morgan Kaufmann Publishers Inc.}, + address = {San Francisco, CA, USA}, + booktitle = {Proceedings of the Second International Conference on Human Language Technology Research}, + pages = {138–145}, + numpages = {8}, + location = {San Diego, California}, + series = {HLT '02} +} +""" + +_DESCRIPTION = """\ +DARPA commissioned NIST to develop an MT evaluation facility based on the BLEU +score. The official script used by NIST to compute BLEU and NIST score is +mteval-14.pl. The main differences are: + + - BLEU uses geometric mean of the ngram precisions, NIST uses arithmetic mean. + - NIST has a different brevity penalty + - NIST score from mteval-14.pl has a self-contained tokenizer (in the Hugging Face implementation we rely on NLTK's +implementation of the NIST-specific tokenizer) +""" + + +_KWARGS_DESCRIPTION = """ +Computes NIST score of translated segments against one or more references. +Args: + predictions: predictions to score (list of str) + references: potentially multiple references for each prediction (list of str or list of list of str) + n: highest n-gram order + lowercase: whether to lowercase the data (only applicable if 'western_lang' is True) + western_lang: whether the current language is a Western language, which will enable some specific tokenization + rules with respect to, e.g., punctuation + +Returns: + 'nist_mt': nist_mt score +Examples: + >>> nist_mt = evaluate.load("nist_mt") + >>> hypothesis = "It is a guide to action which ensures that the military always obeys the commands of the party" + >>> reference1 = "It is a guide to action that ensures that the military will forever heed Party commands" + >>> reference2 = "It is the guiding principle which guarantees the military forces always being under the command of the Party" + >>> reference3 = "It is the practical guide for the army always to heed the directions of the party" + >>> nist_mt.compute(predictions=[hypothesis], references=[[reference1, reference2, reference3]]) + {'nist_mt': 3.3709935957649324} + >>> nist_mt.compute(predictions=[hypothesis], references=[reference1]) + {'nist_mt': 2.4477124183006533} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class NistMt(evaluate.Metric): + """A wrapper around NLTK's NIST implementation.""" + + def _info(self): + return evaluate.MetricInfo( + module_type="metric", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": Value("string", id="prediction"), + "references": Sequence(Value("string", id="reference"), id="references"), + } + ), + datasets.Features( + { + "predictions": Value("string", id="prediction"), + "references": Value("string", id="reference"), + } + ), + ], + homepage="https://www.nltk.org/api/nltk.translate.nist_score.html", + codebase_urls=["https://github.com/nltk/nltk/blob/develop/nltk/translate/nist_score.py"], + reference_urls=["https://en.wikipedia.org/wiki/NIST_(metric)"], + ) + + def _compute(self, predictions, references, n: int = 5, lowercase=False, western_lang=True): + tokenizer = NISTTokenizer() + + # Account for single reference cases: references always need to have one more dimension than predictions + if isinstance(references[0], str): + references = [[ref] for ref in references] + + predictions = [ + tokenizer.tokenize(pred, return_str=False, lowercase=lowercase, western_lang=western_lang) + for pred in predictions + ] + references = [ + [ + tokenizer.tokenize(ref, return_str=False, lowercase=lowercase, western_lang=western_lang) + for ref in ref_sentences + ] + for ref_sentences in references + ] + return {"nist_mt": corpus_nist(list_of_references=references, hypotheses=predictions, n=n)} diff --git a/TextClassification/TextClassification/evaluate/metrics/nist_mt/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/nist_mt/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5844221307e93baf24d64fb2b6df6622b2375a --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/nist_mt/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +nltk diff --git a/TextClassification/TextClassification/evaluate/metrics/nist_mt/tests.py b/TextClassification/TextClassification/evaluate/metrics/nist_mt/tests.py new file mode 100644 index 0000000000000000000000000000000000000000..525287274f6137b655c504082debdd10eae5f982 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/nist_mt/tests.py @@ -0,0 +1,34 @@ +from _pytest.fixtures import fixture +from nist_mt import Nist_mt + + +nist = Nist_mt() + + +@fixture +def hypothesis_sent(): + return "It is a guide to action which ensures that the military always obeys the commands of the party" + + +@fixture +def reference_sent1(): + return "It is a guide to action that ensures that the military will forever heed Party commands" + + +@fixture +def reference_sent2(): + return ( + "It is the guiding principle which guarantees the military forces always being under the command of the Party" + ) + + +@fixture +def reference_sent3(): + return "It is the practical guide for the army always to heed the directions of the party" + + +def test_nist_sentence(hypothesis_sent, reference_sent1, reference_sent2, reference_sent3): + nist_score = nist.compute( + predictions=[hypothesis_sent], references=[[reference_sent1, reference_sent2, reference_sent3]] + ) + assert abs(nist_score["nist_mt"] - 3.3709935957649324) < 1e-6 diff --git a/TextClassification/TextClassification/evaluate/metrics/pearsonr/README.md b/TextClassification/TextClassification/evaluate/metrics/pearsonr/README.md new file mode 100644 index 0000000000000000000000000000000000000000..391d65267014015c0f84ce91ab30a982e6bf6bac --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/pearsonr/README.md @@ -0,0 +1,119 @@ +--- +title: Pearson Correlation Coefficient +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Pearson correlation coefficient and p-value for testing non-correlation. + The Pearson correlation coefficient measures the linear relationship between two datasets. The calculation of the p-value relies on the assumption that each dataset is normally distributed. Like other correlation coefficients, this one varies between -1 and +1 with 0 implying no correlation. Correlations of -1 or +1 imply an exact linear relationship. Positive correlations imply that as x increases, so does y. Negative correlations imply that as x increases, y decreases. + The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets. +--- + +# Metric Card for Pearson Correlation Coefficient (pearsonr) + + +## Metric Description + +Pearson correlation coefficient and p-value for testing non-correlation. +The Pearson correlation coefficient measures the linear relationship between two datasets. The calculation of the p-value relies on the assumption that each dataset is normally distributed. Like other correlation coefficients, this one varies between -1 and +1 with 0 implying no correlation. Correlations of -1 or +1 imply an exact linear relationship. Positive correlations imply that as x increases, so does y. Negative correlations imply that as x increases, y decreases. +The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets. + + +## How to Use + +This metric takes a list of predictions and a list of references as input + +```python +>>> pearsonr_metric = evaluate.load("pearsonr") +>>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5]) +>>> print(round(results['pearsonr']), 2) +['-0.74'] +``` + + +### Inputs +- **predictions** (`list` of `int`): Predicted class labels, as returned by a model. +- **references** (`list` of `int`): Ground truth labels. +- **return_pvalue** (`boolean`): If `True`, returns the p-value, along with the correlation coefficient. If `False`, returns only the correlation coefficient. Defaults to `False`. + + +### Output Values +- **pearsonr**(`float`): Pearson correlation coefficient. Minimum possible value is -1. Maximum possible value is 1. Values of 1 and -1 indicate exact linear positive and negative relationships, respectively. A value of 0 implies no correlation. +- **p-value**(`float`): P-value, which roughly indicates the probability of an The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets. Minimum possible value is 0. Maximum possible value is 1. Higher values indicate higher probabilities. + +Like other correlation coefficients, this one varies between -1 and +1 with 0 implying no correlation. Correlations of -1 or +1 imply an exact linear relationship. Positive correlations imply that as x increases, so does y. Negative correlations imply that as x increases, y decreases. + +Output Example(s): +```python +{'pearsonr': -0.7} +``` +```python +{'p-value': 0.15} +``` + +#### Values from Popular Papers + +### Examples + +Example 1-A simple example using only predictions and references. +```python +>>> pearsonr_metric = evaluate.load("pearsonr") +>>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5]) +>>> print(round(results['pearsonr'], 2)) +-0.74 +``` + +Example 2-The same as Example 1, but that also returns the `p-value`. +```python +>>> pearsonr_metric = evaluate.load("pearsonr") +>>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5], return_pvalue=True) +>>> print(sorted(list(results.keys()))) +['p-value', 'pearsonr'] +>>> print(round(results['pearsonr'], 2)) +-0.74 +>>> print(round(results['p-value'], 2)) +0.15 +``` + + +## Limitations and Bias + +As stated above, the calculation of the p-value relies on the assumption that each data set is normally distributed. This is not always the case, so verifying the true distribution of datasets is recommended. + + +## Citation(s) +```bibtex +@article{2020SciPy-NMeth, +author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and + Haberland, Matt and Reddy, Tyler and Cournapeau, David and + Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and + Bright, Jonathan and {van der Walt}, St{\'e}fan J. and + Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and + Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and + Kern, Robert and Larson, Eric and Carey, C J and + Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and + {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and + Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and + Harris, Charles R. and Archibald, Anne M. and + Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and + {van Mulbregt}, Paul and {SciPy 1.0 Contributors}}, +title = {{{SciPy} 1.0: Fundamental Algorithms for Scientific + Computing in Python}}, +journal = {Nature Methods}, +year = {2020}, +volume = {17}, +pages = {261--272}, +adsurl = {https://rdcu.be/b08Wh}, +doi = {10.1038/s41592-019-0686-2}, +} +``` + + +## Further References diff --git a/TextClassification/TextClassification/evaluate/metrics/pearsonr/app.py b/TextClassification/TextClassification/evaluate/metrics/pearsonr/app.py new file mode 100644 index 0000000000000000000000000000000000000000..2123f9f62a2ae6a5f2cb772b2c92e05f4bc03f90 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/pearsonr/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("pearsonr") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/pearsonr/pearsonr.py b/TextClassification/TextClassification/evaluate/metrics/pearsonr/pearsonr.py new file mode 100644 index 0000000000000000000000000000000000000000..5ed0e76206f2876d52106eb9fd01185ab2bb41fd --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/pearsonr/pearsonr.py @@ -0,0 +1,107 @@ +# Copyright 2021 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Pearson correlation coefficient metric.""" + +import datasets +from scipy.stats import pearsonr + +import evaluate + + +_DESCRIPTION = """ +Pearson correlation coefficient and p-value for testing non-correlation. +The Pearson correlation coefficient measures the linear relationship between two datasets. The calculation of the p-value relies on the assumption that each dataset is normally distributed. Like other correlation coefficients, this one varies between -1 and +1 with 0 implying no correlation. Correlations of -1 or +1 imply an exact linear relationship. Positive correlations imply that as x increases, so does y. Negative correlations imply that as x increases, y decreases. +The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets. +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions (`list` of `int`): Predicted class labels, as returned by a model. + references (`list` of `int`): Ground truth labels. + return_pvalue (`boolean`): If `True`, returns the p-value, along with the correlation coefficient. If `False`, returns only the correlation coefficient. Defaults to `False`. + +Returns: + pearsonr (`float`): Pearson correlation coefficient. Minimum possible value is -1. Maximum possible value is 1. Values of 1 and -1 indicate exact linear positive and negative relationships, respectively. A value of 0 implies no correlation. + p-value (`float`): P-value, which roughly indicates the probability of an The p-value roughly indicates the probability of an uncorrelated system producing datasets that have a Pearson correlation at least as extreme as the one computed from these datasets. Minimum possible value is 0. Maximum possible value is 1. Higher values indicate higher probabilities. + +Examples: + + Example 1-A simple example using only predictions and references. + >>> pearsonr_metric = evaluate.load("pearsonr") + >>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5]) + >>> print(round(results['pearsonr'], 2)) + -0.74 + + Example 2-The same as Example 1, but that also returns the `p-value`. + >>> pearsonr_metric = evaluate.load("pearsonr") + >>> results = pearsonr_metric.compute(predictions=[10, 9, 2.5, 6, 4], references=[1, 2, 3, 4, 5], return_pvalue=True) + >>> print(sorted(list(results.keys()))) + ['p-value', 'pearsonr'] + >>> print(round(results['pearsonr'], 2)) + -0.74 + >>> print(round(results['p-value'], 2)) + 0.15 +""" + + +_CITATION = """ +@article{2020SciPy-NMeth, +author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and + Haberland, Matt and Reddy, Tyler and Cournapeau, David and + Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and + Bright, Jonathan and {van der Walt}, St{\'e}fan J. and + Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and + Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and + Kern, Robert and Larson, Eric and Carey, C J and + Polat, Ilhan and Feng, Yu and Moore, Eric W. and + {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and + Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and + Harris, Charles R. and Archibald, Anne M. and + Ribeiro, Antonio H. and Pedregosa, Fabian and + {van Mulbregt}, Paul and {SciPy 1.0 Contributors}}, +title = {{{SciPy} 1.0: Fundamental Algorithms for Scientific + Computing in Python}}, +journal = {Nature Methods}, +year = {2020}, +volume = {17}, +pages = {261--272}, +adsurl = {https://rdcu.be/b08Wh}, +doi = {10.1038/s41592-019-0686-2}, +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Pearsonr(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("float"), + "references": datasets.Value("float"), + } + ), + reference_urls=["https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.pearsonr.html"], + ) + + def _compute(self, predictions, references, return_pvalue=False): + if return_pvalue: + results = pearsonr(references, predictions) + return {"pearsonr": results[0], "p-value": results[1]} + else: + return {"pearsonr": float(pearsonr(references, predictions)[0])} diff --git a/TextClassification/TextClassification/evaluate/metrics/pearsonr/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/pearsonr/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a410b4ca49fe91f97db76956af15309d2dec7bb8 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/pearsonr/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scipy \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/perplexity/README.md b/TextClassification/TextClassification/evaluate/metrics/perplexity/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7948097f8903b2e7f6fef3ae36f8b6153991cc1d --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/perplexity/README.md @@ -0,0 +1,118 @@ +--- +title: Perplexity +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Perplexity (PPL) is one of the most common metrics for evaluating language models. + It is defined as the exponentiated average negative log-likelihood of a sequence, calculated with exponent base `e`. + + For more information on perplexity, see [this tutorial](https://huggingface.co/docs/transformers/perplexity). +--- + +# Metric Card for Perplexity + +## Metric Description +Given a model and an input text sequence, perplexity measures how likely the model is to generate the input text sequence. + +As a metric, it can be used to evaluate how well the model has learned the distribution of the text it was trained on. + +In this case, `model_id` should be the trained model to be evaluated, and the input texts should be the text that the model was trained on. + +This implementation of perplexity is calculated with log base `e`, as in `perplexity = e**(sum(losses) / num_tokenized_tokens)`, following recent convention in deep learning frameworks. + +## Intended Uses +Any language generation task. + +## How to Use + +The metric takes a list of text as input, as well as the name of the model used to compute the metric: + +```python +from evaluate import load +perplexity = load("perplexity", module_type="metric") +results = perplexity.compute(predictions=predictions, model_id='gpt2') +``` + +### Inputs +- **model_id** (str): model used for calculating Perplexity. NOTE: Perplexity can only be calculated for causal language models. + - This includes models such as gpt2, causal variations of bert, causal versions of t5, and more (the full list can be found in the AutoModelForCausalLM documentation here: https://huggingface.co/docs/transformers/master/en/model_doc/auto#transformers.AutoModelForCausalLM ) +- **predictions** (list of str): input text, where each separate text snippet is one list entry. +- **batch_size** (int): the batch size to run texts through the model. Defaults to 16. +- **add_start_token** (bool): whether to add the start token to the texts, so the perplexity can include the probability of the first word. Defaults to True. +- **device** (str): device to run on, defaults to `cuda` when available + +### Output Values +This metric outputs a dictionary with the perplexity scores for the text input in the list, and the average perplexity. +If one of the input texts is longer than the max input length of the model, then it is truncated to the max length for the perplexity computation. + +``` +{'perplexities': [8.182524681091309, 33.42122268676758, 27.012239456176758], 'mean_perplexity': 22.871995608011883} +``` + +The range of this metric is [0, inf). A lower score is better. + +#### Values from Popular Papers + + +### Examples +Calculating perplexity on predictions defined here: +```python +perplexity = evaluate.load("perplexity", module_type="metric") +input_texts = ["lorem ipsum", "Happy Birthday!", "Bienvenue"] +results = perplexity.compute(model_id='gpt2', + add_start_token=False, + predictions=input_texts) +print(list(results.keys())) +>>>['perplexities', 'mean_perplexity'] +print(round(results["mean_perplexity"], 2)) +>>>646.75 +print(round(results["perplexities"][0], 2)) +>>>32.25 +``` +Calculating perplexity on predictions loaded in from a dataset: +```python +perplexity = evaluate.load("perplexity", module_type="metric") +input_texts = datasets.load_dataset("wikitext", + "wikitext-2-raw-v1", + split="test")["text"][:50] +input_texts = [s for s in input_texts if s!=''] +results = perplexity.compute(model_id='gpt2', + predictions=input_texts) +print(list(results.keys())) +>>>['perplexities', 'mean_perplexity'] +print(round(results["mean_perplexity"], 2)) +>>>576.76 +print(round(results["perplexities"][0], 2)) +>>>889.28 +``` + +## Limitations and Bias +Note that the output value is based heavily on what text the model was trained on. This means that perplexity scores are not comparable between models or datasets. + +See Meister and Cotterell, ["Language Model Evaluation Beyond Perplexity"]( https://arxiv.org/abs/2106.00085) (2021) for more information about alternative model evaluation strategies. + +## Citation + +```bibtex +@article{jelinek1977perplexity, +title={Perplexity—a measure of the difficulty of speech recognition tasks}, +author={Jelinek, Fred and Mercer, Robert L and Bahl, Lalit R and Baker, James K}, +journal={The Journal of the Acoustical Society of America}, +volume={62}, +number={S1}, +pages={S63--S63}, +year={1977}, +publisher={Acoustical Society of America} +} +``` + +## Further References +- [Hugging Face Perplexity Blog Post](https://huggingface.co/docs/transformers/perplexity) diff --git a/TextClassification/TextClassification/evaluate/metrics/perplexity/app.py b/TextClassification/TextClassification/evaluate/metrics/perplexity/app.py new file mode 100644 index 0000000000000000000000000000000000000000..cf4eee44294f2c28e9aea9adc0b7f8abedfb2e0b --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/perplexity/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("perplexity", module_type="metric") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/perplexity/perplexity.py b/TextClassification/TextClassification/evaluate/metrics/perplexity/perplexity.py new file mode 100644 index 0000000000000000000000000000000000000000..ad307e8ad5cf09d8cfd19390ad539f24187631d0 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/perplexity/perplexity.py @@ -0,0 +1,192 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Perplexity Metric.""" + +import datasets +import numpy as np +import torch +from torch.nn import CrossEntropyLoss +from transformers import AutoModelForCausalLM, AutoTokenizer + +import evaluate +from evaluate import logging + + +_CITATION = """\ + +""" + +_DESCRIPTION = """ +Perplexity (PPL) is one of the most common metrics for evaluating language models. +It is defined as the exponentiated average negative log-likelihood of a sequence, calculated with exponent base `e`. + +For more information, see https://huggingface.co/docs/transformers/perplexity +""" + +_KWARGS_DESCRIPTION = """ +Args: + model_id (str): model used for calculating Perplexity + NOTE: Perplexity can only be calculated for causal language models. + This includes models such as gpt2, causal variations of bert, + causal versions of t5, and more (the full list can be found + in the AutoModelForCausalLM documentation here: + https://huggingface.co/docs/transformers/master/en/model_doc/auto#transformers.AutoModelForCausalLM ) + + predictions (list of str): input text, each separate text snippet + is one list entry. + batch_size (int): the batch size to run texts through the model. Defaults to 16. + add_start_token (bool): whether to add the start token to the texts, + so the perplexity can include the probability of the first word. Defaults to True. + device (str): device to run on, defaults to 'cuda' when available +Returns: + perplexity: dictionary containing the perplexity scores for the texts + in the input list, as well as the mean perplexity. If one of the input texts is + longer than the max input length of the model, then it is truncated to the + max length for the perplexity computation. +Examples: + Example 1: + >>> perplexity = evaluate.load("perplexity", module_type="metric") + >>> input_texts = ["lorem ipsum", "Happy Birthday!", "Bienvenue"] + >>> results = perplexity.compute(model_id='gpt2', + ... add_start_token=False, + ... predictions=input_texts) # doctest:+ELLIPSIS + >>> print(list(results.keys())) + ['perplexities', 'mean_perplexity'] + >>> print(round(results["mean_perplexity"], 0)) + 647.0 + >>> print(round(results["perplexities"][0], 0)) + 32.0 + + Example 2: + >>> from datasets import load_dataset + >>> perplexity = evaluate.load("perplexity", module_type="metric") + >>> input_texts = load_dataset("wikitext", "wikitext-2-raw-v1", split="test")["text"][:10] # doctest: +SKIP + >>> input_texts = [s for s in input_texts if s!=''] + >>> results = perplexity.compute(model_id='gpt2', + ... predictions=input_texts) + >>> print(list(results.keys())) + ['perplexities', 'mean_perplexity'] + >>> print(round(results["mean_perplexity"], 2)) # doctest: +SKIP + 576.76 + >>> print(round(results["perplexities"][0], 2)) # doctest: +SKIP + 889.28 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Perplexity(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + module_type="metric", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("string"), + } + ), + reference_urls=["https://huggingface.co/docs/transformers/perplexity"], + ) + + def _compute( + self, predictions, model_id, batch_size: int = 16, add_start_token: bool = True, device=None, max_length=None + ): + + if device is not None: + assert device in ["gpu", "cpu", "cuda"], "device should be either gpu or cpu." + if device == "gpu": + device = "cuda" + else: + device = "cuda" if torch.cuda.is_available() else "cpu" + + model = AutoModelForCausalLM.from_pretrained(model_id) + model = model.to(device) + + tokenizer = AutoTokenizer.from_pretrained(model_id) + + # if batch_size > 1 (which generally leads to padding being required), and + # if there is not an already assigned pad_token, assign an existing + # special token to also be the padding token + if tokenizer.pad_token is None and batch_size > 1: + existing_special_tokens = list(tokenizer.special_tokens_map_extended.values()) + # check that the model already has at least one special token defined + assert ( + len(existing_special_tokens) > 0 + ), "If batch_size > 1, model must have at least one special token to use for padding. Please use a different model or set batch_size=1." + # assign one of the special tokens to also be the pad token + tokenizer.add_special_tokens({"pad_token": existing_special_tokens[0]}) + + if add_start_token and max_length: + # leave room for token to be added: + assert ( + tokenizer.bos_token is not None + ), "Input model must already have a BOS token if using add_start_token=True. Please use a different model, or set add_start_token=False" + max_tokenized_len = max_length - 1 + else: + max_tokenized_len = max_length + + encodings = tokenizer( + predictions, + add_special_tokens=False, + padding=True, + truncation=True if max_tokenized_len else False, + max_length=max_tokenized_len, + return_tensors="pt", + return_attention_mask=True, + ).to(device) + + encoded_texts = encodings["input_ids"] + attn_masks = encodings["attention_mask"] + + # check that each input is long enough: + if add_start_token: + assert torch.all(torch.ge(attn_masks.sum(1), 1)), "Each input text must be at least one token long." + else: + assert torch.all( + torch.ge(attn_masks.sum(1), 2) + ), "When add_start_token=False, each input text must be at least two tokens long. Run with add_start_token=True if inputting strings of only one token, and remove all empty input strings." + + ppls = [] + loss_fct = CrossEntropyLoss(reduction="none") + + for start_index in logging.tqdm(range(0, len(encoded_texts), batch_size)): + end_index = min(start_index + batch_size, len(encoded_texts)) + encoded_batch = encoded_texts[start_index:end_index] + attn_mask = attn_masks[start_index:end_index] + + if add_start_token: + bos_tokens_tensor = torch.tensor([[tokenizer.bos_token_id]] * encoded_batch.size(dim=0)).to(device) + encoded_batch = torch.cat([bos_tokens_tensor, encoded_batch], dim=1) + attn_mask = torch.cat( + [torch.ones(bos_tokens_tensor.size(), dtype=torch.int64).to(device), attn_mask], dim=1 + ) + + labels = encoded_batch + + with torch.no_grad(): + out_logits = model(encoded_batch, attention_mask=attn_mask).logits + + shift_logits = out_logits[..., :-1, :].contiguous() + shift_labels = labels[..., 1:].contiguous() + shift_attention_mask_batch = attn_mask[..., 1:].contiguous() + + perplexity_batch = torch.exp( + (loss_fct(shift_logits.transpose(1, 2), shift_labels) * shift_attention_mask_batch).sum(1) + / shift_attention_mask_batch.sum(1) + ) + + ppls += perplexity_batch.tolist() + + return {"perplexities": ppls, "mean_perplexity": np.mean(ppls)} diff --git a/TextClassification/TextClassification/evaluate/metrics/perplexity/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/perplexity/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..036c7211c00774608a613a2354aaae763c03bbb5 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/perplexity/requirements.txt @@ -0,0 +1,4 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +torch +torch +transformers \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/poseval/README.md b/TextClassification/TextClassification/evaluate/metrics/poseval/README.md new file mode 100644 index 0000000000000000000000000000000000000000..470300f2e1f3c0fc9114d9c95c029ed0919e14ce --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/poseval/README.md @@ -0,0 +1,117 @@ +--- +title: poseval +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + The poseval metric can be used to evaluate POS taggers. Since seqeval does not work well with POS data + that is not in IOB format the poseval is an alternative. It treats each token in the dataset as independant + observation and computes the precision, recall and F1-score irrespective of sentences. It uses scikit-learns's + classification report to compute the scores. +--- + +# Metric Card for peqeval + +## Metric description + +The poseval metric can be used to evaluate POS taggers. Since seqeval does not work well with POS data (see e.g. [here](https://stackoverflow.com/questions/71327693/how-to-disable-seqeval-label-formatting-for-pos-tagging)) that is not in IOB format the poseval is an alternative. It treats each token in the dataset as independant observation and computes the precision, recall and F1-score irrespective of sentences. It uses scikit-learns's [classification report](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.classification_report.html) to compute the scores. + + +## How to use + +Poseval produces labelling scores along with its sufficient statistics from a source against references. + +It takes two mandatory arguments: + +`predictions`: a list of lists of predicted labels, i.e. estimated targets as returned by a tagger. + +`references`: a list of lists of reference labels, i.e. the ground truth/target values. + +It can also take several optional arguments: + +`zero_division`: Which value to substitute as a metric value when encountering zero division. Should be one of [`0`,`1`,`"warn"`]. `"warn"` acts as `0`, but the warning is raised. + + +```python +>>> predictions = [['INTJ', 'ADP', 'PROPN', 'NOUN', 'PUNCT', 'INTJ', 'ADP', 'PROPN', 'VERB', 'SYM']] +>>> references = [['INTJ', 'ADP', 'PROPN', 'PROPN', 'PUNCT', 'INTJ', 'ADP', 'PROPN', 'PROPN', 'SYM']] +>>> poseval = evaluate.load("poseval") +>>> results = poseval.compute(predictions=predictions, references=references) +>>> print(list(results.keys())) +['ADP', 'INTJ', 'NOUN', 'PROPN', 'PUNCT', 'SYM', 'VERB', 'accuracy', 'macro avg', 'weighted avg'] +>>> print(results["accuracy"]) +0.8 +>>> print(results["PROPN"]["recall"]) +0.5 +``` + +## Output values + +This metric returns a a classification report as a dictionary with a summary of scores for overall and per type: + +Overall (weighted and macro avg): + +`accuracy`: the average [accuracy](https://huggingface.co/metrics/accuracy), on a scale between 0.0 and 1.0. + +`precision`: the average [precision](https://huggingface.co/metrics/precision), on a scale between 0.0 and 1.0. + +`recall`: the average [recall](https://huggingface.co/metrics/recall), on a scale between 0.0 and 1.0. + +`f1`: the average [F1 score](https://huggingface.co/metrics/f1), which is the harmonic mean of the precision and recall. It also has a scale of 0.0 to 1.0. + +Per type (e.g. `MISC`, `PER`, `LOC`,...): + +`precision`: the average [precision](https://huggingface.co/metrics/precision), on a scale between 0.0 and 1.0. + +`recall`: the average [recall](https://huggingface.co/metrics/recall), on a scale between 0.0 and 1.0. + +`f1`: the average [F1 score](https://huggingface.co/metrics/f1), on a scale between 0.0 and 1.0. + + +## Examples + +```python +>>> predictions = [['INTJ', 'ADP', 'PROPN', 'NOUN', 'PUNCT', 'INTJ', 'ADP', 'PROPN', 'VERB', 'SYM']] +>>> references = [['INTJ', 'ADP', 'PROPN', 'PROPN', 'PUNCT', 'INTJ', 'ADP', 'PROPN', 'PROPN', 'SYM']] +>>> poseval = evaluate.load("poseval") +>>> results = poseval.compute(predictions=predictions, references=references) +>>> print(list(results.keys())) +['ADP', 'INTJ', 'NOUN', 'PROPN', 'PUNCT', 'SYM', 'VERB', 'accuracy', 'macro avg', 'weighted avg'] +>>> print(results["accuracy"]) +0.8 +>>> print(results["PROPN"]["recall"]) +0.5 +``` + +## Limitations and bias + +In contrast to [seqeval](https://github.com/chakki-works/seqeval), the poseval metric treats each token independently and computes the classification report over all concatenated sequences.. + + +## Citation + +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + +## Further References +- [README for seqeval at GitHub](https://github.com/chakki-works/seqeval) +- [Classification report](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.classification_report.html) +- [Issues with seqeval](https://stackoverflow.com/questions/71327693/how-to-disable-seqeval-label-formatting-for-pos-tagging) \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/poseval/app.py b/TextClassification/TextClassification/evaluate/metrics/poseval/app.py new file mode 100644 index 0000000000000000000000000000000000000000..bf11e456a33c0cf6fcacbe5ff0ff20aed4514e32 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/poseval/app.py @@ -0,0 +1,7 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("poseval") + +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/poseval/poseval.py b/TextClassification/TextClassification/evaluate/metrics/poseval/poseval.py new file mode 100644 index 0000000000000000000000000000000000000000..124146cd024c05e01116403d6c4a164165288bd3 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/poseval/poseval.py @@ -0,0 +1,113 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" seqeval metric. """ + +from typing import Union + +import datasets +from sklearn.metrics import classification_report + +import evaluate + + +_CITATION = """\ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + +_DESCRIPTION = """\ +The poseval metric can be used to evaluate POS taggers. Since seqeval does not work well with POS data \ +(see e.g. [here](https://stackoverflow.com/questions/71327693/how-to-disable-seqeval-label-formatting-for-pos-tagging))\ +that is not in IOB format the poseval metric is an alternative. It treats each token in the dataset as independant \ +observation and computes the precision, recall and F1-score irrespective of sentences. It uses scikit-learns's \ +[classification report](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.classification_report.html) \ +to compute the scores. + +""" + +_KWARGS_DESCRIPTION = """ +Computes the poseval metric. + +Args: + predictions: List of List of predicted labels (Estimated targets as returned by a tagger) + references: List of List of reference labels (Ground truth (correct) target values) + zero_division: Which value to substitute as a metric value when encountering zero division. Should be on of 0, 1, + "warn". "warn" acts as 0, but the warning is raised. + +Returns: + 'scores': dict. Summary of the scores for overall and per type + Overall (weighted and macro avg): + 'accuracy': accuracy, + 'precision': precision, + 'recall': recall, + 'f1': F1 score, also known as balanced F-score or F-measure, + Per type: + 'precision': precision, + 'recall': recall, + 'f1': F1 score, also known as balanced F-score or F-measure +Examples: + + >>> predictions = [['INTJ', 'ADP', 'PROPN', 'NOUN', 'PUNCT', 'INTJ', 'ADP', 'PROPN', 'VERB', 'SYM']] + >>> references = [['INTJ', 'ADP', 'PROPN', 'PROPN', 'PUNCT', 'INTJ', 'ADP', 'PROPN', 'PROPN', 'SYM']] + >>> poseval = evaluate.load("poseval") + >>> results = poseval.compute(predictions=predictions, references=references) + >>> print(list(results.keys())) + ['ADP', 'INTJ', 'NOUN', 'PROPN', 'PUNCT', 'SYM', 'VERB', 'accuracy', 'macro avg', 'weighted avg'] + >>> print(results["accuracy"]) + 0.8 + >>> print(results["PROPN"]["recall"]) + 0.5 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Poseval(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + homepage="https://scikit-learn.org", + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("string", id="label"), id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="label"), id="sequence"), + } + ), + codebase_urls=["https://github.com/scikit-learn/scikit-learn"], + ) + + def _compute( + self, + predictions, + references, + zero_division: Union[str, int] = "warn", + ): + report = classification_report( + y_true=[label for ref in references for label in ref], + y_pred=[label for pred in predictions for label in pred], + output_dict=True, + zero_division=zero_division, + ) + + return report diff --git a/TextClassification/TextClassification/evaluate/metrics/poseval/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/poseval/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/poseval/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/precision/README.md b/TextClassification/TextClassification/evaluate/metrics/precision/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a88670c72854c493656d8174ab5bec02ea28fbb7 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/precision/README.md @@ -0,0 +1,142 @@ +--- +title: Precision +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Precision is the fraction of correctly labeled positive examples out of all of the examples that were labeled as positive. It is computed via the equation: + Precision = TP / (TP + FP) + where TP is the True positives (i.e. the examples correctly labeled as positive) and FP is the False positive examples (i.e. the examples incorrectly labeled as positive). +--- + +# Metric Card for Precision + + +## Metric Description + +Precision is the fraction of correctly labeled positive examples out of all of the examples that were labeled as positive. It is computed via the equation: +Precision = TP / (TP + FP) +where TP is the True positives (i.e. the examples correctly labeled as positive) and FP is the False positive examples (i.e. the examples incorrectly labeled as positive). + + +## How to Use + +At minimum, precision takes as input a list of predicted labels, `predictions`, and a list of output labels, `references`. + +```python +>>> precision_metric = evaluate.load("precision") +>>> results = precision_metric.compute(references=[0, 1], predictions=[0, 1]) +>>> print(results) +{'precision': 1.0} +``` + + +### Inputs +- **predictions** (`list` of `int`): Predicted class labels. +- **references** (`list` of `int`): Actual class labels. +- **labels** (`list` of `int`): The set of labels to include when `average` is not set to `'binary'`. If `average` is `None`, it should be the label order. Labels present in the data can be excluded, for example to calculate a multiclass average ignoring a majority negative class. Labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in `predictions` and `references` are used in sorted order. Defaults to None. +- **pos_label** (`int`): The class to be considered the positive class, in the case where `average` is set to `binary`. Defaults to 1. +- **average** (`string`): This parameter is required for multiclass/multilabel targets. If set to `None`, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`. + - 'binary': Only report results for the class specified by `pos_label`. This is applicable only if the classes found in `predictions` and `references` are binary. + - 'micro': Calculate metrics globally by counting the total true positives, false negatives and false positives. + - 'macro': Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. + - 'weighted': Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. This option can result in an F-score that is not between precision and recall. + - 'samples': Calculate metrics for each instance, and find their average (only meaningful for multilabel classification). +- **sample_weight** (`list` of `float`): Sample weights Defaults to None. +- **zero_division** (): Sets the value to return when there is a zero division. Defaults to . + - 0: Returns 0 when there is a zero division. + - 1: Returns 1 when there is a zero division. + - 'warn': Raises warnings and then returns 0 when there is a zero division. + + +### Output Values +- **precision**(`float` or `array` of `float`): Precision score or list of precision scores, depending on the value passed to `average`. Minimum possible value is 0. Maximum possible value is 1. Higher values indicate that fewer negative examples were incorrectly labeled as positive, which means that, generally, higher scores are better. + +Output Example(s): +```python +{'precision': 0.2222222222222222} +``` +```python +{'precision': array([0.66666667, 0.0, 0.0])} +``` + + + + +#### Values from Popular Papers + + +### Examples + +Example 1-A simple binary example +```python +>>> precision_metric = evaluate.load("precision") +>>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0]) +>>> print(results) +{'precision': 0.5} +``` + +Example 2-The same simple binary example as in Example 1, but with `pos_label` set to `0`. +```python +>>> precision_metric = evaluate.load("precision") +>>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], pos_label=0) +>>> print(round(results['precision'], 2)) +0.67 +``` + +Example 3-The same simple binary example as in Example 1, but with `sample_weight` included. +```python +>>> precision_metric = evaluate.load("precision") +>>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], sample_weight=[0.9, 0.5, 3.9, 1.2, 0.3]) +>>> print(results) +{'precision': 0.23529411764705882} +``` + +Example 4-A multiclass example, with different values for the `average` input. +```python +>>> predictions = [0, 2, 1, 0, 0, 1] +>>> references = [0, 1, 2, 0, 1, 2] +>>> results = precision_metric.compute(predictions=predictions, references=references, average='macro') +>>> print(results) +{'precision': 0.2222222222222222} +>>> results = precision_metric.compute(predictions=predictions, references=references, average='micro') +>>> print(results) +{'precision': 0.3333333333333333} +>>> results = precision_metric.compute(predictions=predictions, references=references, average='weighted') +>>> print(results) +{'precision': 0.2222222222222222} +>>> results = precision_metric.compute(predictions=predictions, references=references, average=None) +>>> print([round(res, 2) for res in results['precision']]) +[0.67, 0.0, 0.0] +``` + + +## Limitations and Bias + +[Precision](https://huggingface.co/metrics/precision) and [recall](https://huggingface.co/metrics/recall) are complementary and can be used to measure different aspects of model performance -- using both of them (or an averaged measure like [F1 score](https://huggingface.co/metrics/F1) to better represent different aspects of performance. See [Wikipedia](https://en.wikipedia.org/wiki/Precision_and_recall) for more information. + +## Citation(s) +```bibtex +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +``` + + +## Further References +- [Wikipedia -- Precision and recall](https://en.wikipedia.org/wiki/Precision_and_recall) diff --git a/TextClassification/TextClassification/evaluate/metrics/precision/app.py b/TextClassification/TextClassification/evaluate/metrics/precision/app.py new file mode 100644 index 0000000000000000000000000000000000000000..ec80edf5a54da759b8e1509650826f0a8e8b8bd7 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/precision/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("precision") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/precision/precision.py b/TextClassification/TextClassification/evaluate/metrics/precision/precision.py new file mode 100644 index 0000000000000000000000000000000000000000..4b35aa7e44f262ec9bbc83500bc55471beb465bd --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/precision/precision.py @@ -0,0 +1,145 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Precision metric.""" + +import datasets +from sklearn.metrics import precision_score + +import evaluate + + +_DESCRIPTION = """ +Precision is the fraction of correctly labeled positive examples out of all of the examples that were labeled as positive. It is computed via the equation: +Precision = TP / (TP + FP) +where TP is the True positives (i.e. the examples correctly labeled as positive) and FP is the False positive examples (i.e. the examples incorrectly labeled as positive). +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions (`list` of `int`): Predicted class labels. + references (`list` of `int`): Actual class labels. + labels (`list` of `int`): The set of labels to include when `average` is not set to `'binary'`. If `average` is `None`, it should be the label order. Labels present in the data can be excluded, for example to calculate a multiclass average ignoring a majority negative class. Labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in `predictions` and `references` are used in sorted order. Defaults to None. + pos_label (`int`): The class to be considered the positive class, in the case where `average` is set to `binary`. Defaults to 1. + average (`string`): This parameter is required for multiclass/multilabel targets. If set to `None`, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`. + + - 'binary': Only report results for the class specified by `pos_label`. This is applicable only if the classes found in `predictions` and `references` are binary. + - 'micro': Calculate metrics globally by counting the total true positives, false negatives and false positives. + - 'macro': Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. + - 'weighted': Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. This option can result in an F-score that is not between precision and recall. + - 'samples': Calculate metrics for each instance, and find their average (only meaningful for multilabel classification). + sample_weight (`list` of `float`): Sample weights Defaults to None. + zero_division (`int` or `string`): Sets the value to return when there is a zero division. Defaults to 'warn'. + + - 0: Returns 0 when there is a zero division. + - 1: Returns 1 when there is a zero division. + - 'warn': Raises warnings and then returns 0 when there is a zero division. + +Returns: + precision (`float` or `array` of `float`): Precision score or list of precision scores, depending on the value passed to `average`. Minimum possible value is 0. Maximum possible value is 1. Higher values indicate that fewer negative examples were incorrectly labeled as positive, which means that, generally, higher scores are better. + +Examples: + + Example 1-A simple binary example + >>> precision_metric = evaluate.load("precision") + >>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0]) + >>> print(results) + {'precision': 0.5} + + Example 2-The same simple binary example as in Example 1, but with `pos_label` set to `0`. + >>> precision_metric = evaluate.load("precision") + >>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], pos_label=0) + >>> print(round(results['precision'], 2)) + 0.67 + + Example 3-The same simple binary example as in Example 1, but with `sample_weight` included. + >>> precision_metric = evaluate.load("precision") + >>> results = precision_metric.compute(references=[0, 1, 0, 1, 0], predictions=[0, 0, 1, 1, 0], sample_weight=[0.9, 0.5, 3.9, 1.2, 0.3]) + >>> print(results) + {'precision': 0.23529411764705882} + + Example 4-A multiclass example, with different values for the `average` input. + >>> predictions = [0, 2, 1, 0, 0, 1] + >>> references = [0, 1, 2, 0, 1, 2] + >>> results = precision_metric.compute(predictions=predictions, references=references, average='macro') + >>> print(results) + {'precision': 0.2222222222222222} + >>> results = precision_metric.compute(predictions=predictions, references=references, average='micro') + >>> print(results) + {'precision': 0.3333333333333333} + >>> results = precision_metric.compute(predictions=predictions, references=references, average='weighted') + >>> print(results) + {'precision': 0.2222222222222222} + >>> results = precision_metric.compute(predictions=predictions, references=references, average=None) + >>> print([round(res, 2) for res in results['precision']]) + [0.67, 0.0, 0.0] +""" + + +_CITATION = """ +@article{scikit-learn, + title={Scikit-learn: Machine Learning in {P}ython}, + author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. + and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. + and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and + Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, + journal={Journal of Machine Learning Research}, + volume={12}, + pages={2825--2830}, + year={2011} +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Precision(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("int32")), + "references": datasets.Sequence(datasets.Value("int32")), + } + if self.config_name == "multilabel" + else { + "predictions": datasets.Value("int32"), + "references": datasets.Value("int32"), + } + ), + reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_score.html"], + ) + + def _compute( + self, + predictions, + references, + labels=None, + pos_label=1, + average="binary", + sample_weight=None, + zero_division="warn", + ): + score = precision_score( + references, + predictions, + labels=labels, + pos_label=pos_label, + average=average, + sample_weight=sample_weight, + zero_division=zero_division, + ) + return {"precision": float(score) if score.size == 1 else score} diff --git a/TextClassification/TextClassification/evaluate/metrics/precision/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/precision/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/precision/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/r_squared/README.md b/TextClassification/TextClassification/evaluate/metrics/r_squared/README.md new file mode 100644 index 0000000000000000000000000000000000000000..00c97c6d6a8c1d63ced5ab59047577ab8174ec55 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/r_squared/README.md @@ -0,0 +1,86 @@ +--- +title: r_squared +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.0.2 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + The R^2 (R Squared) metric is a measure of the goodness of fit of a linear regression model. It is the proportion of the variance in the dependent variable that is predictable from the independent variable. +--- + +# Metric Card for R^2 + +## Metric description + +An R-squared value of 1 indicates that the model perfectly explains the variance of the dependent variable. A value of 0 means that the model does not explain any of the variance. Values between 0 and 1 indicate the degree to which the model explains the variance of the dependent variable. + +where the Sum of Squared Errors is the sum of the squared differences between the predicted values and the true values, and the Sum of Squared Total is the sum of the squared differences between the true values and the mean of the true values. + +For example, if an R-squared value for a model is 0.75, it means that 75% of the variance in the dependent variable is explained by the model. + +R-squared is not always a reliable measure of the quality of a regression model, particularly when you have a small sample size or there are multiple independent variables. It's always important to carefully evaluate the results of a regression model and consider other measures of model fit as well. + +R squared can be calculated using the following formula: + +```python +r_squared = 1 - (Sum of Squared Errors / Sum of Squared Total) +``` + +* Calculate the residual sum of squares (RSS), which is the sum of the squared differences between the predicted values and the actual values. +* Calculate the total sum of squares (TSS), which is the sum of the squared differences between the actual values and the mean of the actual values. +* Calculate the R-squared value by taking 1 - (RSS / TSS). + +Here's an example of how to calculate the R-squared value: +```python +r_squared = 1 - (SSR/SST) +``` + +### How to Use Examples: + +The R2 class in the evaluate module can be used to compute the R^2 value for a given set of predictions and references. (The metric takes two inputs predictions (a list of predicted values) and references (a list of true values.)) + +```python +from evaluate import load +>>> r2_metric = evaluate.load("r_squared") +>>> r_squared = r2_metric.compute(predictions=[1, 2, 3, 4], references=[0.9, 2.1, 3.2, 3.8]) +>>> print(r_squared) +0.98 +``` + +Alternatively, if you want to see an example where there is a perfect match between the prediction and reference: +```python +>>> from evaluate import load +>>> r2_metric = evaluate.load("r_squared") +>>> r_squared = r2_metric.compute(predictions=[1, 2, 3, 4], references=[1, 2, 3, 4]) +>>> print(r_squared) +1.0 +``` + +## Limitations and Bias +R^2 is a statistical measure of the goodness of fit of a regression model. It represents the proportion of the variance in the dependent variable that is predictable from the independent variables. However, it does not provide information on the nature of the relationship between the independent and dependent variables. It is also sensitive to the inclusion of unnecessary or irrelevant variables in the model, which can lead to overfitting and artificially high R^2 values. + +## Citation + +```bibtex +@article{r_squared_model, + title={The R^2 Model Metric: A Comprehensive Guide}, + author={John Doe}, + journal={Journal of Model Evaluation}, + volume={10}, + number={2}, + pages={101-112}, + year={2022}, + publisher={Model Evaluation Society}} +``` + +## Further References + +- [The Open University: R-Squared](https://www.open.edu/openlearn/ocw/mod/oucontent/view.php?id=55450§ion=3.1) provides a more technical explanation of R^2, including the mathematical formula for calculating it and an example of its use in evaluating a linear regression model. + +- [Khan Academy: R-Squared](https://www.khanacademy.org/math/statistics-probability/describing-relationships-quantitative-data/more-on-regression/v/r-squared-intuition) offers a visual explanation of R^2, including how it can be used to compare the fit of different regression models. diff --git a/TextClassification/TextClassification/evaluate/metrics/r_squared/app.py b/TextClassification/TextClassification/evaluate/metrics/r_squared/app.py new file mode 100644 index 0000000000000000000000000000000000000000..c03a6103bd583453f86e8e53bb84f8784b754ecb --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/r_squared/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("r_squared") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/r_squared/r_squared.py b/TextClassification/TextClassification/evaluate/metrics/r_squared/r_squared.py new file mode 100644 index 0000000000000000000000000000000000000000..75e0f7e00fca51bff42f924b4d98042d0848f864 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/r_squared/r_squared.py @@ -0,0 +1,115 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""R squared metric.""" + + +import datasets +import numpy as np + +import evaluate + + +_CITATION = """ +@article{williams2006relationship, +title={The relationship between R2 and the correlation coefficient}, +author={Williams, James}, +journal={Journal of Statistics Education}, +volume={14}, +number={2}, +year={2006} +} +""" + +_DESCRIPTION = """ +R^2 (R Squared) is a statistical measure of the goodness of fit of a regression model. It represents the proportion of the variance in the dependent variable that is predictable from the independent variables. + +The R^2 value ranges from 0 to 1, with a higher value indicating a better fit. A value of 0 means that the model does not explain any of the variance in the dependent variable, while a value of 1 means that the model explains all of the variance. + +R^2 can be calculated using the following formula: + +r_squared = 1 - (Sum of Squared Errors / Sum of Squared Total) + +where the Sum of Squared Errors is the sum of the squared differences between the predicted values and the true values, and the Sum of Squared Total is the sum of the squared differences between the true values and the mean of the true values. +""" + +_KWARGS_DESCRIPTION = """ +Computes the R Squared metric. + +Args: + predictions: List of predicted values of the dependent variable + references: List of true values of the dependent variable + zero_division: Which value to substitute as a metric value when encountering zero division. Should be one of 0, 1, + "warn". "warn" acts as 0, but the warning is raised. + +Returns: + R^2 value ranging from 0 to 1, with a higher value indicating a better fit. + +Examples: + >>> r2_metric = evaluate.load("r_squared") + >>> r_squared = r2_metric.compute(predictions=[1, 2, 3, 4], references=[0.9, 2.1, 3.2, 3.8]) + >>> print(r_squared) + 0.98 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class r_squared(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("float", id="sequence"), + "references": datasets.Value("float", id="sequence"), + } + ), + codebase_urls=["https://github.com/scikit-learn/scikit-learn/"], + reference_urls=[ + "https://en.wikipedia.org/wiki/Coefficient_of_determination", + ], + ) + + def _compute(self, predictions=None, references=None): + """ + Computes the coefficient of determination (R-squared) of predictions with respect to references. + + Parameters: + predictions (List or np.ndarray): The predicted values. + references (List or np.ndarray): The true/reference values. + + Returns: + float: The R-squared value, rounded to 3 decimal places. + """ + predictions = np.array(predictions) + references = np.array(references) + + # Calculate mean of the references + mean_references = np.mean(references) + + # Calculate sum of squared residuals + ssr = np.sum((predictions - references) ** 2) + + # Calculate sum of squared total + sst = np.sum((references - mean_references) ** 2) + + # Calculate R Squared + r_squared = 1 - (ssr / sst) + + # Round off to 3 decimal places + rounded_r_squared = round(r_squared, 3) + + return rounded_r_squared diff --git a/TextClassification/TextClassification/evaluate/metrics/r_squared/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/r_squared/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0b77fbc416b18f5668d2f466f2bb0f12957bdde9 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/r_squared/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} diff --git a/TextClassification/TextClassification/evaluate/metrics/recall/README.md b/TextClassification/TextClassification/evaluate/metrics/recall/README.md new file mode 100644 index 0000000000000000000000000000000000000000..58b021c0709484732e0e0524735e7a6a51e44f3f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/recall/README.md @@ -0,0 +1,132 @@ +--- +title: Recall +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Recall is the fraction of the positive examples that were correctly labeled by the model as positive. It can be computed with the equation: + Recall = TP / (TP + FN) + Where TP is the true positives and FN is the false negatives. +--- + +# Metric Card for Recall + + +## Metric Description + +Recall is the fraction of the positive examples that were correctly labeled by the model as positive. It can be computed with the equation: +Recall = TP / (TP + FN) +Where TP is the number of true positives and FN is the number of false negatives. + + +## How to Use + +At minimum, this metric takes as input two `list`s, each containing `int`s: predictions and references. + +```python +>>> recall_metric = evaluate.load('recall') +>>> results = recall_metric.compute(references=[0, 1], predictions=[0, 1]) +>>> print(results) +["{'recall': 1.0}"] +``` + + +### Inputs +- **predictions** (`list` of `int`): The predicted labels. +- **references** (`list` of `int`): The ground truth labels. +- **labels** (`list` of `int`): The set of labels to include when `average` is not set to `binary`, and their order when average is `None`. Labels present in the data can be excluded in this input, for example to calculate a multiclass average ignoring a majority negative class, while labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in y_true and y_pred are used in sorted order. Defaults to None. +- **pos_label** (`int`): The class label to use as the 'positive class' when calculating the recall. Defaults to `1`. +- **average** (`string`): This parameter is required for multiclass/multilabel targets. If None, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`. + - `'binary'`: Only report results for the class specified by `pos_label`. This is applicable only if the target labels and predictions are binary. + - `'micro'`: Calculate metrics globally by counting the total true positives, false negatives, and false positives. + - `'macro'`: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. + - `'weighted'`: Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. Note that it can result in an F-score that is not between precision and recall. + - `'samples'`: Calculate metrics for each instance, and find their average (only meaningful for multilabel classification). +- **sample_weight** (`list` of `float`): Sample weights Defaults to `None`. +- **zero_division** (): Sets the value to return when there is a zero division. Defaults to . + - `'warn'`: If there is a zero division, the return value is `0`, but warnings are also raised. + - `0`: If there is a zero division, the return value is `0`. + - `1`: If there is a zero division, the return value is `1`. + + +### Output Values +- **recall**(`float`, or `array` of `float`, for multiclass targets): Either the general recall score, or the recall scores for individual classes, depending on the values input to `labels` and `average`. Minimum possible value is 0. Maximum possible value is 1. A higher recall means that more of the positive examples have been labeled correctly. Therefore, a higher recall is generally considered better. + +Output Example(s): +```python +{'recall': 1.0} +``` +```python +{'recall': array([1., 0., 0.])} +``` + +This metric outputs a dictionary with one entry, `'recall'`. + + +#### Values from Popular Papers + + +### Examples + +Example 1-A simple example with some errors +```python +>>> recall_metric = evaluate.load('recall') +>>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1]) +>>> print(results) +{'recall': 0.6666666666666666} +``` + +Example 2-The same example as Example 1, but with `pos_label=0` instead of the default `pos_label=1`. +```python +>>> recall_metric = evaluate.load('recall') +>>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1], pos_label=0) +>>> print(results) +{'recall': 0.5} +``` + +Example 3-The same example as Example 1, but with `sample_weight` included. +```python +>>> recall_metric = evaluate.load('recall') +>>> sample_weight = [0.9, 0.2, 0.9, 0.3, 0.8] +>>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1], sample_weight=sample_weight) +>>> print(results) +{'recall': 0.55} +``` + +Example 4-A multiclass example, using different averages. +```python +>>> recall_metric = evaluate.load('recall') +>>> predictions = [0, 2, 1, 0, 0, 1] +>>> references = [0, 1, 2, 0, 1, 2] +>>> results = recall_metric.compute(predictions=predictions, references=references, average='macro') +>>> print(results) +{'recall': 0.3333333333333333} +>>> results = recall_metric.compute(predictions=predictions, references=references, average='micro') +>>> print(results) +{'recall': 0.3333333333333333} +>>> results = recall_metric.compute(predictions=predictions, references=references, average='weighted') +>>> print(results) +{'recall': 0.3333333333333333} +>>> results = recall_metric.compute(predictions=predictions, references=references, average=None) +>>> print(results) +{'recall': array([1., 0., 0.])} +``` + + +## Limitations and Bias + + +## Citation(s) +```bibtex +@article{scikit-learn, title={Scikit-learn: Machine Learning in {P}ython}, author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, journal={Journal of Machine Learning Research}, volume={12}, pages={2825--2830}, year={2011} +``` + + +## Further References diff --git a/TextClassification/TextClassification/evaluate/metrics/recall/app.py b/TextClassification/TextClassification/evaluate/metrics/recall/app.py new file mode 100644 index 0000000000000000000000000000000000000000..3b273568b99eb1c2ddfe2c685bb2ec85afbe1b56 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/recall/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("recall") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/recall/recall.py b/TextClassification/TextClassification/evaluate/metrics/recall/recall.py new file mode 100644 index 0000000000000000000000000000000000000000..8522cfcf66563befa3b8fa3ba5be1f1145fadb7e --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/recall/recall.py @@ -0,0 +1,135 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Recall metric.""" + +import datasets +from sklearn.metrics import recall_score + +import evaluate + + +_DESCRIPTION = """ +Recall is the fraction of the positive examples that were correctly labeled by the model as positive. It can be computed with the equation: +Recall = TP / (TP + FN) +Where TP is the true positives and FN is the false negatives. +""" + + +_KWARGS_DESCRIPTION = """ +Args: +- **predictions** (`list` of `int`): The predicted labels. +- **references** (`list` of `int`): The ground truth labels. +- **labels** (`list` of `int`): The set of labels to include when `average` is not set to `binary`, and their order when average is `None`. Labels present in the data can be excluded in this input, for example to calculate a multiclass average ignoring a majority negative class, while labels not present in the data will result in 0 components in a macro average. For multilabel targets, labels are column indices. By default, all labels in y_true and y_pred are used in sorted order. Defaults to None. +- **pos_label** (`int`): The class label to use as the 'positive class' when calculating the recall. Defaults to `1`. +- **average** (`string`): This parameter is required for multiclass/multilabel targets. If None, the scores for each class are returned. Otherwise, this determines the type of averaging performed on the data. Defaults to `'binary'`. + - `'binary'`: Only report results for the class specified by `pos_label`. This is applicable only if the target labels and predictions are binary. + - `'micro'`: Calculate metrics globally by counting the total true positives, false negatives, and false positives. + - `'macro'`: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. + - `'weighted'`: Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters `'macro'` to account for label imbalance. Note that it can result in an F-score that is not between precision and recall. + - `'samples'`: Calculate metrics for each instance, and find their average (only meaningful for multilabel classification). +- **sample_weight** (`list` of `float`): Sample weights Defaults to `None`. +- **zero_division** (): Sets the value to return when there is a zero division. Defaults to . + - `'warn'`: If there is a zero division, the return value is `0`, but warnings are also raised. + - `0`: If there is a zero division, the return value is `0`. + - `1`: If there is a zero division, the return value is `1`. + +Returns: +- **recall** (`float`, or `array` of `float`): Either the general recall score, or the recall scores for individual classes, depending on the values input to `labels` and `average`. Minimum possible value is 0. Maximum possible value is 1. A higher recall means that more of the positive examples have been labeled correctly. Therefore, a higher recall is generally considered better. + +Examples: + + Example 1-A simple example with some errors + >>> recall_metric = evaluate.load('recall') + >>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1]) + >>> print(results) + {'recall': 0.6666666666666666} + + Example 2-The same example as Example 1, but with `pos_label=0` instead of the default `pos_label=1`. + >>> recall_metric = evaluate.load('recall') + >>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1], pos_label=0) + >>> print(results) + {'recall': 0.5} + + Example 3-The same example as Example 1, but with `sample_weight` included. + >>> recall_metric = evaluate.load('recall') + >>> sample_weight = [0.9, 0.2, 0.9, 0.3, 0.8] + >>> results = recall_metric.compute(references=[0, 0, 1, 1, 1], predictions=[0, 1, 0, 1, 1], sample_weight=sample_weight) + >>> print(results) + {'recall': 0.55} + + Example 4-A multiclass example, using different averages. + >>> recall_metric = evaluate.load('recall') + >>> predictions = [0, 2, 1, 0, 0, 1] + >>> references = [0, 1, 2, 0, 1, 2] + >>> results = recall_metric.compute(predictions=predictions, references=references, average='macro') + >>> print(results) + {'recall': 0.3333333333333333} + >>> results = recall_metric.compute(predictions=predictions, references=references, average='micro') + >>> print(results) + {'recall': 0.3333333333333333} + >>> results = recall_metric.compute(predictions=predictions, references=references, average='weighted') + >>> print(results) + {'recall': 0.3333333333333333} + >>> results = recall_metric.compute(predictions=predictions, references=references, average=None) + >>> print(results) + {'recall': array([1., 0., 0.])} +""" + + +_CITATION = """ +@article{scikit-learn, title={Scikit-learn: Machine Learning in {P}ython}, author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, journal={Journal of Machine Learning Research}, volume={12}, pages={2825--2830}, year={2011} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Recall(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("int32")), + "references": datasets.Sequence(datasets.Value("int32")), + } + if self.config_name == "multilabel" + else { + "predictions": datasets.Value("int32"), + "references": datasets.Value("int32"), + } + ), + reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.recall_score.html"], + ) + + def _compute( + self, + predictions, + references, + labels=None, + pos_label=1, + average="binary", + sample_weight=None, + zero_division="warn", + ): + score = recall_score( + references, + predictions, + labels=labels, + pos_label=pos_label, + average=average, + sample_weight=sample_weight, + zero_division=zero_division, + ) + return {"recall": float(score) if score.size == 1 else score} diff --git a/TextClassification/TextClassification/evaluate/metrics/recall/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/recall/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/recall/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/rl_reliability/README.md b/TextClassification/TextClassification/evaluate/metrics/rl_reliability/README.md new file mode 100644 index 0000000000000000000000000000000000000000..372302c049278bc49ee848e45a61bc71e4ad17b6 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/rl_reliability/README.md @@ -0,0 +1,118 @@ +--- +title: RL Reliability +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Computes the RL reliability metrics from a set of experiments. There is an `"online"` and `"offline"` configuration for evaluation. +--- + +# Metric Card for RL Reliability + +## Metric Description +The RL Reliability Metrics library provides a set of metrics for measuring the reliability of reinforcement learning (RL) algorithms. + +## How to Use + +```python +import evaluate +import numpy as np + +rl_reliability = evaluate.load("rl_reliability", "online") +results = rl_reliability.compute( + timesteps=[np.linspace(0, 2000000, 1000)], + rewards=[np.linspace(0, 100, 1000)] + ) + +rl_reliability = evaluate.load("rl_reliability", "offline") +results = rl_reliability.compute( + timesteps=[np.linspace(0, 2000000, 1000)], + rewards=[np.linspace(0, 100, 1000)] + ) +``` + + +### Inputs +- **timesteps** *(List[int]): For each run a an list/array with its timesteps.* +- **rewards** *(List[float]): For each run a an list/array with its rewards.* + +KWARGS: +- **baseline="default"** *(Union[str, float]) Normalization used for curves. When `"default"` is passed the curves are normalized by their range in the online setting and by the median performance across runs in the offline case. When a float is passed the curves are divided by that value.* +- **eval_points=[50000, 150000, ..., 2000000]** *(List[int]) Statistics will be computed at these points* +- **freq_thresh=0.01** *(float) Frequency threshold for low-pass filtering.* +- **window_size=100000** *(int) Defines a window centered at each eval point.* +- **window_size_trimmed=99000** *(int) To handle shortened curves due to differencing* +- **alpha=0.05** *(float)The "value at risk" (VaR) cutoff point, a float in the range [0,1].* + +### Output Values + +In `"online"` mode: +- HighFreqEnergyWithinRuns: High Frequency across Time (DT) +- IqrWithinRuns: IQR across Time (DT) +- MadWithinRuns: 'MAD across Time (DT) +- StddevWithinRuns: Stddev across Time (DT) +- LowerCVaROnDiffs: Lower CVaR on Differences (SRT) +- UpperCVaROnDiffs: Upper CVaR on Differences (SRT) +- MaxDrawdown: Max Drawdown (LRT) +- LowerCVaROnDrawdown: Lower CVaR on Drawdown (LRT) +- UpperCVaROnDrawdown: Upper CVaR on Drawdown (LRT) +- LowerCVaROnRaw: Lower CVaR on Raw +- UpperCVaROnRaw: Upper CVaR on Raw +- IqrAcrossRuns: IQR across Runs (DR) +- MadAcrossRuns: MAD across Runs (DR) +- StddevAcrossRuns: Stddev across Runs (DR) +- LowerCVaROnAcross: Lower CVaR across Runs (RR) +- UpperCVaROnAcross: Upper CVaR across Runs (RR) +- MedianPerfDuringTraining: Median Performance across Runs + +In `"offline"` mode: +- MadAcrossRollouts: MAD across rollouts (DF) +- IqrAcrossRollouts: IQR across rollouts (DF) +- LowerCVaRAcrossRollouts: Lower CVaR across rollouts (RF) +- UpperCVaRAcrossRollouts: Upper CVaR across rollouts (RF) +- MedianPerfAcrossRollouts: Median Performance across rollouts + + +### Examples +First get the sample data from the repository: + +```bash +wget https://storage.googleapis.com/rl-reliability-metrics/data/tf_agents_example_csv_dataset.tgz +tar -xvzf tf_agents_example_csv_dataset.tgz +``` + +Load the sample data: +```python +dfs = [pd.read_csv(f"./csv_data/sac_humanoid_{i}_train.csv") for i in range(1, 4)] +``` + +Compute the metrics: +```python +rl_reliability = evaluate.load("rl_reliability", "online") +rl_reliability.compute(timesteps=[df["Metrics/EnvironmentSteps"] for df in dfs], + rewards=[df["Metrics/AverageReturn"] for df in dfs]) +``` + +## Limitations and Bias +This implementation of RL reliability metrics does not compute permutation tests to determine whether algorithms are statistically different in their metric values and also does not compute bootstrap confidence intervals on the rankings of the algorithms. See the [original library](https://github.com/google-research/rl-reliability-metrics/) for more resources. + +## Citation + +```bibtex +@conference{rl_reliability_metrics, + title = {Measuring the Reliability of Reinforcement Learning Algorithms}, + author = {Stephanie CY Chan, Sam Fishman, John Canny, Anoop Korattikara, and Sergio Guadarrama}, + booktitle = {International Conference on Learning Representations, Addis Ababa, Ethiopia}, + year = 2020, +} +``` + +## Further References +- Homepage: https://github.com/google-research/rl-reliability-metrics diff --git a/TextClassification/TextClassification/evaluate/metrics/rl_reliability/app.py b/TextClassification/TextClassification/evaluate/metrics/rl_reliability/app.py new file mode 100644 index 0000000000000000000000000000000000000000..58f118ded69c2b589eb1ab89e9c5fd93ab7eb41d --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/rl_reliability/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("rl_reliability", "online") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/rl_reliability/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/rl_reliability/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..514080bf1a2b571122069de43e6faefc6a4d5d98 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/rl_reliability/requirements.txt @@ -0,0 +1,5 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +git+https://github.com/google-research/rl-reliability-metrics +scipy +tensorflow +gin-config \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/rl_reliability/rl_reliability.py b/TextClassification/TextClassification/evaluate/metrics/rl_reliability/rl_reliability.py new file mode 100644 index 0000000000000000000000000000000000000000..34a9c4570cbc2fcd7f4392886b32de6fa17e4dfd --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/rl_reliability/rl_reliability.py @@ -0,0 +1,186 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Computes the RL Reliability Metrics.""" + +import datasets +import numpy as np +from rl_reliability_metrics.evaluation import eval_metrics +from rl_reliability_metrics.metrics import metrics_offline, metrics_online + +import evaluate + + +logger = evaluate.logging.get_logger(__name__) + +DEFAULT_EVAL_POINTS = [ + 50000, + 150000, + 250000, + 350000, + 450000, + 550000, + 650000, + 750000, + 850000, + 950000, + 1050000, + 1150000, + 1250000, + 1350000, + 1450000, + 1550000, + 1650000, + 1750000, + 1850000, + 1950000, +] + +N_RUNS_RECOMMENDED = 10 + +_CITATION = """\ +@conference{rl_reliability_metrics, + title = {Measuring the Reliability of Reinforcement Learning Algorithms}, + author = {Stephanie CY Chan, Sam Fishman, John Canny, Anoop Korattikara, and Sergio Guadarrama}, + booktitle = {International Conference on Learning Representations, Addis Ababa, Ethiopia}, + year = 2020, +} +""" + +_DESCRIPTION = """\ +Computes the RL reliability metrics from a set of experiments. There is an `"online"` and `"offline"` configuration for evaluation. +""" + + +_KWARGS_DESCRIPTION = """ +Computes the RL reliability metrics from a set of experiments. There is an `"online"` and `"offline"` configuration for evaluation. +Args: + timestamps: list of timestep lists/arrays that serve as index. + rewards: list of reward lists/arrays of each experiment. +Returns: + dictionary: a set of reliability metrics +Examples: + >>> import numpy as np + >>> rl_reliability = evaluate.load("rl_reliability", "online") + >>> results = rl_reliability.compute( + ... timesteps=[np.linspace(0, 2000000, 1000)], + ... rewards=[np.linspace(0, 100, 1000)] + ... ) + >>> print(results["LowerCVaROnRaw"].round(4)) + [0.0258] +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class RLReliability(evaluate.Metric): + """Computes the RL Reliability Metrics.""" + + def _info(self): + if self.config_name not in ["online", "offline"]: + raise KeyError("""You should supply a configuration name selected in '["online", "offline"]'""") + + return evaluate.MetricInfo( + module_type="metric", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "timesteps": datasets.Sequence(datasets.Value("int64")), + "rewards": datasets.Sequence(datasets.Value("float")), + } + ), + homepage="https://github.com/google-research/rl-reliability-metrics", + ) + + def _compute( + self, + timesteps, + rewards, + baseline="default", + freq_thresh=0.01, + window_size=100000, + window_size_trimmed=99000, + alpha=0.05, + eval_points=None, + ): + if len(timesteps) < N_RUNS_RECOMMENDED: + logger.warning( + f"For robust statistics it is recommended to use at least {N_RUNS_RECOMMENDED} runs whereas you provided {len(timesteps)}." + ) + + curves = [] + for timestep, reward in zip(timesteps, rewards): + curves.append(np.stack([timestep, reward])) + + if self.config_name == "online": + if baseline == "default": + baseline = "curve_range" + if eval_points is None: + eval_points = DEFAULT_EVAL_POINTS + + metrics = [ + metrics_online.HighFreqEnergyWithinRuns(thresh=freq_thresh), + metrics_online.IqrWithinRuns( + window_size=window_size_trimmed, eval_points=eval_points, baseline=baseline + ), + metrics_online.IqrAcrossRuns( + lowpass_thresh=freq_thresh, eval_points=eval_points, window_size=window_size, baseline=baseline + ), + metrics_online.LowerCVaROnDiffs(baseline=baseline), + metrics_online.LowerCVaROnDrawdown(baseline=baseline), + metrics_online.LowerCVaROnAcross( + lowpass_thresh=freq_thresh, eval_points=eval_points, window_size=window_size, baseline=baseline + ), + metrics_online.LowerCVaROnRaw(alpha=alpha, baseline=baseline), + metrics_online.MadAcrossRuns( + lowpass_thresh=freq_thresh, eval_points=eval_points, window_size=window_size, baseline=baseline + ), + metrics_online.MadWithinRuns( + eval_points=eval_points, window_size=window_size_trimmed, baseline=baseline + ), + metrics_online.MaxDrawdown(), + metrics_online.StddevAcrossRuns( + lowpass_thresh=freq_thresh, eval_points=eval_points, window_size=window_size, baseline=baseline + ), + metrics_online.StddevWithinRuns( + eval_points=eval_points, window_size=window_size_trimmed, baseline=baseline + ), + metrics_online.UpperCVaROnAcross( + alpha=alpha, + lowpass_thresh=freq_thresh, + eval_points=eval_points, + window_size=window_size, + baseline=baseline, + ), + metrics_online.UpperCVaROnDiffs(alpha=alpha, baseline=baseline), + metrics_online.UpperCVaROnDrawdown(alpha=alpha, baseline=baseline), + metrics_online.UpperCVaROnRaw(alpha=alpha, baseline=baseline), + metrics_online.MedianPerfDuringTraining(window_size=window_size, eval_points=eval_points), + ] + else: + if baseline == "default": + baseline = "median_perf" + + metrics = [ + metrics_offline.MadAcrossRollouts(baseline=baseline), + metrics_offline.IqrAcrossRollouts(baseline=baseline), + metrics_offline.StddevAcrossRollouts(baseline=baseline), + metrics_offline.LowerCVaRAcrossRollouts(alpha=alpha, baseline=baseline), + metrics_offline.UpperCVaRAcrossRollouts(alpha=alpha, baseline=baseline), + metrics_offline.MedianPerfAcrossRollouts(baseline=None), + ] + + evaluator = eval_metrics.Evaluator(metrics=metrics) + result = evaluator.compute_metrics(curves) + return result diff --git a/TextClassification/TextClassification/evaluate/metrics/roc_auc/README.md b/TextClassification/TextClassification/evaluate/metrics/roc_auc/README.md new file mode 100644 index 0000000000000000000000000000000000000000..99e3314c958caed6d05fbaa3f8444e734904780c --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/roc_auc/README.md @@ -0,0 +1,204 @@ +--- +title: ROC AUC +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + This metric computes the area under the curve (AUC) for the Receiver Operating Characteristic Curve (ROC). The return values represent how well the model used is predicting the correct classes, based on the input data. A score of `0.5` means that the model is predicting exactly at chance, i.e. the model's predictions are correct at the same rate as if the predictions were being decided by the flip of a fair coin or the roll of a fair die. A score above `0.5` indicates that the model is doing better than chance, while a score below `0.5` indicates that the model is doing worse than chance. + + This metric has three separate use cases: + - binary: The case in which there are only two different label classes, and each example gets only one label. This is the default implementation. + - multiclass: The case in which there can be more than two different label classes, but each example still gets only one label. + - multilabel: The case in which there can be more than two different label classes, and each example can have more than one label. +--- + +# Metric Card for ROC AUC + + +## Metric Description +This metric computes the area under the curve (AUC) for the Receiver Operating Characteristic Curve (ROC). The return values represent how well the model used is predicting the correct classes, based on the input data. A score of `0.5` means that the model is predicting exactly at chance, i.e. the model's predictions are correct at the same rate as if the predictions were being decided by the flip of a fair coin or the roll of a fair die. A score above `0.5` indicates that the model is doing better than chance, while a score below `0.5` indicates that the model is doing worse than chance. + +This metric has three separate use cases: +- **binary**: The case in which there are only two different label classes, and each example gets only one label. This is the default implementation. +- **multiclass**: The case in which there can be more than two different label classes, but each example still gets only one label. +- **multilabel**: The case in which there can be more than two different label classes, and each example can have more than one label. + + +## How to Use +At minimum, this metric requires references and prediction scores: +```python +>>> roc_auc_score = evaluate.load("roc_auc") +>>> refs = [1, 0, 1, 1, 0, 0] +>>> pred_scores = [0.5, 0.2, 0.99, 0.3, 0.1, 0.7] +>>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores) +>>> print(round(results['roc_auc'], 2)) +0.78 +``` + +The default implementation of this metric is the **binary** implementation. If employing the **multiclass** or **multilabel** use cases, the keyword `"multiclass"` or `"multilabel"` must be specified when loading the metric: +- In the **multiclass** case, the metric is loaded with: + ```python + >>> roc_auc_score = evaluate.load("roc_auc", "multiclass") + ``` +- In the **multilabel** case, the metric is loaded with: + ```python + >>> roc_auc_score = evaluate.load("roc_auc", "multilabel") + ``` + +See the [Examples Section Below](#examples_section) for more extensive examples. + + +### Inputs +- **`references`** (array-like of shape (n_samples,) or (n_samples, n_classes)): Ground truth labels. Expects different inputs based on use case: + - binary: expects an array-like of shape (n_samples,) + - multiclass: expects an array-like of shape (n_samples,) + - multilabel: expects an array-like of shape (n_samples, n_classes) +- **`prediction_scores`** (array-like of shape (n_samples,) or (n_samples, n_classes)): Model predictions. Expects different inputs based on use case: + - binary: expects an array-like of shape (n_samples,) + - multiclass: expects an array-like of shape (n_samples, n_classes). The probability estimates must sum to 1 across the possible classes. + - multilabel: expects an array-like of shape (n_samples, n_classes) +- **`average`** (`str`): Type of average, and is ignored in the binary use case. Defaults to `'macro'`. Options are: + - `'micro'`: Calculates metrics globally by considering each element of the label indicator matrix as a label. Only works with the multilabel use case. + - `'macro'`: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. + - `'weighted'`: Calculate metrics for each label, and find their average, weighted by support (i.e. the number of true instances for each label). + - `'samples'`: Calculate metrics for each instance, and find their average. Only works with the multilabel use case. + - `None`: No average is calculated, and scores for each class are returned. Only works with the multilabels use case. +- **`sample_weight`** (array-like of shape (n_samples,)): Sample weights. Defaults to None. +- **`max_fpr`** (`float`): If not None, the standardized partial AUC over the range [0, `max_fpr`] is returned. Must be greater than `0` and less than or equal to `1`. Defaults to `None`. Note: For the multiclass use case, `max_fpr` should be either `None` or `1.0` as ROC AUC partial computation is not currently supported for `multiclass`. +- **`multi_class`** (`str`): Only used for multiclass targets, in which case it is required. Determines the type of configuration to use. Options are: + - `'ovr'`: Stands for One-vs-rest. Computes the AUC of each class against the rest. This treats the multiclass case in the same way as the multilabel case. Sensitive to class imbalance even when `average == 'macro'`, because class imbalance affects the composition of each of the 'rest' groupings. + - `'ovo'`: Stands for One-vs-one. Computes the average AUC of all possible pairwise combinations of classes. Insensitive to class imbalance when `average == 'macro'`. +- **`labels`** (array-like of shape (n_classes,)): Only used for multiclass targets. List of labels that index the classes in `prediction_scores`. If `None`, the numerical or lexicographical order of the labels in `prediction_scores` is used. Defaults to `None`. + +### Output Values +This metric returns a dict containing the `roc_auc` score. The score is a `float`, unless it is the multilabel case with `average=None`, in which case the score is a numpy `array` with entries of type `float`. + +The output therefore generally takes the following format: +```python +{'roc_auc': 0.778} +``` + +In contrast, though, the output takes the following format in the multilabel case when `average=None`: +```python +{'roc_auc': array([0.83333333, 0.375, 0.94444444])} +``` + +ROC AUC scores can take on any value between `0` and `1`, inclusive. + +#### Values from Popular Papers + + +### Examples +Example 1, the **binary** use case: +```python +>>> roc_auc_score = evaluate.load("roc_auc") +>>> refs = [1, 0, 1, 1, 0, 0] +>>> pred_scores = [0.5, 0.2, 0.99, 0.3, 0.1, 0.7] +>>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores) +>>> print(round(results['roc_auc'], 2)) +0.78 +``` + +Example 2, the **multiclass** use case: +```python +>>> roc_auc_score = evaluate.load("roc_auc", "multiclass") +>>> refs = [1, 0, 1, 2, 2, 0] +>>> pred_scores = [[0.3, 0.5, 0.2], +... [0.7, 0.2, 0.1], +... [0.005, 0.99, 0.005], +... [0.2, 0.3, 0.5], +... [0.1, 0.1, 0.8], +... [0.1, 0.7, 0.2]] +>>> results = roc_auc_score.compute(references=refs, +... prediction_scores=pred_scores, +... multi_class='ovr') +>>> print(round(results['roc_auc'], 2)) +0.85 +``` + +Example 3, the **multilabel** use case: +```python +>>> roc_auc_score = evaluate.load("roc_auc", "multilabel") +>>> refs = [[1, 1, 0], +... [1, 1, 0], +... [0, 1, 0], +... [0, 0, 1], +... [0, 1, 1], +... [1, 0, 1]] +>>> pred_scores = [[0.3, 0.5, 0.2], +... [0.7, 0.2, 0.1], +... [0.005, 0.99, 0.005], +... [0.2, 0.3, 0.5], +... [0.1, 0.1, 0.8], +... [0.1, 0.7, 0.2]] +>>> results = roc_auc_score.compute(references=refs, +... prediction_scores=pred_scores, +... average=None) +>>> print([round(res, 2) for res in results['roc_auc']) +[0.83, 0.38, 0.94] +``` + + +## Limitations and Bias + + +## Citation +```bibtex +@article{doi:10.1177/0272989X8900900307, +author = {Donna Katzman McClish}, +title ={Analyzing a Portion of the ROC Curve}, +journal = {Medical Decision Making}, +volume = {9}, +number = {3}, +pages = {190-195}, +year = {1989}, +doi = {10.1177/0272989X8900900307}, + note ={PMID: 2668680}, +URL = {https://doi.org/10.1177/0272989X8900900307}, +eprint = {https://doi.org/10.1177/0272989X8900900307} +} +``` + +```bibtex +@article{10.1023/A:1010920819831, +author = {Hand, David J. and Till, Robert J.}, +title = {A Simple Generalisation of the Area Under the ROC Curve for Multiple Class Classification Problems}, +year = {2001}, +issue_date = {November 2001}, +publisher = {Kluwer Academic Publishers}, +address = {USA}, +volume = {45}, +number = {2}, +issn = {0885-6125}, +url = {https://doi.org/10.1023/A:1010920819831}, +doi = {10.1023/A:1010920819831}, +journal = {Mach. Learn.}, +month = {oct}, +pages = {171–186}, +numpages = {16}, +keywords = {Gini index, AUC, error rate, ROC curve, receiver operating characteristic} +} +``` + +```bibtex +@article{scikit-learn, +title={Scikit-learn: Machine Learning in {P}ython}, +author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, +journal={Journal of Machine Learning Research}, +volume={12}, +pages={2825--2830}, +year={2011} +} +``` + +## Further References +This implementation is a wrapper around the [Scikit-learn implementation](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_auc_score.html). Much of the documentation here was adapted from their existing documentation, as well. + +The [Guide to ROC and AUC](https://youtu.be/iCZJfO-7C5Q) video from the channel Data Science Bits is also very informative. diff --git a/TextClassification/TextClassification/evaluate/metrics/roc_auc/app.py b/TextClassification/TextClassification/evaluate/metrics/roc_auc/app.py new file mode 100644 index 0000000000000000000000000000000000000000..4830128cf25675a2a6fe810c36f0469ff40f6c67 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/roc_auc/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("roc_auc") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/roc_auc/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/roc_auc/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/roc_auc/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/roc_auc/roc_auc.py b/TextClassification/TextClassification/evaluate/metrics/roc_auc/roc_auc.py new file mode 100644 index 0000000000000000000000000000000000000000..604f93ab3d7b5c0cd6428df1d67a3a1791a954e3 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/roc_auc/roc_auc.py @@ -0,0 +1,191 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Accuracy metric.""" + +import datasets +from sklearn.metrics import roc_auc_score + +import evaluate + + +_DESCRIPTION = """ +This metric computes the area under the curve (AUC) for the Receiver Operating Characteristic Curve (ROC). The return values represent how well the model used is predicting the correct classes, based on the input data. A score of `0.5` means that the model is predicting exactly at chance, i.e. the model's predictions are correct at the same rate as if the predictions were being decided by the flip of a fair coin or the roll of a fair die. A score above `0.5` indicates that the model is doing better than chance, while a score below `0.5` indicates that the model is doing worse than chance. + +This metric has three separate use cases: + - binary: The case in which there are only two different label classes, and each example gets only one label. This is the default implementation. + - multiclass: The case in which there can be more than two different label classes, but each example still gets only one label. + - multilabel: The case in which there can be more than two different label classes, and each example can have more than one label. +""" + +_KWARGS_DESCRIPTION = """ +Args: +- references (array-like of shape (n_samples,) or (n_samples, n_classes)): Ground truth labels. Expects different input based on use case: + - binary: expects an array-like of shape (n_samples,) + - multiclass: expects an array-like of shape (n_samples,) + - multilabel: expects an array-like of shape (n_samples, n_classes) +- prediction_scores (array-like of shape (n_samples,) or (n_samples, n_classes)): Model predictions. Expects different inputs based on use case: + - binary: expects an array-like of shape (n_samples,) + - multiclass: expects an array-like of shape (n_samples, n_classes) + - multilabel: expects an array-like of shape (n_samples, n_classes) +- average (`str`): Type of average, and is ignored in the binary use case. Defaults to 'macro'. Options are: + - `'micro'`: Calculates metrics globally by considering each element of the label indicator matrix as a label. Only works with the multilabel use case. + - `'macro'`: Calculate metrics for each label, and find their unweighted mean. This does not take label imbalance into account. + - `'weighted'`: Calculate metrics for each label, and find their average, weighted by support (i.e. the number of true instances for each label). + - `'samples'`: Calculate metrics for each instance, and find their average. Only works with the multilabel use case. + - `None`: No average is calculated, and scores for each class are returned. Only works with the multilabels use case. +- sample_weight (array-like of shape (n_samples,)): Sample weights. Defaults to None. +- max_fpr (`float`): If not None, the standardized partial AUC over the range [0, `max_fpr`] is returned. Must be greater than `0` and less than or equal to `1`. Defaults to `None`. Note: For the multiclass use case, `max_fpr` should be either `None` or `1.0` as ROC AUC partial computation is not currently supported for `multiclass`. +- multi_class (`str`): Only used for multiclass targets, where it is required. Determines the type of configuration to use. Options are: + - `'ovr'`: Stands for One-vs-rest. Computes the AUC of each class against the rest. This treats the multiclass case in the same way as the multilabel case. Sensitive to class imbalance even when `average == 'macro'`, because class imbalance affects the composition of each of the 'rest' groupings. + - `'ovo'`: Stands for One-vs-one. Computes the average AUC of all possible pairwise combinations of classes. Insensitive to class imbalance when `average == 'macro'`. +- labels (array-like of shape (n_classes,)): Only used for multiclass targets. List of labels that index the classes in + `prediction_scores`. If `None`, the numerical or lexicographical order of the labels in + `prediction_scores` is used. Defaults to `None`. +Returns: + roc_auc (`float` or array-like of shape (n_classes,)): Returns array if in multilabel use case and `average='None'`. Otherwise, returns `float`. +Examples: + Example 1: + >>> roc_auc_score = evaluate.load("roc_auc") + >>> refs = [1, 0, 1, 1, 0, 0] + >>> pred_scores = [0.5, 0.2, 0.99, 0.3, 0.1, 0.7] + >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores) + >>> print(round(results['roc_auc'], 2)) + 0.78 + + Example 2: + >>> roc_auc_score = evaluate.load("roc_auc", "multiclass") + >>> refs = [1, 0, 1, 2, 2, 0] + >>> pred_scores = [[0.3, 0.5, 0.2], + ... [0.7, 0.2, 0.1], + ... [0.005, 0.99, 0.005], + ... [0.2, 0.3, 0.5], + ... [0.1, 0.1, 0.8], + ... [0.1, 0.7, 0.2]] + >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores, multi_class='ovr') + >>> print(round(results['roc_auc'], 2)) + 0.85 + + Example 3: + >>> roc_auc_score = evaluate.load("roc_auc", "multilabel") + >>> refs = [[1, 1, 0], + ... [1, 1, 0], + ... [0, 1, 0], + ... [0, 0, 1], + ... [0, 1, 1], + ... [1, 0, 1]] + >>> pred_scores = [[0.3, 0.5, 0.2], + ... [0.7, 0.2, 0.1], + ... [0.005, 0.99, 0.005], + ... [0.2, 0.3, 0.5], + ... [0.1, 0.1, 0.8], + ... [0.1, 0.7, 0.2]] + >>> results = roc_auc_score.compute(references=refs, prediction_scores=pred_scores, average=None) + >>> print([round(res, 2) for res in results['roc_auc']]) + [0.83, 0.38, 0.94] +""" + +_CITATION = """\ +@article{doi:10.1177/0272989X8900900307, +author = {Donna Katzman McClish}, +title ={Analyzing a Portion of the ROC Curve}, +journal = {Medical Decision Making}, +volume = {9}, +number = {3}, +pages = {190-195}, +year = {1989}, +doi = {10.1177/0272989X8900900307}, + note ={PMID: 2668680}, +URL = {https://doi.org/10.1177/0272989X8900900307}, +eprint = {https://doi.org/10.1177/0272989X8900900307} +} + + +@article{10.1023/A:1010920819831, +author = {Hand, David J. and Till, Robert J.}, +title = {A Simple Generalisation of the Area Under the ROC Curve for Multiple Class Classification Problems}, +year = {2001}, +issue_date = {November 2001}, +publisher = {Kluwer Academic Publishers}, +address = {USA}, +volume = {45}, +number = {2}, +issn = {0885-6125}, +url = {https://doi.org/10.1023/A:1010920819831}, +doi = {10.1023/A:1010920819831}, +journal = {Mach. Learn.}, +month = {oct}, +pages = {171–186}, +numpages = {16}, +keywords = {Gini index, AUC, error rate, ROC curve, receiver operating characteristic} +} + + +@article{scikit-learn, +title={Scikit-learn: Machine Learning in {P}ython}, +author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V. and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P. and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.}, +journal={Journal of Machine Learning Research}, +volume={12}, +pages={2825--2830}, +year={2011} +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class ROCAUC(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "prediction_scores": datasets.Sequence(datasets.Value("float")), + "references": datasets.Value("int32"), + } + if self.config_name == "multiclass" + else { + "references": datasets.Sequence(datasets.Value("int32")), + "prediction_scores": datasets.Sequence(datasets.Value("float")), + } + if self.config_name == "multilabel" + else { + "references": datasets.Value("int32"), + "prediction_scores": datasets.Value("float"), + } + ), + reference_urls=["https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_auc_score.html"], + ) + + def _compute( + self, + references, + prediction_scores, + average="macro", + sample_weight=None, + max_fpr=None, + multi_class="raise", + labels=None, + ): + return { + "roc_auc": roc_auc_score( + references, + prediction_scores, + average=average, + sample_weight=sample_weight, + max_fpr=max_fpr, + multi_class=multi_class, + labels=labels, + ) + } diff --git a/TextClassification/TextClassification/evaluate/metrics/rouge/README.md b/TextClassification/TextClassification/evaluate/metrics/rouge/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5c143c4978d02231681009e82f9b1154c0733414 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/rouge/README.md @@ -0,0 +1,161 @@ +--- +title: ROUGE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + ROUGE, or Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics and a software package used for + evaluating automatic summarization and machine translation software in natural language processing. + The metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation. + + Note that ROUGE is case insensitive, meaning that upper case letters are treated the same way as lower case letters. + + This metrics is a wrapper around Google Research reimplementation of ROUGE: + https://github.com/google-research/google-research/tree/master/rouge +--- + +# Metric Card for ROUGE + +## Metric Description +ROUGE, or Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics and a software package used for evaluating automatic summarization and machine translation software in natural language processing. The metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation. + +Note that ROUGE is case insensitive, meaning that upper case letters are treated the same way as lower case letters. + +This metrics is a wrapper around the [Google Research reimplementation of ROUGE](https://github.com/google-research/google-research/tree/master/rouge) + +## How to Use +At minimum, this metric takes as input a list of predictions and a list of references: +```python +>>> rouge = evaluate.load('rouge') +>>> predictions = ["hello there", "general kenobi"] +>>> references = ["hello there", "general kenobi"] +>>> results = rouge.compute(predictions=predictions, +... references=references) +>>> print(results) +{'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0} +``` + +One can also pass a custom tokenizer which is especially useful for non-latin languages. +```python +>>> results = rouge.compute(predictions=predictions, +... references=references, + tokenizer=lambda x: x.split()) +>>> print(results) +{'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0} +``` + +It can also deal with lists of references for each predictions: +```python +>>> rouge = evaluate.load('rouge') +>>> predictions = ["hello there", "general kenobi"] +>>> references = [["hello", "there"], ["general kenobi", "general yoda"]] +>>> results = rouge.compute(predictions=predictions, +... references=references) +>>> print(results) +{'rouge1': 0.8333, 'rouge2': 0.5, 'rougeL': 0.8333, 'rougeLsum': 0.8333}``` +``` + +### Inputs +- **predictions** (`list`): list of predictions to score. Each prediction + should be a string with tokens separated by spaces. +- **references** (`list` or `list[list]`): list of reference for each prediction or a list of several references per prediction. Each + reference should be a string with tokens separated by spaces. +- **rouge_types** (`list`): A list of rouge types to calculate. Defaults to `['rouge1', 'rouge2', 'rougeL', 'rougeLsum']`. + - Valid rouge types: + - `"rouge1"`: unigram (1-gram) based scoring + - `"rouge2"`: bigram (2-gram) based scoring + - `"rougeL"`: Longest common subsequence based scoring. + - `"rougeLSum"`: splits text using `"\n"` + - See [here](https://github.com/huggingface/datasets/issues/617) for more information +- **use_aggregator** (`boolean`): If True, returns aggregates. Defaults to `True`. +- **use_stemmer** (`boolean`): If `True`, uses Porter stemmer to strip word suffixes. Defaults to `False`. + +### Output Values +The output is a dictionary with one entry for each rouge type in the input list `rouge_types`. If `use_aggregator=False`, each dictionary entry is a list of scores, with one score for each sentence. E.g. if `rouge_types=['rouge1', 'rouge2']` and `use_aggregator=False`, the output is: + +```python +{'rouge1': [0.6666666666666666, 1.0], 'rouge2': [0.0, 1.0]} +``` + +If `rouge_types=['rouge1', 'rouge2']` and `use_aggregator=True`, the output is of the following format: +```python +{'rouge1': 1.0, 'rouge2': 1.0} +``` + +The ROUGE values are in the range of 0 to 1. + + +#### Values from Popular Papers + + +### Examples +An example without aggregation: +```python +>>> rouge = evaluate.load('rouge') +>>> predictions = ["hello goodbye", "ankh morpork"] +>>> references = ["goodbye", "general kenobi"] +>>> results = rouge.compute(predictions=predictions, +... references=references, +... use_aggregator=False) +>>> print(list(results.keys())) +['rouge1', 'rouge2', 'rougeL', 'rougeLsum'] +>>> print(results["rouge1"]) +[0.5, 0.0] +``` + +The same example, but with aggregation: +```python +>>> rouge = evaluate.load('rouge') +>>> predictions = ["hello goodbye", "ankh morpork"] +>>> references = ["goodbye", "general kenobi"] +>>> results = rouge.compute(predictions=predictions, +... references=references, +... use_aggregator=True) +>>> print(list(results.keys())) +['rouge1', 'rouge2', 'rougeL', 'rougeLsum'] +>>> print(results["rouge1"]) +0.25 +``` + +The same example, but only calculating `rouge_1`: +```python +>>> rouge = evaluate.load('rouge') +>>> predictions = ["hello goodbye", "ankh morpork"] +>>> references = ["goodbye", "general kenobi"] +>>> results = rouge.compute(predictions=predictions, +... references=references, +... rouge_types=['rouge_1'], +... use_aggregator=True) +>>> print(list(results.keys())) +['rouge1'] +>>> print(results["rouge1"]) +0.25 +``` + +## Limitations and Bias +See [Schluter (2017)](https://aclanthology.org/E17-2007/) for an in-depth discussion of many of ROUGE's limits. + +## Citation +```bibtex +@inproceedings{lin-2004-rouge, + title = "{ROUGE}: A Package for Automatic Evaluation of Summaries", + author = "Lin, Chin-Yew", + booktitle = "Text Summarization Branches Out", + month = jul, + year = "2004", + address = "Barcelona, Spain", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/W04-1013", + pages = "74--81", +} +``` + +## Further References +- This metrics is a wrapper around the [Google Research reimplementation of ROUGE](https://github.com/google-research/google-research/tree/master/rouge) \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/rouge/app.py b/TextClassification/TextClassification/evaluate/metrics/rouge/app.py new file mode 100644 index 0000000000000000000000000000000000000000..38eb3fd38c21e1715cab970448654c9dc8a4f7bf --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/rouge/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("rouge") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/rouge/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/rouge/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7df8b26f3b8f204880d345de2e27b1349483a06a --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/rouge/requirements.txt @@ -0,0 +1,4 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +absl-py +nltk +rouge_score>=0.1.2 \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/rouge/rouge.py b/TextClassification/TextClassification/evaluate/metrics/rouge/rouge.py new file mode 100644 index 0000000000000000000000000000000000000000..353301cca11fb5c7d4f0b0e70cde1560b4139bc7 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/rouge/rouge.py @@ -0,0 +1,158 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" ROUGE metric from Google Research github repo. """ + +# The dependencies in https://github.com/google-research/google-research/blob/master/rouge/requirements.txt +import absl # Here to have a nice missing dependency error message early on +import datasets +import nltk # Here to have a nice missing dependency error message early on +import numpy # Here to have a nice missing dependency error message early on +import six # Here to have a nice missing dependency error message early on +from rouge_score import rouge_scorer, scoring + +import evaluate + + +_CITATION = """\ +@inproceedings{lin-2004-rouge, + title = "{ROUGE}: A Package for Automatic Evaluation of Summaries", + author = "Lin, Chin-Yew", + booktitle = "Text Summarization Branches Out", + month = jul, + year = "2004", + address = "Barcelona, Spain", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/W04-1013", + pages = "74--81", +} +""" + +_DESCRIPTION = """\ +ROUGE, or Recall-Oriented Understudy for Gisting Evaluation, is a set of metrics and a software package used for +evaluating automatic summarization and machine translation software in natural language processing. +The metrics compare an automatically produced summary or translation against a reference or a set of references (human-produced) summary or translation. + +Note that ROUGE is case insensitive, meaning that upper case letters are treated the same way as lower case letters. + +This metrics is a wrapper around Google Research reimplementation of ROUGE: +https://github.com/google-research/google-research/tree/master/rouge +""" + +_KWARGS_DESCRIPTION = """ +Calculates average rouge scores for a list of hypotheses and references +Args: + predictions: list of predictions to score. Each prediction + should be a string with tokens separated by spaces. + references: list of reference for each prediction. Each + reference should be a string with tokens separated by spaces. + rouge_types: A list of rouge types to calculate. + Valid names: + `"rouge{n}"` (e.g. `"rouge1"`, `"rouge2"`) where: {n} is the n-gram based scoring, + `"rougeL"`: Longest common subsequence based scoring. + `"rougeLsum"`: rougeLsum splits text using `"\n"`. + See details in https://github.com/huggingface/datasets/issues/617 + use_stemmer: Bool indicating whether Porter stemmer should be used to strip word suffixes. + use_aggregator: Return aggregates if this is set to True +Returns: + rouge1: rouge_1 (f1), + rouge2: rouge_2 (f1), + rougeL: rouge_l (f1), + rougeLsum: rouge_lsum (f1) +Examples: + + >>> rouge = evaluate.load('rouge') + >>> predictions = ["hello there", "general kenobi"] + >>> references = ["hello there", "general kenobi"] + >>> results = rouge.compute(predictions=predictions, references=references) + >>> print(results) + {'rouge1': 1.0, 'rouge2': 1.0, 'rougeL': 1.0, 'rougeLsum': 1.0} +""" + + +class Tokenizer: + """Helper class to wrap a callable into a class with a `tokenize` method as used by rouge-score.""" + + def __init__(self, tokenizer_func): + self.tokenizer_func = tokenizer_func + + def tokenize(self, text): + return self.tokenizer_func(text) + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Rouge(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence")), + } + ), + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + ], + codebase_urls=["https://github.com/google-research/google-research/tree/master/rouge"], + reference_urls=[ + "https://en.wikipedia.org/wiki/ROUGE_(metric)", + "https://github.com/google-research/google-research/tree/master/rouge", + ], + ) + + def _compute( + self, predictions, references, rouge_types=None, use_aggregator=True, use_stemmer=False, tokenizer=None + ): + if rouge_types is None: + rouge_types = ["rouge1", "rouge2", "rougeL", "rougeLsum"] + + multi_ref = isinstance(references[0], list) + + if tokenizer is not None: + tokenizer = Tokenizer(tokenizer) + + scorer = rouge_scorer.RougeScorer(rouge_types=rouge_types, use_stemmer=use_stemmer, tokenizer=tokenizer) + if use_aggregator: + aggregator = scoring.BootstrapAggregator() + else: + scores = [] + + for ref, pred in zip(references, predictions): + if multi_ref: + score = scorer.score_multi(ref, pred) + else: + score = scorer.score(ref, pred) + if use_aggregator: + aggregator.add_scores(score) + else: + scores.append(score) + + if use_aggregator: + result = aggregator.aggregate() + for key in result: + result[key] = result[key].mid.fmeasure + + else: + result = {} + for key in scores[0]: + result[key] = list(score[key].fmeasure for score in scores) + + return result diff --git a/TextClassification/TextClassification/evaluate/metrics/sacrebleu/README.md b/TextClassification/TextClassification/evaluate/metrics/sacrebleu/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a86a070022c235a2e9297597c4301b95045689a0 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/sacrebleu/README.md @@ -0,0 +1,119 @@ +--- +title: SacreBLEU +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + SacreBLEU provides hassle-free computation of shareable, comparable, and reproducible BLEU scores. + Inspired by Rico Sennrich's `multi-bleu-detok.perl`, it produces the official WMT scores but works with plain text. + It also knows all the standard test sets and handles downloading, processing, and tokenization for you. + + See the [README.md] file at https://github.com/mjpost/sacreBLEU for more information. +--- + +# Metric Card for SacreBLEU + + +## Metric Description +SacreBLEU provides hassle-free computation of shareable, comparable, and reproducible BLEU scores. Inspired by Rico Sennrich's `multi-bleu-detok.perl`, it produces the official Workshop on Machine Translation (WMT) scores but works with plain text. It also knows all the standard test sets and handles downloading, processing, and tokenization. + +See the [README.md] file at https://github.com/mjpost/sacreBLEU for more information. + +## How to Use +This metric takes a set of predictions and a set of references as input, along with various optional parameters. + + +```python +>>> predictions = ["hello there general kenobi", "foo bar foobar"] +>>> references = [["hello there general kenobi", "hello there !"], +... ["foo bar foobar", "foo bar foobar"]] +>>> sacrebleu = evaluate.load("sacrebleu") +>>> results = sacrebleu.compute(predictions=predictions, +... references=references) +>>> print(list(results.keys())) +['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len'] +>>> print(round(results["score"], 1)) +100.0 +``` + +### Inputs +- **`predictions`** (`list` of `str`): list of translations to score. Each translation should be tokenized into a list of tokens. +- **`references`** (`list` of `list` of `str`): A list of lists of references. The contents of the first sub-list are the references for the first prediction, the contents of the second sub-list are for the second prediction, etc. Note that there must be the same number of references for each prediction (i.e. all sub-lists must be of the same length). +- **`smooth_method`** (`str`): The smoothing method to use, defaults to `'exp'`. Possible values are: + - `'none'`: no smoothing + - `'floor'`: increment zero counts + - `'add-k'`: increment num/denom by k for n>1 + - `'exp'`: exponential decay +- **`smooth_value`** (`float`): The smoothing value. Only valid when `smooth_method='floor'` (in which case `smooth_value` defaults to `0.1`) or `smooth_method='add-k'` (in which case `smooth_value` defaults to `1`). +- **`tokenize`** (`str`): Tokenization method to use for BLEU. If not provided, defaults to `'zh'` for Chinese, `'ja-mecab'` for Japanese and `'13a'` (mteval) otherwise. Possible values are: + - `'none'`: No tokenization. + - `'zh'`: Chinese tokenization. + - `'13a'`: mimics the `mteval-v13a` script from Moses. + - `'intl'`: International tokenization, mimics the `mteval-v14` script from Moses + - `'char'`: Language-agnostic character-level tokenization. + - `'ja-mecab'`: Japanese tokenization. Uses the [MeCab tokenizer](https://pypi.org/project/mecab-python3). +- **`lowercase`** (`bool`): If `True`, lowercases the input, enabling case-insensitivity. Defaults to `False`. +- **`force`** (`bool`): If `True`, insists that your tokenized input is actually detokenized. Defaults to `False`. +- **`use_effective_order`** (`bool`): If `True`, stops including n-gram orders for which precision is 0. This should be `True`, if sentence-level BLEU will be computed. Defaults to `False`. + +### Output Values +- `score`: BLEU score +- `counts`: Counts +- `totals`: Totals +- `precisions`: Precisions +- `bp`: Brevity penalty +- `sys_len`: predictions length +- `ref_len`: reference length + +The output is in the following format: +```python +{'score': 39.76353643835252, 'counts': [6, 4, 2, 1], 'totals': [10, 8, 6, 4], 'precisions': [60.0, 50.0, 33.333333333333336, 25.0], 'bp': 1.0, 'sys_len': 10, 'ref_len': 7} +``` +The score can take any value between `0.0` and `100.0`, inclusive. + +#### Values from Popular Papers + + +### Examples + +```python +>>> predictions = ["hello there general kenobi", +... "on our way to ankh morpork"] +>>> references = [["hello there general kenobi", "hello there !"], +... ["goodbye ankh morpork", "ankh morpork"]] +>>> sacrebleu = evaluate.load("sacrebleu") +>>> results = sacrebleu.compute(predictions=predictions, +... references=references) +>>> print(list(results.keys())) +['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len'] +>>> print(round(results["score"], 1)) +39.8 +``` + +## Limitations and Bias +Because what this metric calculates is BLEU scores, it has the same limitations as that metric, except that sacreBLEU is more easily reproducible. + +## Citation +```bibtex +@inproceedings{post-2018-call, + title = "A Call for Clarity in Reporting {BLEU} Scores", + author = "Post, Matt", + booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", + month = oct, + year = "2018", + address = "Belgium, Brussels", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/W18-6319", + pages = "186--191", +} +``` + +## Further References +- See the [sacreBLEU README.md file](https://github.com/mjpost/sacreBLEU) for more information. \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/sacrebleu/app.py b/TextClassification/TextClassification/evaluate/metrics/sacrebleu/app.py new file mode 100644 index 0000000000000000000000000000000000000000..5c77b0098c7f80e74cd3b2197f12b6e087cab35e --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/sacrebleu/app.py @@ -0,0 +1,11 @@ +import sys + +import evaluate +from evaluate.utils import launch_gradio_widget + + +sys.path = [p for p in sys.path if p != "/home/user/app"] +module = evaluate.load("sacrebleu") +sys.path = ["/home/user/app"] + sys.path + +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/sacrebleu/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/sacrebleu/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..9d3e3c80b77038f530d173d82e6419d76b5e72f0 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/sacrebleu/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +sacrebleu \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/sacrebleu/sacrebleu.py b/TextClassification/TextClassification/evaluate/metrics/sacrebleu/sacrebleu.py new file mode 100644 index 0000000000000000000000000000000000000000..6e756f4d4c9bc78390e3bb0d104f0f4515c2a0b7 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/sacrebleu/sacrebleu.py @@ -0,0 +1,178 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" SACREBLEU metric. """ + +import datasets +import sacrebleu as scb +from packaging import version + +import evaluate + + +_CITATION = """\ +@inproceedings{post-2018-call, + title = "A Call for Clarity in Reporting {BLEU} Scores", + author = "Post, Matt", + booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", + month = oct, + year = "2018", + address = "Belgium, Brussels", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/W18-6319", + pages = "186--191", +} +""" + +_DESCRIPTION = """\ +SacreBLEU provides hassle-free computation of shareable, comparable, and reproducible BLEU scores. +Inspired by Rico Sennrich's `multi-bleu-detok.perl`, it produces the official WMT scores but works with plain text. +It also knows all the standard test sets and handles downloading, processing, and tokenization for you. + +See the [README.md] file at https://github.com/mjpost/sacreBLEU for more information. +""" + +_KWARGS_DESCRIPTION = """ +Produces BLEU scores along with its sufficient statistics +from a source against one or more references. + +Args: + predictions (`list` of `str`): list of translations to score. Each translation should be tokenized into a list of tokens. + references (`list` of `list` of `str`): A list of lists of references. The contents of the first sub-list are the references for the first prediction, the contents of the second sub-list are for the second prediction, etc. Note that there must be the same number of references for each prediction (i.e. all sub-lists must be of the same length). + smooth_method (`str`): The smoothing method to use, defaults to `'exp'`. Possible values are: + - `'none'`: no smoothing + - `'floor'`: increment zero counts + - `'add-k'`: increment num/denom by k for n>1 + - `'exp'`: exponential decay + smooth_value (`float`): The smoothing value. Only valid when `smooth_method='floor'` (in which case `smooth_value` defaults to `0.1`) or `smooth_method='add-k'` (in which case `smooth_value` defaults to `1`). + tokenize (`str`): Tokenization method to use for BLEU. If not provided, defaults to `'zh'` for Chinese, `'ja-mecab'` for Japanese and `'13a'` (mteval) otherwise. Possible values are: + - `'none'`: No tokenization. + - `'zh'`: Chinese tokenization. + - `'13a'`: mimics the `mteval-v13a` script from Moses. + - `'intl'`: International tokenization, mimics the `mteval-v14` script from Moses + - `'char'`: Language-agnostic character-level tokenization. + - `'ja-mecab'`: Japanese tokenization. Uses the [MeCab tokenizer](https://pypi.org/project/mecab-python3). + lowercase (`bool`): If `True`, lowercases the input, enabling case-insensitivity. Defaults to `False`. + force (`bool`): If `True`, insists that your tokenized input is actually detokenized. Defaults to `False`. + use_effective_order (`bool`): If `True`, stops including n-gram orders for which precision is 0. This should be `True`, if sentence-level BLEU will be computed. Defaults to `False`. + +Returns: + 'score': BLEU score, + 'counts': Counts, + 'totals': Totals, + 'precisions': Precisions, + 'bp': Brevity penalty, + 'sys_len': predictions length, + 'ref_len': reference length, + +Examples: + + Example 1: + >>> predictions = ["hello there general kenobi", "foo bar foobar"] + >>> references = [["hello there general kenobi", "hello there !"], ["foo bar foobar", "foo bar foobar"]] + >>> sacrebleu = evaluate.load("sacrebleu") + >>> results = sacrebleu.compute(predictions=predictions, references=references) + >>> print(list(results.keys())) + ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len'] + >>> print(round(results["score"], 1)) + 100.0 + + Example 2: + >>> predictions = ["hello there general kenobi", + ... "on our way to ankh morpork"] + >>> references = [["hello there general kenobi", "hello there !"], + ... ["goodbye ankh morpork", "ankh morpork"]] + >>> sacrebleu = evaluate.load("sacrebleu") + >>> results = sacrebleu.compute(predictions=predictions, + ... references=references) + >>> print(list(results.keys())) + ['score', 'counts', 'totals', 'precisions', 'bp', 'sys_len', 'ref_len'] + >>> print(round(results["score"], 1)) + 39.8 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Sacrebleu(evaluate.Metric): + def _info(self): + if version.parse(scb.__version__) < version.parse("1.4.12"): + raise ImportWarning( + "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n" + 'You can install it with `pip install "sacrebleu>=1.4.12"`.' + ) + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + homepage="https://github.com/mjpost/sacreBLEU", + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), + } + ), + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + ], + codebase_urls=["https://github.com/mjpost/sacreBLEU"], + reference_urls=[ + "https://github.com/mjpost/sacreBLEU", + "https://en.wikipedia.org/wiki/BLEU", + "https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213", + ], + ) + + def _compute( + self, + predictions, + references, + smooth_method="exp", + smooth_value=None, + force=False, + lowercase=False, + tokenize=None, + use_effective_order=False, + ): + # if only one reference is provided make sure we still use list of lists + if isinstance(references[0], str): + references = [[ref] for ref in references] + + references_per_prediction = len(references[0]) + if any(len(refs) != references_per_prediction for refs in references): + raise ValueError("Sacrebleu requires the same number of references for each prediction") + transformed_references = [[refs[i] for refs in references] for i in range(references_per_prediction)] + output = scb.corpus_bleu( + predictions, + transformed_references, + smooth_method=smooth_method, + smooth_value=smooth_value, + force=force, + lowercase=lowercase, + use_effective_order=use_effective_order, + **(dict(tokenize=tokenize) if tokenize else {}), + ) + output_dict = { + "score": output.score, + "counts": output.counts, + "totals": output.totals, + "precisions": output.precisions, + "bp": output.bp, + "sys_len": output.sys_len, + "ref_len": output.ref_len, + } + return output_dict diff --git a/TextClassification/TextClassification/evaluate/metrics/sari/README.md b/TextClassification/TextClassification/evaluate/metrics/sari/README.md new file mode 100644 index 0000000000000000000000000000000000000000..eea74d817b8b61a34fe71cf275bcde6e2c547f58 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/sari/README.md @@ -0,0 +1,146 @@ +--- +title: SARI +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + SARI is a metric used for evaluating automatic text simplification systems. + The metric compares the predicted simplified sentences against the reference + and the source sentences. It explicitly measures the goodness of words that are + added, deleted and kept by the system. + Sari = (F1_add + F1_keep + P_del) / 3 + where + F1_add: n-gram F1 score for add operation + F1_keep: n-gram F1 score for keep operation + P_del: n-gram precision score for delete operation + n = 4, as in the original paper. + + This implementation is adapted from Tensorflow's tensor2tensor implementation [3]. + It has two differences with the original GitHub [1] implementation: + (1) Defines 0/0=1 instead of 0 to give higher scores for predictions that match + a target exactly. + (2) Fixes an alleged bug [2] in the keep score computation. + [1] https://github.com/cocoxu/simplification/blob/master/SARI.py + (commit 0210f15) + [2] https://github.com/cocoxu/simplification/issues/6 + [3] https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py +--- + +# Metric Card for SARI + + +## Metric description +SARI (***s**ystem output **a**gainst **r**eferences and against the **i**nput sentence*) is a metric used for evaluating automatic text simplification systems. + +The metric compares the predicted simplified sentences against the reference and the source sentences. It explicitly measures the goodness of words that are added, deleted and kept by the system. + +SARI can be computed as: + +`sari = ( F1_add + F1_keep + P_del) / 3` + +where + +`F1_add` is the n-gram F1 score for add operations + +`F1_keep` is the n-gram F1 score for keep operations + +`P_del` is the n-gram precision score for delete operations + +The number of n grams, `n`, is equal to 4, as in the original paper. + +This implementation is adapted from [Tensorflow's tensor2tensor implementation](https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py). +It has two differences with the [original GitHub implementation](https://github.com/cocoxu/simplification/blob/master/SARI.py): + +1) It defines 0/0=1 instead of 0 to give higher scores for predictions that match a target exactly. +2) It fixes an [alleged bug](https://github.com/cocoxu/simplification/issues/6) in the keep score computation. + + + +## How to use + +The metric takes 3 inputs: sources (a list of source sentence strings), predictions (a list of predicted sentence strings) and references (a list of lists of reference sentence strings) + +```python +from evaluate import load +sari = load("sari") +sources=["About 95 species are currently accepted."] +predictions=["About 95 you now get in."] +references=[["About 95 species are currently known.","About 95 species are now accepted.","95 species are now accepted."]] +sari_score = sari.compute(sources=sources, predictions=predictions, references=references) +``` +## Output values + +This metric outputs a dictionary with the SARI score: + +``` +print(sari_score) +{'sari': 26.953601953601954} +``` + +The range of values for the SARI score is between 0 and 100 -- the higher the value, the better the performance of the model being evaluated, with a SARI of 100 being a perfect score. + +### Values from popular papers + +The [original paper that proposes the SARI metric](https://aclanthology.org/Q16-1029.pdf) reports scores ranging from 26 to 43 for different simplification systems and different datasets. They also find that the metric ranks all of the simplification systems and human references in the same order as the human assessment used as a comparison, and that it correlates reasonably with human judgments. + +More recent SARI scores for text simplification can be found on leaderboards for datasets such as [TurkCorpus](https://paperswithcode.com/sota/text-simplification-on-turkcorpus) and [Newsela](https://paperswithcode.com/sota/text-simplification-on-newsela). + +## Examples + +Perfect match between prediction and reference: + +```python +from evaluate import load +sari = load("sari") +sources=["About 95 species are currently accepted ."] +predictions=["About 95 species are currently accepted ."] +references=[["About 95 species are currently accepted ."]] +sari_score = sari.compute(sources=sources, predictions=predictions, references=references) +print(sari_score) +{'sari': 100.0} +``` + +Partial match between prediction and reference: + +```python +from evaluate import load +sari = load("sari") +sources=["About 95 species are currently accepted ."] +predictions=["About 95 you now get in ."] +references=[["About 95 species are currently known .","About 95 species are now accepted .","95 species are now accepted ."]] +sari_score = sari.compute(sources=sources, predictions=predictions, references=references) +print(sari_score) +{'sari': 26.953601953601954} +``` + +## Limitations and bias + +SARI is a valuable measure for comparing different text simplification systems as well as one that can assist the iterative development of a system. + +However, while the [original paper presenting SARI](https://aclanthology.org/Q16-1029.pdf) states that it captures "the notion of grammaticality and meaning preservation", this is a difficult claim to empirically validate. + +## Citation + +```bibtex +@inproceedings{xu-etal-2016-optimizing, +title = {Optimizing Statistical Machine Translation for Text Simplification}, +authors={Xu, Wei and Napoles, Courtney and Pavlick, Ellie and Chen, Quanze and Callison-Burch, Chris}, +journal = {Transactions of the Association for Computational Linguistics}, +volume = {4}, +year={2016}, +url = {https://www.aclweb.org/anthology/Q16-1029}, +pages = {401--415}, +} +``` + +## Further References + +- [NLP Progress -- Text Simplification](http://nlpprogress.com/english/simplification.html) +- [Hugging Face Hub -- Text Simplification Models](https://huggingface.co/datasets?filter=task_ids:text-simplification) diff --git a/TextClassification/TextClassification/evaluate/metrics/sari/app.py b/TextClassification/TextClassification/evaluate/metrics/sari/app.py new file mode 100644 index 0000000000000000000000000000000000000000..0666a386de190bab2906c58fef504a081cf52727 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/sari/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("sari") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/sari/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/sari/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7a85cf386b67ac245901a4fe6a22bf7edf6143b1 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/sari/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +sacrebleu +sacremoses \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/sari/sari.py b/TextClassification/TextClassification/evaluate/metrics/sari/sari.py new file mode 100644 index 0000000000000000000000000000000000000000..7d021184d0f45ae1fb01d86f8c5320a4a62fdf89 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/sari/sari.py @@ -0,0 +1,289 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" SARI metric.""" + +from collections import Counter + +import datasets +import sacrebleu +import sacremoses +from packaging import version + +import evaluate + + +_CITATION = """\ +@inproceedings{xu-etal-2016-optimizing, +title = {Optimizing Statistical Machine Translation for Text Simplification}, +authors={Xu, Wei and Napoles, Courtney and Pavlick, Ellie and Chen, Quanze and Callison-Burch, Chris}, +journal = {Transactions of the Association for Computational Linguistics}, +volume = {4}, +year={2016}, +url = {https://www.aclweb.org/anthology/Q16-1029}, +pages = {401--415}, +} +""" + +_DESCRIPTION = """\ +SARI is a metric used for evaluating automatic text simplification systems. +The metric compares the predicted simplified sentences against the reference +and the source sentences. It explicitly measures the goodness of words that are +added, deleted and kept by the system. +Sari = (F1_add + F1_keep + P_del) / 3 +where +F1_add: n-gram F1 score for add operation +F1_keep: n-gram F1 score for keep operation +P_del: n-gram precision score for delete operation +n = 4, as in the original paper. + +This implementation is adapted from Tensorflow's tensor2tensor implementation [3]. +It has two differences with the original GitHub [1] implementation: + (1) Defines 0/0=1 instead of 0 to give higher scores for predictions that match + a target exactly. + (2) Fixes an alleged bug [2] in the keep score computation. +[1] https://github.com/cocoxu/simplification/blob/master/SARI.py + (commit 0210f15) +[2] https://github.com/cocoxu/simplification/issues/6 +[3] https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py +""" + + +_KWARGS_DESCRIPTION = """ +Calculates sari score (between 0 and 100) given a list of source and predicted +sentences, and a list of lists of reference sentences. +Args: + sources: list of source sentences where each sentence should be a string. + predictions: list of predicted sentences where each sentence should be a string. + references: list of lists of reference sentences where each sentence should be a string. +Returns: + sari: sari score +Examples: + >>> sources=["About 95 species are currently accepted ."] + >>> predictions=["About 95 you now get in ."] + >>> references=[["About 95 species are currently known .","About 95 species are now accepted .","95 species are now accepted ."]] + >>> sari = evaluate.load("sari") + >>> results = sari.compute(sources=sources, predictions=predictions, references=references) + >>> print(results) + {'sari': 26.953601953601954} +""" + + +def SARIngram(sgrams, cgrams, rgramslist, numref): + rgramsall = [rgram for rgrams in rgramslist for rgram in rgrams] + rgramcounter = Counter(rgramsall) + + sgramcounter = Counter(sgrams) + sgramcounter_rep = Counter() + for sgram, scount in sgramcounter.items(): + sgramcounter_rep[sgram] = scount * numref + + cgramcounter = Counter(cgrams) + cgramcounter_rep = Counter() + for cgram, ccount in cgramcounter.items(): + cgramcounter_rep[cgram] = ccount * numref + + # KEEP + keepgramcounter_rep = sgramcounter_rep & cgramcounter_rep + keepgramcountergood_rep = keepgramcounter_rep & rgramcounter + keepgramcounterall_rep = sgramcounter_rep & rgramcounter + + keeptmpscore1 = 0 + keeptmpscore2 = 0 + for keepgram in keepgramcountergood_rep: + keeptmpscore1 += keepgramcountergood_rep[keepgram] / keepgramcounter_rep[keepgram] + # Fix an alleged bug [2] in the keep score computation. + # keeptmpscore2 += keepgramcountergood_rep[keepgram] / keepgramcounterall_rep[keepgram] + keeptmpscore2 += keepgramcountergood_rep[keepgram] + # Define 0/0=1 instead of 0 to give higher scores for predictions that match + # a target exactly. + keepscore_precision = 1 + keepscore_recall = 1 + if len(keepgramcounter_rep) > 0: + keepscore_precision = keeptmpscore1 / len(keepgramcounter_rep) + if len(keepgramcounterall_rep) > 0: + # Fix an alleged bug [2] in the keep score computation. + # keepscore_recall = keeptmpscore2 / len(keepgramcounterall_rep) + keepscore_recall = keeptmpscore2 / sum(keepgramcounterall_rep.values()) + keepscore = 0 + if keepscore_precision > 0 or keepscore_recall > 0: + keepscore = 2 * keepscore_precision * keepscore_recall / (keepscore_precision + keepscore_recall) + + # DELETION + delgramcounter_rep = sgramcounter_rep - cgramcounter_rep + delgramcountergood_rep = delgramcounter_rep - rgramcounter + delgramcounterall_rep = sgramcounter_rep - rgramcounter + deltmpscore1 = 0 + deltmpscore2 = 0 + for delgram in delgramcountergood_rep: + deltmpscore1 += delgramcountergood_rep[delgram] / delgramcounter_rep[delgram] + deltmpscore2 += delgramcountergood_rep[delgram] / delgramcounterall_rep[delgram] + # Define 0/0=1 instead of 0 to give higher scores for predictions that match + # a target exactly. + delscore_precision = 1 + if len(delgramcounter_rep) > 0: + delscore_precision = deltmpscore1 / len(delgramcounter_rep) + + # ADDITION + addgramcounter = set(cgramcounter) - set(sgramcounter) + addgramcountergood = set(addgramcounter) & set(rgramcounter) + addgramcounterall = set(rgramcounter) - set(sgramcounter) + + addtmpscore = 0 + for addgram in addgramcountergood: + addtmpscore += 1 + + # Define 0/0=1 instead of 0 to give higher scores for predictions that match + # a target exactly. + addscore_precision = 1 + addscore_recall = 1 + if len(addgramcounter) > 0: + addscore_precision = addtmpscore / len(addgramcounter) + if len(addgramcounterall) > 0: + addscore_recall = addtmpscore / len(addgramcounterall) + addscore = 0 + if addscore_precision > 0 or addscore_recall > 0: + addscore = 2 * addscore_precision * addscore_recall / (addscore_precision + addscore_recall) + + return (keepscore, delscore_precision, addscore) + + +def SARIsent(ssent, csent, rsents): + numref = len(rsents) + + s1grams = ssent.split(" ") + c1grams = csent.split(" ") + s2grams = [] + c2grams = [] + s3grams = [] + c3grams = [] + s4grams = [] + c4grams = [] + + r1gramslist = [] + r2gramslist = [] + r3gramslist = [] + r4gramslist = [] + for rsent in rsents: + r1grams = rsent.split(" ") + r2grams = [] + r3grams = [] + r4grams = [] + r1gramslist.append(r1grams) + for i in range(0, len(r1grams) - 1): + if i < len(r1grams) - 1: + r2gram = r1grams[i] + " " + r1grams[i + 1] + r2grams.append(r2gram) + if i < len(r1grams) - 2: + r3gram = r1grams[i] + " " + r1grams[i + 1] + " " + r1grams[i + 2] + r3grams.append(r3gram) + if i < len(r1grams) - 3: + r4gram = r1grams[i] + " " + r1grams[i + 1] + " " + r1grams[i + 2] + " " + r1grams[i + 3] + r4grams.append(r4gram) + r2gramslist.append(r2grams) + r3gramslist.append(r3grams) + r4gramslist.append(r4grams) + + for i in range(0, len(s1grams) - 1): + if i < len(s1grams) - 1: + s2gram = s1grams[i] + " " + s1grams[i + 1] + s2grams.append(s2gram) + if i < len(s1grams) - 2: + s3gram = s1grams[i] + " " + s1grams[i + 1] + " " + s1grams[i + 2] + s3grams.append(s3gram) + if i < len(s1grams) - 3: + s4gram = s1grams[i] + " " + s1grams[i + 1] + " " + s1grams[i + 2] + " " + s1grams[i + 3] + s4grams.append(s4gram) + + for i in range(0, len(c1grams) - 1): + if i < len(c1grams) - 1: + c2gram = c1grams[i] + " " + c1grams[i + 1] + c2grams.append(c2gram) + if i < len(c1grams) - 2: + c3gram = c1grams[i] + " " + c1grams[i + 1] + " " + c1grams[i + 2] + c3grams.append(c3gram) + if i < len(c1grams) - 3: + c4gram = c1grams[i] + " " + c1grams[i + 1] + " " + c1grams[i + 2] + " " + c1grams[i + 3] + c4grams.append(c4gram) + + (keep1score, del1score, add1score) = SARIngram(s1grams, c1grams, r1gramslist, numref) + (keep2score, del2score, add2score) = SARIngram(s2grams, c2grams, r2gramslist, numref) + (keep3score, del3score, add3score) = SARIngram(s3grams, c3grams, r3gramslist, numref) + (keep4score, del4score, add4score) = SARIngram(s4grams, c4grams, r4gramslist, numref) + avgkeepscore = sum([keep1score, keep2score, keep3score, keep4score]) / 4 + avgdelscore = sum([del1score, del2score, del3score, del4score]) / 4 + avgaddscore = sum([add1score, add2score, add3score, add4score]) / 4 + finalscore = (avgkeepscore + avgdelscore + avgaddscore) / 3 + return finalscore + + +def normalize(sentence, lowercase: bool = True, tokenizer: str = "13a", return_str: bool = True): + + # Normalization is requried for the ASSET dataset (one of the primary + # datasets in sentence simplification) to allow using space + # to split the sentence. Even though Wiki-Auto and TURK datasets, + # do not require normalization, we do it for consistency. + # Code adapted from the EASSE library [1] written by the authors of the ASSET dataset. + # [1] https://github.com/feralvam/easse/blob/580bba7e1378fc8289c663f864e0487188fe8067/easse/utils/preprocessing.py#L7 + + if lowercase: + sentence = sentence.lower() + + if tokenizer in ["13a", "intl"]: + if version.parse(sacrebleu.__version__).major >= 2: + normalized_sent = sacrebleu.metrics.bleu._get_tokenizer(tokenizer)()(sentence) + else: + normalized_sent = sacrebleu.TOKENIZERS[tokenizer]()(sentence) + elif tokenizer == "moses": + normalized_sent = sacremoses.MosesTokenizer().tokenize(sentence, return_str=True, escape=False) + elif tokenizer == "penn": + normalized_sent = sacremoses.MosesTokenizer().penn_tokenize(sentence, return_str=True) + else: + normalized_sent = sentence + + if not return_str: + normalized_sent = normalized_sent.split() + + return normalized_sent + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Sari(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "sources": datasets.Value("string", id="sequence"), + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), + } + ), + codebase_urls=[ + "https://github.com/cocoxu/simplification/blob/master/SARI.py", + "https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py", + ], + reference_urls=["https://www.aclweb.org/anthology/Q16-1029.pdf"], + ) + + def _compute(self, sources, predictions, references): + + if not (len(sources) == len(predictions) == len(references)): + raise ValueError("Sources length must match predictions and references lengths.") + sari_score = 0 + for src, pred, refs in zip(sources, predictions, references): + sari_score += SARIsent(normalize(src), normalize(pred), [normalize(sent) for sent in refs]) + sari_score = sari_score / len(predictions) + return {"sari": 100 * sari_score} diff --git a/TextClassification/TextClassification/evaluate/metrics/seqeval/README.md b/TextClassification/TextClassification/evaluate/metrics/seqeval/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e4442315f079863e802dfc7413e0166ac7d01071 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/seqeval/README.md @@ -0,0 +1,164 @@ +--- +title: seqeval +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + seqeval is a Python framework for sequence labeling evaluation. + seqeval can evaluate the performance of chunking tasks such as named-entity recognition, part-of-speech tagging, semantic role labeling and so on. + + This is well-tested by using the Perl script conlleval, which can be used for + measuring the performance of a system that has processed the CoNLL-2000 shared task data. + + seqeval supports following formats: + IOB1 + IOB2 + IOE1 + IOE2 + IOBES + + See the [README.md] file at https://github.com/chakki-works/seqeval for more information. +--- + +# Metric Card for seqeval + +## Metric description + +seqeval is a Python framework for sequence labeling evaluation. seqeval can evaluate the performance of chunking tasks such as named-entity recognition, part-of-speech tagging, semantic role labeling and so on. + + +## How to use + +Seqeval produces labelling scores along with its sufficient statistics from a source against one or more references. + +It takes two mandatory arguments: + +`predictions`: a list of lists of predicted labels, i.e. estimated targets as returned by a tagger. + +`references`: a list of lists of reference labels, i.e. the ground truth/target values. + +It can also take several optional arguments: + +`suffix` (boolean): `True` if the IOB tag is a suffix (after type) instead of a prefix (before type), `False` otherwise. The default value is `False`, i.e. the IOB tag is a prefix (before type). + +`scheme`: the target tagging scheme, which can be one of [`IOB1`, `IOB2`, `IOE1`, `IOE2`, `IOBES`, `BILOU`]. The default value is `None`. + +`mode`: whether to count correct entity labels with incorrect I/B tags as true positives or not. If you want to only count exact matches, pass `mode="strict"` and a specific `scheme` value. The default is `None`. + +`sample_weight`: An array-like of shape (n_samples,) that provides weights for individual samples. The default is `None`. + +`zero_division`: Which value to substitute as a metric value when encountering zero division. Should be one of [`0`,`1`,`"warn"`]. `"warn"` acts as `0`, but the warning is raised. + + +```python +>>> seqeval = evaluate.load('seqeval') +>>> predictions = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] +>>> references = [['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] +>>> results = seqeval.compute(predictions=predictions, references=references) +``` + +## Output values + +This metric returns a dictionary with a summary of scores for overall and per type: + +Overall: + +`accuracy`: the average [accuracy](https://huggingface.co/metrics/accuracy), on a scale between 0.0 and 1.0. + +`precision`: the average [precision](https://huggingface.co/metrics/precision), on a scale between 0.0 and 1.0. + +`recall`: the average [recall](https://huggingface.co/metrics/recall), on a scale between 0.0 and 1.0. + +`f1`: the average [F1 score](https://huggingface.co/metrics/f1), which is the harmonic mean of the precision and recall. It also has a scale of 0.0 to 1.0. + +Per type (e.g. `MISC`, `PER`, `LOC`,...): + +`precision`: the average [precision](https://huggingface.co/metrics/precision), on a scale between 0.0 and 1.0. + +`recall`: the average [recall](https://huggingface.co/metrics/recall), on a scale between 0.0 and 1.0. + +`f1`: the average [F1 score](https://huggingface.co/metrics/f1), on a scale between 0.0 and 1.0. + + +### Values from popular papers +The 1995 "Text Chunking using Transformation-Based Learning" [paper](https://aclanthology.org/W95-0107) reported a baseline recall of 81.9% and a precision of 78.2% using non Deep Learning-based methods. + +More recently, seqeval continues being used for reporting performance on tasks such as [named entity detection](https://www.mdpi.com/2306-5729/6/8/84/htm) and [information extraction](https://ieeexplore.ieee.org/abstract/document/9697942/). + + +## Examples + +Maximal values (full match) : + +```python +>>> seqeval = evaluate.load('seqeval') +>>> predictions = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] +>>> references = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] +>>> results = seqeval.compute(predictions=predictions, references=references) +>>> print(results) +{'MISC': {'precision': 1.0, 'recall': 1.0, 'f1': 1.0, 'number': 1}, 'PER': {'precision': 1.0, 'recall': 1.0, 'f1': 1.0, 'number': 1}, 'overall_precision': 1.0, 'overall_recall': 1.0, 'overall_f1': 1.0, 'overall_accuracy': 1.0} + +``` + +Minimal values (no match): + +```python +>>> seqeval = evaluate.load('seqeval') +>>> predictions = [['O', 'B-MISC', 'I-MISC'], ['B-PER', 'I-PER', 'O']] +>>> references = [['B-MISC', 'O', 'O'], ['I-PER', '0', 'I-PER']] +>>> results = seqeval.compute(predictions=predictions, references=references) +>>> print(results) +{'MISC': {'precision': 0.0, 'recall': 0.0, 'f1': 0.0, 'number': 1}, 'PER': {'precision': 0.0, 'recall': 0.0, 'f1': 0.0, 'number': 2}, '_': {'precision': 0.0, 'recall': 0.0, 'f1': 0.0, 'number': 1}, 'overall_precision': 0.0, 'overall_recall': 0.0, 'overall_f1': 0.0, 'overall_accuracy': 0.0} +``` + +Partial match: + +```python +>>> seqeval = evaluate.load('seqeval') +>>> predictions = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] +>>> references = [['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] +>>> results = seqeval.compute(predictions=predictions, references=references) +>>> print(results) +{'MISC': {'precision': 0.0, 'recall': 0.0, 'f1': 0.0, 'number': 1}, 'PER': {'precision': 1.0, 'recall': 1.0, 'f1': 1.0, 'number': 1}, 'overall_precision': 0.5, 'overall_recall': 0.5, 'overall_f1': 0.5, 'overall_accuracy': 0.8} +``` + +## Limitations and bias + +seqeval supports following IOB formats (short for inside, outside, beginning) : `IOB1`, `IOB2`, `IOE1`, `IOE2`, `IOBES`, `IOBES` (only in strict mode) and `BILOU` (only in strict mode). + +For more information about IOB formats, refer to the [Wikipedia page](https://en.wikipedia.org/wiki/Inside%E2%80%93outside%E2%80%93beginning_(tagging)) and the description of the [CoNLL-2000 shared task](https://aclanthology.org/W02-2024). + + +## Citation + +```bibtex +@inproceedings{ramshaw-marcus-1995-text, + title = "Text Chunking using Transformation-Based Learning", + author = "Ramshaw, Lance and + Marcus, Mitch", + booktitle = "Third Workshop on Very Large Corpora", + year = "1995", + url = "https://www.aclweb.org/anthology/W95-0107", +} +``` + +```bibtex +@misc{seqeval, + title={{seqeval}: A Python framework for sequence labeling evaluation}, + url={https://github.com/chakki-works/seqeval}, + note={Software available from https://github.com/chakki-works/seqeval}, + author={Hiroki Nakayama}, + year={2018}, +} +``` + +## Further References +- [README for seqeval at GitHub](https://github.com/chakki-works/seqeval) +- [CoNLL-2000 shared task](https://www.clips.uantwerpen.be/conll2002/ner/bin/conlleval.txt) diff --git a/TextClassification/TextClassification/evaluate/metrics/seqeval/app.py b/TextClassification/TextClassification/evaluate/metrics/seqeval/app.py new file mode 100644 index 0000000000000000000000000000000000000000..7c05d59ec96e2789c3850ce0d656b932564d34af --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/seqeval/app.py @@ -0,0 +1,11 @@ +import sys + +import evaluate +from evaluate.utils import launch_gradio_widget + + +sys.path = [p for p in sys.path if p != "/home/user/app"] +module = evaluate.load("seqeval") +sys.path = ["/home/user/app"] + sys.path + +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/seqeval/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/seqeval/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ac681b46ff66dba1b0520056ef07c2fd32566d14 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/seqeval/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +seqeval \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/seqeval/seqeval.py b/TextClassification/TextClassification/evaluate/metrics/seqeval/seqeval.py new file mode 100644 index 0000000000000000000000000000000000000000..252d16b6c6f4f45ae41bb982c00771ae3d707441 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/seqeval/seqeval.py @@ -0,0 +1,164 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" seqeval metric. """ + +import importlib +from typing import List, Optional, Union + +import datasets +from seqeval.metrics import accuracy_score, classification_report + +import evaluate + + +_CITATION = """\ +@inproceedings{ramshaw-marcus-1995-text, + title = "Text Chunking using Transformation-Based Learning", + author = "Ramshaw, Lance and + Marcus, Mitch", + booktitle = "Third Workshop on Very Large Corpora", + year = "1995", + url = "https://www.aclweb.org/anthology/W95-0107", +} +@misc{seqeval, + title={{seqeval}: A Python framework for sequence labeling evaluation}, + url={https://github.com/chakki-works/seqeval}, + note={Software available from https://github.com/chakki-works/seqeval}, + author={Hiroki Nakayama}, + year={2018}, +} +""" + +_DESCRIPTION = """\ +seqeval is a Python framework for sequence labeling evaluation. +seqeval can evaluate the performance of chunking tasks such as named-entity recognition, part-of-speech tagging, semantic role labeling and so on. + +This is well-tested by using the Perl script conlleval, which can be used for +measuring the performance of a system that has processed the CoNLL-2000 shared task data. + +seqeval supports following formats: +IOB1 +IOB2 +IOE1 +IOE2 +IOBES + +See the [README.md] file at https://github.com/chakki-works/seqeval for more information. +""" + +_KWARGS_DESCRIPTION = """ +Produces labelling scores along with its sufficient statistics +from a source against one or more references. + +Args: + predictions: List of List of predicted labels (Estimated targets as returned by a tagger) + references: List of List of reference labels (Ground truth (correct) target values) + suffix: True if the IOB prefix is after type, False otherwise. default: False + scheme: Specify target tagging scheme. Should be one of ["IOB1", "IOB2", "IOE1", "IOE2", "IOBES", "BILOU"]. + default: None + mode: Whether to count correct entity labels with incorrect I/B tags as true positives or not. + If you want to only count exact matches, pass mode="strict". default: None. + sample_weight: Array-like of shape (n_samples,), weights for individual samples. default: None + zero_division: Which value to substitute as a metric value when encountering zero division. Should be on of 0, 1, + "warn". "warn" acts as 0, but the warning is raised. + +Returns: + 'scores': dict. Summary of the scores for overall and per type + Overall: + 'accuracy': accuracy, + 'precision': precision, + 'recall': recall, + 'f1': F1 score, also known as balanced F-score or F-measure, + Per type: + 'precision': precision, + 'recall': recall, + 'f1': F1 score, also known as balanced F-score or F-measure +Examples: + + >>> predictions = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] + >>> references = [['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']] + >>> seqeval = evaluate.load("seqeval") + >>> results = seqeval.compute(predictions=predictions, references=references) + >>> print(list(results.keys())) + ['MISC', 'PER', 'overall_precision', 'overall_recall', 'overall_f1', 'overall_accuracy'] + >>> print(results["overall_f1"]) + 0.5 + >>> print(results["PER"]["f1"]) + 1.0 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Seqeval(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + homepage="https://github.com/chakki-works/seqeval", + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Sequence(datasets.Value("string", id="label"), id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="label"), id="sequence"), + } + ), + codebase_urls=["https://github.com/chakki-works/seqeval"], + reference_urls=["https://github.com/chakki-works/seqeval"], + ) + + def _compute( + self, + predictions, + references, + suffix: bool = False, + scheme: Optional[str] = None, + mode: Optional[str] = None, + sample_weight: Optional[List[int]] = None, + zero_division: Union[str, int] = "warn", + ): + if scheme is not None: + try: + scheme_module = importlib.import_module("seqeval.scheme") + scheme = getattr(scheme_module, scheme) + except AttributeError: + raise ValueError(f"Scheme should be one of [IOB1, IOB2, IOE1, IOE2, IOBES, BILOU], got {scheme}") + report = classification_report( + y_true=references, + y_pred=predictions, + suffix=suffix, + output_dict=True, + scheme=scheme, + mode=mode, + sample_weight=sample_weight, + zero_division=zero_division, + ) + report.pop("macro avg") + report.pop("weighted avg") + overall_score = report.pop("micro avg") + + scores = { + type_name: { + "precision": score["precision"], + "recall": score["recall"], + "f1": score["f1-score"], + "number": score["support"], + } + for type_name, score in report.items() + } + scores["overall_precision"] = overall_score["precision"] + scores["overall_recall"] = overall_score["recall"] + scores["overall_f1"] = overall_score["f1-score"] + scores["overall_accuracy"] = accuracy_score(y_true=references, y_pred=predictions) + + return scores diff --git a/TextClassification/TextClassification/evaluate/metrics/smape/README.md b/TextClassification/TextClassification/evaluate/metrics/smape/README.md new file mode 100644 index 0000000000000000000000000000000000000000..904e17ff28b5c38a27953d130f359e2d37ede60c --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/smape/README.md @@ -0,0 +1,112 @@ +--- +title: sMAPE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Symmetric Mean Absolute Percentage Error (sMAPE) is the symmetric mean percentage error difference between the predicted and actual values defined by Chen and Yang (2004). +--- + +# Metric Card for sMAPE + + +## Metric Description + +Symmetric Mean Absolute Error (sMAPE) is the symmetric mean of the percentage error of difference between the predicted $x_i$ and actual $y_i$ numeric values: + +![image](https://user-images.githubusercontent.com/8100/200009801-ae8be6c8-facf-401b-8df0-3f80a458b9f4.png) + + +## How to Use + +At minimum, this metric requires predictions and references as inputs. + +```python +>>> smape_metric = evaluate.load("smape") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> results = smape_metric.compute(predictions=predictions, references=references) +``` + +### Inputs + +Mandatory inputs: +- `predictions`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the estimated target values. +- `references`: numeric array-like of shape (`n_samples,`) or (`n_samples`, `n_outputs`), representing the ground truth (correct) target values. + +Optional arguments: +- `sample_weight`: numeric array-like of shape (`n_samples,`) representing sample weights. The default is `None`. +- `multioutput`: `raw_values`, `uniform_average` or numeric array-like of shape (`n_outputs,`), which defines the aggregation of multiple output values. The default value is `uniform_average`. + - `raw_values` returns a full set of errors in case of multioutput input. + - `uniform_average` means that the errors of all outputs are averaged with uniform weight. + - the array-like value defines weights used to average errors. + +### Output Values +This metric outputs a dictionary, containing the mean absolute error score, which is of type: +- `float`: if multioutput is `uniform_average` or an ndarray of weights, then the weighted average of all output errors is returned. +- numeric array-like of shape (`n_outputs,`): if multioutput is `raw_values`, then the score is returned for each output separately. + +Each sMAPE `float` value ranges from `0.0` to `2.0`, with the best value being 0.0. + +Output Example(s): +```python +{'smape': 0.5} +``` + +If `multioutput="raw_values"`: +```python +{'smape': array([0.5, 1.5 ])} +``` + +#### Values from Popular Papers + + +### Examples + +Example with the `uniform_average` config: +```python +>>> smape_metric = evaluate.load("smape") +>>> predictions = [2.5, 0.0, 2, 8] +>>> references = [3, -0.5, 2, 7] +>>> results = smape_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'smape': 0.5787...} +``` + +Example with multi-dimensional lists, and the `raw_values` config: +```python +>>> smape_metric = evaluate.load("smape", "multilist") +>>> predictions = [[0.5, 1], [-1, 1], [7, -6]] +>>> references = [[0.1, 2], [-1, 2], [8, -5]] +>>> results = smape_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'smape': 0.8874...} +>>> results = smape_metric.compute(predictions=predictions, references=references, multioutput='raw_values') +>>> print(results) +{'smape': array([1.3749..., 0.4])} +``` + +## Limitations and Bias +This metric is called a measure of "percentage error" even though there is no multiplier of 100. The range is between (0, 2) with it being two when the target and prediction are both zero. + +## Citation(s) + +```bibtex +@article{article, + author = {Chen, Zhuo and Yang, Yuhong}, + year = {2004}, + month = {04}, + pages = {}, + title = {Assessing forecast accuracy measures} +} +``` + +## Further References +- [Symmetric Mean absolute percentage error - Wikipedia](https://en.wikipedia.org/wiki/Symmetric_mean_absolute_percentage_error) diff --git a/TextClassification/TextClassification/evaluate/metrics/smape/app.py b/TextClassification/TextClassification/evaluate/metrics/smape/app.py new file mode 100644 index 0000000000000000000000000000000000000000..e6e08d5961ebcda924f2cac983b4f01ef9ad1b37 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/smape/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("smape") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/smape/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/smape/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..c9c2c1e3344c2dec052782f879af2e3a1f065df8 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/smape/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn diff --git a/TextClassification/TextClassification/evaluate/metrics/smape/smape.py b/TextClassification/TextClassification/evaluate/metrics/smape/smape.py new file mode 100644 index 0000000000000000000000000000000000000000..b8292c37c20f427f9621445e44d59b3fdf79053c --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/smape/smape.py @@ -0,0 +1,158 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""sMAPE - Symmetric Mean Absolute Percentage Error Metric""" + +import datasets +import numpy as np +from sklearn.metrics._regression import _check_reg_targets +from sklearn.utils.validation import check_consistent_length + +import evaluate + + +_CITATION = """\ +@article{article, + author = {Chen, Zhuo and Yang, Yuhong}, + year = {2004}, + month = {04}, + pages = {}, + title = {Assessing forecast accuracy measures} +} +""" + +_DESCRIPTION = """\ +Symmetric Mean Absolute Percentage Error (sMAPE) is the symmetric mean percentage error +difference between the predicted and actual values as defined by Chen and Yang (2004), +based on the metric by Armstrong (1985) and Makridakis (1993). +""" + + +_KWARGS_DESCRIPTION = """ +Args: + predictions: array-like of shape (n_samples,) or (n_samples, n_outputs) + Estimated target values. + references: array-like of shape (n_samples,) or (n_samples, n_outputs) + Ground truth (correct) target values. + sample_weight: array-like of shape (n_samples,), default=None + Sample weights. + multioutput: {"raw_values", "uniform_average"} or array-like of shape (n_outputs,), default="uniform_average" + Defines aggregating of multiple output values. Array-like value defines weights used to average errors. + + "raw_values" : Returns a full set of errors in case of multioutput input. + + "uniform_average" : Errors of all outputs are averaged with uniform weight. + +Returns: + smape : symmetric mean absolute percentage error. + If multioutput is "raw_values", then symmetric mean absolute percentage error is returned for each output separately. If multioutput is "uniform_average" or an ndarray of weights, then the weighted average of all output errors is returned. + sMAPE output is non-negative floating point in the range (0, 2). The best value is 0.0. +Examples: + + >>> smape_metric = evaluate.load("smape") + >>> predictions = [2.5, 0.0, 2, 8] + >>> references = [3, -0.5, 2, 7] + >>> results = smape_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'smape': 0.5787878787878785} + + If you're using multi-dimensional lists, then set the config as follows : + + >>> smape_metric = evaluate.load("smape", "multilist") + >>> predictions = [[0.5, 1], [-1, 1], [7, -6]] + >>> references = [[0.1, 2], [-1, 2], [8, -5]] + >>> results = smape_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'smape': 0.49696969558995985} + >>> results = smape_metric.compute(predictions=predictions, references=references, multioutput='raw_values') + >>> print(results) + {'smape': array([0.48888889, 0.50505051])} +""" + + +def symmetric_mean_absolute_percentage_error(y_true, y_pred, *, sample_weight=None, multioutput="uniform_average"): + """Symmetric Mean absolute percentage error (sMAPE) metric using sklearn's api and helpers. + + Parameters + ---------- + y_true : array-like of shape (n_samples,) or (n_samples, n_outputs) + Ground truth (correct) target values. + y_pred : array-like of shape (n_samples,) or (n_samples, n_outputs) + Estimated target values. + sample_weight : array-like of shape (n_samples,), default=None + Sample weights. + multioutput : {'raw_values', 'uniform_average'} or array-like + Defines aggregating of multiple output values. + Array-like value defines weights used to average errors. + If input is list then the shape must be (n_outputs,). + 'raw_values' : + Returns a full set of errors in case of multioutput input. + 'uniform_average' : + Errors of all outputs are averaged with uniform weight. + Returns + ------- + loss : float or ndarray of floats + If multioutput is 'raw_values', then mean absolute percentage error + is returned for each output separately. + If multioutput is 'uniform_average' or an ndarray of weights, then the + weighted average of all output errors is returned. + sMAPE output is non-negative floating point. The best value is 0.0. + """ + y_type, y_true, y_pred, multioutput = _check_reg_targets(y_true, y_pred, multioutput) + check_consistent_length(y_true, y_pred, sample_weight) + epsilon = np.finfo(np.float64).eps + smape = 2 * np.abs(y_pred - y_true) / (np.maximum(np.abs(y_true), epsilon) + np.maximum(np.abs(y_pred), epsilon)) + output_errors = np.average(smape, weights=sample_weight, axis=0) + if isinstance(multioutput, str): + if multioutput == "raw_values": + return output_errors + elif multioutput == "uniform_average": + # pass None as weights to np.average: uniform mean + multioutput = None + + return np.average(output_errors, weights=multioutput) + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Smape(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features(self._get_feature_types()), + reference_urls=["https://robjhyndman.com/hyndsight/smape/"], + ) + + def _get_feature_types(self): + if self.config_name == "multilist": + return { + "predictions": datasets.Sequence(datasets.Value("float")), + "references": datasets.Sequence(datasets.Value("float")), + } + else: + return { + "predictions": datasets.Value("float"), + "references": datasets.Value("float"), + } + + def _compute(self, predictions, references, sample_weight=None, multioutput="uniform_average"): + + smape_score = symmetric_mean_absolute_percentage_error( + references, + predictions, + sample_weight=sample_weight, + multioutput=multioutput, + ) + + return {"smape": smape_score} diff --git a/TextClassification/TextClassification/evaluate/metrics/spearmanr/README.md b/TextClassification/TextClassification/evaluate/metrics/spearmanr/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bc42bdfa7b485f61a39fd790c5f824b9c029e2a7 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/spearmanr/README.md @@ -0,0 +1,146 @@ +--- +title: Spearman Correlation Coefficient Metric +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + The Spearman rank-order correlation coefficient is a measure of the + relationship between two datasets. Like other correlation coefficients, + this one varies between -1 and +1 with 0 implying no correlation. + Positive correlations imply that as data in dataset x increases, so + does data in dataset y. Negative correlations imply that as x increases, + y decreases. Correlations of -1 or +1 imply an exact monotonic relationship. + + Unlike the Pearson correlation, the Spearman correlation does not + assume that both datasets are normally distributed. + + The p-value roughly indicates the probability of an uncorrelated system + producing datasets that have a Spearman correlation at least as extreme + as the one computed from these datasets. The p-values are not entirely + reliable but are probably reasonable for datasets larger than 500 or so. +--- + +# Metric Card for Spearman Correlation Coefficient Metric (spearmanr) + + +## Metric Description +The Spearman rank-order correlation coefficient is a measure of the +relationship between two datasets. Like other correlation coefficients, +this one varies between -1 and +1 with 0 implying no correlation. +Positive correlations imply that as data in dataset x increases, so +does data in dataset y. Negative correlations imply that as x increases, +y decreases. Correlations of -1 or +1 imply an exact monotonic relationship. + +Unlike the Pearson correlation, the Spearman correlation does not +assume that both datasets are normally distributed. + +The p-value roughly indicates the probability of an uncorrelated system +producing datasets that have a Spearman correlation at least as extreme +as the one computed from these datasets. The p-values are not entirely +reliable but are probably reasonable for datasets larger than 500 or so. + + +## How to Use +At minimum, this metric only requires a `list` of predictions and a `list` of references: + +```python +>>> spearmanr_metric = evaluate.load("spearmanr") +>>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], predictions=[10, 9, 2.5, 6, 4]) +>>> print(results) +{'spearmanr': -0.7} +``` + +### Inputs +- **`predictions`** (`list` of `float`): Predicted labels, as returned by a model. +- **`references`** (`list` of `float`): Ground truth labels. +- **`return_pvalue`** (`bool`): If `True`, returns the p-value. If `False`, returns + only the spearmanr score. Defaults to `False`. + +### Output Values +- **`spearmanr`** (`float`): Spearman correlation coefficient. +- **`p-value`** (`float`): p-value. **Note**: is only returned + if `return_pvalue=True` is input. + +If `return_pvalue=False`, the output is a `dict` with one value, as below: +```python +{'spearmanr': -0.7} +``` + +Otherwise, if `return_pvalue=True`, the output is a `dict` containing a the `spearmanr` value as well as the corresponding `pvalue`: +```python +{'spearmanr': -0.7, 'spearmanr_pvalue': 0.1881204043741873} +``` + +Spearman rank-order correlations can take on any value from `-1` to `1`, inclusive. + +The p-values can take on any value from `0` to `1`, inclusive. + +#### Values from Popular Papers + + +### Examples +A basic example: +```python +>>> spearmanr_metric = evaluate.load("spearmanr") +>>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], predictions=[10, 9, 2.5, 6, 4]) +>>> print(results) +{'spearmanr': -0.7} +``` + +The same example, but that also returns the pvalue: +```python +>>> spearmanr_metric = evaluate.load("spearmanr") +>>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], predictions=[10, 9, 2.5, 6, 4], return_pvalue=True) +>>> print(results) +{'spearmanr': -0.7, 'spearmanr_pvalue': 0.1881204043741873 +>>> print(results['spearmanr']) +-0.7 +>>> print(results['spearmanr_pvalue']) +0.1881204043741873 +``` + +## Limitations and Bias + + +## Citation +```bibtex +@book{kokoska2000crc, + title={CRC standard probability and statistics tables and formulae}, + author={Kokoska, Stephen and Zwillinger, Daniel}, + year={2000}, + publisher={Crc Press} +} +@article{2020SciPy-NMeth, + author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and + Haberland, Matt and Reddy, Tyler and Cournapeau, David and + Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and + Bright, Jonathan and {van der Walt}, St{\'e}fan J. and + Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and + Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and + Kern, Robert and Larson, Eric and Carey, C J and + Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and + {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and + Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and + Harris, Charles R. and Archibald, Anne M. and + Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and + {van Mulbregt}, Paul and {SciPy 1.0 Contributors}}, + title = {{{SciPy} 1.0: Fundamental Algorithms for Scientific + Computing in Python}}, + journal = {Nature Methods}, + year = {2020}, + volume = {17}, + pages = {261--272}, + adsurl = {https://rdcu.be/b08Wh}, + doi = {10.1038/s41592-019-0686-2}, +} +``` + +## Further References +*Add any useful further references.* diff --git a/TextClassification/TextClassification/evaluate/metrics/spearmanr/app.py b/TextClassification/TextClassification/evaluate/metrics/spearmanr/app.py new file mode 100644 index 0000000000000000000000000000000000000000..6cccd91ec9e4b6e27579abad648291d0d6b339ff --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/spearmanr/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("spearmanr") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/spearmanr/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/spearmanr/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..a410b4ca49fe91f97db76956af15309d2dec7bb8 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/spearmanr/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scipy \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/spearmanr/spearmanr.py b/TextClassification/TextClassification/evaluate/metrics/spearmanr/spearmanr.py new file mode 100644 index 0000000000000000000000000000000000000000..44d372638f51aab0b358217cd059dd1353a7f494 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/spearmanr/spearmanr.py @@ -0,0 +1,120 @@ +# Copyright 2021 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Spearman correlation coefficient metric.""" + +import datasets +from scipy.stats import spearmanr + +import evaluate + + +_DESCRIPTION = """ +The Spearman rank-order correlation coefficient is a measure of the +relationship between two datasets. Like other correlation coefficients, +this one varies between -1 and +1 with 0 implying no correlation. +Positive correlations imply that as data in dataset x increases, so +does data in dataset y. Negative correlations imply that as x increases, +y decreases. Correlations of -1 or +1 imply an exact monotonic relationship. + +Unlike the Pearson correlation, the Spearman correlation does not +assume that both datasets are normally distributed. + +The p-value roughly indicates the probability of an uncorrelated system +producing datasets that have a Spearman correlation at least as extreme +as the one computed from these datasets. The p-values are not entirely +reliable but are probably reasonable for datasets larger than 500 or so. +""" + +_KWARGS_DESCRIPTION = """ +Args: + predictions (`List[float]`): Predicted labels, as returned by a model. + references (`List[float]`): Ground truth labels. + return_pvalue (`bool`): If `True`, returns the p-value. If `False`, returns + only the spearmanr score. Defaults to `False`. +Returns: + spearmanr (`float`): Spearman correlation coefficient. + p-value (`float`): p-value. **Note**: is only returned if `return_pvalue=True` is input. +Examples: + Example 1: + >>> spearmanr_metric = evaluate.load("spearmanr") + >>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], predictions=[10, 9, 2.5, 6, 4]) + >>> print(results) + {'spearmanr': -0.7} + + Example 2: + >>> spearmanr_metric = evaluate.load("spearmanr") + >>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], + ... predictions=[10, 9, 2.5, 6, 4], + ... return_pvalue=True) + >>> print(results['spearmanr']) + -0.7 + >>> print(round(results['spearmanr_pvalue'], 2)) + 0.19 +""" + +_CITATION = r"""\ +@book{kokoska2000crc, + title={CRC standard probability and statistics tables and formulae}, + author={Kokoska, Stephen and Zwillinger, Daniel}, + year={2000}, + publisher={Crc Press} +} +@article{2020SciPy-NMeth, + author = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and + Haberland, Matt and Reddy, Tyler and Cournapeau, David and + Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and + Bright, Jonathan and {van der Walt}, St{\'e}fan J. and + Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and + Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and + Kern, Robert and Larson, Eric and Carey, C J and + Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and + {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and + Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and + Harris, Charles R. and Archibald, Anne M. and + Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and + {van Mulbregt}, Paul and {SciPy 1.0 Contributors}}, + title = {{{SciPy} 1.0: Fundamental Algorithms for Scientific + Computing in Python}}, + journal = {Nature Methods}, + year = {2020}, + volume = {17}, + pages = {261--272}, + adsurl = {https://rdcu.be/b08Wh}, + doi = {10.1038/s41592-019-0686-2}, +} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Spearmanr(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("float"), + "references": datasets.Value("float"), + } + ), + reference_urls=["https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.spearmanr.html"], + ) + + def _compute(self, predictions, references, return_pvalue=False): + results = spearmanr(references, predictions) + if return_pvalue: + return {"spearmanr": results[0], "spearmanr_pvalue": results[1]} + else: + return {"spearmanr": results[0]} diff --git a/TextClassification/TextClassification/evaluate/metrics/squad/README.md b/TextClassification/TextClassification/evaluate/metrics/squad/README.md new file mode 100644 index 0000000000000000000000000000000000000000..08e030c599698bd0bdf7aa986f1bc0c14bb792cf --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/squad/README.md @@ -0,0 +1,110 @@ +--- +title: SQuAD +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + This metric wrap the official scoring script for version 1 of the Stanford Question Answering Dataset (SQuAD). + + Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by + crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, + from the corresponding reading passage, or the question might be unanswerable. +--- + +# Metric Card for SQuAD + +## Metric description +This metric wraps the official scoring script for version 1 of the [Stanford Question Answering Dataset (SQuAD)](https://huggingface.co/datasets/squad). + +SQuAD is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable. + +## How to use + +The metric takes two files or two lists of question-answers dictionaries as inputs : one with the predictions of the model and the other with the references to be compared to: + +```python +from evaluate import load +squad_metric = load("squad") +results = squad_metric.compute(predictions=predictions, references=references) +``` +## Output values + +This metric outputs a dictionary with two values: the average exact match score and the average [F1 score](https://huggingface.co/metrics/f1). + +``` +{'exact_match': 100.0, 'f1': 100.0} +``` + +The range of `exact_match` is 0-100, where 0.0 means no answers were matched and 100.0 means all answers were matched. + +The range of `f1` is 0-1 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. + +### Values from popular papers +The [original SQuAD paper](https://nlp.stanford.edu/pubs/rajpurkar2016squad.pdf) reported an F1 score of 51.0% and an Exact Match score of 40.0%. They also report that human performance on the dataset represents an F1 score of 90.5% and an Exact Match score of 80.3%. + +For more recent model performance, see the [dataset leaderboard](https://paperswithcode.com/dataset/squad). + +## Examples + +Maximal values for both exact match and F1 (perfect match): + +```python +from evaluate import load +squad_metric = load("squad") +predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22'}] +references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] +results = squad_metric.compute(predictions=predictions, references=references) +results +{'exact_match': 100.0, 'f1': 100.0} +``` + +Minimal values for both exact match and F1 (no match): + +```python +from evaluate import load +squad_metric = load("squad") +predictions = [{'prediction_text': '1999', 'id': '56e10a3be3433e1400422b22'}] +references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] +results = squad_metric.compute(predictions=predictions, references=references) +results +{'exact_match': 0.0, 'f1': 0.0} +``` + +Partial match (2 out of 3 answers correct) : + +```python +from evaluate import load +squad_metric = load("squad") +predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22'}, {'prediction_text': 'Beyonce', 'id': '56d2051ce7d4791d0090260b'}, {'prediction_text': 'climate change', 'id': '5733b5344776f419006610e1'}] +references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}, {'answers': {'answer_start': [233], 'text': ['Beyoncé and Bruno Mars']}, 'id': '56d2051ce7d4791d0090260b'}, {'answers': {'answer_start': [891], 'text': ['climate change']}, 'id': '5733b5344776f419006610e1'}] +results = squad_metric.compute(predictions=predictions, references=references) +results +{'exact_match': 66.66666666666667, 'f1': 66.66666666666667} +``` + +## Limitations and bias +This metric works only with datasets that have the same format as [SQuAD v.1 dataset](https://huggingface.co/datasets/squad). + +The SQuAD dataset does contain a certain amount of noise, such as duplicate questions as well as missing answers, but these represent a minority of the 100,000 question-answer pairs. Also, neither exact match nor F1 score reflect whether models do better on certain types of questions (e.g. who questions) or those that cover a certain gender or geographical area -- carrying out more in-depth error analysis can complement these numbers. + + +## Citation + + @inproceedings{Rajpurkar2016SQuAD10, + title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text}, + author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang}, + booktitle={EMNLP}, + year={2016} + } + +## Further References + +- [The Stanford Question Answering Dataset: Background, Challenges, Progress (blog post)](https://rajpurkar.github.io/mlx/qa-and-squad/) +- [Hugging Face Course -- Question Answering](https://huggingface.co/course/chapter7/7) diff --git a/TextClassification/TextClassification/evaluate/metrics/squad/app.py b/TextClassification/TextClassification/evaluate/metrics/squad/app.py new file mode 100644 index 0000000000000000000000000000000000000000..7e22ac6a4baf025348645eb91d8f48fd206f715a --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/squad/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("squad") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/squad/compute_score.py b/TextClassification/TextClassification/evaluate/metrics/squad/compute_score.py new file mode 100644 index 0000000000000000000000000000000000000000..e60acfd1044319e43a59ffe8db75e63b68785ba7 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/squad/compute_score.py @@ -0,0 +1,92 @@ +""" Official evaluation script for v1.1 of the SQuAD dataset. """ + +import argparse +import json +import re +import string +import sys +from collections import Counter + + +def normalize_answer(s): + """Lower text and remove punctuation, articles and extra whitespace.""" + + def remove_articles(text): + return re.sub(r"\b(a|an|the)\b", " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def f1_score(prediction, ground_truth): + prediction_tokens = normalize_answer(prediction).split() + ground_truth_tokens = normalize_answer(ground_truth).split() + common = Counter(prediction_tokens) & Counter(ground_truth_tokens) + num_same = sum(common.values()) + if num_same == 0: + return 0 + precision = 1.0 * num_same / len(prediction_tokens) + recall = 1.0 * num_same / len(ground_truth_tokens) + f1 = (2 * precision * recall) / (precision + recall) + return f1 + + +def exact_match_score(prediction, ground_truth): + return normalize_answer(prediction) == normalize_answer(ground_truth) + + +def metric_max_over_ground_truths(metric_fn, prediction, ground_truths): + scores_for_ground_truths = [] + for ground_truth in ground_truths: + score = metric_fn(prediction, ground_truth) + scores_for_ground_truths.append(score) + return max(scores_for_ground_truths) + + +def compute_score(dataset, predictions): + f1 = exact_match = total = 0 + for article in dataset: + for paragraph in article["paragraphs"]: + for qa in paragraph["qas"]: + total += 1 + if qa["id"] not in predictions: + message = "Unanswered question " + qa["id"] + " will receive score 0." + print(message, file=sys.stderr) + continue + ground_truths = list(map(lambda x: x["text"], qa["answers"])) + prediction = predictions[qa["id"]] + exact_match += metric_max_over_ground_truths(exact_match_score, prediction, ground_truths) + f1 += metric_max_over_ground_truths(f1_score, prediction, ground_truths) + + exact_match = 100.0 * exact_match / total + f1 = 100.0 * f1 / total + + return {"exact_match": exact_match, "f1": f1} + + +if __name__ == "__main__": + expected_version = "1.1" + parser = argparse.ArgumentParser(description="Evaluation for SQuAD " + expected_version) + parser.add_argument("dataset_file", help="Dataset file") + parser.add_argument("prediction_file", help="Prediction File") + args = parser.parse_args() + with open(args.dataset_file) as dataset_file: + dataset_json = json.load(dataset_file) + if dataset_json["version"] != expected_version: + print( + "Evaluation expects v-" + expected_version + ", but got dataset with v-" + dataset_json["version"], + file=sys.stderr, + ) + dataset = dataset_json["data"] + with open(args.prediction_file) as prediction_file: + predictions = json.load(prediction_file) + print(json.dumps(compute_score(dataset, predictions))) diff --git a/TextClassification/TextClassification/evaluate/metrics/squad/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/squad/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a135afacbdd142a40cb9bcfe6073b76cc62d995 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/squad/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/squad/squad.py b/TextClassification/TextClassification/evaluate/metrics/squad/squad.py new file mode 100644 index 0000000000000000000000000000000000000000..84658b125f47aed592b6da4659ec60b22e02fe34 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/squad/squad.py @@ -0,0 +1,111 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" SQuAD metric. """ + +import datasets + +import evaluate + +from .compute_score import compute_score + + +_CITATION = """\ +@inproceedings{Rajpurkar2016SQuAD10, + title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text}, + author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang}, + booktitle={EMNLP}, + year={2016} +} +""" + +_DESCRIPTION = """ +This metric wrap the official scoring script for version 1 of the Stanford Question Answering Dataset (SQuAD). + +Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by +crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, +from the corresponding reading passage, or the question might be unanswerable. +""" + +_KWARGS_DESCRIPTION = """ +Computes SQuAD scores (F1 and EM). +Args: + predictions: List of question-answers dictionaries with the following key-values: + - 'id': id of the question-answer pair as given in the references (see below) + - 'prediction_text': the text of the answer + references: List of question-answers dictionaries with the following key-values: + - 'id': id of the question-answer pair (see above), + - 'answers': a Dict in the SQuAD dataset format + { + 'text': list of possible texts for the answer, as a list of strings + 'answer_start': list of start positions for the answer, as a list of ints + } + Note that answer_start values are not taken into account to compute the metric. +Returns: + 'exact_match': Exact match (the normalized answer exactly match the gold answer) + 'f1': The F-score of predicted tokens versus the gold answer +Examples: + + >>> predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22'}] + >>> references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] + >>> squad_metric = evaluate.load("squad") + >>> results = squad_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'exact_match': 100.0, 'f1': 100.0} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Squad(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": {"id": datasets.Value("string"), "prediction_text": datasets.Value("string")}, + "references": { + "id": datasets.Value("string"), + "answers": datasets.features.Sequence( + { + "text": datasets.Value("string"), + "answer_start": datasets.Value("int32"), + } + ), + }, + } + ), + codebase_urls=["https://rajpurkar.github.io/SQuAD-explorer/"], + reference_urls=["https://rajpurkar.github.io/SQuAD-explorer/"], + ) + + def _compute(self, predictions, references): + pred_dict = {prediction["id"]: prediction["prediction_text"] for prediction in predictions} + dataset = [ + { + "paragraphs": [ + { + "qas": [ + { + "answers": [{"text": answer_text} for answer_text in ref["answers"]["text"]], + "id": ref["id"], + } + for ref in references + ] + } + ] + } + ] + score = compute_score(dataset=dataset, predictions=pred_dict) + return score diff --git a/TextClassification/TextClassification/evaluate/metrics/squad_v2/README.md b/TextClassification/TextClassification/evaluate/metrics/squad_v2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6c4a7e2435106940f05565b5523f3dc53c98ed1d --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/squad_v2/README.md @@ -0,0 +1,142 @@ +--- +title: SQuAD v2 +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + This metric wrap the official scoring script for version 2 of the Stanford Question Answering Dataset (SQuAD). + + Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by + crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, + from the corresponding reading passage, or the question might be unanswerable. + + SQuAD2.0 combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions + written adversarially by crowdworkers to look similar to answerable ones. + To do well on SQuAD2.0, systems must not only answer questions when possible, but also + determine when no answer is supported by the paragraph and abstain from answering. +--- + +# Metric Card for SQuAD v2 + +## Metric description +This metric wraps the official scoring script for version 2 of the [Stanford Question Answering Dataset (SQuAD)](https://huggingface.co/datasets/squad_v2). + +SQuAD is a reading comprehension dataset, consisting of questions posed by crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable. + +SQuAD 2.0 combines the 100,000 questions in SQuAD 1.1 with over 50,000 unanswerable questions written adversarially by crowdworkers to look similar to answerable ones. To do well on SQuAD2.0, systems must not only answer questions when possible, but also determine when no answer is supported by the paragraph and abstain from answering. + +## How to use + +The metric takes two files or two lists - one representing model predictions and the other the references to compare them to. + +*Predictions* : List of triple for question-answers to score with the following key-value pairs: +* `'id'`: the question-answer identification field of the question and answer pair +* `'prediction_text'` : the text of the answer +* `'no_answer_probability'` : the probability that the question has no answer + +*References*: List of question-answers dictionaries with the following key-value pairs: +* `'id'`: id of the question-answer pair (see above), +* `'answers'`: a list of Dict {'text': text of the answer as a string} +* `'no_answer_threshold'`: the probability threshold to decide that a question has no answer. + +```python +from evaluate import load +squad_metric = load("squad_v2") +results = squad_metric.compute(predictions=predictions, references=references) +``` +## Output values + +This metric outputs a dictionary with 13 values: +* `'exact'`: Exact match (the normalized answer exactly match the gold answer) (see the `exact_match` metric (forthcoming)) +* `'f1'`: The average F1-score of predicted tokens versus the gold answer (see the [F1 score](https://huggingface.co/metrics/f1) metric) +* `'total'`: Number of scores considered +* `'HasAns_exact'`: Exact match (the normalized answer exactly match the gold answer) +* `'HasAns_f1'`: The F-score of predicted tokens versus the gold answer +* `'HasAns_total'`: How many of the questions have answers +* `'NoAns_exact'`: Exact match (the normalized answer exactly match the gold answer) +* `'NoAns_f1'`: The F-score of predicted tokens versus the gold answer +* `'NoAns_total'`: How many of the questions have no answers +* `'best_exact'` : Best exact match (with varying threshold) +* `'best_exact_thresh'`: No-answer probability threshold associated to the best exact match +* `'best_f1'`: Best F1 score (with varying threshold) +* `'best_f1_thresh'`: No-answer probability threshold associated to the best F1 + + +The range of `exact_match` is 0-100, where 0.0 means no answers were matched and 100.0 means all answers were matched. + +The range of `f1` is 0-1 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. + +The range of `total` depends on the length of predictions/references: its minimal value is 0, and maximal value is the total number of questions in the predictions and references. + +### Values from popular papers +The [SQuAD v2 paper](https://arxiv.org/pdf/1806.03822.pdf) reported an F1 score of 66.3% and an Exact Match score of 63.4%. +They also report that human performance on the dataset represents an F1 score of 89.5% and an Exact Match score of 86.9%. + +For more recent model performance, see the [dataset leaderboard](https://paperswithcode.com/dataset/squad). + +## Examples + +Maximal values for both exact match and F1 (perfect match): + +```python +from evaluate import load +squad_v2_ metric = load("squad_v2") +predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22', 'no_answer_probability': 0.}] +references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] +results = squad_v2_metric.compute(predictions=predictions, references=references) +results +{'exact': 100.0, 'f1': 100.0, 'total': 1, 'HasAns_exact': 100.0, 'HasAns_f1': 100.0, 'HasAns_total': 1, 'best_exact': 100.0, 'best_exact_thresh': 0.0, 'best_f1': 100.0, 'best_f1_thresh': 0.0} +``` + +Minimal values for both exact match and F1 (no match): + +```python +from evaluate import load +squad_metric = load("squad_v2") +predictions = [{'prediction_text': '1999', 'id': '56e10a3be3433e1400422b22', 'no_answer_probability': 0.}] +references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] +results = squad_v2_metric.compute(predictions=predictions, references=references) +results +{'exact': 0.0, 'f1': 0.0, 'total': 1, 'HasAns_exact': 0.0, 'HasAns_f1': 0.0, 'HasAns_total': 1, 'best_exact': 0.0, 'best_exact_thresh': 0.0, 'best_f1': 0.0, 'best_f1_thresh': 0.0} +``` + +Partial match (2 out of 3 answers correct) : + +```python +from evaluate import load +squad_metric = load("squad_v2") +predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22', 'no_answer_probability': 0.}, {'prediction_text': 'Beyonce', 'id': '56d2051ce7d4791d0090260b', 'no_answer_probability': 0.}, {'prediction_text': 'climate change', 'id': '5733b5344776f419006610e1', 'no_answer_probability': 0.}] +references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}, {'answers': {'answer_start': [233], 'text': ['Beyoncé and Bruno Mars']}, 'id': '56d2051ce7d4791d0090260b'}, {'answers': {'answer_start': [891], 'text': ['climate change']}, 'id': '5733b5344776f419006610e1'}] +results = squad_v2_metric.compute(predictions=predictions, references=references) +results +{'exact': 66.66666666666667, 'f1': 66.66666666666667, 'total': 3, 'HasAns_exact': 66.66666666666667, 'HasAns_f1': 66.66666666666667, 'HasAns_total': 3, 'best_exact': 66.66666666666667, 'best_exact_thresh': 0.0, 'best_f1': 66.66666666666667, 'best_f1_thresh': 0.0} +``` + +## Limitations and bias +This metric works only with the datasets in the same format as the [SQuAD v.2 dataset](https://huggingface.co/datasets/squad_v2). + +The SQuAD datasets do contain a certain amount of noise, such as duplicate questions as well as missing answers, but these represent a minority of the 100,000 question-answer pairs. Also, neither exact match nor F1 score reflect whether models do better on certain types of questions (e.g. who questions) or those that cover a certain gender or geographical area -- carrying out more in-depth error analysis can complement these numbers. + + +## Citation + +```bibtex +@inproceedings{Rajpurkar2018SQuAD2, +title={Know What You Don't Know: Unanswerable Questions for SQuAD}, +author={Pranav Rajpurkar and Jian Zhang and Percy Liang}, +booktitle={ACL 2018}, +year={2018} +} +``` + +## Further References + +- [The Stanford Question Answering Dataset: Background, Challenges, Progress (blog post)](https://rajpurkar.github.io/mlx/qa-and-squad/) +- [Hugging Face Course -- Question Answering](https://huggingface.co/course/chapter7/7) diff --git a/TextClassification/TextClassification/evaluate/metrics/squad_v2/app.py b/TextClassification/TextClassification/evaluate/metrics/squad_v2/app.py new file mode 100644 index 0000000000000000000000000000000000000000..6e6ccafc20ab508661ed8a2964e51772e7012fc4 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/squad_v2/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("squad_v2") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/squad_v2/compute_score.py b/TextClassification/TextClassification/evaluate/metrics/squad_v2/compute_score.py new file mode 100644 index 0000000000000000000000000000000000000000..3b512ae92484772a8d10bdf43f4df68520cdf3ed --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/squad_v2/compute_score.py @@ -0,0 +1,323 @@ +"""Official evaluation script for SQuAD version 2.0. + +In addition to basic functionality, we also compute additional statistics and +plot precision-recall curves if an additional na_prob.json file is provided. +This file is expected to map question ID's to the model's predicted probability +that a question is unanswerable. +""" +import argparse +import collections +import json +import os +import re +import string +import sys + +import numpy as np + + +ARTICLES_REGEX = re.compile(r"\b(a|an|the)\b", re.UNICODE) + +OPTS = None + + +def parse_args(): + parser = argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0.") + parser.add_argument("data_file", metavar="data.json", help="Input data JSON file.") + parser.add_argument("pred_file", metavar="pred.json", help="Model predictions.") + parser.add_argument( + "--out-file", "-o", metavar="eval.json", help="Write accuracy metrics to file (default is stdout)." + ) + parser.add_argument( + "--na-prob-file", "-n", metavar="na_prob.json", help="Model estimates of probability of no answer." + ) + parser.add_argument( + "--na-prob-thresh", + "-t", + type=float, + default=1.0, + help='Predict "" if no-answer probability exceeds this (default = 1.0).', + ) + parser.add_argument( + "--out-image-dir", "-p", metavar="out_images", default=None, help="Save precision-recall curves to directory." + ) + parser.add_argument("--verbose", "-v", action="store_true") + if len(sys.argv) == 1: + parser.print_help() + sys.exit(1) + return parser.parse_args() + + +def make_qid_to_has_ans(dataset): + qid_to_has_ans = {} + for article in dataset: + for p in article["paragraphs"]: + for qa in p["qas"]: + qid_to_has_ans[qa["id"]] = bool(qa["answers"]["text"]) + return qid_to_has_ans + + +def normalize_answer(s): + """Lower text and remove punctuation, articles and extra whitespace.""" + + def remove_articles(text): + return ARTICLES_REGEX.sub(" ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def get_tokens(s): + if not s: + return [] + return normalize_answer(s).split() + + +def compute_exact(a_gold, a_pred): + return int(normalize_answer(a_gold) == normalize_answer(a_pred)) + + +def compute_f1(a_gold, a_pred): + gold_toks = get_tokens(a_gold) + pred_toks = get_tokens(a_pred) + common = collections.Counter(gold_toks) & collections.Counter(pred_toks) + num_same = sum(common.values()) + if len(gold_toks) == 0 or len(pred_toks) == 0: + # If either is no-answer, then F1 is 1 if they agree, 0 otherwise + return int(gold_toks == pred_toks) + if num_same == 0: + return 0 + precision = 1.0 * num_same / len(pred_toks) + recall = 1.0 * num_same / len(gold_toks) + f1 = (2 * precision * recall) / (precision + recall) + return f1 + + +def get_raw_scores(dataset, preds): + exact_scores = {} + f1_scores = {} + for article in dataset: + for p in article["paragraphs"]: + for qa in p["qas"]: + qid = qa["id"] + gold_answers = [t for t in qa["answers"]["text"] if normalize_answer(t)] + if not gold_answers: + # For unanswerable questions, only correct answer is empty string + gold_answers = [""] + if qid not in preds: + print(f"Missing prediction for {qid}") + continue + a_pred = preds[qid] + # Take max over all gold answers + exact_scores[qid] = max(compute_exact(a, a_pred) for a in gold_answers) + f1_scores[qid] = max(compute_f1(a, a_pred) for a in gold_answers) + return exact_scores, f1_scores + + +def apply_no_ans_threshold(scores, na_probs, qid_to_has_ans, na_prob_thresh): + new_scores = {} + for qid, s in scores.items(): + pred_na = na_probs[qid] > na_prob_thresh + if pred_na: + new_scores[qid] = float(not qid_to_has_ans[qid]) + else: + new_scores[qid] = s + return new_scores + + +def make_eval_dict(exact_scores, f1_scores, qid_list=None): + if not qid_list: + total = len(exact_scores) + return collections.OrderedDict( + [ + ("exact", 100.0 * sum(exact_scores.values()) / total), + ("f1", 100.0 * sum(f1_scores.values()) / total), + ("total", total), + ] + ) + else: + total = len(qid_list) + return collections.OrderedDict( + [ + ("exact", 100.0 * sum(exact_scores[k] for k in qid_list) / total), + ("f1", 100.0 * sum(f1_scores[k] for k in qid_list) / total), + ("total", total), + ] + ) + + +def merge_eval(main_eval, new_eval, prefix): + for k in new_eval: + main_eval[f"{prefix}_{k}"] = new_eval[k] + + +def plot_pr_curve(precisions, recalls, out_image, title): + plt.step(recalls, precisions, color="b", alpha=0.2, where="post") + plt.fill_between(recalls, precisions, step="post", alpha=0.2, color="b") + plt.xlabel("Recall") + plt.ylabel("Precision") + plt.xlim([0.0, 1.05]) + plt.ylim([0.0, 1.05]) + plt.title(title) + plt.savefig(out_image) + plt.clf() + + +def make_precision_recall_eval(scores, na_probs, num_true_pos, qid_to_has_ans, out_image=None, title=None): + qid_list = sorted(na_probs, key=lambda k: na_probs[k]) + true_pos = 0.0 + cur_p = 1.0 + cur_r = 0.0 + precisions = [1.0] + recalls = [0.0] + avg_prec = 0.0 + for i, qid in enumerate(qid_list): + if qid_to_has_ans[qid]: + true_pos += scores[qid] + cur_p = true_pos / float(i + 1) + cur_r = true_pos / float(num_true_pos) + if i == len(qid_list) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]: + # i.e., if we can put a threshold after this point + avg_prec += cur_p * (cur_r - recalls[-1]) + precisions.append(cur_p) + recalls.append(cur_r) + if out_image: + plot_pr_curve(precisions, recalls, out_image, title) + return {"ap": 100.0 * avg_prec} + + +def run_precision_recall_analysis(main_eval, exact_raw, f1_raw, na_probs, qid_to_has_ans, out_image_dir): + if out_image_dir and not os.path.exists(out_image_dir): + os.makedirs(out_image_dir) + num_true_pos = sum(1 for v in qid_to_has_ans.values() if v) + if num_true_pos == 0: + return + pr_exact = make_precision_recall_eval( + exact_raw, + na_probs, + num_true_pos, + qid_to_has_ans, + out_image=os.path.join(out_image_dir, "pr_exact.png"), + title="Precision-Recall curve for Exact Match score", + ) + pr_f1 = make_precision_recall_eval( + f1_raw, + na_probs, + num_true_pos, + qid_to_has_ans, + out_image=os.path.join(out_image_dir, "pr_f1.png"), + title="Precision-Recall curve for F1 score", + ) + oracle_scores = {k: float(v) for k, v in qid_to_has_ans.items()} + pr_oracle = make_precision_recall_eval( + oracle_scores, + na_probs, + num_true_pos, + qid_to_has_ans, + out_image=os.path.join(out_image_dir, "pr_oracle.png"), + title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)", + ) + merge_eval(main_eval, pr_exact, "pr_exact") + merge_eval(main_eval, pr_f1, "pr_f1") + merge_eval(main_eval, pr_oracle, "pr_oracle") + + +def histogram_na_prob(na_probs, qid_list, image_dir, name): + if not qid_list: + return + x = [na_probs[k] for k in qid_list] + weights = np.ones_like(x) / float(len(x)) + plt.hist(x, weights=weights, bins=20, range=(0.0, 1.0)) + plt.xlabel("Model probability of no-answer") + plt.ylabel("Proportion of dataset") + plt.title(f"Histogram of no-answer probability: {name}") + plt.savefig(os.path.join(image_dir, f"na_prob_hist_{name}.png")) + plt.clf() + + +def find_best_thresh(preds, scores, na_probs, qid_to_has_ans): + num_no_ans = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]) + cur_score = num_no_ans + best_score = cur_score + best_thresh = 0.0 + qid_list = sorted(na_probs, key=lambda k: na_probs[k]) + for i, qid in enumerate(qid_list): + if qid not in scores: + continue + if qid_to_has_ans[qid]: + diff = scores[qid] + else: + if preds[qid]: + diff = -1 + else: + diff = 0 + cur_score += diff + if cur_score > best_score: + best_score = cur_score + best_thresh = na_probs[qid] + return 100.0 * best_score / len(scores), best_thresh + + +def find_all_best_thresh(main_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans): + best_exact, exact_thresh = find_best_thresh(preds, exact_raw, na_probs, qid_to_has_ans) + best_f1, f1_thresh = find_best_thresh(preds, f1_raw, na_probs, qid_to_has_ans) + main_eval["best_exact"] = best_exact + main_eval["best_exact_thresh"] = exact_thresh + main_eval["best_f1"] = best_f1 + main_eval["best_f1_thresh"] = f1_thresh + + +def main(): + with open(OPTS.data_file) as f: + dataset_json = json.load(f) + dataset = dataset_json["data"] + with open(OPTS.pred_file) as f: + preds = json.load(f) + if OPTS.na_prob_file: + with open(OPTS.na_prob_file) as f: + na_probs = json.load(f) + else: + na_probs = {k: 0.0 for k in preds} + qid_to_has_ans = make_qid_to_has_ans(dataset) # maps qid to True/False + has_ans_qids = [k for k, v in qid_to_has_ans.items() if v] + no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v] + exact_raw, f1_raw = get_raw_scores(dataset, preds) + exact_thresh = apply_no_ans_threshold(exact_raw, na_probs, qid_to_has_ans, OPTS.na_prob_thresh) + f1_thresh = apply_no_ans_threshold(f1_raw, na_probs, qid_to_has_ans, OPTS.na_prob_thresh) + out_eval = make_eval_dict(exact_thresh, f1_thresh) + if has_ans_qids: + has_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=has_ans_qids) + merge_eval(out_eval, has_ans_eval, "HasAns") + if no_ans_qids: + no_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=no_ans_qids) + merge_eval(out_eval, no_ans_eval, "NoAns") + if OPTS.na_prob_file: + find_all_best_thresh(out_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans) + if OPTS.na_prob_file and OPTS.out_image_dir: + run_precision_recall_analysis(out_eval, exact_raw, f1_raw, na_probs, qid_to_has_ans, OPTS.out_image_dir) + histogram_na_prob(na_probs, has_ans_qids, OPTS.out_image_dir, "hasAns") + histogram_na_prob(na_probs, no_ans_qids, OPTS.out_image_dir, "noAns") + if OPTS.out_file: + with open(OPTS.out_file, "w") as f: + json.dump(out_eval, f) + else: + print(json.dumps(out_eval, indent=2)) + + +if __name__ == "__main__": + OPTS = parse_args() + if OPTS.out_image_dir: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + main() diff --git a/TextClassification/TextClassification/evaluate/metrics/squad_v2/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/squad_v2/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a135afacbdd142a40cb9bcfe6073b76cc62d995 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/squad_v2/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/squad_v2/squad_v2.py b/TextClassification/TextClassification/evaluate/metrics/squad_v2/squad_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..cb9ba1ae85756f2b1d2bd1713c87c92b87e4c0f8 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/squad_v2/squad_v2.py @@ -0,0 +1,137 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" SQuAD v2 metric. """ + +import datasets + +import evaluate + +from .compute_score import ( + apply_no_ans_threshold, + find_all_best_thresh, + get_raw_scores, + make_eval_dict, + make_qid_to_has_ans, + merge_eval, +) + + +_CITATION = """\ +@inproceedings{Rajpurkar2016SQuAD10, + title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text}, + author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang}, + booktitle={EMNLP}, + year={2016} +} +""" + +_DESCRIPTION = """ +This metric wrap the official scoring script for version 2 of the Stanford Question +Answering Dataset (SQuAD). + +Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by +crowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span, +from the corresponding reading passage, or the question might be unanswerable. + +SQuAD2.0 combines the 100,000 questions in SQuAD1.1 with over 50,000 unanswerable questions +written adversarially by crowdworkers to look similar to answerable ones. +To do well on SQuAD2.0, systems must not only answer questions when possible, but also +determine when no answer is supported by the paragraph and abstain from answering. +""" + +_KWARGS_DESCRIPTION = """ +Computes SQuAD v2 scores (F1 and EM). +Args: + predictions: List of triple for question-answers to score with the following elements: + - the question-answer 'id' field as given in the references (see below) + - the text of the answer + - the probability that the question has no answer + references: List of question-answers dictionaries with the following key-values: + - 'id': id of the question-answer pair (see above), + - 'answers': a list of Dict {'text': text of the answer as a string} + no_answer_threshold: float + Probability threshold to decide that a question has no answer. +Returns: + 'exact': Exact match (the normalized answer exactly match the gold answer) + 'f1': The F-score of predicted tokens versus the gold answer + 'total': Number of score considered + 'HasAns_exact': Exact match (the normalized answer exactly match the gold answer) + 'HasAns_f1': The F-score of predicted tokens versus the gold answer + 'HasAns_total': Number of score considered + 'NoAns_exact': Exact match (the normalized answer exactly match the gold answer) + 'NoAns_f1': The F-score of predicted tokens versus the gold answer + 'NoAns_total': Number of score considered + 'best_exact': Best exact match (with varying threshold) + 'best_exact_thresh': No-answer probability threshold associated to the best exact match + 'best_f1': Best F1 (with varying threshold) + 'best_f1_thresh': No-answer probability threshold associated to the best F1 +Examples: + + >>> predictions = [{'prediction_text': '1976', 'id': '56e10a3be3433e1400422b22', 'no_answer_probability': 0.}] + >>> references = [{'answers': {'answer_start': [97], 'text': ['1976']}, 'id': '56e10a3be3433e1400422b22'}] + >>> squad_v2_metric = evaluate.load("squad_v2") + >>> results = squad_v2_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'exact': 100.0, 'f1': 100.0, 'total': 1, 'HasAns_exact': 100.0, 'HasAns_f1': 100.0, 'HasAns_total': 1, 'best_exact': 100.0, 'best_exact_thresh': 0.0, 'best_f1': 100.0, 'best_f1_thresh': 0.0} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class SquadV2(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": { + "id": datasets.Value("string"), + "prediction_text": datasets.Value("string"), + "no_answer_probability": datasets.Value("float32"), + }, + "references": { + "id": datasets.Value("string"), + "answers": datasets.features.Sequence( + {"text": datasets.Value("string"), "answer_start": datasets.Value("int32")} + ), + }, + } + ), + codebase_urls=["https://rajpurkar.github.io/SQuAD-explorer/"], + reference_urls=["https://rajpurkar.github.io/SQuAD-explorer/"], + ) + + def _compute(self, predictions, references, no_answer_threshold=1.0): + no_answer_probabilities = {p["id"]: p["no_answer_probability"] for p in predictions} + dataset = [{"paragraphs": [{"qas": references}]}] + predictions = {p["id"]: p["prediction_text"] for p in predictions} + + qid_to_has_ans = make_qid_to_has_ans(dataset) # maps qid to True/False + has_ans_qids = [k for k, v in qid_to_has_ans.items() if v] + no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v] + + exact_raw, f1_raw = get_raw_scores(dataset, predictions) + exact_thresh = apply_no_ans_threshold(exact_raw, no_answer_probabilities, qid_to_has_ans, no_answer_threshold) + f1_thresh = apply_no_ans_threshold(f1_raw, no_answer_probabilities, qid_to_has_ans, no_answer_threshold) + out_eval = make_eval_dict(exact_thresh, f1_thresh) + + if has_ans_qids: + has_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=has_ans_qids) + merge_eval(out_eval, has_ans_eval, "HasAns") + if no_ans_qids: + no_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=no_ans_qids) + merge_eval(out_eval, no_ans_eval, "NoAns") + find_all_best_thresh(out_eval, predictions, exact_raw, f1_raw, no_answer_probabilities, qid_to_has_ans) + return dict(out_eval) diff --git a/TextClassification/TextClassification/evaluate/metrics/super_glue/README.md b/TextClassification/TextClassification/evaluate/metrics/super_glue/README.md new file mode 100644 index 0000000000000000000000000000000000000000..edcb4b8c8c224cbc3c041b96cf0ad4f1de897c1b --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/super_glue/README.md @@ -0,0 +1,128 @@ +--- +title: SuperGLUE +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after GLUE with a new set of more difficult language understanding tasks, improved resources, and a new public leaderboard. +--- + +# Metric Card for SuperGLUE + +## Metric description +This metric is used to compute the SuperGLUE evaluation metric associated to each of the subsets of the [SuperGLUE dataset](https://huggingface.co/datasets/super_glue). + +SuperGLUE is a new benchmark styled after GLUE with a new set of more difficult language understanding tasks, improved resources, and a new public leaderboard. + + +## How to use + +There are two steps: (1) loading the SuperGLUE metric relevant to the subset of the dataset being used for evaluation; and (2) calculating the metric. + +1. **Loading the relevant SuperGLUE metric** : the subsets of SuperGLUE are the following: `boolq`, `cb`, `copa`, `multirc`, `record`, `rte`, `wic`, `wsc`, `wsc.fixed`, `axb`, `axg`. + +More information about the different subsets of the SuperGLUE dataset can be found on the [SuperGLUE dataset page](https://huggingface.co/datasets/super_glue) and on the [official dataset website](https://super.gluebenchmark.com/). + +2. **Calculating the metric**: the metric takes two inputs : one list with the predictions of the model to score and one list of reference labels. The structure of both inputs depends on the SuperGlUE subset being used: + +Format of `predictions`: +- for `record`: list of question-answer dictionaries with the following keys: + - `idx`: index of the question as specified by the dataset + - `prediction_text`: the predicted answer text +- for `multirc`: list of question-answer dictionaries with the following keys: + - `idx`: index of the question-answer pair as specified by the dataset + - `prediction`: the predicted answer label +- otherwise: list of predicted labels + +Format of `references`: +- for `record`: list of question-answers dictionaries with the following keys: + - `idx`: index of the question as specified by the dataset + - `answers`: list of possible answers +- otherwise: list of reference labels + +```python +from evaluate import load +super_glue_metric = load('super_glue', 'copa') +predictions = [0, 1] +references = [0, 1] +results = super_glue_metric.compute(predictions=predictions, references=references) +``` +## Output values + +The output of the metric depends on the SuperGLUE subset chosen, consisting of a dictionary that contains one or several of the following metrics: + +`exact_match`: A given predicted string's exact match score is 1 if it is the exact same as its reference string, and is 0 otherwise. (See [Exact Match](https://huggingface.co/metrics/exact_match) for more information). + +`f1`: the harmonic mean of the precision and recall (see [F1 score](https://huggingface.co/metrics/f1) for more information). Its range is 0-1 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. + +`matthews_correlation`: a measure of the quality of binary and multiclass classifications (see [Matthews Correlation](https://huggingface.co/metrics/matthews_correlation) for more information). Its range of values is between -1 and +1, where a coefficient of +1 represents a perfect prediction, 0 an average random prediction and -1 an inverse prediction. + +### Values from popular papers +The [original SuperGLUE paper](https://arxiv.org/pdf/1905.00537.pdf) reported average scores ranging from 47 to 71.5%, depending on the model used (with all evaluation values scaled by 100 to make computing the average possible). + +For more recent model performance, see the [dataset leaderboard](https://super.gluebenchmark.com/leaderboard). + +## Examples + +Maximal values for the COPA subset (which outputs `accuracy`): + +```python +from evaluate import load +super_glue_metric = load('super_glue', 'copa') # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"] +predictions = [0, 1] +references = [0, 1] +results = super_glue_metric.compute(predictions=predictions, references=references) +print(results) +{'accuracy': 1.0} +``` + +Minimal values for the MultiRC subset (which outputs `pearson` and `spearmanr`): + +```python +from evaluate import load +super_glue_metric = load('super_glue', 'multirc') +predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}] +references = [1,0] +results = super_glue_metric.compute(predictions=predictions, references=references) +print(results) +{'exact_match': 0.0, 'f1_m': 0.0, 'f1_a': 0.0} +``` + +Partial match for the COLA subset (which outputs `matthews_correlation`) + +```python +from evaluate import load +super_glue_metric = load('super_glue', 'axb') +references = [0, 1] +predictions = [1,1] +results = super_glue_metric.compute(predictions=predictions, references=references) +print(results) +{'matthews_correlation': 0.0} +``` + +## Limitations and bias +This metric works only with datasets that have the same format as the [SuperGLUE dataset](https://huggingface.co/datasets/super_glue). + +The dataset also includes Winogender, a subset of the dataset that is designed to measure gender bias in coreference resolution systems. However, as noted in the SuperGLUE paper, this subset has its limitations: *"It offers only positive predictive value: A poor bias score is clear evidence that a model exhibits gender bias, but a good score does not mean that the model is unbiased.[...] Also, Winogender does not cover all forms of social bias, or even all forms of gender. For instance, the version of the data used here offers no coverage of gender-neutral they or non-binary pronouns." + +## Citation + +```bibtex +@article{wang2019superglue, + title={Super{GLUE}: A Stickier Benchmark for General-Purpose Language Understanding Systems}, + author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R}, + journal={arXiv preprint arXiv:1905.00537}, + year={2019} +} +``` + +## Further References + +- [SuperGLUE benchmark homepage](https://super.gluebenchmark.com/) diff --git a/TextClassification/TextClassification/evaluate/metrics/super_glue/app.py b/TextClassification/TextClassification/evaluate/metrics/super_glue/app.py new file mode 100644 index 0000000000000000000000000000000000000000..0d11b1f706b3d99f16537dcec21c0a46b85b5bf4 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/super_glue/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("super_glue", "copa") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/super_glue/record_evaluation.py b/TextClassification/TextClassification/evaluate/metrics/super_glue/record_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..396d7ac8abc6d52180fb8d960eddad9b03919a21 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/super_glue/record_evaluation.py @@ -0,0 +1,111 @@ +""" +Official evaluation script for ReCoRD v1.0. +(Some functions are adopted from the SQuAD evaluation script.) +""" + + +import argparse +import json +import re +import string +import sys +from collections import Counter + + +def normalize_answer(s): + """Lower text and remove punctuation, articles and extra whitespace.""" + + def remove_articles(text): + return re.sub(r"\b(a|an|the)\b", " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def f1_score(prediction, ground_truth): + prediction_tokens = normalize_answer(prediction).split() + ground_truth_tokens = normalize_answer(ground_truth).split() + common = Counter(prediction_tokens) & Counter(ground_truth_tokens) + num_same = sum(common.values()) + if num_same == 0: + return 0 + precision = 1.0 * num_same / len(prediction_tokens) + recall = 1.0 * num_same / len(ground_truth_tokens) + f1 = (2 * precision * recall) / (precision + recall) + return f1 + + +def exact_match_score(prediction, ground_truth): + return normalize_answer(prediction) == normalize_answer(ground_truth) + + +def metric_max_over_ground_truths(metric_fn, prediction, ground_truths): + scores_for_ground_truths = [] + for ground_truth in ground_truths: + score = metric_fn(prediction, ground_truth) + scores_for_ground_truths.append(score) + return max(scores_for_ground_truths) + + +def evaluate(dataset, predictions): + f1 = exact_match = total = 0 + correct_ids = [] + for passage in dataset: + for qa in passage["qas"]: + total += 1 + if qa["id"] not in predictions: + message = f'Unanswered question {qa["id"]} will receive score 0.' + print(message, file=sys.stderr) + continue + + ground_truths = list(map(lambda x: x["text"], qa["answers"])) + prediction = predictions[qa["id"]] + + _exact_match = metric_max_over_ground_truths(exact_match_score, prediction, ground_truths) + if int(_exact_match) == 1: + correct_ids.append(qa["id"]) + exact_match += _exact_match + + f1 += metric_max_over_ground_truths(f1_score, prediction, ground_truths) + + exact_match = exact_match / total + f1 = f1 / total + + return {"exact_match": exact_match, "f1": f1}, correct_ids + + +if __name__ == "__main__": + expected_version = "1.0" + parser = argparse.ArgumentParser("Official evaluation script for ReCoRD v1.0.") + parser.add_argument("data_file", help="The dataset file in JSON format.") + parser.add_argument("pred_file", help="The model prediction file in JSON format.") + parser.add_argument("--output_correct_ids", action="store_true", help="Output the correctly answered query IDs.") + args = parser.parse_args() + + with open(args.data_file) as data_file: + dataset_json = json.load(data_file) + if dataset_json["version"] != expected_version: + print( + f'Evaluation expects v-{expected_version}, but got dataset with v-{dataset_json["version"]}', + file=sys.stderr, + ) + dataset = dataset_json["data"] + + with open(args.pred_file) as pred_file: + predictions = json.load(pred_file) + + metrics, correct_ids = evaluate(dataset, predictions) + + if args.output_correct_ids: + print(f"Output {len(correct_ids)} correctly answered question IDs.") + with open("correct_ids.json", "w") as f: + json.dump(correct_ids, f) diff --git a/TextClassification/TextClassification/evaluate/metrics/super_glue/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/super_glue/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/super_glue/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/super_glue/super_glue.py b/TextClassification/TextClassification/evaluate/metrics/super_glue/super_glue.py new file mode 100644 index 0000000000000000000000000000000000000000..993a003b31c7f60536a42e2f4746f26382147ce9 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/super_glue/super_glue.py @@ -0,0 +1,237 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The SuperGLUE benchmark metric.""" + +import datasets +from sklearn.metrics import f1_score, matthews_corrcoef + +import evaluate + +from .record_evaluation import evaluate as evaluate_record + + +_CITATION = """\ +@article{wang2019superglue, + title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems}, + author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R}, + journal={arXiv preprint arXiv:1905.00537}, + year={2019} +} +""" + +_DESCRIPTION = """\ +SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after +GLUE with a new set of more difficult language understanding tasks, improved +resources, and a new public leaderboard. +""" + +_KWARGS_DESCRIPTION = """ +Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset. +Args: + predictions: list of predictions to score. Depending on the SuperGlUE subset: + - for 'record': list of question-answer dictionaries with the following keys: + - 'idx': index of the question as specified by the dataset + - 'prediction_text': the predicted answer text + - for 'multirc': list of question-answer dictionaries with the following keys: + - 'idx': index of the question-answer pair as specified by the dataset + - 'prediction': the predicted answer label + - otherwise: list of predicted labels + references: list of reference labels. Depending on the SuperGLUE subset: + - for 'record': list of question-answers dictionaries with the following keys: + - 'idx': index of the question as specified by the dataset + - 'answers': list of possible answers + - otherwise: list of reference labels +Returns: depending on the SuperGLUE subset: + - for 'record': + - 'exact_match': Exact match between answer and gold answer + - 'f1': F1 score + - for 'multirc': + - 'exact_match': Exact match between answer and gold answer + - 'f1_m': Per-question macro-F1 score + - 'f1_a': Average F1 score over all answers + - for 'axb': + 'matthews_correlation': Matthew Correlation + - for 'cb': + - 'accuracy': Accuracy + - 'f1': F1 score + - for all others: + - 'accuracy': Accuracy +Examples: + + >>> super_glue_metric = evaluate.load('super_glue', 'copa') # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"] + >>> predictions = [0, 1] + >>> references = [0, 1] + >>> results = super_glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'accuracy': 1.0} + + >>> super_glue_metric = evaluate.load('super_glue', 'cb') + >>> predictions = [0, 1] + >>> references = [0, 1] + >>> results = super_glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'accuracy': 1.0, 'f1': 1.0} + + >>> super_glue_metric = evaluate.load('super_glue', 'record') + >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}] + >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}] + >>> results = super_glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'exact_match': 1.0, 'f1': 1.0} + + >>> super_glue_metric = evaluate.load('super_glue', 'multirc') + >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}] + >>> references = [0, 1] + >>> results = super_glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0} + + >>> super_glue_metric = evaluate.load('super_glue', 'axb') + >>> references = [0, 1] + >>> predictions = [0, 1] + >>> results = super_glue_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'matthews_correlation': 1.0} +""" + + +def simple_accuracy(preds, labels): + return float((preds == labels).mean()) + + +def acc_and_f1(preds, labels, f1_avg="binary"): + acc = simple_accuracy(preds, labels) + f1 = float(f1_score(y_true=labels, y_pred=preds, average=f1_avg)) + return { + "accuracy": acc, + "f1": f1, + } + + +def evaluate_multirc(ids_preds, labels): + """ + Computes F1 score and Exact Match for MultiRC predictions. + """ + question_map = {} + for id_pred, label in zip(ids_preds, labels): + question_id = f'{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}' + pred = id_pred["prediction"] + if question_id in question_map: + question_map[question_id].append((pred, label)) + else: + question_map[question_id] = [(pred, label)] + f1s, ems = [], [] + for question, preds_labels in question_map.items(): + question_preds, question_labels = zip(*preds_labels) + f1 = f1_score(y_true=question_labels, y_pred=question_preds, average="macro") + f1s.append(f1) + em = int(sum(p == l for p, l in preds_labels) == len(preds_labels)) + ems.append(em) + f1_m = float(sum(f1s) / len(f1s)) + em = sum(ems) / len(ems) + f1_a = float(f1_score(y_true=labels, y_pred=[id_pred["prediction"] for id_pred in ids_preds])) + return {"exact_match": em, "f1_m": f1_m, "f1_a": f1_a} + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class SuperGlue(evaluate.Metric): + def _info(self): + if self.config_name not in [ + "boolq", + "cb", + "copa", + "multirc", + "record", + "rte", + "wic", + "wsc", + "wsc.fixed", + "axb", + "axg", + ]: + raise KeyError( + "You should supply a configuration name selected in " + '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]' + ) + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features(self._get_feature_types()), + codebase_urls=[], + reference_urls=[], + format="numpy" if not self.config_name == "record" and not self.config_name == "multirc" else None, + ) + + def _get_feature_types(self): + if self.config_name == "record": + return { + "predictions": { + "idx": { + "passage": datasets.Value("int64"), + "query": datasets.Value("int64"), + }, + "prediction_text": datasets.Value("string"), + }, + "references": { + "idx": { + "passage": datasets.Value("int64"), + "query": datasets.Value("int64"), + }, + "answers": datasets.Sequence(datasets.Value("string")), + }, + } + elif self.config_name == "multirc": + return { + "predictions": { + "idx": { + "answer": datasets.Value("int64"), + "paragraph": datasets.Value("int64"), + "question": datasets.Value("int64"), + }, + "prediction": datasets.Value("int64"), + }, + "references": datasets.Value("int64"), + } + else: + return { + "predictions": datasets.Value("int64"), + "references": datasets.Value("int64"), + } + + def _compute(self, predictions, references): + if self.config_name == "axb": + return {"matthews_correlation": matthews_corrcoef(references, predictions)} + elif self.config_name == "cb": + return acc_and_f1(predictions, references, f1_avg="macro") + elif self.config_name == "record": + dataset = [ + { + "qas": [ + {"id": ref["idx"]["query"], "answers": [{"text": ans} for ans in ref["answers"]]} + for ref in references + ] + } + ] + predictions = {pred["idx"]["query"]: pred["prediction_text"] for pred in predictions} + return evaluate_record(dataset, predictions)[0] + elif self.config_name == "multirc": + return evaluate_multirc(predictions, references) + elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]: + return {"accuracy": simple_accuracy(predictions, references)} + else: + raise KeyError( + "You should supply a configuration name selected in " + '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]' + ) diff --git a/TextClassification/TextClassification/evaluate/metrics/ter/README.md b/TextClassification/TextClassification/evaluate/metrics/ter/README.md new file mode 100644 index 0000000000000000000000000000000000000000..84aa2f948a00c2613002720a04abb183e2892a07 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/ter/README.md @@ -0,0 +1,175 @@ +--- +title: TER +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + TER (Translation Edit Rate, also called Translation Error Rate) is a metric to quantify the edit operations that a + hypothesis requires to match a reference translation. We use the implementation that is already present in sacrebleu + (https://github.com/mjpost/sacreBLEU#ter), which in turn is inspired by the TERCOM implementation, which can be found + here: https://github.com/jhclark/tercom. + + The implementation here is slightly different from sacrebleu in terms of the required input format. The length of + the references and hypotheses lists need to be the same, so you may need to transpose your references compared to + sacrebleu's required input format. See https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534 + + See the README.md file at https://github.com/mjpost/sacreBLEU#ter for more information. +--- + +# Metric Card for TER + +## Metric Description +TER (Translation Edit Rate, also called Translation Error Rate) is a metric to quantify the edit operations that a hypothesis requires to match a reference translation. We use the implementation that is already present in [sacrebleu](https://github.com/mjpost/sacreBLEU#ter), which in turn is inspired by the [TERCOM implementation](https://github.com/jhclark/tercom). + +The implementation here is slightly different from sacrebleu in terms of the required input format. The length of the references and hypotheses lists need to be the same, so you may need to transpose your references compared to sacrebleu's required input format. See [this github issue](https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534). + +See the README.md file at https://github.com/mjpost/sacreBLEU#ter for more information. + + +## How to Use +This metric takes, at minimum, predicted sentences and reference sentences: +```python +>>> predictions = ["does this sentence match??", +... "what about this sentence?", +... "What did the TER metric user say to the developer?"] +>>> references = [["does this sentence match", "does this sentence match!?!"], +... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"], +... ["Your jokes are...", "...TERrible"]] +>>> ter = evaluate.load("ter") +>>> results = ter.compute(predictions=predictions, +... references=references, +... case_sensitive=True) +>>> print(results) +{'score': 150.0, 'num_edits': 15, 'ref_length': 10.0} +``` + +### Inputs +This metric takes the following as input: +- **`predictions`** (`list` of `str`): The system stream (a sequence of segments). +- **`references`** (`list` of `list` of `str`): A list of one or more reference streams (each a sequence of segments). +- **`normalized`** (`boolean`): If `True`, applies basic tokenization and normalization to sentences. Defaults to `False`. +- **`ignore_punct`** (`boolean`): If `True`, applies basic tokenization and normalization to sentences. Defaults to `False`. +- **`support_zh_ja_chars`** (`boolean`): If `True`, tokenization/normalization supports processing of Chinese characters, as well as Japanese Kanji, Hiragana, Katakana, and Phonetic Extensions of Katakana. Only applies if `normalized = True`. Defaults to `False`. +- **`case_sensitive`** (`boolean`): If `False`, makes all predictions and references lowercase to ignore differences in case. Defaults to `False`. + +### Output Values +This metric returns the following: +- **`score`** (`float`): TER score (num_edits / sum_ref_lengths * 100) +- **`num_edits`** (`int`): The cumulative number of edits +- **`ref_length`** (`float`): The cumulative average reference length + +The output takes the following form: +```python +{'score': ter_score, 'num_edits': num_edits, 'ref_length': ref_length} +``` + +The metric can take on any value `0` and above. `0` is a perfect score, meaning the predictions exactly match the references and no edits were necessary. Higher scores are worse. Scores above 100 mean that the cumulative number of edits, `num_edits`, is higher than the cumulative length of the references, `ref_length`. + +#### Values from Popular Papers + + +### Examples +Basic example with only predictions and references as inputs: +```python +>>> predictions = ["does this sentence match??", +... "what about this sentence?"] +>>> references = [["does this sentence match", "does this sentence match!?!"], +... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]] +>>> ter = evaluate.load("ter") +>>> results = ter.compute(predictions=predictions, +... references=references, +... case_sensitive=True) +>>> print(results) +{'score': 62.5, 'num_edits': 5, 'ref_length': 8.0} +``` + +Example with `normalization = True`: +```python +>>> predictions = ["does this sentence match??", +... "what about this sentence?"] +>>> references = [["does this sentence match", "does this sentence match!?!"], +... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]] +>>> ter = evaluate.load("ter") +>>> results = ter.compute(predictions=predictions, +... references=references, +... normalized=True, +... case_sensitive=True) +>>> print(results) +{'score': 57.14285714285714, 'num_edits': 6, 'ref_length': 10.5} +``` + +Example ignoring punctuation and capitalization, and everything matches: +```python +>>> predictions = ["does this sentence match??", +... "what about this sentence?"] +>>> references = [["does this sentence match", "does this sentence match!?!"], +... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]] +>>> ter = evaluate.load("ter") +>>> results = ter.compute(predictions=predictions, +... references=references, +... ignore_punct=True, +... case_sensitive=False) +>>> print(results) +{'score': 0.0, 'num_edits': 0, 'ref_length': 8.0} +``` + +Example ignoring punctuation and capitalization, but with an extra (incorrect) sample: +```python +>>> predictions = ["does this sentence match??", +... "what about this sentence?", +... "What did the TER metric user say to the developer?"] +>>> references = [["does this sentence match", "does this sentence match!?!"], +... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"], +... ["Your jokes are...", "...TERrible"]] +>>> ter = evaluate.load("ter") +>>> results = ter.compute(predictions=predictions, +... references=references, +... ignore_punct=True, +... case_sensitive=False) +>>> print(results) +{'score': 100.0, 'num_edits': 10, 'ref_length': 10.0} +``` + + +## Limitations and Bias + + +## Citation +```bibtex +@inproceedings{snover-etal-2006-study, + title = "A Study of Translation Edit Rate with Targeted Human Annotation", + author = "Snover, Matthew and + Dorr, Bonnie and + Schwartz, Rich and + Micciulla, Linnea and + Makhoul, John", + booktitle = "Proceedings of the 7th Conference of the Association for Machine Translation in the Americas: Technical Papers", + month = aug # " 8-12", + year = "2006", + address = "Cambridge, Massachusetts, USA", + publisher = "Association for Machine Translation in the Americas", + url = "https://aclanthology.org/2006.amta-papers.25", + pages = "223--231", +} +@inproceedings{post-2018-call, + title = "A Call for Clarity in Reporting {BLEU} Scores", + author = "Post, Matt", + booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", + month = oct, + year = "2018", + address = "Belgium, Brussels", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/W18-6319", + pages = "186--191", +} +``` + +## Further References +- See [the sacreBLEU github repo](https://github.com/mjpost/sacreBLEU#ter) for more information. diff --git a/TextClassification/TextClassification/evaluate/metrics/ter/app.py b/TextClassification/TextClassification/evaluate/metrics/ter/app.py new file mode 100644 index 0000000000000000000000000000000000000000..063222ebb9814a282c1b479b84e58adc8bd3bf0e --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/ter/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("ter") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/ter/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/ter/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..9d3e3c80b77038f530d173d82e6419d76b5e72f0 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/ter/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +sacrebleu \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/ter/ter.py b/TextClassification/TextClassification/evaluate/metrics/ter/ter.py new file mode 100644 index 0000000000000000000000000000000000000000..4adb9986cc4c5cd08fe03246d4dc8f54a31e0839 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/ter/ter.py @@ -0,0 +1,212 @@ +# Copyright 2021 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TER metric as available in sacrebleu. """ +import datasets +import sacrebleu as scb +from packaging import version +from sacrebleu import TER + +import evaluate + + +_CITATION = """\ +@inproceedings{snover-etal-2006-study, + title = "A Study of Translation Edit Rate with Targeted Human Annotation", + author = "Snover, Matthew and + Dorr, Bonnie and + Schwartz, Rich and + Micciulla, Linnea and + Makhoul, John", + booktitle = "Proceedings of the 7th Conference of the Association for Machine Translation in the Americas: Technical Papers", + month = aug # " 8-12", + year = "2006", + address = "Cambridge, Massachusetts, USA", + publisher = "Association for Machine Translation in the Americas", + url = "https://aclanthology.org/2006.amta-papers.25", + pages = "223--231", +} +@inproceedings{post-2018-call, + title = "A Call for Clarity in Reporting {BLEU} Scores", + author = "Post, Matt", + booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", + month = oct, + year = "2018", + address = "Belgium, Brussels", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/W18-6319", + pages = "186--191", +} +""" + +_DESCRIPTION = """\ +TER (Translation Edit Rate, also called Translation Error Rate) is a metric to quantify the edit operations that a +hypothesis requires to match a reference translation. We use the implementation that is already present in sacrebleu +(https://github.com/mjpost/sacreBLEU#ter), which in turn is inspired by the TERCOM implementation, which can be found +here: https://github.com/jhclark/tercom. + +The implementation here is slightly different from sacrebleu in terms of the required input format. The length of +the references and hypotheses lists need to be the same, so you may need to transpose your references compared to +sacrebleu's required input format. See https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534 + +See the README.md file at https://github.com/mjpost/sacreBLEU#ter for more information. +""" + +_KWARGS_DESCRIPTION = """ +Produces TER scores alongside the number of edits and reference length. + +Args: + predictions (list of str): The system stream (a sequence of segments). + references (list of list of str): A list of one or more reference streams (each a sequence of segments). + normalized (boolean): If `True`, applies basic tokenization and normalization to sentences. Defaults to `False`. + ignore_punct (boolean): If `True`, applies basic tokenization and normalization to sentences. Defaults to `False`. + support_zh_ja_chars (boolean): If `True`, tokenization/normalization supports processing of Chinese characters, + as well as Japanese Kanji, Hiragana, Katakana, and Phonetic Extensions of Katakana. + Only applies if `normalized = True`. Defaults to `False`. + case_sensitive (boolean): If `False`, makes all predictions and references lowercase to ignore differences in case. Defaults to `False`. + +Returns: + 'score' (float): TER score (num_edits / sum_ref_lengths * 100) + 'num_edits' (int): The cumulative number of edits + 'ref_length' (float): The cumulative average reference length + +Examples: + Example 1: + >>> predictions = ["does this sentence match??", + ... "what about this sentence?", + ... "What did the TER metric user say to the developer?"] + >>> references = [["does this sentence match", "does this sentence match!?!"], + ... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"], + ... ["Your jokes are...", "...TERrible"]] + >>> ter = evaluate.load("ter") + >>> results = ter.compute(predictions=predictions, + ... references=references, + ... case_sensitive=True) + >>> print(results) + {'score': 150.0, 'num_edits': 15, 'ref_length': 10.0} + + Example 2: + >>> predictions = ["does this sentence match??", + ... "what about this sentence?"] + >>> references = [["does this sentence match", "does this sentence match!?!"], + ... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]] + >>> ter = evaluate.load("ter") + >>> results = ter.compute(predictions=predictions, + ... references=references, + ... case_sensitive=True) + >>> print(results) + {'score': 62.5, 'num_edits': 5, 'ref_length': 8.0} + + Example 3: + >>> predictions = ["does this sentence match??", + ... "what about this sentence?"] + >>> references = [["does this sentence match", "does this sentence match!?!"], + ... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]] + >>> ter = evaluate.load("ter") + >>> results = ter.compute(predictions=predictions, + ... references=references, + ... normalized=True, + ... case_sensitive=True) + >>> print(results) + {'score': 57.14285714285714, 'num_edits': 6, 'ref_length': 10.5} + + Example 4: + >>> predictions = ["does this sentence match??", + ... "what about this sentence?"] + >>> references = [["does this sentence match", "does this sentence match!?!"], + ... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"]] + >>> ter = evaluate.load("ter") + >>> results = ter.compute(predictions=predictions, + ... references=references, + ... ignore_punct=True, + ... case_sensitive=False) + >>> print(results) + {'score': 0.0, 'num_edits': 0, 'ref_length': 8.0} + + Example 5: + >>> predictions = ["does this sentence match??", + ... "what about this sentence?", + ... "What did the TER metric user say to the developer?"] + >>> references = [["does this sentence match", "does this sentence match!?!"], + ... ["wHaT aBoUt ThIs SeNtEnCe?", "wHaT aBoUt ThIs SeNtEnCe?"], + ... ["Your jokes are...", "...TERrible"]] + >>> ter = evaluate.load("ter") + >>> results = ter.compute(predictions=predictions, + ... references=references, + ... ignore_punct=True, + ... case_sensitive=False) + >>> print(results) + {'score': 100.0, 'num_edits': 10, 'ref_length': 10.0} +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Ter(evaluate.Metric): + def _info(self): + if version.parse(scb.__version__) < version.parse("1.4.12"): + raise ImportWarning( + "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n" + 'You can install it with `pip install "sacrebleu>=1.4.12"`.' + ) + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + homepage="http://www.cs.umd.edu/~snover/tercom/", + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), + } + ), + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + ], + codebase_urls=["https://github.com/mjpost/sacreBLEU#ter"], + reference_urls=[ + "https://github.com/jhclark/tercom", + ], + ) + + def _compute( + self, + predictions, + references, + normalized: bool = False, + ignore_punct: bool = False, + support_zh_ja_chars: bool = False, + case_sensitive: bool = False, + ): + # if only one reference is provided make sure we still use list of lists + if isinstance(references[0], str): + references = [[ref] for ref in references] + + references_per_prediction = len(references[0]) + if any(len(refs) != references_per_prediction for refs in references): + raise ValueError("Sacrebleu requires the same number of references for each prediction") + transformed_references = [[refs[i] for refs in references] for i in range(references_per_prediction)] + + sb_ter = TER( + normalized=normalized, + no_punct=ignore_punct, + asian_support=support_zh_ja_chars, + case_sensitive=case_sensitive, + ) + output = sb_ter.corpus_score(predictions, transformed_references) + + return {"score": output.score, "num_edits": output.num_edits, "ref_length": output.ref_length} diff --git a/TextClassification/TextClassification/evaluate/metrics/trec_eval/README.md b/TextClassification/TextClassification/evaluate/metrics/trec_eval/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7512eb55ee41469e83db4a637bc9c61b6afe4e41 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/trec_eval/README.md @@ -0,0 +1,149 @@ +--- +title: TREC Eval +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + The TREC Eval metric combines a number of information retrieval metrics such as precision and nDCG. It is used to score rankings of retrieved documents with reference values. +--- + +# Metric Card for TREC Eval + +## Metric Description + +The TREC Eval metric combines a number of information retrieval metrics such as precision and normalized Discounted Cumulative Gain (nDCG). It is used to score rankings of retrieved documents with reference values. + +## How to Use +```Python +from evaluate import load +trec_eval = load("trec_eval") +results = trec_eval.compute(predictions=[run], references=[qrel]) +``` + +### Inputs +- **predictions** *(dict): a single retrieval run.* + - **query** *(int): Query ID.* + - **q0** *(str): Literal `"q0"`.* + - **docid** *(str): Document ID.* + - **rank** *(int): Rank of document.* + - **score** *(float): Score of document.* + - **system** *(str): Tag for current run.* +- **references** *(dict): a single qrel.* + - **query** *(int): Query ID.* + - **q0** *(str): Literal `"q0"`.* + - **docid** *(str): Document ID.* + - **rel** *(int): Relevance of document.* + +### Output Values +- **runid** *(str): Run name.* +- **num_ret** *(int): Number of retrieved documents.* +- **num_rel** *(int): Number of relevant documents.* +- **num_rel_ret** *(int): Number of retrieved relevant documents.* +- **num_q** *(int): Number of queries.* +- **map** *(float): Mean average precision.* +- **gm_map** *(float): geometric mean average precision.* +- **bpref** *(float): binary preference score.* +- **Rprec** *(float): precision@R, where R is number of relevant documents.* +- **recip_rank** *(float): reciprocal rank* +- **P@k** *(float): precision@k (k in [5, 10, 15, 20, 30, 100, 200, 500, 1000]).* +- **NDCG@k** *(float): nDCG@k (k in [5, 10, 15, 20, 30, 100, 200, 500, 1000]).* + +### Examples + +A minimal example of looks as follows: +```Python +qrel = { + "query": [0], + "q0": ["q0"], + "docid": ["doc_1"], + "rel": [2] +} +run = { + "query": [0, 0], + "q0": ["q0", "q0"], + "docid": ["doc_2", "doc_1"], + "rank": [0, 1], + "score": [1.5, 1.2], + "system": ["test", "test"] +} + +trec_eval = evaluate.load("trec_eval") +results = trec_eval.compute(references=[qrel], predictions=[run]) +results["P@5"] +0.2 +``` + +A more realistic use case with an examples from [`trectools`](https://github.com/joaopalotti/trectools): + +```python +qrel = pd.read_csv("robust03_qrels.txt", sep="\s+", names=["query", "q0", "docid", "rel"]) +qrel["q0"] = qrel["q0"].astype(str) +qrel = qrel.to_dict(orient="list") + +run = pd.read_csv("input.InexpC2", sep="\s+", names=["query", "q0", "docid", "rank", "score", "system"]) +run = run.to_dict(orient="list") + +trec_eval = evaluate.load("trec_eval") +result = trec_eval.compute(run=[run], qrel=[qrel]) +``` + +```python +result + +{'runid': 'InexpC2', + 'num_ret': 100000, + 'num_rel': 6074, + 'num_rel_ret': 3198, + 'num_q': 100, + 'map': 0.22485930431817494, + 'gm_map': 0.10411523825735523, + 'bpref': 0.217511695914079, + 'Rprec': 0.2502547201167236, + 'recip_rank': 0.6646545943335417, + 'P@5': 0.44, + 'P@10': 0.37, + 'P@15': 0.34600000000000003, + 'P@20': 0.30999999999999994, + 'P@30': 0.2563333333333333, + 'P@100': 0.1428, + 'P@200': 0.09510000000000002, + 'P@500': 0.05242, + 'P@1000': 0.03198, + 'NDCG@5': 0.4101480395089769, + 'NDCG@10': 0.3806761417784469, + 'NDCG@15': 0.37819463408955706, + 'NDCG@20': 0.3686080836061317, + 'NDCG@30': 0.352474353427451, + 'NDCG@100': 0.3778329431025776, + 'NDCG@200': 0.4119129817248979, + 'NDCG@500': 0.4585354576461375, + 'NDCG@1000': 0.49092149290805653} +``` + +## Limitations and Bias +The `trec_eval` metric requires the inputs to be in the TREC run and qrel formats for predictions and references. + + +## Citation + +```bibtex +@inproceedings{palotti2019, + author = {Palotti, Joao and Scells, Harrisen and Zuccon, Guido}, + title = {TrecTools: an open-source Python library for Information Retrieval practitioners involved in TREC-like campaigns}, + series = {SIGIR'19}, + year = {2019}, + location = {Paris, France}, + publisher = {ACM} +} +``` + +## Further References + +- Homepage: https://github.com/joaopalotti/trectools \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/trec_eval/app.py b/TextClassification/TextClassification/evaluate/metrics/trec_eval/app.py new file mode 100644 index 0000000000000000000000000000000000000000..a15c15c2439a4a063da49e3c32745574ada77fea --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/trec_eval/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("trec_eval") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/trec_eval/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/trec_eval/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..8a8124560867182142d4cb3c0d94b0646105080c --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/trec_eval/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +trectools \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/trec_eval/trec_eval.py b/TextClassification/TextClassification/evaluate/metrics/trec_eval/trec_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..462374e01a1972146dc87c0fa70864711046ee2e --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/trec_eval/trec_eval.py @@ -0,0 +1,139 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module to compute TREC evaluation scores.""" + +import datasets +import pandas as pd +from trectools import TrecEval, TrecQrel, TrecRun + +import evaluate + + +_CITATION = """\ +@inproceedings{palotti2019, + author = {Palotti, Joao and Scells, Harrisen and Zuccon, Guido}, + title = {TrecTools: an open-source Python library for Information Retrieval practitioners involved in TREC-like campaigns}, + series = {SIGIR'19}, + year = {2019}, + location = {Paris, France}, + publisher = {ACM} +} +""" + +# TODO: Add description of the module here +_DESCRIPTION = """\ +The TREC Eval metric combines a number of information retrieval metrics such as \ +precision and nDCG. It is used to score rankings of retrieved documents with reference values.""" + + +# TODO: Add description of the arguments of the module here +_KWARGS_DESCRIPTION = """ +Calculates TREC evaluation scores based on a run and qrel. +Args: + predictions: list containing a single run. + references: list containing a single qrel. +Returns: + dict: TREC evaluation scores. +Examples: + >>> trec = evaluate.load("trec_eval") + >>> qrel = { + ... "query": [0], + ... "q0": ["0"], + ... "docid": ["doc_1"], + ... "rel": [2] + ... } + >>> run = { + ... "query": [0, 0], + ... "q0": ["q0", "q0"], + ... "docid": ["doc_2", "doc_1"], + ... "rank": [0, 1], + ... "score": [1.5, 1.2], + ... "system": ["test", "test"] + ... } + >>> results = trec.compute(references=[qrel], predictions=[run]) + >>> print(results["P@5"]) + 0.2 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class TRECEval(evaluate.Metric): + """Compute TREC evaluation scores.""" + + def _info(self): + return evaluate.MetricInfo( + module_type="metric", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": { + "query": datasets.Sequence(datasets.Value("int64")), + "q0": datasets.Sequence(datasets.Value("string")), + "docid": datasets.Sequence(datasets.Value("string")), + "rank": datasets.Sequence(datasets.Value("int64")), + "score": datasets.Sequence(datasets.Value("float")), + "system": datasets.Sequence(datasets.Value("string")), + }, + "references": { + "query": datasets.Sequence(datasets.Value("int64")), + "q0": datasets.Sequence(datasets.Value("string")), + "docid": datasets.Sequence(datasets.Value("string")), + "rel": datasets.Sequence(datasets.Value("int64")), + }, + } + ), + homepage="https://github.com/joaopalotti/trectools", + ) + + def _compute(self, references, predictions): + """Returns the TREC evaluation scores.""" + + if len(predictions) > 1 or len(references) > 1: + raise ValueError( + f"You can only pass one prediction and reference per evaluation. You passed {len(predictions)} prediction(s) and {len(references)} reference(s)." + ) + + df_run = pd.DataFrame(predictions[0]) + df_qrel = pd.DataFrame(references[0]) + + trec_run = TrecRun() + trec_run.filename = "placeholder.file" + trec_run.run_data = df_run + + trec_qrel = TrecQrel() + trec_qrel.filename = "placeholder.file" + trec_qrel.qrels_data = df_qrel + + trec_eval = TrecEval(trec_run, trec_qrel) + + result = {} + result["runid"] = trec_eval.run.get_runid() + result["num_ret"] = trec_eval.get_retrieved_documents(per_query=False) + result["num_rel"] = trec_eval.get_relevant_documents(per_query=False) + result["num_rel_ret"] = trec_eval.get_relevant_retrieved_documents(per_query=False) + result["num_q"] = len(trec_eval.run.topics()) + result["map"] = trec_eval.get_map(depth=10000, per_query=False, trec_eval=True) + result["gm_map"] = trec_eval.get_geometric_map(depth=10000, trec_eval=True) + result["bpref"] = trec_eval.get_bpref(depth=1000, per_query=False, trec_eval=True) + result["Rprec"] = trec_eval.get_rprec(depth=1000, per_query=False, trec_eval=True) + result["recip_rank"] = trec_eval.get_reciprocal_rank(depth=1000, per_query=False, trec_eval=True) + + for v in [5, 10, 15, 20, 30, 100, 200, 500, 1000]: + result[f"P@{v}"] = trec_eval.get_precision(depth=v, per_query=False, trec_eval=True) + for v in [5, 10, 15, 20, 30, 100, 200, 500, 1000]: + result[f"NDCG@{v}"] = trec_eval.get_ndcg(depth=v, per_query=False, trec_eval=True) + + return result diff --git a/TextClassification/TextClassification/evaluate/metrics/wer/README.md b/TextClassification/TextClassification/evaluate/metrics/wer/README.md new file mode 100644 index 0000000000000000000000000000000000000000..21f83429fce2e58cf36c210d057de761640b2281 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/wer/README.md @@ -0,0 +1,158 @@ +--- +title: WER +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + Word error rate (WER) is a common metric of the performance of an automatic speech recognition system. + + The general difficulty of measuring performance lies in the fact that the recognized word sequence can have a different length from the reference word sequence (supposedly the correct one). The WER is derived from the Levenshtein distance, working at the word level instead of the phoneme level. The WER is a valuable tool for comparing different systems as well as for evaluating improvements within one system. This kind of measurement, however, provides no details on the nature of translation errors and further work is therefore required to identify the main source(s) of error and to focus any research effort. + + This problem is solved by first aligning the recognized word sequence with the reference (spoken) word sequence using dynamic string alignment. Examination of this issue is seen through a theory called the power law that states the correlation between perplexity and word error rate. + + Word error rate can then be computed as: + + WER = (S + D + I) / N = (S + D + I) / (S + D + C) + + where + + S is the number of substitutions, + D is the number of deletions, + I is the number of insertions, + C is the number of correct words, + N is the number of words in the reference (N=S+D+C). + + This value indicates the average number of errors per reference word. The lower the value, the better the + performance of the ASR system with a WER of 0 being a perfect score. +--- + +# Metric Card for WER + +## Metric description +Word error rate (WER) is a common metric of the performance of an automatic speech recognition (ASR) system. + +The general difficulty of measuring the performance of ASR systems lies in the fact that the recognized word sequence can have a different length from the reference word sequence (supposedly the correct one). The WER is derived from the [Levenshtein distance](https://en.wikipedia.org/wiki/Levenshtein_distance), working at the word level. + +This problem is solved by first aligning the recognized word sequence with the reference (spoken) word sequence using dynamic string alignment. Examination of this issue is seen through a theory called the power law that states the correlation between [perplexity](https://huggingface.co/metrics/perplexity) and word error rate (see [this article](https://www.cs.cmu.edu/~roni/papers/eval-metrics-bntuw-9802.pdf) for further information). + +Word error rate can then be computed as: + +`WER = (S + D + I) / N = (S + D + I) / (S + D + C)` + +where + +`S` is the number of substitutions, + +`D` is the number of deletions, + +`I` is the number of insertions, + +`C` is the number of correct words, + +`N` is the number of words in the reference (`N=S+D+C`). + + +## How to use + +The metric takes two inputs: references (a list of references for each speech input) and predictions (a list of transcriptions to score). + + +```python +from evaluate import load +wer = load("wer") +wer_score = wer.compute(predictions=predictions, references=references) +``` +## Output values + +This metric outputs a float representing the word error rate. + +``` +print(wer_score) +0.5 +``` + +This value indicates the average number of errors per reference word. + +The **lower** the value, the **better** the performance of the ASR system, with a WER of 0 being a perfect score. + +### Values from popular papers + +This metric is highly dependent on the content and quality of the dataset, and therefore users can expect very different values for the same model but on different datasets. + +For example, datasets such as [LibriSpeech](https://huggingface.co/datasets/librispeech_asr) report a WER in the 1.8-3.3 range, whereas ASR models evaluated on [Timit](https://huggingface.co/datasets/timit_asr) report a WER in the 8.3-20.4 range. +See the leaderboards for [LibriSpeech](https://paperswithcode.com/sota/speech-recognition-on-librispeech-test-clean) and [Timit](https://paperswithcode.com/sota/speech-recognition-on-timit) for the most recent values. + +## Examples + +Perfect match between prediction and reference: + +```python +from evaluate import load +wer = load("wer") +predictions = ["hello world", "good night moon"] +references = ["hello world", "good night moon"] +wer_score = wer.compute(predictions=predictions, references=references) +print(wer_score) +0.0 +``` + +Partial match between prediction and reference: + +```python +from evaluate import load +wer = load("wer") +predictions = ["this is the prediction", "there is an other sample"] +references = ["this is the reference", "there is another one"] +wer_score = wer.compute(predictions=predictions, references=references) +print(wer_score) +0.5 +``` + +No match between prediction and reference: + +```python +from evaluate import load +wer = load("wer") +predictions = ["hello world", "good night moon"] +references = ["hi everyone", "have a great day"] +wer_score = wer.compute(predictions=predictions, references=references) +print(wer_score) +1.0 +``` + +## Limitations and bias + +WER is a valuable tool for comparing different systems as well as for evaluating improvements within one system. This kind of measurement, however, provides no details on the nature of translation errors and further work is therefore required to identify the main source(s) of error and to focus any research effort. + +## Citation + +```bibtex +@inproceedings{woodard1982, +author = {Woodard, J.P. and Nelson, J.T., +year = {1982}, +journal = {Workshop on standardisation for speech I/O technology, Naval Air Development Center, Warminster, PA}, +title = {An information theoretic measure of speech recognition performance} +} +``` + +```bibtex +@inproceedings{morris2004, +author = {Morris, Andrew and Maier, Viktoria and Green, Phil}, +year = {2004}, +month = {01}, +pages = {}, +title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.} +} +``` + +## Further References + +- [Word Error Rate -- Wikipedia](https://en.wikipedia.org/wiki/Word_error_rate) +- [Hugging Face Tasks -- Automatic Speech Recognition](https://huggingface.co/tasks/automatic-speech-recognition) diff --git a/TextClassification/TextClassification/evaluate/metrics/wer/app.py b/TextClassification/TextClassification/evaluate/metrics/wer/app.py new file mode 100644 index 0000000000000000000000000000000000000000..30b8fe447f13fd199a4d6388fb689450975db7b7 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/wer/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("wer") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/wer/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/wer/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..aeeda9ad86ac3257a56a69d358c7d863236ac87f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/wer/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +jiwer \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/wer/wer.py b/TextClassification/TextClassification/evaluate/metrics/wer/wer.py new file mode 100644 index 0000000000000000000000000000000000000000..214d5b22e2afbe6bf7d78747dd3f00b9c76b8a3e --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/wer/wer.py @@ -0,0 +1,106 @@ +# Copyright 2021 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Word Error Ratio (WER) metric. """ + +import datasets +from jiwer import compute_measures + +import evaluate + + +_CITATION = """\ +@inproceedings{inproceedings, + author = {Morris, Andrew and Maier, Viktoria and Green, Phil}, + year = {2004}, + month = {01}, + pages = {}, + title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.} +} +""" + +_DESCRIPTION = """\ +Word error rate (WER) is a common metric of the performance of an automatic speech recognition system. + +The general difficulty of measuring performance lies in the fact that the recognized word sequence can have a different length from the reference word sequence (supposedly the correct one). The WER is derived from the Levenshtein distance, working at the word level instead of the phoneme level. The WER is a valuable tool for comparing different systems as well as for evaluating improvements within one system. This kind of measurement, however, provides no details on the nature of translation errors and further work is therefore required to identify the main source(s) of error and to focus any research effort. + +This problem is solved by first aligning the recognized word sequence with the reference (spoken) word sequence using dynamic string alignment. Examination of this issue is seen through a theory called the power law that states the correlation between perplexity and word error rate. + +Word error rate can then be computed as: + +WER = (S + D + I) / N = (S + D + I) / (S + D + C) + +where + +S is the number of substitutions, +D is the number of deletions, +I is the number of insertions, +C is the number of correct words, +N is the number of words in the reference (N=S+D+C). + +This value indicates the average number of errors per reference word. The lower the value, the better the +performance of the ASR system with a WER of 0 being a perfect score. +""" + +_KWARGS_DESCRIPTION = """ +Compute WER score of transcribed segments against references. + +Args: + references: List of references for each speech input. + predictions: List of transcriptions to score. + concatenate_texts (bool, default=False): Whether to concatenate all input texts or compute WER iteratively. + +Returns: + (float): the word error rate + +Examples: + + >>> predictions = ["this is the prediction", "there is an other sample"] + >>> references = ["this is the reference", "there is another one"] + >>> wer = evaluate.load("wer") + >>> wer_score = wer.compute(predictions=predictions, references=references) + >>> print(wer_score) + 0.5 +""" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class WER(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + codebase_urls=["https://github.com/jitsi/jiwer/"], + reference_urls=[ + "https://en.wikipedia.org/wiki/Word_error_rate", + ], + ) + + def _compute(self, predictions=None, references=None, concatenate_texts=False): + if concatenate_texts: + return compute_measures(references, predictions)["wer"] + else: + incorrect = 0 + total = 0 + for prediction, reference in zip(predictions, references): + measures = compute_measures(reference, prediction) + incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"] + total += measures["substitutions"] + measures["deletions"] + measures["hits"] + return incorrect / total diff --git a/TextClassification/TextClassification/evaluate/metrics/wiki_split/README.md b/TextClassification/TextClassification/evaluate/metrics/wiki_split/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6736bcc7d4b33c4772d4243a9fa05986598f4bb1 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/wiki_split/README.md @@ -0,0 +1,120 @@ +--- +title: WikiSplit +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + WIKI_SPLIT is the combination of three metrics SARI, EXACT and SACREBLEU + It can be used to evaluate the quality of machine-generated texts. +--- + +# Metric Card for WikiSplit + +## Metric description + +WikiSplit is the combination of three metrics: [SARI](https://huggingface.co/metrics/sari), [exact match](https://huggingface.co/metrics/exact_match) and [SacreBLEU](https://huggingface.co/metrics/sacrebleu). + +It can be used to evaluate the quality of sentence splitting approaches, which require rewriting a long sentence into two or more coherent short sentences, e.g. based on the [WikiSplit dataset](https://huggingface.co/datasets/wiki_split). + +## How to use + +The WIKI_SPLIT metric takes three inputs: + +`sources`: a list of source sentences, where each sentence should be a string. + +`predictions`: a list of predicted sentences, where each sentence should be a string. + +`references`: a list of lists of reference sentences, where each sentence should be a string. + +```python +>>> wiki_split = evaluate.load("wiki_split") +>>> sources = ["About 95 species are currently accepted ."] +>>> predictions = ["About 95 you now get in ."] +>>> references= [["About 95 species are currently known ."]] +>>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references) +``` +## Output values + +This metric outputs a dictionary containing three scores: + +`sari`: the [SARI](https://huggingface.co/metrics/sari) score, whose range is between `0.0` and `100.0` -- the higher the value, the better the performance of the model being evaluated, with a SARI of 100 being a perfect score. + +`sacrebleu`: the [SacreBLEU](https://huggingface.co/metrics/sacrebleu) score, which can take any value between `0.0` and `100.0`, inclusive. + +`exact`: the [exact match](https://huggingface.co/metrics/exact_match) score, which represents the sum of all of the individual exact match scores in the set, divided by the total number of predictions in the set. It ranges from `0.0` to `100`, inclusive. Here, `0.0` means no prediction/reference pairs were matches, while `100.0` means they all were. + +```python +>>> print(results) +{'sari': 21.805555555555557, 'sacrebleu': 14.535768424205482, 'exact': 0.0} +``` + +### Values from popular papers + +This metric was initially used by [Rothe et al.(2020)](https://arxiv.org/pdf/1907.12461.pdf) to evaluate the performance of different split-and-rephrase approaches on the [WikiSplit dataset](https://huggingface.co/datasets/wiki_split). They reported a SARI score of 63.5, a SacreBLEU score of 77.2, and an EXACT_MATCH score of 16.3. + +## Examples + +Perfect match between prediction and reference: + +```python +>>> wiki_split = evaluate.load("wiki_split") +>>> sources = ["About 95 species are currently accepted ."] +>>> predictions = ["About 95 species are currently accepted ."] +>>> references= [["About 95 species are currently accepted ."]] +>>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references) +>>> print(results) +{'sari': 100.0, 'sacrebleu': 100.00000000000004, 'exact': 100.0 +``` + +Partial match between prediction and reference: + +```python +>>> wiki_split = evaluate.load("wiki_split") +>>> sources = ["About 95 species are currently accepted ."] +>>> predictions = ["About 95 you now get in ."] +>>> references= [["About 95 species are currently known ."]] +>>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references) +>>> print(results) +{'sari': 21.805555555555557, 'sacrebleu': 14.535768424205482, 'exact': 0.0} +``` + +No match between prediction and reference: + +```python +>>> wiki_split = evaluate.load("wiki_split") +>>> sources = ["About 95 species are currently accepted ."] +>>> predictions = ["Hello world ."] +>>> references= [["About 95 species are currently known ."]] +>>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references) +>>> print(results) +{'sari': 14.047619047619046, 'sacrebleu': 0.0, 'exact': 0.0} +``` +## Limitations and bias + +This metric is not the official metric to evaluate models on the [WikiSplit dataset](https://huggingface.co/datasets/wiki_split). It was initially proposed by [Rothe et al.(2020)](https://arxiv.org/pdf/1907.12461.pdf), whereas the [original paper introducing the WikiSplit dataset (2018)](https://aclanthology.org/D18-1080.pdf) uses different metrics to evaluate performance, such as corpus-level [BLEU](https://huggingface.co/metrics/bleu) and sentence-level BLEU. + +## Citation + +```bibtex +@article{rothe2020leveraging, + title={Leveraging pre-trained checkpoints for sequence generation tasks}, + author={Rothe, Sascha and Narayan, Shashi and Severyn, Aliaksei}, + journal={Transactions of the Association for Computational Linguistics}, + volume={8}, + pages={264--280}, + year={2020}, + publisher={MIT Press} +} +``` + +## Further References + +- [WikiSplit dataset](https://huggingface.co/datasets/wiki_split) +- [WikiSplit paper (Botha et al., 2018)](https://aclanthology.org/D18-1080.pdf) diff --git a/TextClassification/TextClassification/evaluate/metrics/wiki_split/app.py b/TextClassification/TextClassification/evaluate/metrics/wiki_split/app.py new file mode 100644 index 0000000000000000000000000000000000000000..6e077faacdec6e69fa5d9c696350847ccce311c4 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/wiki_split/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("wiki_split") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/wiki_split/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/wiki_split/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7a85cf386b67ac245901a4fe6a22bf7edf6143b1 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/wiki_split/requirements.txt @@ -0,0 +1,3 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +sacrebleu +sacremoses \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/wiki_split/wiki_split.py b/TextClassification/TextClassification/evaluate/metrics/wiki_split/wiki_split.py new file mode 100644 index 0000000000000000000000000000000000000000..be83681b741e8767b58ae3924aaf51b78f8e314b --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/wiki_split/wiki_split.py @@ -0,0 +1,366 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" WIKI_SPLIT metric.""" + +import re +import string +from collections import Counter + +import datasets +import sacrebleu +import sacremoses +from packaging import version + +import evaluate + + +_CITATION = """ +@inproceedings{xu-etal-2016-optimizing, + title = {Optimizing Statistical Machine Translation for Text Simplification}, + authors={Xu, Wei and Napoles, Courtney and Pavlick, Ellie and Chen, Quanze and Callison-Burch, Chris}, + journal = {Transactions of the Association for Computational Linguistics}, + volume = {4}, + year={2016}, + url = {https://www.aclweb.org/anthology/Q16-1029}, + pages = {401--415 +}, +@inproceedings{post-2018-call, + title = "A Call for Clarity in Reporting {BLEU} Scores", + author = "Post, Matt", + booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", + month = oct, + year = "2018", + address = "Belgium, Brussels", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/W18-6319", + pages = "186--191", +} +""" + +_DESCRIPTION = """\ +WIKI_SPLIT is the combination of three metrics SARI, EXACT and SACREBLEU +It can be used to evaluate the quality of machine-generated texts. +""" + + +_KWARGS_DESCRIPTION = """ +Calculates sari score (between 0 and 100) given a list of source and predicted +sentences, and a list of lists of reference sentences. It also computes the BLEU score as well as the exact match score. +Args: + sources: list of source sentences where each sentence should be a string. + predictions: list of predicted sentences where each sentence should be a string. + references: list of lists of reference sentences where each sentence should be a string. +Returns: + sari: sari score + sacrebleu: sacrebleu score + exact: exact score + +Examples: + >>> sources=["About 95 species are currently accepted ."] + >>> predictions=["About 95 you now get in ."] + >>> references=[["About 95 species are currently known ."]] + >>> wiki_split = evaluate.load("wiki_split") + >>> results = wiki_split.compute(sources=sources, predictions=predictions, references=references) + >>> print(results) + {'sari': 21.805555555555557, 'sacrebleu': 14.535768424205482, 'exact': 0.0} +""" + + +def normalize_answer(s): + """Lower text and remove punctuation, articles and extra whitespace.""" + + def remove_articles(text): + regex = re.compile(r"\b(a|an|the)\b", re.UNICODE) + return re.sub(regex, " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def compute_exact(a_gold, a_pred): + return int(normalize_answer(a_gold) == normalize_answer(a_pred)) + + +def compute_em(predictions, references): + scores = [any([compute_exact(ref, pred) for ref in refs]) for pred, refs in zip(predictions, references)] + return (sum(scores) / len(scores)) * 100 + + +def SARIngram(sgrams, cgrams, rgramslist, numref): + rgramsall = [rgram for rgrams in rgramslist for rgram in rgrams] + rgramcounter = Counter(rgramsall) + + sgramcounter = Counter(sgrams) + sgramcounter_rep = Counter() + for sgram, scount in sgramcounter.items(): + sgramcounter_rep[sgram] = scount * numref + + cgramcounter = Counter(cgrams) + cgramcounter_rep = Counter() + for cgram, ccount in cgramcounter.items(): + cgramcounter_rep[cgram] = ccount * numref + + # KEEP + keepgramcounter_rep = sgramcounter_rep & cgramcounter_rep + keepgramcountergood_rep = keepgramcounter_rep & rgramcounter + keepgramcounterall_rep = sgramcounter_rep & rgramcounter + + keeptmpscore1 = 0 + keeptmpscore2 = 0 + for keepgram in keepgramcountergood_rep: + keeptmpscore1 += keepgramcountergood_rep[keepgram] / keepgramcounter_rep[keepgram] + # Fix an alleged bug [2] in the keep score computation. + # keeptmpscore2 += keepgramcountergood_rep[keepgram] / keepgramcounterall_rep[keepgram] + keeptmpscore2 += keepgramcountergood_rep[keepgram] + # Define 0/0=1 instead of 0 to give higher scores for predictions that match + # a target exactly. + keepscore_precision = 1 + keepscore_recall = 1 + if len(keepgramcounter_rep) > 0: + keepscore_precision = keeptmpscore1 / len(keepgramcounter_rep) + if len(keepgramcounterall_rep) > 0: + # Fix an alleged bug [2] in the keep score computation. + # keepscore_recall = keeptmpscore2 / len(keepgramcounterall_rep) + keepscore_recall = keeptmpscore2 / sum(keepgramcounterall_rep.values()) + keepscore = 0 + if keepscore_precision > 0 or keepscore_recall > 0: + keepscore = 2 * keepscore_precision * keepscore_recall / (keepscore_precision + keepscore_recall) + + # DELETION + delgramcounter_rep = sgramcounter_rep - cgramcounter_rep + delgramcountergood_rep = delgramcounter_rep - rgramcounter + delgramcounterall_rep = sgramcounter_rep - rgramcounter + deltmpscore1 = 0 + deltmpscore2 = 0 + for delgram in delgramcountergood_rep: + deltmpscore1 += delgramcountergood_rep[delgram] / delgramcounter_rep[delgram] + deltmpscore2 += delgramcountergood_rep[delgram] / delgramcounterall_rep[delgram] + # Define 0/0=1 instead of 0 to give higher scores for predictions that match + # a target exactly. + delscore_precision = 1 + if len(delgramcounter_rep) > 0: + delscore_precision = deltmpscore1 / len(delgramcounter_rep) + + # ADDITION + addgramcounter = set(cgramcounter) - set(sgramcounter) + addgramcountergood = set(addgramcounter) & set(rgramcounter) + addgramcounterall = set(rgramcounter) - set(sgramcounter) + + addtmpscore = 0 + for addgram in addgramcountergood: + addtmpscore += 1 + + # Define 0/0=1 instead of 0 to give higher scores for predictions that match + # a target exactly. + addscore_precision = 1 + addscore_recall = 1 + if len(addgramcounter) > 0: + addscore_precision = addtmpscore / len(addgramcounter) + if len(addgramcounterall) > 0: + addscore_recall = addtmpscore / len(addgramcounterall) + addscore = 0 + if addscore_precision > 0 or addscore_recall > 0: + addscore = 2 * addscore_precision * addscore_recall / (addscore_precision + addscore_recall) + + return (keepscore, delscore_precision, addscore) + + +def SARIsent(ssent, csent, rsents): + numref = len(rsents) + + s1grams = ssent.split(" ") + c1grams = csent.split(" ") + s2grams = [] + c2grams = [] + s3grams = [] + c3grams = [] + s4grams = [] + c4grams = [] + + r1gramslist = [] + r2gramslist = [] + r3gramslist = [] + r4gramslist = [] + for rsent in rsents: + r1grams = rsent.split(" ") + r2grams = [] + r3grams = [] + r4grams = [] + r1gramslist.append(r1grams) + for i in range(0, len(r1grams) - 1): + if i < len(r1grams) - 1: + r2gram = r1grams[i] + " " + r1grams[i + 1] + r2grams.append(r2gram) + if i < len(r1grams) - 2: + r3gram = r1grams[i] + " " + r1grams[i + 1] + " " + r1grams[i + 2] + r3grams.append(r3gram) + if i < len(r1grams) - 3: + r4gram = r1grams[i] + " " + r1grams[i + 1] + " " + r1grams[i + 2] + " " + r1grams[i + 3] + r4grams.append(r4gram) + r2gramslist.append(r2grams) + r3gramslist.append(r3grams) + r4gramslist.append(r4grams) + + for i in range(0, len(s1grams) - 1): + if i < len(s1grams) - 1: + s2gram = s1grams[i] + " " + s1grams[i + 1] + s2grams.append(s2gram) + if i < len(s1grams) - 2: + s3gram = s1grams[i] + " " + s1grams[i + 1] + " " + s1grams[i + 2] + s3grams.append(s3gram) + if i < len(s1grams) - 3: + s4gram = s1grams[i] + " " + s1grams[i + 1] + " " + s1grams[i + 2] + " " + s1grams[i + 3] + s4grams.append(s4gram) + + for i in range(0, len(c1grams) - 1): + if i < len(c1grams) - 1: + c2gram = c1grams[i] + " " + c1grams[i + 1] + c2grams.append(c2gram) + if i < len(c1grams) - 2: + c3gram = c1grams[i] + " " + c1grams[i + 1] + " " + c1grams[i + 2] + c3grams.append(c3gram) + if i < len(c1grams) - 3: + c4gram = c1grams[i] + " " + c1grams[i + 1] + " " + c1grams[i + 2] + " " + c1grams[i + 3] + c4grams.append(c4gram) + + (keep1score, del1score, add1score) = SARIngram(s1grams, c1grams, r1gramslist, numref) + (keep2score, del2score, add2score) = SARIngram(s2grams, c2grams, r2gramslist, numref) + (keep3score, del3score, add3score) = SARIngram(s3grams, c3grams, r3gramslist, numref) + (keep4score, del4score, add4score) = SARIngram(s4grams, c4grams, r4gramslist, numref) + avgkeepscore = sum([keep1score, keep2score, keep3score, keep4score]) / 4 + avgdelscore = sum([del1score, del2score, del3score, del4score]) / 4 + avgaddscore = sum([add1score, add2score, add3score, add4score]) / 4 + finalscore = (avgkeepscore + avgdelscore + avgaddscore) / 3 + return finalscore + + +def normalize(sentence, lowercase: bool = True, tokenizer: str = "13a", return_str: bool = True): + + # Normalization is requried for the ASSET dataset (one of the primary + # datasets in sentence simplification) to allow using space + # to split the sentence. Even though Wiki-Auto and TURK datasets, + # do not require normalization, we do it for consistency. + # Code adapted from the EASSE library [1] written by the authors of the ASSET dataset. + # [1] https://github.com/feralvam/easse/blob/580bba7e1378fc8289c663f864e0487188fe8067/easse/utils/preprocessing.py#L7 + + if lowercase: + sentence = sentence.lower() + + if tokenizer in ["13a", "intl"]: + if version.parse(sacrebleu.__version__).major >= 2: + normalized_sent = sacrebleu.metrics.bleu._get_tokenizer(tokenizer)()(sentence) + else: + normalized_sent = sacrebleu.TOKENIZERS[tokenizer]()(sentence) + elif tokenizer == "moses": + normalized_sent = sacremoses.MosesTokenizer().tokenize(sentence, return_str=True, escape=False) + elif tokenizer == "penn": + normalized_sent = sacremoses.MosesTokenizer().penn_tokenize(sentence, return_str=True) + else: + normalized_sent = sentence + + if not return_str: + normalized_sent = normalized_sent.split() + + return normalized_sent + + +def compute_sari(sources, predictions, references): + + if not (len(sources) == len(predictions) == len(references)): + raise ValueError("Sources length must match predictions and references lengths.") + sari_score = 0 + for src, pred, refs in zip(sources, predictions, references): + sari_score += SARIsent(normalize(src), normalize(pred), [normalize(sent) for sent in refs]) + sari_score = sari_score / len(predictions) + return 100 * sari_score + + +def compute_sacrebleu( + predictions, + references, + smooth_method="exp", + smooth_value=None, + force=False, + lowercase=False, + use_effective_order=False, +): + references_per_prediction = len(references[0]) + if any(len(refs) != references_per_prediction for refs in references): + raise ValueError("Sacrebleu requires the same number of references for each prediction") + transformed_references = [[refs[i] for refs in references] for i in range(references_per_prediction)] + output = sacrebleu.corpus_bleu( + predictions, + transformed_references, + smooth_method=smooth_method, + smooth_value=smooth_value, + force=force, + lowercase=lowercase, + use_effective_order=use_effective_order, + ) + return output.score + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class WikiSplit(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=[ + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), + } + ), + datasets.Features( + { + "predictions": datasets.Value("string", id="sequence"), + "references": datasets.Value("string", id="sequence"), + } + ), + ], + codebase_urls=[ + "https://github.com/huggingface/transformers/blob/master/src/transformers/data/metrics/squad_metrics.py", + "https://github.com/cocoxu/simplification/blob/master/SARI.py", + "https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py", + "https://github.com/mjpost/sacreBLEU", + ], + reference_urls=[ + "https://www.aclweb.org/anthology/Q16-1029.pdf", + "https://github.com/mjpost/sacreBLEU", + "https://en.wikipedia.org/wiki/BLEU", + "https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213", + ], + ) + + def _compute(self, sources, predictions, references): + # if only one reference is provided make sure we still use list of lists + if isinstance(references[0], str): + references = [[ref] for ref in references] + result = {} + result.update({"sari": compute_sari(sources=sources, predictions=predictions, references=references)}) + result.update({"sacrebleu": compute_sacrebleu(predictions=predictions, references=references)}) + result.update({"exact": compute_em(predictions=predictions, references=references)}) + return result diff --git a/TextClassification/TextClassification/evaluate/metrics/xnli/README.md b/TextClassification/TextClassification/evaluate/metrics/xnli/README.md new file mode 100644 index 0000000000000000000000000000000000000000..beb4066f40b440470f67e29ab2f5908820d234eb --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/xnli/README.md @@ -0,0 +1,119 @@ +--- +title: XNLI +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + XNLI is a subset of a few thousand examples from MNLI which has been translated + into a 14 different languages (some low-ish resource). As with MNLI, the goal is + to predict textual entailment (does sentence A imply/contradict/neither sentence + B) and is a classification task (given two sentences, predict one of three + labels). +--- + +# Metric Card for XNLI + +## Metric description + +The XNLI metric allows to evaluate a model's score on the [XNLI dataset](https://huggingface.co/datasets/xnli), which is a subset of a few thousand examples from the [MNLI dataset](https://huggingface.co/datasets/glue/viewer/mnli) that have been translated into a 14 different languages, some of which are relatively low resource such as Swahili and Urdu. + +As with MNLI, the task is to predict textual entailment (does sentence A imply/contradict/neither sentence B) and is a classification task (given two sentences, predict one of three labels). + +## How to use + +The XNLI metric is computed based on the `predictions` (a list of predicted labels) and the `references` (a list of ground truth labels). + +```python +from evaluate import load +xnli_metric = load("xnli") +predictions = [0, 1] +references = [0, 1] +results = xnli_metric.compute(predictions=predictions, references=references) +``` + +## Output values + +The output of the XNLI metric is simply the `accuracy`, i.e. the proportion of correct predictions among the total number of cases processed, with a range between 0 and 1 (see [accuracy](https://huggingface.co/metrics/accuracy) for more information). + +### Values from popular papers +The [original XNLI paper](https://arxiv.org/pdf/1809.05053.pdf) reported accuracies ranging from 59.3 (for `ur`) to 73.7 (for `en`) for the BiLSTM-max model. + +For more recent model performance, see the [dataset leaderboard](https://paperswithcode.com/dataset/xnli). + +## Examples + +Maximal values: + +```python +>>> from evaluate import load +>>> xnli_metric = load("xnli") +>>> predictions = [0, 1] +>>> references = [0, 1] +>>> results = xnli_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'accuracy': 1.0} +``` + +Minimal values: + +```python +>>> from evaluate import load +>>> xnli_metric = load("xnli") +>>> predictions = [1, 0] +>>> references = [0, 1] +>>> results = xnli_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'accuracy': 0.0} +``` + +Partial match: + +```python +>>> from evaluate import load +>>> xnli_metric = load("xnli") +>>> predictions = [1, 0, 1] +>>> references = [1, 0, 0] +>>> results = xnli_metric.compute(predictions=predictions, references=references) +>>> print(results) +{'accuracy': 0.6666666666666666} +``` + +## Limitations and bias + +While accuracy alone does give a certain indication of performance, it can be supplemented by error analysis and a better understanding of the model's mistakes on each of the categories represented in the dataset, especially if they are unbalanced. + +While the XNLI dataset is multilingual and represents a diversity of languages, in reality, cross-lingual sentence understanding goes beyond translation, given that there are many cultural differences that have an impact on human sentiment annotations. Since the XNLI dataset was obtained by translation based on English sentences, it does not capture these cultural differences. + + + +## Citation + +```bibtex +@InProceedings{conneau2018xnli, + author = "Conneau, Alexis + and Rinott, Ruty + and Lample, Guillaume + and Williams, Adina + and Bowman, Samuel R. + and Schwenk, Holger + and Stoyanov, Veselin", + title = "XNLI: Evaluating Cross-lingual Sentence Representations", + booktitle = "Proceedings of the 2018 Conference on Empirical Methods + in Natural Language Processing", + year = "2018", + publisher = "Association for Computational Linguistics", + location = "Brussels, Belgium", +} +``` + +## Further References + +- [XNI Dataset GitHub](https://github.com/facebookresearch/XNLI) +- [HuggingFace Tasks -- Text Classification](https://huggingface.co/tasks/text-classification) diff --git a/TextClassification/TextClassification/evaluate/metrics/xnli/app.py b/TextClassification/TextClassification/evaluate/metrics/xnli/app.py new file mode 100644 index 0000000000000000000000000000000000000000..a44a7ae86b81a933d2b90ffab37d28a11ecc2d99 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/xnli/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("xnli") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/xnli/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/xnli/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a135afacbdd142a40cb9bcfe6073b76cc62d995 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/xnli/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/xnli/xnli.py b/TextClassification/TextClassification/evaluate/metrics/xnli/xnli.py new file mode 100644 index 0000000000000000000000000000000000000000..cc631d55fbe40190f4b72fc98bc66070f151d727 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/xnli/xnli.py @@ -0,0 +1,88 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" XNLI benchmark metric. """ + +import datasets + +import evaluate + + +_CITATION = """\ +@InProceedings{conneau2018xnli, + author = "Conneau, Alexis + and Rinott, Ruty + and Lample, Guillaume + and Williams, Adina + and Bowman, Samuel R. + and Schwenk, Holger + and Stoyanov, Veselin", + title = "XNLI: Evaluating Cross-lingual Sentence Representations", + booktitle = "Proceedings of the 2018 Conference on Empirical Methods + in Natural Language Processing", + year = "2018", + publisher = "Association for Computational Linguistics", + location = "Brussels, Belgium", +} +""" + +_DESCRIPTION = """\ +XNLI is a subset of a few thousand examples from MNLI which has been translated +into a 14 different languages (some low-ish resource). As with MNLI, the goal is +to predict textual entailment (does sentence A imply/contradict/neither sentence +B) and is a classification task (given two sentences, predict one of three +labels). +""" + +_KWARGS_DESCRIPTION = """ +Computes XNLI score which is just simple accuracy. +Args: + predictions: Predicted labels. + references: Ground truth labels. +Returns: + 'accuracy': accuracy +Examples: + + >>> predictions = [0, 1] + >>> references = [0, 1] + >>> xnli_metric = evaluate.load("xnli") + >>> results = xnli_metric.compute(predictions=predictions, references=references) + >>> print(results) + {'accuracy': 1.0} +""" + + +def simple_accuracy(preds, labels): + return (preds == labels).mean() + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class Xnli(evaluate.Metric): + def _info(self): + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + { + "predictions": datasets.Value("int64" if self.config_name != "sts-b" else "float32"), + "references": datasets.Value("int64" if self.config_name != "sts-b" else "float32"), + } + ), + codebase_urls=[], + reference_urls=[], + format="numpy", + ) + + def _compute(self, predictions, references): + return {"accuracy": simple_accuracy(predictions, references)} diff --git a/TextClassification/TextClassification/evaluate/metrics/xtreme_s/README.md b/TextClassification/TextClassification/evaluate/metrics/xtreme_s/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c93cbfe6fc388666fb6ec5308767fbc083219447 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/xtreme_s/README.md @@ -0,0 +1,141 @@ +--- +title: XTREME-S +emoji: 🤗 +colorFrom: blue +colorTo: red +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +tags: +- evaluate +- metric +description: >- + XTREME-S is a benchmark to evaluate universal cross-lingual speech representations in many languages. + XTREME-S covers four task families: speech recognition, classification, speech-to-text translation and retrieval. +--- + +# Metric Card for XTREME-S + + +## Metric Description + +The XTREME-S metric aims to evaluate model performance on the Cross-lingual TRansfer Evaluation of Multilingual Encoders for Speech (XTREME-S) benchmark. + +This benchmark was designed to evaluate speech representations across languages, tasks, domains and data regimes. It covers 102 languages from 10+ language families, 3 different domains and 4 task families: speech recognition, translation, classification and retrieval. + +## How to Use + +There are two steps: (1) loading the XTREME-S metric relevant to the subset of the benchmark being used for evaluation; and (2) calculating the metric. + +1. **Loading the relevant XTREME-S metric** : the subsets of XTREME-S are the following: `mls`, `voxpopuli`, `covost2`, `fleurs-asr`, `fleurs-lang_id`, `minds14` and `babel`. More information about the different subsets can be found on the [XTREME-S benchmark page](https://huggingface.co/datasets/google/xtreme_s). + + +```python +>>> xtreme_s_metric = evaluate.load('xtreme_s', 'mls') +``` + +2. **Calculating the metric**: the metric takes two inputs : + +- `predictions`: a list of predictions to score, with each prediction a `str`. + +- `references`: a list of lists of references for each translation, with each reference a `str`. + +```python +>>> references = ["it is sunny here", "paper and pen are essentials"] +>>> predictions = ["it's sunny", "paper pen are essential"] +>>> results = xtreme_s_metric.compute(predictions=predictions, references=references) +``` + +It also has two optional arguments: + +- `bleu_kwargs`: a `dict` of keywords to be passed when computing the `bleu` metric for the `covost2` subset. Keywords can be one of `smooth_method`, `smooth_value`, `force`, `lowercase`, `tokenize`, `use_effective_order`. + +- `wer_kwargs`: optional dict of keywords to be passed when computing `wer` and `cer`, which are computed for the `mls`, `fleurs-asr`, `voxpopuli`, and `babel` subsets. Keywords are `concatenate_texts`. + +## Output values + +The output of the metric depends on the XTREME-S subset chosen, consisting of a dictionary that contains one or several of the following metrics: + +- `accuracy`: the proportion of correct predictions among the total number of cases processed, with a range between 0 and 1 (see [accuracy](https://huggingface.co/metrics/accuracy) for more information). This is returned for the `fleurs-lang_id` and `minds14` subsets. + +- `f1`: the harmonic mean of the precision and recall (see [F1 score](https://huggingface.co/metrics/f1) for more information). Its range is 0-1 -- its lowest possible value is 0, if either the precision or the recall is 0, and its highest possible value is 1.0, which means perfect precision and recall. It is returned for the `minds14` subset. + +- `wer`: Word error rate (WER) is a common metric of the performance of an automatic speech recognition system. The lower the value, the better the performance of the ASR system, with a WER of 0 being a perfect score (see [WER score](https://huggingface.co/metrics/wer) for more information). It is returned for the `mls`, `fleurs-asr`, `voxpopuli` and `babel` subsets of the benchmark. + +- `cer`: Character error rate (CER) is similar to WER, but operates on character instead of word. The lower the CER value, the better the performance of the ASR system, with a CER of 0 being a perfect score (see [CER score](https://huggingface.co/metrics/cer) for more information). It is returned for the `mls`, `fleurs-asr`, `voxpopuli` and `babel` subsets of the benchmark. + +- `bleu`: the BLEU score, calculated according to the SacreBLEU metric approach. It can take any value between 0.0 and 100.0, inclusive, with higher values being better (see [SacreBLEU](https://huggingface.co/metrics/sacrebleu) for more details). This is returned for the `covost2` subset. + + +### Values from popular papers +The [original XTREME-S paper](https://arxiv.org/pdf/2203.10752.pdf) reported average WERs ranging from 9.2 to 14.6, a BLEU score of 20.6, an accuracy of 73.3 and F1 score of 86.9, depending on the subsets of the dataset tested on. + +## Examples + +For the `mls` subset (which outputs `wer` and `cer`): + +```python +>>> xtreme_s_metric = evaluate.load('xtreme_s', 'mls') +>>> references = ["it is sunny here", "paper and pen are essentials"] +>>> predictions = ["it's sunny", "paper pen are essential"] +>>> results = xtreme_s_metric.compute(predictions=predictions, references=references) +>>> print({k: round(v, 2) for k, v in results.items()}) +{'wer': 0.56, 'cer': 0.27} +``` + +For the `covost2` subset (which outputs `bleu`): + +```python +>>> xtreme_s_metric = evaluate.load('xtreme_s', 'covost2') +>>> references = ["bonjour paris", "il est necessaire de faire du sport de temps en temp"] +>>> predictions = ["bonjour paris", "il est important de faire du sport souvent"] +>>> results = xtreme_s_metric.compute(predictions=predictions, references=references) +>>> print({k: round(v, 2) for k, v in results.items()}) +{'bleu': 31.65} +``` + +For the `fleurs-lang_id` subset (which outputs `accuracy`): + +```python +>>> xtreme_s_metric = evaluate.load('xtreme_s', 'fleurs-lang_id') +>>> references = [0, 1, 0, 0, 1] +>>> predictions = [0, 1, 1, 0, 0] +>>> results = xtreme_s_metric.compute(predictions=predictions, references=references) +>>> print({k: round(v, 2) for k, v in results.items()}) +{'accuracy': 0.6} + ``` + +For the `minds14` subset (which outputs `f1` and `accuracy`): + +```python +>>> xtreme_s_metric = evaluate.load('xtreme_s', 'minds14') +>>> references = [0, 1, 0, 0, 1] +>>> predictions = [0, 1, 1, 0, 0] +>>> results = xtreme_s_metric.compute(predictions=predictions, references=references) +>>> print({k: round(v, 2) for k, v in results.items()}) +{'f1': 0.58, 'accuracy': 0.6} +``` + +## Limitations and bias +This metric works only with datasets that have the same format as the [XTREME-S dataset](https://huggingface.co/datasets/google/xtreme_s). + +While the XTREME-S dataset is meant to represent a variety of languages and tasks, it has inherent biases: it is missing many languages that are important and under-represented in NLP datasets. + +It also has a particular focus on read-speech because common evaluation benchmarks like CoVoST-2 or LibriSpeech evaluate on this type of speech, which results in a mismatch between performance obtained in a read-speech setting and a more noisy setting (in production or live deployment, for instance). + +## Citation + +```bibtex +@article{conneau2022xtreme, + title={XTREME-S: Evaluating Cross-lingual Speech Representations}, + author={Conneau, Alexis and Bapna, Ankur and Zhang, Yu and Ma, Min and von Platen, Patrick and Lozhkov, Anton and Cherry, Colin and Jia, Ye and Rivera, Clara and Kale, Mihir and others}, + journal={arXiv preprint arXiv:2203.10752}, + year={2022} +} +``` + +## Further References + +- [XTREME-S dataset](https://huggingface.co/datasets/google/xtreme_s) +- [XTREME-S github repository](https://github.com/google-research/xtreme) diff --git a/TextClassification/TextClassification/evaluate/metrics/xtreme_s/app.py b/TextClassification/TextClassification/evaluate/metrics/xtreme_s/app.py new file mode 100644 index 0000000000000000000000000000000000000000..e42af6746ed8c80b9b1aefb132e6afcaf8ef8ef2 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/xtreme_s/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("xtreme_s", "mls") +launch_gradio_widget(module) diff --git a/TextClassification/TextClassification/evaluate/metrics/xtreme_s/requirements.txt b/TextClassification/TextClassification/evaluate/metrics/xtreme_s/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..52b0e8aed7c551e2e5d173651d6f25565b6f0f0f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/xtreme_s/requirements.txt @@ -0,0 +1,2 @@ +git+https://github.com/huggingface/evaluate@{COMMIT_PLACEHOLDER} +scikit-learn \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/metrics/xtreme_s/xtreme_s.py b/TextClassification/TextClassification/evaluate/metrics/xtreme_s/xtreme_s.py new file mode 100644 index 0000000000000000000000000000000000000000..b4c052fc50310d1577290f23345e269669d78db1 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/metrics/xtreme_s/xtreme_s.py @@ -0,0 +1,271 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" XTREME-S benchmark metric. """ + +from typing import List + +import datasets +from datasets.config import PY_VERSION +from packaging import version +from sklearn.metrics import f1_score + +import evaluate + + +if PY_VERSION < version.parse("3.8"): + import importlib_metadata +else: + import importlib.metadata as importlib_metadata + + +# TODO(Patrick/Anton) +_CITATION = """\ +""" + +_DESCRIPTION = """\ + XTREME-S is a benchmark to evaluate universal cross-lingual speech representations in many languages. + XTREME-S covers four task families: speech recognition, classification, speech-to-text translation and retrieval. +""" + +_KWARGS_DESCRIPTION = """ +Compute XTREME-S evaluation metric associated to each XTREME-S dataset. +Args: + predictions: list of predictions to score. + Each translation should be tokenized into a list of tokens. + references: list of lists of references for each translation. + Each reference should be tokenized into a list of tokens. + bleu_kwargs: optional dict of keywords to be passed when computing 'bleu'. + Keywords include Dict can be one of 'smooth_method', 'smooth_value', 'force', 'lowercase', + 'tokenize', 'use_effective_order'. + wer_kwargs: optional dict of keywords to be passed when computing 'wer' and 'cer'. + Keywords include 'concatenate_texts'. +Returns: depending on the XTREME-S task, one or several of: + "accuracy": Accuracy - for 'fleurs-lang_id', 'minds14' + "f1": F1 score - for 'minds14' + "wer": Word error rate - for 'mls', 'fleurs-asr', 'voxpopuli', 'babel' + "cer": Character error rate - for 'mls', 'fleurs-asr', 'voxpopuli', 'babel' + "bleu": BLEU score according to the `sacrebleu` metric - for 'covost2' +Examples: + + >>> xtreme_s_metric = evaluate.load('xtreme_s', 'mls') # 'mls', 'voxpopuli', 'fleurs-asr' or 'babel' + >>> references = ["it is sunny here", "paper and pen are essentials"] + >>> predictions = ["it's sunny", "paper pen are essential"] + >>> results = xtreme_s_metric.compute(predictions=predictions, references=references) + >>> print({k: round(v, 2) for k, v in results.items()}) + {'wer': 0.56, 'cer': 0.27} + + >>> xtreme_s_metric = evaluate.load('xtreme_s', 'covost2') + >>> references = ["bonjour paris", "il est necessaire de faire du sport de temps en temp"] + >>> predictions = ["bonjour paris", "il est important de faire du sport souvent"] + >>> results = xtreme_s_metric.compute(predictions=predictions, references=references) + >>> print({k: round(v, 2) for k, v in results.items()}) + {'bleu': 31.65} + + >>> xtreme_s_metric = evaluate.load('xtreme_s', 'fleurs-lang_id') + >>> references = [0, 1, 0, 0, 1] + >>> predictions = [0, 1, 1, 0, 0] + >>> results = xtreme_s_metric.compute(predictions=predictions, references=references) + >>> print({k: round(v, 2) for k, v in results.items()}) + {'accuracy': 0.6} + + >>> xtreme_s_metric = evaluate.load('xtreme_s', 'minds14') + >>> references = [0, 1, 0, 0, 1] + >>> predictions = [0, 1, 1, 0, 0] + >>> results = xtreme_s_metric.compute(predictions=predictions, references=references) + >>> print({k: round(v, 2) for k, v in results.items()}) + {'f1': 0.58, 'accuracy': 0.6} +""" + +_CONFIG_NAMES = ["fleurs-asr", "mls", "voxpopuli", "babel", "covost2", "fleurs-lang_id", "minds14"] +SENTENCE_DELIMITER = "" + +try: + from jiwer import transforms as tr + + _jiwer_available = True +except ImportError: + _jiwer_available = False + +if _jiwer_available and version.parse(importlib_metadata.version("jiwer")) < version.parse("2.3.0"): + + class SentencesToListOfCharacters(tr.AbstractTransform): + def __init__(self, sentence_delimiter: str = " "): + self.sentence_delimiter = sentence_delimiter + + def process_string(self, s: str): + return list(s) + + def process_list(self, inp: List[str]): + chars = [] + for sent_idx, sentence in enumerate(inp): + chars.extend(self.process_string(sentence)) + if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(inp) - 1: + chars.append(self.sentence_delimiter) + return chars + + cer_transform = tr.Compose( + [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)] + ) +elif _jiwer_available: + cer_transform = tr.Compose( + [ + tr.RemoveMultipleSpaces(), + tr.Strip(), + tr.ReduceToSingleSentence(SENTENCE_DELIMITER), + tr.ReduceToListOfListOfChars(), + ] + ) +else: + cer_transform = None + + +def simple_accuracy(preds, labels): + return float((preds == labels).mean()) + + +def f1_and_simple_accuracy(preds, labels): + return { + "f1": float(f1_score(y_true=labels, y_pred=preds, average="macro")), + "accuracy": simple_accuracy(preds, labels), + } + + +def bleu( + preds, + labels, + smooth_method="exp", + smooth_value=None, + force=False, + lowercase=False, + tokenize=None, + use_effective_order=False, +): + # xtreme-s can only have one label + labels = [[label] for label in labels] + preds = list(preds) + try: + import sacrebleu as scb + except ImportError: + raise ValueError( + "sacrebleu has to be installed in order to apply the bleu metric for covost2." + "You can install it via `pip install sacrebleu`." + ) + + if version.parse(scb.__version__) < version.parse("1.4.12"): + raise ImportWarning( + "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n" + 'You can install it with `pip install "sacrebleu>=1.4.12"`.' + ) + + references_per_prediction = len(labels[0]) + if any(len(refs) != references_per_prediction for refs in labels): + raise ValueError("Sacrebleu requires the same number of references for each prediction") + transformed_references = [[refs[i] for refs in labels] for i in range(references_per_prediction)] + output = scb.corpus_bleu( + preds, + transformed_references, + smooth_method=smooth_method, + smooth_value=smooth_value, + force=force, + lowercase=lowercase, + use_effective_order=use_effective_order, + **(dict(tokenize=tokenize) if tokenize else {}), + ) + return {"bleu": output.score} + + +def wer_and_cer(preds, labels, concatenate_texts, config_name): + try: + from jiwer import compute_measures + except ImportError: + raise ValueError( + f"jiwer has to be installed in order to apply the wer metric for {config_name}." + "You can install it via `pip install jiwer`." + ) + + if concatenate_texts: + wer = compute_measures(labels, preds)["wer"] + + cer = compute_measures(labels, preds, truth_transform=cer_transform, hypothesis_transform=cer_transform)["wer"] + return {"wer": wer, "cer": cer} + else: + + def compute_score(preds, labels, score_type="wer"): + incorrect = 0 + total = 0 + for prediction, reference in zip(preds, labels): + if score_type == "wer": + measures = compute_measures(reference, prediction) + elif score_type == "cer": + measures = compute_measures( + reference, prediction, truth_transform=cer_transform, hypothesis_transform=cer_transform + ) + incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"] + total += measures["substitutions"] + measures["deletions"] + measures["hits"] + return incorrect / total + + return {"wer": compute_score(preds, labels, "wer"), "cer": compute_score(preds, labels, "cer")} + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class XtremeS(evaluate.Metric): + def _info(self): + if self.config_name not in _CONFIG_NAMES: + raise KeyError(f"You should supply a configuration name selected in {_CONFIG_NAMES}") + + pred_type = "int64" if self.config_name in ["fleurs-lang_id", "minds14"] else "string" + + return evaluate.MetricInfo( + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + features=datasets.Features( + {"predictions": datasets.Value(pred_type), "references": datasets.Value(pred_type)} + ), + codebase_urls=[], + reference_urls=[], + format="numpy", + ) + + def _compute(self, predictions, references, bleu_kwargs=None, wer_kwargs=None): + + bleu_kwargs = bleu_kwargs if bleu_kwargs is not None else {} + wer_kwargs = wer_kwargs if wer_kwargs is not None else {} + + if self.config_name == "fleurs-lang_id": + return {"accuracy": simple_accuracy(predictions, references)} + elif self.config_name == "minds14": + return f1_and_simple_accuracy(predictions, references) + elif self.config_name == "covost2": + smooth_method = bleu_kwargs.pop("smooth_method", "exp") + smooth_value = bleu_kwargs.pop("smooth_value", None) + force = bleu_kwargs.pop("force", False) + lowercase = bleu_kwargs.pop("lowercase", False) + tokenize = bleu_kwargs.pop("tokenize", None) + use_effective_order = bleu_kwargs.pop("use_effective_order", False) + return bleu( + preds=predictions, + labels=references, + smooth_method=smooth_method, + smooth_value=smooth_value, + force=force, + lowercase=lowercase, + tokenize=tokenize, + use_effective_order=use_effective_order, + ) + elif self.config_name in ["fleurs-asr", "mls", "voxpopuli", "babel"]: + concatenate_texts = wer_kwargs.pop("concatenate_texts", False) + return wer_and_cer(predictions, references, concatenate_texts, self.config_name) + else: + raise KeyError(f"You should supply a configuration name selected in {_CONFIG_NAMES}") diff --git a/TextClassification/TextClassification/evaluate/setup.cfg b/TextClassification/TextClassification/evaluate/setup.cfg new file mode 100644 index 0000000000000000000000000000000000000000..1e2614e9aaa957acf7da9a3586cd7ac6920eae7b --- /dev/null +++ b/TextClassification/TextClassification/evaluate/setup.cfg @@ -0,0 +1,20 @@ +[metadata] +license_file = LICENSE + +[isort] +ensure_newline_before_comments = True +force_grid_wrap = 0 +include_trailing_comma = True +line_length = 119 +lines_after_imports = 2 +multi_line_output = 3 +use_parentheses = True + +[flake8] +ignore = E203, E501, W503 +max-line-length = 119 +exclude = + src/datasets/datasets + src/datasets/metrics +per-file-ignores = + metrics/*:F401 diff --git a/TextClassification/TextClassification/evaluate/setup.py b/TextClassification/TextClassification/evaluate/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..6e47429dde1a2592bccc227429f70dc2f6935e32 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/setup.py @@ -0,0 +1,158 @@ +# Lint as: python3 +""" HuggingFace/Evaluate is an open library for evaluation. + +Note: + + VERSION needs to be formatted following the MAJOR.MINOR.PATCH convention + (we need to follow this convention to be able to retrieve versioned scripts) + +To create the package for pypi. + +1. Open a PR and change the version in: + - __init__.py + - setup.py + Then merge the PR once it's approved. + +3. Add a tag "vVERSION" (e.g. v0.4.1) in git to mark the release : "git tag vVERSION -m 'Add tag vVERSION for pypi'" + Push the tag to remote: git push --tags origin main + Then verify that the 'Python release' CI job runs and succeeds. + +4. Fill release notes in the tag in github once everything is looking hunky-dory. + +5. Open a PR to change the version in __init__.py and setup.py to X.X.X+1.dev0 (e.g. VERSION=0.4.1 -> 0.4.2.dev0). + Then merge the PR once it's approved. +""" + +import os + +from setuptools import find_packages, setup + + +REQUIRED_PKGS = [ + # We need datasets as a backend + "datasets>=2.0.0", + # We use numpy>=1.17 to have np.random.Generator (Dataset shuffling) + "numpy>=1.17", + # For smart caching dataset processing + "dill", + # For performance gains with apache arrow + "pandas", + # for downloading datasets over HTTPS + "requests>=2.19.0", + # progress bars in download and scripts + "tqdm>=4.62.1", + # for fast hashing + "xxhash", + # for better multiprocessing + "multiprocess", + # to get metadata of optional dependencies such as torch or tensorflow for Python versions that don't have it + "importlib_metadata;python_version<'3.8'", + # to save datasets locally or on any filesystem + # minimum 2021.05.0 to have the AbstractArchiveFileSystem + "fsspec[http]>=2021.05.0", + # To get datasets from the Datasets Hub on huggingface.co + "huggingface-hub>=0.7.0", + # Utilities from PyPA to e.g., compare versions + "packaging", +] + +TEMPLATE_REQUIRE = [ + # to populate metric template + "cookiecutter", + # for the gradio widget + "gradio>=3.0.0" +] + +EVALUATOR_REQUIRE = [ + "transformers", + # for bootstrap computations in Evaluator + "scipy>=1.7.1", +] + +TESTS_REQUIRE = [ + # test dependencies + "absl-py", + "charcut>=1.1.1", # for charcut_mt + "cer>=1.2.0", # for characTER + "nltk", # for NIST and probably others + "pytest", + "pytest-datadir", + "pytest-xdist", + # optional dependencies + "tensorflow>=2.3,!=2.6.0,!=2.6.1, <=2.10", + "torch", + # metrics dependencies + "accelerate", # for frugalscore (calls transformers' Trainer) + "bert_score>=0.3.6", + "rouge_score>=0.1.2", + "sacrebleu", + "sacremoses", + "scipy>=1.10.0", + "seqeval", + "scikit-learn", + "jiwer", + "sentencepiece", # for bleurt + "transformers", # for evaluator + "mauve-text", + "trectools", + # to speed up pip backtracking + "toml>=0.10.1", + "requests_file>=1.5.1", + "tldextract>=3.1.0", + "texttable>=1.6.3", + "unidecode>=1.3.4", + "Werkzeug>=1.0.1", + "six~=1.15.0", +] + +QUALITY_REQUIRE = ["black~=22.0", "flake8>=3.8.3", "isort>=5.0.0", "pyyaml>=5.3.1"] + + +EXTRAS_REQUIRE = { + "tensorflow": ["tensorflow>=2.2.0,!=2.6.0,!=2.6.1"], + "tensorflow_gpu": ["tensorflow-gpu>=2.2.0,!=2.6.0,!=2.6.1"], + "torch": ["torch"], + "dev": TESTS_REQUIRE + QUALITY_REQUIRE, + "tests": TESTS_REQUIRE, + "quality": QUALITY_REQUIRE, + "docs": [ + # Might need to add doc-builder and some specific deps in the future + "s3fs", + ], + "template": TEMPLATE_REQUIRE, + "evaluator": EVALUATOR_REQUIRE +} + +setup( + name="evaluate", + version="0.4.2", # expected format is one of x.y.z.dev0, or x.y.z.rc1 or x.y.z (no to dashes, yes to dots) + description="HuggingFace community-driven open-source library of evaluation", + long_description=open("README.md", encoding="utf-8").read(), + long_description_content_type="text/markdown", + author="HuggingFace Inc.", + author_email="leandro@huggingface.co", + url="https://github.com/huggingface/evaluate", + download_url="https://github.com/huggingface/evaluate/tags", + license="Apache 2.0", + package_dir={"": "src"}, + packages=find_packages("src"), + entry_points={"console_scripts": ["evaluate-cli=evaluate.commands.evaluate_cli:main"]}, + install_requires=REQUIRED_PKGS, + extras_require=EXTRAS_REQUIRE, + python_requires=">=3.8.0", + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + ], + keywords="metrics machine learning evaluate evaluation", + zip_safe=False, # Required for mypy to find the py.typed file +) diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/__init__.py b/TextClassification/TextClassification/evaluate/src/evaluate/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..acd8330050c6fa934021c0d2536547d674d60c72 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/__init__.py @@ -0,0 +1,51 @@ +# flake8: noqa +# Copyright 2020 The HuggingFace Evaluate Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +# pylint: enable=line-too-long +# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position + +__version__ = "0.4.2" + +from packaging import version + + +SCRIPTS_VERSION = "main" if version.parse(__version__).is_devrelease else __version__ + +del version + +from .evaluation_suite import EvaluationSuite +from .evaluator import ( + AudioClassificationEvaluator, + AutomaticSpeechRecognitionEvaluator, + Evaluator, + ImageClassificationEvaluator, + QuestionAnsweringEvaluator, + SummarizationEvaluator, + Text2TextGenerationEvaluator, + TextClassificationEvaluator, + TextGenerationEvaluator, + TokenClassificationEvaluator, + TranslationEvaluator, + evaluator, +) +from .hub import push_to_hub +from .info import ComparisonInfo, EvaluationModuleInfo, MeasurementInfo, MetricInfo +from .inspect import inspect_evaluation_module, list_evaluation_modules +from .loading import load +from .module import CombinedEvaluations, Comparison, EvaluationModule, Measurement, Metric, combine +from .saving import save +from .utils import * +from .utils import gradio, logging diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/commands/__init__.py b/TextClassification/TextClassification/evaluate/src/evaluate/commands/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/commands/evaluate_cli.py b/TextClassification/TextClassification/evaluate/src/evaluate/commands/evaluate_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..80593c4dfa0f96c8d3ea5ff6131c13c0a94181eb --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/commands/evaluate_cli.py @@ -0,0 +1,137 @@ +import argparse +import os +import subprocess +from pathlib import Path + +from cookiecutter.main import cookiecutter +from huggingface_hub import HfApi, Repository, create_repo + +from evaluate.utils.logging import get_logger + + +logger = get_logger(__name__) + +INSTRUCTIONS = """\ +A new repository for your module "{module_name}" of type "{module_type}" has been created at {output_dir} and pushed to the Hugging Face Hub: {repo_url}. + +Here are the next steps: +- implement the module logic in {module_slug}/{module_slug}.py +- document your module in {module_slug}/README.md +- add test cases for your module in {module_slug}/tests.py +- if your module has any dependencies update them in {module_slug}/requirements.txt + +You can test your module's widget locally by running: + +``` +python {output_dir}/{module_slug}/app.py +``` + +When you are happy with your changes you can push your changes with the following commands to the Hugging Face Hub: + +``` +cd {output_dir}/{module_slug} +git add . +git commit -m "Updating module" +git push +``` + +You should then see the update widget on the Hugging Face Hub: {repo_url} +And you can load your module in Python with the following code: + +``` +from evaluate import load +module = load("{namespace}/{module_slug}") +``` +""" + + +def main(): + parser = argparse.ArgumentParser("HuggingFace Evaluate CLI tool", usage="evaluate-cli []") + subparsers = parser.add_subparsers() + parser_create = subparsers.add_parser("create", help="Create new evaluation module.") + parser_create.add_argument( + "module_name", type=str, help='Pretty name of new evaluation module, e.g. "Recall" or "Exact Match".' + ) + parser_create.add_argument( + "--module_type", + default="metric", + type=str, + help="Type of module, has to be one of [metric|comparison|measurement].", + ) + parser_create.add_argument( + "--dataset_name", default="", type=str, help="Name of dataset if evaluation module is dataset specific." + ) + parser_create.add_argument("--module_description", type=str, help="Short description of evaluation module.") + parser_create.add_argument("--output_dir", default=Path.cwd(), type=str, help="Path to output directory.") + parser_create.add_argument( + "--organization", default=None, type=str, help="Organization on the Hub to push evaluation module to." + ) + parser_create.add_argument("--private", action="store_true", help="Sets evaluation module repository to private.") + args = vars(parser.parse_args()) + + if args["module_type"] not in ["metric", "comparison", "measurement"]: + raise ValueError("The module_type needs to be one of metric, comparison, or measurement") + + if "-" in args["module_name"]: + raise ValueError("Hyphens ('-') are not allowed in module names.") + + output_dir = Path(args["output_dir"]) + organization = args["organization"] + module_slug = args["module_name"].lower().replace(" ", "_") + + if organization is None: + hfapi = HfApi() + namespace = hfapi.whoami()["name"] + else: + namespace = organization + args["namespace"] = namespace + repo_url = f"https://huggingface.co/spaces/{namespace}/{module_slug}" + + try: + create_repo(namespace + "/" + module_slug, repo_type="space", space_sdk="gradio", private=args["private"]) + except Exception as exception: + logger.error( + f"Could not create Space for module at hf.co/spaces/{namespace}/{module_slug}. Make sure this space does not exist already." + ) + raise exception + subprocess.run( + f"git clone {repo_url}".split(), + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + check=True, + encoding="utf-8", + cwd=output_dir, + env=os.environ.copy(), + ) + + repo = Repository( + local_dir=output_dir / module_slug, + ) + + cookiecutter( + "https://github.com/huggingface/evaluate/", + directory="templates", + no_input=True, + extra_context=args, + output_dir=output_dir, + overwrite_if_exists=True, + ) + + repo.git_add() + repo.git_commit("add module default template") + repo.git_push() + + print( + INSTRUCTIONS.format( + module_name=args["module_name"], + module_type=args["module_type"], + module_slug=module_slug, + namespace=namespace, + repo_url=repo_url, + output_dir=output_dir, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/config.py b/TextClassification/TextClassification/evaluate/src/evaluate/config.py new file mode 100644 index 0000000000000000000000000000000000000000..4909fa251ff82893d7c3c536bb111ae947735a8a --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/config.py @@ -0,0 +1,192 @@ +import importlib +import os +import platform +from pathlib import Path + +from packaging import version + +from .utils.logging import get_logger + + +logger = get_logger(__name__) + + +# Metrics +S3_METRICS_BUCKET_PREFIX = "https://s3.amazonaws.com/datasets.huggingface.co/datasets/metrics" +CLOUDFRONT_METRICS_DISTRIB_PREFIX = "https://cdn-datasets.huggingface.co/datasets/metric" +REPO_METRICS_URL = "https://raw.githubusercontent.com/huggingface/evaluate/{revision}/metrics/{path}/{name}" +REPO_MEASUREMENTS_URL = "https://raw.githubusercontent.com/huggingface/evaluate/{revision}/measurements/{path}/{name}" +REPO_COMPARISONS_URL = "https://raw.githubusercontent.com/huggingface/evaluate/{revision}/comparisons/{path}/{name}" + +# Evaluation module types +EVALUATION_MODULE_TYPES = ["metric", "comparison", "measurement"] + +# Hub +HF_ENDPOINT = os.environ.get("HF_ENDPOINT", "https://huggingface.co") +HF_LIST_ENDPOINT = HF_ENDPOINT + "/api/spaces?filter={type}" +HUB_EVALUATE_URL = HF_ENDPOINT + "/spaces/{path}/resolve/{revision}/{name}" +HUB_DEFAULT_VERSION = "main" + +PY_VERSION = version.parse(platform.python_version()) + +if PY_VERSION < version.parse("3.8"): + import importlib_metadata +else: + import importlib.metadata as importlib_metadata + +# General environment variables accepted values for booleans +ENV_VARS_TRUE_VALUES = {"1", "ON", "YES", "TRUE"} +ENV_VARS_TRUE_AND_AUTO_VALUES = ENV_VARS_TRUE_VALUES.union({"AUTO"}) + + +# Imports +PANDAS_VERSION = version.parse(importlib_metadata.version("pandas")) +PYARROW_VERSION = version.parse(importlib_metadata.version("pyarrow")) + +USE_TF = os.environ.get("USE_TF", "AUTO").upper() +USE_TORCH = os.environ.get("USE_TORCH", "AUTO").upper() +USE_JAX = os.environ.get("USE_JAX", "AUTO").upper() + +TORCH_VERSION = "N/A" +TORCH_AVAILABLE = False + +if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES: + TORCH_AVAILABLE = importlib.util.find_spec("torch") is not None + if TORCH_AVAILABLE: + try: + TORCH_VERSION = version.parse(importlib_metadata.version("torch")) + logger.info(f"PyTorch version {TORCH_VERSION} available.") + except importlib_metadata.PackageNotFoundError: + pass +else: + logger.info("Disabling PyTorch because USE_TF is set") + +TF_VERSION = "N/A" +TF_AVAILABLE = False + +if USE_TF in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TORCH not in ENV_VARS_TRUE_VALUES: + TF_AVAILABLE = importlib.util.find_spec("tensorflow") is not None + if TF_AVAILABLE: + # For the metadata, we have to look for both tensorflow and tensorflow-cpu + for package in [ + "tensorflow", + "tensorflow-cpu", + "tensorflow-gpu", + "tf-nightly", + "tf-nightly-cpu", + "tf-nightly-gpu", + "intel-tensorflow", + "tensorflow-rocm", + "tensorflow-macos", + ]: + try: + TF_VERSION = version.parse(importlib_metadata.version(package)) + except importlib_metadata.PackageNotFoundError: + continue + else: + break + else: + TF_AVAILABLE = False + if TF_AVAILABLE: + if TF_VERSION.major < 2: + logger.info(f"TensorFlow found but with version {TF_VERSION}. `datasets` requires version 2 minimum.") + TF_AVAILABLE = False + else: + logger.info(f"TensorFlow version {TF_VERSION} available.") +else: + logger.info("Disabling Tensorflow because USE_TORCH is set") + + +JAX_VERSION = "N/A" +JAX_AVAILABLE = False + +if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES: + JAX_AVAILABLE = importlib.util.find_spec("jax") is not None + if JAX_AVAILABLE: + try: + JAX_VERSION = version.parse(importlib_metadata.version("jax")) + logger.info(f"JAX version {JAX_VERSION} available.") + except importlib_metadata.PackageNotFoundError: + pass +else: + logger.info("Disabling JAX because USE_JAX is set to False") + + +# Cache location +DEFAULT_XDG_CACHE_HOME = "~/.cache" +XDG_CACHE_HOME = os.getenv("XDG_CACHE_HOME", DEFAULT_XDG_CACHE_HOME) +DEFAULT_HF_CACHE_HOME = os.path.join(XDG_CACHE_HOME, "huggingface") +HF_CACHE_HOME = os.path.expanduser(os.getenv("HF_HOME", DEFAULT_HF_CACHE_HOME)) + +DEFAULT_HF_EVALUATE_CACHE = os.path.join(HF_CACHE_HOME, "evaluate") +HF_EVALUATE_CACHE = Path(os.getenv("HF_EVALUATE_CACHE", DEFAULT_HF_EVALUATE_CACHE)) + +DEFAULT_HF_METRICS_CACHE = os.path.join(HF_CACHE_HOME, "metrics") +HF_METRICS_CACHE = Path(os.getenv("HF_METRICS_CACHE", DEFAULT_HF_METRICS_CACHE)) + +DEFAULT_HF_MODULES_CACHE = os.path.join(HF_CACHE_HOME, "modules") +HF_MODULES_CACHE = Path(os.getenv("HF_MODULES_CACHE", DEFAULT_HF_MODULES_CACHE)) + +DOWNLOADED_DATASETS_DIR = "downloads" +DEFAULT_DOWNLOADED_EVALUATE_PATH = os.path.join(HF_EVALUATE_CACHE, DOWNLOADED_DATASETS_DIR) +DOWNLOADED_EVALUATE_PATH = Path(os.getenv("HF_DATASETS_DOWNLOADED_EVALUATE_PATH", DEFAULT_DOWNLOADED_EVALUATE_PATH)) + +EXTRACTED_EVALUATE_DIR = "extracted" +DEFAULT_EXTRACTED_EVALUATE_PATH = os.path.join(DEFAULT_DOWNLOADED_EVALUATE_PATH, EXTRACTED_EVALUATE_DIR) +EXTRACTED_EVALUATE_PATH = Path(os.getenv("HF_DATASETS_EXTRACTED_EVALUATE_PATH", DEFAULT_EXTRACTED_EVALUATE_PATH)) + +# Download count for the website +HF_UPDATE_DOWNLOAD_COUNTS = ( + os.environ.get("HF_UPDATE_DOWNLOAD_COUNTS", "AUTO").upper() in ENV_VARS_TRUE_AND_AUTO_VALUES +) + +# Offline mode +HF_EVALUATE_OFFLINE = os.environ.get("HF_EVALUATE_OFFLINE", "AUTO").upper() in ENV_VARS_TRUE_VALUES + + +# File names +LICENSE_FILENAME = "LICENSE" +METRIC_INFO_FILENAME = "metric_info.json" +DATASETDICT_JSON_FILENAME = "dataset_dict.json" + +MODULE_NAME_FOR_DYNAMIC_MODULES = "evaluate_modules" + +HF_HUB_ALLOWED_TASKS = [ + "image-classification", + "translation", + "image-segmentation", + "fill-mask", + "automatic-speech-recognition", + "token-classification", + "sentence-similarity", + "audio-classification", + "question-answering", + "summarization", + "zero-shot-classification", + "table-to-text", + "feature-extraction", + "other", + "multiple-choice", + "text-classification", + "text-to-image", + "text2text-generation", + "zero-shot-image-classification", + "tabular-classification", + "tabular-regression", + "image-to-image", + "tabular-to-text", + "unconditional-image-generation", + "text-retrieval", + "text-to-speech", + "object-detection", + "audio-to-audio", + "text-generation", + "conversational", + "table-question-answering", + "visual-question-answering", + "image-to-text", + "reinforcement-learning", + "voice-activity-detection", + "time-series-forecasting", + "document-question-answering", +] diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/evaluation_suite/__init__.py b/TextClassification/TextClassification/evaluate/src/evaluate/evaluation_suite/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a306d8068a6d66b60fbcc5420bf0cbb334c36305 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/evaluation_suite/__init__.py @@ -0,0 +1,128 @@ +import importlib +import inspect +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Dict, Optional, Union + +from datasets import Dataset, DownloadConfig, DownloadMode, load_dataset +from datasets.utils.version import Version + +from ..evaluator import evaluator +from ..loading import evaluation_module_factory +from ..utils.logging import get_logger + + +logger = get_logger(__name__) + + +@dataclass +class SubTask: + task_type: str + data: Optional[Union[str, Dataset]] = None + subset: Optional[str] = None + split: Optional[str] = None + data_preprocessor: Optional[Callable] = None + args_for_task: Optional[dict] = None + + def __post_init__(self): + if type(self.task_type) is not str: + raise ValueError(f"'task_type' must be type 'str', got {type(self.task_type)}") + if type(self.data) not in [Dataset, str]: + raise ValueError( + f"'data' must be an already-instantiated Dataset object or type 'str', got {type(self.data)}" + ) + if self.subset and type(self.subset) is not str: + raise ValueError(f"'subset' must be type 'str', got {type(self.subset)}") + if self.split and type(self.split) is not str: + raise ValueError(f"'split' must be type 'str', got {type(self.split)}") + if self.data_preprocessor and not callable(self.data_preprocessor): + raise ValueError(f"'data_preprocessor' must be a Callable', got {self.data_preprocessor}") + if self.args_for_task and type(self.args_for_task) is not dict: + raise ValueError(f"'args_for_task' must be type 'dict', got {type(self.args_for_task)}") + + +def import_main_class(module_path): + """Import a module at module_path and return the EvaluationSuite class""" + module = importlib.import_module(module_path) + + module_main_cls = None + for name, obj in module.__dict__.items(): + if isinstance(obj, type) and obj.__name__ == "Suite": + if inspect.isabstract(obj): + continue + module_main_cls = obj + break + + return module_main_cls + + +class EvaluationSuite: + """ + This class instantiates an evaluation suite made up of multiple tasks, where each task consists of a dataset and + an associated metric, and runs evaluation on a model or pipeline. Evaluation suites can be a Python script found + either locally or uploaded as a Space on the Hugging Face Hub. + Usage: + ```python + from evaluate import EvaluationSuite + suite = EvaluationSuite.load("evaluate/evaluation-suite-ci") + results = suite.run("lvwerra/distilbert-imdb") + ``` + """ + + def __init__(self, name): + self.name = name + + @staticmethod + def load( + path: str, + download_mode: Optional[DownloadMode] = None, + revision: Optional[Union[str, Version]] = None, + download_config: Optional[DownloadConfig] = None, + ): + download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) + evaluation_module = evaluation_module_factory( + path, module_type=None, revision=revision, download_config=download_config, download_mode=download_mode + ) + name = Path(path).stem + evaluation_cls = import_main_class(evaluation_module.module_path) + evaluation_instance = evaluation_cls(name) + + return evaluation_instance + + def __repr__(self): + self.tasks = [str(task) for task in self.suite] + return f'EvaluationSuite name: "{self.name}", ' f"Tasks: {self.tasks})" + + def assert_suite_nonempty(self): + if not self.suite: + raise ValueError( + "No evaluation tasks found. The EvaluationSuite must include at least one SubTask definition." + ) + + def run( + self, model_or_pipeline: Union[str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel"] # noqa: F821 + ) -> Dict[str, float]: + + self.assert_suite_nonempty() + + results_all = [] + for task in self.suite: + + task_name = task.data + + if task.data_preprocessor: # task requires extra preprocessing + ds = load_dataset(task.data, name=task.subset, split=task.split) + task.data = ds.map(task.data_preprocessor) + + task_evaluator = evaluator(task.task_type) + args_for_task = task.args_for_task + args_for_task["model_or_pipeline"] = model_or_pipeline + args_for_task["data"] = task.data + args_for_task["subset"] = task.subset + args_for_task["split"] = task.split + results = task_evaluator.compute(**args_for_task) + + results["task_name"] = task_name + "/" + task.subset if task.subset else task_name + results["data_preprocessor"] = str(task.data_preprocessor) if task.data_preprocessor is not None else None + results_all.append(results) + return results_all diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/__init__.py b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a2fe4be8a1332417fb8515f019c1b7e8c41a58bf --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/__init__.py @@ -0,0 +1,140 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +try: + from transformers.pipelines import SUPPORTED_TASKS as SUPPORTED_PIPELINE_TASKS + from transformers.pipelines import TASK_ALIASES + from transformers.pipelines import check_task as check_pipeline_task + + TRANSFORMERS_AVAILABLE = True +except ImportError: + TRANSFORMERS_AVAILABLE = False + +from typing import Dict, List + +from .audio_classification import AudioClassificationEvaluator +from .automatic_speech_recognition import AutomaticSpeechRecognitionEvaluator +from .base import Evaluator +from .image_classification import ImageClassificationEvaluator +from .question_answering import QuestionAnsweringEvaluator +from .text2text_generation import SummarizationEvaluator, Text2TextGenerationEvaluator, TranslationEvaluator +from .text_classification import TextClassificationEvaluator +from .text_generation import TextGenerationEvaluator +from .token_classification import TokenClassificationEvaluator + + +SUPPORTED_EVALUATOR_TASKS = { + "text-classification": { + "implementation": TextClassificationEvaluator, + "default_metric_name": "accuracy", + }, + "image-classification": { + "implementation": ImageClassificationEvaluator, + "default_metric_name": "accuracy", + }, + "question-answering": { + "implementation": QuestionAnsweringEvaluator, + "default_metric_name": "squad", + }, + "token-classification": { + "implementation": TokenClassificationEvaluator, + "default_metric_name": "seqeval", + }, + "text-generation": { + "implementation": TextGenerationEvaluator, + "default_metric_name": "word_count", + }, + "text2text-generation": { + "implementation": Text2TextGenerationEvaluator, + "default_metric_name": "bleu", + }, + "summarization": { + "implementation": SummarizationEvaluator, + "default_metric_name": "rouge", + }, + "translation": { + "implementation": TranslationEvaluator, + "default_metric_name": "bleu", + }, + "automatic-speech-recognition": { + "implementation": AutomaticSpeechRecognitionEvaluator, + "default_metric_name": "wer", + }, + "audio-classification": { + "implementation": AudioClassificationEvaluator, + "default_metric_name": "accuracy", + }, +} + + +def get_supported_tasks() -> List[str]: + """ + Returns a list of supported task strings. + """ + return list(SUPPORTED_EVALUATOR_TASKS.keys()) + + +def check_task(task: str) -> Dict: + """ + Checks an incoming task string, to validate it's correct and returns the default Evaluator class and default metric + name. It first performs a check to validata that the string is a valid `Pipeline` task, then it checks if it's a + valid `Evaluator` task. `Evaluator` tasks are a substet of `Pipeline` tasks. + Args: + task (`str`): + The task defining which evaluator will be returned. Currently accepted tasks are: + - `"image-classification"` + - `"question-answering"` + - `"text-classification"` (alias `"sentiment-analysis"` available) + - `"token-classification"` + Returns: + task_defaults: `dict`, contains the implementasion class of a give Evaluator and the default metric name. + """ + if task in TASK_ALIASES: + task = TASK_ALIASES[task] + if not check_pipeline_task(task): + raise KeyError(f"Unknown task {task}, available tasks are: {get_supported_tasks()}.") + if task in SUPPORTED_EVALUATOR_TASKS.keys() and task in SUPPORTED_PIPELINE_TASKS.keys(): + return SUPPORTED_EVALUATOR_TASKS[task] + raise KeyError(f"Unknown task {task}, available tasks are: {get_supported_tasks()}.") + + +def evaluator(task: str = None) -> Evaluator: + """ + Utility factory method to build an [`Evaluator`]. + Evaluators encapsulate a task and a default metric name. They leverage `pipeline` functionality from `transformers` + to simplify the evaluation of multiple combinations of models, datasets and metrics for a given task. + Args: + task (`str`): + The task defining which evaluator will be returned. Currently accepted tasks are: + - `"image-classification"`: will return a [`ImageClassificationEvaluator`]. + - `"question-answering"`: will return a [`QuestionAnsweringEvaluator`]. + - `"text-classification"` (alias `"sentiment-analysis"` available): will return a [`TextClassificationEvaluator`]. + - `"token-classification"`: will return a [`TokenClassificationEvaluator`]. + Returns: + [`Evaluator`]: An evaluator suitable for the task. + Examples: + ```python + >>> from evaluate import evaluator + >>> # Sentiment analysis evaluator + >>> evaluator("sentiment-analysis") + ```""" + if not TRANSFORMERS_AVAILABLE: + raise ImportError( + "If you want to use the `Evaluator` you need `transformers`. Run `pip install evaluate[transformers]`." + ) + targeted_task = check_task(task) + evaluator_class = targeted_task["implementation"] + default_metric_name = targeted_task["default_metric_name"] + return evaluator_class(task=task, default_metric_name=default_metric_name) diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/audio_classification.py b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/audio_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..685fb9fd8515f8506b89e9375948fea181f79a8f --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/audio_classification.py @@ -0,0 +1,151 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from numbers import Number +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Union + +from datasets import Dataset +from typing_extensions import Literal + +from ..module import EvaluationModule +from ..utils.file_utils import add_end_docstrings, add_start_docstrings +from .base import EVALUATOR_COMPUTE_RETURN_DOCSTRING, EVALUTOR_COMPUTE_START_DOCSTRING, Evaluator + + +if TYPE_CHECKING: + from transformers import FeatureExtractionMixin, Pipeline, PreTrainedModel, PreTrainedTokenizer, TFPreTrainedModel + + +TASK_DOCUMENTATION = r""" + Examples: + + + + Remember that, in order to process audio files, you need ffmpeg installed (https://ffmpeg.org/download.html) + + + + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + + >>> task_evaluator = evaluator("audio-classification") + >>> data = load_dataset("superb", 'ks', split="test[:40]") + >>> results = task_evaluator.compute( + >>> model_or_pipeline=""superb/wav2vec2-base-superb-ks"", + >>> data=data, + >>> label_column="label", + >>> input_column="file", + >>> metric="accuracy", + >>> label_mapping={0: "yes", 1: "no", 2: "up", 3: "down"} + >>> ) + ``` + + + + The evaluator supports raw audio data as well, in the form of a numpy array. However, be aware that calling + the audio column automatically decodes and resamples the audio files, which can be slow for large datasets. + + + + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + + >>> task_evaluator = evaluator("audio-classification") + >>> data = load_dataset("superb", 'ks', split="test[:40]") + >>> data = data.map(lambda example: {"audio": example["audio"]["array"]}) + >>> results = task_evaluator.compute( + >>> model_or_pipeline=""superb/wav2vec2-base-superb-ks"", + >>> data=data, + >>> label_column="label", + >>> input_column="audio", + >>> metric="accuracy", + >>> label_mapping={0: "yes", 1: "no", 2: "up", 3: "down"} + >>> ) + ``` +""" + + +class AudioClassificationEvaluator(Evaluator): + """ + Audio classification evaluator. + This audio classification evaluator can currently be loaded from [`evaluator`] using the default task name + `audio-classification`. + Methods in this class assume a data format compatible with the [`transformers.AudioClassificationPipeline`]. + """ + + PIPELINE_KWARGS = {} + + def __init__(self, task="audio-classification", default_metric_name=None): + super().__init__(task, default_metric_name=default_metric_name) + + def predictions_processor(self, predictions, label_mapping): + pred_label = [max(pred, key=lambda x: x["score"])["label"] for pred in predictions] + pred_label = [label_mapping[pred] if label_mapping is not None else pred for pred in pred_label] + + return {"predictions": pred_label} + + @add_start_docstrings(EVALUTOR_COMPUTE_START_DOCSTRING) + @add_end_docstrings(EVALUATOR_COMPUTE_RETURN_DOCSTRING, TASK_DOCUMENTATION) + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: Optional[str] = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + feature_extractor: Optional[Union[str, "FeatureExtractionMixin"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: int = None, + random_state: Optional[int] = None, + input_column: str = "file", + label_column: str = "label", + label_mapping: Optional[Dict[str, Number]] = None, + ) -> Tuple[Dict[str, float], Any]: + + """ + input_column (`str`, defaults to `"file"`): + The name of the column containing either the audio files or a raw waveform, represented as a numpy array, in the dataset specified by `data`. + label_column (`str`, defaults to `"label"`): + The name of the column containing the labels in the dataset specified by `data`. + label_mapping (`Dict[str, Number]`, *optional*, defaults to `None`): + We want to map class labels defined by the model in the pipeline to values consistent with those + defined in the `label_column` of the `data` dataset. + """ + + result = super().compute( + model_or_pipeline=model_or_pipeline, + data=data, + subset=subset, + split=split, + metric=metric, + tokenizer=tokenizer, + feature_extractor=feature_extractor, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + device=device, + random_state=random_state, + input_column=input_column, + label_column=label_column, + label_mapping=label_mapping, + ) + + return result diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/automatic_speech_recognition.py b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/automatic_speech_recognition.py new file mode 100644 index 0000000000000000000000000000000000000000..ee423826cdd7bac384080b3db8a369cc59a53283 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/automatic_speech_recognition.py @@ -0,0 +1,112 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Union + +from datasets import Dataset +from typing_extensions import Literal + +from ..module import EvaluationModule +from ..utils.file_utils import add_end_docstrings, add_start_docstrings +from .base import EVALUATOR_COMPUTE_RETURN_DOCSTRING, EVALUTOR_COMPUTE_START_DOCSTRING, Evaluator + + +if TYPE_CHECKING: + from transformers import Pipeline, PreTrainedModel, PreTrainedTokenizer, TFPreTrainedModel + + +TASK_DOCUMENTATION = r""" + Examples: + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + >>> task_evaluator = evaluator("automatic-speech-recognition") + >>> data = load_dataset("mozilla-foundation/common_voice_11_0", "en", split="validation[:40]") + >>> results = task_evaluator.compute( + >>> model_or_pipeline="https://huggingface.co/openai/whisper-tiny.en", + >>> data=data, + >>> input_column="path", + >>> label_column="sentence", + >>> metric="wer", + >>> ) + ``` +""" + + +class AutomaticSpeechRecognitionEvaluator(Evaluator): + """ + Automatic speech recognition evaluator. + This automatic speech recognition evaluator can currently be loaded from [`evaluator`] using the default task name + `automatic-speech-recognition`. + Methods in this class assume a data format compatible with the [`AutomaticSpeechRecognitionPipeline`]. + """ + + PIPELINE_KWARGS = {"truncation": True} + + def __init__(self, task="automatic-speech-recognition", default_metric_name=None): + super().__init__(task, default_metric_name=default_metric_name) + + def predictions_processor(self, predictions, label_mapping): + return {"predictions": [pred["text"] for pred in predictions]} + + @add_start_docstrings(EVALUTOR_COMPUTE_START_DOCSTRING) + @add_end_docstrings(EVALUATOR_COMPUTE_RETURN_DOCSTRING, TASK_DOCUMENTATION) + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: Optional[str] = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: int = None, + random_state: Optional[int] = None, + input_column: str = "path", + label_column: str = "sentence", + generation_kwargs: dict = None, + ) -> Tuple[Dict[str, float], Any]: + """ + input_column (`str`, defaults to `"path"`): + the name of the column containing the input audio path in the dataset specified by `data`. + label_column (`str`, defaults to `"sentence"`): + the name of the column containing the labels in the dataset specified by `data`. + generation_kwargs (`Dict`, *optional*, defaults to `None`): + The generation kwargs are passed to the pipeline and set the text generation strategy. + """ + + if generation_kwargs is not None: + self.PIPELINE_KWARGS.update(generation_kwargs) + + result = super().compute( + model_or_pipeline=model_or_pipeline, + data=data, + subset=subset, + split=split, + metric=metric, + tokenizer=tokenizer, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + device=device, + random_state=random_state, + input_column=input_column, + label_column=label_column, + ) + + return result diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/base.py b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/base.py new file mode 100644 index 0000000000000000000000000000000000000000..09de31f19dedda5dbe38ce0295d8e5b0d962f389 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/base.py @@ -0,0 +1,544 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from numbers import Number +from typing import Any, Callable, Dict, List, Optional, Union + +# Lint as: python3 +from datasets import Dataset, load_dataset + +from evaluate.evaluator.utils import choose_split + + +try: + from scipy.stats import bootstrap + + SCIPY_AVAILABLE = True +except ImportError: + SCIPY_AVAILABLE = False + +try: + import transformers + from transformers import Pipeline, pipeline + + TRANSFORMERS_AVAILABLE = True +except ImportError: + TRANSFORMERS_AVAILABLE = False + +from time import perf_counter + +from typing_extensions import Literal + +from ..loading import load +from ..module import EvaluationModule +from ..utils.logging import get_logger +from .utils import DatasetColumn + + +logger = get_logger(__name__) + + +EVALUTOR_COMPUTE_START_DOCSTRING = r""" + Compute the metric for a given pipeline and dataset combination. + Args: + model_or_pipeline (`str` or `Pipeline` or `Callable` or `PreTrainedModel` or `TFPreTrainedModel`, defaults to `None`): + If the argument in not specified, we initialize the default pipeline for the task (in this case + `text-classification` or its alias - `sentiment-analysis`). If the argument is of the type `str` or + is a model instance, we use it to initialize a new `Pipeline` with the given model. Otherwise we assume the + argument specifies a pre-initialized pipeline. + data (`str` or `Dataset`, defaults to `None`): + Specifies the dataset we will run evaluation on. If it is of type `str`, we treat it as the dataset + name, and load it. Otherwise we assume it represents a pre-loaded dataset. + subset (`str`, defaults to `None`): + Defines which dataset subset to load. If `None` is passed the default subset is loaded. + split (`str`, defaults to `None`): + Defines which dataset split to load. If `None` is passed, infers based on the `choose_split` function. + metric (`str` or `EvaluationModule`, defaults to `None`): + Specifies the metric we use in evaluator. If it is of type `str`, we treat it as the metric name, and + load it. Otherwise we assume it represents a pre-loaded metric. + tokenizer (`str` or `PreTrainedTokenizer`, *optional*, defaults to `None`): + Argument can be used to overwrite a default tokenizer if `model_or_pipeline` represents a model for + which we build a pipeline. If `model_or_pipeline` is `None` or a pre-initialized pipeline, we ignore + this argument. + strategy (`Literal["simple", "bootstrap"]`, defaults to "simple"): + specifies the evaluation strategy. Possible values are: + - `"simple"` - we evaluate the metric and return the scores. + - `"bootstrap"` - on top of computing the metric scores, we calculate the confidence interval for each + of the returned metric keys, using `scipy`'s `bootstrap` method + https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html. + confidence_level (`float`, defaults to `0.95`): + The `confidence_level` value passed to `bootstrap` if `"bootstrap"` strategy is chosen. + n_resamples (`int`, defaults to `9999`): + The `n_resamples` value passed to `bootstrap` if `"bootstrap"` strategy is chosen. + device (`int`, defaults to `None`): + Device ordinal for CPU/GPU support of the pipeline. Setting this to -1 will leverage CPU, a positive + integer will run the model on the associated CUDA device ID. If `None` is provided it will be inferred and + CUDA:0 used if available, CPU otherwise. + random_state (`int`, *optional*, defaults to `None`): + The `random_state` value passed to `bootstrap` if `"bootstrap"` strategy is chosen. Useful for + debugging. +""" + +EVALUATOR_COMPUTE_RETURN_DOCSTRING = r""" + Return: + A `Dict`. The keys represent metric keys calculated for the `metric` spefied in function arguments. For the + `"simple"` strategy, the value is the metric score. For the `"bootstrap"` strategy, the value is a `Dict` + containing the score, the confidence interval and the standard error calculated for each metric key. +""" + + +class Evaluator(ABC): + """ + The [`Evaluator`] class is the class from which all evaluators inherit. Refer to this class for methods shared across + different evaluators. + Base class implementing evaluator operations. + """ + + PIPELINE_KWARGS = {} + METRIC_KWARGS = {} + + def __init__(self, task: str, default_metric_name: str = None): + if not TRANSFORMERS_AVAILABLE: + raise ImportError( + "If you want to use the `Evaluator` you need `transformers`. Run `pip install evaluate[evaluator]`." + ) + if not SCIPY_AVAILABLE: + raise ImportError( + "If you want to use the `Evaluator` you need `scipy>=1.7.1`. Run `pip install evaluate[evaluator]`." + ) + self.task = task + self.default_metric_name = default_metric_name + + @staticmethod + def _compute_confidence_interval( + metric, + metric_inputs, + metric_keys: List[str], + confidence_level: float = 0.95, + n_resamples: int = 9999, + random_state: Optional[int] = None, + ) -> Dict[str, Any]: + """ + A utility function enabling the confidence interval calculation for metrics computed + by the evaluator based on `scipy`'s `bootstrap` method. + """ + + # bootstrap only works with functions that use args and no kwargs + def build_args_metric(metric, key, **kwargs): + def args_metric(*args): + return metric.compute(**{k: v for k, v in zip(kwargs.keys(), args)})[key] + + return args_metric + + bootstrap_dict = {} + for key in metric_keys: + bs = bootstrap( + data=list(metric_inputs.values()), + statistic=build_args_metric(metric, key, **metric_inputs), + paired=True, + vectorized=False, + confidence_level=confidence_level, + n_resamples=n_resamples, + random_state=random_state, + ) + bootstrap_dict[key] = { + "confidence_interval": (bs.confidence_interval.low, bs.confidence_interval.high), + "standard_error": bs.standard_error, + } + return bootstrap_dict + + @staticmethod + def _compute_time_perf(start_time: float, end_time: float, num_samples: int) -> Dict[str, Any]: + """ + A utility function computing time performance metrics: + - `total_time_in_seconds` - pipeline inference runtime for the evaluation data in seconds, + - `samples_per_second` - pipeline throughput in the number of samples per second. + - `latency_in_seconds` - pipeline inference runtime for the evaluation data in seconds per sample, + + """ + latency = end_time - start_time + throughput = num_samples / latency + latency_sample = 1.0 / throughput + + return { + "total_time_in_seconds": latency, + "samples_per_second": throughput, + "latency_in_seconds": latency_sample, + } + + @staticmethod + def _infer_device() -> int: + """Helper function to check if GPU or CPU is available for inference.""" + # try infer with torch first + try: + import torch + + if torch.cuda.is_available(): + device = 0 # first GPU + else: + device = -1 # CPU + except ImportError: + # if not available try TF + try: + import tensorflow as tf + + if len(tf.config.list_physical_devices("GPU")) > 0: + device = 0 # first GPU + else: + device = -1 # CPU + except ImportError: + device = -1 + + if device == -1: + logger.info("No GPU found. The default device for pipeline inference is set to CPU.") + else: + logger.info("GPU found. The default device for pipeline inference is set to GPU (CUDA:0).") + + return device + + @abstractmethod + def predictions_processor(self, *args, **kwargs): + """ + A core method of the `Evaluator` class, which processes the pipeline outputs for compatibility with the metric. + """ + raise NotImplementedError() + + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: Optional[str] = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + feature_extractor: Optional[Union[str, "FeatureExtractionMixin"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: int = None, + random_state: Optional[int] = None, + input_column: str = "text", + label_column: str = "label", + label_mapping: Optional[Dict[str, Number]] = None, + ) -> Dict[str, float]: + + result = {} + + self.check_for_mismatch_in_device_setup(device, model_or_pipeline) + + # Prepare inputs + data = self.load_data(data=data, subset=subset, split=split) + metric_inputs, pipe_inputs = self.prepare_data(data=data, input_column=input_column, label_column=label_column) + pipe = self.prepare_pipeline( + model_or_pipeline=model_or_pipeline, + tokenizer=tokenizer, + feature_extractor=feature_extractor, + device=device, + ) + metric = self.prepare_metric(metric) + + # Compute predictions + predictions, perf_results = self.call_pipeline(pipe, pipe_inputs) + predictions = self.predictions_processor(predictions, label_mapping) + + metric_inputs.update(predictions) + + # Compute metrics from references and predictions + metric_results = self.compute_metric( + metric=metric, + metric_inputs=metric_inputs, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + random_state=random_state, + ) + + # TODO: To clarify why `wer` and `cer` return float + # even though metric.compute contract says that it + # returns Optional[dict]. + if type(metric_results) is float: + metric_results = {metric.name: metric_results} + + result.update(metric_results) + result.update(perf_results) + + return result + + @staticmethod + def check_for_mismatch_in_device_setup(device, model_or_pipeline): + if device is not None and device != -1 and isinstance(model_or_pipeline, Pipeline): + if model_or_pipeline.device.type == "cpu": + raise ValueError( + "The value of the `device` kwarg passed to `compute` suggests that this pipe should be run on an " + "accelerator, but the pipe was instantiated on CPU. Pass `device` to the pipeline during " + "initialization to use an accelerator, or pass `device=None` to `compute`. " + ) + elif device != model_or_pipeline.device.index: + raise ValueError( + f"This pipeline was instantiated on device {model_or_pipeline.device.index} but device={device} was passed to `compute`." + ) + + def check_required_columns(self, data: Union[str, Dataset], columns_names: Dict[str, str]): + """ + Ensure the columns required for the evaluation are present in the dataset. + + Args: + data (`str` or [`Dataset`]): + Specifies the dataset we will run evaluation on. + columns_names (`List[str]`): + List of column names to check in the dataset. The keys are the arguments to the [`evaluate.EvaluationModule.compute`] method, + while the values are the column names to check. + + Example: + + ```py + >>> from datasets import load_dataset + >>> from evaluate import evaluator + >>> data = load_dataset("rotten_tomatoes', split="train") + >>> evaluator.check_required_columns(data, {"input_column": "text", "label_column": "label"}) + ``` + """ + for input_name, column_name in columns_names.items(): + if column_name not in data.column_names: + raise ValueError( + f"Invalid `{input_name}` {column_name} specified. The dataset contains the following columns: {data.column_names}." + ) + + @staticmethod + def get_dataset_split(data, subset=None, split=None): + """ + Infers which split to use if `None` is given. + + Args: + data (`str`): + Name of dataset. + subset (`str`): + Name of config for datasets with multiple configurations (e.g. 'glue/cola'). + split (`str`, defaults to `None`): + Split to use. + Returns: + `split`: `str` containing which split to use + + Example: + + ```py + >>> from evaluate import evaluator + >>> evaluator("text-classification").get_dataset_split(data="rotten_tomatoes") + WARNING:evaluate.evaluator.base:Dataset split not defined! Automatically evaluating with split: TEST + 'test' + ``` + """ + if split is None: + split = choose_split(data, subset) + logger.warning(f"Dataset split not defined! Automatically evaluating with split: {split.upper()}") + return split + + def load_data(self, data: Union[str, Dataset], subset: str = None, split: str = None): + """ + Load dataset with given subset and split. + Args: + data ([`Dataset`] or `str`, defaults to `None`): + Specifies the dataset we will run evaluation on. If it is of + type `str`, we treat it as the dataset name, and load it. Otherwise we assume it represents a pre-loaded dataset. + subset (`str`, defaults to `None`): + Specifies dataset subset to be passed to `name` in `load_dataset`. To be + used with datasets with several configurations (e.g. glue/sst2). + split (`str`, defaults to `None`): + User-defined dataset split by name (e.g. train, validation, test). Supports slice-split (`test[:n]`). + If not defined and data is a `str` type, will automatically select the best one via `choose_split()`. + Returns: + data ([`Dataset`]): Loaded dataset which will be used for evaluation. + + Example: + + ```py + >>> from evaluate import evaluator + >>> evaluator("text-classification").load_data(data="rotten_tomatoes", split="train") + Dataset({ + features: ['text', 'label'], + num_rows: 8530 + }) + ``` + """ + if isinstance(data, str): + split = self.get_dataset_split(data, subset, split) + data = load_dataset(data, name=subset, split=split) + return data + elif isinstance(data, Dataset): + if split is not None or subset is not None: + logger.warning("`data` is a preloaded Dataset! Ignoring `subset` and `split`.") + return data + else: + raise ValueError( + "Please specify a valid `data` object - either a `str` with a name or a `Dataset` object." + ) + + def prepare_data(self, data: Dataset, input_column: str, label_column: str, *args, **kwargs): + """ + Prepare data. + + Args: + data ([`Dataset`]): + Specifies the dataset we will run evaluation on. + input_column (`str`, defaults to `"text"`): + The name of the column containing the text feature in the dataset specified by `data`. + second_input_column(`str`, *optional*): + The name of the column containing the second text feature if there is one. Otherwise, set to `None`. + label_column (`str`, defaults to `"label"`): + The name of the column containing the labels in the dataset specified by `data`. + Returns: + `dict`: metric inputs. + `list`: pipeline inputs. + + Example: + + ```py + >>> from evaluate import evaluator + >>> from datasets import load_dataset + + >>> ds = load_dataset("rotten_tomatoes", split="train") + >>> evaluator("text-classification").prepare_data(ds, input_column="text", second_input_column=None, label_column="label") + ``` + """ + + self.check_required_columns(data, {"input_column": input_column, "label_column": label_column}) + + return {"references": data[label_column]}, DatasetColumn(data, input_column) + + def prepare_pipeline( + self, + model_or_pipeline: Union[str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel"], # noqa: F821 + tokenizer: Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"] = None, # noqa: F821 + feature_extractor: Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"] = None, # noqa: F821 + device: int = None, + ): + """ + Prepare pipeline. + + Args: + model_or_pipeline (`str` or [`~transformers.Pipeline`] or `Callable` or [`~transformers.PreTrainedModel`] or [`~transformers.TFPreTrainedModel`], defaults to `None`): + If the argument in not specified, we initialize the default pipeline for the task. If the argument is of the type `str` or + is a model instance, we use it to initialize a new [`~transformers.Pipeline`] with the given model. Otherwise we assume the + argument specifies a pre-initialized pipeline. + preprocessor ([`~transformers.PreTrainedTokenizerBase`] or [`~transformers.FeatureExtractionMixin`], *optional*, defaults to `None`): + Argument can be used to overwrite a default preprocessor if `model_or_pipeline` represents a model for + which we build a pipeline. If `model_or_pipeline` is `None` or a pre-initialized pipeline, we ignore + this argument. + Returns: + The initialized pipeline. + + Example: + + ```py + >>> from evaluate import evaluator + >>> evaluator("text-classification").prepare_pipeline(model_or_pipeline="distilbert-base-uncased") + ``` + """ + + if device is None: + device = self._infer_device() + + if ( + isinstance(model_or_pipeline, str) + or isinstance(model_or_pipeline, transformers.PreTrainedModel) + or isinstance(model_or_pipeline, transformers.TFPreTrainedModel) + ): + pipe = pipeline( + self.task, + model=model_or_pipeline, + tokenizer=tokenizer, + feature_extractor=feature_extractor, + device=device, + ) + else: + if model_or_pipeline is None: + pipe = pipeline(self.task, device=device) + else: + pipe = model_or_pipeline + if tokenizer is not None and feature_extractor is not None: + logger.warning("Ignoring the value of the preprocessor argument (`tokenizer` or `feature_extractor`).") + if (pipe.task != self.task) and not (self.task == "translation" and pipe.task.startswith("translation")): + raise ValueError( + f"Incompatible `model_or_pipeline`. Please specify `model_or_pipeline` compatible with the `{self.task}` task." + ) + return pipe + + def prepare_metric(self, metric: Union[str, EvaluationModule]): + """ + Prepare metric. + + Args: + metric (`str` or [`EvaluationModule`], defaults to `None`): + Specifies the metric we use in evaluator. If it is of type `str`, we treat it as the metric name, and + load it. Otherwise we assume it represents a pre-loaded metric. + + Returns: + The loaded metric. + + Example: + + ```py + >>> from evaluate import evaluator + >>> evaluator("text-classification").prepare_metric("accuracy") + ``` + """ + # Prepare metric. + if metric is None: + if self.default_metric_name is None: + raise ValueError( + "`Evaluator` doesn't specify a default metric. Please specify a valid `metric` argument." + ) + metric = load(self.default_metric_name) + elif isinstance(metric, str): + metric = load(metric) + + return metric + + def call_pipeline(self, pipe, *args, **kwargs): + start_time = perf_counter() + pipe_output = pipe(*args, **kwargs, **self.PIPELINE_KWARGS) + end_time = perf_counter() + return pipe_output, self._compute_time_perf(start_time, end_time, len(pipe_output)) + + def compute_metric( + self, + metric: EvaluationModule, + metric_inputs: Dict, + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + random_state: Optional[int] = None, + ): + """Compute and return metrics.""" + result = metric.compute(**metric_inputs, **self.METRIC_KWARGS) + + if strategy == "bootstrap": + metric_keys = result.keys() + bootstrap_dict = self._compute_confidence_interval( + metric, + metric_inputs, + metric_keys, + confidence_level, + n_resamples, + random_state, + ) + for key in metric_keys: + bootstrap_dict[key]["score"] = result[key] + + return bootstrap_dict + + return result diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/image_classification.py b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/image_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..82831458bb8789ce9c9418d6c19d4af4ba5b35a2 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/image_classification.py @@ -0,0 +1,119 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from numbers import Number +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Union + +from datasets import Dataset +from typing_extensions import Literal + +from ..module import EvaluationModule +from ..utils.file_utils import add_end_docstrings, add_start_docstrings +from .base import EVALUATOR_COMPUTE_RETURN_DOCSTRING, EVALUTOR_COMPUTE_START_DOCSTRING, Evaluator + + +if TYPE_CHECKING: + from transformers import FeatureExtractionMixin, Pipeline, PreTrainedModel, PreTrainedTokenizer, TFPreTrainedModel + + +TASK_DOCUMENTATION = r""" + Examples: + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + >>> task_evaluator = evaluator("image-classification") + >>> data = load_dataset("beans", split="test[:40]") + >>> results = task_evaluator.compute( + >>> model_or_pipeline="nateraw/vit-base-beans", + >>> data=data, + >>> label_column="labels", + >>> metric="accuracy", + >>> label_mapping={'angular_leaf_spot': 0, 'bean_rust': 1, 'healthy': 2}, + >>> strategy="bootstrap" + >>> ) + ``` +""" + + +class ImageClassificationEvaluator(Evaluator): + """ + Image classification evaluator. + This image classification evaluator can currently be loaded from [`evaluator`] using the default task name + `image-classification`. + Methods in this class assume a data format compatible with the [`ImageClassificationPipeline`]. + """ + + PIPELINE_KWARGS = {} + + def __init__(self, task="image-classification", default_metric_name=None): + super().__init__(task, default_metric_name=default_metric_name) + + def predictions_processor(self, predictions, label_mapping): + pred_label = [max(pred, key=lambda x: x["score"])["label"] for pred in predictions] + pred_label = [label_mapping[pred] if label_mapping is not None else pred for pred in pred_label] + + return {"predictions": pred_label} + + @add_start_docstrings(EVALUTOR_COMPUTE_START_DOCSTRING) + @add_end_docstrings(EVALUATOR_COMPUTE_RETURN_DOCSTRING, TASK_DOCUMENTATION) + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: Optional[str] = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + feature_extractor: Optional[Union[str, "FeatureExtractionMixin"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: int = None, + random_state: Optional[int] = None, + input_column: str = "image", + label_column: str = "label", + label_mapping: Optional[Dict[str, Number]] = None, + ) -> Tuple[Dict[str, float], Any]: + + """ + input_column (`str`, defaults to `"image"`): + The name of the column containing the images as PIL ImageFile in the dataset specified by `data`. + label_column (`str`, defaults to `"label"`): + The name of the column containing the labels in the dataset specified by `data`. + label_mapping (`Dict[str, Number]`, *optional*, defaults to `None`): + We want to map class labels defined by the model in the pipeline to values consistent with those + defined in the `label_column` of the `data` dataset. + """ + + result = super().compute( + model_or_pipeline=model_or_pipeline, + data=data, + subset=subset, + split=split, + metric=metric, + tokenizer=tokenizer, + feature_extractor=feature_extractor, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + device=device, + random_state=random_state, + input_column=input_column, + label_column=label_column, + label_mapping=label_mapping, + ) + + return result diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/question_answering.py b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/question_answering.py new file mode 100644 index 0000000000000000000000000000000000000000..99b4190eebdda4e90617d0979fe23af2965d3204 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/question_answering.py @@ -0,0 +1,239 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union + +# Lint as: python3 +from datasets import Dataset + + +try: + TRANSFORMERS_AVAILABLE = True +except ImportError: + TRANSFORMERS_AVAILABLE = False + +from typing_extensions import Literal + +from ..module import EvaluationModule +from ..utils.file_utils import add_end_docstrings, add_start_docstrings +from ..utils.logging import get_logger +from .base import EVALUATOR_COMPUTE_RETURN_DOCSTRING, EVALUTOR_COMPUTE_START_DOCSTRING, Evaluator +from .utils import DatasetColumn + + +if TYPE_CHECKING: + from transformers import Pipeline, PreTrainedModel, PreTrainedTokenizer, TFPreTrainedModel + + +logger = get_logger(__name__) + + +TASK_DOCUMENTATION = r""" + Examples: + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + >>> task_evaluator = evaluator("question-answering") + >>> data = load_dataset("squad", split="validation[:2]") + >>> results = task_evaluator.compute( + >>> model_or_pipeline="sshleifer/tiny-distilbert-base-cased-distilled-squad", + >>> data=data, + >>> metric="squad", + >>> ) + ``` + + + + Datasets where the answer may be missing in the context are supported, for example SQuAD v2 dataset. In this case, it is safer to pass `squad_v2_format=True` to + the compute() call. + + + + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + >>> task_evaluator = evaluator("question-answering") + >>> data = load_dataset("squad_v2", split="validation[:2]") + >>> results = task_evaluator.compute( + >>> model_or_pipeline="mrm8488/bert-tiny-finetuned-squadv2", + >>> data=data, + >>> metric="squad_v2", + >>> squad_v2_format=True, + >>> ) + ``` +""" + + +class QuestionAnsweringEvaluator(Evaluator): + """ + Question answering evaluator. This evaluator handles + [**extractive** question answering](https://huggingface.co/docs/transformers/task_summary#extractive-question-answering), + where the answer to the question is extracted from a context. + + This question answering evaluator can currently be loaded from [`evaluator`] using the default task name + `question-answering`. + + Methods in this class assume a data format compatible with the + [`~transformers.QuestionAnsweringPipeline`]. + """ + + PIPELINE_KWARGS = {} + + def __init__(self, task="question-answering", default_metric_name=None): + super().__init__(task, default_metric_name=default_metric_name) + + def prepare_data( + self, data: Dataset, question_column: str, context_column: str, id_column: str, label_column: str + ): + """Prepare data.""" + if data is None: + raise ValueError( + "Please specify a valid `data` object - either a `str` with a name or a `Dataset` object." + ) + self.check_required_columns( + data, + { + "question_column": question_column, + "context_column": context_column, + "id_column": id_column, + "label_column": label_column, + }, + ) + + metric_inputs = dict() + metric_inputs["references"] = [ + {"id": element[id_column], "answers": element[label_column]} for element in data + ] + + return metric_inputs, { + "question": DatasetColumn(data, question_column), + "context": DatasetColumn(data, context_column), + } + + def is_squad_v2_format(self, data: Dataset, label_column: str = "answers"): + """ + Check if the provided dataset follows the squad v2 data schema, namely possible samples where the answer is not in the context. + In this case, the answer text list should be `[]`. + """ + original_num_rows = data.num_rows + nonempty_num_rows = data.filter( + lambda x: len(x[label_column]["text"]) > 0, load_from_cache_file=False + ).num_rows + if original_num_rows > nonempty_num_rows: + return True + else: + return False + + def predictions_processor(self, predictions: List, squad_v2_format: bool, ids: List): + result = [] + for i in range(len(predictions)): + pred = {"prediction_text": predictions[i]["answer"], "id": ids[i]} + if squad_v2_format: + pred["no_answer_probability"] = predictions[i]["score"] + result.append(pred) + return {"predictions": result} + + @add_start_docstrings(EVALUTOR_COMPUTE_START_DOCSTRING) + @add_end_docstrings(EVALUATOR_COMPUTE_RETURN_DOCSTRING, TASK_DOCUMENTATION) + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: Optional[str] = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: int = None, + random_state: Optional[int] = None, + question_column: str = "question", + context_column: str = "context", + id_column: str = "id", + label_column: str = "answers", + squad_v2_format: Optional[bool] = None, + ) -> Tuple[Dict[str, float], Any]: + """ + question_column (`str`, defaults to `"question"`): + The name of the column containing the question in the dataset specified by `data`. + context_column (`str`, defaults to `"context"`): + The name of the column containing the context in the dataset specified by `data`. + id_column (`str`, defaults to `"id"`): + The name of the column containing the identification field of the question and answer pair in the + dataset specified by `data`. + label_column (`str`, defaults to `"answers"`): + The name of the column containing the answers in the dataset specified by `data`. + squad_v2_format (`bool`, *optional*, defaults to `None`): + Whether the dataset follows the format of squad_v2 dataset. This is the case when the provided dataset + has questions where the answer is not in the context, more specifically when are answers as + `{"text": [], "answer_start": []}` in the answer column. If all questions have at least one answer, this parameter + should be set to `False`. If this parameter is not provided, the format will be automatically inferred. + """ + result = {} + self.check_for_mismatch_in_device_setup(device, model_or_pipeline) + + data = self.load_data(data=data, subset=subset, split=split) + metric_inputs, pipe_inputs = self.prepare_data( + data=data, + question_column=question_column, + context_column=context_column, + id_column=id_column, + label_column=label_column, + ) + + if squad_v2_format is None: + squad_v2_format = self.is_squad_v2_format(data=data, label_column=label_column) + logger.warning( + f"`squad_v2_format` parameter not provided to QuestionAnsweringEvaluator.compute(). Automatically inferred `squad_v2_format` as {squad_v2_format}." + ) + pipe = self.prepare_pipeline(model_or_pipeline=model_or_pipeline, tokenizer=tokenizer, device=device) + + metric = self.prepare_metric(metric) + + if squad_v2_format and metric.name == "squad": + logger.warning( + "The dataset has SQuAD v2 format but you are using the SQuAD metric. Consider passing the 'squad_v2' metric." + ) + if not squad_v2_format and metric.name == "squad_v2": + logger.warning( + "The dataset has SQuAD v1 format but you are using the SQuAD v2 metric. Consider passing the 'squad' metric." + ) + + if squad_v2_format: + self.PIPELINE_KWARGS["handle_impossible_answer"] = True + else: + self.PIPELINE_KWARGS["handle_impossible_answer"] = False + + # Compute predictions + predictions, perf_results = self.call_pipeline(pipe, **pipe_inputs) + predictions = self.predictions_processor(predictions, squad_v2_format=squad_v2_format, ids=data[id_column]) + metric_inputs.update(predictions) + + # Compute metrics from references and predictions + metric_results = self.compute_metric( + metric=metric, + metric_inputs=metric_inputs, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + random_state=random_state, + ) + + result.update(metric_results) + result.update(perf_results) + + return result diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/text2text_generation.py b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/text2text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..6dfd2c035695b38c1e4f0d9d4929b12c6be30920 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/text2text_generation.py @@ -0,0 +1,267 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Union + +from datasets import Dataset +from typing_extensions import Literal + +from ..module import EvaluationModule +from ..utils.file_utils import add_start_docstrings +from .base import EVALUATOR_COMPUTE_RETURN_DOCSTRING, EVALUTOR_COMPUTE_START_DOCSTRING, Evaluator + + +if TYPE_CHECKING: + from transformers import Pipeline, PreTrainedModel, PreTrainedTokenizer, TFPreTrainedModel + + +TASK_DOCUMENTATION_KWARGS = r""" + input_column (`str`, defaults to `"text"`): + the name of the column containing the input text in the dataset specified by `data`. + label_column (`str`, defaults to `"label"`): + the name of the column containing the labels in the dataset specified by `data`. + generation_kwargs (`Dict`, *optional*, defaults to `None`): + The generation kwargs are passed to the pipeline and set the text generation strategy. +""" + +TEXT2TEXT_TASK_DOCSTRING_EXAMPLE = r""" + Examples: + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + >>> task_evaluator = evaluator("text2text-generation") + >>> data = load_dataset("cnn_dailymail", "3.0.0", split="validation[:40]") + >>> results = task_evaluator.compute( + >>> model_or_pipeline="facebook/bart-large-cnn", + >>> data=data, + >>> input_column="article", + >>> label_column="highlights", + >>> metric="rouge", + >>> ) + ``` +""" + +SUMMARIZATION_TASK_DOCSTRING_EXAMPLE = r""" + Examples: + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + >>> task_evaluator = evaluator("summarization") + >>> data = load_dataset("cnn_dailymail", "3.0.0", split="validation[:40]") + >>> results = task_evaluator.compute( + >>> model_or_pipeline="facebook/bart-large-cnn", + >>> data=data, + >>> input_column="article", + >>> label_column="highlights", + >>> ) + ``` +""" + + +TRANSLATION_TASK_DOCSTRING_EXAMPLE = r""" + Examples: + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + >>> task_evaluator = evaluator("translation") + >>> data = load_dataset("wmt19", "fr-de", split="validation[:40]") + >>> data = data.map(lambda x: {"text": x["translation"]["de"], "label": x["translation"]["fr"]}) + >>> results = task_evaluator.compute( + >>> model_or_pipeline="Helsinki-NLP/opus-mt-de-fr", + >>> data=data, + >>> ) + ``` +""" + + +class Text2TextGenerationEvaluator(Evaluator): + """ + Text2Text generation evaluator. + This Text2Text generation evaluator can currently be loaded from [`evaluator`] using the default task name + `text2text-generation`. + Methods in this class assume a data format compatible with the [`~transformers.Text2TextGenerationPipeline`]. + """ + + PREDICTION_PREFIX = "generated" + PIPELINE_KWARGS = {"truncation": True} + + def __init__(self, task="text2text-generation", default_metric_name=None): + super().__init__(task, default_metric_name=default_metric_name) + + def predictions_processor(self, predictions, label_mapping): + return {"predictions": [pred[f"{self.PREDICTION_PREFIX}_text"] for pred in predictions]} + + @add_start_docstrings( + EVALUTOR_COMPUTE_START_DOCSTRING, + TASK_DOCUMENTATION_KWARGS, + EVALUATOR_COMPUTE_RETURN_DOCSTRING, + TEXT2TEXT_TASK_DOCSTRING_EXAMPLE, + ) + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: Optional[str] = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: int = None, + random_state: Optional[int] = None, + input_column: str = "text", + label_column: str = "label", + generation_kwargs: dict = None, + ) -> Tuple[Dict[str, float], Any]: + if generation_kwargs is not None: + self.PIPELINE_KWARGS.update(generation_kwargs) + + result = super().compute( + model_or_pipeline=model_or_pipeline, + data=data, + subset=subset, + split=split, + metric=metric, + tokenizer=tokenizer, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + device=device, + random_state=random_state, + input_column=input_column, + label_column=label_column, + ) + + return result + + +class SummarizationEvaluator(Text2TextGenerationEvaluator): + """ + Text summarization evaluator. + This text summarization evaluator can currently be loaded from [`evaluator`] using the default task name + `summarization`. + Methods in this class assume a data format compatible with the [`SummarizationEvaluator`]. + """ + + PREDICTION_PREFIX = "summary" + PIPELINE_KWARGS = {"truncation": True} + + def __init__(self, task="summarization", default_metric_name=None): + super().__init__(task, default_metric_name=default_metric_name) + + @add_start_docstrings( + EVALUTOR_COMPUTE_START_DOCSTRING, + TASK_DOCUMENTATION_KWARGS, + EVALUATOR_COMPUTE_RETURN_DOCSTRING, + SUMMARIZATION_TASK_DOCSTRING_EXAMPLE, + ) + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: Optional[str] = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: int = None, + random_state: Optional[int] = None, + input_column: str = "text", + label_column: str = "label", + generation_kwargs: dict = None, + ) -> Tuple[Dict[str, float], Any]: + result = super().compute( + model_or_pipeline=model_or_pipeline, + data=data, + subset=subset, + split=split, + metric=metric, + tokenizer=tokenizer, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + device=device, + random_state=random_state, + input_column=input_column, + label_column=label_column, + generation_kwargs=generation_kwargs, + ) + + return result + + +class TranslationEvaluator(Text2TextGenerationEvaluator): + """ + Translation evaluator. + This translation generation evaluator can currently be loaded from [`evaluator`] using the default task name + `translation`. + Methods in this class assume a data format compatible with the [`~transformers.TranslationPipeline`]. + """ + + PREDICTION_PREFIX = "translation" + PIPELINE_KWARGS = {"truncation": True} + + def __init__(self, task="translation", default_metric_name=None): + super().__init__(task, default_metric_name=default_metric_name) + + @add_start_docstrings( + EVALUTOR_COMPUTE_START_DOCSTRING, + TASK_DOCUMENTATION_KWARGS, + EVALUATOR_COMPUTE_RETURN_DOCSTRING, + TRANSLATION_TASK_DOCSTRING_EXAMPLE, + ) + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: Optional[str] = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: int = None, + random_state: Optional[int] = None, + input_column: str = "text", + label_column: str = "label", + generation_kwargs: dict = None, + ) -> Tuple[Dict[str, float], Any]: + result = super().compute( + model_or_pipeline=model_or_pipeline, + data=data, + subset=subset, + split=split, + metric=metric, + tokenizer=tokenizer, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + device=device, + random_state=random_state, + input_column=input_column, + label_column=label_column, + generation_kwargs=generation_kwargs, + ) + + return result diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/text_classification.py b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/text_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..200eb01d70336148db473edebebc96e3137c5799 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/text_classification.py @@ -0,0 +1,160 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from numbers import Number +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Union + +from datasets import Dataset, load_dataset +from typing_extensions import Literal + +from ..module import EvaluationModule +from ..utils.file_utils import add_end_docstrings, add_start_docstrings +from .base import EVALUATOR_COMPUTE_RETURN_DOCSTRING, EVALUTOR_COMPUTE_START_DOCSTRING, Evaluator +from .utils import DatasetColumnPair + + +if TYPE_CHECKING: + from transformers import FeatureExtractionMixin, Pipeline, PreTrainedModel, PreTrainedTokenizer, TFPreTrainedModel + + +TASK_DOCUMENTATION = r""" + Examples: + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + >>> task_evaluator = evaluator("text-classification") + >>> data = load_dataset("imdb", split="test[:2]") + >>> results = task_evaluator.compute( + >>> model_or_pipeline="huggingface/prunebert-base-uncased-6-finepruned-w-distil-mnli", + >>> data=data, + >>> metric="accuracy", + >>> label_mapping={"LABEL_0": 0.0, "LABEL_1": 1.0}, + >>> strategy="bootstrap", + >>> n_resamples=10, + >>> random_state=0 + >>> ) + ``` +""" + + +class TextClassificationEvaluator(Evaluator): + """ + Text classification evaluator. + This text classification evaluator can currently be loaded from [`evaluator`] using the default task name + `text-classification` or with a `"sentiment-analysis"` alias. + Methods in this class assume a data format compatible with the [`~transformers.TextClassificationPipeline`] - a single textual + feature as input and a categorical label as output. + """ + + PIPELINE_KWARGS = {"truncation": True} + + def __init__(self, task="text-classification", default_metric_name=None): + super().__init__(task, default_metric_name=default_metric_name) + + def prepare_data(self, data: Union[str, Dataset], input_column: str, second_input_column: str, label_column: str): + if data is None: + raise ValueError( + "Please specify a valid `data` object - either a `str` with a name or a `Dataset` object." + ) + + self.check_required_columns(data, {"input_column": input_column, "label_column": label_column}) + + if second_input_column is not None: + self.check_required_columns(data, {"second_input_column": second_input_column}) + + data = load_dataset(data) if isinstance(data, str) else data + + return {"references": data[label_column]}, DatasetColumnPair( + data, input_column, second_input_column, "text", "text_pair" + ) + + def predictions_processor(self, predictions, label_mapping): + predictions = [ + label_mapping[element["label"]] if label_mapping is not None else element["label"] + for element in predictions + ] + return {"predictions": predictions} + + @add_start_docstrings(EVALUTOR_COMPUTE_START_DOCSTRING) + @add_end_docstrings(EVALUATOR_COMPUTE_RETURN_DOCSTRING, TASK_DOCUMENTATION) + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: Optional[str] = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + feature_extractor: Optional[Union[str, "FeatureExtractionMixin"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: int = None, + random_state: Optional[int] = None, + input_column: str = "text", + second_input_column: Optional[str] = None, + label_column: str = "label", + label_mapping: Optional[Dict[str, Number]] = None, + ) -> Tuple[Dict[str, float], Any]: + """ + input_column (`str`, *optional*, defaults to `"text"`): + The name of the column containing the text feature in the dataset specified by `data`. + second_input_column (`str`, *optional*, defaults to `None`): + The name of the second column containing the text features. This may be useful for classification tasks + as MNLI, where two columns are used. + label_column (`str`, defaults to `"label"`): + The name of the column containing the labels in the dataset specified by `data`. + label_mapping (`Dict[str, Number]`, *optional*, defaults to `None`): + We want to map class labels defined by the model in the pipeline to values consistent with those + defined in the `label_column` of the `data` dataset. + """ + + result = {} + + self.check_for_mismatch_in_device_setup(device, model_or_pipeline) + + # Prepare inputs + data = self.load_data(data=data, subset=subset, split=split) + metric_inputs, pipe_inputs = self.prepare_data( + data=data, input_column=input_column, second_input_column=second_input_column, label_column=label_column + ) + pipe = self.prepare_pipeline( + model_or_pipeline=model_or_pipeline, + tokenizer=tokenizer, + feature_extractor=feature_extractor, + device=device, + ) + metric = self.prepare_metric(metric) + + # Compute predictions + predictions, perf_results = self.call_pipeline(pipe, pipe_inputs) + predictions = self.predictions_processor(predictions, label_mapping) + metric_inputs.update(predictions) + + # Compute metrics from references and predictions + metric_results = self.compute_metric( + metric=metric, + metric_inputs=metric_inputs, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + random_state=random_state, + ) + + result.update(metric_results) + result.update(perf_results) + + return result diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/text_generation.py b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..15522e860f7eb6fc693780f637337c0fdb22a21c --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/text_generation.py @@ -0,0 +1,69 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Tuple + +from datasets import Dataset + +from .base import Evaluator +from .utils import DatasetColumn + + +TASK_DOCUMENTATION_KWARGS = r""" + input_column (`str`, defaults to `"text"`): + the name of the column containing the input text in the dataset specified by `data`. + generation_kwargs (`Dict`, *optional*, defaults to `None`): + The generation kwargs are passed to the pipeline and set the text generation strategy. +""" + + +class TextGenerationEvaluator(Evaluator): + """ + Text generation evaluator. + This Text generation evaluator can currently be loaded from [`evaluator`] using the default task name + `text-generation`. + Methods in this class assume a data format compatible with the [`~transformers.TextGenerationPipeline`]. + """ + + def predictions_processor(self, predictions, *args, **kwargs): + """ + Args: + predictions: A list of lists of dicts + + Returns: + `dict`: All the generated texts are flattened and stored under the "data" key. + """ + return {"data": [pred[f"{self.predictions_prefix}_text"] for pred_list in predictions for pred in pred_list]} + + def __init__(self, task="text-generation", default_metric_name=None, predictions_prefix: str = "generated"): + super().__init__(task=task, default_metric_name=default_metric_name) + self.predictions_prefix = predictions_prefix + + def prepare_data(self, data: Dataset, input_column: str, *args, **kwargs) -> Tuple[Dict, DatasetColumn]: + """ + Prepare data. + + Args: + data ([`Dataset`]): + Specifies the dataset we will run evaluation on. + input_column (`str`, defaults to `"text"`): + The name of the column containing the text feature in the dataset specified by `data`. + Returns: + `dict`: metric inputs. + `list`: pipeline inputs. + """ + + self.check_required_columns(data, {"input_column": input_column}) + + return {}, DatasetColumn(data, input_column) diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/token_classification.py b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/token_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..ba08ebd58d72417eed4e20c93a46c53adaa49811 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/token_classification.py @@ -0,0 +1,278 @@ +# Copyright 2022 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union + +from datasets import ClassLabel, Dataset, Sequence +from typing_extensions import Literal + +from ..module import EvaluationModule +from ..utils.file_utils import add_end_docstrings, add_start_docstrings +from .base import EVALUATOR_COMPUTE_RETURN_DOCSTRING, EVALUTOR_COMPUTE_START_DOCSTRING, Evaluator +from .utils import DatasetColumn + + +if TYPE_CHECKING: + from transformers import Pipeline, PreTrainedModel, PreTrainedTokenizer, TFPreTrainedModel + + +TASK_DOCUMENTATION = r""" + The dataset input and label columns are expected to be formatted as a list of words and a list of labels respectively, following [conll2003 dataset](https://huggingface.co/datasets/conll2003). Datasets whose inputs are single strings, and labels are a list of offset are not supported. + + Examples: + ```python + >>> from evaluate import evaluator + >>> from datasets import load_dataset + >>> task_evaluator = evaluator("token-classification") + >>> data = load_dataset("conll2003", split="validation[:2]") + >>> results = task_evaluator.compute( + >>> model_or_pipeline="elastic/distilbert-base-uncased-finetuned-conll03-english", + >>> data=data, + >>> metric="seqeval", + >>> ) + ``` + + + + For example, the following dataset format is accepted by the evaluator: + + ```python + dataset = Dataset.from_dict( + mapping={ + "tokens": [["New", "York", "is", "a", "city", "and", "Felix", "a", "person", "."]], + "ner_tags": [[1, 2, 0, 0, 0, 0, 3, 0, 0, 0]], + }, + features=Features({ + "tokens": Sequence(feature=Value(dtype="string")), + "ner_tags": Sequence(feature=ClassLabel(names=["O", "B-LOC", "I-LOC", "B-PER", "I-PER"])), + }), + ) + ``` + + + + + + For example, the following dataset format is **not** accepted by the evaluator: + + ```python + dataset = Dataset.from_dict( + mapping={ + "tokens": [["New York is a city and Felix a person."]], + "starts": [[0, 23]], + "ends": [[7, 27]], + "ner_tags": [["LOC", "PER"]], + }, + features=Features({ + "tokens": Value(dtype="string"), + "starts": Sequence(feature=Value(dtype="int32")), + "ends": Sequence(feature=Value(dtype="int32")), + "ner_tags": Sequence(feature=Value(dtype="string")), + }), + ) + ``` + + +""" + + +class TokenClassificationEvaluator(Evaluator): + """ + Token classification evaluator. + + This token classification evaluator can currently be loaded from [`evaluator`] using the default task name + `token-classification`. + + Methods in this class assume a data format compatible with the [`~transformers.TokenClassificationPipeline`]. + """ + + PIPELINE_KWARGS = {"ignore_labels": []} + + def __init__(self, task="token-classification", default_metric_name=None): + super().__init__(task, default_metric_name=default_metric_name) + + def predictions_processor(self, predictions: List[List[Dict]], words: List[List[str]], join_by: str): + """ + Transform the pipeline predictions into a list of predicted labels of the same length as the true labels. + + Args: + predictions (`List[List[Dict]]`): + List of pipeline predictions, where each token has been labeled. + words (`List[List[str]]`): + Original input data to the pipeline, used to build predicted labels of the same length. + join_by (`str`): + String to use to join two words. In English, it will typically be " ". + + Returns: + `dict`: a dictionary holding the predictions + """ + preds = [] + + # iterate over the data rows + for i, prediction in enumerate(predictions): + pred_processed = [] + + # get a list of tuples giving the indexes of the start and end character of each word + words_offsets = self.words_to_offsets(words[i], join_by) + + token_index = 0 + for word_offset in words_offsets: + # for each word, we may keep only the predicted label for the first token, discard the others + while prediction[token_index]["start"] < word_offset[0]: + token_index += 1 + + if prediction[token_index]["start"] > word_offset[0]: # bad indexing + pred_processed.append("O") + elif prediction[token_index]["start"] == word_offset[0]: + pred_processed.append(prediction[token_index]["entity"]) + + preds.append(pred_processed) + + return {"predictions": preds} + + def words_to_offsets(self, words: List[str], join_by: str): + """ + Convert a list of words to a list of offsets, where word are joined by `join_by`. + + Args: + words (`List[str]`): + List of words to get offsets from. + join_by (`str`): + String to insert between words. + + Returns: + `List[Tuple[int, int]]`: List of the characters (start index, end index) for each of the words. + """ + offsets = [] + + start = 0 + for word in words: + end = start + len(word) - 1 + offsets.append((start, end)) + start = end + len(join_by) + 1 + + return offsets + + def prepare_data(self, data: Union[str, Dataset], input_column: str, label_column: str, join_by: str): + super().prepare_data(data, input_column, label_column) + + if not isinstance(data.features[input_column], Sequence) or not isinstance( + data.features[label_column], Sequence + ): + raise ValueError( + "TokenClassificationEvaluator expects the input and label columns to be provided as lists." + ) + + # If the labels are of type ClassLabel, they are already integers and we have the map stored somewhere. + # Otherwise, we have to get the list of labels manually. + labels_are_int = isinstance(data.features[label_column].feature, ClassLabel) + if labels_are_int: + label_list = data.features[label_column].feature.names # list of string labels + id_to_label = {i: label for i, label in enumerate(label_list)} + references = [[id_to_label[label_id] for label_id in label_ids] for label_ids in data[label_column]] + elif data.features[label_column].feature.dtype.startswith("int"): + raise NotImplementedError( + "References provided as integers, but the reference column is not a Sequence of ClassLabels." + ) + else: + # In the event the labels are not a `Sequence[ClassLabel]`, we have already labels as strings + # An example is labels as ["PER", "PER", "O", "LOC", "O", "LOC", "O"], e.g. in polyglot_ner dataset + references = data[label_column] + + metric_inputs = {"references": references} + data = data.map(lambda x: {input_column: join_by.join(x[input_column])}) + pipeline_inputs = DatasetColumn(data, input_column) + + return metric_inputs, pipeline_inputs + + def prepare_pipeline( + self, + model_or_pipeline: Union[str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel"], # noqa: F821 + tokenizer: Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"] = None, # noqa: F821 + feature_extractor: Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"] = None, # noqa: F821 + device: int = None, + ): + pipe = super().prepare_pipeline(model_or_pipeline, tokenizer, feature_extractor, device) + + # check the pipeline outputs start characters in its predictions + dummy_output = pipe(["2003 New York Gregory"], **self.PIPELINE_KWARGS) + if dummy_output[0][0]["start"] is None: + raise ValueError( + "TokenClassificationEvaluator supports only pipelines giving 'start' index as a pipeline output (got None). " + "Transformers pipelines with a slow tokenizer will raise this error." + ) + + return pipe + + @add_start_docstrings(EVALUTOR_COMPUTE_START_DOCSTRING) + @add_end_docstrings(EVALUATOR_COMPUTE_RETURN_DOCSTRING, TASK_DOCUMENTATION) + def compute( + self, + model_or_pipeline: Union[ + str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel" # noqa: F821 + ] = None, + data: Union[str, Dataset] = None, + subset: Optional[str] = None, + split: str = None, + metric: Union[str, EvaluationModule] = None, + tokenizer: Optional[Union[str, "PreTrainedTokenizer"]] = None, # noqa: F821 + strategy: Literal["simple", "bootstrap"] = "simple", + confidence_level: float = 0.95, + n_resamples: int = 9999, + device: Optional[int] = None, + random_state: Optional[int] = None, + input_column: str = "tokens", + label_column: str = "ner_tags", + join_by: Optional[str] = " ", + ) -> Tuple[Dict[str, float], Any]: + """ + input_column (`str`, defaults to `"tokens"`): + The name of the column containing the tokens feature in the dataset specified by `data`. + label_column (`str`, defaults to `"label"`): + The name of the column containing the labels in the dataset specified by `data`. + join_by (`str`, *optional*, defaults to `" "`): + This evaluator supports dataset whose input column is a list of words. This parameter specifies how to join + words to generate a string input. This is especially useful for languages that do not separate words by a space. + """ + result = {} + + self.check_for_mismatch_in_device_setup(device, model_or_pipeline) + + # Prepare inputs + data = self.load_data(data=data, subset=subset, split=split) + metric_inputs, pipe_inputs = self.prepare_data( + data=data, input_column=input_column, label_column=label_column, join_by=join_by + ) + pipe = self.prepare_pipeline(model_or_pipeline=model_or_pipeline, tokenizer=tokenizer, device=device) + metric = self.prepare_metric(metric) + + # Compute predictions + predictions, perf_results = self.call_pipeline(pipe, pipe_inputs) + predictions = self.predictions_processor(predictions, data[input_column], join_by) + metric_inputs.update(predictions) + + # Compute metrics from references and predictions + metric_results = self.compute_metric( + metric=metric, + metric_inputs=metric_inputs, + strategy=strategy, + confidence_level=confidence_level, + n_resamples=n_resamples, + random_state=random_state, + ) + + result.update(metric_results) + result.update(perf_results) + + return result diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/utils.py b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e364276d008b689d726b8dbbea1402fa93886d9b --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/evaluator/utils.py @@ -0,0 +1,84 @@ +from datasets import Dataset, get_dataset_split_names + + +class DatasetColumn(list): + """Helper class to avoid loading a dataset column into memory when accessing it.""" + + def __init__(self, dataset: Dataset, key: str): + self.dataset = dataset + self.key = key + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, i): + return self.dataset[i][self.key] + + def __iter__(self): + return (self.dataset[i][self.key] for i in range(len(self))) + + +def choose_split(data, subset=None): + available_splits = get_dataset_split_names(data, subset) + preferred_split_order = [ + "test", + "testing", + "eval", + "evaluation", + "validation", + "val", + "valid", + "dev", + "train", + "training", + ] + for split in preferred_split_order: + if split in available_splits: + return split + raise ValueError("No dataset split defined! Pass an explicit value to the `split` kwarg.") + + +class DatasetColumnPair(list): + """Helper class to avoid loading two dataset columns into memory when accessing it.""" + + def __init__( + self, + dataset: Dataset, + first_col: str, + second_col: str, + first_key: str, + second_key: str, + ): + """ + Args: + dataset (Dataset): dataset to build an iterator on + first_col (str): first column name to use in the dataset + second_col (str): second column name to use in the dataset + first_key (str): key name used for the first column in the returned dictionary + second_key (str): key name used for the second column in the returned dictionary + """ + self.dataset = dataset + + self.first_col = first_col + self.second_col = second_col + + self.first_key = first_key + self.second_key = second_key + + def __len__(self): + return len(self.dataset) + + def __getitem__(self, i): + return { + self.first_key: self.dataset[i][self.first_col], + self.second_key: self.dataset[i][self.second_col] if self.second_col else None, + } + + def __iter__(self): + return ( + { + self.first_key: self.dataset[i][self.first_col], + self.second_key: self.dataset[i][self.second_col] if self.second_col else None, + } + for i in range(len(self)) + ) diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/hub.py b/TextClassification/TextClassification/evaluate/src/evaluate/hub.py new file mode 100644 index 0000000000000000000000000000000000000000..86118332c6d2293f475e84e80726364dcc63e292 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/hub.py @@ -0,0 +1,133 @@ +from typing import Dict + +import requests +from huggingface_hub import dataset_info, model_info +from huggingface_hub.repocard import metadata_update + +from .config import HF_HUB_ALLOWED_TASKS +from .utils.logging import get_logger + + +logger = get_logger(__name__) + + +def push_to_hub( + model_id: str, + task_type: str, + dataset_type: str, + dataset_name: str, + metric_type: str, + metric_name: str, + metric_value: float, + task_name: str = None, + dataset_config: str = None, + dataset_split: str = None, + dataset_revision: str = None, + dataset_args: Dict[str, int] = None, + metric_config: str = None, + metric_args: Dict[str, int] = None, + overwrite: bool = False, +): + r""" + Pushes the result of a metric to the metadata of a model repository in the Hub. + + Args: + model_id (`str`): + Model id from https://hf.co/models. + task_type (`str`): + Task id, refer to the [Hub allowed tasks](https://github.com/huggingface/evaluate/blob/main/src/evaluate/config.py#L154) for allowed values. + dataset_type (`str`): + Dataset id from https://hf.co/datasets. + dataset_name (`str`): + Pretty name for the dataset. + metric_type (`str`): + Metric id from https://hf.co/metrics. + metric_name (`str`): + Pretty name for the metric. + metric_value (`float`): + Computed metric value. + task_name (`str`, *optional*): + Pretty name for the task. + dataset_config (`str`, *optional*): + Dataset configuration used in [`~datasets.load_dataset`]. + See [`~datasets.load_dataset`] for more info. + dataset_split (`str`, *optional*): + Name of split used for metric computation. + dataset_revision (`str`, *optional*): + Git hash for the specific version of the dataset. + dataset_args (`dict[str, int]`, *optional*): + Additional arguments passed to [`~datasets.load_dataset`]. + metric_config (`str`, *optional*): + Configuration for the metric (e.g. the GLUE metric has a configuration for each subset). + metric_args (`dict[str, int]`, *optional*): + Arguments passed during [`~evaluate.EvaluationModule.compute`]. + overwrite (`bool`, *optional*, defaults to `False`): + If set to `True` an existing metric field can be overwritten, otherwise + attempting to overwrite any existing fields will cause an error. + + Example: + + ```python + >>> push_to_hub( + ... model_id="huggingface/gpt2-wikitext2", + ... metric_value=0.5 + ... metric_type="bleu", + ... metric_name="BLEU", + ... dataset_name="WikiText", + ... dataset_type="wikitext", + ... dataset_split="test", + ... task_type="text-generation", + ... task_name="Text Generation" + ... ) + ```""" + if task_type not in HF_HUB_ALLOWED_TASKS: + raise ValueError(f"Task type not supported. Task has to be one of {HF_HUB_ALLOWED_TASKS}") + + try: + dataset_info(dataset_type) + except requests.exceptions.HTTPError: + logger.warning(f"Dataset {dataset_type} not found on the Hub at hf.co/datasets/{dataset_type}") + + try: + model_info(model_id) + except requests.exceptions.HTTPError: + raise ValueError(f"Model {model_id} not found on the Hub at hf.co/{model_id}") + + result = { + "task": { + "type": task_type, + }, + "dataset": { + "type": dataset_type, + "name": dataset_name, + }, + "metrics": [ + { + "type": metric_type, + "value": metric_value, + }, + ], + } + + if dataset_config is not None: + result["dataset"]["config"] = dataset_config + if dataset_split is not None: + result["dataset"]["split"] = dataset_split + if dataset_revision is not None: + result["dataset"]["revision"] = dataset_revision + if dataset_args is not None: + result["dataset"]["args"] = dataset_args + + if task_name is not None: + result["task"]["name"] = task_name + + if metric_name is not None: + result["metrics"][0]["name"] = metric_name + if metric_config is not None: + result["metrics"][0]["config"] = metric_config + if metric_args is not None: + result["metrics"][0]["args"] = metric_args + + metadata = {"model-index": [{"results": [result]}]} + + return metadata_update(repo_id=model_id, metadata=metadata, overwrite=overwrite) diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/info.py b/TextClassification/TextClassification/evaluate/src/evaluate/info.py new file mode 100644 index 0000000000000000000000000000000000000000..cc095784e4f1c1f473dd85955447d97d5fdc4e65 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/info.py @@ -0,0 +1,157 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +""" EvaluationModuleInfo records information we know about a dataset and a metric. +""" + +import dataclasses +import json +import os +from dataclasses import asdict, dataclass, field +from typing import List, Optional, Union + +from datasets.features import Features, Value + +from . import config +from .utils.logging import get_logger + + +logger = get_logger(__name__) + + +@dataclass +class EvaluationModuleInfo: + """Base class to store information about an evaluation used for `MetricInfo`, `ComparisonInfo`, + and `MeasurementInfo`. + + `EvaluationModuleInfo` documents an evaluation, including its name, version, and features. + See the constructor arguments and properties for a full list. + + Note: Not all fields are known on construction and may be updated later. + """ + + # Set in the dataset scripts + description: str + citation: str + features: Union[Features, List[Features]] + inputs_description: str = field(default_factory=str) + homepage: str = field(default_factory=str) + license: str = field(default_factory=str) + codebase_urls: List[str] = field(default_factory=list) + reference_urls: List[str] = field(default_factory=list) + streamable: bool = False + format: Optional[str] = None + module_type: str = "metric" # deprecate this in the future + + # Set later by the builder + module_name: Optional[str] = None + config_name: Optional[str] = None + experiment_id: Optional[str] = None + + def __post_init__(self): + if self.format is not None: + for key, value in self.features.items(): + if not isinstance(value, Value): + raise ValueError( + f"When using 'numpy' format, all features should be a `datasets.Value` feature. " + f"Here {key} is an instance of {value.__class__.__name__}" + ) + + def write_to_directory(self, metric_info_dir): + """Write `EvaluationModuleInfo` as JSON to `metric_info_dir`. + Also save the license separately in LICENSE. + + Args: + metric_info_dir (`str`): + The directory to save `metric_info_dir` to. + + Example: + + ```py + >>> my_metric.info.write_to_directory("/path/to/directory/") + ``` + """ + with open(os.path.join(metric_info_dir, config.METRIC_INFO_FILENAME), "w", encoding="utf-8") as f: + json.dump(asdict(self), f) + + with open(os.path.join(metric_info_dir, config.LICENSE_FILENAME), "w", encoding="utf-8") as f: + f.write(self.license) + + @classmethod + def from_directory(cls, metric_info_dir) -> "EvaluationModuleInfo": + """Create `EvaluationModuleInfo` from the JSON file in `metric_info_dir`. + + Args: + metric_info_dir (`str`): + The directory containing the `metric_info` JSON file. This + should be the root directory of a specific metric version. + + Example: + + ```py + >>> my_metric = EvaluationModuleInfo.from_directory("/path/to/directory/") + ``` + """ + logger.info(f"Loading Metric info from {metric_info_dir}") + if not metric_info_dir: + raise ValueError("Calling EvaluationModuleInfo.from_directory() with undefined metric_info_dir.") + + with open(os.path.join(metric_info_dir, config.METRIC_INFO_FILENAME), encoding="utf-8") as f: + metric_info_dict = json.load(f) + return cls.from_dict(metric_info_dict) + + @classmethod + def from_dict(cls, metric_info_dict: dict) -> "EvaluationModuleInfo": + field_names = {f.name for f in dataclasses.fields(cls)} + return cls(**{k: v for k, v in metric_info_dict.items() if k in field_names}) + + +@dataclass +class MetricInfo(EvaluationModuleInfo): + """Information about a metric. + + `EvaluationModuleInfo` documents a metric, including its name, version, and features. + See the constructor arguments and properties for a full list. + + Note: Not all fields are known on construction and may be updated later. + """ + + module_type: str = "metric" + + +@dataclass +class ComparisonInfo(EvaluationModuleInfo): + """Information about a comparison. + + `EvaluationModuleInfo` documents a comparison, including its name, version, and features. + See the constructor arguments and properties for a full list. + + Note: Not all fields are known on construction and may be updated later. + """ + + module_type: str = "comparison" + + +@dataclass +class MeasurementInfo(EvaluationModuleInfo): + """Information about a measurement. + + `EvaluationModuleInfo` documents a measurement, including its name, version, and features. + See the constructor arguments and properties for a full list. + + Note: Not all fields are known on construction and may be updated later. + """ + + module_type: str = "measurement" diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/inspect.py b/TextClassification/TextClassification/evaluate/src/evaluate/inspect.py new file mode 100644 index 0000000000000000000000000000000000000000..20e2af28ed4df4e99c6d67cccdd24dda1c8cecf9 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/inspect.py @@ -0,0 +1,129 @@ +# Copyright 2020 The HuggingFace Evaluate Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +""" List and inspect metrics.""" + +from typing import Optional + +import requests +from datasets import DownloadConfig + +from .config import EVALUATION_MODULE_TYPES, HF_LIST_ENDPOINT +from .loading import evaluation_module_factory +from .utils.logging import get_logger + + +logger = get_logger(__name__) + + +class SplitsNotFoundError(ValueError): + pass + + +def list_evaluation_modules(module_type=None, include_community=True, with_details=False): + """List all evaluation modules available on the Hugging Face Hub. + + Args: + module_type (`str`, *optional*, defaults to `None`): + Type of evaluation modules to list. Has to be one of `'metric'`, `'comparison'`, or `'measurement'`. If `None`, all types are listed. + include_community (`bool`, *optional*, defaults to `True`): + Include community modules in the list. + with_details (`bool`, *optional*, defaults to `False`): + Return the full details on the metrics instead of only the ID. + + Returns: + `List[Union[str, dict]]` + + Example: + + ```py + >>> from evaluate import list_evaluation_modules + >>> list_evaluation_modules(module_type="metric") + ``` + """ + + if module_type is None: + evaluations_list = [] + for module_type in EVALUATION_MODULE_TYPES: + evaluations_list.extend( + _list_evaluation_modules_type( + module_type, include_community=include_community, with_details=with_details + ) + ) + else: + if module_type not in EVALUATION_MODULE_TYPES: + raise ValueError(f"Invalid module type '{module_type}'. Has to be one of {EVALUATION_MODULE_TYPES}.") + evaluations_list = _list_evaluation_modules_type( + module_type, include_community=include_community, with_details=with_details + ) + return evaluations_list + + +def _list_evaluation_modules_type(module_type, include_community=True, with_details=False): + + r = requests.get(HF_LIST_ENDPOINT.format(type=module_type)) + r.raise_for_status() + d = r.json() + + if not include_community: + d = [element for element in d if element["id"].split("/")[0] == f"evaluate-{module_type}"] + + # remove namespace for canonical modules and add community tag + for element in d: + if element["id"].split("/")[0] == f"evaluate-{module_type}": + element["id"] = element["id"].split("/")[1] + element["community"] = False + else: + element["community"] = True + + if with_details: + return [ + { + "name": element["id"], + "type": module_type, + "community": element["community"], + "likes": element.get("likes", 0), + } + for element in d + ] + else: + return [element["id"] for element in d] + + +def inspect_evaluation_module( + path: str, local_path: str, download_config: Optional[DownloadConfig] = None, **download_kwargs +): + r""" + Allow inspection/modification of a evaluation script by copying it on local drive at local_path. + + Args: + path (``str``): path to the evaluation script. Can be either: + + - a local path to script or the directory containing the script (if the script has the same name as the directory), + e.g. ``'./metrics/accuracy'`` or ``'./metrics/accuracy/accuracy.py'`` + - a dataset identifier on the Hugging Face Hub (list all available datasets and ids with ``evaluate.list_evaluation_modules()``) + e.g. ``'accuracy'``, ``'bleu'`` or ``'word_length'`` + local_path (``str``): path to the local folder to copy the datset script to. + download_config (Optional ``datasets.DownloadConfig``: specific download configuration parameters. + **download_kwargs: optional attributes for DownloadConfig() which will override the attributes in download_config if supplied. + """ + evaluation_module = evaluation_module_factory( + path, download_config=download_config, force_local_path=local_path, **download_kwargs + ) + print( + f"The processing scripts for metric {path} can be inspected at {local_path}. " + f"The main class is in {evaluation_module.module_path}. " + f"You can modify this processing scripts and use it with `evaluate.load({local_path})`." + ) diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/loading.py b/TextClassification/TextClassification/evaluate/src/evaluate/loading.py new file mode 100644 index 0000000000000000000000000000000000000000..a9cbbe60de07fed296cd27fd1954ef14cf0de032 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/loading.py @@ -0,0 +1,771 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Access datasets.""" +import filecmp +import importlib +import inspect +import json +import os +import re +import shutil +import time +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Tuple, Type, Union +from urllib.parse import urlparse + +from datasets import DownloadConfig, DownloadMode +from datasets.builder import DatasetBuilder +from datasets.packaged_modules import _EXTENSION_TO_MODULE, _hash_python_lines +from datasets.utils.filelock import FileLock +from datasets.utils.version import Version + +from . import SCRIPTS_VERSION, config +from .module import EvaluationModule +from .utils.file_utils import ( + cached_path, + head_hf_s3, + hf_hub_url, + init_hf_modules, + is_relative_path, + relative_to_absolute_path, + url_or_path_join, +) +from .utils.logging import get_logger + + +logger = get_logger(__name__) + + +ALL_ALLOWED_EXTENSIONS = list(_EXTENSION_TO_MODULE.keys()) + ["zip"] + + +def init_dynamic_modules( + name: str = config.MODULE_NAME_FOR_DYNAMIC_MODULES, hf_modules_cache: Optional[Union[Path, str]] = None +): + """ + Create a module with name `name` in which you can add dynamic modules + such as metrics or datasets. The module can be imported using its name. + The module is created in the HF_MODULE_CACHE directory by default (~/.cache/huggingface/modules) but it can + be overriden by specifying a path to another directory in `hf_modules_cache`. + """ + hf_modules_cache = init_hf_modules(hf_modules_cache) + dynamic_modules_path = os.path.join(hf_modules_cache, name) + os.makedirs(dynamic_modules_path, exist_ok=True) + if not os.path.exists(os.path.join(dynamic_modules_path, "__init__.py")): + with open(os.path.join(dynamic_modules_path, "__init__.py"), "w"): + pass + return dynamic_modules_path + + +def import_main_class(module_path) -> Optional[Union[Type[DatasetBuilder], Type[EvaluationModule]]]: + """Import a module at module_path and return its main class, a Metric by default""" + module = importlib.import_module(module_path) + main_cls_type = EvaluationModule + + # Find the main class in our imported module + module_main_cls = None + for name, obj in module.__dict__.items(): + if isinstance(obj, type) and issubclass(obj, main_cls_type): + if inspect.isabstract(obj): + continue + module_main_cls = obj + break + + return module_main_cls + + +def files_to_hash(file_paths: List[str]) -> str: + """ + Convert a list of scripts or text files provided in file_paths into a hashed filename in a repeatable way. + """ + # List all python files in directories if directories are supplied as part of external imports + to_use_files: List[Union[Path, str]] = [] + for file_path in file_paths: + if os.path.isdir(file_path): + to_use_files.extend(list(Path(file_path).rglob("*.[pP][yY]"))) + else: + to_use_files.append(file_path) + + # Get the code from all these files + lines = [] + for file_path in to_use_files: + with open(file_path, encoding="utf-8") as f: + lines.extend(f.readlines()) + return _hash_python_lines(lines) + + +def convert_github_url(url_path: str) -> Tuple[str, Optional[str]]: + """Convert a link to a file on a github repo in a link to the raw github object.""" + parsed = urlparse(url_path) + sub_directory = None + if parsed.scheme in ("http", "https", "s3") and parsed.netloc == "github.com": + if "blob" in url_path: + if not url_path.endswith(".py"): + raise ValueError(f"External import from github at {url_path} should point to a file ending with '.py'") + url_path = url_path.replace("blob", "raw") # Point to the raw file + else: + # Parse github url to point to zip + github_path = parsed.path[1:] + repo_info, branch = github_path.split("/tree/") if "/tree/" in github_path else (github_path, "master") + repo_owner, repo_name = repo_info.split("/") + url_path = f"https://github.com/{repo_owner}/{repo_name}/archive/{branch}.zip" + sub_directory = f"{repo_name}-{branch}" + return url_path, sub_directory + + +def increase_load_count(name: str, resource_type: str): + """Update the download count of a dataset or metric.""" + if not config.HF_EVALUATE_OFFLINE and config.HF_UPDATE_DOWNLOAD_COUNTS: + try: + head_hf_s3(name, filename=name + ".py", dataset=(resource_type == "dataset")) + except Exception: + pass + + +def get_imports(file_path: str) -> Tuple[str, str, str, str]: + """Find whether we should import or clone additional files for a given processing script. + And list the import. + + We allow: + - library dependencies, + - local dependencies and + - external dependencies whose url is specified with a comment starting from "# From:' followed by the raw url to a file, an archive or a github repository. + external dependencies will be downloaded (and extracted if needed in the dataset folder). + We also add an `__init__.py` to each sub-folder of a downloaded folder so the user can import from them in the script. + + Note that only direct import in the dataset processing script will be handled + We don't recursively explore the additional import to download further files. + + Example:: + + import tensorflow + import .c4_utils + import .clicr.dataset-code.build_json_dataset # From: https://raw.githubusercontent.com/clips/clicr/master/dataset-code/build_json_dataset + """ + lines = [] + with open(file_path, encoding="utf-8") as f: + lines.extend(f.readlines()) + + logger.debug(f"Checking {file_path} for additional imports.") + imports: List[Tuple[str, str, str, Optional[str]]] = [] + is_in_docstring = False + for line in lines: + docstr_start_match = re.findall(r'[\s\S]*?"""[\s\S]*?', line) + + if len(docstr_start_match) == 1: + # flip True <=> False only if doctstring + # starts at line without finishing + is_in_docstring = not is_in_docstring + + if is_in_docstring: + # import statements in doctstrings should + # not be added as required dependencies + continue + + match = re.match(r"^import\s+(\.?)([^\s\.]+)[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*)", line, flags=re.MULTILINE) + if match is None: + match = re.match( + r"^from\s+(\.?)([^\s\.]+)(?:[^\s]*)\s+import\s+[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*)", + line, + flags=re.MULTILINE, + ) + if match is None: + continue + if match.group(1): + # The import starts with a '.', we will download the relevant file + if any(imp[1] == match.group(2) for imp in imports): + # We already have this import + continue + if match.group(3): + # The import has a comment with 'From:', we'll retrieve it from the given url + url_path = match.group(3) + url_path, sub_directory = convert_github_url(url_path) + imports.append(("external", match.group(2), url_path, sub_directory)) + elif match.group(2): + # The import should be at the same place as the file + imports.append(("internal", match.group(2), match.group(2), None)) + else: + if match.group(3): + # The import has a comment with `From: git+https:...`, asks user to pip install from git. + url_path = match.group(3) + imports.append(("library", match.group(2), url_path, None)) + else: + imports.append(("library", match.group(2), match.group(2), None)) + + return imports + + +def _download_additional_modules( + name: str, base_path: str, imports: Tuple[str, str, str, str], download_config: Optional[DownloadConfig] +) -> List[Tuple[str, str]]: + """ + Download additional module for a module .py at URL (or local path) /.py + The imports must have been parsed first using ``get_imports``. + + If some modules need to be installed with pip, an error is raised showing how to install them. + This function return the list of downloaded modules as tuples (import_name, module_file_path). + + The downloaded modules can then be moved into an importable directory with ``_copy_script_and_other_resources_in_importable_dir``. + """ + local_imports = [] + library_imports = [] + download_config = download_config.copy() + if download_config.download_desc is None: + download_config.download_desc = "Downloading extra modules" + for import_type, import_name, import_path, sub_directory in imports: + if import_type == "library": + library_imports.append((import_name, import_path)) # Import from a library + continue + + if import_name == name: + raise ValueError( + f"Error in the {name} script, importing relative {import_name} module " + f"but {import_name} is the name of the script. " + f"Please change relative import {import_name} to another name and add a '# From: URL_OR_PATH' " + f"comment pointing to the original relative import file path." + ) + if import_type == "internal": + url_or_filename = url_or_path_join(base_path, import_path + ".py") + elif import_type == "external": + url_or_filename = import_path + else: + raise ValueError("Wrong import_type") + + local_import_path = cached_path( + url_or_filename, + download_config=download_config, + ) + if sub_directory is not None: + local_import_path = os.path.join(local_import_path, sub_directory) + local_imports.append((import_name, local_import_path)) + + # Check library imports + needs_to_be_installed = set() + for library_import_name, library_import_path in library_imports: + try: + lib = importlib.import_module(library_import_name) # noqa F841 + except ImportError: + library_import_name = "scikit-learn" if library_import_name == "sklearn" else library_import_name + needs_to_be_installed.add((library_import_name, library_import_path)) + if needs_to_be_installed: + raise ImportError( + f"To be able to use {name}, you need to install the following dependencies" + f"{[lib_name for lib_name, lib_path in needs_to_be_installed]} using 'pip install " + f"{' '.join([lib_path for lib_name, lib_path in needs_to_be_installed])}' for instance'" + ) + return local_imports + + +def _copy_script_and_other_resources_in_importable_dir( + name: str, + importable_directory_path: str, + subdirectory_name: str, + original_local_path: str, + local_imports: List[Tuple[str, str]], + additional_files: List[Tuple[str, str]], + download_mode: Optional[DownloadMode], +) -> str: + """Copy a script and its required imports to an importable directory + + Args: + name (str): name of the resource to load + importable_directory_path (str): path to the loadable folder in the dynamic modules directory + subdirectory_name (str): name of the subdirectory in importable_directory_path in which to place the script + original_local_path (str): local path to the resource script + local_imports (List[Tuple[str, str]]): list of (destination_filename, import_file_to_copy) + additional_files (List[Tuple[str, str]]): list of (destination_filename, additional_file_to_copy) + download_mode (Optional[DownloadMode]): download mode + + Return: + importable_local_file: path to an importable module with importlib.import_module + """ + + # Define a directory with a unique name in our dataset or metric folder + # path is: ./datasets|metrics/dataset|metric_name/hash_from_code/script.py + # we use a hash as subdirectory_name to be able to have multiple versions of a dataset/metric processing file together + importable_subdirectory = os.path.join(importable_directory_path, subdirectory_name) + importable_local_file = os.path.join(importable_subdirectory, name + ".py") + + # Prevent parallel disk operations + lock_path = importable_directory_path + ".lock" + with FileLock(lock_path): + # Create main dataset/metrics folder if needed + if download_mode == DownloadMode.FORCE_REDOWNLOAD and os.path.exists(importable_directory_path): + shutil.rmtree(importable_directory_path) + os.makedirs(importable_directory_path, exist_ok=True) + + # add an __init__ file to the main dataset folder if needed + init_file_path = os.path.join(importable_directory_path, "__init__.py") + if not os.path.exists(init_file_path): + with open(init_file_path, "w"): + pass + + # Create hash dataset folder if needed + os.makedirs(importable_subdirectory, exist_ok=True) + # add an __init__ file to the hash dataset folder if needed + init_file_path = os.path.join(importable_subdirectory, "__init__.py") + if not os.path.exists(init_file_path): + with open(init_file_path, "w"): + pass + + # Copy dataset.py file in hash folder if needed + if not os.path.exists(importable_local_file): + shutil.copyfile(original_local_path, importable_local_file) + + # Record metadata associating original dataset path with local unique folder + meta_path = importable_local_file.split(".py")[0] + ".json" + if not os.path.exists(meta_path): + meta = {"original file path": original_local_path, "local file path": importable_local_file} + # the filename is *.py in our case, so better rename to filenam.json instead of filename.py.json + with open(meta_path, "w", encoding="utf-8") as meta_file: + json.dump(meta, meta_file) + + # Copy all the additional imports + for import_name, import_path in local_imports: + if os.path.isfile(import_path): + full_path_local_import = os.path.join(importable_subdirectory, import_name + ".py") + if not os.path.exists(full_path_local_import): + shutil.copyfile(import_path, full_path_local_import) + elif os.path.isdir(import_path): + full_path_local_import = os.path.join(importable_subdirectory, import_name) + if not os.path.exists(full_path_local_import): + shutil.copytree(import_path, full_path_local_import) + else: + raise OSError(f"Error with local import at {import_path}") + + # Copy aditional files like dataset infos file if needed + for file_name, original_path in additional_files: + destination_additional_path = os.path.join(importable_subdirectory, file_name) + if not os.path.exists(destination_additional_path) or not filecmp.cmp( + original_path, destination_additional_path + ): + shutil.copyfile(original_path, destination_additional_path) + return importable_local_file + + +def _create_importable_file( + local_path: str, + local_imports: List[Tuple[str, str]], + additional_files: List[Tuple[str, str]], + dynamic_modules_path: str, + module_namespace: str, + name: str, + download_mode: DownloadMode, +) -> Tuple[str, str]: + importable_directory_path = os.path.join(dynamic_modules_path, module_namespace, name.replace("/", "--")) + Path(importable_directory_path).mkdir(parents=True, exist_ok=True) + (Path(importable_directory_path).parent / "__init__.py").touch(exist_ok=True) + hash = files_to_hash([local_path] + [loc[1] for loc in local_imports]) + importable_local_file = _copy_script_and_other_resources_in_importable_dir( + name=name.split("/")[-1], + importable_directory_path=importable_directory_path, + subdirectory_name=hash, + original_local_path=local_path, + local_imports=local_imports, + additional_files=additional_files, + download_mode=download_mode, + ) + logger.debug(f"Created importable dataset file at {importable_local_file}") + module_path = ".".join( + [os.path.basename(dynamic_modules_path), module_namespace, name.replace("/", "--"), hash, name.split("/")[-1]] + ) + return module_path, hash + + +@dataclass +class ImportableModule: + module_path: str + hash: str + + +class _EvaluationModuleFactory: + def get_module(self) -> ImportableModule: + raise NotImplementedError + + +class LocalEvaluationModuleFactory(_EvaluationModuleFactory): + """Get the module of a local metric. The metric script is loaded from a local script.""" + + def __init__( + self, + path: str, + module_type: str = "metrics", + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[DownloadMode] = None, + dynamic_modules_path: Optional[str] = None, + ): + self.path = path + self.module_type = module_type + self.name = Path(path).stem + self.download_config = download_config or DownloadConfig() + self.download_mode = download_mode + self.dynamic_modules_path = dynamic_modules_path + + def get_module(self) -> ImportableModule: + # get script and other files + imports = get_imports(self.path) + local_imports = _download_additional_modules( + name=self.name, + base_path=str(Path(self.path).parent), + imports=imports, + download_config=self.download_config, + ) + # copy the script and the files in an importable directory + dynamic_modules_path = self.dynamic_modules_path if self.dynamic_modules_path else init_dynamic_modules() + module_path, hash = _create_importable_file( + local_path=self.path, + local_imports=local_imports, + additional_files=[], + dynamic_modules_path=dynamic_modules_path, + module_namespace=self.module_type, + name=self.name, + download_mode=self.download_mode, + ) + # make the new module to be noticed by the import system + importlib.invalidate_caches() + return ImportableModule(module_path, hash) + + +class HubEvaluationModuleFactory(_EvaluationModuleFactory): + """Get the module of a metric from a metric repository on the Hub.""" + + def __init__( + self, + name: str, + module_type: str = "metrics", + revision: Optional[Union[str, Version]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[DownloadMode] = None, + dynamic_modules_path: Optional[str] = None, + ): + self.name = name + self.module_type = module_type + self.revision = revision + self.download_config = download_config or DownloadConfig() + self.download_mode = download_mode + self.dynamic_modules_path = dynamic_modules_path + assert self.name.count("/") == 1 + increase_load_count(name, resource_type="metric") + + def download_loading_script(self, revision) -> str: + file_path = hf_hub_url(path=self.name, name=self.name.split("/")[1] + ".py", revision=revision) + download_config = self.download_config.copy() + if download_config.download_desc is None: + download_config.download_desc = "Downloading builder script" + return cached_path(file_path, download_config=download_config) + + def get_module(self) -> ImportableModule: + revision = self.revision or os.getenv("HF_SCRIPTS_VERSION", SCRIPTS_VERSION) + + if re.match(r"\d*\.\d*\.\d*", revision): # revision is version number (three digits separated by full stops) + revision = "v" + revision # tagging convention on evaluate repository starts with v + + # get script and other files + try: + local_path = self.download_loading_script(revision) + except FileNotFoundError as err: + # if there is no file found with current revision tag try to load main + if self.revision is None and os.getenv("HF_SCRIPTS_VERSION", SCRIPTS_VERSION) != "main": + revision = "main" + local_path = self.download_loading_script(revision) + else: + raise err + + imports = get_imports(local_path) + local_imports = _download_additional_modules( + name=self.name, + base_path=hf_hub_url(path=self.name, name="", revision=revision), + imports=imports, + download_config=self.download_config, + ) + # copy the script and the files in an importable directory + dynamic_modules_path = self.dynamic_modules_path if self.dynamic_modules_path else init_dynamic_modules() + module_path, hash = _create_importable_file( + local_path=local_path, + local_imports=local_imports, + additional_files=[], + dynamic_modules_path=dynamic_modules_path, + module_namespace=self.module_type, + name=self.name, + download_mode=self.download_mode, + ) + # make the new module to be noticed by the import system + importlib.invalidate_caches() + return ImportableModule(module_path, hash) + + +class CachedEvaluationModuleFactory(_EvaluationModuleFactory): + """ + Get the module of a metric that has been loaded once already and cached. + The script that is loaded from the cache is the most recent one with a matching name. + """ + + def __init__( + self, + name: str, + module_type: str = "metrics", + dynamic_modules_path: Optional[str] = None, + ): + self.name = name + self.module_type = module_type + self.dynamic_modules_path = dynamic_modules_path + assert self.name.count("/") == 0 + + def get_module(self) -> ImportableModule: + dynamic_modules_path = self.dynamic_modules_path if self.dynamic_modules_path else init_dynamic_modules() + importable_directory_path = os.path.join(dynamic_modules_path, self.module_type, self.name) + hashes = ( + [h for h in os.listdir(importable_directory_path) if len(h) == 64] + if os.path.isdir(importable_directory_path) + else None + ) + if not hashes: + raise FileNotFoundError(f"Metric {self.name} is not cached in {dynamic_modules_path}") + # get most recent + + def _get_modification_time(module_hash): + return ( + (Path(importable_directory_path) / module_hash / (self.name.split("--")[-1] + ".py")).stat().st_mtime + ) + + hash = sorted(hashes, key=_get_modification_time)[-1] + logger.warning( + f"Using the latest cached version of the module from {os.path.join(importable_directory_path, hash)} " + f"(last modified on {time.ctime(_get_modification_time(hash))}) since it " + f"couldn't be found locally at {self.name}, or remotely on the Hugging Face Hub." + ) + # make the new module to be noticed by the import system + module_path = ".".join( + [os.path.basename(dynamic_modules_path), self.module_type, self.name, hash, self.name.split("--")[-1]] + ) + importlib.invalidate_caches() + return ImportableModule(module_path, hash) + + +def evaluation_module_factory( + path: str, + module_type: Optional[str] = None, + revision: Optional[Union[str, Version]] = None, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[DownloadMode] = None, + force_local_path: Optional[str] = None, + dynamic_modules_path: Optional[str] = None, + **download_kwargs, +) -> ImportableModule: + """ + Download/extract/cache a metric module. + + Metrics codes are cached inside the the dynamic modules cache to allow easy import (avoid ugly sys.path tweaks). + + Args: + + path (str): Path or name of the metric script. + + - if ``path`` is a local metric script or a directory containing a local metric script (if the script has the same name as the directory): + -> load the module from the metric script + e.g. ``'./metrics/accuracy'`` or ``'./metrics/accuracy/accuracy.py'``. + - if ``path`` is a metric on the Hugging Face Hub (ex: `glue`, `squad`) + -> load the module from the metric script in the github repository at huggingface/datasets + e.g. ``'accuracy'`` or ``'rouge'``. + + revision (Optional ``Union[str, datasets.Version]``): + If specified, the module will be loaded from the datasets repository at this version. + By default: + - it is set to the local version of the lib. + - it will also try to load it from the master branch if it's not available at the local version of the lib. + Specifying a version that is different from your local version of the lib might cause compatibility issues. + download_config (:class:`DownloadConfig`, optional): Specific download configuration parameters. + download_mode (:class:`DownloadMode`, default ``REUSE_DATASET_IF_EXISTS``): Download/generate mode. + force_local_path (Optional str): Optional path to a local path to download and prepare the script to. + Used to inspect or modify the script folder. + dynamic_modules_path (Optional str, defaults to HF_MODULES_CACHE / "datasets_modules", i.e. ~/.cache/huggingface/modules/datasets_modules): + Optional path to the directory in which the dynamic modules are saved. It must have been initialized with :obj:`init_dynamic_modules`. + By default the datasets and metrics are stored inside the `datasets_modules` module. + download_kwargs: optional attributes for DownloadConfig() which will override the attributes in download_config if supplied. + + Returns: + ImportableModule + """ + if download_config is None: + download_config = DownloadConfig(**download_kwargs) + download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) + download_config.extract_compressed_file = True + download_config.force_extract = True + + filename = list(filter(lambda x: x, path.replace(os.sep, "/").split("/")))[-1] + if not filename.endswith(".py"): + filename = filename + ".py" + combined_path = os.path.join(path, filename) + # Try locally + if path.endswith(filename): + if os.path.isfile(path): + return LocalEvaluationModuleFactory( + path, download_mode=download_mode, dynamic_modules_path=dynamic_modules_path + ).get_module() + else: + raise FileNotFoundError(f"Couldn't find a metric script at {relative_to_absolute_path(path)}") + elif os.path.isfile(combined_path): + return LocalEvaluationModuleFactory( + combined_path, download_mode=download_mode, dynamic_modules_path=dynamic_modules_path + ).get_module() + elif is_relative_path(path) and path.count("/") <= 1 and not force_local_path: + try: + # load a canonical evaluation module from hub + if path.count("/") == 0: + # if no type provided look through all possible modules + if module_type is None: + for current_type in ["metric", "comparison", "measurement"]: + try: + return HubEvaluationModuleFactory( + f"evaluate-{current_type}/{path}", + revision=revision, + download_config=download_config, + download_mode=download_mode, + dynamic_modules_path=dynamic_modules_path, + ).get_module() + except ConnectionError: + pass + raise FileNotFoundError + # if module_type provided load specific module_type + else: + return HubEvaluationModuleFactory( + f"evaluate-{module_type}/{path}", + revision=revision, + download_config=download_config, + download_mode=download_mode, + dynamic_modules_path=dynamic_modules_path, + ).get_module() + # load community evaluation module from hub + elif path.count("/") == 1: + return HubEvaluationModuleFactory( + path, + revision=revision, + download_config=download_config, + download_mode=download_mode, + dynamic_modules_path=dynamic_modules_path, + ).get_module() + except Exception as e1: # noqa: all the attempts failed, before raising the error we should check if the module is already cached. + # if it's a canonical module we need to check if it's any of the types + if path.count("/") == 0: + for current_type in ["metric", "comparison", "measurement"]: + try: + return CachedEvaluationModuleFactory( + f"evaluate-{current_type}--{path}", dynamic_modules_path=dynamic_modules_path + ).get_module() + except Exception as e2: # noqa: if it's not in the cache, then it doesn't exist. + pass + # if it's a community module we just need to check on path + elif path.count("/") == 1: + try: + return CachedEvaluationModuleFactory( + path.replace("/", "--"), dynamic_modules_path=dynamic_modules_path + ).get_module() + except Exception as e2: # noqa: if it's not in the cache, then it doesn't exist. + pass + if not isinstance(e1, (ConnectionError, FileNotFoundError)): + raise e1 from None + raise FileNotFoundError( + f"Couldn't find a module script at {relative_to_absolute_path(combined_path)}. " + f"Module '{path}' doesn't exist on the Hugging Face Hub either." + ) from None + else: + raise FileNotFoundError(f"Couldn't find a module script at {relative_to_absolute_path(combined_path)}.") + + +def load( + path: str, + config_name: Optional[str] = None, + module_type: Optional[str] = None, + process_id: int = 0, + num_process: int = 1, + cache_dir: Optional[str] = None, + experiment_id: Optional[str] = None, + keep_in_memory: bool = False, + download_config: Optional[DownloadConfig] = None, + download_mode: Optional[DownloadMode] = None, + revision: Optional[Union[str, Version]] = None, + **init_kwargs, +) -> EvaluationModule: + """Load a [`~evaluate.EvaluationModule`]. + + Args: + + path (`str`): + Path to the evaluation processing script with the evaluation builder. Can be either: + - a local path to processing script or the directory containing the script (if the script has the same name as the directory), + e.g. `'./metrics/rouge'` or `'./metrics/rouge/rouge.py'` + - a evaluation module identifier on the HuggingFace evaluate repo e.g. `'rouge'` or `'bleu'` that are in either `'metrics/'`, + `'comparisons/'`, or `'measurements/'` depending on the provided `module_type` + config_name (`str`, *optional*): + Selecting a configuration for the metric (e.g. the GLUE metric has a configuration for each subset). + module_type (`str`, default `'metric'`): + Type of evaluation module, can be one of `'metric'`, `'comparison'`, or `'measurement'`. + process_id (`int`, *optional*): + For distributed evaluation: id of the process. + num_process (`int`, *optional*): + For distributed evaluation: total number of processes. + cache_dir (`str`, *optional*): + Path to store the temporary predictions and references (default to `~/.cache/huggingface/evaluate/`). + experiment_id (`str`): + A specific experiment id. This is used if several distributed evaluations share the same file system. + This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). + keep_in_memory (`bool`): + Whether to store the temporary results in memory (defaults to `False`). + download_config ([`~evaluate.DownloadConfig`], *optional*): + Specific download configuration parameters. + download_mode ([`DownloadMode`], defaults to `REUSE_DATASET_IF_EXISTS`): + Download/generate mode. + revision (`Union[str, evaluate.Version]`, *optional*): + If specified, the module will be loaded from the datasets repository + at this version. By default it is set to the local version of the lib. Specifying a version that is different from + your local version of the lib might cause compatibility issues. + + Returns: + [`evaluate.EvaluationModule`] + + Example: + + ```py + >>> from evaluate import load + >>> accuracy = load("accuracy") + ``` + """ + download_mode = DownloadMode(download_mode or DownloadMode.REUSE_DATASET_IF_EXISTS) + evaluation_module = evaluation_module_factory( + path, module_type=module_type, revision=revision, download_config=download_config, download_mode=download_mode + ) + evaluation_cls = import_main_class(evaluation_module.module_path) + evaluation_instance = evaluation_cls( + config_name=config_name, + process_id=process_id, + num_process=num_process, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + experiment_id=experiment_id, + hash=evaluation_module.hash, + **init_kwargs, + ) + + if module_type and module_type != evaluation_instance.module_type: + raise TypeError( + f"No module of module type '{module_type}' not found for '{path}' locally, or on the Hugging Face Hub. Found module of module type '{evaluation_instance.module_type}' instead." + ) + + # Download and prepare resources for the metric + evaluation_instance.download_and_prepare(download_config=download_config) + + return evaluation_instance diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/module.py b/TextClassification/TextClassification/evaluate/src/evaluate/module.py new file mode 100644 index 0000000000000000000000000000000000000000..b4d73e5eaedad1aef7feb1304098ef5dce421831 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/module.py @@ -0,0 +1,1034 @@ +# Copyright 2020 The HuggingFace Datasets Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +""" EvaluationModule base class.""" +import collections +import itertools +import os +import types +import uuid +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np +import pyarrow as pa +from datasets import DatasetInfo, DownloadConfig, DownloadManager +from datasets.arrow_dataset import Dataset +from datasets.arrow_reader import ArrowReader +from datasets.arrow_writer import ArrowWriter +from datasets.features import Features, Sequence, Value +from datasets.features.features import _check_non_null_non_empty_recursive +from datasets.utils.filelock import BaseFileLock, FileLock, Timeout +from datasets.utils.py_utils import copyfunc, temp_seed, zip_dict + +from . import config +from .info import EvaluationModuleInfo +from .naming import camelcase_to_snakecase +from .utils.logging import get_logger + + +logger = get_logger(__name__) + + +class FileFreeLock(BaseFileLock): + """Thread lock until a file **cannot** be locked""" + + def __init__(self, lock_file, *args, **kwargs): + self.filelock = FileLock(lock_file) + super().__init__(lock_file, *args, **kwargs) + self._lock_file_fd = None + + def _acquire(self): + try: + self.filelock.acquire(timeout=0.01, poll_interval=0.02) # Try to lock once + except Timeout: + # We couldn't acquire the lock, the file is locked! + self._lock_file_fd = self.filelock.lock_file + else: + # We were able to acquire the lock, the file is not yet locked! + self.filelock.release() + self._lock_file_fd = None + + def _release(self): + self._lock_file_fd = None + + @property + def is_locked(self) -> bool: + return self._lock_file_fd is not None + + +# lists - summarize long lists similarly to NumPy +# arrays/tensors - let the frameworks control formatting +def summarize_if_long_list(obj): + if type(obj) is not list or len(obj) <= 6: + return f"{obj}" + + def format_chunk(chunk): + return ", ".join(repr(x) for x in chunk) + + return f"[{format_chunk(obj[:3])}, ..., {format_chunk(obj[-3:])}]" + + +class EvaluationModuleInfoMixin: + """This base class exposes some attributes of EvaluationModuleInfo + at the base level of the EvaluationModule for easy access. + """ + + def __init__(self, info: EvaluationModuleInfo): + self._module_info = info + + @property + def info(self): + """:class:`evaluate.EvaluationModuleInfo` object containing all the metadata in the evaluation module.""" + return self._module_info + + @property + def name(self) -> str: + return self._module_info.module_name + + @property + def experiment_id(self) -> Optional[str]: + return self._module_info.experiment_id + + @property + def description(self) -> str: + return self._module_info.description + + @property + def citation(self) -> str: + return self._module_info.citation + + @property + def features(self) -> Features: + return self._module_info.features + + @property + def inputs_description(self) -> str: + return self._module_info.inputs_description + + @property + def homepage(self) -> Optional[str]: + return self._module_info.homepage + + @property + def license(self) -> str: + return self._module_info.license + + @property + def codebase_urls(self) -> Optional[List[str]]: + return self._module_info.codebase_urls + + @property + def reference_urls(self) -> Optional[List[str]]: + return self._module_info.reference_urls + + @property + def streamable(self) -> bool: + return self._module_info.streamable + + @property + def format(self) -> Optional[str]: + return self._module_info.format + + @property + def module_type(self) -> str: + return self._module_info.module_type + + +class EvaluationModule(EvaluationModuleInfoMixin): + """A `EvaluationModule` is the base class and common API for metrics, comparisons, and measurements. + + Args: + config_name (`str`): + This is used to define a hash specific to a module computation script and prevents the module's data + to be overridden when the module loading script is modified. + keep_in_memory (`bool`): + Keep all predictions and references in memory. Not possible in distributed settings. + cache_dir (`str`): + Path to a directory in which temporary prediction/references data will be stored. + The data directory should be located on a shared file-system in distributed setups. + num_process (`int`): + Specify the total number of nodes in a distributed settings. + This is useful to compute module in distributed setups (in particular non-additive modules like F1). + process_id (`int`): + Specify the id of the current process in a distributed setup (between 0 and num_process-1) + This is useful to compute module in distributed setups (in particular non-additive metrics like F1). + seed (`int`, optional): + If specified, this will temporarily set numpy's random seed when [`~evaluate.EvaluationModule.compute`] is run. + experiment_id (`str`): + A specific experiment id. This is used if several distributed evaluations share the same file system. + This is useful to compute module in distributed setups (in particular non-additive metrics like F1). + hash (`str`): + Used to identify the evaluation module according to the hashed file contents. + max_concurrent_cache_files (`int`): + Max number of concurrent module cache files (default `10000`). + timeout (`Union[int, float]`): + Timeout in second for distributed setting synchronization. + """ + + def __init__( + self, + config_name: Optional[str] = None, + keep_in_memory: bool = False, + cache_dir: Optional[str] = None, + num_process: int = 1, + process_id: int = 0, + seed: Optional[int] = None, + experiment_id: Optional[str] = None, + hash: str = None, + max_concurrent_cache_files: int = 10000, + timeout: Union[int, float] = 100, + **kwargs, + ): + # prepare info + self.config_name = config_name or "default" + info = self._info() + info.module_name = camelcase_to_snakecase(self.__class__.__name__) + info.config_name = self.config_name + info.experiment_id = experiment_id or "default_experiment" + EvaluationModuleInfoMixin.__init__(self, info) # For easy access on low level + + # Safety checks on num_process and process_id + if not isinstance(process_id, int) or process_id < 0: + raise ValueError("'process_id' should be a number greater than 0") + if not isinstance(num_process, int) or num_process <= process_id: + raise ValueError("'num_process' should be a number greater than process_id") + if keep_in_memory and num_process != 1: + raise ValueError("Using 'keep_in_memory' is not possible in distributed setting (num_process > 1).") + + self.num_process = num_process + self.process_id = process_id + self.max_concurrent_cache_files = max_concurrent_cache_files + + self.keep_in_memory = keep_in_memory + self._data_dir_root = os.path.expanduser(cache_dir or config.HF_METRICS_CACHE) + self.data_dir = self._build_data_dir() + if seed is None: + _, seed, pos, *_ = np.random.get_state() + self.seed: int = seed[pos] if pos < 624 else seed[0] + else: + self.seed: int = seed + self.timeout: Union[int, float] = timeout + + # Update 'compute' and 'add' docstring + # methods need to be copied otherwise it changes the docstrings of every instance + self.compute = types.MethodType(copyfunc(self.compute), self) + self.add_batch = types.MethodType(copyfunc(self.add_batch), self) + self.add = types.MethodType(copyfunc(self.add), self) + self.compute.__func__.__doc__ += self.info.inputs_description + self.add_batch.__func__.__doc__ += self.info.inputs_description + self.add.__func__.__doc__ += self.info.inputs_description + + # self.arrow_schema = pa.schema(field for field in self.info.features.type) + self.selected_feature_format = None + self.buf_writer = None + self.writer = None + self.writer_batch_size = None + self.data = None + + # This is the cache file we store our predictions/references in + # Keep it None for now so we can (cloud)pickle the object + self.cache_file_name = None + self.filelock = None + self.rendez_vous_lock = None + + # This is all the cache files on which we have a lock when we are in a distributed setting + self.file_paths = None + self.filelocks = None + + # This fingerprints the evaluation module according to the hashed contents of the module code + self._hash = hash + + def __len__(self): + """Return the number of examples (predictions or predictions/references pair) + currently stored in the evaluation module's cache. + """ + return 0 if self.writer is None else len(self.writer) + + def __repr__(self): + return ( + f'EvaluationModule(name: "{self.name}", module_type: "{self.module_type}", ' + f'features: {self.features}, usage: """{self.inputs_description}""", ' + f"stored examples: {len(self)})" + ) + + def _build_data_dir(self): + """Path of this evaluation module in cache_dir: + Will be: + self._data_dir_root/self.name/self.config_name/self.hash (if not none)/ + If any of these element is missing or if ``with_version=False`` the corresponding subfolders are dropped. + """ + builder_data_dir = self._data_dir_root + builder_data_dir = os.path.join(builder_data_dir, self.name, self.config_name) + os.makedirs(builder_data_dir, exist_ok=True) + return builder_data_dir + + def _create_cache_file(self, timeout=1) -> Tuple[str, FileLock]: + """Create a new cache file. If the default cache file is used, we generated a new hash.""" + file_path = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-{self.process_id}.arrow") + filelock = None + for i in range(self.max_concurrent_cache_files): + filelock = FileLock(file_path + ".lock") + try: + filelock.acquire(timeout=timeout) + except Timeout: + # If we have reached the max number of attempts or we are not allow to find a free name (distributed setup) + # We raise an error + if self.num_process != 1: + raise ValueError( + f"Error in _create_cache_file: another evaluation module instance is already using the local cache file at {file_path}. " + f"Please specify an experiment_id (currently: {self.experiment_id}) to avoid collision " + f"between distributed evaluation module instances." + ) from None + if i == self.max_concurrent_cache_files - 1: + raise ValueError( + f"Cannot acquire lock, too many evaluation module instance are operating concurrently on this file system." + f"You should set a larger value of max_concurrent_cache_files when creating the evaluation module " + f"(current value is {self.max_concurrent_cache_files})." + ) from None + # In other cases (allow to find new file name + not yet at max num of attempts) we can try to sample a new hashing name. + file_uuid = str(uuid.uuid4()) + file_path = os.path.join( + self.data_dir, f"{self.experiment_id}-{file_uuid}-{self.num_process}-{self.process_id}.arrow" + ) + else: + break + + return file_path, filelock + + def _get_all_cache_files(self) -> Tuple[List[str], List[FileLock]]: + """Get a lock on all the cache files in a distributed setup. + We wait for timeout second to let all the distributed node finish their tasks (default is 100 seconds). + """ + if self.num_process == 1: + if self.cache_file_name is None: + raise ValueError( + "Evaluation module cache file doesn't exist. Please make sure that you call `add` or `add_batch` " + "at least once before calling `compute`." + ) + file_paths = [self.cache_file_name] + else: + file_paths = [ + os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-{process_id}.arrow") + for process_id in range(self.num_process) + ] + + # Let's acquire a lock on each process files to be sure they are finished writing + filelocks = [] + for process_id, file_path in enumerate(file_paths): + if process_id == 0: # process 0 already has its lock file + filelocks.append(self.filelock) + else: + filelock = FileLock(file_path + ".lock") + try: + filelock.acquire(timeout=self.timeout) + except Timeout: + raise ValueError( + f"Cannot acquire lock on cached file {file_path} for process {process_id}." + ) from None + else: + filelocks.append(filelock) + + return file_paths, filelocks + + def _check_all_processes_locks(self): + expected_lock_file_names = [ + os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-{process_id}.arrow.lock") + for process_id in range(self.num_process) + ] + for expected_lock_file_name in expected_lock_file_names: + nofilelock = FileFreeLock(expected_lock_file_name) + try: + nofilelock.acquire(timeout=self.timeout) + except Timeout: + raise ValueError( + f"Expected to find locked file {expected_lock_file_name} from process {self.process_id} but it doesn't exist." + ) from None + else: + nofilelock.release() + + def _check_rendez_vous(self): + expected_lock_file_name = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-0.arrow.lock") + nofilelock = FileFreeLock(expected_lock_file_name) + try: + nofilelock.acquire(timeout=self.timeout) + except Timeout: + raise ValueError( + f"Expected to find locked file {expected_lock_file_name} from process {self.process_id} but it doesn't exist." + ) from None + else: + nofilelock.release() + lock_file_name = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-rdv.lock") + rendez_vous_lock = FileLock(lock_file_name) + try: + rendez_vous_lock.acquire(timeout=self.timeout) + except Timeout: + raise ValueError(f"Couldn't acquire lock on {lock_file_name} from process {self.process_id}.") from None + else: + rendez_vous_lock.release() + + def _finalize(self): + """Close all the writing process and load/gather the data + from all the nodes if main node or all_process is True. + """ + if self.writer is not None: + self.writer.finalize() + self.writer = None + # release the locks of the processes > 0 so that process 0 can lock them to read + delete the data + if self.filelock is not None and self.process_id > 0: + self.filelock.release() + + if self.keep_in_memory: + # Read the predictions and references + reader = ArrowReader(path=self.data_dir, info=DatasetInfo(features=self.selected_feature_format)) + self.data = Dataset.from_buffer(self.buf_writer.getvalue()) + + elif self.process_id == 0: + # Let's acquire a lock on each node files to be sure they are finished writing + file_paths, filelocks = self._get_all_cache_files() + + # Read the predictions and references + try: + reader = ArrowReader(path="", info=DatasetInfo(features=self.selected_feature_format)) + self.data = Dataset(**reader.read_files([{"filename": f} for f in file_paths])) + except FileNotFoundError: + raise ValueError( + "Error in finalize: another evaluation module instance is already using the local cache file. " + "Please specify an experiment_id to avoid collision between distributed evaluation module instances." + ) from None + + # Store file paths and locks and we will release/delete them after the computation. + self.file_paths = file_paths + self.filelocks = filelocks + + def compute(self, *, predictions=None, references=None, **kwargs) -> Optional[dict]: + """Compute the evaluation module. + + Usage of positional arguments is not allowed to prevent mistakes. + + Args: + predictions (`list/array/tensor`, *optional*): + Predictions. + references (`list/array/tensor`, *optional*): + References. + **kwargs (optional): + Keyword arguments that will be forwarded to the evaluation module [`~evaluate.EvaluationModule.compute`] + method (see details in the docstring). + + Return: + `dict` or `None` + + - Dictionary with the results if this evaluation module is run on the main process (`process_id == 0`). + - `None` if the evaluation module is not run on the main process (`process_id != 0`). + + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> accuracy.compute(predictions=[0, 1, 1, 0], references=[0, 1, 0, 1]) + ``` + """ + all_kwargs = {"predictions": predictions, "references": references, **kwargs} + if predictions is None and references is None: + missing_kwargs = {k: None for k in self._feature_names() if k not in all_kwargs} + all_kwargs.update(missing_kwargs) + else: + missing_inputs = [k for k in self._feature_names() if k not in all_kwargs] + if missing_inputs: + raise ValueError( + f"Evaluation module inputs are missing: {missing_inputs}. All required inputs are {list(self._feature_names())}" + ) + inputs = {input_name: all_kwargs[input_name] for input_name in self._feature_names()} + compute_kwargs = {k: kwargs[k] for k in kwargs if k not in self._feature_names()} + + if any(v is not None for v in inputs.values()): + self.add_batch(**inputs) + self._finalize() + + self.cache_file_name = None + self.filelock = None + self.selected_feature_format = None + + if self.process_id == 0: + self.data.set_format(type=self.info.format) + + inputs = {input_name: self.data[input_name] for input_name in self._feature_names()} + with temp_seed(self.seed): + output = self._compute(**inputs, **compute_kwargs) + + if self.buf_writer is not None: + self.buf_writer = None + del self.data + self.data = None + else: + # Release locks and delete all the cache files. Process 0 is released last. + for filelock, file_path in reversed(list(zip(self.filelocks, self.file_paths))): + logger.info(f"Removing {file_path}") + del self.data + self.data = None + del self.writer + self.writer = None + os.remove(file_path) + filelock.release() + + return output + else: + return None + + def add_batch(self, *, predictions=None, references=None, **kwargs): + """Add a batch of predictions and references for the evaluation module's stack. + + Args: + predictions (`list/array/tensor`, *optional*): + Predictions. + references (`list/array/tensor`, *optional*): + References. + + Example: + + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> for refs, preds in zip([[0,1],[0,1]], [[1,0],[0,1]]): + ... accuracy.add_batch(references=refs, predictions=preds) + ``` + """ + bad_inputs = [input_name for input_name in kwargs if input_name not in self._feature_names()] + if bad_inputs: + raise ValueError( + f"Bad inputs for evaluation module: {bad_inputs}. All required inputs are {list(self._feature_names())}" + ) + batch = {"predictions": predictions, "references": references, **kwargs} + batch = {input_name: batch[input_name] for input_name in self._feature_names()} + if self.writer is None: + self.selected_feature_format = self._infer_feature_from_batch(batch) + self._init_writer() + try: + for key, column in batch.items(): + if len(column) > 0: + self._enforce_nested_string_type(self.selected_feature_format[key], column[0]) + batch = self.selected_feature_format.encode_batch(batch) + self.writer.write_batch(batch) + except (pa.ArrowInvalid, TypeError): + if any(len(batch[c]) != len(next(iter(batch.values()))) for c in batch): + col0 = next(iter(batch)) + bad_col = [c for c in batch if len(batch[c]) != len(batch[col0])][0] + error_msg = ( + f"Mismatch in the number of {col0} ({len(batch[col0])}) and {bad_col} ({len(batch[bad_col])})" + ) + elif set(self.selected_feature_format) != {"references", "predictions"}: + error_msg = ( + f"Module inputs don't match the expected format.\n" + f"Expected format: {self.selected_feature_format },\n" + ) + error_msg_inputs = ",\n".join( + f"Input {input_name}: {summarize_if_long_list(batch[input_name])}" + for input_name in self.selected_feature_format + ) + error_msg += error_msg_inputs + else: + error_msg = ( + f"Predictions and/or references don't match the expected format.\n" + f"Expected format: {self.selected_feature_format },\n" + f"Input predictions: {summarize_if_long_list(predictions)},\n" + f"Input references: {summarize_if_long_list(references)}" + ) + raise ValueError(error_msg) from None + + def add(self, *, prediction=None, reference=None, **kwargs): + """Add one prediction and reference for the evaluation module's stack. + + Args: + prediction (`list/array/tensor`, *optional*): + Predictions. + reference (`list/array/tensor`, *optional*): + References. + + Example: + + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> accuracy.add(references=[0,1], predictions=[1,0]) + ``` + """ + bad_inputs = [input_name for input_name in kwargs if input_name not in self._feature_names()] + if bad_inputs: + raise ValueError( + f"Bad inputs for evaluation module: {bad_inputs}. All required inputs are {list(self._feature_names())}" + ) + example = {"predictions": prediction, "references": reference, **kwargs} + example = {input_name: example[input_name] for input_name in self._feature_names()} + if self.writer is None: + self.selected_feature_format = self._infer_feature_from_example(example) + self._init_writer() + try: + self._enforce_nested_string_type(self.selected_feature_format, example) + example = self.selected_feature_format.encode_example(example) + self.writer.write(example) + except (pa.ArrowInvalid, TypeError): + error_msg = ( + f"Evaluation module inputs don't match the expected format.\n" + f"Expected format: {self.selected_feature_format},\n" + ) + error_msg_inputs = ",\n".join( + f"Input {input_name}: {summarize_if_long_list(example[input_name])}" + for input_name in self.selected_feature_format + ) + error_msg += error_msg_inputs + raise ValueError(error_msg) from None + + def _infer_feature_from_batch(self, batch): + if isinstance(self.features, Features): + return self.features + else: + example = dict([(k, v[0]) for k, v in batch.items()]) + return self._infer_feature_from_example(example) + + def _infer_feature_from_example(self, example): + if isinstance(self.features, Features): + return self.features + else: + for features in self.features: + try: + self._enforce_nested_string_type(features, example) + features.encode_example(example) + return features + except (ValueError, TypeError): + continue + feature_strings = "\n".join([f"Feature option {i}: {feature}" for i, feature in enumerate(self.features)]) + error_msg = ( + f"Predictions and/or references don't match the expected format.\n" + f"Expected format:\n{feature_strings},\n" + f"Input predictions: {summarize_if_long_list(example['predictions'])},\n" + f"Input references: {summarize_if_long_list(example['references'])}" + ) + raise ValueError(error_msg) from None + + def _feature_names(self): + if isinstance(self.features, list): + feature_names = list(self.features[0].keys()) + else: + feature_names = list(self.features.keys()) + return feature_names + + def _init_writer(self, timeout=1): + if self.num_process > 1: + if self.process_id == 0: + file_path = os.path.join(self.data_dir, f"{self.experiment_id}-{self.num_process}-rdv.lock") + self.rendez_vous_lock = FileLock(file_path) + try: + self.rendez_vous_lock.acquire(timeout=timeout) + except TimeoutError: + raise ValueError( + f"Error in _init_writer: another evalution module instance is already using the local cache file at {file_path}. " + f"Please specify an experiment_id (currently: {self.experiment_id}) to avoid collision " + f"between distributed evaluation module instances." + ) from None + + if self.keep_in_memory: + self.buf_writer = pa.BufferOutputStream() + self.writer = ArrowWriter( + features=self.selected_feature_format, stream=self.buf_writer, writer_batch_size=self.writer_batch_size + ) + else: + self.buf_writer = None + + # Get cache file name and lock it + if self.cache_file_name is None or self.filelock is None: + cache_file_name, filelock = self._create_cache_file() # get ready + self.cache_file_name = cache_file_name + self.filelock = filelock + + self.writer = ArrowWriter( + features=self.selected_feature_format, + path=self.cache_file_name, + writer_batch_size=self.writer_batch_size, + ) + # Setup rendez-vous here if + if self.num_process > 1: + if self.process_id == 0: + self._check_all_processes_locks() # wait for everyone to be ready + self.rendez_vous_lock.release() # let everyone go + else: + self._check_rendez_vous() # wait for master to be ready and to let everyone go + + def _info(self) -> EvaluationModuleInfo: + """Construct the EvaluationModuleInfo object. See `EvaluationModuleInfo` for details. + + Warning: This function is only called once and the result is cached for all + following .info() calls. + + Returns: + info: (EvaluationModuleInfo) The EvaluationModule information + """ + raise NotImplementedError + + def download_and_prepare( + self, + download_config: Optional[DownloadConfig] = None, + dl_manager: Optional[DownloadManager] = None, + ): + """Downloads and prepares evaluation module for reading. + + Args: + download_config ([`DownloadConfig`], *optional*): + Specific download configuration parameters. + dl_manager ([`DownloadManager`], *optional*): + Specific download manager to use. + + Example: + + ```py + >>> import evaluate + ``` + """ + if dl_manager is None: + if download_config is None: + download_config = DownloadConfig() + download_config.cache_dir = os.path.join(self.data_dir, "downloads") + download_config.force_download = False + + dl_manager = DownloadManager( + dataset_name=self.name, download_config=download_config, data_dir=self.data_dir + ) + + self._download_and_prepare(dl_manager) + + def _download_and_prepare(self, dl_manager): + """Downloads and prepares resources for the evaluation module. + + This is the internal implementation to overwrite called when user calls + `download_and_prepare`. It should download all required resources for the evaluation module. + + Args: + dl_manager (:class:`DownloadManager`): `DownloadManager` used to download and cache data. + """ + return None + + def _compute(self, *, predictions=None, references=None, **kwargs) -> Dict[str, Any]: + """This method defines the common API for all the evaluation module in the library""" + raise NotImplementedError + + def __del__(self): + if hasattr(self, "filelock") and self.filelock is not None: + self.filelock.release() + if hasattr(self, "rendez_vous_lock") and self.rendez_vous_lock is not None: + self.rendez_vous_lock.release() + if hasattr(self, "writer"): # in case it was already deleted + del self.writer + if hasattr(self, "data"): # in case it was already deleted + del self.data + + def _enforce_nested_string_type(self, schema, obj): + """ + Recursively checks if there is any Value feature of type string and throws TypeError if corresponding object is not a string. + Since any Python object can be cast to string this avoids implicitly casting wrong input types (e.g. lists) to string without error. + """ + # Nested structures: we allow dict, list, tuples, sequences + if isinstance(schema, dict): + return [self._enforce_nested_string_type(sub_schema, o) for k, (sub_schema, o) in zip_dict(schema, obj)] + + elif isinstance(schema, (list, tuple)): + sub_schema = schema[0] + return [self._enforce_nested_string_type(sub_schema, o) for o in obj] + elif isinstance(schema, Sequence): + # We allow to reverse list of dict => dict of list for compatiblity with tfds + if isinstance(schema.feature, dict): + if isinstance(obj, (list, tuple)): + # obj is a list of dict + for k, dict_tuples in zip_dict(schema.feature, *obj): + for sub_obj in dict_tuples[1:]: + if _check_non_null_non_empty_recursive(sub_obj, dict_tuples[0]): + self._enforce_nested_string_type(dict_tuples[0], sub_obj) + break + return None + else: + # obj is a single dict + for k, (sub_schema, sub_objs) in zip_dict(schema.feature, obj): + for sub_obj in sub_objs: + if _check_non_null_non_empty_recursive(sub_obj, sub_schema): + self._enforce_nested_string_type(sub_schema, sub_obj) + break + return None + # schema.feature is not a dict + if isinstance(obj, str): # don't interpret a string as a list + raise ValueError(f"Got a string but expected a list instead: '{obj}'") + if obj is None: + return None + else: + if len(obj) > 0: + for first_elmt in obj: + if _check_non_null_non_empty_recursive(first_elmt, schema.feature): + break + if not isinstance(first_elmt, list): + return self._enforce_nested_string_type(schema.feature, first_elmt) + + elif isinstance(schema, Value): + if pa.types.is_string(schema.pa_type) and not isinstance(obj, str): + raise TypeError(f"Expected type str but got {type(obj)}.") + + +class Metric(EvaluationModule): + """A Metric is the base class and common API for all metrics. + + Args: + config_name (`str`): + This is used to define a hash specific to a metric computation script and prevents the metric's data + to be overridden when the metric loading script is modified. + keep_in_memory (`bool`): + Keep all predictions and references in memory. Not possible in distributed settings. + cache_dir (`str`): + Path to a directory in which temporary prediction/references data will be stored. + The data directory should be located on a shared file-system in distributed setups. + num_process (`int`): + Specify the total number of nodes in a distributed settings. + This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). + process_id (`int`): + Specify the id of the current process in a distributed setup (between 0 and num_process-1) + This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). + seed (`int`, *optional*): + If specified, this will temporarily set numpy's random seed when [`~evaluate.Metric.compute`] is run. + experiment_id (`str`): + A specific experiment id. This is used if several distributed evaluations share the same file system. + This is useful to compute metrics in distributed setups (in particular non-additive metrics like F1). + max_concurrent_cache_files (`int`): + Max number of concurrent metric cache files (default `10000`). + timeout (`Union[int, float]`): + Timeout in second for distributed setting synchronization. + """ + + +class Comparison(EvaluationModule): + """A Comparison is the base class and common API for all comparisons. + + Args: + config_name (`str`): + This is used to define a hash specific to a comparison computation script and prevents the comparison's data + to be overridden when the comparison loading script is modified. + keep_in_memory (`bool`): + Keep all predictions and references in memory. Not possible in distributed settings. + cache_dir (`str`): + Path to a directory in which temporary prediction/references data will be stored. + The data directory should be located on a shared file-system in distributed setups. + num_process (`int`): + Specify the total number of nodes in a distributed settings. + This is useful to compute comparisons in distributed setups (in particular non-additive comparisons). + process_id (`int`): + Specify the id of the current process in a distributed setup (between 0 and num_process-1) + This is useful to compute comparisons in distributed setups (in particular non-additive comparisons). + seed (`int`, *optional*): + If specified, this will temporarily set numpy's random seed when [`~evaluate.Comparison.compute`] is run. + experiment_id (`str`): + A specific experiment id. This is used if several distributed evaluations share the same file system. + This is useful to compute comparisons in distributed setups (in particular non-additive comparisons). + max_concurrent_cache_files (`int`): + Max number of concurrent comparison cache files (default `10000`). + timeout (`Union[int, float]`): + Timeout in second for distributed setting synchronization. + """ + + +class Measurement(EvaluationModule): + """A Measurement is the base class and common API for all measurements. + + Args: + config_name (`str`): + This is used to define a hash specific to a measurement computation script and prevents the measurement's data + to be overridden when the measurement loading script is modified. + keep_in_memory (`bool`): + Keep all predictions and references in memory. Not possible in distributed settings. + cache_dir (`str`): + Path to a directory in which temporary prediction/references data will be stored. + The data directory should be located on a shared file-system in distributed setups. + num_process (`int`): + Specify the total number of nodes in a distributed settings. + This is useful to compute measurements in distributed setups (in particular non-additive measurements). + process_id (`int`): + Specify the id of the current process in a distributed setup (between 0 and num_process-1) + This is useful to compute measurements in distributed setups (in particular non-additive measurements). + seed (`int`, *optional*): + If specified, this will temporarily set numpy's random seed when [`~evaluate.Measurement.compute`] is run. + experiment_id (`str`): + A specific experiment id. This is used if several distributed evaluations share the same file system. + This is useful to compute measurements in distributed setups (in particular non-additive measurements). + max_concurrent_cache_files (`int`): + Max number of concurrent measurement cache files (default `10000`). + timeout (`Union[int, float]`): + Timeout in second for distributed setting synchronization. + """ + + +class CombinedEvaluations: + def __init__(self, evaluation_modules, force_prefix=False): + from .loading import load # avoid circular imports + + self.evaluation_module_names = None + if isinstance(evaluation_modules, list): + self.evaluation_modules = evaluation_modules + elif isinstance(evaluation_modules, dict): + self.evaluation_modules = list(evaluation_modules.values()) + self.evaluation_module_names = list(evaluation_modules.keys()) + loaded_modules = [] + + for module in self.evaluation_modules: + if isinstance(module, str): + module = load(module) + loaded_modules.append(module) + self.evaluation_modules = loaded_modules + + if self.evaluation_module_names is None: + self.evaluation_module_names = [module.name for module in self.evaluation_modules] + + self.force_prefix = force_prefix + + def add(self, prediction=None, reference=None, **kwargs): + """Add one prediction and reference for each evaluation module's stack. + + Args: + predictions (`list/array/tensor`, *optional*): + Predictions. + references (`list/array/tensor`, *optional*): + References. + + Example: + + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> f1 = evaluate.load("f1") + >>> clf_metrics = combine(["accuracy", "f1"]) + >>> for ref, pred in zip([0,1,0,1], [1,0,0,1]): + ... clf_metrics.add(references=ref, predictions=pred) + ``` + """ + for evaluation_module in self.evaluation_modules: + batch = {"predictions": prediction, "references": reference, **kwargs} + batch = {input_name: batch[input_name] for input_name in evaluation_module._feature_names()} + evaluation_module.add(**batch) + + def add_batch(self, predictions=None, references=None, **kwargs): + """Add a batch of predictions and references for each evaluation module's stack. + + Args: + predictions (`list/array/tensor`, *optional*): + Predictions. + references (`list/array/tensor`, *optional*): + References. + + Example: + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> f1 = evaluate.load("f1") + >>> clf_metrics = combine(["accuracy", "f1"]) + >>> for refs, preds in zip([[0,1],[0,1]], [[1,0],[0,1]]): + ... clf_metrics.add(references=refs, predictions=preds) + ``` + """ + for evaluation_module in self.evaluation_modules: + batch = {"predictions": predictions, "references": references, **kwargs} + batch = {input_name: batch[input_name] for input_name in evaluation_module._feature_names()} + evaluation_module.add_batch(**batch) + + def compute(self, predictions=None, references=None, **kwargs): + """Compute each evaluation module. + + Usage of positional arguments is not allowed to prevent mistakes. + + Args: + predictions (`list/array/tensor`, *optional*): + Predictions. + references (`list/array/tensor`, *optional*): + References. + **kwargs (*optional*): + Keyword arguments that will be forwarded to the evaluation module [`~evaluate.EvaluationModule.compute`] + method (see details in the docstring). + + Return: + `dict` or `None` + + - Dictionary with the results if this evaluation module is run on the main process (`process_id == 0`). + - `None` if the evaluation module is not run on the main process (`process_id != 0`). + + Example: + + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> f1 = evaluate.load("f1") + >>> clf_metrics = combine(["accuracy", "f1"]) + >>> clf_metrics.compute(predictions=[0,1], references=[1,1]) + {'accuracy': 0.5, 'f1': 0.6666666666666666} + ``` + """ + results = [] + + for evaluation_module in self.evaluation_modules: + batch = {"predictions": predictions, "references": references, **kwargs} + results.append(evaluation_module.compute(**batch)) + + return self._merge_results(results) + + def _merge_results(self, results): + merged_results = {} + results_keys = list(itertools.chain.from_iterable([r.keys() for r in results])) + duplicate_keys = {item for item, count in collections.Counter(results_keys).items() if count > 1} + + duplicate_names = [ + item for item, count in collections.Counter(self.evaluation_module_names).items() if count > 1 + ] + duplicate_counter = {name: 0 for name in duplicate_names} + + for module_name, result in zip(self.evaluation_module_names, results): + for k, v in result.items(): + if k not in duplicate_keys and not self.force_prefix: + merged_results[f"{k}"] = v + elif module_name in duplicate_counter: + merged_results[f"{module_name}_{duplicate_counter[module_name]}_{k}"] = v + else: + merged_results[f"{module_name}_{k}"] = v + + if module_name in duplicate_counter: + duplicate_counter[module_name] += 1 + + return merged_results + + +def combine(evaluations, force_prefix=False): + """Combines several metrics, comparisons, or measurements into a single `CombinedEvaluations` object that + can be used like a single evaluation module. + + If two scores have the same name, then they are prefixed with their module names. + And if two modules have the same name, please use a dictionary to give them different names, otherwise an integer id is appended to the prefix. + + Args: + evaluations (`Union[list, dict]`): + A list or dictionary of evaluation modules. The modules can either be passed + as strings or loaded `EvaluationModule`s. If a dictionary is passed its keys are the names used and the values the modules. + The names are used as prefix in case there are name overlaps in the returned results of each module or if `force_prefix=True`. + force_prefix (`bool`, *optional*, defaults to `False`): + If `True` all scores from the modules are prefixed with their name. If + a dictionary is passed the keys are used as name otherwise the module's name. + + Examples: + + ```py + >>> import evaluate + >>> accuracy = evaluate.load("accuracy") + >>> f1 = evaluate.load("f1") + >>> clf_metrics = combine(["accuracy", "f1"]) + ``` + """ + + return CombinedEvaluations(evaluations, force_prefix=force_prefix) diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/naming.py b/TextClassification/TextClassification/evaluate/src/evaluate/naming.py new file mode 100644 index 0000000000000000000000000000000000000000..6335cf1b0ff47f0f2a409d6641fd5c528a31e949 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/naming.py @@ -0,0 +1,82 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Utilities for file names.""" + +import itertools +import os +import re + + +_uppercase_uppercase_re = re.compile(r"([A-Z]+)([A-Z][a-z])") +_lowercase_uppercase_re = re.compile(r"([a-z\d])([A-Z])") + +_single_underscore_re = re.compile(r"(?>> import evaluate + >>> result = {"bleu": 0.7} + >>> params = {"model": "gpt-2"} + >>> evaluate.save("./results/", **result, **params) + ``` + """ + current_time = datetime.now() + + file_path = _setup_path(path_or_file, current_time) + + data["_timestamp"] = current_time.isoformat() + data["_git_commit_hash"] = _git_commit_hash() + data["_evaluate_version"] = __version__ + data["_python_version"] = sys.version + data["_interpreter_path"] = sys.executable + + with FileLock(str(file_path) + ".lock"): + with open(file_path, "w") as f: + json.dump(data, f) + + # cleanup lock file + try: + os.remove(str(file_path) + ".lock") + except FileNotFoundError: + pass + + return file_path + + +def _setup_path(path_or_file, current_time): + path_or_file = Path(path_or_file) + is_file = len(path_or_file.suffix) > 0 + if is_file: + folder = path_or_file.parent + file_name = path_or_file.name + else: + folder = path_or_file + file_name = "result-" + current_time.strftime("%Y_%m_%d-%H_%M_%S") + ".json" + folder.mkdir(parents=True, exist_ok=True) + return folder / file_name + + +def _git_commit_hash(): + res = subprocess.run("git rev-parse --is-inside-work-tree".split(), cwd="./", stdout=subprocess.PIPE) + if res.stdout.decode().strip() == "true": + res = subprocess.run("git rev-parse HEAD".split(), cwd=os.getcwd(), stdout=subprocess.PIPE) + return res.stdout.decode().strip() + else: + return None diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/utils/__init__.py b/TextClassification/TextClassification/evaluate/src/evaluate/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..fac692eeb80def645bce9901324902ac9e1899f5 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/utils/__init__.py @@ -0,0 +1,39 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# flake8: noqa +# Lint as: python3 +"""Util import.""" + +__all__ = [ + "disable_progress_bar", + "enable_progress_bar", + "is_progress_bar_enabled", + "infer_gradio_input_types", + "json_to_string_type", + "parse_readme", + "parse_gradio_data", + "parse_test_cases", + "launch_gradio_widget", +] + +from .gradio import ( + infer_gradio_input_types, + json_to_string_type, + launch_gradio_widget, + parse_gradio_data, + parse_readme, + parse_test_cases, +) +from .logging import disable_progress_bar, enable_progress_bar, is_progress_bar_enabled diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/utils/file_utils.py b/TextClassification/TextClassification/evaluate/src/evaluate/utils/file_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c52d44a958e328e498ce55562767cc0a81e63980 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/utils/file_utils.py @@ -0,0 +1,618 @@ +""" +Utilities for working with the local dataset cache. +This file is adapted from the AllenNLP library at https://github.com/allenai/allennlp +Copyright by the AllenNLP authors. +""" + +import copy +import io +import json +import os +import posixpath +import re +import shutil +import sys +import tempfile +import time +import urllib +from contextlib import closing, contextmanager +from functools import partial +from hashlib import sha256 +from pathlib import Path +from typing import List, Optional, Type, TypeVar, Union +from urllib.parse import urljoin, urlparse + +import requests +from datasets import DownloadConfig +from datasets.utils.extract import ExtractManager +from datasets.utils.filelock import FileLock + +from .. import __version__, config +from . import logging + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +INCOMPLETE_SUFFIX = ".incomplete" + +T = TypeVar("T", str, Path) + + +def init_hf_modules(hf_modules_cache: Optional[Union[Path, str]] = None) -> str: + """ + Add hf_modules_cache to the python path. + By default hf_modules_cache='~/.cache/huggingface/modules'. + It can also be set with the environment variable HF_MODULES_CACHE. + This is used to add modules such as `datasets_modules` + """ + hf_modules_cache = hf_modules_cache if hf_modules_cache is not None else config.HF_MODULES_CACHE + hf_modules_cache = str(hf_modules_cache) + if hf_modules_cache not in sys.path: + sys.path.append(hf_modules_cache) + + os.makedirs(hf_modules_cache, exist_ok=True) + if not os.path.exists(os.path.join(hf_modules_cache, "__init__.py")): + with open(os.path.join(hf_modules_cache, "__init__.py"), "w"): + pass + return hf_modules_cache + + +def is_remote_url(url_or_filename: str) -> bool: + parsed = urlparse(url_or_filename) + return parsed.scheme in ("http", "https", "s3", "gs", "hdfs", "ftp") + + +def is_local_path(url_or_filename: str) -> bool: + # On unix the scheme of a local path is empty (for both absolute and relative), + # while on windows the scheme is the drive name (ex: "c") for absolute paths. + # for details on the windows behavior, see https://bugs.python.org/issue42215 + return urlparse(url_or_filename).scheme == "" or os.path.ismount(urlparse(url_or_filename).scheme + ":/") + + +def is_relative_path(url_or_filename: str) -> bool: + return urlparse(url_or_filename).scheme == "" and not os.path.isabs(url_or_filename) + + +def relative_to_absolute_path(path: T) -> T: + """Convert relative path to absolute path.""" + abs_path_str = os.path.abspath(os.path.expanduser(os.path.expandvars(str(path)))) + return Path(abs_path_str) if isinstance(path, Path) else abs_path_str + + +def hf_bucket_url(identifier: str, filename: str, use_cdn=False, dataset=True) -> str: + if dataset: + endpoint = config.CLOUDFRONT_DATASETS_DISTRIB_PREFIX if use_cdn else config.S3_DATASETS_BUCKET_PREFIX + else: + endpoint = config.CLOUDFRONT_METRICS_DISTRIB_PREFIX if use_cdn else config.S3_METRICS_BUCKET_PREFIX + return "/".join((endpoint, identifier, filename)) + + +def head_hf_s3( + identifier: str, filename: str, use_cdn=False, dataset=True, max_retries=0 +) -> Union[requests.Response, Exception]: + return http_head( + hf_bucket_url(identifier=identifier, filename=filename, use_cdn=use_cdn, dataset=dataset), + max_retries=max_retries, + ) + + +def hf_hub_url(path: str, name: str, revision: Optional[str] = None) -> str: + revision = revision or config.HUB_DEFAULT_VERSION + return config.HUB_EVALUATE_URL.format(path=path, name=name, revision=revision) + + +def url_or_path_join(base_name: str, *pathnames: str) -> str: + if is_remote_url(base_name): + return posixpath.join(base_name, *(str(pathname).replace(os.sep, "/").lstrip("/") for pathname in pathnames)) + else: + return Path(base_name, *pathnames).as_posix() + + +def url_or_path_parent(url_or_path: str) -> str: + if is_remote_url(url_or_path): + return url_or_path[: url_or_path.rindex("/")] + else: + return os.path.dirname(url_or_path) + + +def hash_url_to_filename(url, etag=None): + """ + Convert `url` into a hashed filename in a repeatable way. + If `etag` is specified, append its hash to the url's, delimited + by a period. + If the url ends with .h5 (Keras HDF5 weights) adds '.h5' to the name + so that TF 2.0 can identify it as a HDF5 file + (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380) + """ + url_bytes = url.encode("utf-8") + url_hash = sha256(url_bytes) + filename = url_hash.hexdigest() + + if etag: + etag_bytes = etag.encode("utf-8") + etag_hash = sha256(etag_bytes) + filename += "." + etag_hash.hexdigest() + + if url.endswith(".py"): + filename += ".py" + + return filename + + +def cached_path( + url_or_filename, + download_config=None, + **download_kwargs, +) -> str: + """ + Given something that might be a URL (or might be a local path), + determine which. If it's a URL, download the file and cache it, and + return the path to the cached file. If it's already a local path, + make sure the file exists and then return the path. + + Return: + Local path (string) + + Raises: + FileNotFoundError: in case of non-recoverable file + (non-existent or no cache on disk) + ConnectionError: in case of unreachable url + and no cache on disk + ValueError: if it couldn't parse the url or filename correctly + requests.exceptions.ConnectionError: in case of internet connection issue + """ + if download_config is None: + download_config = DownloadConfig(**download_kwargs) + + cache_dir = download_config.cache_dir or config.DOWNLOADED_EVALUATE_PATH + if isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + if isinstance(url_or_filename, Path): + url_or_filename = str(url_or_filename) + + if is_remote_url(url_or_filename): + # URL, so get it from the cache (downloading if necessary) + output_path = get_from_cache( + url_or_filename, + cache_dir=cache_dir, + force_download=download_config.force_download, + proxies=download_config.proxies, + resume_download=download_config.resume_download, + user_agent=download_config.user_agent, + local_files_only=download_config.local_files_only, + use_etag=download_config.use_etag, + max_retries=download_config.max_retries, + use_auth_token=download_config.use_auth_token, + ignore_url_params=download_config.ignore_url_params, + download_desc=download_config.download_desc, + ) + elif os.path.exists(url_or_filename): + # File, and it exists. + output_path = url_or_filename + elif is_local_path(url_or_filename): + # File, but it doesn't exist. + raise FileNotFoundError(f"Local file {url_or_filename} doesn't exist") + else: + # Something unknown + raise ValueError(f"unable to parse {url_or_filename} as a URL or as a local path") + + if output_path is None: + return output_path + + if download_config.extract_compressed_file: + output_path = ExtractManager(cache_dir=download_config.cache_dir).extract( + output_path, force_extract=download_config.force_extract + ) + + return output_path + + +def get_datasets_user_agent(user_agent: Optional[Union[str, dict]] = None) -> str: + ua = f"datasets/{__version__}; python/{config.PY_VERSION}" + ua += f"; pyarrow/{config.PYARROW_VERSION}" + if config.TORCH_AVAILABLE: + ua += f"; torch/{config.TORCH_VERSION}" + if config.TF_AVAILABLE: + ua += f"; tensorflow/{config.TF_VERSION}" + if config.JAX_AVAILABLE: + ua += f"; jax/{config.JAX_VERSION}" + if isinstance(user_agent, dict): + ua += f"; {'; '.join(f'{k}/{v}' for k, v in user_agent.items())}" + elif isinstance(user_agent, str): + ua += "; " + user_agent + return ua + + +def get_authentication_headers_for_url(url: str, use_auth_token: Optional[Union[str, bool]] = None) -> dict: + """Handle the HF authentication""" + headers = {} + if url.startswith(config.HF_ENDPOINT): + token = None + if isinstance(use_auth_token, str): + token = use_auth_token + elif bool(use_auth_token): + from huggingface_hub import hf_api + + token = hf_api.HfFolder.get_token() + if token: + headers["authorization"] = f"Bearer {token}" + return headers + + +class OfflineModeIsEnabled(ConnectionError): + pass + + +def _raise_if_offline_mode_is_enabled(msg: Optional[str] = None): + """Raise an OfflineModeIsEnabled error (subclass of ConnectionError) if HF_EVALUATE_OFFLINE is True.""" + if config.HF_EVALUATE_OFFLINE: + raise OfflineModeIsEnabled( + "Offline mode is enabled." if msg is None else "Offline mode is enabled. " + str(msg) + ) + + +def _retry( + func, + func_args: Optional[tuple] = None, + func_kwargs: Optional[dict] = None, + exceptions: Type[requests.exceptions.RequestException] = requests.exceptions.RequestException, + status_codes: Optional[List[int]] = None, + max_retries: int = 0, + base_wait_time: float = 0.5, + max_wait_time: float = 2, +): + func_args = func_args or () + func_kwargs = func_kwargs or {} + retry = 0 + while True: + try: + return func(*func_args, **func_kwargs) + except exceptions as err: + if retry >= max_retries or (status_codes and err.response.status_code not in status_codes): + raise err + else: + sleep_time = min(max_wait_time, base_wait_time * 2**retry) # Exponential backoff + logger.info(f"{func} timed out, retrying in {sleep_time}s... [{retry/max_retries}]") + time.sleep(sleep_time) + retry += 1 + + +def _request_with_retry( + method: str, + url: str, + max_retries: int = 0, + base_wait_time: float = 0.5, + max_wait_time: float = 2, + timeout: float = 10.0, + **params, +) -> requests.Response: + """Wrapper around requests to retry in case it fails with a ConnectTimeout, with exponential backoff. + + Note that if the environment variable HF_EVALUATE_OFFLINE is set to 1, then a OfflineModeIsEnabled error is raised. + + Args: + method (str): HTTP method, such as 'GET' or 'HEAD'. + url (str): The URL of the resource to fetch. + max_retries (int): Maximum number of retries, defaults to 0 (no retries). + base_wait_time (float): Duration (in seconds) to wait before retrying the first time. Wait time between + retries then grows exponentially, capped by max_wait_time. + max_wait_time (float): Maximum amount of time between two retries, in seconds. + **params: Params to pass to :obj:`requests.request`. + """ + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + tries, success = 0, False + while not success: + tries += 1 + try: + response = requests.request(method=method.upper(), url=url, timeout=timeout, **params) + success = True + except (requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError) as err: + if tries > max_retries: + raise err + else: + logger.info(f"{method} request to {url} timed out, retrying... [{tries/max_retries}]") + sleep_time = min(max_wait_time, base_wait_time * 2 ** (tries - 1)) # Exponential backoff + time.sleep(sleep_time) + return response + + +def ftp_head(url, timeout=10.0): + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + try: + with closing(urllib.request.urlopen(url, timeout=timeout)) as r: + r.read(1) + except Exception: + return False + return True + + +def ftp_get(url, temp_file, timeout=10.0): + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + try: + logger.info(f"Getting through FTP {url} into {temp_file.name}") + with closing(urllib.request.urlopen(url, timeout=timeout)) as r: + shutil.copyfileobj(r, temp_file) + except urllib.error.URLError as e: + raise ConnectionError(e) from None + + +def http_get( + url, temp_file, proxies=None, resume_size=0, headers=None, cookies=None, timeout=100.0, max_retries=0, desc=None +): + headers = copy.deepcopy(headers) or {} + headers["user-agent"] = get_datasets_user_agent(user_agent=headers.get("user-agent")) + if resume_size > 0: + headers["Range"] = f"bytes={resume_size:d}-" + response = _request_with_retry( + method="GET", + url=url, + stream=True, + proxies=proxies, + headers=headers, + cookies=cookies, + max_retries=max_retries, + timeout=timeout, + ) + if response.status_code == 416: # Range not satisfiable + return + content_length = response.headers.get("Content-Length") + total = resume_size + int(content_length) if content_length is not None else None + with logging.tqdm( + unit="B", + unit_scale=True, + total=total, + initial=resume_size, + desc=desc or "Downloading", + disable=not logging.is_progress_bar_enabled(), + ) as progress: + for chunk in response.iter_content(chunk_size=1024): + progress.update(len(chunk)) + temp_file.write(chunk) + + +def http_head( + url, proxies=None, headers=None, cookies=None, allow_redirects=True, timeout=10.0, max_retries=0 +) -> requests.Response: + headers = copy.deepcopy(headers) or {} + headers["user-agent"] = get_datasets_user_agent(user_agent=headers.get("user-agent")) + response = _request_with_retry( + method="HEAD", + url=url, + proxies=proxies, + headers=headers, + cookies=cookies, + allow_redirects=allow_redirects, + timeout=timeout, + max_retries=max_retries, + ) + return response + + +def request_etag(url: str, use_auth_token: Optional[Union[str, bool]] = None) -> Optional[str]: + headers = get_authentication_headers_for_url(url, use_auth_token=use_auth_token) + response = http_head(url, headers=headers, max_retries=3) + response.raise_for_status() + etag = response.headers.get("ETag") if response.ok else None + return etag + + +def get_from_cache( + url, + cache_dir=None, + force_download=False, + proxies=None, + etag_timeout=100, + resume_download=False, + user_agent=None, + local_files_only=False, + use_etag=True, + max_retries=0, + use_auth_token=None, + ignore_url_params=False, + download_desc=None, +) -> str: + """ + Given a URL, look for the corresponding file in the local cache. + If it's not there, download it. Then return the path to the cached file. + + Return: + Local path (string) + + Raises: + FileNotFoundError: in case of non-recoverable file + (non-existent or no cache on disk) + ConnectionError: in case of unreachable url + and no cache on disk + """ + if cache_dir is None: + cache_dir = config.HF_EVALUATE_CACHE + if isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + + os.makedirs(cache_dir, exist_ok=True) + + if ignore_url_params: + # strip all query parameters and #fragments from the URL + cached_url = urljoin(url, urlparse(url).path) + else: + cached_url = url # additional parameters may be added to the given URL + + connected = False + response = None + cookies = None + etag = None + head_error = None + + # Try a first time to file the file on the local file system without eTag (None) + # if we don't ask for 'force_download' then we spare a request + filename = hash_url_to_filename(cached_url, etag=None) + cache_path = os.path.join(cache_dir, filename) + + if os.path.exists(cache_path) and not force_download and not use_etag: + return cache_path + + # Prepare headers for authentication + headers = get_authentication_headers_for_url(url, use_auth_token=use_auth_token) + if user_agent is not None: + headers["user-agent"] = user_agent + + # We don't have the file locally or we need an eTag + if not local_files_only: + if url.startswith("ftp://"): + connected = ftp_head(url) + try: + response = http_head( + url, + allow_redirects=True, + proxies=proxies, + timeout=etag_timeout, + max_retries=max_retries, + headers=headers, + ) + if response.status_code == 200: # ok + etag = response.headers.get("ETag") if use_etag else None + for k, v in response.cookies.items(): + # In some edge cases, we need to get a confirmation token + if k.startswith("download_warning") and "drive.google.com" in url: + url += "&confirm=" + v + cookies = response.cookies + connected = True + # Fix Google Drive URL to avoid Virus scan warning + if "drive.google.com" in url and "confirm=" not in url: + url += "&confirm=t" + # In some edge cases, head request returns 400 but the connection is actually ok + elif ( + (response.status_code == 400 and "firebasestorage.googleapis.com" in url) + or (response.status_code == 405 and "drive.google.com" in url) + or ( + response.status_code == 403 + and ( + re.match(r"^https?://github.com/.*?/.*?/releases/download/.*?/.*?$", url) + or re.match(r"^https://.*?s3.*?amazonaws.com/.*?$", response.url) + ) + ) + or (response.status_code == 403 and "ndownloader.figstatic.com" in url) + ): + connected = True + logger.info(f"Couldn't get ETag version for url {url}") + elif response.status_code == 401 and config.HF_ENDPOINT in url and use_auth_token is None: + raise ConnectionError( + f"Unauthorized for URL {url}. Please use the parameter ``use_auth_token=True`` after logging in with ``huggingface-cli login``" + ) + except (OSError, requests.exceptions.Timeout) as e: + # not connected + head_error = e + pass + + # connected == False = we don't have a connection, or url doesn't exist, or is otherwise inaccessible. + # try to get the last downloaded one + if not connected: + if os.path.exists(cache_path) and not force_download: + return cache_path + if local_files_only: + raise FileNotFoundError( + f"Cannot find the requested files in the cached path at {cache_path} and outgoing traffic has been" + " disabled. To enable file online look-ups, set 'local_files_only' to False." + ) + elif response is not None and response.status_code == 404: + raise FileNotFoundError(f"Couldn't find file at {url}") + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + if head_error is not None: + raise ConnectionError(f"Couldn't reach {url} ({repr(head_error)})") + elif response is not None: + raise ConnectionError(f"Couldn't reach {url} (error {response.status_code})") + else: + raise ConnectionError(f"Couldn't reach {url}") + + # Try a second time + filename = hash_url_to_filename(cached_url, etag) + cache_path = os.path.join(cache_dir, filename) + + if os.path.exists(cache_path) and not force_download: + return cache_path + + # From now on, connected is True. + # Prevent parallel downloads of the same file with a lock. + lock_path = cache_path + ".lock" + with FileLock(lock_path): + + if resume_download: + incomplete_path = cache_path + ".incomplete" + + @contextmanager + def _resumable_file_manager(): + with open(incomplete_path, "a+b") as f: + yield f + + temp_file_manager = _resumable_file_manager + if os.path.exists(incomplete_path): + resume_size = os.stat(incomplete_path).st_size + else: + resume_size = 0 + else: + temp_file_manager = partial(tempfile.NamedTemporaryFile, dir=cache_dir, delete=False) + resume_size = 0 + + # Download to temporary file, then copy to cache dir once finished. + # Otherwise you get corrupt cache entries if the download gets interrupted. + with temp_file_manager() as temp_file: + logger.info(f"{url} not found in cache or force_download set to True, downloading to {temp_file.name}") + + # GET file object + if url.startswith("ftp://"): + ftp_get(url, temp_file) + else: + http_get( + url, + temp_file, + proxies=proxies, + resume_size=resume_size, + headers=headers, + cookies=cookies, + max_retries=max_retries, + desc=download_desc, + ) + + logger.info(f"storing {url} in cache at {cache_path}") + shutil.move(temp_file.name, cache_path) + + logger.info(f"creating metadata file for {cache_path}") + meta = {"url": url, "etag": etag} + meta_path = cache_path + ".json" + with open(meta_path, "w", encoding="utf-8") as meta_file: + json.dump(meta, meta_file) + + return cache_path + + +def add_start_docstrings(*docstr): + def docstring_decorator(fn): + fn.__doc__ = "".join(docstr) + "\n\n" + (fn.__doc__ if fn.__doc__ is not None else "") + return fn + + return docstring_decorator + + +def add_end_docstrings(*docstr): + def docstring_decorator(fn): + fn.__doc__ = (fn.__doc__ if fn.__doc__ is not None else "") + "\n\n" + "".join(docstr) + return fn + + return docstring_decorator + + +def estimate_dataset_size(paths): + return sum(path.stat().st_size for path in paths) + + +def readline(f: io.RawIOBase): + # From: https://github.com/python/cpython/blob/d27e2f4d118e7a9909b6a3e5da06c5ff95806a85/Lib/_pyio.py#L525 + res = bytearray() + while True: + b = f.read(1) + if not b: + break + res += b + if res.endswith(b"\n"): + break + return bytes(res) diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/utils/gradio.py b/TextClassification/TextClassification/evaluate/src/evaluate/utils/gradio.py new file mode 100644 index 0000000000000000000000000000000000000000..3b73d9c67e711caad66edaf0f66808fff296aabd --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/utils/gradio.py @@ -0,0 +1,131 @@ +import json +import os +import re +import sys +from pathlib import Path + +import numpy as np +from datasets import Value + +from .logging import get_logger + + +logger = get_logger(__name__) + +REGEX_YAML_BLOCK = re.compile(r"---[\n\r]+([\S\s]*?)[\n\r]+---[\n\r]") + + +def infer_gradio_input_types(feature_types): + """ + Maps metric feature types to input types for gradio Dataframes: + - float/int -> numbers + - string -> strings + - any other -> json + Note that json is not a native gradio type but will be treated as string that + is then parsed as a json. + """ + input_types = [] + for feature_type in feature_types: + input_type = "json" + if isinstance(feature_type, Value): + if feature_type.dtype.startswith("int") or feature_type.dtype.startswith("float"): + input_type = "number" + elif feature_type.dtype == "string": + input_type = "str" + input_types.append(input_type) + return input_types + + +def json_to_string_type(input_types): + """Maps json input type to str.""" + return ["str" if i == "json" else i for i in input_types] + + +def parse_readme(filepath): + """Parses a repositories README and removes""" + if not os.path.exists(filepath): + return "No README.md found." + with open(filepath, "r") as f: + text = f.read() + match = REGEX_YAML_BLOCK.search(text) + if match: + text = text[match.end() :] + return text + + +def parse_gradio_data(data, input_types): + """Parses data from gradio Dataframe for use in metric.""" + metric_inputs = {} + data.replace("", np.nan, inplace=True) + data.dropna(inplace=True) + for feature_name, input_type in zip(data, input_types): + if input_type == "json": + metric_inputs[feature_name] = [json.loads(d) for d in data[feature_name].to_list()] + elif input_type == "str": + metric_inputs[feature_name] = [d.strip('"') for d in data[feature_name].to_list()] + else: + metric_inputs[feature_name] = data[feature_name] + return metric_inputs + + +def parse_test_cases(test_cases, feature_names, input_types): + """ + Parses test cases to be used in gradio Dataframe. Note that an apostrophe is added + to strings to follow the format in json. + """ + if len(test_cases) == 0: + return None + examples = [] + for test_case in test_cases: + parsed_cases = [] + for feat, input_type in zip(feature_names, input_types): + if input_type == "json": + parsed_cases.append([str(element) for element in test_case[feat]]) + elif input_type == "str": + parsed_cases.append(['"' + element + '"' for element in test_case[feat]]) + else: + parsed_cases.append(test_case[feat]) + examples.append([list(i) for i in zip(*parsed_cases)]) + return examples + + +def launch_gradio_widget(metric): + """Launches `metric` widget with Gradio.""" + + try: + import gradio as gr + except ImportError as error: + logger.error("To create a metric widget with Gradio make sure gradio is installed.") + raise error + + local_path = Path(sys.path[0]) + # if there are several input types, use first as default. + if isinstance(metric.features, list): + (feature_names, feature_types) = zip(*metric.features[0].items()) + else: + (feature_names, feature_types) = zip(*metric.features.items()) + gradio_input_types = infer_gradio_input_types(feature_types) + + def compute(data): + return metric.compute(**parse_gradio_data(data, gradio_input_types)) + + iface = gr.Interface( + fn=compute, + inputs=gr.inputs.Dataframe( + headers=feature_names, + col_count=len(feature_names), + row_count=1, + datatype=json_to_string_type(gradio_input_types), + ), + outputs=gr.outputs.Textbox(label=metric.name), + description=( + metric.info.description + "\nIf this is a text-based metric, make sure to wrap you input in double quotes." + " Alternatively you can use a JSON-formatted list as input." + ), + title=f"Metric: {metric.name}", + article=parse_readme(local_path / "README.md"), + # TODO: load test cases and use them to populate examples + # examples=[parse_test_cases(test_cases, feature_names, gradio_input_types)] + ) + + iface.launch() diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/utils/logging.py b/TextClassification/TextClassification/evaluate/src/evaluate/utils/logging.py new file mode 100644 index 0000000000000000000000000000000000000000..8df58d3dcfb4c8b903b78c244387561dc659e423 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/utils/logging.py @@ -0,0 +1,234 @@ +# Copyright 2020 Optuna, Hugging Face +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Logging utilities. """ + +import logging +import os +from logging import CRITICAL # NOQA +from logging import DEBUG # NOQA +from logging import ERROR # NOQA +from logging import FATAL # NOQA +from logging import INFO # NOQA +from logging import NOTSET # NOQA +from logging import WARN # NOQA +from logging import WARNING # NOQA +from typing import Optional + +from tqdm import auto as tqdm_lib + + +log_levels = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, +} + +_default_log_level = logging.WARNING + + +def _get_default_logging_level(): + """ + If EVALUATE_VERBOSITY env var is set to one of the valid choices return that as the new default level. + If it is not - fall back to ``_default_log_level`` + """ + env_level_str = os.getenv("EVALUATE_VERBOSITY", None) + if env_level_str: + if env_level_str in log_levels: + return log_levels[env_level_str] + else: + logging.getLogger().warning( + f"Unknown option EVALUATE_VERBOSITY={env_level_str}, " + f"has to be one of: { ', '.join(log_levels.keys()) }" + ) + return _default_log_level + + +def _get_library_name() -> str: + return __name__.split(".")[0] + + +def _get_library_root_logger() -> logging.Logger: + return logging.getLogger(_get_library_name()) + + +def _configure_library_root_logger() -> None: + # Apply our default configuration to the library root logger. + library_root_logger = _get_library_root_logger() + library_root_logger.setLevel(_get_default_logging_level()) + + +def _reset_library_root_logger() -> None: + library_root_logger = _get_library_root_logger() + library_root_logger.setLevel(logging.NOTSET) + + +def get_logger(name: Optional[str] = None) -> logging.Logger: + """Return a logger with the specified name.""" + if name is None: + name = _get_library_name() + return logging.getLogger(name) + + +def get_verbosity() -> int: + """Return the current level for the Hugging Face Evaluate library's root logger. + Returns: + Logging level, e.g., `evaluate.logging.DEBUG` and `evaluate.logging.INFO`. + + + + Hugging Face Evaluate library has following logging levels: + - `evaluate.logging.CRITICAL`, `evaluate.logging.FATAL` + - `evaluate.logging.ERROR` + - `evaluate.logging.WARNING`, `evaluate.logging.WARN` + - `evaluate.logging.INFO` + - `evaluate.logging.DEBUG` + + + """ + return _get_library_root_logger().getEffectiveLevel() + + +def set_verbosity(verbosity: int) -> None: + """Set the level for the Hugging Face Evaluate library's root logger. + Args: + verbosity: + Logging level, e.g., `evaluate.logging.DEBUG` and `evaluate.logging.INFO`. + """ + _get_library_root_logger().setLevel(verbosity) + + +def set_verbosity_info(): + """Set the level for the Hugging Face Evaluate library's root logger to `INFO`. + + This will display most of the logging information and tqdm bars. + + Shortcut to `evaluate.logging.set_verbosity(evaluate.logging.INFO)`. + """ + return set_verbosity(INFO) + + +def set_verbosity_warning(): + """Set the level for the Hugging Face Evaluate library's root logger to `WARNING`. + + This will display only the warning and errors logging information and tqdm bars. + + Shortcut to `evaluate.logging.set_verbosity(evaluate.logging.WARNING)`. + """ + return set_verbosity(WARNING) + + +def set_verbosity_debug(): + """Set the level for the Hugging Face Evaluate library's root logger to `DEBUG`. + + This will display all the logging information and tqdm bars. + + Shortcut to `evaluate.logging.set_verbosity(evaluate.logging.DEBUG)`. + """ + return set_verbosity(DEBUG) + + +def set_verbosity_error(): + """Set the level for the Hugging Face Evaluate library's root logger to `ERROR`. + + This will display only the errors logging information and tqdm bars. + + Shortcut to `evaluate.logging.set_verbosity(evaluate.logging.ERROR)`. + """ + return set_verbosity(ERROR) + + +def disable_propagation() -> None: + """Disable propagation of the library log outputs. + Note that log propagation is disabled by default. + """ + _get_library_root_logger().propagate = False + + +def enable_propagation() -> None: + """Enable propagation of the library log outputs. + Please disable the Hugging Face Evaluate library's default handler to prevent double logging if the root logger has + been configured. + """ + _get_library_root_logger().propagate = True + + +# Configure the library root logger at the module level (singleton-like) +_configure_library_root_logger() + + +class EmptyTqdm: + """Dummy tqdm which doesn't do anything.""" + + def __init__(self, *args, **kwargs): # pylint: disable=unused-argument + self._iterator = args[0] if args else None + + def __iter__(self): + return iter(self._iterator) + + def __getattr__(self, _): + """Return empty function.""" + + def empty_fn(*args, **kwargs): # pylint: disable=unused-argument + return + + return empty_fn + + def __enter__(self): + return self + + def __exit__(self, type_, value, traceback): + return + + +_tqdm_active = True + + +class _tqdm_cls: + def __call__(self, *args, **kwargs): + if _tqdm_active: + return tqdm_lib.tqdm(*args, **kwargs) + else: + return EmptyTqdm(*args, **kwargs) + + def set_lock(self, *args, **kwargs): + self._lock = None + if _tqdm_active: + return tqdm_lib.tqdm.set_lock(*args, **kwargs) + + def get_lock(self): + if _tqdm_active: + return tqdm_lib.tqdm.get_lock() + + +tqdm = _tqdm_cls() + + +def is_progress_bar_enabled() -> bool: + """Return a boolean indicating whether tqdm progress bars are enabled.""" + global _tqdm_active + return bool(_tqdm_active) + + +def enable_progress_bar(): + """Enable tqdm progress bar.""" + global _tqdm_active + _tqdm_active = True + + +def disable_progress_bar(): + """Enable tqdm progress bar.""" + global _tqdm_active + _tqdm_active = False diff --git a/TextClassification/TextClassification/evaluate/src/evaluate/visualization.py b/TextClassification/TextClassification/evaluate/src/evaluate/visualization.py new file mode 100644 index 0000000000000000000000000000000000000000..b8be8605805e4d11ef93a2911ab32afe934a78bb --- /dev/null +++ b/TextClassification/TextClassification/evaluate/src/evaluate/visualization.py @@ -0,0 +1,230 @@ +import textwrap + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd + + +class ComplexRadar: + """Create a complex radar chart with different scales for each variable + Args: + fig (`matplotlib.figure`) : A matplotlib figure object to add the axes on. + variables (`list`) : a list of variables to. plot + ranges (`list` of `tuples`): A list of ranges (min, max) for each variable + n_ring_levels (`int): Number of ordinate or ring levels to draw. + Default: 5. + show_scales (`bool`): Indicates if we the ranges for each variable are plotted. + Default: True. + format_cfg (`dict`): A dictionary with formatting configurations. + Default: None. + Returns: + `matplotlib.figure.Figure`: a radar plot. + """ + + def __init__(self, fig, variables, ranges, n_ring_levels=5, show_scales=True, format_cfg=None): + + self.format_cfg = format_cfg + + # Calculate angles and create for each variable an axes + # Consider here the trick with having the first axes element twice (len+1) + angles = np.arange(0, 360, 360.0 / len(variables)) + axes = [ + fig.add_axes([0.1, 0.1, 0.9, 0.9], polar=True, label="axes{}".format(i), **self.format_cfg["axes_args"]) + for i in range(len(variables) + 1) + ] + + # Ensure clockwise rotation (first variable at the top N) + for ax in axes: + ax.set_theta_zero_location("N") + ax.set_theta_direction(-1) + ax.set_axisbelow(True) + + # Writing the ranges on each axes + for i, ax in enumerate(axes): + + # Here we do the trick by repeating the first iteration + j = 0 if (i == 0 or i == 1) else i - 1 + ax.set_ylim(*ranges[j]) + # Set endpoint to True if you like to have values right before the last circle + grid = np.linspace(*ranges[j], num=n_ring_levels, endpoint=self.format_cfg["incl_endpoint"]) + gridlabel = ["{}".format(round(x, 2)) for x in grid] + gridlabel[0] = "" # remove values from the center + lines, labels = ax.set_rgrids( + grid, labels=gridlabel, angle=angles[j], **self.format_cfg["rgrid_tick_lbls_args"] + ) + + ax.set_ylim(*ranges[j]) + ax.spines["polar"].set_visible(False) + ax.grid(visible=False) + + if show_scales is False: + ax.set_yticklabels([]) + + # Set all axes except the first one unvisible + for ax in axes[1:]: + ax.patch.set_visible(False) + ax.xaxis.set_visible(False) + + # Setting the attributes + self.angle = np.deg2rad(np.r_[angles, angles[0]]) + self.ranges = ranges + self.ax = axes[0] + self.ax1 = axes[1] + self.plot_counter = 0 + + # Draw (inner) circles and lines + self.ax.yaxis.grid(**self.format_cfg["rad_ln_args"]) + # Draw outer circle + self.ax.spines["polar"].set(**self.format_cfg["outer_ring"]) + # Draw angle lines + self.ax.xaxis.grid(**self.format_cfg["angle_ln_args"]) + + # ax1 is the duplicate of axes[0] (self.ax) + # Remove everything from ax1 except the plot itself + self.ax1.axis("off") + self.ax1.set_zorder(9) + + # Create the outer labels for each variable + l, text = self.ax.set_thetagrids(angles, labels=variables) + + # Beautify them + labels = [t.get_text() for t in self.ax.get_xticklabels()] + labels = [ + "\n".join( + textwrap.wrap( + label, + self.format_cfg["theta_tick_lbls_txt_wrap"], + break_long_words=self.format_cfg["theta_tick_lbls_brk_lng_wrds"], + ) + ) + for label in labels + ] + self.ax.set_xticklabels(labels, **self.format_cfg["theta_tick_lbls"]) + + for t, a in zip(self.ax.get_xticklabels(), angles): + if a == 0: + t.set_ha("center") + elif a > 0 and a < 180: + t.set_ha("left") + elif a == 180: + t.set_ha("center") + else: + t.set_ha("right") + + self.ax.tick_params(axis="both", pad=self.format_cfg["theta_tick_lbls_pad"]) + + def _scale_data(self, data, ranges): + """Scales data[1:] to ranges[0]""" + for d, (y1, y2) in zip(data[1:], ranges[1:]): + assert (y1 <= d <= y2) or (y2 <= d <= y1) + x1, x2 = ranges[0] + d = data[0] + sdata = [d] + for d, (y1, y2) in zip(data[1:], ranges[1:]): + sdata.append((d - y1) / (y2 - y1) * (x2 - x1) + x1) + return sdata + + def plot(self, data, *args, **kwargs): + """Plots a line""" + sdata = self._scale_data(data, self.ranges) + self.ax1.plot(self.angle, np.r_[sdata, sdata[0]], *args, **kwargs) + self.plot_counter = self.plot_counter + 1 + + def use_legend(self, *args, **kwargs): + """Shows a legend""" + self.ax1.legend(*args, **kwargs) + + +def radar_plot(data, model_names, invert_range=[], config=None, fig=None): + """Create a complex radar chart with different scales for each variable + Source: https://towardsdatascience.com/how-to-create-and-visualize-complex-radar-charts-f7764d0f3652 + + Args: + data (`List[dict]`): the results (list of metric + value pairs). + E.g. data = [{"accuracy": 0.9, "precision":0.8},{"accuracy": 0.7, "precision":0.6}] + names (`List[dict]`): model names. + E.g. names = ["model1", "model 2", ...] + invert_range (`List[dict]`, optional): the metrics to invert (in cases when smaller is better, e.g. speed) + E.g. invert_range=["latency_in_seconds"] + config (`dict`, optional) : a specification of the formatting configurations, namely: + + - rad_ln_args (`dict`, default `{"visible": True}`): The visibility of the radial (circle) lines. + + - outer_ring (`dict`, default `{"visible": True}`): The visibility of the outer ring. + + - angle_ln_args (`dict`, default `{"visible": True}`): The visibility of the angle lines. + + - rgrid_tick_lbls_args (`dict`, default `{"fontsize": 12}`): The font size of the tick labels on the scales. + + - theta_tick_lbls (`dict`, default `{"fontsize": 12}`): The font size of the variable labels on the plot. + + - theta_tick_lbls_pad (`int`, default `3`): The padding of the variable labels on the plot. + + - theta_tick_lbls_brk_lng_wrds (`bool`, default `True` ): Whether long words in the label are broken up or not. + + - theta_tick_lbls_txt_wrap (`int`, default `15`): Text wrap for tick labels + + - incl_endpoint (`bool`, default `False`): Include value endpoints on calse + + - marker (`str`, default `"o"`): the shape of the marker used in the radar plot. + + - markersize (`int`, default `3`): the shape of the marker used in the radar plot. + + - legend_loc (`str`, default `"upper right"`): the location of the legend in the radar plot. Must be one of: 'upper left', 'upper right', 'lower left', 'lower right'. + + - bbox_to_anchor (`tuple`, default `(2, 1)`: anchor for the legend. + fig (`matplotlib.figure.Figure`, optional): figure used to plot the radar plot. + + Returns: + `matplotlib.figure.Figure` + """ + data = pd.DataFrame(data) + data.index = model_names + variables = data.keys() + if all(x in variables for x in invert_range) is False: + raise ValueError("All of the metrics in `invert_range` should be in the data provided.") + min_max_per_variable = data.describe().T[["min", "max"]] + min_max_per_variable["min"] = min_max_per_variable["min"] - 0.1 * ( + min_max_per_variable["max"] - min_max_per_variable["min"] + ) + min_max_per_variable["max"] = min_max_per_variable["max"] + 0.1 * ( + min_max_per_variable["max"] - min_max_per_variable["min"] + ) + + ranges = list(min_max_per_variable.itertuples(index=False, name=None)) + ranges = [ + (max_value, min_value) if var in invert_range else (min_value, max_value) + for var, (min_value, max_value) in zip(variables, ranges) + ] + format_cfg = { + "axes_args": {}, + "rad_ln_args": {"visible": True}, + "outer_ring": {"visible": True}, + "angle_ln_args": {"visible": True}, + "rgrid_tick_lbls_args": {"fontsize": 12}, + "theta_tick_lbls": {"fontsize": 12}, + "theta_tick_lbls_pad": 3, + "theta_tick_lbls_brk_lng_wrds": True, + "theta_tick_lbls_txt_wrap": 15, + "incl_endpoint": False, + "marker": "o", + "markersize": 3, + "legend_loc": "upper right", + "bbox_to_anchor": (2, 1), + } + if config is not None: + format_cfg.update(config) + if fig is None: + fig = plt.figure() + radar = ComplexRadar( + fig, + variables, + ranges, + n_ring_levels=3, + show_scales=True, + format_cfg=format_cfg, + ) + for g in zip(data.index): + radar.plot(data.loc[g].values, label=g, marker=format_cfg["marker"], markersize=format_cfg["markersize"]) + radar.use_legend(**{"loc": format_cfg["legend_loc"], "bbox_to_anchor": format_cfg["bbox_to_anchor"]}) + return fig diff --git a/TextClassification/TextClassification/evaluate/templates/cookiecutter.json b/TextClassification/TextClassification/evaluate/templates/cookiecutter.json new file mode 100644 index 0000000000000000000000000000000000000000..44ad5052d08a72f946798ef6e572059541bb73ec --- /dev/null +++ b/TextClassification/TextClassification/evaluate/templates/cookiecutter.json @@ -0,0 +1,9 @@ +{ + "module_name": "Awesome Module", + "module_type": "module", + "module_description": "This new module is designed to solve this great ML task and is crafted with a lot of care and love.", + "module_slug": "{{ cookiecutter.module_name|lower|replace(' ', '_') }}", + "module_class_name": "{{ cookiecutter.module_name|replace(' ', '') }}", + "namespace": "", + "dataset_name": "" +} \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/README.md b/TextClassification/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1168061ad8adf4105e06a21741c615ec93acea99 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/README.md @@ -0,0 +1,50 @@ +--- +title: {{ cookiecutter.module_name }} +datasets: +- {{ cookiecutter.dataset_name }} +tags: +- evaluate +- {{ cookiecutter.module_type }} +description: "TODO: add a description here" +sdk: gradio +sdk_version: 3.19.1 +app_file: app.py +pinned: false +--- + +# {{ cookiecutter.module_type|capitalize }} Card for {{ cookiecutter.module_name }} + +***Module Card Instructions:*** *Fill out the following subsections. Feel free to take a look at existing {{ cookiecutter.module_type }} cards if you'd like examples.* + +## {{ cookiecutter.module_type|capitalize }} Description +*Give a brief overview of this {{ cookiecutter.module_type }}, including what task(s) it is usually used for, if any.* + +## How to Use +*Give general statement of how to use the {{ cookiecutter.module_type }}* + +*Provide simplest possible example for using the {{ cookiecutter.module_type }}* + +### Inputs +*List all input arguments in the format below* +- **input_field** *(type): Definition of input, with explanation if necessary. State any default value(s).* + +### Output Values + +*Explain what this {{ cookiecutter.module_type }} outputs and provide an example of what the {{ cookiecutter.module_type }} output looks like. Modules should return a dictionary with one or multiple key-value pairs, e.g. {"bleu" : 6.02}* + +*State the range of possible values that the {{ cookiecutter.module_type }}'s output can take, as well as what in that range is considered good. For example: "This {{ cookiecutter.module_type }} can take on any value between 0 and 100, inclusive. Higher scores are better."* + +#### Values from Popular Papers +*Give examples, preferrably with links to leaderboards or publications, to papers that have reported this {{ cookiecutter.module_type }}, along with the values they have reported.* + +### Examples +*Give code examples of the {{ cookiecutter.module_type }} being used. Try to include examples that clear up any potential ambiguity left from the {{ cookiecutter.module_type }} description above. If possible, provide a range of examples that show both typical and atypical results, as well as examples where a variety of input parameters are passed.* + +## Limitations and Bias +*Note any known limitations or biases that the {{ cookiecutter.module_type }} has, with links and references if possible.* + +## Citation +*Cite the source where this {{ cookiecutter.module_type }} was introduced.* + +## Further References +*Add any useful further references.* diff --git a/TextClassification/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/app.py b/TextClassification/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/app.py new file mode 100644 index 0000000000000000000000000000000000000000..6582ec47a21ff99a2b3e5696820f3b029b99a486 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/app.py @@ -0,0 +1,6 @@ +import evaluate +from evaluate.utils import launch_gradio_widget + + +module = evaluate.load("{{ cookiecutter.namespace }}/{{ cookiecutter.module_slug }}") +launch_gradio_widget(module) \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/requirements.txt b/TextClassification/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..630637afffa554a9ac5588a774f4e0bf8b81a2fe --- /dev/null +++ b/TextClassification/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/requirements.txt @@ -0,0 +1 @@ +git+https://github.com/huggingface/evaluate@main \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/tests.py b/TextClassification/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/tests.py new file mode 100644 index 0000000000000000000000000000000000000000..601ed757507caebec67493462d11eb4c8901c2a1 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/tests.py @@ -0,0 +1,17 @@ +test_cases = [ + { + "predictions": [0, 0], + "references": [1, 1], + "result": {"metric_score": 0} + }, + { + "predictions": [1, 1], + "references": [1, 1], + "result": {"metric_score": 1} + }, + { + "predictions": [1, 0], + "references": [1, 1], + "result": {"metric_score": 0.5} + } +] \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/{{ cookiecutter.module_slug }}.py b/TextClassification/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/{{ cookiecutter.module_slug }}.py new file mode 100644 index 0000000000000000000000000000000000000000..578ebd2e846fb3519c8cd3efb284b4c373eaa4ce --- /dev/null +++ b/TextClassification/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/{{ cookiecutter.module_slug }}.py @@ -0,0 +1,95 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""TODO: Add a description here.""" + +import evaluate +import datasets + + +# TODO: Add BibTeX citation +_CITATION = """\ +@InProceedings{huggingface:module, +title = {A great new module}, +authors={huggingface, Inc.}, +year={2020} +} +""" + +# TODO: Add description of the module here +_DESCRIPTION = """\ +This new module is designed to solve this great ML task and is crafted with a lot of care. +""" + + +# TODO: Add description of the arguments of the module here +_KWARGS_DESCRIPTION = """ +Calculates how good are predictions given some references, using certain scores +Args: + predictions: list of predictions to score. Each predictions + should be a string with tokens separated by spaces. + references: list of reference for each prediction. Each + reference should be a string with tokens separated by spaces. +Returns: + accuracy: description of the first score, + another_score: description of the second score, +Examples: + Examples should be written in doctest format, and should illustrate how + to use the function. + + >>> my_new_module = evaluate.load("my_new_module") + >>> results = my_new_module.compute(references=[0, 1], predictions=[0, 1]) + >>> print(results) + {'accuracy': 1.0} +""" + +# TODO: Define external resources urls if needed +BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt" + + +@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) +class {{ cookiecutter.module_class_name }}(evaluate.{{ cookiecutter.module_type | capitalize}}): + """TODO: Short description of my evaluation module.""" + + def _info(self): + # TODO: Specifies the evaluate.EvaluationModuleInfo object + return evaluate.{{ cookiecutter.module_type | capitalize}}Info( + # This is the description that will appear on the modules page. + module_type="{{ cookiecutter.module_type}}", + description=_DESCRIPTION, + citation=_CITATION, + inputs_description=_KWARGS_DESCRIPTION, + # This defines the format of each prediction and reference + features=datasets.Features({ + 'predictions': datasets.Value('int64'), + 'references': datasets.Value('int64'), + }), + # Homepage of the module for documentation + homepage="http://module.homepage", + # Additional links to the codebase or references + codebase_urls=["http://github.com/path/to/codebase/of/new_module"], + reference_urls=["http://path.to.reference.url/new_module"] + ) + + def _download_and_prepare(self, dl_manager): + """Optional: download external resources useful to compute the scores""" + # TODO: Download external resources if needed + pass + + def _compute(self, predictions, references): + """Returns the scores""" + # TODO: Compute the different scores of the module + accuracy = sum(i == j for i, j in zip(predictions, references)) / len(predictions) + return { + "accuracy": accuracy, + } \ No newline at end of file diff --git a/TextClassification/TextClassification/evaluate/tests/__init__.py b/TextClassification/TextClassification/evaluate/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/TextClassification/TextClassification/evaluate/tests/conftest.py b/TextClassification/TextClassification/evaluate/tests/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..1d0496df66452348c6490a37d2a7c9914f9d25e4 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/tests/conftest.py @@ -0,0 +1,466 @@ +import csv +import json +import lzma +import os +import tarfile +import textwrap + +import datasets +import pyarrow as pa +import pyarrow.parquet as pq +import pytest +from datasets import config +from datasets.arrow_dataset import Dataset +from datasets.features import ClassLabel, Features, Sequence, Value + + +@pytest.fixture(autouse=True) +def set_test_cache_config(tmp_path_factory, monkeypatch): + # test_hf_cache_home = tmp_path_factory.mktemp("cache") # TODO: why a cache dir per test function does not work? + test_hf_cache_home = tmp_path_factory.getbasetemp() / "cache" + test_hf_evaluate_cache = test_hf_cache_home / "datasets" + test_hf_metrics_cache = test_hf_cache_home / "metrics" + test_hf_modules_cache = test_hf_cache_home / "modules" + monkeypatch.setattr("evaluate.config.HF_EVALUATE_CACHE", str(test_hf_evaluate_cache)) + monkeypatch.setattr("evaluate.config.HF_METRICS_CACHE", str(test_hf_metrics_cache)) + monkeypatch.setattr("evaluate.config.HF_MODULES_CACHE", str(test_hf_modules_cache)) + test_DOWNLOADED_EVALUATE_PATH = test_hf_evaluate_cache / "downloads" + monkeypatch.setattr("evaluate.config.DOWNLOADED_EVALUATE_PATH", str(test_DOWNLOADED_EVALUATE_PATH)) + test_EXTRACTED_EVALUATE_PATH = test_hf_evaluate_cache / "downloads" / "extracted" + monkeypatch.setattr("evaluate.config.EXTRACTED_EVALUATE_PATH", str(test_EXTRACTED_EVALUATE_PATH)) + + +@pytest.fixture(autouse=True, scope="session") +def disable_tqdm_output(): + datasets.disable_progress_bar() + + +@pytest.fixture(autouse=True) +def set_update_download_counts_to_false(monkeypatch): + # don't take tests into account when counting downloads + monkeypatch.setattr("evaluate.config.HF_UPDATE_DOWNLOAD_COUNTS", False) + monkeypatch.setattr("datasets.config.HF_UPDATE_DOWNLOAD_COUNTS", False) + + +FILE_CONTENT = """\ + Text data. + Second line of data.""" + + +@pytest.fixture(scope="session") +def dataset(): + n = 10 + features = Features( + { + "tokens": Sequence(Value("string")), + "labels": Sequence(ClassLabel(names=["negative", "positive"])), + "answers": Sequence( + { + "text": Value("string"), + "answer_start": Value("int32"), + } + ), + "id": Value("int64"), + } + ) + dataset = Dataset.from_dict( + { + "tokens": [["foo"] * 5] * n, + "labels": [[1] * 5] * n, + "answers": [{"answer_start": [97], "text": ["1976"]}] * 10, + "id": list(range(n)), + }, + features=features, + ) + return dataset + + +@pytest.fixture(scope="session") +def arrow_file(tmp_path_factory, dataset): + filename = str(tmp_path_factory.mktemp("data") / "file.arrow") + dataset.map(cache_file_name=filename) + return filename + + +@pytest.fixture(scope="session") +def text_file(tmp_path_factory): + filename = tmp_path_factory.mktemp("data") / "file.txt" + data = FILE_CONTENT + with open(filename, "w") as f: + f.write(data) + return filename + + +@pytest.fixture(scope="session") +def xz_file(tmp_path_factory): + filename = tmp_path_factory.mktemp("data") / "file.txt.xz" + data = bytes(FILE_CONTENT, "utf-8") + with lzma.open(filename, "wb") as f: + f.write(data) + return filename + + +@pytest.fixture(scope="session") +def gz_file(tmp_path_factory): + import gzip + + path = str(tmp_path_factory.mktemp("data") / "file.txt.gz") + data = bytes(FILE_CONTENT, "utf-8") + with gzip.open(path, "wb") as f: + f.write(data) + return path + + +@pytest.fixture(scope="session") +def bz2_file(tmp_path_factory): + import bz2 + + path = tmp_path_factory.mktemp("data") / "file.txt.bz2" + data = bytes(FILE_CONTENT, "utf-8") + with bz2.open(path, "wb") as f: + f.write(data) + return path + + +@pytest.fixture(scope="session") +def zstd_file(tmp_path_factory): + if config.ZSTANDARD_AVAILABLE: + import zstandard as zstd + + path = tmp_path_factory.mktemp("data") / "file.txt.zst" + data = bytes(FILE_CONTENT, "utf-8") + with zstd.open(path, "wb") as f: + f.write(data) + return path + + +@pytest.fixture(scope="session") +def lz4_file(tmp_path_factory): + if config.LZ4_AVAILABLE: + import lz4.frame + + path = tmp_path_factory.mktemp("data") / "file.txt.lz4" + data = bytes(FILE_CONTENT, "utf-8") + with lz4.frame.open(path, "wb") as f: + f.write(data) + return path + + +@pytest.fixture(scope="session") +def xml_file(tmp_path_factory): + filename = tmp_path_factory.mktemp("data") / "file.xml" + data = textwrap.dedent( + """\ + + +
+ + + Contingut 1 + Content 1 + + + Contingut 2 + Content 2 + + + Contingut 3 + Content 3 + + + Contingut 4 + Content 4 + + + Contingut 5 + Content 5 + + + """ + ) + with open(filename, "w") as f: + f.write(data) + return filename + + +DATA = [ + {"col_1": "0", "col_2": 0, "col_3": 0.0}, + {"col_1": "1", "col_2": 1, "col_3": 1.0}, + {"col_1": "2", "col_2": 2, "col_3": 2.0}, + {"col_1": "3", "col_2": 3, "col_3": 3.0}, +] +DATA2 = [ + {"col_1": "4", "col_2": 4, "col_3": 4.0}, + {"col_1": "5", "col_2": 5, "col_3": 5.0}, +] +DATA_DICT_OF_LISTS = { + "col_1": ["0", "1", "2", "3"], + "col_2": [0, 1, 2, 3], + "col_3": [0.0, 1.0, 2.0, 3.0], +} + +DATA_312 = [ + {"col_3": 0.0, "col_1": "0", "col_2": 0}, + {"col_3": 1.0, "col_1": "1", "col_2": 1}, +] + +DATA_STR = [ + {"col_1": "s0", "col_2": 0, "col_3": 0.0}, + {"col_1": "s1", "col_2": 1, "col_3": 1.0}, + {"col_1": "s2", "col_2": 2, "col_3": 2.0}, + {"col_1": "s3", "col_2": 3, "col_3": 3.0}, +] + + +@pytest.fixture(scope="session") +def dataset_dict(): + return DATA_DICT_OF_LISTS + + +@pytest.fixture(scope="session") +def arrow_path(tmp_path_factory): + dataset = Dataset.from_dict(DATA_DICT_OF_LISTS) + path = str(tmp_path_factory.mktemp("data") / "dataset.arrow") + dataset.map(cache_file_name=path) + return path + + +@pytest.fixture(scope="session") +def csv_path(tmp_path_factory): + path = str(tmp_path_factory.mktemp("data") / "dataset.csv") + with open(path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=["col_1", "col_2", "col_3"]) + writer.writeheader() + for item in DATA: + writer.writerow(item) + return path + + +@pytest.fixture(scope="session") +def csv2_path(tmp_path_factory): + path = str(tmp_path_factory.mktemp("data") / "dataset2.csv") + with open(path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=["col_1", "col_2", "col_3"]) + writer.writeheader() + for item in DATA: + writer.writerow(item) + return path + + +@pytest.fixture(scope="session") +def bz2_csv_path(csv_path, tmp_path_factory): + import bz2 + + path = tmp_path_factory.mktemp("data") / "dataset.csv.bz2" + with open(csv_path, "rb") as f: + data = f.read() + # data = bytes(FILE_CONTENT, "utf-8") + with bz2.open(path, "wb") as f: + f.write(data) + return path + + +@pytest.fixture(scope="session") +def zip_csv_path(csv_path, csv2_path, tmp_path_factory): + import zipfile + + path = tmp_path_factory.mktemp("data") / "dataset.csv.zip" + with zipfile.ZipFile(path, "w") as f: + f.write(csv_path, arcname=os.path.basename(csv_path)) + f.write(csv2_path, arcname=os.path.basename(csv2_path)) + return path + + +@pytest.fixture(scope="session") +def zip_csv_with_dir_path(csv_path, csv2_path, tmp_path_factory): + import zipfile + + path = tmp_path_factory.mktemp("data") / "dataset_with_dir.csv.zip" + with zipfile.ZipFile(path, "w") as f: + f.write(csv_path, arcname=os.path.join("main_dir", os.path.basename(csv_path))) + f.write(csv2_path, arcname=os.path.join("main_dir", os.path.basename(csv2_path))) + return path + + +@pytest.fixture(scope="session") +def parquet_path(tmp_path_factory): + path = str(tmp_path_factory.mktemp("data") / "dataset.parquet") + schema = pa.schema( + { + "col_1": pa.string(), + "col_2": pa.int64(), + "col_3": pa.float64(), + } + ) + with open(path, "wb") as f: + writer = pq.ParquetWriter(f, schema=schema) + pa_table = pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(DATA))] for k in DATA[0]}, schema=schema) + writer.write_table(pa_table) + writer.close() + return path + + +@pytest.fixture(scope="session") +def json_list_of_dicts_path(tmp_path_factory): + path = str(tmp_path_factory.mktemp("data") / "dataset.json") + data = {"data": DATA} + with open(path, "w") as f: + json.dump(data, f) + return path + + +@pytest.fixture(scope="session") +def json_dict_of_lists_path(tmp_path_factory): + path = str(tmp_path_factory.mktemp("data") / "dataset.json") + data = {"data": DATA_DICT_OF_LISTS} + with open(path, "w") as f: + json.dump(data, f) + return path + + +@pytest.fixture(scope="session") +def jsonl_path(tmp_path_factory): + path = str(tmp_path_factory.mktemp("data") / "dataset.jsonl") + with open(path, "w") as f: + for item in DATA: + f.write(json.dumps(item) + "\n") + return path + + +@pytest.fixture(scope="session") +def jsonl2_path(tmp_path_factory): + path = str(tmp_path_factory.mktemp("data") / "dataset2.jsonl") + with open(path, "w") as f: + for item in DATA: + f.write(json.dumps(item) + "\n") + return path + + +@pytest.fixture(scope="session") +def jsonl_312_path(tmp_path_factory): + path = str(tmp_path_factory.mktemp("data") / "dataset_312.jsonl") + with open(path, "w") as f: + for item in DATA_312: + f.write(json.dumps(item) + "\n") + return path + + +@pytest.fixture(scope="session") +def jsonl_str_path(tmp_path_factory): + path = str(tmp_path_factory.mktemp("data") / "dataset-str.jsonl") + with open(path, "w") as f: + for item in DATA_STR: + f.write(json.dumps(item) + "\n") + return path + + +@pytest.fixture(scope="session") +def text_gz_path(tmp_path_factory, text_path): + import gzip + + path = str(tmp_path_factory.mktemp("data") / "dataset.txt.gz") + with open(text_path, "rb") as orig_file: + with gzip.open(path, "wb") as zipped_file: + zipped_file.writelines(orig_file) + return path + + +@pytest.fixture(scope="session") +def jsonl_gz_path(tmp_path_factory, jsonl_path): + import gzip + + path = str(tmp_path_factory.mktemp("data") / "dataset.jsonl.gz") + with open(jsonl_path, "rb") as orig_file: + with gzip.open(path, "wb") as zipped_file: + zipped_file.writelines(orig_file) + return path + + +@pytest.fixture(scope="session") +def zip_jsonl_path(jsonl_path, jsonl2_path, tmp_path_factory): + import zipfile + + path = tmp_path_factory.mktemp("data") / "dataset.jsonl.zip" + with zipfile.ZipFile(path, "w") as f: + f.write(jsonl_path, arcname=os.path.basename(jsonl_path)) + f.write(jsonl2_path, arcname=os.path.basename(jsonl2_path)) + return path + + +@pytest.fixture(scope="session") +def zip_jsonl_with_dir_path(jsonl_path, jsonl2_path, tmp_path_factory): + import zipfile + + path = tmp_path_factory.mktemp("data") / "dataset_with_dir.jsonl.zip" + with zipfile.ZipFile(path, "w") as f: + f.write(jsonl_path, arcname=os.path.join("main_dir", os.path.basename(jsonl_path))) + f.write(jsonl2_path, arcname=os.path.join("main_dir", os.path.basename(jsonl2_path))) + return path + + +@pytest.fixture(scope="session") +def tar_jsonl_path(jsonl_path, jsonl2_path, tmp_path_factory): + path = tmp_path_factory.mktemp("data") / "dataset.jsonl.tar" + with tarfile.TarFile(path, "w") as f: + f.add(jsonl_path, arcname=os.path.basename(jsonl_path)) + f.add(jsonl2_path, arcname=os.path.basename(jsonl2_path)) + return path + + +@pytest.fixture(scope="session") +def tar_nested_jsonl_path(tar_jsonl_path, jsonl_path, jsonl2_path, tmp_path_factory): + path = tmp_path_factory.mktemp("data") / "dataset_nested.jsonl.tar" + with tarfile.TarFile(path, "w") as f: + f.add(tar_jsonl_path, arcname=os.path.join("nested", os.path.basename(tar_jsonl_path))) + return path + + +@pytest.fixture(scope="session") +def text_path(tmp_path_factory): + data = ["0", "1", "2", "3"] + path = str(tmp_path_factory.mktemp("data") / "dataset.txt") + with open(path, "w") as f: + for item in data: + f.write(item + "\n") + return path + + +@pytest.fixture(scope="session") +def text2_path(tmp_path_factory): + data = ["0", "1", "2", "3"] + path = str(tmp_path_factory.mktemp("data") / "dataset2.txt") + with open(path, "w") as f: + for item in data: + f.write(item + "\n") + return path + + +@pytest.fixture(scope="session") +def zip_text_path(text_path, text2_path, tmp_path_factory): + import zipfile + + path = tmp_path_factory.mktemp("data") / "dataset.text.zip" + with zipfile.ZipFile(path, "w") as f: + f.write(text_path, arcname=os.path.basename(text_path)) + f.write(text2_path, arcname=os.path.basename(text2_path)) + return path + + +@pytest.fixture(scope="session") +def zip_text_with_dir_path(text_path, text2_path, tmp_path_factory): + import zipfile + + path = tmp_path_factory.mktemp("data") / "dataset_with_dir.text.zip" + with zipfile.ZipFile(path, "w") as f: + f.write(text_path, arcname=os.path.join("main_dir", os.path.basename(text_path))) + f.write(text2_path, arcname=os.path.join("main_dir", os.path.basename(text2_path))) + return path + + +@pytest.fixture(scope="session") +def text_path_with_unicode_new_lines(tmp_path_factory): + text = "\n".join(["First", "Second\u2029with Unicode new line", "Third"]) + path = str(tmp_path_factory.mktemp("data") / "dataset_with_unicode_new_lines.txt") + with open(path, "w", encoding="utf-8") as f: + f.write(text) + return path diff --git a/TextClassification/TextClassification/evaluate/tests/test_evaluation_suite.py b/TextClassification/TextClassification/evaluate/tests/test_evaluation_suite.py new file mode 100644 index 0000000000000000000000000000000000000000..ad77c9d71bb55b5e0bcaab4f90d2364adcb4ef62 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/tests/test_evaluation_suite.py @@ -0,0 +1,31 @@ +from unittest import TestCase + +from evaluate import EvaluationSuite +from tests.test_evaluator import DummyTextClassificationPipeline + + +class TestEvaluationSuite(TestCase): + def setUp(self): + # Check that the EvaluationSuite loads successfully + self.evaluation_suite = EvaluationSuite.load("evaluate/evaluation-suite-ci") + + # Setup a dummy model for usage with the EvaluationSuite + self.dummy_model = DummyTextClassificationPipeline() + + def test_running_evaluation_suite(self): + + # Check that the evaluation suite successfully runs + results = self.evaluation_suite.run(self.dummy_model) + + # Check that the results are correct + for r in results: + self.assertEqual(r["accuracy"], 0.5) + + # Check that correct number of tasks were run + self.assertEqual(len(results), 2) + + def test_empty_suite(self): + + self.empty_suite = self.evaluation_suite + self.empty_suite.suite = [] + self.assertRaises(ValueError, self.empty_suite.run, self.dummy_model) diff --git a/TextClassification/TextClassification/evaluate/tests/test_evaluator.py b/TextClassification/TextClassification/evaluate/tests/test_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..259b5c7b9ddd9704b65d747bda9907df2479fa4c --- /dev/null +++ b/TextClassification/TextClassification/evaluate/tests/test_evaluator.py @@ -0,0 +1,1145 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 + +from time import sleep +from unittest import TestCase, mock + +import numpy as np +from datasets import ClassLabel, Dataset, Features, Sequence, Value +from PIL import Image +from transformers import ( + AutoConfig, + AutoFeatureExtractor, + AutoModelForAudioClassification, + AutoModelForImageClassification, + AutoModelForQuestionAnswering, + AutoModelForSequenceClassification, + AutoModelForTokenClassification, + AutoTokenizer, + pipeline, +) + +from evaluate import ( + AudioClassificationEvaluator, + AutomaticSpeechRecognitionEvaluator, + Evaluator, + ImageClassificationEvaluator, + QuestionAnsweringEvaluator, + Text2TextGenerationEvaluator, + TextClassificationEvaluator, + TextGenerationEvaluator, + TokenClassificationEvaluator, + evaluator, + load, +) + +from .utils import slow + + +class DummyTextGenerationPipeline: + def __init__(self, prefix="generated", task="text-generation", num_return_sequences=1): + self.task = task + self.prefix = prefix + self.num_return_sequences = num_return_sequences + + def __call__(self, inputs, **kwargs): + return [[{f"{self.prefix}_text": "Lorem ipsum"} for _ in range(self.num_return_sequences)] for _ in inputs] + + +class DummyText2TextGenerationPipeline: + def __init__(self, prefix="generated", task="text2text-generation"): + self.task = task + self.prefix = prefix + + def __call__(self, inputs, **kwargs): + return [{f"{self.prefix}_text": "Lorem ipsum"} for _ in inputs] + + +class DummyTextClassificationPipeline: + def __init__(self, sleep_time=None): + self.task = "text-classification" + self.sleep_time = sleep_time + + def __call__(self, inputs, **kwargs): + if self.sleep_time is not None: + sleep(self.sleep_time) + return [{"label": "NEGATIVE"} if i % 2 == 1 else {"label": "POSITIVE"} for i, _ in enumerate(inputs)] + + +class DummyImageClassificationPipeline: + def __init__(self): + self.task = "image-classification" + + def __call__(self, images, **kwargs): + return [[{"score": 0.9, "label": "yurt"}, {"score": 0.1, "label": "umbrella"}] for i, _ in enumerate(images)] + + +class DummyQuestionAnsweringPipeline: + def __init__(self, v2: bool): + self.task = "question-answering" + self.v2 = v2 + + def __call__(self, question, context, **kwargs): + if self.v2: + return [ + {"score": 0.95, "start": 31, "end": 39, "answer": "Felix"} + if i % 2 == 0 + else {"score": 0.95, "start": 0, "end": 0, "answer": ""} + for i in range(len(question)) + ] + else: + return [{"score": 0.95, "start": 31, "end": 39, "answer": "Felix"} for _ in question] + + +class DummyTokenClassificationPipeline: + def __init__(self): + self.task = "token-classification" + + def __call__(self, inputs, **kwargs): + result = [ + {"start": 0, "entity": "B-LOC"}, + {"start": 2, "entity": "I-LOC"}, + {"start": 4, "entity": "I-LOC"}, + {"start": 9, "entity": "O"}, + {"start": 11, "entity": "O"}, + {"start": 16, "entity": "B-LOC"}, + {"start": 21, "entity": "O"}, + ] + + return [result] + + +class DummyAutomaticSpeechRecognitionPipeline: + def __init__(self) -> None: + self.task = "automatic-speech-recognition" + + def __call__(self, inputs, **kwargs): + return [{"text": "Lorem ipsum"} for _ in inputs] + + +class DummyAudioClassificationPipeline: + def __init__(self): + self.task = "audio-classification" + + def __call__(self, audio, **kwargs): + return [[{"score": 0.9, "label": "yes"}, {"score": 0.1, "label": "no"}] for i, _ in enumerate(audio)] + + +class TestEvaluator(TestCase): + def setUp(self): + self.data = Dataset.from_dict({"label": [1, 0], "text": ["great movie", "horrible movie"]}) + self.default_ckpt = "hf-internal-testing/tiny-random-bert" + self.default_model = AutoModelForSequenceClassification.from_pretrained(self.default_ckpt, num_labels=2) + self.default_tokenizer = AutoTokenizer.from_pretrained(self.default_ckpt) + self.pipe = pipeline("text-classification", model=self.default_model, tokenizer=self.default_tokenizer) + self.evaluator = evaluator("text-classification") + self.data = Dataset.from_dict({"label": [1, 0], "text": ["great movie", "horrible movie"]}) + self.label_mapping = {"LABEL_0": 0.0, "LABEL_1": 1.0} + + def test_wrong_task(self): + self.assertRaises(KeyError, evaluator, "bad_task") + + def test_device_placement(self): + orig_import = __import__ + + pt_mock = mock.Mock() + tf_mock = mock.Mock() + + # mock import of torch and tensorflow + def import_pt_tf_mock(name, *args): + if name == "torch": + if pt_available: + return pt_mock + else: + raise ImportError + if name == "tensorflow": + if tf_available: + return tf_mock + else: + raise ImportError + return orig_import(name, *args) + + with mock.patch("builtins.__import__", side_effect=import_pt_tf_mock): + # neither pt or tf are available + pt_available = False + tf_available = False + self.assertEqual(Evaluator._infer_device(), -1) + + # pt available but no GPU + pt_available = True + pt_mock.cuda.is_available.return_value = False + self.assertEqual(Evaluator._infer_device(), -1) + + # pt available and GPU found + pt_mock.cuda.is_available.return_value = True + self.assertEqual(Evaluator._infer_device(), 0) + + # tf available but no GPU + pt_available = False + tf_available = True + tf_mock.config.list_physical_devices.return_value = [] + self.assertEqual(Evaluator._infer_device(), -1) + + # tf available and GPU found + tf_mock.config.list_physical_devices.return_value = ["GPU:0", "GPU:1"] + self.assertEqual(Evaluator._infer_device(), 0) + + # pt accelerator found and pipeline instantiated on CPU + pt_mock.cuda.is_available.return_value = True + self.assertRaises( + ValueError, Evaluator.check_for_mismatch_in_device_setup, Evaluator._infer_device(), self.pipe + ) + + # tf accelerator found and pipeline instantiated on CPU + pt_available = False + tf_available = True + self.assertRaises( + ValueError, Evaluator.check_for_mismatch_in_device_setup, Evaluator._infer_device(), self.pipe + ) + + def test_pipe_init(self): + self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + input_column="text", + label_column="label", + label_mapping=self.label_mapping, + ) + + def test_model_init(self): + self.evaluator.compute( + model_or_pipeline=self.default_model, + tokenizer=self.default_tokenizer, + data=self.data, + input_column="text", + label_column="label", + label_mapping=self.label_mapping, + ) + + def test_model_str_init(self): + self.evaluator.compute( + model_or_pipeline=self.default_ckpt, + data=self.data, + input_column="text", + label_column="label", + label_mapping=self.label_mapping, + ) + + +class TestTextClassificationEvaluator(TestCase): + def setUp(self): + self.data = Dataset.from_dict({"label": [1, 0], "text": ["great movie", "horrible movie"]}) + self.default_model = "lvwerra/distilbert-imdb" + self.input_column = "text" + self.label_column = "label" + self.pipe = DummyTextClassificationPipeline() + self.perf_pipe = DummyTextClassificationPipeline(sleep_time=0.1) + self.evaluator = evaluator("text-classification") + self.label_mapping = {"NEGATIVE": 0.0, "POSITIVE": 1.0} + + def test_pipe_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + input_column="text", + label_column="label", + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 1.0) + + @slow + def test_model_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.default_model, + data=self.data, + metric="accuracy", + input_column=self.input_column, + label_column=self.label_column, + label_mapping=self.label_mapping, + ) + + model = AutoModelForSequenceClassification.from_pretrained(self.default_model) + tokenizer = AutoTokenizer.from_pretrained(self.default_model) + + self.assertEqual(results["accuracy"], 1.0) + results = self.evaluator.compute( + model_or_pipeline=model, + data=self.data, + metric="accuracy", + tokenizer=tokenizer, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 1.0) + + def test_class_init(self): + evaluator = TextClassificationEvaluator() + self.assertEqual(evaluator.task, "text-classification") + self.assertIsNone(evaluator.default_metric_name) + + results = evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="f1", + label_mapping=self.label_mapping, + ) + self.assertEqual(results["f1"], 1.0) + + @slow + def test_default_pipe_init(self): + results = self.evaluator.compute( + data=self.data, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 1.0) + + def test_data_loading(self): + + # Test passing in dataset by name with split + data = self.evaluator.load_data("evaluate/imdb-ci", split="test[:1]") + self.evaluator.prepare_data(data=data, input_column="text", label_column="label", second_input_column=None) + + # Test passing in dataset by name without split and inferring the optimal split + data = self.evaluator.load_data("evaluate/imdb-ci") + self.evaluator.prepare_data(data=data, input_column="text", label_column="label", second_input_column=None) + + # Test that it chooses the correct one (e.g. imdb only has train and test, but no validation) + self.assertEqual(data.split, "test") + + # Test that the data point returned is correct; this maps to the first example in the dataset + self.assertEqual(data[0]["text"], "I love movies about whales!") + + # Test loading subset of a dataset with the `name` field + data = self.evaluator.load_data("evaluate/glue-ci", subset="cola", split="test") + self.assertEqual(isinstance(data, Dataset), True) + + # Test loading subset of a dataset with the `name` field and having it infer the split + data = self.evaluator.load_data("evaluate/glue-ci", subset="cola") + self.assertEqual(isinstance(data, Dataset), True) + + def test_overwrite_default_metric(self): + accuracy = load("accuracy") + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric=accuracy, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 1.0) + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="accuracy", + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 1.0) + + def test_bootstrap(self): + data = Dataset.from_dict({"label": [1, 0, 0], "text": ["great movie", "great movie", "horrible movie"]}) + + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=data, + metric="accuracy", + label_mapping=self.label_mapping, + strategy="bootstrap", + n_resamples=10, + random_state=0, + ) + self.assertAlmostEqual(results["accuracy"]["score"], 0.666666, 5) + self.assertAlmostEqual(results["accuracy"]["confidence_interval"][0], 0.33557, 5) + self.assertAlmostEqual(results["accuracy"]["confidence_interval"][1], 1.0, 5) + self.assertAlmostEqual(results["accuracy"]["standard_error"], 0.22498, 5) + + def test_perf(self): + results = self.evaluator.compute( + model_or_pipeline=self.perf_pipe, + data=self.data, + metric="accuracy", + input_column=self.input_column, + label_column=self.label_column, + label_mapping=self.label_mapping, + n_resamples=10, + random_state=0, + ) + self.assertEqual(results["accuracy"], 1.0) + self.assertAlmostEqual(results["total_time_in_seconds"], 0.1, 1) + self.assertAlmostEqual(results["samples_per_second"], len(self.data) / results["total_time_in_seconds"], 5) + self.assertAlmostEqual(results["latency_in_seconds"], results["total_time_in_seconds"] / len(self.data), 5) + + def test_bootstrap_and_perf(self): + data = Dataset.from_dict({"label": [1, 0, 0], "text": ["great movie", "great movie", "horrible movie"]}) + + results = self.evaluator.compute( + model_or_pipeline=self.perf_pipe, + data=data, + metric="accuracy", + input_column=self.input_column, + label_column=self.label_column, + label_mapping=self.label_mapping, + strategy="bootstrap", + n_resamples=10, + random_state=0, + ) + self.assertAlmostEqual(results["accuracy"]["score"], 0.666666, 5) + self.assertAlmostEqual(results["accuracy"]["confidence_interval"][0], 0.33557, 5) + self.assertAlmostEqual(results["accuracy"]["confidence_interval"][1], 1.0, 5) + self.assertAlmostEqual(results["accuracy"]["standard_error"], 0.22498285, 5) + self.assertAlmostEqual(results["total_time_in_seconds"], 0.1, 1) + self.assertAlmostEqual(results["samples_per_second"], len(data) / results["total_time_in_seconds"], 5) + self.assertAlmostEqual(results["latency_in_seconds"], results["total_time_in_seconds"] / len(data), 5) + + +class TestTextClassificationEvaluatorTwoColumns(TestCase): + def setUp(self): + self.data = Dataset.from_dict( + { + "label": [1, 0], + "premise": ["great car", "great movie"], + "hypothesis": ["great vehicle", "horrible movie"], + } + ) + self.default_model = "prajjwal1/bert-tiny-mnli" + self.input_column = "premise" + self.second_input_column = "hypothesis" + self.label_column = "label" + self.pipe = DummyTextClassificationPipeline() + self.evaluator = evaluator("text-classification") + self.label_mapping = {"NEGATIVE": 0.0, "POSITIVE": 1.0} + self.label_mapping2 = {"LABEL_0": 0, "LABEL_1": 1, "LABEL_2": 2} + + def test_pipe_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + input_column=self.input_column, + second_input_column=self.second_input_column, + label_column="label", + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 1.0) + + @slow + def test_model_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.default_model, + data=self.data, + metric="accuracy", + input_column=self.input_column, + second_input_column=self.second_input_column, + label_column=self.label_column, + label_mapping=self.label_mapping2, + ) + self.assertEqual(results["accuracy"], 1.0) + + model = AutoModelForSequenceClassification.from_pretrained(self.default_model) + tokenizer = AutoTokenizer.from_pretrained(self.default_model) + + results = self.evaluator.compute( + model_or_pipeline=model, + data=self.data, + metric="accuracy", + input_column=self.input_column, + second_input_column=self.second_input_column, + tokenizer=tokenizer, + label_mapping=self.label_mapping2, + ) + self.assertEqual(results["accuracy"], 1.0) + + +class TestImageClassificationEvaluator(TestCase): + def setUp(self): + self.data = Dataset.from_dict( + { + "label": [2, 2], + "image": [Image.new("RGB", (500, 500), (255, 255, 255)), Image.new("RGB", (500, 500), (170, 95, 170))], + } + ) + self.default_model = "lysandre/tiny-vit-random" + self.pipe = DummyImageClassificationPipeline() + self.evaluator = evaluator("image-classification") + self.label_mapping = AutoConfig.from_pretrained(self.default_model).label2id + + def test_pipe_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + @slow + def test_model_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.default_model, + data=self.data, + metric="accuracy", + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + model = AutoModelForImageClassification.from_pretrained(self.default_model) + feature_extractor = AutoFeatureExtractor.from_pretrained(self.default_model) + + results = self.evaluator.compute( + model_or_pipeline=model, + data=self.data, + metric="accuracy", + feature_extractor=feature_extractor, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + def test_class_init(self): + evaluator = ImageClassificationEvaluator() + self.assertEqual(evaluator.task, "image-classification") + self.assertIsNone(evaluator.default_metric_name) + + results = evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="accuracy", + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + @slow + def test_default_pipe_init(self): + results = self.evaluator.compute( + data=self.data, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + def test_overwrite_default_metric(self): + accuracy = load("accuracy") + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric=accuracy, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="accuracy", + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + +class TestQuestionAnsweringEvaluator(TestCase): + def setUp(self): + self.data = Dataset.from_dict( + { + "id": ["56be4db0acb8001400a502ec", "56be4db0acb8001400a502ed"], + "context": ["My name is Felix and I love cookies!", "Misa name is Felix and misa love cookies!"], + "answers": [{"text": ["Felix"], "answer_start": [11]}, {"text": ["Felix"], "answer_start": [13]}], + "question": ["What is my name?", "What is my name?"], + } + ) + self.data_v2 = Dataset.from_dict( + { + "id": ["56be4db0acb8001400a502ec", "56be4db0acb8001400a502ed"], + "context": ["My name is Felix and I love cookies!", "Let's explore the city!"], + "answers": [{"text": ["Felix"], "answer_start": [11]}, {"text": [], "answer_start": []}], + "question": ["What is my name?", "What is my name?"], + } + ) + + self.default_model = "mrm8488/bert-tiny-finetuned-squadv2" + self.pipe = DummyQuestionAnsweringPipeline(v2=False) + self.pipe_v2 = DummyQuestionAnsweringPipeline(v2=True) + self.evaluator = evaluator("question-answering") + + def test_pipe_init(self): + # squad_v1-like dataset + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + ) + self.assertEqual(results["exact_match"], 100.0) + self.assertEqual(results["f1"], 100.0) + + @slow + def test_model_init(self): + # squad_v1-like dataset + results = self.evaluator.compute( + model_or_pipeline=self.default_model, + data=self.data, + metric="squad", + ) + self.assertEqual(results["exact_match"], 0) + self.assertEqual(results["f1"], 100 / 3) + + model = AutoModelForQuestionAnswering.from_pretrained(self.default_model) + tokenizer = AutoTokenizer.from_pretrained(self.default_model) + + results = self.evaluator.compute( + model_or_pipeline=model, + data=self.data, + metric="squad", + tokenizer=tokenizer, + ) + self.assertEqual(results["exact_match"], 0) + self.assertEqual(results["f1"], 100 / 3) + + def test_class_init(self): + # squad_v1-like dataset + evaluator = QuestionAnsweringEvaluator() + self.assertEqual(evaluator.task, "question-answering") + self.assertIsNone(evaluator.default_metric_name) + + results = evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="squad", + ) + self.assertEqual(results["exact_match"], 100.0) + self.assertEqual(results["f1"], 100.0) + + # squad_v2-like dataset + evaluator = QuestionAnsweringEvaluator() + self.assertEqual(evaluator.task, "question-answering") + self.assertIsNone(evaluator.default_metric_name) + + results = evaluator.compute( + model_or_pipeline=self.pipe_v2, + data=self.data_v2, + metric="squad_v2", + ) + self.assertDictEqual( + {key: results[key] for key in ["HasAns_f1", "NoAns_f1"]}, {"HasAns_f1": 100.0, "NoAns_f1": 100.0} + ) + + @slow + def test_default_pipe_init(self): + # squad_v1-like dataset + results = self.evaluator.compute( + data=self.data, + ) + self.assertEqual(results["exact_match"], 100.0) + self.assertEqual(results["f1"], 100.0) + + # squad_v2-like dataset + results = self.evaluator.compute( + data=self.data_v2, + metric="squad_v2", + ) + self.assertDictEqual( + {key: results[key] for key in ["HasAns_f1", "NoAns_f1"]}, {"HasAns_f1": 100.0, "NoAns_f1": 0.0} + ) + + def test_data_loading(self): + # Test passing in dataset by name with data_split + data = self.evaluator.load_data("evaluate/squad-ci", split="validation[:1]") + self.evaluator.prepare_data( + data=data, question_column="question", context_column="context", id_column="id", label_column="answers" + ) + + # Test passing in dataset by name without data_split and inferring the optimal split + data = self.evaluator.load_data("evaluate/squad-ci") + self.evaluator.prepare_data( + data=data, question_column="question", context_column="context", id_column="id", label_column="answers" + ) + + # Test that it chooses the correct one (e.g. squad only has train and validation, but no test) + self.assertEqual(data.split, "validation") + + # Test that the data point returned is correct; this maps to the first example in the squad-ci dataset + self.assertEqual(data[0]["id"], "56be4db0acb8001400a502ec") + + def test_overwrite_default_metric(self): + # squad_v1-like dataset + squad = load("squad") + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric=squad, + ) + self.assertEqual(results["exact_match"], 100.0) + self.assertEqual(results["f1"], 100.0) + + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="squad", + ) + self.assertEqual(results["exact_match"], 100.0) + self.assertEqual(results["f1"], 100.0) + + +class TestTokenClassificationEvaluator(TestCase): + def setUp(self): + features = Features( + { + "tokens": Sequence(feature=Value(dtype="string")), + "ner_tags": Sequence(feature=ClassLabel(names=["O", "B-LOC", "I-LOC"])), + } + ) + + self.data = Dataset.from_dict( + { + "tokens": [["New", "York", "a", "nice", "City", "."]], + "ner_tags": [[1, 2, 0, 0, 1, 0]], + }, + features=features, + ) + self.default_model = "hf-internal-testing/tiny-bert-for-token-classification" + self.pipe = DummyTokenClassificationPipeline() + self.evaluator = evaluator("token-classification") + + @slow + def test_model_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.default_model, + data=self.data, + metric="seqeval", + ) + self.assertEqual(results["overall_accuracy"], 0.5) + + model = AutoModelForTokenClassification.from_pretrained(self.default_model) + tokenizer = AutoTokenizer.from_pretrained(self.default_model) + results = self.evaluator.compute( + model_or_pipeline=model, + data=self.data, + metric="seqeval", + tokenizer=tokenizer, + ) + self.assertEqual(results["overall_accuracy"], 0.5) + + def test_class_init(self): + evaluator = TokenClassificationEvaluator() + self.assertEqual(evaluator.task, "token-classification") + self.assertIsNone(evaluator.default_metric_name) + + results = evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="seqeval", + ) + self.assertEqual(results["overall_accuracy"], 1.0) + + @slow + def test_default_pipe_init(self): + results = self.evaluator.compute( + data=self.data, + ) + self.assertEqual(results["overall_accuracy"], 2 / 3) + + def test_overwrite_default_metric(self): + accuracy = load("seqeval") + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric=accuracy, + ) + self.assertEqual(results["overall_accuracy"], 1.0) + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="seqeval", + ) + self.assertEqual(results["overall_accuracy"], 1.0) + + def test_data_loading(self): + # Test passing in dataset by name with data_split + data = self.evaluator.load_data("evaluate/conll2003-ci", split="validation[:1]") + self.evaluator.prepare_data( + data=data, + input_column="tokens", + label_column="ner_tags", + join_by=" ", + ) + + # Test passing in dataset by name without data_split and inferring the optimal split + data = self.evaluator.load_data("evaluate/conll2003-ci") + self.evaluator.prepare_data( + data=data, + input_column="tokens", + label_column="ner_tags", + join_by=" ", + ) + + # Test that it chooses the correct one (e.g. conll2003 has train, validation, test but should select test) + self.assertEqual(data.split, "test") + + # Test that the data point returned is correct; this maps to the first example in the dataset + self.assertEqual(data[0]["id"], "0") + + def test_wrong_task(self): + self.assertRaises(KeyError, evaluator, "bad_task") + + def test_words_to_offsets(self): + task_evaluator = evaluator("token-classification") + + words = ["This", "is", "a", "test", "."] + join_by = " " + + offsets = task_evaluator.words_to_offsets(words, join_by) + + self.assertListEqual([(0, 3), (5, 6), (8, 8), (10, 13), (15, 15)], offsets) + + words = ["日", "本", "語", "はなせるの?"] + join_by = "" + + offsets = task_evaluator.words_to_offsets(words, join_by) + + self.assertListEqual([(0, 0), (1, 1), (2, 2), (3, 8)], offsets) + + def test_predictions_processor(self): + task_evaluator = evaluator("token-classification") + join_by = " " + words = [["New", "York", "a", "nice", "City", "."]] + + # aligned start and words + predictions = [ + [ + {"start": 0, "entity": "B-LOC"}, + {"start": 2, "entity": "I-LOC"}, + {"start": 4, "entity": "I-LOC"}, + {"start": 9, "entity": "O"}, + {"start": 11, "entity": "O"}, + {"start": 16, "entity": "B-LOC"}, + {"start": 21, "entity": "O"}, + ] + ] + predictions = task_evaluator.predictions_processor(predictions, words, join_by) + self.assertListEqual(predictions["predictions"][0], ["B-LOC", "I-LOC", "O", "O", "B-LOC", "O"]) + + # non-aligned start and words + predictions = [ + [ + {"start": 0, "entity": "B-LOC"}, + {"start": 2, "entity": "I-LOC"}, + {"start": 9, "entity": "O"}, + {"start": 11, "entity": "O"}, + {"start": 16, "entity": "B-LOC"}, + {"start": 21, "entity": "O"}, + ] + ] + predictions = task_evaluator.predictions_processor(predictions, words, join_by) + self.assertListEqual(predictions["predictions"][0], ["B-LOC", "O", "O", "O", "B-LOC", "O"]) + + # non-aligned start and words + predictions = [ + [ + {"start": 0, "entity": "B-LOC"}, + {"start": 6, "entity": "I-LOC"}, + {"start": 9, "entity": "O"}, + {"start": 11, "entity": "O"}, + {"start": 16, "entity": "B-LOC"}, + {"start": 21, "entity": "O"}, + ] + ] + predictions = task_evaluator.predictions_processor(predictions, words, join_by) + self.assertListEqual(predictions["predictions"][0], ["B-LOC", "O", "O", "O", "B-LOC", "O"]) + + # non-aligned start and words + predictions = [ + [ + {"start": 0, "entity": "B-LOC"}, + {"start": 9, "entity": "O"}, + {"start": 11, "entity": "O"}, + {"start": 16, "entity": "B-LOC"}, + {"start": 21, "entity": "O"}, + ] + ] + predictions = task_evaluator.predictions_processor(predictions, words, join_by) + self.assertListEqual(predictions["predictions"][0], ["B-LOC", "O", "O", "O", "B-LOC", "O"]) + + +class TestTextGenerationEvaluator(TestCase): + def setUp(self): + self.data = Dataset.from_dict({"text": ["Lorem ipsum"]}) + self.pipe = DummyTextGenerationPipeline(num_return_sequences=4) + self.evaluator = evaluator("text-generation") + + def test_class_init(self): + evaluator = TextGenerationEvaluator() + self.assertEqual(evaluator.task, "text-generation") + self.assertIsNone(evaluator.default_metric_name) + + results = evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="word_count", + ) + self.assertIsInstance(results["unique_words"], int) + + @slow + def test_default_pipe_init(self): + results = self.evaluator.compute(data=self.data) + self.assertIsInstance(results["unique_words"], int) + + def test_overwrite_default_metric(self): + word_length = load("word_length") + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric=word_length, + ) + self.assertIsInstance(results["average_word_length"], int) + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="word_length", + ) + self.assertIsInstance(results["average_word_length"], int) + + def test_process_predictions_multiple_return_sequences(self): + processed_predictions = self.evaluator.predictions_processor( + [ + [{"generated_text": "A"}, {"generated_text": "B"}], + [{"generated_text": "C"}, {"generated_text": "D"}], + ] + ) + self.assertEqual(processed_predictions, {"data": ["A", "B", "C", "D"]}) + + +class TestText2TextGenerationEvaluator(TestCase): + def setUp(self): + self.data = Dataset.from_dict( + { + "text": ["Lorem ipsum"] * 4, + "label": ["Ipsum Lorem"] * 4, + } + ) + self.pipe = DummyText2TextGenerationPipeline() + self.evaluator = evaluator("text2text-generation") + + def test_pipe_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + ) + self.assertEqual(results["bleu"], 0) + + def test_class_init(self): + evaluator = Text2TextGenerationEvaluator() + self.assertEqual(evaluator.task, "text2text-generation") + self.assertIsNone(evaluator.default_metric_name) + + results = evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="bleu", + ) + self.assertEqual(results["bleu"], 0) + + @slow + def test_default_pipe_init(self): + results = self.evaluator.compute(data=self.data) + self.assertEqual(results["bleu"], 0) + + def test_overwrite_default_metric(self): + rouge = load("rouge") + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric=rouge, + ) + self.assertEqual(results["rouge1"], 1.0) + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="rouge", + ) + self.assertEqual(results["rouge1"], 1.0) + + def test_summarization(self): + pipe = DummyText2TextGenerationPipeline(task="summarization", prefix="summary") + e = evaluator("summarization") + + results = e.compute( + model_or_pipeline=pipe, + data=self.data, + ) + self.assertEqual(results["rouge1"], 1.0) + + def test_translation(self): + pipe = DummyText2TextGenerationPipeline(task="translation", prefix="translation") + e = evaluator("translation") + + results = e.compute( + model_or_pipeline=pipe, + data=self.data, + ) + self.assertEqual(results["bleu"], 0) + + +class TestAutomaticSpeechRecognitionEvaluator(TestCase): + def setUp(self): + self.data = Dataset.from_dict( + { + "path": [ + # Examples copied from default speech model of + # `automic-speech-recognition` pipeline: + # https://huggingface.co/facebook/wav2vec2-base-960h + # https://github.com/huggingface/transformers/blob/main/src/transformers/pipelines/__init__.py#L161 + "https://cdn-media.huggingface.co/speech_samples/sample1.flac", + "https://cdn-media.huggingface.co/speech_samples/sample2.flac", + ], + "sentence": ["Ipsum Lorem"] * 2, + } + ) + self.pipe = DummyAutomaticSpeechRecognitionPipeline() + self.evaluator = evaluator("automatic-speech-recognition") + + def test_pipe_init(self): + print(self.evaluator) + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + ) + print(results) + self.assertEqual(results["wer"], 1.0) + + def test_class_init(self): + evaluator = AutomaticSpeechRecognitionEvaluator() + self.assertEqual(evaluator.task, "automatic-speech-recognition") + self.assertIsNone(evaluator.default_metric_name) + + results = evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="wer", + ) + self.assertEqual(results["wer"], 1.0) + + @slow + def test_default_pipe_init(self): + results = self.evaluator.compute(data=self.data) + self.assertGreater(results["wer"], 1.0) + + def test_overwrite_default_metric(self): + cer = load("cer") + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric=cer, + ) + self.assertEqual(results["cer"], 0.7272727272727273) + + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="cer", + ) + self.assertEqual(results["cer"], 0.7272727272727273) + + +class TestAudioClassificationEvaluator(TestCase): + def setUp(self): + self.data = Dataset.from_dict( + {"file": ["https://huggingface.co/datasets/Narsil/asr_dummy/resolve/main/1.flac"], "label": [11]} + ) + self.raw_data = Dataset.from_dict( + { + "audio": [ + np.array( + [-0.00048828, -0.00018311, -0.00137329, 0.00079346, 0.00091553, 0.00085449], dtype=np.float32 + ) + ], + "label": [11], + } + ) + self.default_model = "superb/wav2vec2-base-superb-ks" + self.pipe = DummyAudioClassificationPipeline() + self.evaluator = evaluator("audio-classification") + self.label_mapping = AutoConfig.from_pretrained(self.default_model).label2id + + def test_pipe_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + def test_raw_pipe_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.pipe, data=self.raw_data, label_mapping=self.label_mapping, input_column="audio" + ) + self.assertEqual(results["accuracy"], 0) + + @slow + def test_model_init(self): + results = self.evaluator.compute( + model_or_pipeline=self.default_model, + data=self.data, + metric="accuracy", + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + model = AutoModelForAudioClassification.from_pretrained(self.default_model) + feature_extractor = AutoFeatureExtractor.from_pretrained(self.default_model) + + results = self.evaluator.compute( + model_or_pipeline=model, + data=self.data, + metric="accuracy", + feature_extractor=feature_extractor, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + def test_class_init(self): + evaluator = AudioClassificationEvaluator() + self.assertEqual(evaluator.task, "audio-classification") + self.assertIsNone(evaluator.default_metric_name) + + results = evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="accuracy", + label_mapping=self.label_mapping, + ) + results_raw = evaluator.compute( + model_or_pipeline=self.pipe, + data=self.raw_data, + label_mapping=self.label_mapping, + metric="accuracy", + input_column="audio", + ) + self.assertEqual(results_raw["accuracy"], 0) + self.assertEqual(results["accuracy"], 0) + + @slow + def test_default_pipe_init(self): + results = self.evaluator.compute( + data=self.data, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + + def test_overwrite_default_metric(self): + accuracy = load("accuracy") + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric=accuracy, + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) + results = self.evaluator.compute( + model_or_pipeline=self.pipe, + data=self.data, + metric="accuracy", + label_mapping=self.label_mapping, + ) + self.assertEqual(results["accuracy"], 0) diff --git a/TextClassification/TextClassification/evaluate/tests/test_file_utils.py b/TextClassification/TextClassification/evaluate/tests/test_file_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1b1cdccd4d6498b9ea0db08177222def4f088abb --- /dev/null +++ b/TextClassification/TextClassification/evaluate/tests/test_file_utils.py @@ -0,0 +1,56 @@ +import os +from pathlib import Path +from unittest.mock import patch + +import pytest + +from evaluate.utils.file_utils import OfflineModeIsEnabled, cached_path, ftp_get, ftp_head, http_get, http_head + + +FILE_CONTENT = """\ + Text data. + Second line of data.""" + + +def test_cached_path_local(text_file): + # absolute path + text_file = str(Path(text_file).resolve()) + assert cached_path(text_file) == text_file + # relative path + text_file = str(Path(__file__).resolve().relative_to(Path(os.getcwd()))) + assert cached_path(text_file) == text_file + + +def test_cached_path_missing_local(tmp_path): + # absolute path + missing_file = str(tmp_path.resolve() / "__missing_file__.txt") + with pytest.raises(FileNotFoundError): + cached_path(missing_file) + # relative path + missing_file = "./__missing_file__.txt" + with pytest.raises(FileNotFoundError): + cached_path(missing_file) + + +@patch("evaluate.config.HF_EVALUATE_OFFLINE", True) +def test_cached_path_offline(): + with pytest.raises(OfflineModeIsEnabled): + cached_path("https://huggingface.co") + + +@patch("evaluate.config.HF_EVALUATE_OFFLINE", True) +def test_http_offline(tmp_path_factory): + filename = tmp_path_factory.mktemp("data") / "file.html" + with pytest.raises(OfflineModeIsEnabled): + http_get("https://huggingface.co", temp_file=filename) + with pytest.raises(OfflineModeIsEnabled): + http_head("https://huggingface.co") + + +@patch("evaluate.config.HF_EVALUATE_OFFLINE", True) +def test_ftp_offline(tmp_path_factory): + filename = tmp_path_factory.mktemp("data") / "file.html" + with pytest.raises(OfflineModeIsEnabled): + ftp_get("ftp://huggingface.co", temp_file=filename) + with pytest.raises(OfflineModeIsEnabled): + ftp_head("ftp://huggingface.co") diff --git a/TextClassification/TextClassification/evaluate/tests/test_hub.py b/TextClassification/TextClassification/evaluate/tests/test_hub.py new file mode 100644 index 0000000000000000000000000000000000000000..055996b1092d1c7d064d448c8f1ebb84781928b9 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/tests/test_hub.py @@ -0,0 +1,187 @@ +import glob +from unittest import TestCase +from unittest.mock import patch + +import pytest +import requests +import yaml + +from evaluate.hub import push_to_hub +from tests.test_metric import DummyMetric + + +minimum_metadata = { + "model-index": [ + { + "results": [ + { + "task": {"type": "dummy-task"}, + "dataset": {"type": "dataset_type", "name": "dataset_name"}, + "metrics": [ + {"type": "dummy_metric", "value": 1.0, "name": "Pretty Metric Name"}, + ], + } + ] + } + ] +} + +extras_metadata = { + "model-index": [ + { + "results": [ + { + "task": {"type": "dummy-task", "name": "task_name"}, + "dataset": { + "type": "dataset_type", + "name": "dataset_name", + "config": "fr", + "split": "test", + "revision": "abc", + "args": {"a": 1, "b": 2}, + }, + "metrics": [ + { + "type": "dummy_metric", + "value": 1.0, + "name": "Pretty Metric Name", + "config": "default", + "args": {"hello": 1, "world": 2}, + }, + ], + } + ] + } + ] +} + + +@patch("evaluate.hub.HF_HUB_ALLOWED_TASKS", ["dummy-task"]) +@patch("evaluate.hub.dataset_info", lambda x: True) +@patch("evaluate.hub.model_info", lambda x: True) +@patch("evaluate.hub.metadata_update") +class TestHub(TestCase): + @pytest.fixture(autouse=True) + def inject_fixtures(self, caplog): + self._caplog = caplog + + def setUp(self): + self.metric = DummyMetric() + self.metric.add() + self.args = {"hello": 1, "world": 2} + self.result = self.metric.compute() + + def test_push_metric_required_arguments(self, metadata_update): + push_to_hub( + model_id="username/repo", + metric_value=self.result["accuracy"], + metric_name="Pretty Metric Name", + metric_type=self.metric.name, + dataset_name="dataset_name", + dataset_type="dataset_type", + task_type="dummy-task", + ) + + metadata_update.assert_called_once_with(repo_id="username/repo", metadata=minimum_metadata, overwrite=False) + + def test_push_metric_missing_arguments(self, metadata_update): + with pytest.raises(TypeError): + push_to_hub( + model_id="username/repo", + metric_value=self.result["accuracy"], + metric_name="Pretty Metric Name", + metric_type=self.metric.name, + dataset_name="dataset_name", + dataset_type="dummy-task", + ) + + def test_push_metric_invalid_arguments(self, metadata_update): + with pytest.raises(TypeError): + push_to_hub( + model_id="username/repo", + metric_value=self.result["accuracy"], + metric_name="Pretty Metric Name", + metric_type=self.metric.name, + dataset_name="dataset_name", + dataset_type="dataset_type", + task_type="dummy-task", + random_value="incorrect", + ) + + def test_push_metric_extra_arguments(self, metadata_update): + push_to_hub( + model_id="username/repo", + metric_value=self.result["accuracy"], + metric_name="Pretty Metric Name", + metric_type=self.metric.name, + dataset_name="dataset_name", + dataset_type="dataset_type", + dataset_config="fr", + dataset_split="test", + dataset_revision="abc", + dataset_args={"a": 1, "b": 2}, + task_type="dummy-task", + task_name="task_name", + metric_config=self.metric.config_name, + metric_args=self.args, + ) + + metadata_update.assert_called_once_with(repo_id="username/repo", metadata=extras_metadata, overwrite=False) + + def test_push_metric_invalid_task_type(self, metadata_update): + with pytest.raises(ValueError): + push_to_hub( + model_id="username/repo", + metric_value=self.result["accuracy"], + metric_name="Pretty Metric Name", + metric_type=self.metric.name, + dataset_name="dataset_name", + dataset_type="dataset_type", + task_type="audio-classification", + ) + + def test_push_metric_invalid_dataset_type(self, metadata_update): + with patch("evaluate.hub.dataset_info") as mock_dataset_info: + mock_dataset_info.side_effect = requests.HTTPError() + push_to_hub( + model_id="username/repo", + metric_value=self.result["accuracy"], + metric_name="Pretty Metric Name", + metric_type=self.metric.name, + dataset_name="dataset_name", + dataset_type="dataset_type", + task_type="dummy-task", + ) + + assert "Dataset dataset_type not found on the Hub at hf.co/datasets/dataset_type" in self._caplog.text + metadata_update.assert_called_once_with( + repo_id="username/repo", metadata=minimum_metadata, overwrite=False + ) + + def test_push_metric_invalid_model_id(self, metadata_update): + with patch("evaluate.hub.model_info") as mock_model_info: + mock_model_info.side_effect = requests.HTTPError() + with pytest.raises(ValueError): + push_to_hub( + model_id="username/bad-repo", + metric_value=self.result["accuracy"], + metric_name="Pretty Metric Name", + metric_type=self.metric.name, + dataset_name="dataset_name", + dataset_type="dataset_type", + task_type="dummy-task", + ) + + +class ValidateYaml(TestCase): + def setUp(self): + pass + + def testLoadingCards(self): + readme_filepaths = [] + for glob_path in ["measurements/*/README.md", "metrics/*/README.md", "comparisons/*/README.md"]: + readme_filepaths.extend(glob.glob(glob_path)) + for readme_file in readme_filepaths: + with open(readme_file, encoding="utf8") as f_yaml: + x = yaml.safe_load_all(f_yaml) + self.assertIsInstance(next(x), dict) diff --git a/TextClassification/TextClassification/evaluate/tests/test_load.py b/TextClassification/TextClassification/evaluate/tests/test_load.py new file mode 100644 index 0000000000000000000000000000000000000000..e20ea671fcb52874b498b5cb933818066296d9e2 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/tests/test_load.py @@ -0,0 +1,140 @@ +import importlib +import os +import tempfile +from unittest import TestCase + +import pytest +from datasets import DownloadConfig + +import evaluate +from evaluate.loading import ( + CachedEvaluationModuleFactory, + HubEvaluationModuleFactory, + LocalEvaluationModuleFactory, + evaluation_module_factory, +) + +from .utils import OfflineSimulationMode, offline + + +SAMPLE_METRIC_IDENTIFIER = "lvwerra/test" + +METRIC_LOADING_SCRIPT_NAME = "__dummy_metric1__" + +METRIC_LOADING_SCRIPT_CODE = """ +import evaluate +from evaluate import EvaluationModuleInfo +from datasets import Features, Value + +class __DummyMetric1__(evaluate.EvaluationModule): + + def _info(self): + return EvaluationModuleInfo(features=Features({"predictions": Value("int"), "references": Value("int")})) + + def _compute(self, predictions, references): + return {"__dummy_metric1__": sum(int(p == r) for p, r in zip(predictions, references))} +""" + + +@pytest.fixture +def metric_loading_script_dir(tmp_path): + script_name = METRIC_LOADING_SCRIPT_NAME + script_dir = tmp_path / script_name + script_dir.mkdir() + script_path = script_dir / f"{script_name}.py" + with open(script_path, "w") as f: + f.write(METRIC_LOADING_SCRIPT_CODE) + return str(script_dir) + + +class ModuleFactoryTest(TestCase): + @pytest.fixture(autouse=True) + def inject_fixtures(self, metric_loading_script_dir): + self._metric_loading_script_dir = metric_loading_script_dir + + def setUp(self): + self.hf_modules_cache = tempfile.mkdtemp() + self.cache_dir = tempfile.mkdtemp() + self.download_config = DownloadConfig(cache_dir=self.cache_dir) + self.dynamic_modules_path = evaluate.loading.init_dynamic_modules( + name="test_datasets_modules_" + os.path.basename(self.hf_modules_cache), + hf_modules_cache=self.hf_modules_cache, + ) + + def test_HubEvaluationModuleFactory_with_internal_import(self): + # "squad_v2" requires additional imports (internal) + factory = HubEvaluationModuleFactory( + "evaluate-metric/squad_v2", + module_type="metric", + download_config=self.download_config, + dynamic_modules_path=self.dynamic_modules_path, + ) + module_factory_result = factory.get_module() + assert importlib.import_module(module_factory_result.module_path) is not None + + def test_HubEvaluationModuleFactory_with_external_import(self): + # "bleu" requires additional imports (external from github) + factory = HubEvaluationModuleFactory( + "evaluate-metric/bleu", + module_type="metric", + download_config=self.download_config, + dynamic_modules_path=self.dynamic_modules_path, + ) + module_factory_result = factory.get_module() + assert importlib.import_module(module_factory_result.module_path) is not None + + def test_HubEvaluationModuleFactoryWithScript(self): + factory = HubEvaluationModuleFactory( + SAMPLE_METRIC_IDENTIFIER, + download_config=self.download_config, + dynamic_modules_path=self.dynamic_modules_path, + ) + module_factory_result = factory.get_module() + assert importlib.import_module(module_factory_result.module_path) is not None + + def test_LocalMetricModuleFactory(self): + path = os.path.join(self._metric_loading_script_dir, f"{METRIC_LOADING_SCRIPT_NAME}.py") + factory = LocalEvaluationModuleFactory( + path, download_config=self.download_config, dynamic_modules_path=self.dynamic_modules_path + ) + module_factory_result = factory.get_module() + assert importlib.import_module(module_factory_result.module_path) is not None + + def test_CachedMetricModuleFactory(self): + path = os.path.join(self._metric_loading_script_dir, f"{METRIC_LOADING_SCRIPT_NAME}.py") + factory = LocalEvaluationModuleFactory( + path, download_config=self.download_config, dynamic_modules_path=self.dynamic_modules_path + ) + module_factory_result = factory.get_module() + for offline_mode in OfflineSimulationMode: + with offline(offline_mode): + factory = CachedEvaluationModuleFactory( + METRIC_LOADING_SCRIPT_NAME, + dynamic_modules_path=self.dynamic_modules_path, + ) + module_factory_result = factory.get_module() + assert importlib.import_module(module_factory_result.module_path) is not None + + def test_cache_with_remote_canonical_module(self): + metric = "accuracy" + evaluation_module_factory( + metric, download_config=self.download_config, dynamic_modules_path=self.dynamic_modules_path + ) + + for offline_mode in OfflineSimulationMode: + with offline(offline_mode): + evaluation_module_factory( + metric, download_config=self.download_config, dynamic_modules_path=self.dynamic_modules_path + ) + + def test_cache_with_remote_community_module(self): + metric = "lvwerra/test" + evaluation_module_factory( + metric, download_config=self.download_config, dynamic_modules_path=self.dynamic_modules_path + ) + + for offline_mode in OfflineSimulationMode: + with offline(offline_mode): + evaluation_module_factory( + metric, download_config=self.download_config, dynamic_modules_path=self.dynamic_modules_path + ) diff --git a/TextClassification/TextClassification/evaluate/tests/test_metric.py b/TextClassification/TextClassification/evaluate/tests/test_metric.py new file mode 100644 index 0000000000000000000000000000000000000000..598b0f9295a903002b367a629bb3cffbfd47402e --- /dev/null +++ b/TextClassification/TextClassification/evaluate/tests/test_metric.py @@ -0,0 +1,759 @@ +import os +import pickle +import tempfile +import time +from multiprocessing import Pool +from unittest import TestCase, mock + +import pytest +from datasets.features import Features, Sequence, Value + +from evaluate.module import EvaluationModule, EvaluationModuleInfo, combine + +from .utils import require_tf, require_torch + + +class DummyMetric(EvaluationModule): + def _info(self): + return EvaluationModuleInfo( + description="dummy metric for tests", + citation="insert citation here", + features=Features({"predictions": Value("int64"), "references": Value("int64")}), + ) + + def _compute(self, predictions, references): + result = {} + if not predictions: + return result + else: + result["accuracy"] = sum(i == j for i, j in zip(predictions, references)) / len(predictions) + try: + result["set_equality"] = set(predictions) == set(references) + except TypeError: + result["set_equality"] = None + return result + + @classmethod + def predictions_and_references(cls): + return ([1, 2, 3, 4], [1, 2, 4, 3]) + + @classmethod + def predictions_and_references_strings(cls): + return (["a", "b", "c", "d"], ["a", "b", "d", "c"]) + + @classmethod + def expected_results(cls): + return {"accuracy": 0.5, "set_equality": True} + + @classmethod + def other_predictions_and_references(cls): + return ([1, 3, 4, 5], [1, 2, 3, 4]) + + @classmethod + def other_expected_results(cls): + return {"accuracy": 0.25, "set_equality": False} + + @classmethod + def distributed_predictions_and_references(cls): + return ([1, 2, 3, 4], [1, 2, 3, 4]), ([1, 2, 4, 5], [1, 2, 3, 4]) + + @classmethod + def distributed_expected_results(cls): + return {"accuracy": 0.75, "set_equality": False} + + @classmethod + def separate_predictions_and_references(cls): + return ([1, 2, 3, 4], [1, 2, 3, 4]), ([1, 2, 4, 5], [1, 2, 3, 4]) + + @classmethod + def separate_expected_results(cls): + return [{"accuracy": 1.0, "set_equality": True}, {"accuracy": 0.5, "set_equality": False}] + + +class AnotherDummyMetric(EvaluationModule): + def _info(self): + return EvaluationModuleInfo( + description="another dummy metric for tests", + citation="insert citation here", + features=Features({"predictions": Value("int64"), "references": Value("int64")}), + ) + + def _compute(self, predictions, references): + return {"set_equality": False} + + @classmethod + def expected_results(cls): + return {"set_equality": False} + + +def properly_del_metric(metric): + """properly delete a metric on windows if the process is killed during multiprocessing""" + if metric is not None: + if metric.filelock is not None: + metric.filelock.release() + if metric.rendez_vous_lock is not None: + metric.rendez_vous_lock.release() + del metric.writer + del metric.data + del metric + + +def metric_compute(arg): + """Thread worker function for distributed evaluation testing. + On base level to be pickable. + """ + metric = None + try: + num_process, process_id, preds, refs, exp_id, cache_dir, wait = arg + metric = DummyMetric( + num_process=num_process, process_id=process_id, experiment_id=exp_id, cache_dir=cache_dir, timeout=5 + ) + time.sleep(wait) + results = metric.compute(predictions=preds, references=refs) + return results + finally: + properly_del_metric(metric) + + +def metric_add_batch_and_compute(arg): + """Thread worker function for distributed evaluation testing. + On base level to be pickable. + """ + metric = None + try: + num_process, process_id, preds, refs, exp_id, cache_dir, wait = arg + metric = DummyMetric( + num_process=num_process, process_id=process_id, experiment_id=exp_id, cache_dir=cache_dir, timeout=5 + ) + metric.add_batch(predictions=preds, references=refs) + time.sleep(wait) + results = metric.compute() + return results + finally: + properly_del_metric(metric) + + +def metric_add_and_compute(arg): + """Thread worker function for distributed evaluation testing. + On base level to be pickable. + """ + metric = None + try: + num_process, process_id, preds, refs, exp_id, cache_dir, wait = arg + metric = DummyMetric( + num_process=num_process, process_id=process_id, experiment_id=exp_id, cache_dir=cache_dir, timeout=5 + ) + for pred, ref in zip(preds, refs): + metric.add(prediction=pred, reference=ref) + time.sleep(wait) + results = metric.compute() + return results + finally: + properly_del_metric(metric) + + +class TestMetric(TestCase): + def test_dummy_metric(self): + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + + metric = DummyMetric(experiment_id="test_dummy_metric") + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + del metric + + metric = DummyMetric(experiment_id="test_dummy_metric") + metric.add_batch(predictions=preds, references=refs) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + metric = DummyMetric(experiment_id="test_dummy_metric") + for pred, ref in zip(preds, refs): + metric.add(prediction=pred, reference=ref) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + # With keep_in_memory + metric = DummyMetric(keep_in_memory=True, experiment_id="test_dummy_metric") + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + del metric + + metric = DummyMetric(keep_in_memory=True, experiment_id="test_dummy_metric") + metric.add_batch(predictions=preds, references=refs) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + metric = DummyMetric(keep_in_memory=True, experiment_id="test_dummy_metric") + for pred, ref in zip(preds, refs): + metric.add(prediction=pred, reference=ref) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + metric = DummyMetric(keep_in_memory=True, experiment_id="test_dummy_metric") + self.assertDictEqual({}, metric.compute(predictions=[], references=[])) + del metric + + metric = DummyMetric(keep_in_memory=True, experiment_id="test_dummy_metric") + with self.assertRaisesRegex(ValueError, "Mismatch in the number"): + metric.add_batch(predictions=[1, 2, 3], references=[1, 2, 3, 4]) + del metric + + def test_metric_with_cache_dir(self): + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + + with tempfile.TemporaryDirectory() as tmp_dir: + metric = DummyMetric(experiment_id="test_dummy_metric", cache_dir=tmp_dir) + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + del metric + + def test_concurrent_metrics(self): + preds, refs = DummyMetric.predictions_and_references() + other_preds, other_refs = DummyMetric.other_predictions_and_references() + expected_results = DummyMetric.expected_results() + other_expected_results = DummyMetric.other_expected_results() + + metric = DummyMetric(experiment_id="test_concurrent_metrics") + other_metric = DummyMetric( + experiment_id="test_concurrent_metrics", + ) + + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + self.assertDictEqual( + other_expected_results, other_metric.compute(predictions=other_preds, references=other_refs) + ) + del metric, other_metric + + metric = DummyMetric( + experiment_id="test_concurrent_metrics", + ) + other_metric = DummyMetric( + experiment_id="test_concurrent_metrics", + ) + metric.add_batch(predictions=preds, references=refs) + other_metric.add_batch(predictions=other_preds, references=other_refs) + self.assertDictEqual(expected_results, metric.compute()) + self.assertDictEqual(other_expected_results, other_metric.compute()) + + for pred, ref, other_pred, other_ref in zip(preds, refs, other_preds, other_refs): + metric.add(prediction=pred, reference=ref) + other_metric.add(prediction=other_pred, reference=other_ref) + self.assertDictEqual(expected_results, metric.compute()) + self.assertDictEqual(other_expected_results, other_metric.compute()) + del metric, other_metric + + # With keep_in_memory + metric = DummyMetric(experiment_id="test_concurrent_metrics", keep_in_memory=True) + other_metric = DummyMetric(experiment_id="test_concurrent_metrics", keep_in_memory=True) + + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + self.assertDictEqual( + other_expected_results, other_metric.compute(predictions=other_preds, references=other_refs) + ) + + metric = DummyMetric(experiment_id="test_concurrent_metrics", keep_in_memory=True) + other_metric = DummyMetric(experiment_id="test_concurrent_metrics", keep_in_memory=True) + metric.add_batch(predictions=preds, references=refs) + other_metric.add_batch(predictions=other_preds, references=other_refs) + self.assertDictEqual(expected_results, metric.compute()) + self.assertDictEqual(other_expected_results, other_metric.compute()) + + for pred, ref, other_pred, other_ref in zip(preds, refs, other_preds, other_refs): + metric.add(prediction=pred, reference=ref) + other_metric.add(prediction=other_pred, reference=other_ref) + self.assertDictEqual(expected_results, metric.compute()) + self.assertDictEqual(other_expected_results, other_metric.compute()) + del metric, other_metric + + def test_separate_experiments_in_parallel(self): + with tempfile.TemporaryDirectory() as tmp_dir: + (preds_0, refs_0), (preds_1, refs_1) = DummyMetric.separate_predictions_and_references() + expected_results = DummyMetric.separate_expected_results() + + pool = Pool(processes=2) + + results = pool.map( + metric_compute, + [ + (1, 0, preds_0, refs_0, None, tmp_dir, 0), + (1, 0, preds_1, refs_1, None, tmp_dir, 0), + ], + ) + self.assertDictEqual(expected_results[0], results[0]) + self.assertDictEqual(expected_results[1], results[1]) + del results + + # more than one sec of waiting so that the second metric has to sample a new hashing name + results = pool.map( + metric_compute, + [ + (1, 0, preds_0, refs_0, None, tmp_dir, 2), + (1, 0, preds_1, refs_1, None, tmp_dir, 2), + ], + ) + self.assertDictEqual(expected_results[0], results[0]) + self.assertDictEqual(expected_results[1], results[1]) + del results + + results = pool.map( + metric_add_and_compute, + [ + (1, 0, preds_0, refs_0, None, tmp_dir, 0), + (1, 0, preds_1, refs_1, None, tmp_dir, 0), + ], + ) + self.assertDictEqual(expected_results[0], results[0]) + self.assertDictEqual(expected_results[1], results[1]) + del results + + results = pool.map( + metric_add_batch_and_compute, + [ + (1, 0, preds_0, refs_0, None, tmp_dir, 0), + (1, 0, preds_1, refs_1, None, tmp_dir, 0), + ], + ) + self.assertDictEqual(expected_results[0], results[0]) + self.assertDictEqual(expected_results[1], results[1]) + del results + + def test_distributed_metrics(self): + with tempfile.TemporaryDirectory() as tmp_dir: + (preds_0, refs_0), (preds_1, refs_1) = DummyMetric.distributed_predictions_and_references() + expected_results = DummyMetric.distributed_expected_results() + + pool = Pool(processes=4) + + results = pool.map( + metric_compute, + [ + (2, 0, preds_0, refs_0, "test_distributed_metrics_0", tmp_dir, 0), + (2, 1, preds_1, refs_1, "test_distributed_metrics_0", tmp_dir, 0.5), + ], + ) + self.assertDictEqual(expected_results, results[0]) + self.assertIsNone(results[1]) + del results + + results = pool.map( + metric_compute, + [ + (2, 0, preds_0, refs_0, "test_distributed_metrics_0", tmp_dir, 0.5), + (2, 1, preds_1, refs_1, "test_distributed_metrics_0", tmp_dir, 0), + ], + ) + self.assertDictEqual(expected_results, results[0]) + self.assertIsNone(results[1]) + del results + + results = pool.map( + metric_add_and_compute, + [ + (2, 0, preds_0, refs_0, "test_distributed_metrics_1", tmp_dir, 0), + (2, 1, preds_1, refs_1, "test_distributed_metrics_1", tmp_dir, 0), + ], + ) + self.assertDictEqual(expected_results, results[0]) + self.assertIsNone(results[1]) + del results + + results = pool.map( + metric_add_batch_and_compute, + [ + (2, 0, preds_0, refs_0, "test_distributed_metrics_2", tmp_dir, 0), + (2, 1, preds_1, refs_1, "test_distributed_metrics_2", tmp_dir, 0), + ], + ) + self.assertDictEqual(expected_results, results[0]) + self.assertIsNone(results[1]) + del results + + # To use several distributed metrics on the same local file system, need to specify an experiment_id + try: + results = pool.map( + metric_add_and_compute, + [ + (2, 0, preds_0, refs_0, "test_distributed_metrics_3", tmp_dir, 0), + (2, 1, preds_1, refs_1, "test_distributed_metrics_3", tmp_dir, 0), + (2, 0, preds_0, refs_0, "test_distributed_metrics_3", tmp_dir, 0), + (2, 1, preds_1, refs_1, "test_distributed_metrics_3", tmp_dir, 0), + ], + ) + except ValueError: + # We are fine with either raising a ValueError or computing well the metric + # Being sure we raise the error would means making the dummy dataset bigger + # and the test longer... + pass + else: + self.assertDictEqual(expected_results, results[0]) + self.assertDictEqual(expected_results, results[2]) + self.assertIsNone(results[1]) + self.assertIsNone(results[3]) + del results + + results = pool.map( + metric_add_and_compute, + [ + (2, 0, preds_0, refs_0, "exp_0", tmp_dir, 0), + (2, 1, preds_1, refs_1, "exp_0", tmp_dir, 0), + (2, 0, preds_0, refs_0, "exp_1", tmp_dir, 0), + (2, 1, preds_1, refs_1, "exp_1", tmp_dir, 0), + ], + ) + self.assertDictEqual(expected_results, results[0]) + self.assertDictEqual(expected_results, results[2]) + self.assertIsNone(results[1]) + self.assertIsNone(results[3]) + del results + + # With keep_in_memory is not allowed + with self.assertRaises(ValueError): + DummyMetric( + experiment_id="test_distributed_metrics_4", + keep_in_memory=True, + num_process=2, + process_id=0, + cache_dir=tmp_dir, + ) + + def test_dummy_metric_pickle(self): + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_file = os.path.join(tmp_dir, "metric.pt") + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + + metric = DummyMetric(experiment_id="test_dummy_metric_pickle") + + with open(tmp_file, "wb") as f: + pickle.dump(metric, f) + del metric + + with open(tmp_file, "rb") as f: + metric = pickle.load(f) + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + del metric + + def test_input_numpy(self): + import numpy as np + + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + preds, refs = np.array(preds), np.array(refs) + + metric = DummyMetric(experiment_id="test_input_numpy") + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + del metric + + metric = DummyMetric(experiment_id="test_input_numpy") + metric.add_batch(predictions=preds, references=refs) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + metric = DummyMetric(experiment_id="test_input_numpy") + for pred, ref in zip(preds, refs): + metric.add(prediction=pred, reference=ref) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + @require_torch + def test_input_torch(self): + import torch + + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + preds, refs = torch.tensor(preds), torch.tensor(refs) + + metric = DummyMetric(experiment_id="test_input_torch") + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + del metric + + metric = DummyMetric(experiment_id="test_input_torch") + metric.add_batch(predictions=preds, references=refs) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + metric = DummyMetric(experiment_id="test_input_torch") + for pred, ref in zip(preds, refs): + metric.add(prediction=pred, reference=ref) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + @require_tf + def test_input_tf(self): + import tensorflow as tf + + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + preds, refs = tf.constant(preds), tf.constant(refs) + + metric = DummyMetric(experiment_id="test_input_tf") + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + del metric + + metric = DummyMetric(experiment_id="test_input_tf") + metric.add_batch(predictions=preds, references=refs) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + metric = DummyMetric(experiment_id="test_input_tf") + for pred, ref in zip(preds, refs): + metric.add(prediction=pred, reference=ref) + self.assertDictEqual(expected_results, metric.compute()) + del metric + + def test_string_casting(self): + metric = DummyMetric(experiment_id="test_string_casting") + metric.info.features = Features({"predictions": Value("string"), "references": Value("string")}) + metric.compute(predictions=["a"], references=["a"]) + with self.assertRaises(ValueError): + metric.compute(predictions=[1], references=[1]) + + metric = DummyMetric(experiment_id="test_string_casting_2") + metric.info.features = Features( + {"predictions": Sequence(Value("string")), "references": Sequence(Value("string"))} + ) + metric.compute(predictions=[["a"]], references=[["a"]]) + with self.assertRaises(ValueError): + metric.compute(predictions=["a"], references=["a"]) + + def test_string_casting_tested_once(self): + + self.counter = 0 + + def checked_fct(fct): # wrapper function that increases a counter on each call + def wrapped(*args, **kwargs): + self.counter += 1 + return fct(*args, **kwargs) + + return wrapped + + with mock.patch( + "evaluate.EvaluationModule._enforce_nested_string_type", + checked_fct(DummyMetric._enforce_nested_string_type), + ): + metric = DummyMetric(experiment_id="test_string_casting_called_once") + metric.info.features = Features( + {"references": Sequence(Value("string")), "predictions": Sequence(Value("string"))} + ) + refs = [["test"] * 10] * 10 + preds = [["test"] * 10] * 10 + + metric.add_batch(references=refs, predictions=preds) + metric.add_batch(references=refs, predictions=preds) + + # the function is called twice for every batch's input: once on the + # sequence and then recursively agin on the first input of the sequence + self.assertEqual(self.counter, 8) + + def test_multiple_features(self): + metric = DummyMetric() + metric.info.features = [ + Features({"predictions": Value("int64"), "references": Value("int64")}), + Features({"predictions": Value("string"), "references": Value("string")}), + ] + + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + + metric.info.features = [ + Features({"predictions": Value("string"), "references": Value("string")}), + Features({"predictions": Value("int64"), "references": Value("int64")}), + ] + + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + self.assertDictEqual(expected_results, metric.compute(predictions=preds, references=refs)) + + del metric + + +class MetricWithMultiLabel(EvaluationModule): + def _info(self): + return EvaluationModuleInfo( + description="dummy metric for tests", + citation="insert citation here", + features=Features( + {"predictions": Sequence(Value("int64")), "references": Sequence(Value("int64"))} + if self.config_name == "multilabel" + else {"predictions": Value("int64"), "references": Value("int64")} + ), + ) + + def _compute(self, predictions=None, references=None): + return ( + { + "accuracy": sum(i == j for i, j in zip(predictions, references)) / len(predictions), + } + if predictions + else {} + ) + + +@pytest.mark.parametrize( + "config_name, predictions, references, expected", + [ + (None, [1, 2, 3, 4], [1, 2, 4, 3], 0.5), # Multiclass: Value("int64") + ( + "multilabel", + [[1, 0], [1, 0], [1, 0], [1, 0]], + [[1, 0], [0, 1], [1, 1], [0, 0]], + 0.25, + ), # Multilabel: Sequence(Value("int64")) + ], +) +def test_metric_with_multilabel(config_name, predictions, references, expected, tmp_path): + cache_dir = tmp_path / "cache" + metric = MetricWithMultiLabel(config_name, cache_dir=cache_dir) + results = metric.compute(predictions=predictions, references=references) + assert results["accuracy"] == expected + + +def test_safety_checks_process_vars(): + with pytest.raises(ValueError): + _ = DummyMetric(process_id=-2) + + with pytest.raises(ValueError): + _ = DummyMetric(num_process=2, process_id=3) + + +class AccuracyWithNonStandardFeatureNames(EvaluationModule): + def _info(self): + return EvaluationModuleInfo( + description="dummy metric for tests", + citation="insert citation here", + features=Features({"inputs": Value("int64"), "targets": Value("int64")}), + ) + + def _compute(self, inputs, targets): + return ( + { + "accuracy": sum(i == j for i, j in zip(inputs, targets)) / len(targets), + } + if targets + else {} + ) + + @classmethod + def inputs_and_targets(cls): + return ([1, 2, 3, 4], [1, 2, 4, 3]) + + @classmethod + def expected_results(cls): + return {"accuracy": 0.5} + + +def test_metric_with_non_standard_feature_names_add(tmp_path): + cache_dir = tmp_path / "cache" + inputs, targets = AccuracyWithNonStandardFeatureNames.inputs_and_targets() + metric = AccuracyWithNonStandardFeatureNames(cache_dir=cache_dir) + for input, target in zip(inputs, targets): + metric.add(inputs=input, targets=target) + results = metric.compute() + assert results == AccuracyWithNonStandardFeatureNames.expected_results() + + +def test_metric_with_non_standard_feature_names_add_batch(tmp_path): + cache_dir = tmp_path / "cache" + inputs, targets = AccuracyWithNonStandardFeatureNames.inputs_and_targets() + metric = AccuracyWithNonStandardFeatureNames(cache_dir=cache_dir) + metric.add_batch(inputs=inputs, targets=targets) + results = metric.compute() + assert results == AccuracyWithNonStandardFeatureNames.expected_results() + + +def test_metric_with_non_standard_feature_names_compute(tmp_path): + cache_dir = tmp_path / "cache" + inputs, targets = AccuracyWithNonStandardFeatureNames.inputs_and_targets() + metric = AccuracyWithNonStandardFeatureNames(cache_dir=cache_dir) + results = metric.compute(inputs=inputs, targets=targets) + assert results == AccuracyWithNonStandardFeatureNames.expected_results() + + +class TestEvaluationcombined_evaluation(TestCase): + def test_single_module(self): + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + + combined_evaluation = combine([DummyMetric()]) + + self.assertDictEqual(expected_results, combined_evaluation.compute(predictions=preds, references=refs)) + + def test_add(self): + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + + combined_evaluation = combine([DummyMetric()]) + + for pred, ref in zip(preds, refs): + combined_evaluation.add(pred, ref) + self.assertDictEqual(expected_results, combined_evaluation.compute()) + + def test_add_batch(self): + preds, refs = DummyMetric.predictions_and_references() + expected_results = DummyMetric.expected_results() + + combined_evaluation = combine([DummyMetric()]) + + combined_evaluation.add_batch(predictions=preds, references=refs) + self.assertDictEqual(expected_results, combined_evaluation.compute()) + + def test_force_prefix_with_dict(self): + prefix = "test_prefix" + preds, refs = DummyMetric.predictions_and_references() + + expected_results = DummyMetric.expected_results() + expected_results[f"{prefix}_accuracy"] = expected_results.pop("accuracy") + expected_results[f"{prefix}_set_equality"] = expected_results.pop("set_equality") + + combined_evaluation = combine({prefix: DummyMetric()}, force_prefix=True) + + self.assertDictEqual(expected_results, combined_evaluation.compute(predictions=preds, references=refs)) + + def test_duplicate_module(self): + preds, refs = DummyMetric.predictions_and_references() + dummy_metric = DummyMetric() + dummy_result = DummyMetric.expected_results() + combined_evaluation = combine([dummy_metric, dummy_metric]) + + expected_results = {} + for i in range(2): + for k in dummy_result: + expected_results[f"{dummy_metric.name}_{i}_{k}"] = dummy_result[k] + self.assertDictEqual(expected_results, combined_evaluation.compute(predictions=preds, references=refs)) + + def test_two_modules_with_same_score_name(self): + preds, refs = DummyMetric.predictions_and_references() + dummy_metric = DummyMetric() + another_dummy_metric = AnotherDummyMetric() + + dummy_result_1 = DummyMetric.expected_results() + dummy_result_2 = AnotherDummyMetric.expected_results() + + dummy_result_1[dummy_metric.name + "_set_equality"] = dummy_result_1.pop("set_equality") + dummy_result_1[another_dummy_metric.name + "_set_equality"] = dummy_result_2["set_equality"] + + combined_evaluation = combine([dummy_metric, another_dummy_metric]) + + self.assertDictEqual(dummy_result_1, combined_evaluation.compute(predictions=preds, references=refs)) + + def test_modules_from_string(self): + expected_result = {"accuracy": 0.5, "recall": 0.5, "precision": 1.0} + predictions = [0, 1] + references = [1, 1] + + combined_evaluation = combine(["accuracy", "recall", "precision"]) + + self.assertDictEqual( + expected_result, combined_evaluation.compute(predictions=predictions, references=references) + ) + + def test_modules_from_string_poslabel(self): + expected_result = {"recall": 1.0, "precision": 0.5} + predictions = [0, 1, 0] + references = [1, 1, 0] + + combined_evaluation = combine(["recall", "precision"]) + + self.assertDictEqual( + expected_result, combined_evaluation.compute(predictions=predictions, references=references, pos_label=0) + ) diff --git a/TextClassification/TextClassification/evaluate/tests/test_metric_common.py b/TextClassification/TextClassification/evaluate/tests/test_metric_common.py new file mode 100644 index 0000000000000000000000000000000000000000..014dc0b32b5eb21808862301613360902bcacae1 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/tests/test_metric_common.py @@ -0,0 +1,227 @@ +# Copyright 2020 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import doctest +import glob +import importlib +import inspect +import os +import re +from contextlib import contextmanager +from functools import wraps +from unittest.mock import patch + +import numpy as np +import pytest +from absl.testing import parameterized + +import evaluate +from evaluate import load + +from .utils import _run_slow_tests, for_all_test_methods, local, slow + + +REQUIRE_FAIRSEQ = {"comet"} +_has_fairseq = importlib.util.find_spec("fairseq") is not None + +UNSUPPORTED_ON_WINDOWS = {"code_eval"} +_on_windows = os.name == "nt" + +SLOW_METRIC = {"perplexity", "regard", "toxicity"} + + +def skip_if_metric_requires_fairseq(test_case): + @wraps(test_case) + def wrapper(self, evaluation_module_name, evaluation_module_type): + if not _has_fairseq and evaluation_module_name in REQUIRE_FAIRSEQ: + self.skipTest('"test requires Fairseq"') + else: + test_case(self, evaluation_module_name, evaluation_module_type) + + return wrapper + + +def skip_on_windows_if_not_windows_compatible(test_case): + @wraps(test_case) + def wrapper(self, evaluation_module_name, evaluation_module_type): + if _on_windows and evaluation_module_name in UNSUPPORTED_ON_WINDOWS: + self.skipTest('"test not supported on Windows"') + else: + test_case(self, evaluation_module_name, evaluation_module_type) + + return wrapper + + +def skip_slow_metrics(test_case): + @wraps(test_case) + def wrapper(self, evaluation_module_name, evaluation_module_type): + if not _run_slow_tests and evaluation_module_name in SLOW_METRIC: + self.skipTest('"test is slow"') + else: + test_case(self, evaluation_module_name, evaluation_module_type) + + return wrapper + + +def get_local_module_names(): + metrics = [metric_dir.split(os.sep)[-2] for metric_dir in glob.glob("./metrics/*/")] + comparisons = [metric_dir.split(os.sep)[-2] for metric_dir in glob.glob("./comparisons/*/")] + measurements = [metric_dir.split(os.sep)[-2] for metric_dir in glob.glob("./measurements/*/")] + + evaluation_modules = metrics + comparisons + measurements + evaluation_module_types = ( + ["metric"] * len(metrics) + ["comparison"] * len(comparisons) + ["measurement"] * len(measurements) + ) + + return [ + {"testcase_name": f"{t}_{x}", "evaluation_module_name": x, "evaluation_module_type": t} + for x, t in zip(evaluation_modules, evaluation_module_types) + if x != "gleu" # gleu is unfinished + ] + + +@parameterized.named_parameters(get_local_module_names()) +@for_all_test_methods(skip_if_metric_requires_fairseq, skip_on_windows_if_not_windows_compatible, skip_slow_metrics) +@local +class LocalModuleTest(parameterized.TestCase): + INTENSIVE_CALLS_PATCHER = {} + evaluation_module_name = None + evaluation_module_type = None + + def test_load(self, evaluation_module_name, evaluation_module_type): + doctest.ELLIPSIS_MARKER = "[...]" + evaluation_module = importlib.import_module( + evaluate.loading.evaluation_module_factory( + os.path.join(evaluation_module_type + "s", evaluation_module_name), module_type=evaluation_module_type + ).module_path + ) + evaluation_instance = evaluate.loading.import_main_class(evaluation_module.__name__) + # check parameters + parameters = inspect.signature(evaluation_instance._compute).parameters + self.assertTrue(all([p.kind != p.VAR_KEYWORD for p in parameters.values()])) # no **kwargs + # run doctest + with self.patch_intensive_calls(evaluation_module_name, evaluation_module.__name__): + with self.use_local_metrics(evaluation_module_type): + try: + results = doctest.testmod(evaluation_module, verbose=True, raise_on_error=True) + except doctest.UnexpectedException as e: + raise e.exc_info[1] # raise the exception that doctest caught + self.assertEqual(results.failed, 0) + self.assertGreater(results.attempted, 1) + + @slow + def test_load_real_metric(self, evaluation_module_name, evaluation_module_type): + doctest.ELLIPSIS_MARKER = "[...]" + metric_module = importlib.import_module( + evaluate.loading.evaluation_module_factory( + os.path.join(evaluation_module_type, evaluation_module_name) + ).module_path + ) + # run doctest + with self.use_local_metrics(): + results = doctest.testmod(metric_module, verbose=True, raise_on_error=True) + self.assertEqual(results.failed, 0) + self.assertGreater(results.attempted, 1) + + @contextmanager + def patch_intensive_calls(self, evaluation_module_name, module_name): + if evaluation_module_name in self.INTENSIVE_CALLS_PATCHER: + with self.INTENSIVE_CALLS_PATCHER[evaluation_module_name](module_name): + yield + else: + yield + + @contextmanager + def use_local_metrics(self, evaluation_module_type): + def load_local_metric(evaluation_module_name, *args, **kwargs): + return load(os.path.join(evaluation_module_type + "s", evaluation_module_name), *args, **kwargs) + + with patch("evaluate.load") as mock_load: + mock_load.side_effect = load_local_metric + yield + + @classmethod + def register_intensive_calls_patcher(cls, evaluation_module_name): + def wrapper(patcher): + patcher = contextmanager(patcher) + cls.INTENSIVE_CALLS_PATCHER[evaluation_module_name] = patcher + return patcher + + return wrapper + + +# Metrics intensive calls patchers +# -------------------------------- + + +@LocalModuleTest.register_intensive_calls_patcher("bleurt") +def patch_bleurt(module_name): + import tensorflow.compat.v1 as tf + from bleurt.score import Predictor + + tf.flags.DEFINE_string("sv", "", "") # handle pytest cli flags + + class MockedPredictor(Predictor): + def predict(self, input_dict): + assert len(input_dict["input_ids"]) == 2 + return np.array([1.03, 1.04]) + + # mock predict_fn which is supposed to do a forward pass with a bleurt model + with patch("bleurt.score._create_predictor") as mock_create_predictor: + mock_create_predictor.return_value = MockedPredictor() + yield + + +@LocalModuleTest.register_intensive_calls_patcher("bertscore") +def patch_bertscore(module_name): + import torch + + def bert_cos_score_idf(model, refs, *args, **kwargs): + return torch.tensor([[1.0, 1.0, 1.0]] * len(refs)) + + # mock get_model which is supposed to do download a bert model + # mock bert_cos_score_idf which is supposed to do a forward pass with a bert model + with patch("bert_score.scorer.get_model"), patch( + "bert_score.scorer.bert_cos_score_idf" + ) as mock_bert_cos_score_idf: + mock_bert_cos_score_idf.side_effect = bert_cos_score_idf + yield + + +@LocalModuleTest.register_intensive_calls_patcher("comet") +def patch_comet(module_name): + def load_from_checkpoint(model_path): + class Model: + def predict(self, data, *args, **kwargs): + assert len(data) == 2 + scores = [0.19, 0.92] + return scores, sum(scores) / len(scores) + + return Model() + + # mock load_from_checkpoint which is supposed to do download a bert model + # mock load_from_checkpoint which is supposed to do download a bert model + with patch("comet.download_model") as mock_download_model: + mock_download_model.return_value = None + with patch("comet.load_from_checkpoint") as mock_load_from_checkpoint: + mock_load_from_checkpoint.side_effect = load_from_checkpoint + yield + + +def test_seqeval_raises_when_incorrect_scheme(): + metric = load(os.path.join("metrics", "seqeval")) + wrong_scheme = "ERROR" + error_message = f"Scheme should be one of [IOB1, IOB2, IOE1, IOE2, IOBES, BILOU], got {wrong_scheme}" + with pytest.raises(ValueError, match=re.escape(error_message)): + metric.compute(predictions=[], references=[], scheme=wrong_scheme) diff --git a/TextClassification/TextClassification/evaluate/tests/test_save.py b/TextClassification/TextClassification/evaluate/tests/test_save.py new file mode 100644 index 0000000000000000000000000000000000000000..adca53feedec431c17efa85bbecc0f71e21d761e --- /dev/null +++ b/TextClassification/TextClassification/evaluate/tests/test_save.py @@ -0,0 +1,44 @@ +import json +import shutil +import tempfile +from pathlib import Path +from unittest import TestCase + +import evaluate + + +result_dict = {"metric": 1.0, "model_name": "x"} + +SAVE_EXTRA_KEYS = ["_timestamp", "_git_commit_hash", "_evaluate_version", "_python_version", "_interpreter_path"] + + +class TestSave(TestCase): + def setUp(self): + self.save_path = Path(tempfile.mkdtemp()) + + def tearDown(self): + shutil.rmtree(self.save_path) + + def test_save_to_folder(self): + file_path = evaluate.save(self.save_path, **result_dict) + with open(file_path, "r") as f: + loaded_result_dict = json.load(f) + for key in SAVE_EXTRA_KEYS: + _ = loaded_result_dict.pop(key) + self.assertDictEqual(result_dict, loaded_result_dict) + + def test_save_to_folder_nested(self): + file_path = evaluate.save(self.save_path / "sub_dir1/sub_dir2", **result_dict) + with open(file_path, "r") as f: + loaded_result_dict = json.load(f) + for key in SAVE_EXTRA_KEYS: + _ = loaded_result_dict.pop(key) + self.assertDictEqual(result_dict, loaded_result_dict) + + def test_save_to_file(self): + _ = evaluate.save(self.save_path / "test.json", **result_dict) + with open(self.save_path / "test.json", "r") as f: + loaded_result_dict = json.load(f) + for key in SAVE_EXTRA_KEYS: + _ = loaded_result_dict.pop(key) + self.assertDictEqual(result_dict, loaded_result_dict) diff --git a/TextClassification/TextClassification/evaluate/tests/test_trainer_evaluator_parity.py b/TextClassification/TextClassification/evaluate/tests/test_trainer_evaluator_parity.py new file mode 100644 index 0000000000000000000000000000000000000000..b513da88663088e18899e770274f102a7c06e546 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/tests/test_trainer_evaluator_parity.py @@ -0,0 +1,313 @@ +import json +import os +import shutil +import subprocess +import tempfile +import unittest + +import numpy as np +import torch +import transformers +from datasets import load_dataset +from transformers import AutoFeatureExtractor, AutoModelForImageClassification, Trainer, TrainingArguments, pipeline + +from evaluate import evaluator, load + +from .utils import slow + + +class TestEvaluatorTrainerParity(unittest.TestCase): + def setUp(self): + self.dir_path = tempfile.mkdtemp("evaluator_trainer_parity_test") + + transformers_version = transformers.__version__ + branch = "" + if not transformers_version.endswith(".dev0"): + branch = f"--branch v{transformers_version}" + subprocess.run( + f"git clone --depth 3 --filter=blob:none --sparse {branch} https://github.com/huggingface/transformers", + shell=True, + cwd=self.dir_path, + ) + + def tearDown(self): + shutil.rmtree(self.dir_path, ignore_errors=True) + + def test_text_classification_parity(self): + model_name = "philschmid/tiny-bert-sst2-distilled" + + subprocess.run( + "git sparse-checkout set examples/pytorch/text-classification", + shell=True, + cwd=os.path.join(self.dir_path, "transformers"), + ) + + subprocess.run( + f"python examples/pytorch/text-classification/run_glue.py" + f" --model_name_or_path {model_name}" + f" --task_name sst2" + f" --do_eval" + f" --max_seq_length 9999999999" # rely on tokenizer.model_max_length for max_length + f" --output_dir {os.path.join(self.dir_path, 'textclassification_sst2_transformers')}" + f" --max_eval_samples 80", + shell=True, + cwd=os.path.join(self.dir_path, "transformers"), + ) + + with open( + f"{os.path.join(self.dir_path, 'textclassification_sst2_transformers', 'eval_results.json')}", "r" + ) as f: + transformers_results = json.load(f) + + eval_dataset = load_dataset("glue", "sst2", split="validation[:80]") + + pipe = pipeline(task="text-classification", model=model_name, tokenizer=model_name) + + task_evaluator = evaluator(task="text-classification") + evaluator_results = task_evaluator.compute( + model_or_pipeline=pipe, + data=eval_dataset, + metric="accuracy", + input_column="sentence", + label_column="label", + label_mapping={"negative": 0, "positive": 1}, + strategy="simple", + ) + + self.assertEqual(transformers_results["eval_accuracy"], evaluator_results["accuracy"]) + + @slow + def test_text_classification_parity_two_columns(self): + model_name = "prajjwal1/bert-tiny-mnli" + max_eval_samples = 150 + + subprocess.run( + "git sparse-checkout set examples/pytorch/text-classification", + shell=True, + cwd=os.path.join(self.dir_path, "transformers"), + ) + + subprocess.run( + f"python examples/pytorch/text-classification/run_glue.py" + f" --model_name_or_path {model_name}" + f" --task_name mnli" + f" --do_eval" + f" --max_seq_length 256" + f" --output_dir {os.path.join(self.dir_path, 'textclassification_mnli_transformers')}" + f" --max_eval_samples {max_eval_samples}", + shell=True, + cwd=os.path.join(self.dir_path, "transformers"), + ) + + with open( + f"{os.path.join(self.dir_path, 'textclassification_mnli_transformers', 'eval_results.json')}", "r" + ) as f: + transformers_results = json.load(f) + + eval_dataset = load_dataset("glue", "mnli", split=f"validation_matched[:{max_eval_samples}]") + + pipe = pipeline(task="text-classification", model=model_name, tokenizer=model_name, max_length=256) + + task_evaluator = evaluator(task="text-classification") + evaluator_results = task_evaluator.compute( + model_or_pipeline=pipe, + data=eval_dataset, + metric="accuracy", + input_column="premise", + second_input_column="hypothesis", + label_column="label", + label_mapping={"LABEL_0": 0, "LABEL_1": 1, "LABEL_2": 2}, + ) + + self.assertEqual(transformers_results["eval_accuracy"], evaluator_results["accuracy"]) + + def test_image_classification_parity(self): + # we can not compare to the Pytorch transformers example, that uses custom preprocessing on the images + model_name = "douwekiela/resnet-18-finetuned-dogfood" + dataset_name = "beans" + max_eval_samples = 120 + + raw_dataset = load_dataset(dataset_name, split="validation") + eval_dataset = raw_dataset.select(range(max_eval_samples)) + + feature_extractor = AutoFeatureExtractor.from_pretrained(model_name) + model = AutoModelForImageClassification.from_pretrained(model_name) + + def collate_fn(examples): + pixel_values = torch.stack( + [torch.tensor(feature_extractor(example["image"])["pixel_values"][0]) for example in examples] + ) + labels = torch.tensor([example["labels"] for example in examples]) + return {"pixel_values": pixel_values, "labels": labels} + + metric = load("accuracy") + trainer = Trainer( + model=model, + args=TrainingArguments( + output_dir=os.path.join(self.dir_path, "imageclassification_beans_transformers"), + remove_unused_columns=False, + ), + train_dataset=None, + eval_dataset=eval_dataset, + compute_metrics=lambda p: metric.compute( + predictions=np.argmax(p.predictions, axis=1), references=p.label_ids + ), + tokenizer=None, + data_collator=collate_fn, + ) + + metrics = trainer.evaluate() + trainer.save_metrics("eval", metrics) + + with open( + f"{os.path.join(self.dir_path, 'imageclassification_beans_transformers', 'eval_results.json')}", "r" + ) as f: + transformers_results = json.load(f) + + pipe = pipeline(task="image-classification", model=model_name, feature_extractor=model_name) + + task_evaluator = evaluator(task="image-classification") + evaluator_results = task_evaluator.compute( + model_or_pipeline=pipe, + data=eval_dataset, + metric="accuracy", + input_column="image", + label_column="labels", + label_mapping=model.config.label2id, + strategy="simple", + ) + + self.assertEqual(transformers_results["eval_accuracy"], evaluator_results["accuracy"]) + + def test_question_answering_parity(self): + model_name_v1 = "anas-awadalla/bert-tiny-finetuned-squad" + model_name_v2 = "mrm8488/bert-tiny-finetuned-squadv2" + + subprocess.run( + "git sparse-checkout set examples/pytorch/question-answering", + shell=True, + cwd=os.path.join(self.dir_path, "transformers"), + ) + + # test squad_v1-like dataset + subprocess.run( + f"python examples/pytorch/question-answering/run_qa.py" + f" --model_name_or_path {model_name_v1}" + f" --dataset_name squad" + f" --do_eval" + f" --output_dir {os.path.join(self.dir_path, 'questionanswering_squad_transformers')}" + f" --max_eval_samples 100" + f" --max_seq_length 384", + shell=True, + cwd=os.path.join(self.dir_path, "transformers"), + ) + + with open( + f"{os.path.join(self.dir_path, 'questionanswering_squad_transformers', 'eval_results.json')}", "r" + ) as f: + transformers_results = json.load(f) + + eval_dataset = load_dataset("squad", split="validation[:100]") + + pipe = pipeline( + task="question-answering", + model=model_name_v1, + tokenizer=model_name_v1, + max_answer_len=30, + padding="max_length", + ) + + task_evaluator = evaluator(task="question-answering") + evaluator_results = task_evaluator.compute( + model_or_pipeline=pipe, + data=eval_dataset, + metric="squad", + strategy="simple", + ) + + self.assertEqual(transformers_results["eval_f1"], evaluator_results["f1"]) + self.assertEqual(transformers_results["eval_exact_match"], evaluator_results["exact_match"]) + + # test squad_v2-like dataset + subprocess.run( + f"python examples/pytorch/question-answering/run_qa.py" + f" --model_name_or_path {model_name_v2}" + f" --dataset_name squad_v2" + f" --version_2_with_negative" + f" --do_eval" + f" --output_dir {os.path.join(self.dir_path, 'questionanswering_squadv2_transformers')}" + f" --max_eval_samples 100" + f" --max_seq_length 384", + shell=True, + cwd=os.path.join(self.dir_path, "transformers"), + ) + + with open( + f"{os.path.join(self.dir_path, 'questionanswering_squadv2_transformers', 'eval_results.json')}", "r" + ) as f: + transformers_results = json.load(f) + + eval_dataset = load_dataset("squad_v2", split="validation[:100]") + + pipe = pipeline( + task="question-answering", + model=model_name_v2, + tokenizer=model_name_v2, + max_answer_len=30, + ) + + task_evaluator = evaluator(task="question-answering") + evaluator_results = task_evaluator.compute( + model_or_pipeline=pipe, + data=eval_dataset, + metric="squad_v2", + strategy="simple", + squad_v2_format=True, + ) + + self.assertEqual(transformers_results["eval_f1"], evaluator_results["f1"]) + self.assertEqual(transformers_results["eval_HasAns_f1"], evaluator_results["HasAns_f1"]) + self.assertEqual(transformers_results["eval_NoAns_f1"], evaluator_results["NoAns_f1"]) + + def test_token_classification_parity(self): + model_name = "hf-internal-testing/tiny-bert-for-token-classification" + n_samples = 500 + + subprocess.run( + "git sparse-checkout set examples/pytorch/token-classification", + shell=True, + cwd=os.path.join(self.dir_path, "transformers"), + ) + + subprocess.run( + f"python examples/pytorch/token-classification/run_ner.py" + f" --model_name_or_path {model_name}" + f" --dataset_name conll2003" + f" --do_eval" + f" --output_dir {os.path.join(self.dir_path, 'tokenclassification_conll2003_transformers')}" + f" --max_eval_samples {n_samples}", + shell=True, + cwd=os.path.join(self.dir_path, "transformers"), + ) + + with open( + os.path.join(self.dir_path, "tokenclassification_conll2003_transformers", "eval_results.json"), "r" + ) as f: + transformers_results = json.load(f) + + eval_dataset = load_dataset("conll2003", split=f"validation[:{n_samples}]") + + pipe = pipeline(task="token-classification", model=model_name) + + e = evaluator(task="token-classification") + evaluator_results = e.compute( + model_or_pipeline=pipe, + data=eval_dataset, + metric="seqeval", + input_column="tokens", + label_column="ner_tags", + strategy="simple", + ) + + self.assertEqual(transformers_results["eval_accuracy"], evaluator_results["overall_accuracy"]) + self.assertEqual(transformers_results["eval_f1"], evaluator_results["overall_f1"]) diff --git a/TextClassification/TextClassification/evaluate/tests/test_viz.py b/TextClassification/TextClassification/evaluate/tests/test_viz.py new file mode 100644 index 0000000000000000000000000000000000000000..c44e5104e377fba28a067337c89991d0df52e5b3 --- /dev/null +++ b/TextClassification/TextClassification/evaluate/tests/test_viz.py @@ -0,0 +1,24 @@ +from unittest import TestCase + +import matplotlib.pyplot as plt + +from evaluate.visualization import radar_plot + + +class TestViz(TestCase): + def test_invert_range(self): + data = [{"accuracy": 0.9, "precision": 0.8}, {"accuracy": 0.7, "precision": 0.6}] + model_names = ["model1", "model2"] + wrong_invert_range = ["latency_in_seconds"] # Value not present in data + with self.assertRaises(ValueError): + radar_plot(data, model_names, wrong_invert_range) + + def test_output_is_plot(self): + data = [ + {"accuracy": 0.9, "precision": 0.8, "latency_in_seconds": 48.1}, + {"accuracy": 0.7, "precision": 0.6, "latency_in_seconds": 51.4}, + ] + model_names = ["model1", "model2"] + invert_range = ["latency_in_seconds"] + out_plt = radar_plot(data, model_names, invert_range) + self.assertIsInstance(out_plt, plt.Figure) diff --git a/TextClassification/TextClassification/evaluate/tests/utils.py b/TextClassification/TextClassification/evaluate/tests/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fc27d9dbbb56b5d39e3b63e54237f13bf4ffacfd --- /dev/null +++ b/TextClassification/TextClassification/evaluate/tests/utils.py @@ -0,0 +1,290 @@ +import os +import tempfile +import unittest +from contextlib import contextmanager +from copy import deepcopy +from distutils.util import strtobool +from enum import Enum +from pathlib import Path +from unittest.mock import patch + +from evaluate import config + + +def parse_flag_from_env(key, default=False): + try: + value = os.environ[key] + except KeyError: + # KEY isn't set, default to `default`. + _value = default + else: + # KEY is set, convert it to True or False. + try: + _value = strtobool(value) + except ValueError: + # More values are supported, but let's keep the message simple. + raise ValueError(f"If set, {key} must be yes or no.") + return _value + + +_run_slow_tests = parse_flag_from_env("RUN_SLOW", default=False) +_run_remote_tests = parse_flag_from_env("RUN_REMOTE", default=False) +_run_local_tests = parse_flag_from_env("RUN_LOCAL", default=True) +_run_packaged_tests = parse_flag_from_env("RUN_PACKAGED", default=True) + + +def require_beam(test_case): + """ + Decorator marking a test that requires Apache Beam. + + These tests are skipped when Apache Beam isn't installed. + + """ + if not config.TORCH_AVAILABLE: + test_case = unittest.skip("test requires PyTorch")(test_case) + return test_case + + +def require_faiss(test_case): + """ + Decorator marking a test that requires Faiss. + + These tests are skipped when Faiss isn't installed. + + """ + try: + import faiss # noqa + except ImportError: + test_case = unittest.skip("test requires faiss")(test_case) + return test_case + + +def require_regex(test_case): + """ + Decorator marking a test that requires regex. + + These tests are skipped when Regex isn't installed. + + """ + try: + import regex # noqa + except ImportError: + test_case = unittest.skip("test requires regex")(test_case) + return test_case + + +def require_elasticsearch(test_case): + """ + Decorator marking a test that requires ElasticSearch. + + These tests are skipped when ElasticSearch isn't installed. + + """ + try: + import elasticsearch # noqa + except ImportError: + test_case = unittest.skip("test requires elasticsearch")(test_case) + return test_case + + +def require_torch(test_case): + """ + Decorator marking a test that requires PyTorch. + + These tests are skipped when PyTorch isn't installed. + + """ + if not config.TORCH_AVAILABLE: + test_case = unittest.skip("test requires PyTorch")(test_case) + return test_case + + +def require_tf(test_case): + """ + Decorator marking a test that requires TensorFlow. + + These tests are skipped when TensorFlow isn't installed. + + """ + if not config.TF_AVAILABLE: + test_case = unittest.skip("test requires TensorFlow")(test_case) + return test_case + + +def require_jax(test_case): + """ + Decorator marking a test that requires JAX. + + These tests are skipped when JAX isn't installed. + + """ + if not config.JAX_AVAILABLE: + test_case = unittest.skip("test requires JAX")(test_case) + return test_case + + +def require_pil(test_case): + """ + Decorator marking a test that requires Pillow. + + These tests are skipped when Pillow isn't installed. + + """ + if not config.PIL_AVAILABLE: + test_case = unittest.skip("test requires Pillow")(test_case) + return test_case + + +def require_transformers(test_case): + """ + Decorator marking a test that requires transformers. + + These tests are skipped when transformers isn't installed. + + """ + try: + import transformers # noqa F401 + except ImportError: + return unittest.skip("test requires transformers")(test_case) + else: + return test_case + + +def slow(test_case): + """ + Decorator marking a test as slow. + + Slow tests are skipped by default. Set the RUN_SLOW environment variable + to a truthy value to run them. + + """ + if not _run_slow_tests or _run_slow_tests == 0: + test_case = unittest.skip("test is slow")(test_case) + return test_case + + +def local(test_case): + """ + Decorator marking a test as local + + Local tests are run by default. Set the RUN_LOCAL environment variable + to a falsy value to not run them. + """ + if not _run_local_tests or _run_local_tests == 0: + test_case = unittest.skip("test is local")(test_case) + return test_case + + +def packaged(test_case): + """ + Decorator marking a test as packaged + + Packaged tests are run by default. Set the RUN_PACKAGED environment variable + to a falsy value to not run them. + """ + if not _run_packaged_tests or _run_packaged_tests == 0: + test_case = unittest.skip("test is packaged")(test_case) + return test_case + + +def remote(test_case): + """ + Decorator marking a test as one that relies on GitHub or the Hugging Face Hub. + + Remote tests are skipped by default. Set the RUN_REMOTE environment variable + to a falsy value to not run them. + """ + if not _run_remote_tests or _run_remote_tests == 0: + test_case = unittest.skip("test requires remote")(test_case) + return test_case + + +def for_all_test_methods(*decorators): + def decorate(cls): + for name, fn in cls.__dict__.items(): + if callable(fn) and name.startswith("test"): + for decorator in decorators: + fn = decorator(fn) + setattr(cls, name, fn) + return cls + + return decorate + + +class RequestWouldHangIndefinitelyError(Exception): + pass + + +class OfflineSimulationMode(Enum): + CONNECTION_FAILS = 0 + CONNECTION_TIMES_OUT = 1 + HF_EVALUATE_OFFLINE_SET_TO_1 = 2 + + +@contextmanager +def offline(mode=OfflineSimulationMode.CONNECTION_FAILS, timeout=1e-16): + """ + Simulate offline mode. + + There are three offline simulatiom modes: + + CONNECTION_FAILS (default mode): a ConnectionError is raised for each network call. + Connection errors are created by mocking socket.socket + CONNECTION_TIMES_OUT: the connection hangs until it times out. + The default timeout value is low (1e-16) to speed up the tests. + Timeout errors are created by mocking requests.request + HF_EVALUATE_OFFLINE_SET_TO_1: the HF_EVALUATE_OFFLINE environment variable is set to 1. + This makes the http/ftp calls of the library instantly fail and raise an OfflineModeEmabled error. + """ + from requests import request as online_request + + def timeout_request(method, url, **kwargs): + # Change the url to an invalid url so that the connection hangs + invalid_url = "https://10.255.255.1" + if kwargs.get("timeout") is None: + raise RequestWouldHangIndefinitelyError( + f"Tried a call to {url} in offline mode with no timeout set. Please set a timeout." + ) + kwargs["timeout"] = timeout + try: + return online_request(method, invalid_url, **kwargs) + except Exception as e: + # The following changes in the error are just here to make the offline timeout error prettier + e.request.url = url + max_retry_error = e.args[0] + max_retry_error.args = (max_retry_error.args[0].replace("10.255.255.1", f"OfflineMock[{url}]"),) + e.args = (max_retry_error,) + raise + + def offline_socket(*args, **kwargs): + raise OSError("Offline mode is enabled.") + + if mode is OfflineSimulationMode.CONNECTION_FAILS: + # inspired from https://stackoverflow.com/a/18601897 + with patch("socket.socket", offline_socket): + yield + elif mode is OfflineSimulationMode.CONNECTION_TIMES_OUT: + # inspired from https://stackoverflow.com/a/904609 + with patch("requests.request", timeout_request): + with patch("requests.api.request", timeout_request): + yield + elif mode is OfflineSimulationMode.HF_EVALUATE_OFFLINE_SET_TO_1: + with patch("evaluate.config.HF_EVALUATE_OFFLINE", True): + yield + else: + raise ValueError("Please use a value from the OfflineSimulationMode enum.") + + +@contextmanager +def set_current_working_directory_to_temp_dir(*args, **kwargs): + original_working_dir = str(Path().resolve()) + with tempfile.TemporaryDirectory(*args, **kwargs) as tmp_dir: + try: + os.chdir(tmp_dir) + yield + finally: + os.chdir(original_working_dir) + + +def is_rng_equal(rng1, rng2): + return deepcopy(rng1).integers(0, 100, 10).tolist() == deepcopy(rng2).integers(0, 100, 10).tolist() diff --git a/TextClassification/TextClassification/experiments/checkpoint-191/config.json b/TextClassification/TextClassification/experiments/checkpoint-191/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41769867e12878cdd43c28177f4e2b10082b7424 --- /dev/null +++ b/TextClassification/TextClassification/experiments/checkpoint-191/config.json @@ -0,0 +1,38 @@ +{ + "_name_or_path": "bert-base-uncased", + "architectures": [ + "BertForSequenceClassification" + ], + "attention_probs_dropout_prob": 0.1, + "classifier_dropout": null, + "finetuning_task": "text-classification", + "gradient_checkpointing": false, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 768, + "id2label": { + "0": "0", + "1": "1", + "2": "None" + }, + "initializer_range": 0.02, + "intermediate_size": 3072, + "label2id": { + "0": 0, + "1": 1, + "None": 2 + }, + "layer_norm_eps": 1e-12, + "max_position_embeddings": 512, + "model_type": "bert", + "num_attention_heads": 12, + "num_hidden_layers": 12, + "pad_token_id": 0, + "position_embedding_type": "absolute", + "problem_type": "single_label_classification", + "torch_dtype": "float32", + "transformers_version": "4.44.2", + "type_vocab_size": 2, + "use_cache": true, + "vocab_size": 30522 +} diff --git a/TextClassification/TextClassification/requirements.txt b/TextClassification/TextClassification/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..1d5cc9e9df3dfbfb376cfac3277ead048419f76f --- /dev/null +++ b/TextClassification/TextClassification/requirements.txt @@ -0,0 +1,7 @@ +accelerate >= 0.12.0 +datasets >= 1.8.0 +sentencepiece != 0.1.92 +scipy +scikit-learn +protobuf +evaluate \ No newline at end of file diff --git a/TextClassification/TextClassification/run_classification.py b/TextClassification/TextClassification/run_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..a5764759a995434808c983dcc481ff8459b5ed0b --- /dev/null +++ b/TextClassification/TextClassification/run_classification.py @@ -0,0 +1,744 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2020 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Finetuning the library models for text classification.""" +# You can also adapt this script on your own text classification task. Pointers for this are left as comments. + +import logging +import os +import random +import sys +import warnings +from dataclasses import dataclass, field +from typing import List, Optional + +import datasets +import evaluate +import numpy as np +from datasets import Value, load_dataset + +import transformers +from transformers import ( + AutoConfig, + AutoModelForSequenceClassification, + AutoTokenizer, + DataCollatorWithPadding, + EvalPrediction, + HfArgumentParser, + Trainer, + TrainingArguments, + default_data_collator, + set_seed, +) +from transformers.trainer_utils import get_last_checkpoint +from transformers.utils import check_min_version, send_example_telemetry +from transformers.utils.versions import require_version + +logger = logging.getLogger(__name__) + + +@dataclass +class DataTrainingArguments: + """ + Arguments pertaining to what data we are going to input our model for training and eval. + + Using `HfArgumentParser` we can turn this class + into argparse arguments to be able to specify them on + the command line. + """ + + dataset_name: Optional[str] = field( + default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."} + ) + dataset_config_name: Optional[str] = field( + default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."} + ) + do_regression: bool = field( + default=None, + metadata={ + "help": "Whether to do regression instead of classification. If None, will be inferred from the dataset." + }, + ) + text_column_names: Optional[str] = field( + default=None, + metadata={ + "help": ( + "The name of the text column in the input dataset or a CSV/JSON file. " + 'If not specified, will use the "sentence" column for single/multi-label classification task.' + ) + }, + ) + text_column_delimiter: Optional[str] = field( + default=" ", metadata={"help": "THe delimiter to use to join text columns into a single sentence."} + ) + train_split_name: Optional[str] = field( + default=None, + metadata={ + "help": 'The name of the train split in the input dataset. If not specified, will use the "train" split when do_train is enabled' + }, + ) + validation_split_name: Optional[str] = field( + default=None, + metadata={ + "help": 'The name of the validation split in the input dataset. If not specified, will use the "validation" split when do_eval is enabled' + }, + ) + test_split_name: Optional[str] = field( + default=None, + metadata={ + "help": 'The name of the test split in the input dataset. If not specified, will use the "test" split when do_predict is enabled' + }, + ) + remove_splits: Optional[str] = field( + default=None, + metadata={"help": "The splits to remove from the dataset. Multiple splits should be separated by commas."}, + ) + remove_columns: Optional[str] = field( + default=None, + metadata={"help": "The columns to remove from the dataset. Multiple columns should be separated by commas."}, + ) + label_column_name: Optional[str] = field( + default=None, + metadata={ + "help": ( + "The name of the label column in the input dataset or a CSV/JSON file. " + 'If not specified, will use the "label" column for single/multi-label classification task' + ) + }, + ) + max_seq_length: int = field( + default=128, + metadata={ + "help": ( + "The maximum total input sequence length after tokenization. Sequences longer " + "than this will be truncated, sequences shorter will be padded." + ) + }, + ) + overwrite_cache: bool = field( + default=False, metadata={"help": "Overwrite the cached preprocessed datasets or not."} + ) + pad_to_max_length: bool = field( + default=True, + metadata={ + "help": ( + "Whether to pad all samples to `max_seq_length`. " + "If False, will pad the samples dynamically when batching to the maximum length in the batch." + ) + }, + ) + shuffle_train_dataset: bool = field( + default=False, metadata={"help": "Whether to shuffle the train dataset or not."} + ) + shuffle_seed: int = field( + default=42, metadata={"help": "Random seed that will be used to shuffle the train dataset."} + ) + max_train_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ) + }, + ) + max_eval_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of evaluation examples to this " + "value if set." + ) + }, + ) + max_predict_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of prediction examples to this " + "value if set." + ) + }, + ) + metric_name: Optional[str] = field(default=None, metadata={"help": "The metric to use for evaluation."}) + train_file: Optional[str] = field( + default=None, metadata={"help": "A csv or a json file containing the training data."} + ) + validation_file: Optional[str] = field( + default=None, metadata={"help": "A csv or a json file containing the validation data."} + ) + test_file: Optional[str] = field(default=None, metadata={"help": "A csv or a json file containing the test data."}) + + def __post_init__(self): + if self.dataset_name is None: + if self.train_file is None or self.validation_file is None: + raise ValueError(" training/validation file or a dataset name.") + + train_extension = self.train_file.split(".")[-1] + assert train_extension in ["csv", "json"], "`train_file` should be a csv or a json file." + validation_extension = self.validation_file.split(".")[-1] + assert ( + validation_extension == train_extension + ), "`validation_file` should have the same extension (csv or json) as `train_file`." + + +@dataclass +class ModelArguments: + """ + Arguments pertaining to which model/config/tokenizer we are going to fine-tune from. + """ + + model_name_or_path: str = field( + metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"} + ) + config_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"} + ) + tokenizer_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"} + ) + cache_dir: Optional[str] = field( + default=None, + metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}, + ) + use_fast_tokenizer: bool = field( + default=True, + metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}, + ) + model_revision: str = field( + default="main", + metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."}, + ) + token: str = field( + default=None, + metadata={ + "help": ( + "The token to use as HTTP bearer authorization for remote files. If not specified, will use the token " + "generated when running `huggingface-cli login` (stored in `~/.huggingface`)." + ) + }, + ) + use_auth_token: bool = field( + default=None, + metadata={ + "help": "The `use_auth_token` argument is deprecated and will be removed in v4.34. Please use `token` instead." + }, + ) + trust_remote_code: bool = field( + default=False, + metadata={ + "help": ( + "Whether or not to allow for custom models defined on the Hub in their own modeling files. This option " + "should only be set to `True` for repositories you trust and in which you have read the code, as it will " + "execute code present on the Hub on your local machine." + ) + }, + ) + ignore_mismatched_sizes: bool = field( + default=False, + metadata={"help": "Will enable to load a pretrained model whose head dimensions are different."}, + ) + + +def get_label_list(raw_dataset, split="train") -> List[str]: + """Get the list of labels from a multi-label dataset""" + + if isinstance(raw_dataset[split]["label"][0], list): + label_list = [label for sample in raw_dataset[split]["label"] for label in sample] + label_list = list(set(label_list)) + else: + label_list = raw_dataset[split].unique("label") + # we will treat the label list as a list of string instead of int, consistent with model.config.label2id + label_list = [str(label) for label in label_list] + return label_list + + +def main(): + # See all possible arguments in src/transformers/training_args.py + # or by passing the --help flag to this script. + # We now keep distinct sets of args, for a cleaner separation of concerns. + + parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)) + if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): + # If we pass only one argument to the script and it's the path to a json file, + # let's parse it to get our arguments. + model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) + else: + model_args, data_args, training_args = parser.parse_args_into_dataclasses() + + if model_args.use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed in v4.34. Please use `token` instead.", + FutureWarning, + ) + if model_args.token is not None: + raise ValueError("`token` and `use_auth_token` are both specified. Please set only the argument `token`.") + model_args.token = model_args.use_auth_token + + # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The + # information sent is the one passed as arguments along with your Python/PyTorch versions. + send_example_telemetry("run_classification", model_args, data_args) + + # Setup logging + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + handlers=[logging.StreamHandler(sys.stdout)], + ) + + if training_args.should_log: + # The default of training_args.log_level is passive, so we set log level at info here to have that default. + transformers.utils.logging.set_verbosity_info() + + log_level = training_args.get_process_log_level() + logger.setLevel(log_level) + datasets.utils.logging.set_verbosity(log_level) + transformers.utils.logging.set_verbosity(log_level) + transformers.utils.logging.enable_default_handler() + transformers.utils.logging.enable_explicit_format() + + # Log on each process the small summary: + logger.warning( + f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, " + + f"distributed training: {training_args.parallel_mode.value == 'distributed'}, 16-bits training: {training_args.fp16}" + ) + logger.info(f"Training/evaluation parameters {training_args}") + + # Detecting last checkpoint. + last_checkpoint = None + if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir: + last_checkpoint = get_last_checkpoint(training_args.output_dir) + if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0: + raise ValueError( + f"Output directory ({training_args.output_dir}) already exists and is not empty. " + "Use --overwrite_output_dir to overcome." + ) + elif last_checkpoint is not None and training_args.resume_from_checkpoint is None: + logger.info( + f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change " + "the `--output_dir` or add `--overwrite_output_dir` to train from scratch." + ) + + # Set seed before initializing model. + set_seed(training_args.seed) + + # Get the datasets: you can either provide your own CSV/JSON training and evaluation files, or specify a dataset name + # to load from huggingface/datasets. In ether case, you can specify a the key of the column(s) containing the text and + # the key of the column containing the label. If multiple columns are specified for the text, they will be joined together + # for the actual text value. + # In distributed training, the load_dataset function guarantee that only one local process can concurrently + # download the dataset. + if data_args.dataset_name is not None: + # Downloading and loading a dataset from the hub. + raw_datasets = load_dataset( + data_args.dataset_name, + data_args.dataset_config_name, + cache_dir=model_args.cache_dir, + token=model_args.token, + ) + # Try print some info about the dataset + logger.info(f"Dataset loaded: {raw_datasets}") + logger.info(raw_datasets) + else: + # Loading a dataset from your local files. + # CSV/JSON training and evaluation files are needed. + data_files = {"train": data_args.train_file, "validation": data_args.validation_file} + + # Get the test dataset: you can provide your own CSV/JSON test file + if training_args.do_predict: + if data_args.test_file is not None: + train_extension = data_args.train_file.split(".")[-1] + test_extension = data_args.test_file.split(".")[-1] + assert ( + test_extension == train_extension + ), "`test_file` should have the same extension (csv or json) as `train_file`." + data_files["test"] = data_args.test_file + else: + raise ValueError("Need either a dataset name or a test file for `do_predict`.") + + for key in data_files.keys(): + logger.info(f"load a local file for {key}: {data_files[key]}") + + if data_args.train_file.endswith(".csv"): + # Loading a dataset from local csv files + raw_datasets = load_dataset( + "csv", + data_files=data_files, + cache_dir=model_args.cache_dir, + token=model_args.token, + ) + else: + # Loading a dataset from local json files + raw_datasets = load_dataset( + "json", + data_files=data_files, + cache_dir=model_args.cache_dir, + token=model_args.token, + ) + + # See more about loading any type of standard or custom dataset at + # https://huggingface.co/docs/datasets/loading_datasets. + + if data_args.remove_splits is not None: + for split in data_args.remove_splits.split(","): + logger.info(f"removing split {split}") + raw_datasets.pop(split) + + if data_args.train_split_name is not None: + logger.info(f"using {data_args.train_split_name} as train set") + raw_datasets["train"] = raw_datasets[data_args.train_split_name] + raw_datasets.pop(data_args.train_split_name) + + if data_args.validation_split_name is not None: + logger.info(f"using {data_args.validation_split_name} as validation set") + raw_datasets["validation"] = raw_datasets[data_args.validation_split_name] + raw_datasets.pop(data_args.validation_split_name) + + if data_args.test_split_name is not None: + logger.info(f"using {data_args.test_split_name} as test set") + raw_datasets["test"] = raw_datasets[data_args.test_split_name] + raw_datasets.pop(data_args.test_split_name) + + if data_args.remove_columns is not None: + for split in raw_datasets.keys(): + for column in data_args.remove_columns.split(","): + logger.info(f"removing column {column} from split {split}") + raw_datasets[split].remove_columns(column) + + if data_args.label_column_name is not None and data_args.label_column_name != "label": + for key in raw_datasets.keys(): + raw_datasets[key] = raw_datasets[key].rename_column(data_args.label_column_name, "label") + + # Trying to have good defaults here, don't hesitate to tweak to your needs. + + is_regression = ( + raw_datasets["train"].features["label"].dtype in ["float32", "float64"] + if data_args.do_regression is None + else data_args.do_regression + ) + + is_multi_label = False + if is_regression: + label_list = None + num_labels = 1 + # regession requires float as label type, let's cast it if needed + for split in raw_datasets.keys(): + if raw_datasets[split].features["label"].dtype not in ["float32", "float64"]: + logger.warning( + f"Label type for {split} set to float32, was {raw_datasets[split].features['label'].dtype}" + ) + features = raw_datasets[split].features + features.update({"label": Value("float32")}) + try: + raw_datasets[split] = raw_datasets[split].cast(features) + except TypeError as error: + logger.error( + f"Unable to cast {split} set to float32, please check the labels are correct, or maybe try with --do_regression=False" + ) + raise error + + else: # classification + if raw_datasets["train"].features["label"].dtype == "list": # multi-label classification + is_multi_label = True + logger.info("Label type is list, doing multi-label classification") + # Trying to find the number of labels in a multi-label classification task + # We have to deal with common cases that labels appear in the training set but not in the validation/test set. + # So we build the label list from the union of labels in train/val/test. + label_list = get_label_list(raw_datasets, split="train") + for split in ["validation", "test"]: + if split in raw_datasets: + val_or_test_labels = get_label_list(raw_datasets, split=split) + diff = set(val_or_test_labels).difference(set(label_list)) + if len(diff) > 0: + # add the labels that appear in val/test but not in train, throw a warning + logger.warning( + f"Labels {diff} in {split} set but not in training set, adding them to the label list" + ) + label_list += list(diff) + # if label is -1, we throw a warning and remove it from the label list + for label in label_list: + if label == -1: + logger.warning("Label -1 found in label list, removing it.") + label_list.remove(label) + + label_list.sort() + num_labels = len(label_list) + if num_labels <= 1: + raise ValueError("You need more than one label to do classification.") + + # Load pretrained model and tokenizer + # In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently + # download model & vocab. + config = AutoConfig.from_pretrained( + model_args.config_name if model_args.config_name else model_args.model_name_or_path, + num_labels=num_labels, + finetuning_task="text-classification", + cache_dir=model_args.cache_dir, + revision=model_args.model_revision, + token=model_args.token, + trust_remote_code=model_args.trust_remote_code, + ) + + if is_regression: + config.problem_type = "regression" + logger.info("setting problem type to regression") + elif is_multi_label: + config.problem_type = "multi_label_classification" + logger.info("setting problem type to multi label classification") + else: + config.problem_type = "single_label_classification" + logger.info("setting problem type to single label classification") + + tokenizer = AutoTokenizer.from_pretrained( + model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path, + cache_dir=model_args.cache_dir, + use_fast=model_args.use_fast_tokenizer, + revision=model_args.model_revision, + token=model_args.token, + trust_remote_code=model_args.trust_remote_code, + ) + model = AutoModelForSequenceClassification.from_pretrained( + model_args.model_name_or_path, + from_tf=bool(".ckpt" in model_args.model_name_or_path), + config=config, + cache_dir=model_args.cache_dir, + revision=model_args.model_revision, + token=model_args.token, + trust_remote_code=model_args.trust_remote_code, + ignore_mismatched_sizes=model_args.ignore_mismatched_sizes, + ) + + # Padding strategy + if data_args.pad_to_max_length: + padding = "max_length" + else: + # We will pad later, dynamically at batch creation, to the max sequence length in each batch + padding = False + + # for training ,we will update the config with label infos, + # if do_train is not set, we will use the label infos in the config + if training_args.do_train and not is_regression: # classification, training + label_to_id = {v: i for i, v in enumerate(label_list)} + # update config with label infos + if model.config.label2id != label_to_id: + logger.warning( + "The label2id key in the model config.json is not equal to the label2id key of this " + "run. You can ignore this if you are doing finetuning." + ) + model.config.label2id = label_to_id + model.config.id2label = {id: label for label, id in label_to_id.items()} + elif not is_regression: # classification, but not training + logger.info("using label infos in the model config") + logger.info("label2id: {}".format(model.config.label2id)) + label_to_id = model.config.label2id + else: # regression + label_to_id = None + + if data_args.max_seq_length > tokenizer.model_max_length: + logger.warning( + f"The max_seq_length passed ({data_args.max_seq_length}) is larger than the maximum length for the " + f"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}." + ) + max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length) + + def multi_labels_to_ids(labels: List[str]) -> List[float]: + ids = [0.0] * len(label_to_id) # BCELoss requires float as target type + for label in labels: + ids[label_to_id[label]] = 1.0 + return ids + + def preprocess_function(examples): + if data_args.text_column_names is not None: + text_column_names = data_args.text_column_names.split(",") + # join together text columns into "sentence" column + examples["sentence"] = examples[text_column_names[0]] + for column in text_column_names[1:]: + for i in range(len(examples[column])): + examples["sentence"][i] += data_args.text_column_delimiter + examples[column][i] + # Tokenize the texts + result = tokenizer(examples["sentence"], padding=padding, max_length=max_seq_length, truncation=True) + if label_to_id is not None and "label" in examples: + if is_multi_label: + result["label"] = [multi_labels_to_ids(l) for l in examples["label"]] + else: + result["label"] = [(label_to_id[str(l)] if l != -1 else -1) for l in examples["label"]] + return result + + # Running the preprocessing pipeline on all the datasets + with training_args.main_process_first(desc="dataset map pre-processing"): + raw_datasets = raw_datasets.map( + preprocess_function, + batched=True, + load_from_cache_file=not data_args.overwrite_cache, + desc="Running tokenizer on dataset", + ) + + if training_args.do_train: + if "train" not in raw_datasets: + raise ValueError("--do_train requires a train dataset.") + train_dataset = raw_datasets["train"] + if data_args.shuffle_train_dataset: + logger.info("Shuffling the training dataset") + train_dataset = train_dataset.shuffle(seed=data_args.shuffle_seed) + if data_args.max_train_samples is not None: + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) + + if training_args.do_eval: + if "validation" not in raw_datasets and "validation_matched" not in raw_datasets: + if "test" not in raw_datasets and "test_matched" not in raw_datasets: + raise ValueError("--do_eval requires a validation or test dataset if validation is not defined.") + else: + logger.warning("Validation dataset not found. Falling back to test dataset for validation.") + eval_dataset = raw_datasets["test"] + else: + eval_dataset = raw_datasets["validation"] + + if data_args.max_eval_samples is not None: + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) + + if training_args.do_predict or data_args.test_file is not None: + if "test" not in raw_datasets: + raise ValueError("--do_predict requires a test dataset") + predict_dataset = raw_datasets["test"] + # remove label column if it exists + if data_args.max_predict_samples is not None: + max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples) + predict_dataset = predict_dataset.select(range(max_predict_samples)) + + # Log a few random samples from the training set: + if training_args.do_train: + for index in random.sample(range(len(train_dataset)), 3): + logger.info(f"Sample {index} of the training set: {train_dataset[index]}.") + + if data_args.metric_name is not None: + metric = ( + evaluate.load(data_args.metric_name, config_name="multilabel", cache_dir=model_args.cache_dir) + if is_multi_label + else evaluate.load(data_args.metric_name, cache_dir=model_args.cache_dir) + ) + logger.info(f"Using metric {data_args.metric_name} for evaluation.") + else: + if is_regression: + metric = evaluate.load("mse", cache_dir=model_args.cache_dir) + logger.info("Using mean squared error (mse) as regression score, you can use --metric_name to overwrite.") + else: + if is_multi_label: + metric = evaluate.load("f1", config_name="multilabel", cache_dir=model_args.cache_dir) + logger.info( + "Using multilabel F1 for multi-label classification task, you can use --metric_name to overwrite." + ) + else: + metric = evaluate.load("accuracy", cache_dir=model_args.cache_dir) + logger.info("Using accuracy as classification score, you can use --metric_name to overwrite.") + + def compute_metrics(p: EvalPrediction): + preds = p.predictions[0] if isinstance(p.predictions, tuple) else p.predictions + if is_regression: + preds = np.squeeze(preds) + result = metric.compute(predictions=preds, references=p.label_ids) + elif is_multi_label: + preds = np.array([np.where(p > 0, 1, 0) for p in preds]) # convert logits to multi-hot encoding + # Micro F1 is commonly used in multi-label classification + result = metric.compute(predictions=preds, references=p.label_ids, average="micro") + else: + preds = np.argmax(preds, axis=1) + result = metric.compute(predictions=preds, references=p.label_ids) + if len(result) > 1: + result["combined_score"] = np.mean(list(result.values())).item() + return result + + # Data collator will default to DataCollatorWithPadding when the tokenizer is passed to Trainer, so we change it if + # we already did the padding. + if data_args.pad_to_max_length: + data_collator = default_data_collator + elif training_args.fp16: + data_collator = DataCollatorWithPadding(tokenizer, pad_to_multiple_of=8) + else: + data_collator = None + + # Initialize our Trainer + trainer = Trainer( + model=model, + args=training_args, + train_dataset=train_dataset if training_args.do_train else None, + eval_dataset=eval_dataset if training_args.do_eval else None, + compute_metrics=compute_metrics, + tokenizer=tokenizer, + data_collator=data_collator, + ) + + # Training + if training_args.do_train: + checkpoint = None + if training_args.resume_from_checkpoint is not None: + checkpoint = training_args.resume_from_checkpoint + elif last_checkpoint is not None: + checkpoint = last_checkpoint + train_result = trainer.train(resume_from_checkpoint=checkpoint) + metrics = train_result.metrics + max_train_samples = ( + data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset) + ) + metrics["train_samples"] = min(max_train_samples, len(train_dataset)) + trainer.save_model() # Saves the tokenizer too for easy upload + trainer.log_metrics("train", metrics) + trainer.save_metrics("train", metrics) + trainer.save_state() + + # Evaluation + if training_args.do_eval: + logger.info("*** Evaluate ***") + metrics = trainer.evaluate(eval_dataset=eval_dataset) + max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset) + metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset)) + trainer.log_metrics("eval", metrics) + trainer.save_metrics("eval", metrics) + + if training_args.do_predict: + logger.info("*** Predict ***") + # Removing the `label` columns if exists because it might contains -1 and Trainer won't like that. + if "label" in predict_dataset.features: + predict_dataset = predict_dataset.remove_columns("label") + predictions = trainer.predict(predict_dataset, metric_key_prefix="predict").predictions + if is_regression: + predictions = np.squeeze(predictions) + elif is_multi_label: + # Convert logits to multi-hot encoding. We compare the logits to 0 instead of 0.5, because the sigmoid is not applied. + # You can also pass `preprocess_logits_for_metrics=lambda logits, labels: nn.functional.sigmoid(logits)` to the Trainer + # and set p > 0.5 below (less efficient in this case) + predictions = np.array([np.where(p > 0, 1, 0) for p in predictions]) + else: + predictions = np.argmax(predictions, axis=1) + output_predict_file = os.path.join(training_args.output_dir, "predict_results.txt") + if trainer.is_world_process_zero(): + with open(output_predict_file, "w") as writer: + logger.info("***** Predict results *****") + writer.write("index\tprediction\n") + for index, item in enumerate(predictions): + if is_regression: + writer.write(f"{index}\t{item:3.3f}\n") + elif is_multi_label: + # recover from multi-hot encoding + item = [label_list[i] for i in range(len(item)) if item[i] == 1] + writer.write(f"{index}\t{item}\n") + else: + item = label_list[item] + writer.write(f"{index}\t{item}\n") + logger.info("Predict results saved at {}".format(output_predict_file)) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/TextClassification/TextClassification/train.sh b/TextClassification/TextClassification/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..751f1578f03ba2ec57971acb1e5297fb78d653b8 --- /dev/null +++ b/TextClassification/TextClassification/train.sh @@ -0,0 +1,18 @@ +python run_classification.py \ + --model_name_or_path bert-base-uncased \ + --train_file data/train.csv \ + --validation_file data/val.csv \ + --test_file data/test.csv \ + --shuffle_train_dataset \ + --metric_name evaluate/metrics/accuracy/accuracy.py \ + --text_column_name "text" \ + --text_column_delimiter "\n" \ + --label_column_name "target" \ + --do_train \ + --do_eval \ + --do_predict \ + --max_seq_length 512 \ + --per_device_train_batch_size 32 \ + --learning_rate 2e-5 \ + --num_train_epochs 1 \ + --output_dir experiments/ \ No newline at end of file diff --git a/TextClassification/__MACOSX/TextClassification/._.DS_Store b/TextClassification/__MACOSX/TextClassification/._.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..a5b28df1cbc6e15bd0d35cdadd0c2e65d5131c7d Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/._.DS_Store differ diff --git a/TextClassification/__MACOSX/TextClassification/._app.py b/TextClassification/__MACOSX/TextClassification/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/._bert-base-uncased b/TextClassification/__MACOSX/TextClassification/._bert-base-uncased new file mode 100644 index 0000000000000000000000000000000000000000..23c7791615fe4fe49b9c2dc2c2065546d01e4a73 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/._bert-base-uncased differ diff --git a/TextClassification/__MACOSX/TextClassification/._data b/TextClassification/__MACOSX/TextClassification/._data new file mode 100644 index 0000000000000000000000000000000000000000..23c7791615fe4fe49b9c2dc2c2065546d01e4a73 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/._data differ diff --git a/TextClassification/__MACOSX/TextClassification/._evaluate b/TextClassification/__MACOSX/TextClassification/._evaluate new file mode 100644 index 0000000000000000000000000000000000000000..23c7791615fe4fe49b9c2dc2c2065546d01e4a73 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/._evaluate differ diff --git a/TextClassification/__MACOSX/TextClassification/._requirements.txt b/TextClassification/__MACOSX/TextClassification/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..23c7791615fe4fe49b9c2dc2c2065546d01e4a73 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/._run_classification.py b/TextClassification/__MACOSX/TextClassification/._run_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/._run_classification.py differ diff --git a/TextClassification/__MACOSX/TextClassification/._train.sh b/TextClassification/__MACOSX/TextClassification/._train.sh new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/._train.sh differ diff --git a/TextClassification/__MACOSX/TextClassification/bert-base-uncased/._config.json b/TextClassification/__MACOSX/TextClassification/bert-base-uncased/._config.json new file mode 100644 index 0000000000000000000000000000000000000000..ba442e3372245b3d958ad2a9812bb2bcc1291902 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/bert-base-uncased/._config.json differ diff --git a/TextClassification/__MACOSX/TextClassification/bert-base-uncased/._pytorch_model.bin b/TextClassification/__MACOSX/TextClassification/bert-base-uncased/._pytorch_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..4523a554435afa3b50a75b82d49d5d6b1084138c --- /dev/null +++ b/TextClassification/__MACOSX/TextClassification/bert-base-uncased/._pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:416ad996a66f55271e745c86fd2c3572f46f0d20a37098aa3ea138cee64b8992 +size 163 diff --git a/TextClassification/__MACOSX/TextClassification/bert-base-uncased/._tokenizer.json b/TextClassification/__MACOSX/TextClassification/bert-base-uncased/._tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/bert-base-uncased/._tokenizer.json differ diff --git a/TextClassification/__MACOSX/TextClassification/bert-base-uncased/._tokenizer_config.json b/TextClassification/__MACOSX/TextClassification/bert-base-uncased/._tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/bert-base-uncased/._tokenizer_config.json differ diff --git a/TextClassification/__MACOSX/TextClassification/bert-base-uncased/._vocab.txt b/TextClassification/__MACOSX/TextClassification/bert-base-uncased/._vocab.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/bert-base-uncased/._vocab.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/data/._test.csv b/TextClassification/__MACOSX/TextClassification/data/._test.csv new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/data/._test.csv differ diff --git a/TextClassification/__MACOSX/TextClassification/data/._train.csv b/TextClassification/__MACOSX/TextClassification/data/._train.csv new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/data/._train.csv differ diff --git a/TextClassification/__MACOSX/TextClassification/data/._val.csv b/TextClassification/__MACOSX/TextClassification/data/._val.csv new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/data/._val.csv differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/._.github b/TextClassification/__MACOSX/TextClassification/evaluate/._.github new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/._.github differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/._.gitignore b/TextClassification/__MACOSX/TextClassification/evaluate/._.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/._.gitignore differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/._AUTHORS b/TextClassification/__MACOSX/TextClassification/evaluate/._AUTHORS new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/._AUTHORS differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/._CODE_OF_CONDUCT.md b/TextClassification/__MACOSX/TextClassification/evaluate/._CODE_OF_CONDUCT.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/._CODE_OF_CONDUCT.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/._CONTRIBUTING.md b/TextClassification/__MACOSX/TextClassification/evaluate/._CONTRIBUTING.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/._CONTRIBUTING.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/._LICENSE b/TextClassification/__MACOSX/TextClassification/evaluate/._LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/._LICENSE differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/._Makefile b/TextClassification/__MACOSX/TextClassification/evaluate/._Makefile new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/._Makefile differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/._additional-tests-requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/._additional-tests-requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/._additional-tests-requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/._comparisons b/TextClassification/__MACOSX/TextClassification/evaluate/._comparisons new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/._comparisons differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/._docs b/TextClassification/__MACOSX/TextClassification/evaluate/._docs new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/._docs differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/._measurements b/TextClassification/__MACOSX/TextClassification/evaluate/._measurements new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/._measurements differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/._metrics b/TextClassification/__MACOSX/TextClassification/evaluate/._metrics new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/._metrics differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/._setup.cfg b/TextClassification/__MACOSX/TextClassification/evaluate/._setup.cfg new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/._setup.cfg differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/._setup.py b/TextClassification/__MACOSX/TextClassification/evaluate/._setup.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/._setup.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/._src b/TextClassification/__MACOSX/TextClassification/evaluate/._src new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/._src differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/._templates b/TextClassification/__MACOSX/TextClassification/evaluate/._templates new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/._templates differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/._tests b/TextClassification/__MACOSX/TextClassification/evaluate/._tests new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/._tests differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/.github/._hub b/TextClassification/__MACOSX/TextClassification/evaluate/.github/._hub new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/.github/._hub differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/.github/._workflows b/TextClassification/__MACOSX/TextClassification/evaluate/.github/._workflows new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/.github/._workflows differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/.github/hub/._push_evaluations_to_hub.py b/TextClassification/__MACOSX/TextClassification/evaluate/.github/hub/._push_evaluations_to_hub.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/.github/hub/._push_evaluations_to_hub.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/.github/hub/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/.github/hub/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/.github/hub/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/.github/workflows/._build_documentation.yml b/TextClassification/__MACOSX/TextClassification/evaluate/.github/workflows/._build_documentation.yml new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/.github/workflows/._build_documentation.yml differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/.github/workflows/._build_pr_documentation.yml b/TextClassification/__MACOSX/TextClassification/evaluate/.github/workflows/._build_pr_documentation.yml new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/.github/workflows/._build_pr_documentation.yml differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/.github/workflows/._ci.yml b/TextClassification/__MACOSX/TextClassification/evaluate/.github/workflows/._ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/.github/workflows/._ci.yml differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/.github/workflows/._delete_doc_comment.yml b/TextClassification/__MACOSX/TextClassification/evaluate/.github/workflows/._delete_doc_comment.yml new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/.github/workflows/._delete_doc_comment.yml differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/.github/workflows/._python-release.yml b/TextClassification/__MACOSX/TextClassification/evaluate/.github/workflows/._python-release.yml new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/.github/workflows/._python-release.yml differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/.github/workflows/._update_spaces.yml b/TextClassification/__MACOSX/TextClassification/evaluate/.github/workflows/._update_spaces.yml new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/.github/workflows/._update_spaces.yml differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/._exact_match b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/._exact_match new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/._exact_match differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/._mcnemar b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/._mcnemar new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/._mcnemar differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/._wilcoxon b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/._wilcoxon new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/._wilcoxon differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/exact_match/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/exact_match/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/exact_match/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/exact_match/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/exact_match/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/exact_match/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/exact_match/._exact_match.py b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/exact_match/._exact_match.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/exact_match/._exact_match.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/exact_match/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/exact_match/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/exact_match/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/mcnemar/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/mcnemar/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/mcnemar/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/mcnemar/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/mcnemar/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/mcnemar/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/mcnemar/._mcnemar.py b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/mcnemar/._mcnemar.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/mcnemar/._mcnemar.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/mcnemar/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/mcnemar/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/mcnemar/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/wilcoxon/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/wilcoxon/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/wilcoxon/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/wilcoxon/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/wilcoxon/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/wilcoxon/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/wilcoxon/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/wilcoxon/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/wilcoxon/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/wilcoxon/._wilcoxon.py b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/wilcoxon/._wilcoxon.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/comparisons/wilcoxon/._wilcoxon.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/docs/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/._source b/TextClassification/__MACOSX/TextClassification/evaluate/docs/._source new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/._source differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/.__toctree.yml b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/.__toctree.yml new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/.__toctree.yml differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._a_quick_tour.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._a_quick_tour.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._a_quick_tour.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._base_evaluator.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._base_evaluator.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._base_evaluator.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._choosing_a_metric.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._choosing_a_metric.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._choosing_a_metric.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._considerations.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._considerations.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._considerations.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._creating_and_sharing.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._creating_and_sharing.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._creating_and_sharing.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._custom_evaluator.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._custom_evaluator.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._custom_evaluator.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._evaluation_suite.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._evaluation_suite.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._evaluation_suite.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._index.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._index.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._index.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._installation.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._installation.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._installation.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._keras_integrations.md b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._keras_integrations.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._keras_integrations.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._package_reference b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._package_reference new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._package_reference differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._sklearn_integrations.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._sklearn_integrations.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._sklearn_integrations.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._transformers_integrations.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._transformers_integrations.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._transformers_integrations.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._types_of_evaluations.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._types_of_evaluations.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/._types_of_evaluations.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._evaluator_classes.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._evaluator_classes.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._evaluator_classes.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._hub_methods.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._hub_methods.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._hub_methods.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._loading_methods.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._loading_methods.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._loading_methods.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._logging_methods.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._logging_methods.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._logging_methods.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._main_classes.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._main_classes.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._main_classes.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._saving_methods.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._saving_methods.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._saving_methods.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._visualization_methods.mdx b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._visualization_methods.mdx new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/docs/source/package_reference/._visualization_methods.mdx differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._honest b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._honest new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._honest differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._label_distribution b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._label_distribution new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._label_distribution differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._perplexity b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._perplexity new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._perplexity differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._regard b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._regard new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._regard differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._text_duplicates b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._text_duplicates new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._text_duplicates differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._toxicity b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._toxicity new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._toxicity differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._word_count b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._word_count new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._word_count differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._word_length b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._word_length new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/._word_length differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/honest/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/honest/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/honest/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/honest/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/honest/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/honest/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/honest/._honest.py b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/honest/._honest.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/honest/._honest.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/honest/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/honest/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/honest/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/label_distribution/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/label_distribution/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/label_distribution/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/label_distribution/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/label_distribution/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/label_distribution/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/label_distribution/._label_distribution.py b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/label_distribution/._label_distribution.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/label_distribution/._label_distribution.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/label_distribution/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/label_distribution/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/label_distribution/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/perplexity/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/perplexity/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/perplexity/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/perplexity/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/perplexity/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/perplexity/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/perplexity/._perplexity.py b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/perplexity/._perplexity.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/perplexity/._perplexity.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/perplexity/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/perplexity/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/perplexity/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/regard/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/regard/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/regard/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/regard/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/regard/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/regard/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/regard/._regard.py b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/regard/._regard.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/regard/._regard.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/regard/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/regard/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/regard/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/text_duplicates/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/text_duplicates/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/text_duplicates/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/text_duplicates/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/text_duplicates/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/text_duplicates/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/text_duplicates/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/text_duplicates/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/text_duplicates/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/text_duplicates/._text_duplicates.py b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/text_duplicates/._text_duplicates.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/text_duplicates/._text_duplicates.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/toxicity/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/toxicity/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/toxicity/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/toxicity/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/toxicity/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/toxicity/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/toxicity/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/toxicity/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/toxicity/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/toxicity/._toxicity.py b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/toxicity/._toxicity.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/toxicity/._toxicity.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_count/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_count/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_count/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_count/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_count/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_count/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_count/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_count/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_count/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_count/._word_count.py b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_count/._word_count.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_count/._word_count.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_length/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_length/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_length/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_length/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_length/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_length/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_length/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_length/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_length/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_length/._word_length.py b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_length/._word_length.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/measurements/word_length/._word_length.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._accuracy b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._accuracy new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._accuracy differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._bertscore b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._bertscore new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._bertscore differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._bleu b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._bleu new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._bleu differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._bleurt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._bleurt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._bleurt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._brier_score b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._brier_score new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._brier_score differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._cer b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._cer new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._cer differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._character b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._character new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._character differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._charcut_mt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._charcut_mt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._charcut_mt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._chrf b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._chrf new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._chrf differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._code_eval b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._code_eval new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._code_eval differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._comet b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._comet new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._comet differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._competition_math b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._competition_math new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._competition_math differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._confusion_matrix b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._confusion_matrix new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._confusion_matrix differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._coval b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._coval new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._coval differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._cuad b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._cuad new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._cuad differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._exact_match b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._exact_match new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._exact_match differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._f1 b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._f1 new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._f1 differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._frugalscore b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._frugalscore new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._frugalscore differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._glue b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._glue new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._glue differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._google_bleu b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._google_bleu new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._google_bleu differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._indic_glue b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._indic_glue new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._indic_glue differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mae b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mae new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mae differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mahalanobis b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mahalanobis new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mahalanobis differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mape b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mape new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mape differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mase b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mase new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mase differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._matthews_correlation b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._matthews_correlation new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._matthews_correlation differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mauve b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mauve new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mauve differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mean_iou b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mean_iou new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mean_iou differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._meteor b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._meteor new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._meteor differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mse b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mse new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._mse differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._nist_mt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._nist_mt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._nist_mt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._pearsonr b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._pearsonr new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._pearsonr differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._perplexity b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._perplexity new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._perplexity differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._poseval b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._poseval new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._poseval differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._precision b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._precision new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._precision differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._r_squared b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._r_squared new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._r_squared differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._recall b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._recall new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._recall differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._rl_reliability b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._rl_reliability new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._rl_reliability differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._roc_auc b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._roc_auc new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._roc_auc differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._rouge b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._rouge new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._rouge differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._sacrebleu b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._sacrebleu new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._sacrebleu differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._sari b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._sari new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._sari differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._seqeval b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._seqeval new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._seqeval differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._smape b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._smape new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._smape differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._spearmanr b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._spearmanr new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._spearmanr differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._squad b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._squad new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._squad differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._squad_v2 b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._squad_v2 new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._squad_v2 differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._super_glue b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._super_glue new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._super_glue differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._ter b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._ter new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._ter differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._trec_eval b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._trec_eval new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._trec_eval differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._wer b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._wer new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._wer differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._wiki_split b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._wiki_split new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._wiki_split differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._xnli b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._xnli new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._xnli differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._xtreme_s b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._xtreme_s new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/._xtreme_s differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/accuracy/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/accuracy/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/accuracy/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/accuracy/._accuracy.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/accuracy/._accuracy.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/accuracy/._accuracy.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/accuracy/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/accuracy/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/accuracy/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/accuracy/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/accuracy/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/accuracy/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bertscore/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bertscore/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bertscore/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bertscore/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bertscore/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bertscore/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bertscore/._bertscore.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bertscore/._bertscore.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bertscore/._bertscore.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bertscore/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bertscore/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bertscore/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleu/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleu/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleu/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleu/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleu/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleu/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleu/._bleu.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleu/._bleu.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleu/._bleu.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleu/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleu/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleu/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleu/._tokenizer_13a.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleu/._tokenizer_13a.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleu/._tokenizer_13a.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleurt/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleurt/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleurt/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleurt/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleurt/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleurt/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleurt/._bleurt.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleurt/._bleurt.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleurt/._bleurt.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleurt/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleurt/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/bleurt/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/brier_score/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/brier_score/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/brier_score/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/brier_score/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/brier_score/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/brier_score/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/brier_score/._brier_score.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/brier_score/._brier_score.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/brier_score/._brier_score.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/brier_score/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/brier_score/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/brier_score/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cer/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cer/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cer/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cer/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cer/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cer/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cer/._cer.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cer/._cer.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cer/._cer.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cer/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cer/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cer/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cer/._test_cer.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cer/._test_cer.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cer/._test_cer.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/character/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/character/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/character/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/character/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/character/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/character/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/character/._character.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/character/._character.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/character/._character.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/character/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/character/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/character/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/charcut_mt/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/charcut_mt/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/charcut_mt/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/charcut_mt/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/charcut_mt/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/charcut_mt/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/charcut_mt/._charcut_mt.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/charcut_mt/._charcut_mt.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/charcut_mt/._charcut_mt.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/charcut_mt/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/charcut_mt/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/charcut_mt/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/chrf/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/chrf/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/chrf/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/chrf/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/chrf/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/chrf/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/chrf/._chrf.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/chrf/._chrf.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/chrf/._chrf.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/chrf/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/chrf/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/chrf/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/code_eval/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/code_eval/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/code_eval/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/code_eval/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/code_eval/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/code_eval/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/code_eval/._code_eval.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/code_eval/._code_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/code_eval/._code_eval.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/code_eval/._execute.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/code_eval/._execute.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/code_eval/._execute.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/code_eval/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/code_eval/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/code_eval/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/comet/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/comet/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/comet/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/comet/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/comet/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/comet/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/comet/._comet.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/comet/._comet.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/comet/._comet.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/comet/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/comet/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/comet/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/competition_math/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/competition_math/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/competition_math/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/competition_math/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/competition_math/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/competition_math/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/competition_math/._competition_math.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/competition_math/._competition_math.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/competition_math/._competition_math.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/competition_math/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/competition_math/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/competition_math/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/confusion_matrix/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/confusion_matrix/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/confusion_matrix/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/confusion_matrix/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/confusion_matrix/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/confusion_matrix/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/confusion_matrix/._confusion_matrix.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/confusion_matrix/._confusion_matrix.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/confusion_matrix/._confusion_matrix.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/confusion_matrix/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/confusion_matrix/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/confusion_matrix/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/coval/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/coval/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/coval/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/coval/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/coval/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/coval/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/coval/._coval.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/coval/._coval.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/coval/._coval.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/coval/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/coval/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/coval/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cuad/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cuad/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cuad/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cuad/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cuad/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cuad/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cuad/._compute_score.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cuad/._compute_score.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cuad/._compute_score.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cuad/._cuad.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cuad/._cuad.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cuad/._cuad.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cuad/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cuad/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/cuad/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/exact_match/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/exact_match/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/exact_match/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/exact_match/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/exact_match/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/exact_match/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/exact_match/._exact_match.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/exact_match/._exact_match.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/exact_match/._exact_match.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/exact_match/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/exact_match/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/exact_match/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/f1/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/f1/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/f1/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/f1/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/f1/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/f1/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/f1/._f1.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/f1/._f1.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/f1/._f1.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/f1/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/f1/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/f1/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/frugalscore/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/frugalscore/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/frugalscore/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/frugalscore/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/frugalscore/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/frugalscore/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/frugalscore/._frugalscore.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/frugalscore/._frugalscore.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/frugalscore/._frugalscore.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/frugalscore/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/frugalscore/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/frugalscore/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/glue/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/glue/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/glue/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/glue/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/glue/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/glue/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/glue/._glue.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/glue/._glue.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/glue/._glue.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/glue/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/glue/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/glue/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/google_bleu/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/google_bleu/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/google_bleu/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/google_bleu/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/google_bleu/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/google_bleu/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/google_bleu/._google_bleu.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/google_bleu/._google_bleu.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/google_bleu/._google_bleu.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/google_bleu/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/google_bleu/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/google_bleu/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/google_bleu/._tokenizer_13a.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/google_bleu/._tokenizer_13a.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/google_bleu/._tokenizer_13a.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/indic_glue/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/indic_glue/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/indic_glue/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/indic_glue/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/indic_glue/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/indic_glue/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/indic_glue/._indic_glue.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/indic_glue/._indic_glue.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/indic_glue/._indic_glue.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/indic_glue/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/indic_glue/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/indic_glue/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mae/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mae/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mae/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mae/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mae/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mae/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mae/._mae.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mae/._mae.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mae/._mae.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mae/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mae/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mae/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mahalanobis/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mahalanobis/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mahalanobis/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mahalanobis/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mahalanobis/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mahalanobis/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mahalanobis/._mahalanobis.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mahalanobis/._mahalanobis.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mahalanobis/._mahalanobis.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mahalanobis/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mahalanobis/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mahalanobis/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mape/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mape/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mape/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mape/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mape/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mape/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mape/._mape.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mape/._mape.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mape/._mape.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mape/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mape/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mape/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mase/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mase/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mase/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mase/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mase/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mase/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mase/._mase.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mase/._mase.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mase/._mase.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mase/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mase/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mase/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/matthews_correlation/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/matthews_correlation/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/matthews_correlation/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/matthews_correlation/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/matthews_correlation/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/matthews_correlation/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/matthews_correlation/._matthews_correlation.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/matthews_correlation/._matthews_correlation.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/matthews_correlation/._matthews_correlation.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/matthews_correlation/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/matthews_correlation/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/matthews_correlation/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mauve/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mauve/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mauve/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mauve/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mauve/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mauve/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mauve/._mauve.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mauve/._mauve.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mauve/._mauve.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mauve/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mauve/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mauve/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mean_iou/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mean_iou/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mean_iou/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mean_iou/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mean_iou/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mean_iou/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mean_iou/._mean_iou.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mean_iou/._mean_iou.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mean_iou/._mean_iou.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mean_iou/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mean_iou/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mean_iou/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/meteor/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/meteor/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/meteor/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/meteor/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/meteor/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/meteor/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/meteor/._meteor.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/meteor/._meteor.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/meteor/._meteor.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/meteor/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/meteor/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/meteor/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mse/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mse/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mse/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mse/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mse/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mse/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mse/._mse.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mse/._mse.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mse/._mse.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mse/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mse/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/mse/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/nist_mt/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/nist_mt/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/nist_mt/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/nist_mt/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/nist_mt/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/nist_mt/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/nist_mt/._nist_mt.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/nist_mt/._nist_mt.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/nist_mt/._nist_mt.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/nist_mt/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/nist_mt/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/nist_mt/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/nist_mt/._tests.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/nist_mt/._tests.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/nist_mt/._tests.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/pearsonr/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/pearsonr/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/pearsonr/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/pearsonr/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/pearsonr/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/pearsonr/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/pearsonr/._pearsonr.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/pearsonr/._pearsonr.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/pearsonr/._pearsonr.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/pearsonr/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/pearsonr/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/pearsonr/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/perplexity/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/perplexity/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/perplexity/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/perplexity/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/perplexity/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/perplexity/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/perplexity/._perplexity.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/perplexity/._perplexity.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/perplexity/._perplexity.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/perplexity/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/perplexity/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/perplexity/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/poseval/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/poseval/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/poseval/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/poseval/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/poseval/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/poseval/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/poseval/._poseval.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/poseval/._poseval.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/poseval/._poseval.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/poseval/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/poseval/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/poseval/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/precision/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/precision/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/precision/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/precision/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/precision/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/precision/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/precision/._precision.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/precision/._precision.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/precision/._precision.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/precision/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/precision/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/precision/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/r_squared/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/r_squared/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/r_squared/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/r_squared/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/r_squared/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/r_squared/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/r_squared/._r_squared.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/r_squared/._r_squared.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/r_squared/._r_squared.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/r_squared/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/r_squared/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/r_squared/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/recall/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/recall/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/recall/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/recall/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/recall/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/recall/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/recall/._recall.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/recall/._recall.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/recall/._recall.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/recall/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/recall/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/recall/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rl_reliability/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rl_reliability/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rl_reliability/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rl_reliability/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rl_reliability/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rl_reliability/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rl_reliability/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rl_reliability/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rl_reliability/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rl_reliability/._rl_reliability.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rl_reliability/._rl_reliability.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rl_reliability/._rl_reliability.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/roc_auc/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/roc_auc/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/roc_auc/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/roc_auc/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/roc_auc/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/roc_auc/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/roc_auc/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/roc_auc/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/roc_auc/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/roc_auc/._roc_auc.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/roc_auc/._roc_auc.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/roc_auc/._roc_auc.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rouge/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rouge/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rouge/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rouge/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rouge/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rouge/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rouge/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rouge/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rouge/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rouge/._rouge.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rouge/._rouge.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/rouge/._rouge.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sacrebleu/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sacrebleu/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sacrebleu/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sacrebleu/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sacrebleu/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sacrebleu/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sacrebleu/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sacrebleu/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sacrebleu/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sacrebleu/._sacrebleu.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sacrebleu/._sacrebleu.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sacrebleu/._sacrebleu.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sari/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sari/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sari/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sari/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sari/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sari/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sari/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sari/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sari/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sari/._sari.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sari/._sari.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/sari/._sari.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/seqeval/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/seqeval/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/seqeval/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/seqeval/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/seqeval/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/seqeval/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/seqeval/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/seqeval/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/seqeval/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/seqeval/._seqeval.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/seqeval/._seqeval.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/seqeval/._seqeval.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/smape/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/smape/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/smape/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/smape/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/smape/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/smape/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/smape/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/smape/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/smape/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/smape/._smape.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/smape/._smape.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/smape/._smape.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/spearmanr/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/spearmanr/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/spearmanr/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/spearmanr/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/spearmanr/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/spearmanr/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/spearmanr/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/spearmanr/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/spearmanr/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/spearmanr/._spearmanr.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/spearmanr/._spearmanr.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/spearmanr/._spearmanr.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/squad/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/squad/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/squad/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/squad/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/squad/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/squad/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/squad/._compute_score.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/squad/._compute_score.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/squad/._compute_score.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/squad/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/squad/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/squad/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/metrics/squad/._squad.py b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/squad/._squad.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/metrics/squad/._squad.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/._evaluate b/TextClassification/__MACOSX/TextClassification/evaluate/src/._evaluate new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/._evaluate differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/.___init__.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/.___init__.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/.___init__.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._commands b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._commands new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._commands differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._config.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._config.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._config.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._evaluation_suite b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._evaluation_suite new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._evaluation_suite differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._evaluator b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._evaluator new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._evaluator differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._hub.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._hub.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._hub.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._info.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._info.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._info.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._inspect.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._inspect.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._inspect.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._loading.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._loading.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._loading.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._module.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._module.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._module.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._naming.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._naming.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._naming.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._saving.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._saving.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._saving.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._utils b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._utils new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._utils differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._visualization.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._visualization.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/._visualization.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/commands/.___init__.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/commands/.___init__.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/commands/.___init__.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/commands/._evaluate_cli.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/commands/._evaluate_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/commands/._evaluate_cli.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluation_suite/.___init__.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluation_suite/.___init__.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluation_suite/.___init__.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/.___init__.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/.___init__.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/.___init__.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._audio_classification.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._audio_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._audio_classification.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._automatic_speech_recognition.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._automatic_speech_recognition.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._automatic_speech_recognition.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._base.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._base.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._base.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._image_classification.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._image_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._image_classification.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._question_answering.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._question_answering.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._question_answering.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._text2text_generation.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._text2text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._text2text_generation.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._text_classification.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._text_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._text_classification.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._text_generation.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._text_generation.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._token_classification.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._token_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._token_classification.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._utils.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._utils.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/evaluator/._utils.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/utils/.___init__.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/utils/.___init__.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/utils/.___init__.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/utils/._file_utils.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/utils/._file_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/utils/._file_utils.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/utils/._gradio.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/utils/._gradio.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/utils/._gradio.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/utils/._logging.py b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/utils/._logging.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/src/evaluate/utils/._logging.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/templates/._cookiecutter.json b/TextClassification/__MACOSX/TextClassification/evaluate/templates/._cookiecutter.json new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/templates/._cookiecutter.json differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/templates/._{{ cookiecutter.module_slug }} b/TextClassification/__MACOSX/TextClassification/evaluate/templates/._{{ cookiecutter.module_slug }} new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/templates/._{{ cookiecutter.module_slug }} differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/._README.md b/TextClassification/__MACOSX/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/._README.md new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/._README.md differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/._app.py b/TextClassification/__MACOSX/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/._app.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/._app.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/._requirements.txt b/TextClassification/__MACOSX/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/._requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/._requirements.txt differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/._tests.py b/TextClassification/__MACOSX/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/._tests.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/._tests.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/._{{ cookiecutter.module_slug }}.py b/TextClassification/__MACOSX/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/._{{ cookiecutter.module_slug }}.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/templates/{{ cookiecutter.module_slug }}/._{{ cookiecutter.module_slug }}.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/tests/.___init__.py b/TextClassification/__MACOSX/TextClassification/evaluate/tests/.___init__.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/tests/.___init__.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/tests/._conftest.py b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._conftest.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_evaluation_suite.py b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_evaluation_suite.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_evaluation_suite.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_evaluator.py b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_evaluator.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_file_utils.py b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_file_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_file_utils.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_hub.py b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_hub.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_hub.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_load.py b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_load.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_load.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_metric.py b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_metric.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_metric.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_metric_common.py b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_metric_common.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_metric_common.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_save.py b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_save.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_save.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_trainer_evaluator_parity.py b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_trainer_evaluator_parity.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_trainer_evaluator_parity.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_viz.py b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_viz.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._test_viz.py differ diff --git a/TextClassification/__MACOSX/TextClassification/evaluate/tests/._utils.py b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._utils.py new file mode 100644 index 0000000000000000000000000000000000000000..df5ca2ab776fc297e04d6dff09972e401f325035 Binary files /dev/null and b/TextClassification/__MACOSX/TextClassification/evaluate/tests/._utils.py differ