<jupyter_start><jupyter_text><jupyter_code>!pip install transformers
from transformers import AutoTokenizer, AutoModelForTokenClassification
from transformers import pipeline

tokenizer = AutoTokenizer.from_pretrained("dslim/bert-base-NER")
model = AutoModelForTokenClassification.from_pretrained("dslim/bert-base-NER")

nlp = pipeline("ner", model=model, tokenizer=tokenizer)
example = """"F.B.I. Agent , Who in Texts, Is Fired - The New York Times SectionsSEARCHSkip to contentSkip to site ’s,
 PaperPolitics|F.B.I. Agent , Who in Texts, Is Firedhttps://www.nytimes.com/2018/08/13/us/politics/peter-strzok-fired-fbi.htmlAdvertisementContinue 
 reading the main storySupported reading the main storyF.B.I. Agent , Who in Texts, Is FiredRead in , a top counterintelligence agent 
 who was taken off the special counsel investigation after his disparaging texts about President were uncovered, was fired., , for and ., , — , 
 the senior counterintelligence agent who disparaged President in inflammatory text messages and helped oversee the email and investigations, has
  been fired for violating bureau policies, Mr. ’s lawyer said ., Mr. and his allies seized on the texts — exchanged during the campaign with a 
  former lawyer, — in assailing the investigation as an illegitimate “witch hunt.”, Mr. , who rose over at the to become one of its most 
  experienced counterintelligence agents, was a key figure in of the inquiry., Along with writing the texts, Mr. was accused of sending a
   highly sensitive search warrant to his personal email account., The had been under immense political pressure by Mr. to dismiss Mr. , who was
    removed from the staff of the special counsel, ., The president has repeatedly denounced Mr. in posts on , and on expressed satisfaction that 
    he had been sacked., Mr. ’s victory traces back to , when Mr. ’s conduct was laid out in a wide-ranging inspector general’s report on how the 
    handled the investigation of ’s emails in the run-up to the election., The report was critical of Mr. ’s conduct in sending the texts, and the
     bureau’s said that Mr. should be suspended for and demoted., Mr. had testified before the in about how he had not allowed his political views 
     to interfere with the investigations he was overseeing., But Mr. ’s lawyer said the deputy director of the F.B.I., , had overruled and fired 
     Mr. ., A spokeswoman for the did not respond to a message seeking comment about why Mr. was dismissed rather than demoted., Firing Mr. , 
     however, removes a favorite target of Mr. from the ranks of the and gives Mr. and the director, , a chance to move beyond the president’s ire.,
      , Mr. ’s lawyer, denounced his client’s dismissal., “The decision to fire Special Agent is not only a departure from typical bureau practice,
       but also contradicts Director ’s testimony to and his assurances that the intended to follow its regular process in this and all personnel 
       matters,” Mr. said., “This decision should be deeply troubling to all ,” Mr. added., “A lengthy investigation and multiple rounds of 
       congressional testimony failed to produce a shred of evidence that Special Agent ’s personal views ever affected his work., ”Mr., ’s 
       text exchanges with Ms. demonstrated animosity toward Mr. ., In , Ms. asks: Trump is “not ever going to become president, right?, Right?!”, 
       Mr. responds: “No., No he won’t., We’ll stop it.”, The inspector general, who uncovered the messages, found no evidence that the pair
        imposed their political views on their investigative decisions but cited that exchange as “not only indicative of a biased state of 
        mind but, even more seriously, implies a willingness to take official action to impact the presidential candidate’s electoral prospects., 
        ”The report by the inspector general, , that preceded Mr. ’s firing not only criticized his conduct in sending the texts but also his use 
        of personal email accounts to handle sensitive information., In"""

ner_results = nlp(example)
print(ner_results)<jupyter_output>Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.12.3)
Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)
Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)
Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.2)
Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.3)
Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.3.2)
Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.1.2)
Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)
Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-p[...]
{'fraction_non_alphanumeric': 0.07654659357870008, 'fraction_numerical': 0.01664056382145654, 'mean_word_length': 4.129518072289157, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '659656', 'license': 'MIT', 'n_tokens_mistral': 1417, 'n_tokens_neox': 1324, 'n_words': 649, 'path': 'dslim_bert_base_NER_.ipynb', 'repo_name': 'anshika002/zee-media-internship', 'size': '5108'}