<jupyter_start><jupyter_text>Install required libs<jupyter_code>!pip install rake-nltk
!pip install nltk
!pip install pysolr
!pip install langdetect<jupyter_output>Requirement already satisfied: rake-nltk in /usr/local/lib/python3.6/dist-packages (1.0.4)
Requirement already satisfied: nltk in /usr/local/lib/python3.6/dist-packages (from rake-nltk) (3.2.5)
Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nltk->rake-nltk) (1.12.0)<jupyter_text>Retrieving Covid Data Connecting to database<jupyter_code>import pysolr
solr = pysolr.Solr('http://librairy.linkeddata.es/solr/covid-paragraphs/', always_commit=True, timeout=1000)<jupyter_output><empty_output><jupyter_text>Function to retrieve all documents<jupyter_code>def getAllDocuments(solr, start, limit_rows):
    results = solr.search('*:*', **{
    'start': str(start),
    'rows': limit_rows,
    })
    return results

print(results)<jupyter_output><empty_output><jupyter_text>Getting documents<jupyter_code>#there are >2M paragraphs. for quick executions use <30k
results = getAllDocuments(solr, 0 , 100000)

print(len(results))<jupyter_output>100000<jupyter_text>Creating the corpus, checking for correct language<jupyter_code>from langdetect import detect

corpus = ""
field_text = ""
for result in results:
        ids = result["id"]
        try: 
          field_text = result["text_t"]
          if detect(field_text)== 'en':
            corpus=corpus+" \n"+field_text
        except:
          print("error in:"+ids)
          continue<jupyter_output>error in:8c9093d1327f8b02a6cc5ca937824549
error in:56609a38be47c54f111613616c35adf2
error in:c8a753e26b270310932dee1e20f8b0a5
error in:8f16df16f272764ee5355be54b321700
error in:4002574e9d1cf770a6597f2c81fcbcda
error in:e106492ff8bdab360162eb5b60c33a85
error in:cfefa4d93f6c15bc08925e6492e14318
error in:cadf283ee05e17315f565189ccc83e98
error in:a13da310fead60fa850113881e28d635
error in:48c1646fd260e00728fb6e847cc2b90e
error in:969c26ec72e820ef236f83605b8142a7
error in:5b38320bec3157661fd96b83fc432746
error in:8c4a042bbf88e1f389bbe5984c3378e1
error in:230fba214e0ea7d9ffdf275e7a0dba00
error in:29ab5b47e2900142718c5a1ddfa4de8d
error in:3df4ef29f791879512fb037ee3ba5c8d
error in:78775b581745b36256299396e6f7d011
error in:c480653d02001c4e7536054af998d777
error in:e1f094e1b69d8ae3ee08e11b2f12c9c2
error in:1ed892a55850f8052d36b2d9a2f5db82
error in:f3a3ae81cfc6857427fae94c433e9c95
error in:5c8c306733e4c2c8c3d72bc6487e0562
error in:c3f170198a7be20e10ba6f5965797e95
error in:dc8baf2a43127bd7468eb141e[...]<jupyter_text>Saving copus in file<jupyter_code>c = open("corpus100.txt", "w")
corpus20 = c.write(corpus)
c.close()<jupyter_output><empty_output><jupyter_text>Executing Rake<jupyter_code>from rake_nltk import Metric, Rake
import string
import nltk

#punctuation list from nltk
punctuations = string.punctuation

#stopword list from nltk
stopwords = nltk.corpus.stopwords.words('english')

#adding extra stopwords based on covid corpora
newStopWords = ['within', 'using', 'figure','table','box', ',|', '-|',')-', 'list',').', '′-','),','.,','would','also', 'another', 'together', 'http://', 'www.','et', '=-', '://', '”),','‐,', '**', '“', 'al','may','%)', ')=', ');', '(+)-', 'many', 'serveral', 'although', 'whether', 'single', 'similar', '].', ',”', '–', '×', '+', '−/−','±', '˘', '…', '.;', '||','-(', '|,', '…,', '…),', '|', '®-', ',"', '®', '-']
stopwords.extend(newStopWords)

#setting Rake with custom parameters
r = Rake(language='english',min_length=2, max_length=4, punctuations=punctuations, stopwords=stopwords, ranking_metric=Metric.WORD_FREQUENCY)

#from downloaded corpus
file = open("corpus100.txt")
corpus100=file.read()

#initializing Rake
r.extract_keywords_from_text(corpus100)<jupyter_output><empty_output><jupyter_text>Printing results<jupyter_code>results = r.get_ranked_phrases_with_scores()
for i in results[0:3000]:
  print(i)<jupyter_output>(32348.0, '2 0 1 1')
(31820.0, 'cells control viral infection')
(31502.0, '1 cells (~ 1')
(30738.0, 'cells following viral infection')
(29970.0, 'virus protein 1')
(29903.0, 'subsequent viral protein 1')
(29801.0, '1 upon virus infection')
(29470.0, 'length virus protein 2')
(29283.0, '1 1 2')
(28686.0, 'viral protein 1')
(28686.0, '1 viral protein')
(28679.0, 'influenza virus infected cells')
(28443.0, 'virus infected cells compared')
(28420.0, '1 virus infection')
(28372.0, '1 infection 1')
(28227.0, 'cell motility protein 1')
(28161.0, 'tim 1 cell protein')
(27881.0, 'human influenza virus infection')
(27704.0, '1 human mast cells')
(27669.0, 'putative viral protein 2')
(27640.0, '1 human infection due')
(27525.0, 'cells including endothelial cells')
(27495.0, '1 influenza virus c')
(27441.0, '2 0 1 5')
(27436.0, 'new human viral infection')
(27403.0, '1 host cell proteins')
(27288.0, 'human immunodeficiency virus 1')
(27288.0, '1 human immunodeficiency virus')
(27265.0, 'human para[...]<jupyter_text>Or saving results in file<jupyter_code>t = open("3kt_100kc_4f.txt", "w")

for i in results[0:3000]:
    t.write(str(i)+" \n")
t.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15214411591932642, 'fraction_numerical': 0.15096925788133933, 'mean_word_length': 6.739393939393939, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '409944', 'license': 'Apache-2.0', 'n_tokens_mistral': 2520, 'n_tokens_neox': 2075, 'n_words': 393, 'path': 'notebooks/main-concepts/Main_Concepts.ipynb', 'repo_name': 'oeg-upm/drugs4covid19-nlp', 'size': '5107'}