<commit_before>from portality import models
from portality.core import app
import esprit
import re

KEYWORDS = {
    "fields": ["id","bibjson.keywords"],
    "query": {
        "filtered": {
            "filter": {
                "exists": {"field": "index.fulltext"}
            },
            "query": {
                "match_all": {}
            }
        }
    },
    "sort": [{
        "in_doaj": "desc"
    }]
}

def rewrite_keywords_art(id):
    article = models.Article.pull(id)
    bib = article.bibjson()
    kwords = [k.lower() for k in bib.keywords]
    bib.set_keywords(kwords)
    article.save()

def rewrite_keywords_journal(id):
    journal = models.Article.pull(id)
    bib = journal.bibjson()
    kwords = [k.lower() for k in bib.keywords]
    bib.set_keywords(kwords)
    journal.save()

if __name__ == "__main__":

    conn = esprit.raw.make_connection(None, app.config["ELASTIC_SEARCH_HOST"], None, app.config["ELASTIC_SEARCH_DB"])

    for j in esprit.tasks.scroll(conn, models.Article.__type__, q=KEYWORDS, page_size=100, keepalive='5m'):
        if not j.has_key('bibjson.keywords'):
            continue
        keywords = j['bibjson.keywords']
        id = j['id'][0]
        print(id, keywords)

        if keywords is not None:
            for key in keywords:
                if bool(re.match(r".*[A-Z].*", key)):
                    rewrite_keywords_art(id)
                    break

    for j in esprit.tasks.scroll(conn, models.Journal.__type__, q=KEYWORDS, page_size=100, keepalive='5m'):
        keywords = j['keywords']
        id = j['id'][0]
        for key in keywords:
            if bool(re.match(r".*[A-Z].*", key)):
                rewrite_keywords_journal(id)
                break
<commit_msg>Update keywords_capital_letters.py<commit_after>from portality import models
from portality.core import app
import esprit
import re

KEYWORDS = {
    "fields": ["id","bibjson.keywords"],
    "query": {
        "filtered": {
            "filter": {
                "exists": {"field": "index.fulltext"}
            },
            "query": {
                "match_all": {}
            }
        }
    },
    "sort": [{
        "in_doaj": "desc"
    }]
}

def rewrite_keywords_art(id):
    article = models.Article.pull(id)
    bib = article.bibjson()
    kwords = [k.lower() for k in bib.keywords]
    bib.set_keywords(kwords)
    article.save()

def rewrite_keywords_journal(id):
    journal = models.Article.pull(id)
    bib = journal.bibjson()
    kwords = [k.lower() for k in bib.keywords]
    bib.set_keywords(kwords)
    journal.save()

if __name__ == "__main__":

    conn = esprit.raw.make_connection(None, app.config["ELASTIC_SEARCH_HOST"], None, app.config["ELASTIC_SEARCH_DB"])

    for j in esprit.tasks.scroll(conn, models.Article.__type__, q=KEYWORDS, page_size=100, keepalive='5m'):
        if not j.has_key('bibjson.keywords'):
            continue
        keywords = j['bibjson.keywords']
        id = j['id'][0]

        if keywords is not None:
            for key in keywords:
                if bool(re.match(r".*[A-Z].*", key)):
                    rewrite_keywords_art(id)
                    break

    for j in esprit.tasks.scroll(conn, models.Journal.__type__, q=KEYWORDS, page_size=100, keepalive='5m'):
        keywords = j['keywords']
        id = j['id'][0]
        for key in keywords:
            if bool(re.match(r".*[A-Z].*", key)):
                rewrite_keywords_journal(id)
                break

{'fraction_non_alphanumeric': 0.14224013820904116, 'fraction_numerical': 0.005758710048949035, 'mean_word_length': 2.1610555050045495, 'pattern_counts': {'":': 22, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6989388', 'n_tokens_mistral': 1192, 'n_tokens_neox': 1105, 'n_words': 229}