<commit_before>import couchdb, os, logging, sys
from pprint import pprint
import time

# run in heroku by a) commiting, b) pushing to heroku, and c) running
# heroku run python extras/couch_maint.py

logging.basicConfig(
    stream=sys.stdout,
    level=logging.DEBUG,
    format='%(levelname)8s %(name)s - %(message)s'
)

logger = logging.getLogger("couch_maint")
  
cloudant_db = os.getenv("CLOUDANT_DB")
cloudant_url = os.getenv("CLOUDANT_URL")

couch = couchdb.Server(url=cloudant_url)
db = couch[cloudant_db]
logger.info("connected to couch at " + cloudant_url + " / " + cloudant_db)

"""
admin/items_with_pmids

function(doc) {
  if (doc.type == "item") {
    if (typeof doc.aliases.pmid != "undefined") {<commit_msg>temporarily point staging couch_maint script to production db<commit_after>import couchdb, os, logging, sys
from pprint import pprint
import time

# run in heroku by a) commiting, b) pushing to heroku, and c) running
# heroku run python extras/couch_maint.py

logging.basicConfig(
    stream=sys.stdout,
    level=logging.DEBUG,
    format='%(levelname)8s %(name)s - %(message)s'
)

logger = logging.getLogger("couch_maint")
  
cloudant_db = "ti"
cloudant_url = "https://app5109761.heroku:TuLL8oXFh4k0iAcAPnDMlSjC@<EMAIL>.cloudant.com"

#cloudant_db = os.getenv("CLOUDANT_DB")
#cloudant_url = os.getenv("CLOUDANT_URL")

couch = couchdb.Server(url=cloudant_url)
db = couch[cloudant_db]
logger.info("connected to couch at " + cloudant_url + " / " + cloudant_db)

"""
admin/items_with_pmids

function(doc) {
  if (doc.type == "item") {
    if (typeof doc.aliases.pmid != "undefined") {
{'fraction_non_alphanumeric': 0.13333333333333333, 'fraction_numerical': 0.007476635514018692, 'mean_word_length': 5.037593984962406, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1739043', 'n_tokens_mistral': 609, 'n_tokens_neox': 597, 'n_words': 129}