<commit_before><commit_msg>Add script to expire existing DocOps lists<commit_after>Settings = require "settings-sharelatex"
rclient = require("redis-sharelatex").createClient(Settings.redis.documentupdater)
keys = Settings.redis.documentupdater.key_schema
async = require "async"
RedisManager = require "./app/js/RedisManager"

getKeysFromNode = (node, pattern, callback) ->
	cursor = 0  # redis iterator
	keySet = {} # use hash to avoid duplicate results
	# scan over all keys looking for pattern
	doIteration = (cb) ->
		node.scan cursor, "MATCH", pattern, "COUNT", 1000, (error, reply) ->
			return callback(error) if error?
			[cursor, keys] = reply
			console.log "SCAN", keys.length
			for key in keys
				keySet[key] = true
			if cursor == '0'  # note redis returns string result not numeric
				return callback(null, Object.keys(keySet))
			else
				doIteration()
	doIteration()

getKeys = (pattern, callback) ->
	nodes = rclient.nodes?('master') || [ rclient ];
	console.log "GOT NODES", nodes.length
	doKeyLookupForNode = (node, cb) ->
		getKeysFromNode node, pattern, cb
	async.concatSeries nodes, doKeyLookupForNode, callback

TTL = 60 * 60 # 1 hour
expireDocOps = (callback) ->
	getKeys keys.docOps(doc_id: "*"), (error, keys) ->
		async.mapSeries keys,
			(key, cb) ->
				console.log "EXPIRE #{key} #{RedisManager.DOC_OPS_TTL}"
				rclient.expire key, RedisManager.DOC_OPS_TTL, cb
			callback

setTimeout () -> #  Give redis a chance to connect
	expireDocOps (error) ->
		throw error if error?
		process.exit()
, 1000
{'fraction_non_alphanumeric': 0.12336814621409922, 'fraction_numerical': 0.0097911227154047, 'mean_word_length': 4.2320819112627985, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4221276', 'n_tokens_mistral': 555, 'n_tokens_neox': 499, 'n_words': 143}