<commit_before>"""
### Usage: ctmigrate [load|dump] [--domain=DOMAIN] [--port=PORT] [--filename=FILENAME]

### Options:
	-h, --help            show this help message and exit
	-d DOMAIN, --domain=DOMAIN
						  domain of target server (default: localhost)
	-p PORT, --port=PORT  port of target server (default: 8080)
	-f FILENAME, --filename=FILENAME
						  name of sqlite data file for dumping/loading to/from
						  (default: dump.db)
"""

import getpass
from optparse import OptionParser
from cantools import db
from cantools.web import fetch, post
from cantools.util import error, log

LIMIT = 500

def load(host, port, session):
	pw = getpass.getpass("admin password? ")
	log("loading database into %s:%s"%(host, port), important=True)
	for model in db.get_schema():
		log("retrieving %s entities"%(model,), important=True)
		mod = db.get_model(model)
		offset = 0
		while 1:
			chunk = db.get_page(model, LIMIT, offset, session=session)
			log(post(host, "/_db", port, {
				"pw": pw,
				"action": "put",
				"data": chunk
			}, ctjson=True))
			offset += LIMIT
			if len(chunk) < LIMIT:
				break
			log("processed %s %s records"%(offset, model), 1)
	log("finished loading data from sqlite dump file")

keys = {}
missing = {}
blobs = []
realz = set()

def fixmissing(orig):
	if orig in missing and orig in keys:
		for d in missing[orig]:
			for k, v in d.items():
				if type(v) is list:
					for i in range(len(v)):
						if v[i] == orig:
							v[i] = keys[orig]
				elif v == orig:
					d[k] = keys[orig]
		del missing[orig]

def _missing(k, d):
	if k not in missing:
		missing[k] = []
	missing[k].append(d)

def _fix_or_miss(d, prop):
	dk = d[prop]
	if type(dk) is list:
		for i in range(len(dk)):
			k = dk[i]
			if k in keys:
				dk[i] = keys[k]
			else:
				_missing(k, d)
	elif dk:
		if dk in keys:
			d[prop] = keys[dk]
		else:
			_missing(dk, d)

def fixkeys(d, schema):
	if "ctkey" in d:
		orig = d["key"]
		old = d["oldkey"]
		d["key"] = keys[orig] = keys[old] = d["ctkey"]
		realz.add(d["key"])
		fixmissing(orig)
		fixmissing(old)
		for prop in schema["_kinds"]:
			_fix_or_miss(d, prop)

def delmissing(orig):
	for d in missing[orig]:
		for k, v in d.items():
			if type(v) is list:
				d[k] = [val for val in v if val != orig]
			elif v == orig:
				d[k] = None
	del missing[orig]

def prune():
	if missing:
		mlen = len(missing.keys())
		log("pruning %s missing keys"%(mlen,), important=True)
		log("searching for matches")
		for oldkey in missing.keys():
			fixmissing(oldkey)
		newlen = len(missing.keys())
		log("matched %s - %s left over"%(mlen - newlen, newlen))
		log("deleting stragglers")
		for oldkey in missing.keys():
			delmissing(oldkey)

def checkblobs(d, schema):
	for key, prop in schema.items():
		if prop == "binary" and d[key]:
			blobs.append(d)
			break

def getblobs(host, port):
	log("retrieving binaries stored on %s records"%(len(blobs),), important=True)
	for d in blobs:
		for key, prop in db.get_schema(d["modelName"]).items():
			if prop == "binary" and d[key]:
				log("fetching %s.%s"%(d["key"], key))
				d[key] = fetch(host, port=port,
					path="/_db?action=blob&key=%s&property=%s"%(d["key"], key))

def dump(host, port, session):
	log("dumping database at %s:%s"%(host, port), important=True)
	mods = {}
	schemas = db.get_schema()
	for model in schemas:
		log("retrieving %s entities"%(model,), important=True)
		schema = schemas[model]
		mods[model] = []
		offset = 0
		while 1:
			chunk = fetch(host, port=port, ctjson=True,
				path="/_db?action=get&modelName=%s&offset=%s&limit=%s"%(model, offset, LIMIT))
			for c in chunk:
				fixkeys(c, schema)
				checkblobs(c, schema)
			mods[model] += chunk
			offset += LIMIT
			if len(chunk) < LIMIT:
				break
			log("got %s %s records"%(offset, model), 1)
		log("found %s %s records"%(len(mods[model]), model))
	log("%s unmatched keys!"%(len(missing.keys()),), important=True)
	prune()
	if blobs:
		getblobs(host, port)
	puts = []
	log("building models", important=True)
	for model in mods:
		mod = db.get_model(model)<commit_msg>migrate dump: grab gae blobs right before put()<commit_after>"""
### Usage: ctmigrate [load|dump] [--domain=DOMAIN] [--port=PORT] [--filename=FILENAME]

### Options:
	-h, --help            show this help message and exit
	-d DOMAIN, --domain=DOMAIN
						  domain of target server (default: localhost)
	-p PORT, --port=PORT  port of target server (default: 8080)
	-f FILENAME, --filename=FILENAME
						  name of sqlite data file for dumping/loading to/from
						  (default: dump.db)
"""

import getpass
from optparse import OptionParser
from cantools import db
from cantools.web import fetch, post
from cantools.util import error, log

LIMIT = 500

def load(host, port, session):
	pw = getpass.getpass("admin password? ")
	log("loading database into %s:%s"%(host, port), important=True)
	for model in db.get_schema():
		log("retrieving %s entities"%(model,), important=True)
		mod = db.get_model(model)
		offset = 0
		while 1:
			chunk = db.get_page(model, LIMIT, offset, session=session)
			log(post(host, "/_db", port, {
				"pw": pw,
				"action": "put",
				"data": chunk
			}, ctjson=True))
			offset += LIMIT
			if len(chunk) < LIMIT:
				break
			log("processed %s %s records"%(offset, model), 1)
	log("finished loading data from sqlite dump file")

keys = {}
missing = {}
blobs = []
realz = set()

def fixmissing(orig):
	if orig in missing and orig in keys:
		for d in missing[orig]:
			for k, v in d.items():
				if type(v) is list:
					for i in range(len(v)):
						if v[i] == orig:
							v[i] = keys[orig]
				elif v == orig:
					d[k] = keys[orig]
		del missing[orig]

def _missing(k, d):
	if k not in missing:
		missing[k] = []
	missing[k].append(d)

def _fix_or_miss(d, prop):
	dk = d[prop]
	if type(dk) is list:
		for i in range(len(dk)):
			k = dk[i]
			if k in keys:
				dk[i] = keys[k]
			else:
				_missing(k, d)
	elif dk:
		if dk in keys:
			d[prop] = keys[dk]
		else:
			_missing(dk, d)

def fixkeys(d, schema):
	if "ctkey" in d:
		orig = d["gaekey"] = d["key"]
		old = d["oldkey"]
		d["key"] = keys[orig] = keys[old] = d["ctkey"]
		realz.add(d["key"])
		fixmissing(orig)
		fixmissing(old)
		for prop in schema["_kinds"]:
			_fix_or_miss(d, prop)

def delmissing(orig):
	for d in missing[orig]:
		for k, v in d.items():
			if type(v) is list:
				d[k] = [val for val in v if val != orig]
			elif v == orig:
				d[k] = None
	del missing[orig]

def prune():
	if missing:
		mlen = len(missing.keys())
		log("pruning %s missing keys"%(mlen,), important=True)
		log("searching for matches")
		for oldkey in missing.keys():
			fixmissing(oldkey)
		newlen = len(missing.keys())
		log("matched %s - %s left over"%(mlen - newlen, newlen))
		log("deleting stragglers")
		for oldkey in missing.keys():
			delmissing(oldkey)

def checkblobs(d, schema):
	for key, prop in schema.items():
		if prop == "blob" and d[key]:
			blobs.append(d)
			break

def getblobs(host, port):
	log("retrieving binaries stored on %s records"%(len(blobs),), important=True)
	for d in blobs:
		for key, prop in db.get_schema(d["modelName"]).items():
			if prop == "blob" and d[key]:
				entkey = d.get("gaekey", d["key"])
				log("fetching %s.%s"%(entkey, key))
				d[key] = fetch(host, port=port,
					path="/_db?action=blob&key=%s&property=%s"%(entkey, key))

def dump(host, port, session):
	log("dumping database at %s:%s"%(host, port), important=True)
	mods = {}
	schemas = db.get_schema()
	for model in schemas:
		log("retrieving %s entities"%(model,), important=True)
		schema = schemas[model]
		mods[model] = []
		offset = 0
		while 1:
			chunk = fetch(host, port=port, ctjson=True,
				path="/_db?action=get&modelName=%s&offset=%s&limit=%s"%(model, offset, LIMIT))
			for c in chunk:
				fixkeys(c, schema)
				checkblobs(c, schema)
			mods[model] += chunk
			offset += LIMIT
			if len(chunk) < LIMIT:
				break
			log("got %s %s records"%(offset, model), 1)
		log("found %s %s records"%(len(mods[model]), model))
	log("%s unmatched keys!"%(len(missing.keys()),), important=True)
	prune()
	if blobs:
		getblobs(host, port)
	puts = []
	log("building models", important=True)
	for model in mods:
		mod = db.get_model(model)
{'fraction_non_alphanumeric': 0.16578850631050115, 'fraction_numerical': 0.003185884082832986, 'mean_word_length': 3.6560182544209927, 'pattern_counts': {'":': 6, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '785907', 'n_tokens_mistral': 3460, 'n_tokens_neox': 3058, 'n_words': 884}