<commit_before>import argparse
import math
import os
import threading
import time

import progressbar

from objects import beatmap
from objects import score
from common.constants import bcolors
from common.db import dbConnector
from common.ripple import userUtils
from constants import rankedStatuses
from helpers import config
from helpers import consoleHelper
from common import generalUtils
from objects import glob

# constants
MAX_WORKERS = 32
MODULE_NAME = "rippoppai"
UNIX = True if os.name == "posix" else False

# Global stuff
args = None

if __name__ == "__main__":
	# Verbose
	glob.debug = False

	def recalcFromScoreData(scoreData):
		"""
		Recalculate pp value for a score.
		Does every check, output and queries needed.

		score -- score+beatmap dictionary (returned from db with JOIN) of score to recalc
		return -- calculated pp value or None
		"""

		# Create score object and set its data
		s = score.score()
		s.setDataFromDict(scoreData)
		if s.scoreID == 0:
			# Make sure the score exists
			if glob.debug:
				consoleHelper.printColored("[!] No score with id {}".format(scoreData["id"]), bcolors.RED)

		# Create beatmap object
		b = beatmap.beatmap()

		# Check if we have data for this song
		if scoreData["song_name"] is None or args.apirefresh == True:
			# If we don't have song data in scoreData, get with get_scores method (mysql, osuapi blabla)
			b.setData(scoreData["beatmap_md5"], 0)
		else:
			# If we have data, set data from dict
			b.setDataFromDict(scoreData)

		# Make sure the beatmap is ranked
		if b.rankedStatus != rankedStatuses.RANKED and b.rankedStatus != rankedStatuses.APPROVED and b.rankedStatus != rankedStatuses.QUALIFIED:
			if glob.debug:
				consoleHelper.printColored("[!] Beatmap {} is not ranked ().".format(s.fileMd5), bcolors.RED)
			# Don't calculate pp if the beatmap is not ranked
			return False

		# Calculate score pp
		s.calculatePP(b)

		# Update score pp in dictionary
		scoreData["pp"] = s.pp
		return True

	class worker:
		"""
		rippoppai recalculator worker
		"""
		def __init__(self, _id, scores):
			"""
			Instantiate a worker

			id -- worker numeric id
			scores -- list of scores+beatmaps dictionaries to recalc
			"""
			self.id = _id
			self.scores = scores
			self.perc = 0.00
			self.current = 0
			self.total = len(self.scores)
			self.done = False

		def doWork(self):
			"""
			Worker's work
			Basically, calculate pp for scores inside self.scores
			"""

			# Make sure scores have been passed
			if self.scores is not None:
				for i in self.scores:
					# Loop through all scores
					# Recalculate pp
					recalcFromScoreData(i)

					# Calculate percentage
					self.perc = (100*self.current)/self.total

					# Update recalculated count
					self.current+=1

				# Recalculation finished, save new pp values in db
				consoleHelper.printColored("[WORKER{}] PP calc for this worker finished. Saving results in db...".format(self.id), bcolors.PINK)
				for i in self.scores:
					# Loop through all scores and update pp in db
					glob.db.execute("UPDATE scores SET pp = %s WHERE id = %s", [i["pp"], i["id"]])

			# This worker has finished his work
			self.done = True

	def massRecalc(scores, workersNum = 0):
		"""
		Recalc pp for scores in scores dictionary.

		scores -- dictionary returned from query. must contain id key with score id
		workersNum -- number of workers. If 0, will spawn 1 worker every 200 scores up to MAX_WORKERS
		"""
		# Get total scores number
		totalScores = len(scores)

		# Calculate number of workers if needed
		if workersNum == 0:
			workersNum = min(math.ceil(totalScores/200), MAX_WORKERS)<commit_msg>PRONTO LA SIGNORA ANNA?<commit_after>import argparse
import math
import os
import threading
import time

import progressbar

from objects import beatmap
from objects import score
from common.constants import bcolors
from common.db import dbConnector
from common.ripple import userUtils
from constants import rankedStatuses
from helpers import config
from helpers import consoleHelper
from common import generalUtils
from objects import glob

# constants
MAX_WORKERS = 32
MODULE_NAME = "rippoppai"
UNIX = True if os.name == "posix" else False

# Global stuff
args = None

if __name__ == "__main__":
	# Verbose
	glob.debug = False

	def recalcFromScoreData(scoreData):
		"""
		Recalculate pp value for a score.
		Does every check, output and queries needed.

		score -- score+beatmap dictionary (returned from db with JOIN) of score to recalc
		return -- calculated pp value or None
		"""

		# Create score object and set its data
		s = score.score()
		s.setDataFromDict(scoreData)
		if s.scoreID == 0:
			# Make sure the score exists
			if glob.debug:
				consoleHelper.printColored("[!] No score with id {}".format(scoreData["id"]), bcolors.RED)

		# Create beatmap object
		b = beatmap.beatmap()

		# Check if we have data for this song
		if scoreData["song_name"] is None or args.apirefresh == True:
			# If we don't have song data in scoreData, get with get_scores method (mysql, osuapi blabla)
			b.setData(scoreData["beatmap_md5"], 0)
		else:
			# If we have data, set data from dict
			b.setDataFromDict(scoreData)

		# Make sure the beatmap is ranked
		if b.rankedStatus != rankedStatuses.RANKED and b.rankedStatus != rankedStatuses.APPROVED and b.rankedStatus != rankedStatuses.QUALIFIED:
			if glob.debug:
				consoleHelper.printColored("[!] Beatmap {} is not ranked ().".format(s.fileMd5), bcolors.RED)
			# Don't calculate pp if the beatmap is not ranked
			return False

		# Calculate score pp
		s.calculatePP(b)

		# Update score pp in dictionary
		scoreData["pp"] = s.pp
		return True

	class worker:
		"""
		rippoppai recalculator worker
		"""
		def __init__(self, _id, scores):
			"""
			Instantiate a worker

			id -- worker numeric id
			scores -- list of scores+beatmaps dictionaries to recalc
			"""
			self.id = _id
			self.scores = scores
			self.perc = 0.00
			self.current = 0
			self.total = len(self.scores)
			self.done = False

		def doWork(self):
			"""
			Worker's work
			Basically, calculate pp for scores inside self.scores
			"""

			# Make sure scores have been passed
			if self.scores is not None:
					for i in self.scores:
						try:
							# Loop through all scores
							# Recalculate pp
							recalcFromScoreData(i)

							# Calculate percentage
							self.perc = (100*self.current)/self.total

							# Update recalculated count
							self.current+=1
						except Exception as e:
							consoleHelper.printColored("LA SIGNORA ANNA È MORTA! ({})".format(e), bcolors.YELLOW)

					# Recalculation finished, save new pp values in db
					consoleHelper.printColored("[WORKER{}] PP calc for this worker finished. Saving results in db...".format(self.id), bcolors.PINK)
					try:
						for i in self.scores:
							# Loop through all scores and update pp in db
							glob.db.execute("UPDATE scores SET pp = %s WHERE id = %s", [i["pp"], i["id"]])
					except:
						consoleHelper.printColored("Errore query stampa piede intensifies", bcolors.RED)

			# This worker has finished his work
			self.done = True

	def massRecalc(scores, workersNum = 0):
		"""
		Recalc pp for scores in scores dictionary.

		scores -- dictionary returned from query. must contain id key with score id
		workersNum -- number of workers. If 0, will spawn 1 worker every 200 scores up to MAX_WORKERS
		"""
		# Get total scores number
		totalScores = len(scores)

		# Calculate number of workers if needed
		if workersNum == 0:
			workersNum = min(math.ceil(totalScores/200), MAX_WORKERS)
{'fraction_non_alphanumeric': 0.08828996282527882, 'fraction_numerical': 0.006372809346787042, 'mean_word_length': 3.3873034362259755, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4064866', 'n_tokens_mistral': 2768, 'n_tokens_neox': 2277, 'n_words': 848}