<commit_before><?php

declare(strict_types=1);

namespace App\Command;

use App\Entity\LearningMaterialInterface;
use App\Repository\CourseLearningMaterialRepository;
use App\Repository\CourseObjectiveRepository;
use App\Repository\LearningMaterialRepository;
use App\Repository\ProgramYearObjectiveRepository;
use App\Repository\SessionLearningMaterialRepository;
use App\Repository\SessionObjectiveRepository;
use App\Repository\SessionRepository;
use Doctrine\ORM\EntityManagerInterface;
use Exception;
use HTMLPurifier;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Helper\Table;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;
use Symfony\Contracts\HttpClient\HttpClientInterface;

/**
 * Cleans up all the strings in the database
 *
 * Class CleanupStringsCommand
 */
class CleanupStringsCommand extends Command
{
    /**
     * @var int where to limit each query for memory management
     */
    private const QUERY_LIMIT = 500;

    public function __construct(
        protected HTMLPurifier $purifier,
        protected EntityManagerInterface $em,
        protected LearningMaterialRepository $learningMaterialRepository,
        protected CourseLearningMaterialRepository $courseLearningMaterialRepository,
        protected SessionLearningMaterialRepository $sessionLearningMaterialRepository,
        protected SessionRepository $sessionRepository,
        protected SessionObjectiveRepository $sessionObjectiveRepository,
        protected CourseObjectiveRepository $courseObjectiveRepository,
        protected ProgramYearObjectiveRepository $programYearObjectiveRepository,
        protected HttpClientInterface $httpClient
    ) {
        parent::__construct();
    }

    protected function configure()
    {
        $this
            ->setName('ilios:cleanup-strings')
            ->setAliases(['ilios:maintenance:cleanup-strings'])
            ->setDescription('Cleans up selected text fields in the database.')
            ->addOption(
                'objective-title',
                null,
                InputOption::VALUE_NONE,
                'Should we purify the markup of objective titles?'
            )
            ->addOption(
                'learningmaterial-description',
                null,
                InputOption::VALUE_NONE,
                'Should we purify the markup of learning material descriptions?'
            )
            ->addOption(
                'learningmaterial-note',
                null,
                InputOption::VALUE_NONE,
                'Should we purify the markup of learning materials notes?'
            )
            ->addOption(
                'session-description',
                null,
                InputOption::VALUE_NONE,
                'Should we purify the markup of session descriptions?'
            )
            ->addOption(
                'learningmaterial-links',
                null,
                InputOption::VALUE_NONE,
                'Should we attempt to correct learning material links?'
            );
    }

    protected function execute(InputInterface $input, OutputInterface $output): int
    {
        if ($input->getOption('objective-title')) {
            $this->purifyObjectiveTitle($output);
        }
        if ($input->getOption('learningmaterial-description')) {
            $this->purifyLearnignMaterialDescription($output);
        }
        if ($input->getOption('learningmaterial-note')) {
            $this->purifyCourseLearningMaterialNote($output);
            $output->writeln('');
            $this->purifySessionLearningMaterialNote($output);
        }
        if ($input->getOption('session-description')) {
            $this->purifySessionDescription($output);
        }
        if ($input->getOption('learningmaterial-links')) {
            $this->correctLearningMaterialLinks($output);
        }

        return 0;
    }

    /**
     * Purify objective titles
     * @throws Exception
     */
    protected function purifyObjectiveTitle(OutputInterface $output)
    {
        $cleanedTitles = 0;
        $limit = self::QUERY_LIMIT;
        $totalObjectives = $this->sessionObjectiveRepository->getTotalObjectiveCount();
        $totalObjectives += $this->courseObjectiveRepository->getTotalObjectiveCount();
        $totalObjectives += $this->programYearObjectiveRepository->getTotalObjectiveCount();

        $progress = new ProgressBar($output, $totalObjectives);
        $progress->setRedrawFrequency(208);
        $output->writeln("<info>Starting cleanup of objective titles...</info>");
        $progress->start();
        $objectiveManagers = [
            $this->sessionObjectiveRepository,
            $this->courseObjectiveRepository,
            $this->programYearObjectiveRepository
        ];

        foreach ($objectiveManagers as $objectiveManager) {
            $offset = 0;
            do {
                $objectives = $objectiveManager->findBy([], ['id' => 'ASC'], $limit, $offset);
                foreach ($objectives as $objective) {
                    $originalTitle = $objective->getTitle();
                    $cleanTitle = $this->purifier->purify($originalTitle);
                    if ($originalTitle != $cleanTitle) {
                        $cleanedTitles++;
                        $objective->setTitle($cleanTitle);
                        $objectiveManager->update($objective, false);
                    }
                    $progress->advance();
                }

                $offset += $limit;
                $this->em->flush();
                $this->em->clear();
            } while (count($objectives) === $limit);
        }

        $progress->finish();
        $output->writeln('');
        $output->writeln("<info>{$cleanedTitles} Objective Titles updated.</info>");
    }

    /**
     * Purify learning material description
     */
    protected function purifyLearnignMaterialDescription(OutputInterface $output)
    {
        $cleaned = 0;
        $offset = 0;
        $limit = self::QUERY_LIMIT;
        $total = $this->learningMaterialRepository->getTotalLearningMaterialCount();
        $progress = new ProgressBar($output, $total);<commit_msg>fixes typo in function name.<commit_after><?php

declare(strict_types=1);

namespace App\Command;

use App\Entity\LearningMaterialInterface;
use App\Repository\CourseLearningMaterialRepository;
use App\Repository\CourseObjectiveRepository;
use App\Repository\LearningMaterialRepository;
use App\Repository\ProgramYearObjectiveRepository;
use App\Repository\SessionLearningMaterialRepository;
use App\Repository\SessionObjectiveRepository;
use App\Repository\SessionRepository;
use Doctrine\ORM\EntityManagerInterface;
use Exception;
use HTMLPurifier;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Helper\Table;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;
use Symfony\Contracts\HttpClient\HttpClientInterface;

/**
 * Cleans up all the strings in the database
 *
 * Class CleanupStringsCommand
 */
class CleanupStringsCommand extends Command
{
    /**
     * @var int where to limit each query for memory management
     */
    private const QUERY_LIMIT = 500;

    public function __construct(
        protected HTMLPurifier $purifier,
        protected EntityManagerInterface $em,
        protected LearningMaterialRepository $learningMaterialRepository,
        protected CourseLearningMaterialRepository $courseLearningMaterialRepository,
        protected SessionLearningMaterialRepository $sessionLearningMaterialRepository,
        protected SessionRepository $sessionRepository,
        protected SessionObjectiveRepository $sessionObjectiveRepository,
        protected CourseObjectiveRepository $courseObjectiveRepository,
        protected ProgramYearObjectiveRepository $programYearObjectiveRepository,
        protected HttpClientInterface $httpClient
    ) {
        parent::__construct();
    }

    protected function configure()
    {
        $this
            ->setName('ilios:cleanup-strings')
            ->setAliases(['ilios:maintenance:cleanup-strings'])
            ->setDescription('Cleans up selected text fields in the database.')
            ->addOption(
                'objective-title',
                null,
                InputOption::VALUE_NONE,
                'Should we purify the markup of objective titles?'
            )
            ->addOption(
                'learningmaterial-description',
                null,
                InputOption::VALUE_NONE,
                'Should we purify the markup of learning material descriptions?'
            )
            ->addOption(
                'learningmaterial-note',
                null,
                InputOption::VALUE_NONE,
                'Should we purify the markup of learning materials notes?'
            )
            ->addOption(
                'session-description',
                null,
                InputOption::VALUE_NONE,
                'Should we purify the markup of session descriptions?'
            )
            ->addOption(
                'learningmaterial-links',
                null,
                InputOption::VALUE_NONE,
                'Should we attempt to correct learning material links?'
            );
    }

    protected function execute(InputInterface $input, OutputInterface $output): int
    {
        if ($input->getOption('objective-title')) {
            $this->purifyObjectiveTitle($output);
        }
        if ($input->getOption('learningmaterial-description')) {
            $this->purifyLearningMaterialDescription($output);
        }
        if ($input->getOption('learningmaterial-note')) {
            $this->purifyCourseLearningMaterialNote($output);
            $output->writeln('');
            $this->purifySessionLearningMaterialNote($output);
        }
        if ($input->getOption('session-description')) {
            $this->purifySessionDescription($output);
        }
        if ($input->getOption('learningmaterial-links')) {
            $this->correctLearningMaterialLinks($output);
        }

        return 0;
    }

    /**
     * Purify objective titles
     * @throws Exception
     */
    protected function purifyObjectiveTitle(OutputInterface $output)
    {
        $cleanedTitles = 0;
        $limit = self::QUERY_LIMIT;
        $totalObjectives = $this->sessionObjectiveRepository->getTotalObjectiveCount();
        $totalObjectives += $this->courseObjectiveRepository->getTotalObjectiveCount();
        $totalObjectives += $this->programYearObjectiveRepository->getTotalObjectiveCount();

        $progress = new ProgressBar($output, $totalObjectives);
        $progress->setRedrawFrequency(208);
        $output->writeln("<info>Starting cleanup of objective titles...</info>");
        $progress->start();
        $objectiveManagers = [
            $this->sessionObjectiveRepository,
            $this->courseObjectiveRepository,
            $this->programYearObjectiveRepository
        ];

        foreach ($objectiveManagers as $objectiveManager) {
            $offset = 0;
            do {
                $objectives = $objectiveManager->findBy([], ['id' => 'ASC'], $limit, $offset);
                foreach ($objectives as $objective) {
                    $originalTitle = $objective->getTitle();
                    $cleanTitle = $this->purifier->purify($originalTitle);
                    if ($originalTitle != $cleanTitle) {
                        $cleanedTitles++;
                        $objective->setTitle($cleanTitle);
                        $objectiveManager->update($objective, false);
                    }
                    $progress->advance();
                }

                $offset += $limit;
                $this->em->flush();
                $this->em->clear();
            } while (count($objectives) === $limit);
        }

        $progress->finish();
        $output->writeln('');
        $output->writeln("<info>{$cleanedTitles} Objective Titles updated.</info>");
    }

    /**
     * Purify learning material description
     */
    protected function purifyLearningMaterialDescription(OutputInterface $output)
    {
        $cleaned = 0;
        $offset = 0;
        $limit = self::QUERY_LIMIT;
        $total = $this->learningMaterialRepository->getTotalLearningMaterialCount();
        $progress = new ProgressBar($output, $total);
{'fraction_non_alphanumeric': 0.10130234734461514, 'fraction_numerical': 0.0018716369024409265, 'mean_word_length': 2.4142705005324814, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 109, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6799523', 'n_tokens_mistral': 3175, 'n_tokens_neox': 2992, 'n_words': 607}