<commit_before><?php

namespace App\Console\Commands\Bulk;

use Aic\Hub\Foundation\AbstractCommand as BaseCommand;

class BulkAll extends BaseCommand
{

    protected $signature = 'bulk:all {skip?}';

    protected $description = "Reset database and import everything";

    public function handle()
    {
        $shouldSkipTo = $this->argument('skip') ?? false;

        // Import all bulkable resources from compliant data services
        foreach (config('resources.inbound') as $source => $endpoints) {
            foreach ($endpoints as $endpoint => $resource) {
                dump("$source >> $endpoint");

                if ($shouldSkipTo && $source !== $shouldSkipTo) {
                    dump("Skipping...");
                    continue;
                }

                $this->call('bulk:import', [
                    'source' => $source,
                    'endpoint' => $endpoint,
                ]);

                $shouldSkipTo = false;
            }
        }

        // EventOccurrence is not included in import:web to avoid duplication
        $this->call('import:web-full', ['--yes' => 'default', 'endpoint' => 'event-occurrences']);

        // Run scout now for faster uptime
        $this->call('scout:import-all');

        // Import non-standard data
        $this->call('import:mobile');
        $this->call('import:sites', ['--yes' => 'default']);
        $this->call('import:ulan');
        $this->call('import:digital-labels');

        // TODO: Use upserts for import:analytics
        $this->call('import:analytics');

        // TODO: Use bulking for import:images
        $this->call('import:images');

        // TODO: Spoof \App\Command records for partial imports
    }

}
<commit_msg>Spoof \App\Command records in `bulk:all`<commit_after><?php

namespace App\Console\Commands\Bulk;

use App\Command;

use Aic\Hub\Foundation\AbstractCommand as BaseCommand;

class BulkAll extends BaseCommand
{

    protected $signature = 'bulk:all {skip?}';

    protected $description = "Reset database and import everything";

    public function handle()
    {
        $shouldSkipTo = $this->argument('skip') ?? false;

        // Import all bulkable resources from compliant data services
        foreach (config('resources.inbound') as $source => $endpoints) {
            foreach ($endpoints as $endpoint => $resource) {
                dump("$source >> $endpoint");

                if ($shouldSkipTo && $source !== $shouldSkipTo) {
                    dump("Skipping...");
                    continue;
                }

                $this->call('bulk:import', [
                    'source' => $source,
                    'endpoint' => $endpoint,
                ]);

                $shouldSkipTo = false;
            }
        }

        // EventOccurrence is not included in import:web to avoid duplication
        $this->call('import:web-full', ['--yes' => 'default', 'endpoint' => 'event-occurrences']);

        // Run scout now for faster uptime
        $this->call('scout:import-all');

        // Import non-standard data
        $this->call('import:mobile');
        $this->call('import:sites', ['--yes' => 'default']);
        $this->call('import:ulan');
        $this->call('import:digital-labels');

        // TODO: Use upserts for import:analytics
        $this->call('import:analytics');

        // TODO: Use bulking for import:images
        $this->call('import:images');

        // Spoof \App\Command records for partial imports
        foreach ([
            'import:collections',
            'import:assets',
            'import:events',
            'import:dsc',
            'import:mobile',
            'import:library',
            'import:archive',
            'import:sites',
            'import:ulan',
            'import:products',
            'import:images',
            'import:analytics',
            'import:web',
        ] as $commandName) {
            $command = Command::where('command', $commandName) ?: new Command([
                'title' => $commandName,
            ]);

            $command->last_attempt_at = $this->command->last_attempt_at;
            $command->last_success_at = $this->command->last_attempt_at;

            $command->save();
        }
    }

}

{'fraction_non_alphanumeric': 0.14520676691729323, 'fraction_numerical': 0.0, 'mean_word_length': 1.9298004129387474, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4537871', 'n_tokens_mistral': 1175, 'n_tokens_neox': 1112, 'n_words': 253}