<commit_before><?php

namespace App\Console\Commands;

use Carbon\Carbon;
use DB;

use App\Models\Dsc\Publication;
use App\Models\Dsc\Section;

class ImportCatalogues extends AbstractImportCommand
{

    protected $signature = 'import:dsc
                            {--y|yes : Answer "yes" to all prompts}';

    protected $description = "Import all catalogue data";


    public function handle()
    {

        // Return false if the user bails out
        if (!$this->option('yes') && !$this->confirm("Running this will delete all existing catalogue data from your database! Are you sure?"))
        {
            return false;
        }

        // Remove all Publications and Sections from the search index
        $this->call("scout:flush", ['model' => Publication::class]);
        $this->call("scout:flush", ['model' => Section::class]);

        // Truncate tables
        DB::table('sections')->truncate();
        DB::table('publications')->truncate();

        $this->info("Truncated catalogue tables.");

        // Reinstall search: flush might not work, since some models might be present in the index, which aren't here
        $this->info("Please manually ensure that your search index mappings are up-to-date.");
        // $this->call("search:uninstall");
        // $this->call("search:install");

        $this->import(Publication::class, 'publications', 1);
        $this->import(Section::class, 'sections', 1);

        $this->info("Imported all Publications and Sections from data service!");

    }


    private function import($model, $endpoint, $current = 1)
    {

        // Abort if the table is already filled
        if( $model::count() > 0 )
        {
            return false;
        }

        // Query for the first page + get page count
        $json = $this->queryService($endpoint, $current);

        $pages = $json->pagination->total_pages;

        while ($current <= $pages)
        {

            foreach ($json->data as $source)
            {

                $this->saveDatum( $source, $model );

            }

            $current++;
            $json = $this->queryService($endpoint, $current);

        }

    }

    private function queryService($endpoint, $page = 1, $limit = 100)
    {
        return $this->query( env('DSC_DATA_SERVICE_URL') . '/' . $endpoint . '?page=' . $page . '&limit=' . $limit );
    }

}<commit_msg>Refactor ImportCatalogues to use ImportsData<commit_after><?php

namespace App\Console\Commands;

use Carbon\Carbon;
use DB;

use App\Models\Dsc\Publication;
use App\Models\Dsc\Section;

class ImportCatalogues extends AbstractImportCommandNew
{

    protected $signature = 'import:dsc
                            {--y|yes : Answer "yes" to all prompts}';

    protected $description = "Import all catalogue data";


    public function handle()
    {

        $this->api = env('DSC_DATA_SERVICE_URL');

        // Return false if the user bails out
        if (!$this->option('yes') && !$this->confirm("Running this will delete all existing catalogue data from your database! Are you sure?"))
        {
            return false;
        }

        // Remove all Publications and Sections from the search index
        $this->call("scout:flush", ['model' => Publication::class]);
        $this->call("scout:flush", ['model' => Section::class]);

        // Truncate tables
        DB::table('sections')->truncate();
        DB::table('publications')->truncate();

        $this->info("Truncated catalogue tables.");

        // Reinstall search: flush might not work, since some models might be present in the index, which aren't here
        $this->info("Please manually ensure that your search index mappings are up-to-date.");
        // $this->call("search:uninstall");
        // $this->call("search:install");

        $this->import(Publication::class, 'publications');
        $this->import(Section::class, 'sections');

        $this->info("Imported all Publications and Sections from data service!");

    }

}
{'fraction_non_alphanumeric': 0.14032016008004003, 'fraction_numerical': 0.002001000500250125, 'mean_word_length': 2.501751313485114, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5649236', 'n_tokens_mistral': 1128, 'n_tokens_neox': 1062, 'n_words': 312}