<commit_before><?php

namespace App\Console\Commands;

use App\Player;
use App\Replay;
use App\Services\ParserService;
use DB;
use Illuminate\Console\Command;
use Illuminate\Support\Collection;

class ReparseReplays extends Command
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
    protected $signature = 'hotsapi:reparse {min_id=0} {max_id=1000000000}';

    /**
     * The console command description.
     *
     * @var string
     */
    protected $description = 'Redo parsing for replays';

    /**
     * @var ParserService
     */
    private $parser;

    /**
     * Create a new command instance.
     *
     * @param ParserService $parser
     */
    public function __construct(ParserService $parser)
    {
        parent::__construct();
        $this->parser = $parser;
    }

    /**
     * Execute the console command.
     *
     * @return mixed
     */
    public function handle()
    {
        $min_id = $this->argument('min_id');
        $max_id = $this->argument('max_id');
        $this->info("Reparsing replays, id from $min_id to $max_id");
        //Replay::where('id', '>=', $min_id)->where('id', '<=', $max_id)->with('players')->chunk(100, function ($x) { return $this->reparse($x); });
        $this->getBrokenReplays()->each(function ($x) { return $this->reparse([$x]); });
    }

    public function getBrokenReplays()
    {
        $noMap = DB::select('SELECT id FROM replays WHERE game_map IS NULL');
        $noHero = DB::select('SELECT DISTINCT(replay_id) AS id FROM players WHERE hero IS NULL');
        $noPlayers = DB::select('SELECT replay_id AS id, count(*) AS cnt FROM players GROUP BY replay_id HAVING cnt != 10');

        $ids = collect($noMap)->merge($noHero)->merge($noPlayers)->pluck('id')->unique();
        $this->info("Broken replay summary:\nNo map: " . count($noMap) . "\nNo hero: " . count($noHero) . "\nNo players: " . count($noPlayers) . "\nUnique: " . count($ids));
        return Replay::whereIn('id', $ids)->get();
    }

    /**
     * Reparse collection of replays
     *
     * @param $replays
     */
    public function reparse($replays)
    {
        foreach ($replays as $replay) {
            $this->info("Parsing replay id=$replay->id, file=$replay->filename");
            $tmpFile = tempnam('', 'replay_');
            try {
                $content = \Storage::cloud()->get("$replay->filename.StormReplay");
                file_put_contents($tmpFile, $content);
                $parseResult = $this->parser->analyze($tmpFile, true);
                if ($parseResult->status != ParserService::STATUS_SUCCESS) {<commit_msg>add no players query<commit_after><?php

namespace App\Console\Commands;

use App\Player;
use App\Replay;
use App\Services\ParserService;
use DB;
use Illuminate\Console\Command;
use Illuminate\Support\Collection;

class ReparseReplays extends Command
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
    protected $signature = 'hotsapi:reparse {min_id=0} {max_id=1000000000}';

    /**
     * The console command description.
     *
     * @var string
     */
    protected $description = 'Redo parsing for replays';

    /**
     * @var ParserService
     */
    private $parser;

    /**
     * Create a new command instance.
     *
     * @param ParserService $parser
     */
    public function __construct(ParserService $parser)
    {
        parent::__construct();
        $this->parser = $parser;
    }

    /**
     * Execute the console command.
     *
     * @return mixed
     */
    public function handle()
    {
        $min_id = $this->argument('min_id');
        $max_id = $this->argument('max_id');
        $this->info("Reparsing replays, id from $min_id to $max_id");
        //Replay::where('id', '>=', $min_id)->where('id', '<=', $max_id)->with('players')->chunk(100, function ($x) { return $this->reparse($x); });
        $this->getBrokenReplays()->each(function ($x) { return $this->reparse([$x]); });
    }

    public function getBrokenReplays()
    {
        $noMap = DB::select('SELECT id FROM replays WHERE game_map IS NULL');
        $noHero = DB::select('SELECT DISTINCT(replay_id) AS id FROM players WHERE hero IS NULL');
        $noPlayers = DB::select('SELECT r.id AS id FROM replays r LEFT JOIN players p ON p.replay_id = r.id WHERE p.id IS NULL');
        $wrongPlayers = DB::select('SELECT replay_id AS id, count(*) AS cnt FROM players GROUP BY replay_id HAVING cnt != 10');

        $ids = collect($noMap)->merge($noHero)->merge($noPlayers)->merge($wrongPlayers)->pluck('id')->unique();
        $this->info("Broken replay summary:\nNo map: " . count($noMap) . "\nNo hero: " . count($noHero) . "\nNo players: " . count($noPlayers) . "\nWrong players: " . count($wrongPlayers) . "\nUnique: " . count($ids));
        return Replay::whereIn('id', $ids)->get();
    }

    /**
     * Reparse collection of replays
     *
     * @param $replays
     */
    public function reparse($replays)
    {
        foreach ($replays as $replay) {
            $this->info("Parsing replay id=$replay->id, file=$replay->filename");
            $tmpFile = tempnam('', 'replay_');
            try {
                $content = \Storage::cloud()->get("$replay->filename.StormReplay");
                file_put_contents($tmpFile, $content);
                $parseResult = $this->parser->analyze($tmpFile, true);
                if ($parseResult->status != ParserService::STATUS_SUCCESS) {
{'fraction_non_alphanumeric': 0.16171917683482062, 'fraction_numerical': 0.005827718084137679, 'mean_word_length': 2.6394963552021204, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '236216', 'n_tokens_mistral': 1755, 'n_tokens_neox': 1680, 'n_words': 393}