<commit_before><?php

final class PhabricatorFilesManagementMigrateWorkflow
  extends PhabricatorFilesManagementWorkflow {

  protected function didConstruct() {
    $this
      ->setName('migrate')
      ->setSynopsis(pht('Migrate files between storage engines.'))
      ->setArguments(
        array(
          array(
            'name'      => 'engine',
            'param'     => 'storage_engine',
            'help'      => pht('Migrate to the named storage engine.'),
          ),
          array(
            'name'      => 'dry-run',
            'help'      => pht('Show what would be migrated.'),
          ),
          array(
            'name' => 'min-size',
            'param' => 'bytes',
            'help' => pht(
              'Do not migrate data for files which are smaller than a given '.
              'filesize.'),
          ),
          array(
            'name' => 'max-size',
            'param' => 'bytes',
            'help' => pht(
              'Do not migrate data for files which are larger than a given '.
              'filesize.'),
          ),
          array(
            'name'      => 'all',
            'help'      => pht('Migrate all files.'),
          ),
          array(
            'name' => 'copy',
            'help' => pht(
              'Copy file data instead of moving it: after migrating, do not '.
              'remove the old data even if it is no longer referenced.'),
          ),
          array(
            'name'      => 'names',
            'wildcard'  => true,
          ),
        ));
  }

  public function execute(PhutilArgumentParser $args) {
    $target_key = $args->getArg('engine');
    if (!$target_key) {
      throw new PhutilArgumentUsageException(
        pht(
          'Specify an engine to migrate to with `%s`. '.
          'Use `%s` to get a list of engines.',
          '--engine',
          'files engines'));
    }

    $target_engine = PhabricatorFile::buildEngine($target_key);

    $iterator = $this->buildIterator($args);
    if (!$iterator) {
      throw new PhutilArgumentUsageException(
        pht(
          'Either specify a list of files to migrate, or use `%s` '.
          'to migrate all files.',
          '--all'));
    }

    $is_dry_run = $args->getArg('dry-run');

    $min_size = (int)$args->getArg('min-size');
    $max_size = (int)$args->getArg('max-size');

    $is_copy = $args->getArg('copy');

    $failed = array();
    $engines = PhabricatorFileStorageEngine::loadAllEngines();
    $total_bytes = 0;
    $total_files = 0;
    foreach ($iterator as $file) {
      $monogram = $file->getMonogram();

      $engine_key = $file->getStorageEngine();
      $engine = idx($engines, $engine_key);

      if (!$engine) {
        echo tsprintf(
          "%s\n",
          pht(
            '%s: Uses unknown storage engine "%s".',
            $monogram,
            $engine_key));
        $failed[] = $file;
        continue;
      }

      if ($engine->isChunkEngine()) {
        echo tsprintf(
          "%s\n",
          pht(
            '%s: Stored as chunks, no data to migrate directly.',
            $monogram));
        continue;
      }

      if ($engine_key === $target_key) {
        echo tsprintf(
          "%s\n",
          pht(<commit_msg>When migrating files between storage engines with "bin/files migrate ...", skip expired temporary files<commit_after><?php

final class PhabricatorFilesManagementMigrateWorkflow
  extends PhabricatorFilesManagementWorkflow {

  protected function didConstruct() {
    $this
      ->setName('migrate')
      ->setSynopsis(pht('Migrate files between storage engines.'))
      ->setArguments(
        array(
          array(
            'name'      => 'engine',
            'param'     => 'storage_engine',
            'help'      => pht('Migrate to the named storage engine.'),
          ),
          array(
            'name'      => 'dry-run',
            'help'      => pht('Show what would be migrated.'),
          ),
          array(
            'name' => 'min-size',
            'param' => 'bytes',
            'help' => pht(
              'Do not migrate data for files which are smaller than a given '.
              'filesize.'),
          ),
          array(
            'name' => 'max-size',
            'param' => 'bytes',
            'help' => pht(
              'Do not migrate data for files which are larger than a given '.
              'filesize.'),
          ),
          array(
            'name'      => 'all',
            'help'      => pht('Migrate all files.'),
          ),
          array(
            'name' => 'copy',
            'help' => pht(
              'Copy file data instead of moving it: after migrating, do not '.
              'remove the old data even if it is no longer referenced.'),
          ),
          array(
            'name'      => 'names',
            'wildcard'  => true,
          ),
        ));
  }

  public function execute(PhutilArgumentParser $args) {
    $target_key = $args->getArg('engine');
    if (!$target_key) {
      throw new PhutilArgumentUsageException(
        pht(
          'Specify an engine to migrate to with `%s`. '.
          'Use `%s` to get a list of engines.',
          '--engine',
          'files engines'));
    }

    $target_engine = PhabricatorFile::buildEngine($target_key);

    $iterator = $this->buildIterator($args);
    if (!$iterator) {
      throw new PhutilArgumentUsageException(
        pht(
          'Either specify a list of files to migrate, or use `%s` '.
          'to migrate all files.',
          '--all'));
    }

    $is_dry_run = $args->getArg('dry-run');

    $min_size = (int)$args->getArg('min-size');
    $max_size = (int)$args->getArg('max-size');

    $is_copy = $args->getArg('copy');

    $failed = array();
    $engines = PhabricatorFileStorageEngine::loadAllEngines();
    $total_bytes = 0;
    $total_files = 0;
    foreach ($iterator as $file) {
      $monogram = $file->getMonogram();

      // See T7148. When we export data for an instance, we copy all the data
      // for Files from S3 into the database dump so that the database dump is
      // a complete, standalone archive of all the data. In the general case,
      // installs may have a similar process using "--copy" to create a more
      // complete backup.

      // When doing this, we may run into temporary files which have been
      // deleted between the time we took the original dump and the current
      // timestamp. These files can't be copied since the data no longer
      // exists: the daemons on the live install already deleted it.

      // Simply avoid this whole mess by declining to migrate expired temporary
      // files. They're as good as dead anyway.

      $ttl = $file->getTTL();
      if ($ttl) {
        if ($ttl < PhabricatorTime::getNow()) {
          echo tsprintf(
            "%s\n",
            pht(
              '%s: Skipping expired temporary file.',
              $monogram));
          continue;
        }
      }

      $engine_key = $file->getStorageEngine();
      $engine = idx($engines, $engine_key);

      if (!$engine) {
        echo tsprintf(
          "%s\n",
          pht(
            '%s: Uses unknown storage engine "%s".',
            $monogram,
            $engine_key));
        $failed[] = $file;
        continue;
      }

      if ($engine->isChunkEngine()) {
        echo tsprintf(
          "%s\n",
          pht(
            '%s: Stored as chunks, no data to migrate directly.',
            $monogram));
        continue;
      }

      if ($engine_key === $target_key) {
        echo tsprintf(
          "%s\n",
          pht(
{'fraction_non_alphanumeric': 0.13303653266989657, 'fraction_numerical': 0.0011784732224695562, 'mean_word_length': 1.653926337734538, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4943374', 'n_tokens_mistral': 2207, 'n_tokens_neox': 2162, 'n_words': 632}