<commit_before>#!/usr/bin/env perl
use strict;
use warnings FATAL => qw/all/;

use FindBin;
use lib "$FindBin::Bin/../lib";

use Encode;

use YATT::Lite::XHF;
use YAML::Tiny;

foreach my $fn (@ARGV) {
  my $parser = YATT::Lite::XHF->new(file => $fn, encoding => 'utf8');

  my @data;
  push @data, scalar $parser->read(skip_comment => 0);
  push @data, $_ while $_ = $parser->read;

  print encode_utf8(YAML::Tiny->new(@data)->write_string);
}<commit_msg>Add -d outdir option. worked fine.<commit_after>#!/usr/bin/env perl
use strict;
use warnings FATAL => qw/all/;
use 5.016;

use FindBin;
use lib "$FindBin::Bin/../lib";

use Encode;
use YAML::Tiny;
use Getopt::Long;
use File::Basename;

use YATT::Lite::XHF;

sub usage {
  die join("\n", @_, <<END);
Usage: $0 [-d OUTDIR]  FILES...
END

}

{

  GetOptions("d|outdir=s", \ (my $o_outdir)
             , "h|help",   \ (my $o_help)
           )
    or usage("Invalid option");
  usage() if $o_help;

  if ($o_outdir) {
    usage("Can't find outdir: $o_outdir") unless -d $o_outdir;
    usage("outdir is not writable: $o_outdir") unless -w $o_outdir;
  }

  foreach my $fn (@ARGV) {
    my $parser = YATT::Lite::XHF->new(file => $fn, encoding => 'utf8');

    # Read toplevel as dictionaries.
    my @data;
    # Metainfo. Maybe empty.
    push @data, scalar $parser->read(skip_comment => 0);
    # body.
    push @data, $_ while $_ = $parser->read;

    my $yaml = YAML::Tiny->new(@data);

    if ($o_outdir) {
      my $outfn = join("/", $o_outdir, basename($fn));
      $outfn =~ s/\.xhf$/.yaml/;
      $yaml->write($outfn);
    } else {
      print encode_utf8($yaml->write_string);
    }
  }
}
{'fraction_non_alphanumeric': 0.19478155339805825, 'fraction_numerical': 0.006674757281553398, 'mean_word_length': 3.1328320802005014, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7207722', 'n_tokens_mistral': 675, 'n_tokens_neox': 649, 'n_words': 157}