<commit_before>#!/opt/local/bin/perl

##TODO
#Verify all URLs to make sure there are no 404s
#Add random string to filenames to make URLs unguessable

use strict;
use warnings;

use File::Temp qw(tempfile tempdir);
use File::Path 'remove_tree';
use File::Copy;
use File::Spec;
use POSIX qw(floor);
use Text::CSV;
use String::Random;
use YAML ();
use Getopt::Long;

my $Dialoger = '/Applications/CocoaDialog.app/Contents/MacOS/CocoaDialog';
Getopt::Long::Configure(qw(bundling ignore_case));

my (%opt, @files, $subtitle, $time, $project_name);
GetOptions(
	   'help' => \$opt{help},
	   'file=s' => \@files,
	   'title=s' => \$project_name,
	   'chunks=s' => \$time,
	   'subtitle=s' => \$subtitle,
	  );
my $usage = "USAGE: assign.pl --file foo.mp3 [--file bar.mp3...] --title Foo --chunks 1:00 [--subtitle 'Foo telephone interview']\n";
die $usage if $opt{help};
my $name_via_command_line;
$name_via_command_line = 1 if $project_name;

@files = @ARGV if not @files;
@files or error_bye("Drag audio files onto the icon");

#Need files in proper order before joining them
@files = sort { filepath_sortable($a) cmp filepath_sortable($b) } @files;
#Need file extensions so we know what files need to be converted to mp3
filebase_split($_) or error_bye("You need a file extension on the file '$_'") foreach @files;
#Escape file names so they can safely be passed to shell within single quotes 
shell_safe($_) or error_bye("Unsafe file name", $_) foreach @files;

my %config = config_file();
$config{$_} or error_bye("Required param '$_' missing from config file", "$ENV{HOME}/.audibleturk") foreach qw(scp url);
#my @fails = $config{scp} =~ /[^\w\s_\@\/\.\:\-]+/g;
#print "DEBUG fail $_\n" foreach @fails;
$config{scp} =~ /^[\w\s_\@\/\.\:\+\-]+$/ or error_bye("Unexpected scp format in config file", "$ENV{HOME}/.audibleturk");
if ($config{local}) {
  $config{local} =~ s/^~\/?/$ENV{HOME}/;
  $config{local} = "$ENV{HOME}/$config{local}" if $config{local} =~ /^[^\/]/;
  $config{local} =~ s/\/$//;
}
$config{local} ||= "$ENV{HOME}/Desktop";

foreach (qw(scp url local)){
    $config{$_} =~ s/\s+$//;
    $config{$_} =~ s/\/$//;
}

shell_safe($config{local}) or error_bye("Unsafe dir name", $config{local});
if (exists $config{randomize}){
    $config{randomize} = 0 if $config{randomize} =~ /\bfalse\b/i;
    $config{randomize} = 0 if $config{randomize} =~ /\bno\b/i;
    $config{randomize} = 0 if $config{randomize} =~ /\b0\b/;
    $config{randomize} = 0 if not defined $config{randomize}
}
else {
    $config{randomize} = 1;
}

#Get split interval
my $default_interval = $config{last_split_interval} || $config{default_split_interval} || '1:00';
$default_interval =~ /^[\d\:\s\.]+$/ or error_bye("Unexpected format to default split interval '$default_interval'");
my $min_dot_seconds;
$min_dot_seconds = to_min_seconds($time) if $time;
until ($min_dot_seconds) {
  my $dialog_input = `$Dialoger standard-inputbox --title "Length" --no-newline --informative-text "Split audio every (hh:mm:ss)" --text '$default_interval'`;
  my ($button, $time) = split /\n/, $dialog_input, 2;
  $button == 1 or exit;
  $min_dot_seconds = to_min_seconds($time) or error_box("Did not understand the time '$time'");
  config_file('last_split_interval' => $time) if $min_dot_seconds;
}


#Get base name for mp3s we'll be outputting
my ($default_name) = filebase_split(filepath_base($files[0]));
$default_name =~ s/(\w+)(\.)?\d+(\.mono)?$/$1/;
until ($project_name && init_project_dir($project_name)) {
  my $filled_in_name = $project_name || $default_name;
  shell_safe($filled_in_name) or error_bye("Potentially unsafe name encountered: $filled_in_name");
  my $dialog_input = `$Dialoger standard-inputbox --title "Name" --no-newline --informative-text "Base name for mp3s:" --text '$filled_in_name'`;
  my $button;
  ($button, $project_name) = split /\n/, $dialog_input, 2;
  $button == 1 or exit;
  $project_name =~ s/\s/_/g;
  $project_name =~ s/[^\w_-]//g;
  if (not($project_name =~ /\w/)) {
    error_box('Name must contain one letter or number');
    $project_name = '';
  }
}

sub init_project_dir{
  my ($project_name) = @_;

  if (mkdir "$config{local}/$project_name") {
    mkdir "$config{local}/$project_name/$_" or error_bye("Failed to create folder", "$config{local}/$project_name/$_") foreach qw(audio csv originals etc);
    return 1;
  } else {
    if ($! =~ /file exists/i) {
      error_box("Name '$project_name' taken, please choose another", "Folder already exists at $config{local}/$project_name");
      return 0;
    } else {
      error_bye("Failed to create folder", "$config{local}/$project_name: $!");
    }
  }
}

#Get optional subtitle, write to etc/
if ($subtitle) {
  write_subtitle($subtitle);
} elsif (not $name_via_command_line) {
  my $dialog_input =  `$Dialoger standard-inputbox --title "Subtitle" --no-newline --informative-text "Subtitle to appear at top of $project_name transcription. Date, meeting location or other notes. Optional."`;
  my $button;
  ($button, $subtitle) = split /\n/, $dialog_input, 2;
  if ($subtitle && $button == 1) {
    write_subtitle($subtitle)
  }
}

sub write_subtitle{
  my ($subtitle) = @_;
  return if not defined $subtitle;
  open(my $fh, '>', "$config{local}/$project_name/etc/subtitle.txt") or error_bye("Error writing subtitle", "$config{local}/$project_name/etc/subtitle.txt: $!");
  print $fh $subtitle;
  close $fh;
}

#Copy audio files to 'originals' folder
foreach my $file (@files){
    my $base = filepath_base($file);
    my $new_path = "$config{local}/$project_name/originals/$base";
    copy($file, $new_path) or error_bye ("Could not copy original audio file", "Could not move $file to $new_path: $!");
    $file = $new_path;
}

#Convert everything to MP3
my %temp_dir;
foreach my $file (@files) {
  my ($pathbase, $ext) = filebase_split($file);
  if ((lc $ext) ne 'mp3') {
    my $file_display_name = filepath_base($file);
    print "Determining original bitrate for $file_display_name\n";
    my $bitrate = file_bitrate($file) || 192;
    $bitrate =~ /^\d+$/ or error_bye("Unexpected bitrate format", $bitrate);
    print "Converting $file_display_name to mp3 at $bitrate kbps\n";
    $temp_dir{convert} ||= tempdir();
    my $temp_file;
    {
      no warnings;
      (undef, $temp_file) = tempfile('audibleturkXXXXXX', SUFFIX => '.mp3', DIR => $temp_dir{convert}, OPEN => 0); 
    }
    shell_safe($temp_file) or error_bye("Unsafe tempfile name", $temp_file);
    system("ffmpeg -i $file -acodec libmp3lame -ab ${bitrate}k -ac 2 '$temp_file' 2> /dev/null") == 0 or error_bye("Could not convert the file $file", "$? / $!");
    $file = $temp_file;
  }
}

#Merge files together
my $wrap_file_base =  "$config{local}/$project_name/audio/$project_name";
my $wrap_file = $wrap_file_base . '.all.mp3'; 
if (@files > 1) {
    print "Merging audio\n";<commit_msg>Fixes unquoted file name in convert step in assign.pl<commit_after>#!/opt/local/bin/perl

##TODO
#Verify all URLs to make sure there are no 404s
#Add random string to filenames to make URLs unguessable

use strict;
use warnings;

use File::Temp qw(tempfile tempdir);
use File::Path 'remove_tree';
use File::Copy;
use File::Spec;
use POSIX qw(floor);
use Text::CSV;
use String::Random;
use YAML ();
use Getopt::Long;

my $Dialoger = '/Applications/CocoaDialog.app/Contents/MacOS/CocoaDialog';
Getopt::Long::Configure(qw(bundling ignore_case));

my (%opt, @files, $subtitle, $time, $project_name);
GetOptions(
	   'help' => \$opt{help},
	   'file=s' => \@files,
	   'title=s' => \$project_name,
	   'chunks=s' => \$time,
	   'subtitle=s' => \$subtitle,
	  );
my $usage = "USAGE: assign.pl --file foo.mp3 [--file bar.mp3...] --title Foo --chunks 1:00 [--subtitle 'Foo telephone interview']\n";
die $usage if $opt{help};
my $name_via_command_line;
$name_via_command_line = 1 if $project_name;

@files = @ARGV if not @files;
@files or error_bye("Drag audio files onto the icon");

#Need files in proper order before joining them
@files = sort { filepath_sortable($a) cmp filepath_sortable($b) } @files;
#Need file extensions so we know what files need to be converted to mp3
filebase_split($_) or error_bye("You need a file extension on the file '$_'") foreach @files;
#Escape file names so they can safely be passed to shell within single quotes 
shell_safe($_) or error_bye("Unsafe file name", $_) foreach @files;

my %config = config_file();
$config{$_} or error_bye("Required param '$_' missing from config file", "$ENV{HOME}/.audibleturk") foreach qw(scp url);
#my @fails = $config{scp} =~ /[^\w\s_\@\/\.\:\-]+/g;
#print "DEBUG fail $_\n" foreach @fails;
$config{scp} =~ /^[\w\s_\@\/\.\:\+\-]+$/ or error_bye("Unexpected scp format in config file", "$ENV{HOME}/.audibleturk");
if ($config{local}) {
  $config{local} =~ s/^~\/?/$ENV{HOME}/;
  $config{local} = "$ENV{HOME}/$config{local}" if $config{local} =~ /^[^\/]/;
  $config{local} =~ s/\/$//;
}
$config{local} ||= "$ENV{HOME}/Desktop";

foreach (qw(scp url local)){
    $config{$_} =~ s/\s+$//;
    $config{$_} =~ s/\/$//;
}

shell_safe($config{local}) or error_bye("Unsafe dir name", $config{local});
if (exists $config{randomize}){
    $config{randomize} = 0 if $config{randomize} =~ /\bfalse\b/i;
    $config{randomize} = 0 if $config{randomize} =~ /\bno\b/i;
    $config{randomize} = 0 if $config{randomize} =~ /\b0\b/;
    $config{randomize} = 0 if not defined $config{randomize}
}
else {
    $config{randomize} = 1;
}

#Get split interval
my $default_interval = $config{last_split_interval} || $config{default_split_interval} || '1:00';
$default_interval =~ /^[\d\:\s\.]+$/ or error_bye("Unexpected format to default split interval '$default_interval'");
my $min_dot_seconds;
$min_dot_seconds = to_min_seconds($time) if $time;
until ($min_dot_seconds) {
  my $dialog_input = `$Dialoger standard-inputbox --title "Length" --no-newline --informative-text "Split audio every (hh:mm:ss)" --text '$default_interval'`;
  my ($button, $time) = split /\n/, $dialog_input, 2;
  $button == 1 or exit;
  $min_dot_seconds = to_min_seconds($time) or error_box("Did not understand the time '$time'");
  config_file('last_split_interval' => $time) if $min_dot_seconds;
}


#Get base name for mp3s we'll be outputting
my ($default_name) = filebase_split(filepath_base($files[0]));
$default_name =~ s/(\w+)(\.)?\d+(\.mono)?$/$1/;
until ($project_name && init_project_dir($project_name)) {
  my $filled_in_name = $project_name || $default_name;
  shell_safe($filled_in_name) or error_bye("Potentially unsafe name encountered: $filled_in_name");
  my $dialog_input = `$Dialoger standard-inputbox --title "Name" --no-newline --informative-text "Base name for mp3s:" --text '$filled_in_name'`;
  my $button;
  ($button, $project_name) = split /\n/, $dialog_input, 2;
  $button == 1 or exit;
  $project_name =~ s/\s/_/g;
  $project_name =~ s/[^\w_-]//g;
  if (not($project_name =~ /\w/)) {
    error_box('Name must contain one letter or number');
    $project_name = '';
  }
}

sub init_project_dir{
  my ($project_name) = @_;

  if (mkdir "$config{local}/$project_name") {
    mkdir "$config{local}/$project_name/$_" or error_bye("Failed to create folder", "$config{local}/$project_name/$_") foreach qw(audio csv originals etc);
    return 1;
  } else {
    if ($! =~ /file exists/i) {
      error_box("Name '$project_name' taken, please choose another", "Folder already exists at $config{local}/$project_name");
      return 0;
    } else {
      error_bye("Failed to create folder", "$config{local}/$project_name: $!");
    }
  }
}

#Get optional subtitle, write to etc/
if ($subtitle) {
  write_subtitle($subtitle);
} elsif (not $name_via_command_line) {
  my $dialog_input =  `$Dialoger standard-inputbox --title "Subtitle" --no-newline --informative-text "Subtitle to appear at top of $project_name transcription. Date, meeting location or other notes. Optional."`;
  my $button;
  ($button, $subtitle) = split /\n/, $dialog_input, 2;
  if ($subtitle && $button == 1) {
    write_subtitle($subtitle)
  }
}

sub write_subtitle{
  my ($subtitle) = @_;
  return if not defined $subtitle;
  open(my $fh, '>', "$config{local}/$project_name/etc/subtitle.txt") or error_bye("Error writing subtitle", "$config{local}/$project_name/etc/subtitle.txt: $!");
  print $fh $subtitle;
  close $fh;
}

#Copy audio files to 'originals' folder
foreach my $file (@files){
    my $base = filepath_base($file);
    my $new_path = "$config{local}/$project_name/originals/$base";
    copy($file, $new_path) or error_bye ("Could not copy original audio file", "Could not move $file to $new_path: $!");
    $file = $new_path;
}

#Convert everything to MP3
my %temp_dir;
foreach my $file (@files) {
  my ($pathbase, $ext) = filebase_split($file);
  if ((lc $ext) ne 'mp3') {
    my $file_display_name = filepath_base($file);
    print "Determining original bitrate for $file_display_name\n";
    my $bitrate = file_bitrate($file) || 192;
    $bitrate =~ /^\d+$/ or error_bye("Unexpected bitrate format", $bitrate);
    print "Converting $file_display_name to mp3 at $bitrate kbps\n";
    $temp_dir{convert} ||= tempdir();
    my $temp_file;
    {
      no warnings;
      (undef, $temp_file) = tempfile('audibleturkXXXXXX', SUFFIX => '.mp3', DIR => $temp_dir{convert}, OPEN => 0); 
    }
    shell_safe($temp_file) or error_bye("Unsafe tempfile name", $temp_file);
    system("ffmpeg -i '$file' -acodec libmp3lame -ab ${bitrate}k -ac 2 '$temp_file' 2> /dev/null") == 0 or error_bye("Could not convert the file $file", "$? / $!");
    $file = $temp_file;
  }
}

#Merge files together
my $wrap_file_base =  "$config{local}/$project_name/audio/$project_name";
my $wrap_file = $wrap_file_base . '.all.mp3'; 
if (@files > 1) {
    print "Merging audio\n";
{'fraction_non_alphanumeric': 0.17898349642179057, 'fraction_numerical': 0.006572221410836863, 'mean_word_length': 4.3959810874704495, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2168129', 'n_tokens_mistral': 5055, 'n_tokens_neox': 4894, 'n_words': 1295}