<commit_before>package LtxMojo;
use Mojo::Base 'Mojolicious';
use Mojo::JSON;
use Mojo::IOLoop;
use Mojo::ByteStream qw(b);

use File::Basename 'dirname';
use File::Spec::Functions qw(catdir catfile);
use File::Temp qw(tempdir);
use File::Path qw(remove_tree);

use Archive::Zip qw(:CONSTANTS :ERROR_CODES);
use IO::String;
use Encode;

use LaTeXML::Util::Config;
use LaTeXML::Util::Pathname qw(pathname_is_literaldata pathname_is_url);
use LaTeXML::Converter;
use LtxMojo::Startup;

our $dbfile  = '.LaTeXML_Mojo.cache';

# Every CPAN module needs a version
our $VERSION = '0.3';

# This method will run once at server start
sub startup {
my $app = shift;
# Switch to installable home directory
$app->home->parse(catdir(dirname(__FILE__), 'LtxMojo'));

# Switch to installable "public" directory
$app->static->paths->[0] = $app->home->rel_dir('public');
# Switch to installable "templates" directory
$app->renderer->paths->[0] = $app->home->rel_dir('templates');

$ENV{MOJO_MAX_MESSAGE_SIZE} = 107374182; # close to 100 MB file upload limit
$ENV{MOJO_REQUEST_TIMEOUT} = 600;# 10 minutes;
$ENV{MOJO_CONNECT_TIMEOUT} = 120; # 2 minutes
$ENV{MOJO_INACTIVITY_TIMEOUT} = 600; # 10 minutes;

# Make signed cookies secure
$app->secret('LaTeXML is the way to go!');

#Prep a LaTeXML Startup instance
my $startup = LtxMojo::Startup->new(dbfile => catfile($app->home,$dbfile));

# Do a one-time check for admin, add if none:
$startup->modify_user('admin', 'admin', 'admin')
  unless $startup->exists_user('admin');

$app->helper(convert_zip => sub {
  my ($self) = @_;
  my $source_dir = tempdir();
  my $destination_dir = tempdir();

  # Make sure we point to the actual source directory
  my $name = $self->req->headers->header('x-file-name');
  $name =~ s/\.zip$//;
  #.zip (can't support others for now)
  my $content_handle = IO::String->new($self->req->body);
  my $zip = Archive::Zip->new();
  $self->render(text=>"Archive is corrupt!") unless
    ($zip->readFromFileHandle( $content_handle ) == AZ_OK);
  $zip->extractTree($name,$source_dir);
  # HTTP GET parameters hold the conversion options
  my @all_params = @{ $self->req->url->query->params || [] };
  my $opts=[];
  # Ugh, disallow 'null' as a value!!! (TODO: Smarter fix??)
  while (my ($key,$value) = splice(@all_params,0,2)) {
    if ($key=~/local|path|destination|directory/) {
      # You don't get to specify harddrive info in the web service
      next; }
    $value = '' if ($value && ($value  eq 'null'));
    push @$opts, ($key,$value); }

  my $ext = 'html';
  my $destination = catfile($destination_dir,"$name.$ext");
  my $config = LaTeXML::Util::Config->new();
  $config->read_keyvals($opts);
  my @latexml_inputs = grep {defined} split(':',($ENV{LATEXMLINPUTS}||''));
  $config->set('paths',[$source_dir,@latexml_inputs]);
  $config->set('sourcedirectory',$source_dir);
  $config->set('sitedirectory',$destination_dir);
  $config->set('destination',$destination);
  $config->set('local',($self->tx->remote_address eq '127.0.0.1'));
  # Only HTML5 for now.
  $config->set('format','html5');
  # Prepare converter
  my $converter = LaTeXML::Converter->get_converter($config);
  $converter->prepare_session($config);
  #Send a request:
  my $response = $converter->convert(catfile($source_dir,"$name.tex"));
  if (defined $response) {
      my ($result, $status, $status_code, $log) = map { $response->{$_} } qw(result status status_code log);
      if ($result) {
        open(OUT,">",$destination) or $self->render(text=>"Couldn't open output file ".$destination.": $!");
        print OUT $result;
        close OUT;
      } else {
        # Delete converter if Fatal occurred
        undef $converter;
      }
      if ($log) {
        my $logfile = $config->get('log');
        if (!$logfile) {
          my $ext = 'log';
          $logfile = catfile($destination_dir,"$name.$ext");
        }
        open(OUT,">",$logfile) or $self->render(text=>"Couldn't open log file ".$logfile.": $!");
        print OUT $log;
        close OUT;
      }
    }
  # Zip and send back
  my $returnzip = Archive::Zip->new();
  my $payload='';
  $returnzip->addTree($destination_dir,$name);
  $content_handle = IO::String->new($payload);
  $self->render(text=>'final Archive creation failed') unless ($returnzip->writeToFileHandle( $content_handle ) == AZ_OK);
  my $headers = Mojo::Headers->new;
  $headers->add('Content-Type',"application/zip;name=$name.zip");
  $headers->add('Content-Disposition',"attachment;filename=$name.zip");
  $self->res->content->headers($headers);
  # Cleanup before returning
  remove_tree($source_dir,$destination_dir);
  # Return
  return $self->render(data=>$payload);
});

  # TODO: Maybe reintegrate IF we support username-based profiles
  # if (!defined $opt->{profile}) {
  #   if (defined $opt->{user}
  #     && $startup->verify_user($opt->{user}, $opt->{password}))
  #   {
  #     $opt->{profile} =
  #       $startup->lookup_user_property($opt->{user}, 'default') || 'custom';
  #   }
  #   else {
  #     $opt->{profile} = 'custom';
  #   }
  # }

$app->helper(convert_string => sub {
  my ($self) = @_;  
  my ($source,$is_jsonp);
  my $get_params = $self->req->url->query->params || [];
  my $post_params = $self->req->body_params->params || [];
  if (scalar(@$post_params) == 1) {
    $source = $post_params->[0];
    $post_params=[];
  } elsif ((scalar(@$post_params) == 2) && ($post_params->[0] !~ /^tex|source$/)) {
    $source = $post_params->[0].$post_params->[1];<commit_msg> updating ZIP workflow to using the built in Pack post-processor<commit_after>package LtxMojo;
use Mojo::Base 'Mojolicious';
use Mojo::JSON;
use Mojo::IOLoop;
use Mojo::ByteStream qw(b);

use File::Basename 'dirname';
use File::Spec::Functions qw(catdir catfile);
use File::Temp qw(tempdir);
use File::Path qw(remove_tree);

use Archive::Zip qw(:CONSTANTS :ERROR_CODES);
use IO::String;
use Encode;

use LaTeXML::Util::Config;
use LaTeXML::Util::Pathname qw(pathname_is_literaldata pathname_is_url);
use LaTeXML::Converter;
use LtxMojo::Startup;

our $dbfile  = '.LaTeXML_Mojo.cache';

# Every CPAN module needs a version
our $VERSION = '0.3';

# This method will run once at server start
sub startup {
my $app = shift;
# Switch to installable home directory
$app->home->parse(catdir(dirname(__FILE__), 'LtxMojo'));

# Switch to installable "public" directory
$app->static->paths->[0] = $app->home->rel_dir('public');
# Switch to installable "templates" directory
$app->renderer->paths->[0] = $app->home->rel_dir('templates');

$ENV{MOJO_MAX_MESSAGE_SIZE} = 107374182; # close to 100 MB file upload limit
$ENV{MOJO_REQUEST_TIMEOUT} = 600;# 10 minutes;
$ENV{MOJO_CONNECT_TIMEOUT} = 120; # 2 minutes
$ENV{MOJO_INACTIVITY_TIMEOUT} = 600; # 10 minutes;

# Make signed cookies secure
$app->secret('LaTeXML is the way to go!');

#Prep a LaTeXML Startup instance
my $startup = LtxMojo::Startup->new(dbfile => catfile($app->home,$dbfile));

# Do a one-time check for admin, add if none:
$startup->modify_user('admin', 'admin', 'admin')
  unless $startup->exists_user('admin');

$app->helper(convert_zip => sub {
  my ($self) = @_;
  my $source_dir = tempdir();
  my $destination_dir = tempdir();

  # Make sure we point to the actual source directory
  my $name = $self->req->headers->header('x-file-name');
  $name =~ s/\.zip$//;
  #.zip (can't support others for now)
  my $content_handle = IO::String->new($self->req->body);
  my $zip = Archive::Zip->new();
  $self->render(text=>"Archive is corrupt!") unless
    ($zip->readFromFileHandle( $content_handle ) == AZ_OK);
  $zip->extractTree($name,$source_dir);
  # HTTP GET parameters hold the conversion options
  my @all_params = @{ $self->req->url->query->params || [] };
  my $opts=[];
  # Ugh, disallow 'null' as a value!!! (TODO: Smarter fix??)
  while (my ($key,$value) = splice(@all_params,0,2)) {
    if ($key=~/local|path|destination|directory/) {
      # You don't get to specify harddrive info in the web service
      next; }
    $value = '' if ($value && ($value  eq 'null'));
    push @$opts, ($key,$value); }

  my $ext = 'zip';
  my $destination = catfile($destination_dir,"$name.$ext");
  my $config = LaTeXML::Util::Config->new();
  $config->read_keyvals($opts);
  my @latexml_inputs = grep {defined} split(':',($ENV{LATEXMLINPUTS}||''));
  $config->set('paths',[$source_dir,@latexml_inputs]);
  $config->set('sourcedirectory',$source_dir);
  $config->set('sitedirectory',$destination_dir);
  $config->set('destination',$destination);
  $config->set('whatsout','archive');
  $config->set('local',($self->tx->remote_address eq '127.0.0.1'));
  # Only HTML5 for now.
  $config->set('format','html5');
  # Prepare converter
  my $converter = LaTeXML::Converter->get_converter($config);
  $converter->prepare_session($config);
  #Send a request:
  my $response = $converter->convert(catfile($source_dir,"$name.tex"));

  $self->render(text=>'Fatal: Internal Conversion Error, please contact the administrator.') unless
    (defined $response && ($response->{result}));
  my $headers = Mojo::Headers->new;
  $headers->add('Content-Type',"application/zip;name=$name.zip");
  $headers->add('Content-Disposition',"attachment;filename=$name.zip");
  $self->res->content->headers($headers);
  # Cleanup before returning
  remove_tree($source_dir,$destination_dir);
  # Return
  return $self->render(data=>$response->{result});
});

  # TODO: Maybe reintegrate IF we support username-based profiles
  # if (!defined $opt->{profile}) {
  #   if (defined $opt->{user}
  #     && $startup->verify_user($opt->{user}, $opt->{password}))
  #   {
  #     $opt->{profile} =
  #       $startup->lookup_user_property($opt->{user}, 'default') || 'custom';
  #   }
  #   else {
  #     $opt->{profile} = 'custom';
  #   }
  # }

$app->helper(convert_string => sub {
  my ($self) = @_;  
  my ($source,$is_jsonp);
  my $get_params = $self->req->url->query->params || [];
  my $post_params = $self->req->body_params->params || [];
  if (scalar(@$post_params) == 1) {
    $source = $post_params->[0];
    $post_params=[];
  } elsif ((scalar(@$post_params) == 2) && ($post_params->[0] !~ /^tex|source$/)) {
    $source = $post_params->[0].$post_params->[1];
{'fraction_non_alphanumeric': 0.19347548393436179, 'fraction_numerical': 0.00903999213913727, 'mean_word_length': 4.604625550660793, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 178, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3571483', 'n_tokens_mistral': 3618, 'n_tokens_neox': 3519, 'n_words': 772}