<commit_before>#!/usr/bin/perl
#
# Align the MS-DOS stub and NE header to sit on 512-byte blocks,
# as observed on Windows 1.x/2.x executables. This means making
# the MS-DOS header 512 bytes long as well.
# (C) 2012 <NAME>

if (@ARGV < 1) {
    print STDERR "win2xalign512.pl <NE image>\n";
    exit 1;
}

my $ne_file = shift @ARGV;
die unless -f $ne_file;

my $tmp;
my $ne_offset = 0;
my $ne_file_tmp = $ne_file.".tmp";

open(NE,"<",$ne_file) || die;
binmode(NE);
seek(NE,0,0);
read(NE,$tmp,2);

if ($tmp ne "MZ") {
    print "Not MS-DOS executable\n";
    exit 1;
}

seek(NE,0x3C,0);
read(NE,$tmp,4);
$ne_offset = unpack("l",$tmp);
print "Extended header at $ne_offset\n";

seek(NE,$ne_offset,0);
read(NE,$tmp,2);
if ($tmp ne "NE") {
    print "Not an NE image\n";
    exit 1;
}

print "NE header at $ne_offset\n";
if ($ne_offset == 1024) { # NTS: Shortcut
    print "Nothing to do\n";
    exit 1;
}

# we copy the contents this time, to keep the code simple
open(NETMP,">$ne_file_tmp") || die;
binmode(NETMP);

# copy the MS-DOS header. Assume at least 0x40 bytes. Patch NE offset field to set it to 1024.
seek(NE,0,0);
seek(NETMP,0,0);
read(NE,$tmp,0x40);
$tmp = substr($tmp,0,0x3C).pack("V",1024);
print NETMP $tmp;

# how big is the original MS-DOS header?
($msdos_last,$msdos_blocks,$relocs,$hdr_para) = unpack("vvvv",substr($tmp,2,8));
$hdr_para *= 16;
print "MS-DOS header: blocks=$msdos_blocks last=$msdos_last header=$hdr_para\n";
$msdos_size = $msdos_blocks * 512;
$msdos_size += $msdos_last - 512 if $msdos_last != 0;
print "..size: $msdos_size\n";
$msdos_size = $ne_offset if $msdos_size > $ne_offset;
print "..stub: $msdos_size\n";
die if $msdos_size <= $hdr_para;
$msdos_size -= $hdr_para;
print "..stub without hdr: $msdos_size\n";
die "original header too big" if $hdr_para > (512-0x40);
die "Relocations not supported" if $relocs != 0;
# finish padding header
print NETMP chr(0) x (512 - 0x40);

# now copy the MS-DOS stub
seek(NE,$hdr_para,0);
read(NE,$tmp,$msdos_size);
print NETMP $tmp;
die "stub too big" if $msdos_size > 512;
# and pad
print NETMP chr(0) x (512 - $msdos_size);

# then the rest of the file.
# NTS: Noted that Windows 1.x/2.x binaries use alignment=16.
read(NE,$tmp,0x1000000);

$shift_align = unpack("v",substr($tmp,0x32,2));
die if $shift_align > 512;
print "NE alignment: ".(1 << $shift_align)."\n";

# locate segment table (we're going to patch it).
# you determine where segment table ends by resource table offset.
$segment_offset = unpack("v",substr($tmp,0x22,2));
print "Segment table at: $segment_offset + $ne_offset\n";
$resource_table_offset = unpack("v",substr($tmp,0x24,2));<commit_msg>minor correction in win2xalign512 script<commit_after>#!/usr/bin/perl
#
# Align the MS-DOS stub and NE header to sit on 512-byte blocks,
# as observed on Windows 1.x/2.x executables. This means making
# the MS-DOS header 512 bytes long as well.
# (C) 2012 <NAME>

if (@ARGV < 1) {
    print STDERR "win2xalign512.pl <NE image>\n";
    exit 1;
}

my $ne_file = shift @ARGV;
die unless -f $ne_file;

my $tmp;
my $ne_offset = 0;
my $ne_file_tmp = $ne_file.".tmp";

open(NE,"<",$ne_file) || die;
binmode(NE);
seek(NE,0,0);
read(NE,$tmp,2);

if ($tmp ne "MZ") {
    print "Not MS-DOS executable\n";
    exit 1;
}

seek(NE,0x3C,0);
read(NE,$tmp,4);
$ne_offset = unpack("l",$tmp);
print "Extended header at $ne_offset\n";

seek(NE,$ne_offset,0);
read(NE,$tmp,2);
if ($tmp ne "NE") {
    print "Not an NE image\n";
    exit 1;
}

print "NE header at $ne_offset\n";
if ($ne_offset == 1024) { # NTS: Shortcut
    print "Nothing to do\n";
    exit 1;
}

# we copy the contents this time, to keep the code simple
open(NETMP,">$ne_file_tmp") || die;
binmode(NETMP);

# copy the MS-DOS header. Assume at least 0x40 bytes. Patch NE offset field to set it to 1024.
seek(NE,0,0);
seek(NETMP,0,0);
read(NE,$tmp,0x40);
$tmp = substr($tmp,0,0x3C).pack("V",1024);
print NETMP $tmp;

# how big is the original MS-DOS header?
($msdos_last,$msdos_blocks,$relocs,$hdr_para) = unpack("vvvv",substr($tmp,2,8));
$hdr_para *= 16;
print "MS-DOS header: blocks=$msdos_blocks last=$msdos_last header=$hdr_para\n";
$msdos_size = $msdos_blocks * 512;
$msdos_size += $msdos_last - 512 if $msdos_last != 0;
print "..size: $msdos_size\n";
$msdos_size = $ne_offset if $msdos_size > $ne_offset;
print "..stub: $msdos_size\n";
die if $msdos_size <= $hdr_para;
$msdos_size -= $hdr_para;
print "..stub without hdr: $msdos_size\n";
die "original header too big" if $hdr_para > (512-0x40);
die "Relocations not supported" if $relocs != 0;
# finish padding header
print NETMP chr(0) x (512 - 0x40);

# now copy the MS-DOS stub
seek(NE,$hdr_para,0);
read(NE,$tmp,$msdos_size);
print NETMP $tmp;
die "stub too big" if $msdos_size > 512;
# and pad
print NETMP chr(0) x (512 - $msdos_size);

# then the rest of the file.
# NTS: Noted that Windows 1.x/2.x binaries use alignment=16.
read(NE,$tmp,0x1000000);

$shift_align = unpack("v",substr($tmp,0x32,2));
die if $shift_align > 9;
print "NE alignment: ".(1 << $shift_align)."\n";

# locate segment table (we're going to patch it).
# you determine where segment table ends by resource table offset.
$segment_offset = unpack("v",substr($tmp,0x22,2));
print "Segment table at: $segment_offset + $ne_offset\n";
$resource_table_offset = unpack("v",substr($tmp,0x24,2));
{'fraction_non_alphanumeric': 0.1696512723845429, 'fraction_numerical': 0.04448633364750235, 'mean_word_length': 4.736216216216216, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5862500', 'n_tokens_mistral': 2304, 'n_tokens_neox': 2102, 'n_words': 609}