<commit_before>#!/usr/bin/env perl

use warnings;
use strict;

use Data::Dumper;

open my $fh, '<', 'download/pci.ids';

my @vendors = ();
my %queries = ();
my %key = (
    0 => sub {
        s/^(....)\s+//; chomp;
        push @vendors, {
            "id" => $1,
            "name" => $_,
            "devices" => [] }
    },
    1 => sub {
        s/^(....)\s+//; chomp;
        push $vendors[-1]{devices}, {
            "id" => $1,
            "name" => $_,
            "subdevices" => [] }
    },
    2 => sub {
        s/^(....)\ (....)\s+//; chomp;
        return if not $2;
        push $vendors[-1]{devices}[-1]{subdevices}, {
            "subvendor" => $1,
            "subdevice" => $2,
            "subsystem_name" => $_ };
    },
);
while (<$fh>) {
    next if m/^#|^\s+$/;
    $key{length $1}->(s/^(\t{0,2})//);
}
my $i = 0;
foreach my $vendor (@vendors) {
    print ++$i . "\n";
#    print "$vendor->{id} $vendor->{name}\n";
    $queries{$vendor->{id}} = {
        "abstract" =>"<i>Vendor</i>: $vendor->{name}",
        "category" => "<i>PCI vendor ID</i>: $vendor->{id}"
    };
    if (scalar @{$vendor->{devices}} > 0) {
        foreach my $device (@{$vendor->{devices}}) {
#            print "$vendor->{id} $device->{id}"
#                . "$device->{name} $vendor->{name}\n";
            $queries{"$vendor->{id} $device->{id}"} = {
                "abstract" => "<i>Vendor</i>: $vendor->{name}<br>"
                            . "<i>Device</i>: $device->{name}",
                "category" => "PCI vendor ID $vendor->{id}"
            };
            if (scalar @{$device->{subdevices}} > 0) {
                foreach my $subdevice (@{$device->{subdevices}}) {
#                    print "$vendor->{id} $device->{id} "
#                        . "$subdevice->{subvendor} "
#                        . "$subdevice->{subdevice} "
#                        . "$vendor->{name} $device->{name} "
#                        . "$subdevice->{subsystem_name}\n";
                    $queries{"$vendor->{id} $device->{id}"
                            . "$subdevice->{subvendor} "
                            . "$subdevice->{subdevice}"} = {
                                "abstract" => "<i>Vendor</i>:"
                                    . "$vendor->{name}<br>"
                                    . "<i>Device</i>: $device->{name}<br>"
                                    . "<i>Subdevice/subvendor</i>: "
                                    . "$subdevice->{subsystem_name}",
                                "category" => "PCI vendor ID "
                                             . "$vendor->{id}"
                    };
                }
            }
        }
    }
}

open my $output, '>', 'output.txt';
map {
    print $output join "\t", (
        $_,                                 # title
        "A",                                # type
        "",                                 # redirect
        "",                                 # otheruses
        $queries{$_}->{category},           # categories
        "",                                 # references
        "",                                 # see_also
        "",                                 # further_reading
        "",                                 # external_links
        "",                                 # disambiguation
        "",                                 # images<commit_msg>removed debugging commands<commit_after>#!/usr/bin/env perl

use warnings;
use strict;

use Data::Dumper;

open my $fh, '<', 'download/pci.ids';

my @vendors = ();
my %queries = ();
my %key = (
    0 => sub {
        s/^(....)\s+//; chomp;
        push @vendors, {
            "id" => $1,
            "name" => $_,
            "devices" => [] }
    },
    1 => sub {
        s/^(....)\s+//; chomp;
        push $vendors[-1]{devices}, {
            "id" => $1,
            "name" => $_,
            "subdevices" => [] }
    },
    2 => sub {
        s/^(....)\ (....)\s+//; chomp;
        return if not $2;
        push $vendors[-1]{devices}[-1]{subdevices}, {
            "subvendor" => $1,
            "subdevice" => $2,
            "subsystem_name" => $_ };
    },
);
while (<$fh>) {
    next if m/^#|^\s+$/;
    $key{length $1}->(s/^(\t{0,2})//);
}
foreach my $vendor (@vendors) {
    $queries{$vendor->{id}} = {
        "abstract" =>"<i>Vendor</i>: $vendor->{name}",
        "category" => "<i>PCI vendor ID</i>: $vendor->{id}"
    };
    if (scalar @{$vendor->{devices}} > 0) {
        foreach my $device (@{$vendor->{devices}}) {
            $queries{"$vendor->{id} $device->{id}"} = {
                "abstract" => "<i>Vendor</i>: $vendor->{name}<br>"
                            . "<i>Device</i>: $device->{name}",
                "category" => "PCI vendor ID $vendor->{id}"
            };
            if (scalar @{$device->{subdevices}} > 0) {
                foreach my $subdevice (@{$device->{subdevices}}) {
                    $queries{"$vendor->{id} $device->{id}"
                            . "$subdevice->{subvendor} "
                            . "$subdevice->{subdevice}"} = {
                                "abstract" => "<i>Vendor</i>:"
                                    . "$vendor->{name}<br>"
                                    . "<i>Device</i>: $device->{name}<br>"
                                    . "<i>Subdevice/subvendor</i>: "
                                    . "$subdevice->{subsystem_name}",
                                "category" => "PCI vendor ID "
                                             . "$vendor->{id}"
                    };
                }
            }
        }
    }
}

open my $output, '>', 'output.txt';
map {
    print $output join "\t", (
        $_,                                 # title
        "A",                                # type
        "",                                 # redirect
        "",                                 # otheruses
        exists $queries{$_}->{category} ?
            $queries{$_}->{category} : "",  # categories
        "",                                 # references
        "",                                 # see_also
        "",                                 # further_reading
        "",                                 # external_links
        "",                                 # disambiguation
        "",                                 # images
{'fraction_non_alphanumeric': 0.19660003148118999, 'fraction_numerical': 0.005194396348181961, 'mean_word_length': 0.9968573224387178, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 139, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 54, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6226264', 'n_tokens_mistral': 1862, 'n_tokens_neox': 1816, 'n_words': 305}