<commit_before>#!/usr/bin/perl
# 2009/Mar/01 @ <NAME>
# Desc: add new version of Slackware into DB and everything 
# that belongs to it.
use lib "/srv/httpd/search.slackware.eu/perl/";
use Slackware::Search::ConfigParser qw(_getConfig);
use Slackware::Search::SupportLib qw(:T1);

use DBI;
use strict;
use warnings;

use constant CFGFILE => '/srv/httpd/search.slackware.eu/conf/config.pl';

my $cfgParser = 'Slackware::Search::ConfigParser';
my %CFG = $cfgParser->_getConfig(CFGFILE);

unless (%CFG || keys(%CFG)) {
	printf("Parsing of config file has failed.\n");
	exit 2;
}

my $dbh = DBI->connect($CFG{DB_DSN},
$CFG{DB_USER},
$CFG{DB_PASS},
	{
		AutoCommit => 0, 
		RaiseError => 1, 
		PrintError => 1
	}
);

die ("Unable to connect to DB.") unless ($dbh);

### MAIN ###
my $numArgs = $#ARGV + 1;

if ($numArgs == 0) {
	printf("Parameter must be Slackware version.\n");
	exit 1;
}

if ($ARGV[0] !~ /^slackware(64)?-([0-9]+\.[0-9]+|current){1}$/i) {
	printf("Parameter '%s' doesn't look like Slackware version to me.\n",
		$ARGV[0]);
	exit 1;
}

my @filesReq = qw(CHECKSUMS.md5 CHECKSUMS.md5.files 
CHECKSUMS.md5.pkgs FILELIST.TXT FILELIST.TXT.files 
FILELIST.TXT.md5 FILELIST.TXT.pkgs);
for my $fileReq (@filesReq) {
	unless ( -e "./$fileReq" ) {
		printf("File '%s' doesn't seem to exist.\n", $fileReq);
		exit 1;
	} # unless -e $fileReq
} # for my $fileReq

my $slib = 'Slackware::Search::SupportLib';
$slib->_set_dbHandler($dbh);

my $sverExists = $slib->existsSlackVer($ARGV[0]);
if ($sverExists == 1) {
	printf("This Slackware version is already in DB.\n");
	printf("Please use update script.\n");
	exit 1;
}
my $idSlackVer = 0;
$idSlackVer = $slib->insertSlackVer($ARGV[0]);
$slib->_set_sverName($ARGV[0]);

### PKGS ###
open(FPKGS, "./FILELIST.TXT.pkgs")
	or die("Unable to open FILELIST.TXT.pkgs");

printf("Processing packages...\n");
my $categoryLast = '';
my $idSerieLast = 0;
my $idCatLast = 0;
my $serieLast = '';
my $idSerie;
while (my $linePkg = <FPKGS>) {
	chomp($linePkg);
	my @arrLine = split(' ', $linePkg);
	unless ($arrLine[7]) {
		next;
	} # unless $arrLine[7]
# 4 - size; 5 - date; 6 - time; 7 pkgname
	if ($arrLine[7] !~ /^\.\// || $arrLine[7] !~ /\.(tgz|txz)$/) {
		next;
	} # if $arrLine[7]
	$arrLine[7] = substr($arrLine[7], 2);
	my @arrPath = split(/\//, $arrLine[7]);

	my $pkgName = pop(@arrPath);
	my $idPkg = $slib->insertPkg($pkgName);

	my $category = shift(@arrPath);
	my $idCategory = 0;
	if ($category eq $categoryLast) {
		$idCategory = $idCatLast;
	} else {
		$idCategory = $slib->insertCategory($category);
		$idCatLast = $idCategory;
		$categoryLast = $category;
	}

	my $serie = join("/", @arrPath);
#	for my $entry (@arrPath) {
#		$serie.= $entry;
#	}
	unless ($serie) {
		$idSerie = 'NULL';
	} else {
		if ($serie eq $serieLast) {
			$idSerie = $idSerieLast;
		} else {
			$idSerie = $slib->insertSerie($serie);
			$idSerieLast = $idSerie;
			$serieLast = $serie;
		}
	}

	my $sql1 = sprintf("INSERT INTO packages (id_slackversion, id_category,
		id_serie, id_package, package_size, package_created) VALUES (%i, %i, %i, 
		%i, %s, '%s %s');", $idSlackVer, $idCategory, $idSerie, $idPkg, 
		$arrLine[4], $arrLine[5], $arrLine[6]);
	$dbh->do($sql1)
		or die("Unable to insert package.");
} # while my $linePkg

close(FPKGS);

### PKGS MD5 ###
open(FPKGS5, "./CHECKSUMS.md5.pkgs")
	or die("Unable to open CHECKSUMS.md5.pkgs");

printf("Processing package's MD5s...\n");
while (my $linePkg5 = <FPKGS5>) {
	chomp($linePkg5);
	my @arrLine = split(' ', $linePkg5);
	if ($arrLine[1] !~ /^\.\// || $arrLine[1] !~ /\.(tgz|txz)$/) {
		next;
	} # unless $arrLine
	$arrLine[1] = substr($arrLine[1], 2);
	my @arrPath = split(/\//, $arrLine[1]);

	my $pkgName = pop(@arrPath);
	my $idPkg = $slib->getPkgId($pkgName);

	my $category = shift(@arrPath);
	my $idCategory = 0;
	if ($category eq $categoryLast) {
		$idCategory = $idCatLast;
	} else {
		$idCategory = $slib->getCategoryId($category);
		$idCatLast = $idCategory;
		$categoryLast = $category;
	}

	my $serie = '';
	for my $entry (@arrPath) {
		$serie.= $entry;
	}
	unless ($serie) {
		$idSerie = 'NULL';
	} else {
		if ($serie eq $serieLast) {
			$idSerie = $idSerieLast;
		} else {
			$idSerie = $slib->getSerieId($serie);
			$idSerieLast = $idSerie;
			$serieLast = $serie;
		}
	}

	my $sql1 = sprintf("UPDATE packages SET package_md5sum = '%s' WHERE 
		id_slackversion = %i AND id_category = %i AND id_serie = %i AND 
	id_package = %i;", $arrLine[0], $idSlackVer, $idCategory, $idSerie);
	$dbh->do($sql1)
		or die("Unable to update package's MD5 sum.");
} # while my $linePkg5

close(FPKGS5);

### SVER DATA FILES ###
open(DFILE, "./FILELIST.TXT.files")
	or die("Unable to open FILELIST.TXT.files.");

printf("Processing data files (pkg desc'n files)...\n");
printf("This is going to take a while...\n");

while (my $lineDF = <DFILE>) {
	chomp($lineDF);
	my @arrLine = split(' ', $lineDF);
	unless ($arrLine[7]) {
		next;
	}
	if ($arrLine[7] !~ /^\.\// || $arrLine[7] =~ /\.md5$/) {
		next;
	}
	if ($arrLine[7] =~ /\.txt$/i) {
		$slib->processPkgDesc($arrLine[7], $idSlackVer);
	}<commit_msg>FIX: db-slackv-update.pl ~ id_serie can be string<commit_after>#!/usr/bin/perl
# 2009/Mar/01 @ <NAME>
# Desc: add new version of Slackware into DB and everything 
# that belongs to it.
use lib "/srv/httpd/search.slackware.eu/perl/";
use Slackware::Search::ConfigParser qw(_getConfig);
use Slackware::Search::SupportLib qw(:T1);

use DBI;
use strict;
use warnings;

use constant CFGFILE => '/srv/httpd/search.slackware.eu/conf/config.pl';

my $cfgParser = 'Slackware::Search::ConfigParser';
my %CFG = $cfgParser->_getConfig(CFGFILE);

unless (%CFG || keys(%CFG)) {
	printf("Parsing of config file has failed.\n");
	exit 2;
}

my $dbh = DBI->connect($CFG{DB_DSN},
$CFG{DB_USER},
$CFG{DB_PASS},
	{
		AutoCommit => 0, 
		RaiseError => 1, 
		PrintError => 1
	}
);

die ("Unable to connect to DB.") unless ($dbh);

### MAIN ###
my $numArgs = $#ARGV + 1;

if ($numArgs == 0) {
	printf("Parameter must be Slackware version.\n");
	exit 1;
}

if ($ARGV[0] !~ /^slackware(64)?-([0-9]+\.[0-9]+|current){1}$/i) {
	printf("Parameter '%s' doesn't look like Slackware version to me.\n",
		$ARGV[0]);
	exit 1;
}

my @filesReq = qw(CHECKSUMS.md5 CHECKSUMS.md5.files 
CHECKSUMS.md5.pkgs FILELIST.TXT FILELIST.TXT.files 
FILELIST.TXT.md5 FILELIST.TXT.pkgs);
for my $fileReq (@filesReq) {
	unless ( -e "./$fileReq" ) {
		printf("File '%s' doesn't seem to exist.\n", $fileReq);
		exit 1;
	} # unless -e $fileReq
} # for my $fileReq

my $slib = 'Slackware::Search::SupportLib';
$slib->_set_dbHandler($dbh);

my $sverExists = $slib->existsSlackVer($ARGV[0]);
if ($sverExists == 1) {
	printf("This Slackware version is already in DB.\n");
	printf("Please use update script.\n");
	exit 1;
}
my $idSlackVer = 0;
$idSlackVer = $slib->insertSlackVer($ARGV[0]);
$slib->_set_sverName($ARGV[0]);

### PKGS ###
open(FPKGS, "./FILELIST.TXT.pkgs")
	or die("Unable to open FILELIST.TXT.pkgs");

printf("Processing packages...\n");
my $categoryLast = '';
my $idSerieLast = 0;
my $idCatLast = 0;
my $serieLast = '';
my $idSerie;
while (my $linePkg = <FPKGS>) {
	chomp($linePkg);
	my @arrLine = split(' ', $linePkg);
	unless ($arrLine[7]) {
		next;
	} # unless $arrLine[7]
# 4 - size; 5 - date; 6 - time; 7 pkgname
	if ($arrLine[7] !~ /^\.\// || $arrLine[7] !~ /\.(tgz|txz)$/) {
		next;
	} # if $arrLine[7]
	$arrLine[7] = substr($arrLine[7], 2);
	my @arrPath = split(/\//, $arrLine[7]);

	my $pkgName = pop(@arrPath);
	my $idPkg = $slib->insertPkg($pkgName);

	my $category = shift(@arrPath);
	my $idCategory = 0;
	if ($category eq $categoryLast) {
		$idCategory = $idCatLast;
	} else {
		$idCategory = $slib->insertCategory($category);
		$idCatLast = $idCategory;
		$categoryLast = $category;
	}

	my $serie = join("/", @arrPath);
#	for my $entry (@arrPath) {
#		$serie.= $entry;
#	}
	unless ($serie) {
		$idSerie = 'NULL';
	} else {
		if ($serie eq $serieLast) {
			$idSerie = $idSerieLast;
		} else {
			$idSerie = $slib->insertSerie($serie);
			$idSerieLast = $idSerie;
			$serieLast = $serie;
		}
	}

	my $sql1 = sprintf("INSERT INTO packages (id_slackversion, id_category,
		id_serie, id_package, package_size, package_created) VALUES (%i, %i, %s, 
		%i, %s, '%s %s');", $idSlackVer, $idCategory, $idSerie, $idPkg, 
		$arrLine[4], $arrLine[5], $arrLine[6]);
	$dbh->do($sql1)
		or die("Unable to insert package.");
} # while my $linePkg

close(FPKGS);

### PKGS MD5 ###
open(FPKGS5, "./CHECKSUMS.md5.pkgs")
	or die("Unable to open CHECKSUMS.md5.pkgs");

printf("Processing package's MD5s...\n");
while (my $linePkg5 = <FPKGS5>) {
	chomp($linePkg5);
	my @arrLine = split(' ', $linePkg5);
	if ($arrLine[1] !~ /^\.\// || $arrLine[1] !~ /\.(tgz|txz)$/) {
		next;
	} # unless $arrLine
	$arrLine[1] = substr($arrLine[1], 2);
	my @arrPath = split(/\//, $arrLine[1]);

	my $pkgName = pop(@arrPath);
	my $idPkg = $slib->getPkgId($pkgName);

	my $category = shift(@arrPath);
	my $idCategory = 0;
	if ($category eq $categoryLast) {
		$idCategory = $idCatLast;
	} else {
		$idCategory = $slib->getCategoryId($category);
		$idCatLast = $idCategory;
		$categoryLast = $category;
	}

	my $serie = '';
	for my $entry (@arrPath) {
		$serie.= $entry;
	}
	unless ($serie) {
		$idSerie = 'NULL';
	} else {
		if ($serie eq $serieLast) {
			$idSerie = $idSerieLast;
		} else {
			$idSerie = $slib->getSerieId($serie);
			$idSerieLast = $idSerie;
			$serieLast = $serie;
		}
	}

	my $sql1 = sprintf("UPDATE packages SET package_md5sum = '%s' WHERE 
		id_slackversion = %i AND id_category = %i AND id_serie = %s AND 
	id_package = %i;", $arrLine[0], $idSlackVer, $idCategory, $idSerie);
	$dbh->do($sql1)
		or die("Unable to update package's MD5 sum.");
} # while my $linePkg5

close(FPKGS5);

### SVER DATA FILES ###
open(DFILE, "./FILELIST.TXT.files")
	or die("Unable to open FILELIST.TXT.files.");

printf("Processing data files (pkg desc'n files)...\n");
printf("This is going to take a while...\n");

while (my $lineDF = <DFILE>) {
	chomp($lineDF);
	my @arrLine = split(' ', $lineDF);
	unless ($arrLine[7]) {
		next;
	}
	if ($arrLine[7] !~ /^\.\// || $arrLine[7] =~ /\.md5$/) {
		next;
	}
	if ($arrLine[7] =~ /\.txt$/i) {
		$slib->processPkgDesc($arrLine[7], $idSlackVer);
	}
{'fraction_non_alphanumeric': 0.20840681951793064, 'fraction_numerical': 0.01665686850872036, 'mean_word_length': 4.052970297029703, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '205786', 'n_tokens_mistral': 4442, 'n_tokens_neox': 4253, 'n_words': 955}