<commit_before>#!/usr/bin/perl

use warnings;
use strict;

use Config::INI::Reader;
use Config::INI::Writer;
use Apache::Solr;
use FindBin;

require "$FindBin::Bin/helper.pl";
our $reIsGlobalPath;
our $ini_pathToFachkatalogGlobal;
our $ini_urlSolrDefault;
our $ini_pathLogFileAlternative;
our $ini_pathIndexfile;

sub updateDelete(@){
	my $pathConfigIni = "$FindBin::Bin/../etc/config.ini";
	my $configs = Config::INI::Reader->read_file($pathConfigIni);
	my @verbuende = @_;
	&logMessage("DEBUG", "verbuende: "."@verbuende");
	
	foreach my $verbund(@verbuende){
		my @deletions = ();
		
		# skip global ini vars
		next if $verbund eq "_";

		if($configs->{$verbund}->{'check'} eq 0){
			&logMessage("WARNING", "($verbund) update/delete will not be done due to check=0!");
			next;
		}
		if($configs->{$verbund}->{'updateIsRunning'} eq '1'){
			&logMessage("WARNING", "($verbund) can not update/delete - update already in progress and locked by config.ini");
			next;
		}
		
		$configs->{$verbund}->{updateIsRunning} = 1;
		Config::INI::Writer->write_file($configs, $pathConfigIni);
		
		my $solr = Apache::Solr->new(server => $ini_urlSolrDefault);
		
		&logMessage("INFO", "($verbund) Applying updates to $verbund ..");
		
		# get and check if the path is relative or global
		my $dirDel = $configs->{$verbund}->{'updates'};
		$dirDel = $ini_pathToFachkatalogGlobal . $dirDel if($dirDel =~ $reIsGlobalPath);
		
		opendir(DIR_DEL, $dirDel) or die $!;
		my @filesWithIDs = ();

		# rename the ending of the file into *.del if necessary
		while(my $file = readdir(DIR_DEL)){

			if($file =~ m/(.*del.+)\.del$/){
				push(@filesWithIDs, $file);
			}
			elsif ($file =~ m/(.*del.+)\.\w{3}$/){
				my $oldFileName = $dirDel."/".$file;
				my $newFileName = $dirDel."/".$1.".del";
				&renameFile($oldFileName, $newFileName, $verbund);
			}
		}
		@filesWithIDs = sort @filesWithIDs;
		&logMessage("INFO", "($verbund) Found " . scalar(@filesWithIDs) . " local deletion files");
		
		# get and check if the path is relative or global
		my $dirUpd = $configs->{$verbund}->{'updates'};
		$dirUpd = $ini_pathToFachkatalogGlobal . $dirUpd if($dirUpd =~ $reIsGlobalPath);
		
		opendir(DIR_UPD, $dirUpd) or die $!;
		my @filesWithUpd = ();
		my $updateFormat = $configs->{$verbund}->{'updateFormat'};
		
		while(my $file = readdir(DIR_UPD)){
			if ($file =~ m/\.$updateFormat$/){
				push(@filesWithUpd, $file);
			}
		}
		@filesWithUpd = sort @filesWithUpd;
		&logMessage("INFO", "($verbund) Found " . scalar(@filesWithUpd) . " local update files");
		
		open(my $alternLOG, ">> $ini_pathLogFileAlternative") or die $!;
		print $alternLOG "\n\n". &getTimeStr() . " ($verbund): \n";
		close $alternLOG;
		
		my $configProperties = $configs->{$verbund}->{'configPropertiesFile'};
		$configProperties = $ini_pathToFachkatalogGlobal . $configProperties if($configProperties =~ $reIsGlobalPath);
		
		my @ids = ();<commit_msg>* applied also del updates to files which had to be renamed<commit_after>#!/usr/bin/perl

use warnings;
use strict;

use Config::INI::Reader;
use Config::INI::Writer;
use Apache::Solr;
use FindBin;

require "$FindBin::Bin/helper.pl";
our $reIsGlobalPath;
our $ini_pathToFachkatalogGlobal;
our $ini_urlSolrDefault;
our $ini_pathLogFileAlternative;
our $ini_pathIndexfile;

sub updateDelete(@){
	my $pathConfigIni = "$FindBin::Bin/../etc/config.ini";
	my $configs = Config::INI::Reader->read_file($pathConfigIni);
	my @verbuende = @_;
	&logMessage("DEBUG", "verbuende: "."@verbuende");
	
	foreach my $verbund(@verbuende){
		my @deletions = ();
		
		# skip global ini vars
		next if $verbund eq "_";

		if($configs->{$verbund}->{'check'} eq 0){
			&logMessage("WARNING", "($verbund) update/delete will not be done due to check=0!");
			next;
		}
		if($configs->{$verbund}->{'updateIsRunning'} eq '1'){
			&logMessage("WARNING", "($verbund) can not update/delete - update already in progress and locked by config.ini");
			next;
		}
		
		$configs->{$verbund}->{updateIsRunning} = 1;
		Config::INI::Writer->write_file($configs, $pathConfigIni);
		
		my $solr = Apache::Solr->new(server => $ini_urlSolrDefault);
		
		&logMessage("INFO", "($verbund) Applying updates to $verbund ..");
		
		# get and check if the path is relative or global
		my $dirDel = $configs->{$verbund}->{'updates'};
		$dirDel = $ini_pathToFachkatalogGlobal . $dirDel if($dirDel =~ $reIsGlobalPath);
		
		opendir(DIR_DEL, $dirDel) or die $!;
		my @filesWithIDs = ();

		# rename the ending of the file into *.del if necessary
		while(my $file = readdir(DIR_DEL)){

			if($file =~ m/(.*del.+)\.del$/){
				push(@filesWithIDs, $file);
			}
			elsif ($file =~ m/(.*del.+)\.\w{3}$/){
				my $oldFileName = $dirDel."/".$file;
				my $newFileName = $dirDel."/".$1.".del";
				&renameFile($oldFileName, $newFileName, $verbund);
				push(@filesWithIDs, $newFileName);
			}
		}
		@filesWithIDs = sort @filesWithIDs;
		&logMessage("INFO", "($verbund) Found " . scalar(@filesWithIDs) . " local deletion files");
		
		# get and check if the path is relative or global
		my $dirUpd = $configs->{$verbund}->{'updates'};
		$dirUpd = $ini_pathToFachkatalogGlobal . $dirUpd if($dirUpd =~ $reIsGlobalPath);
		
		opendir(DIR_UPD, $dirUpd) or die $!;
		my @filesWithUpd = ();
		my $updateFormat = $configs->{$verbund}->{'updateFormat'};
		
		while(my $file = readdir(DIR_UPD)){
			if ($file =~ m/\.$updateFormat$/){
				push(@filesWithUpd, $file);
			}
		}
		@filesWithUpd = sort @filesWithUpd;
		&logMessage("INFO", "($verbund) Found " . scalar(@filesWithUpd) . " local update files");
		
		open(my $alternLOG, ">> $ini_pathLogFileAlternative") or die $!;
		print $alternLOG "\n\n". &getTimeStr() . " ($verbund): \n";
		close $alternLOG;
		
		my $configProperties = $configs->{$verbund}->{'configPropertiesFile'};
		$configProperties = $ini_pathToFachkatalogGlobal . $configProperties if($configProperties =~ $reIsGlobalPath);
		
		my @ids = ();
{'fraction_non_alphanumeric': 0.1852856416341753, 'fraction_numerical': 0.0020342430920494998, 'mean_word_length': 4.393053016453382, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6496518', 'n_tokens_mistral': 2381, 'n_tokens_neox': 2113, 'n_words': 467}