<commit_before>#! /usr/bin/perl -w
# vim:sw=8:ts=8:et:nowrap
use strict;

# $Id: db2xml.pl,v 1.2 2008-12-08 21:08:14 angie Exp $
#
# compares DB content to XML data and updates xml
# 
# Magic numbers, and other properties of the destination schema
# are documented here:
#        http://parl.stand.org.uk/cgi-bin/moin.cgi/DataSchema
#        
# The XML files for Hansard objects come from the Public Whip parser:
#       http://scm.kforge.net/plugins/scmsvn/cgi-bin/viewcvs.cgi/trunk/parlparse/pyscraper/?root=ukparse
# And those for MPs are in (semi-)manually updated files here:
#       http://scm.kforge.net/plugins/scmsvn/cgi-bin/viewcvs.cgi/trunk/parlparse/members/?root=ukparse

use FindBin;
chdir $FindBin::Bin;
use lib "$FindBin::Bin";
use lib "$FindBin::Bin/../../perllib";

use mySociety::Config;
mySociety::Config::set_file("$FindBin::Bin/../conf/general");


use Getopt::Long;
use DBI; 
use XML::Twig;

use vars qw($cronquiet $update_person $personid $members_xml_file $dbh $parldata $lastupdatedir $pwmembers $passer $debug
);

$parldata = mySociety::Config::get('RAWDATA');
$lastupdatedir = mySociety::Config::get('XAPIANDB') . '/../xml2db/';
$pwmembers = mySociety::Config::get('PWMEMBERS');

my $result = GetOptions (
                        "cronquiet" => \$cronquiet,
                        "update_person" => \$update_person,
                        "personid=s" => \$personid,
                        "debug" => \$debug,
                        );

#my $pwmembers = mySociety::Config::get('PWMEMBERS');
#$members_xml_file = $pwmembers . 'all-members.xml';

my $dsn = 'DBI:mysql:database=' . mySociety::Config::get('DB_NAME'). ':host=' . mySociety::Config::get('DB_HOST');
$dbh = DBI->connect($dsn, mySociety::Config::get('DB_USER'), mySociety::Config::get('DB_PASSWORD'), { RaiseError => 1, PrintError => 0 });


if ($update_person && $personid) {
    update_person();
}




sub update_person {

    $passer->{'file_updated'} = '';
    
    unless (db_load_person()) {
        die "couldn't load person";
        #$passer->{'person_db_vals'} now holds mps vals
    }

    unless ($passer->{'person_db_vals'}->{'mp_website'}) {
        die "no url for person in database to update";
    }
    
    my $twig = new XML::Twig(
        twig_handlers => {
        'personinfo' => \&xml_load_person,
    }, output_filter => 'safe',
        pretty_print => 'record',
    );
    
    my $twigfile = ($pwmembers . "websites.xml");
    $twig->parsefile($twigfile);
    
    if ($passer->{'file_updated'} eq $twigfile) {
        print "writing out $twigfile" if $debug;
        open( FH, ">$twigfile") or die "cannot open $twigfile: $!";
        $twig->print( \*FH);
    }
    return 1;

}


sub xml_load_person {
    my ($twig, $mp) = @_;
    #$mp->print;
    my $personinfo_id = $mp->att('id');
    my $pid = '';
    if ($personinfo_id && $personinfo_id =~ m#uk.org.publicwhip/person/(\d+)$#) {
        $pid = $1;
        if ($pid == $personid) {
            if ($mp->att('mp_website') eq $passer->{'person_db_vals'}->{'mp_website'}) {
                print "XML and DB already match, no update needed \n" if $debug;
                return 1;
            }
            my $updated = $mp->set_att( mp_website => $passer->{'person_db_vals'}->{'mp_website'});
            if ($mp->att('mp_website') eq $passer->{'person_db_vals'}->{'mp_website'}) {
                $passer->{'file_updated'} = $pwmembers . "websites.xml";
                print "MP's URL updated \n" if $debug;
            }
        }
    }
    #$twig->purge();<commit_msg>just notes<commit_after>#! /usr/bin/perl -w
# vim:sw=8:ts=8:et:nowrap
use strict;

# $Id: db2xml.pl,v 1.3 2008-12-09 16:13:17 angie Exp $
#
# compares DB content to XML data and updates xml
# 
# Magic numbers, and other properties of the destination schema
# are documented here:
#        http://parl.stand.org.uk/cgi-bin/moin.cgi/DataSchema
#        
# The XML files for Hansard objects come from the Public Whip parser:
#       http://scm.kforge.net/plugins/scmsvn/cgi-bin/viewcvs.cgi/trunk/parlparse/pyscraper/?root=ukparse
# And those for MPs are in (semi-)manually updated files here:
#       http://scm.kforge.net/plugins/scmsvn/cgi-bin/viewcvs.cgi/trunk/parlparse/members/?root=ukparse

use FindBin;
chdir $FindBin::Bin;
use lib "$FindBin::Bin";
use lib "$FindBin::Bin/../../perllib";

use mySociety::Config;
mySociety::Config::set_file("$FindBin::Bin/../conf/general");


use Getopt::Long;
use DBI; 
use XML::Twig;

use vars qw($cronquiet $update_person $personid $members_xml_file $dbh $parldata $lastupdatedir $pwmembers $passer $debug
);

$parldata = mySociety::Config::get('RAWDATA');
$lastupdatedir = mySociety::Config::get('XAPIANDB') . '/../xml2db/';
$pwmembers = mySociety::Config::get('PWMEMBERS');

my $result = GetOptions (
                        "cronquiet" => \$cronquiet,
                        "update_person" => \$update_person,
                        "personid=s" => \$personid,
                        "debug" => \$debug,
                        );

#my $pwmembers = mySociety::Config::get('PWMEMBERS');
#$members_xml_file = $pwmembers . 'all-members.xml';

my $dsn = 'DBI:mysql:database=' . mySociety::Config::get('DB_NAME'). ':host=' . mySociety::Config::get('DB_HOST');
$dbh = DBI->connect($dsn, mySociety::Config::get('DB_USER'), mySociety::Config::get('DB_PASSWORD'), { RaiseError => 1, PrintError => 0 });


if ($update_person && $personid) {
    update_person();
}




sub update_person {

    $passer->{'file_updated'} = '';
    
    unless (db_load_person()) {
        die "couldn't load person";
        #$passer->{'person_db_vals'} now holds mps vals
    }

    unless ($passer->{'person_db_vals'}->{'mp_website'}) {
        die "no url for person in database to update";
    }
    
    my $twig = new XML::Twig(
        twig_handlers => {
        'personinfo' => \&xml_load_person,
    }, output_filter => 'safe',
        pretty_print => 'record',
    );
    
    my $twigfile = ($pwmembers . "websites.xml");
    $twig->parsefile($twigfile);
    
    if ($passer->{'file_updated'} eq $twigfile) {
        print "writing out $twigfile" if $debug;
        open( FH, ">$twigfile") or die "cannot open $twigfile: $!";
        $twig->print( \*FH);
    }
    
    # should we read the file back in here to make sure it's really OK, 
    # or do we trust this code and XML::Twig?
    
    return 1;

}


sub xml_load_person {
    my ($twig, $mp) = @_;
    #$mp->print;
    my $personinfo_id = $mp->att('id');
    my $pid = '';
    if ($personinfo_id && $personinfo_id =~ m#uk.org.publicwhip/person/(\d+)$#) {
        $pid = $1;
        if ($pid == $personid) {
            if ($mp->att('mp_website') eq $passer->{'person_db_vals'}->{'mp_website'}) {
                print "XML and DB already match, no update needed \n" if $debug;
                return 1;
            }
            my $updated = $mp->set_att( mp_website => $passer->{'person_db_vals'}->{'mp_website'});
            if ($mp->att('mp_website') eq $passer->{'person_db_vals'}->{'mp_website'}) {
                $passer->{'file_updated'} = $pwmembers . "websites.xml";
                print "MP's URL updated \n" if $debug;
            }
        }
    }
    #$twig->purge();
{'fraction_non_alphanumeric': 0.1760387811634349, 'fraction_numerical': 0.006925207756232687, 'mean_word_length': 2.7589796980739196, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 20}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3577620', 'n_tokens_mistral': 2684, 'n_tokens_neox': 2513, 'n_words': 530}