<commit_before>#!/usr/bin/env perl

#
# Reads in a CSV of users by day (generated by Splunk)
# and generates summaries including a histogram of # of days
# using kbase.
#

use JSON;
use strict;
use Date::Calc qw(Delta_Days);
use POSIX qw/strftime/;

my $json = JSON->new->allow_nonref;
my $nBuckets=6;
my $users;
my $maxVisits;
my %staff;
my %visits;
my $users;
my $totUsers=0;


# Read in the Staff list
#
open(E,"kbase-staff.lst") or die "Unable to open KBase Staff list";
while(<E>){
  chomp;
  $staff{$_}=1;
  $users->{$_}->{'staff'}=JSON::true;
}
close E;

# Get all the users from the log and skip over some bogus splunk columns
#
$_=<STDIN>;
chomp;
my @userl=split /,/;
shift @userl;
for my $u (@userl){
  next if defined $staff{$u};
  next if $u eq '"-"';
  next if $u eq 'NULL';
  next if $u =~ '_span';
  next if $u =~ '_spandays';
  $users->{$u}->{'staff'}=JSON::false;
}

my $start_date=0;
my $end_date=0;
my $time;
my $counts;

# Go through each day and tally up visits
#
while(<STDIN>){
  chomp;
  my @list=split /,/;
  $time=shift @list;
  $time=~s/T.*//;
  $time=~s/"//;
  my $i=0;
  $start_date=$time unless $start_date ne 0;
  foreach (@list){
    my $user=$userl[$i];
    $i++;
    # Skip if the user isn't in our good list
    #
    next unless defined $users->{$user};
    next if $_ == 0;
    $users->{$user}->{visits}++;
    $users->{$user}->{first}=$time if ! defined $users->{$user}->{first};
    $users->{$user}->{last}=$time;
    $visits{$user}++;
    $counts->{visitors_by_date_all}->{$time}++;
    $counts->{visitors_by_date_ext}->{$time}++ if ! defined $staff{$user};
  }
}
$end_date=$time;

# Calculate user related stats
#
foreach my $u (%{$users}){
  next if ! defined $visits{$u};
  $maxVisits=$visits{$u} if $visits{$u}>$maxVisits;
  my @st=split /-/,$users->{$u}->{first};
  my @en=split /-/,$end_date;
  my $span=Delta_Days(@st,@en)+1;
  $users->{$u}->{span}=$span;
  $users->{$u}->{visitation_rate}=$visits{$u}/$span; 
  $totUsers++ if $visits{$u} > 1;
}

foreach my $t (sort keys %visits){
  next if $visits{$t} < 2;
  my $bucket=int($visits{$t});
  $counts->{'all'}->{$bucket}++;
  $counts->{'nonkbase'}->{$bucket}++ if ! defined $staff{$t};
}

my $end;
my $b=0;
my $histo;
my $tot=0;
my $ext=0;
my $start='';
#$histo->{'buckets'}=$nBuckets;
foreach my $t (2..$maxVisits){
  my $start="$t" if $start eq '';
  my $ct=$counts->{'all'}->{$t};
  my $ex=$counts->{'nonkbase'}->{$t};
  next unless defined $ct;
  $tot+=$ct;
  $ext+=$ex;
  $end=$t;
  if ($tot>($totUsers/$nBuckets) || $t eq $maxVisits){
    my $label="$start to $end visits";
    $label="$end visits" if $start eq $end;
    $histo->[$b]->{label}=$label;
    $histo->[$b]->{range}->{start}=$start;
    $histo->[$b]->{range}->{end}=$end;
    $histo->[$b]->{counts}->{all}=$tot;
    $histo->[$b]->{counts}->{nonkbase}=$ext;
    $tot=0;
    $ext=0;
    $start='';
    $b++;
  }
}

my $jo;
$jo->{by_user}=$users;
$jo->{counts_by_visits}=$counts;
$jo->{histogram}=$histo;
$jo->{range}->{start}=$start_date;
$jo->{range}->{end}=$end_date;
$jo->{meta}->{comments}="Generated from a Splunk query and summarized by user_visits_histogram";<commit_msg>Minor fixes to user_visits_histogram - Initialize start - Use integers for ranges - Prune users that don't have any data<commit_after>#!/usr/bin/env perl

#
# Reads in a CSV of users by day (generated by Splunk)
# and generates summaries including a histogram of # of days
# using kbase.
#

use JSON;
use strict;
use Date::Calc qw(Delta_Days);
use POSIX qw/strftime/;

my $json = JSON->new->allow_nonref;
my $nBuckets=6;
my $users;
my $maxVisits;
my %staff;
my %visits;
my $users;
my $totUsers=0;


# Read in the Staff list
#
open(E,"kbase-staff.lst") or die "Unable to open KBase Staff list";
while(<E>){
  chomp;
  $staff{$_}=1;
  $users->{$_}->{'staff'}=JSON::true;
}
close E;

# Get all the users from the log and skip over some bogus splunk columns
#
$_=<STDIN>;
chomp;
my @userl=split /,/;
shift @userl;
for my $u (@userl){
  next if defined $staff{$u};
  next if $u eq '"-"';
  next if $u eq 'NULL';
  next if $u =~ '_span';
  next if $u =~ '_spandays';
  $users->{$u}->{'staff'}=JSON::false;
}

my $start_date=0;
my $end_date=0;
my $time;
my $counts;

# Go through each day and tally up visits
#
while(<STDIN>){
  chomp;
  my @list=split /,/;
  $time=shift @list;
  $time=~s/T.*//;
  $time=~s/"//;
  my $i=0;
  $start_date=$time unless $start_date ne 0;
  foreach (@list){
    my $user=$userl[$i];
    $i++;
    # Skip if the user isn't in our good list
    #
    next unless defined $users->{$user};
    next if $_ == 0;
    $users->{$user}->{visits}++;
    $users->{$user}->{first}=$time if ! defined $users->{$user}->{first};
    $users->{$user}->{last}=$time;
    $visits{$user}++;
    $counts->{visitors_by_date_all}->{$time}++;
    $counts->{visitors_by_date_ext}->{$time}++ if ! defined $staff{$user};
  }
}
$end_date=$time;

# Calculate user related stats
#
foreach my $u (%{$users}){
  next if ! defined $visits{$u};
  $maxVisits=$visits{$u} if $visits{$u}>$maxVisits;
  my @st=split /-/,$users->{$u}->{first};
  my @en=split /-/,$end_date;
  my $span=Delta_Days(@st,@en)+1;
  $users->{$u}->{span}=$span;
  $users->{$u}->{visitation_rate}=$visits{$u}/$span; 
  $totUsers++ if $visits{$u} > 1;
}

foreach my $t (sort keys %visits){
  next if $visits{$t} < 2;
  my $bucket=int($visits{$t});
  $counts->{'all'}->{$bucket}++;
  $counts->{'nonkbase'}->{$bucket}++ if ! defined $staff{$t};
}

my $end;
my $b=0;
my $histo;
my $tot=0;
my $ext=0;
my $start='';
#$histo->{'buckets'}=$nBuckets;
foreach my $t (2..$maxVisits){
  $start=$t if $start == '';
  my $ct=$counts->{'all'}->{$t};
  my $ex=$counts->{'nonkbase'}->{$t};
  next unless defined $ct;
  $tot+=$ct;
  $ext+=$ex;
  $end=$t;
  if ($tot>($totUsers/$nBuckets) || $t eq $maxVisits){
    my $label="$start to $end visits";
    $label="$end visits" if $start eq $end;
    $histo->[$b]->{label}=$label;
    $histo->[$b]->{range}->{start}=int($start);
    $histo->[$b]->{range}->{end}=int($end);
    $histo->[$b]->{counts}->{all}=int($tot);
    $histo->[$b]->{counts}->{nonkbase}=$ext;
    $tot=0;
    $ext=0;
    $start='';
    $b++;
  }
}

# Remove empty entries from staff list
foreach my $user (keys %staff){
  delete $users->{$user} if ! defined $users->{$user}->{visits};
}

my $jo;
$jo->{by_user}=$users;
$jo->{counts_by_visits}=$counts;
$jo->{histogram}=$histo;
$jo->{range}->{start}=$start_date;
$jo->{range}->{end}=$end_date;
$jo->{meta}->{comments}="Generated from a Splunk query and summarized by user_visits_histogram";
{'fraction_non_alphanumeric': 0.24250305997552019, 'fraction_numerical': 0.005201958384332925, 'mean_word_length': 4.15130023640662, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 132, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7227071', 'n_tokens_mistral': 2805, 'n_tokens_neox': 2745, 'n_words': 633}