#!/usr/bin/perl

# version: 6 (20020904), Roderik Muit <ro@muit.nl>

# Script to counter strange behaviour of ipac-ng on my (sparc64, linux-2.4.18)
# machine. (See end of script for a description of the problem.)
# What it does is go through all files which were created on a certain day
# (default: yesterday), and add up the figures from the previous file
# to this file.
# NOTE: for this script to work correctly, the last file with numbers from the
# previous day should be present in $IPACDIR, or the first file from the day
# processed should contain correct numbers (which is the case for the first
# file produced after 'ipacsum -S')
#
# The files with 'fixed numbers' are written into another dir ($FIXDIR).
# You should run ipacsum with argument '--dir <$FIXDIR>'.
#
# Do not run this script on a set of data where you did 
# 'ipacsum -S' (e.g. by restarting the machine) somewhere in that timeframe.
# If you stop/start IP accounting, I suggest doing the following:
# - stop IP accounting 
# - run this script to generate all 'fixed' files
# - delete the original files (from $IPACDIR), or move them into another
#   dir (not a subdir of $IPACDIR, or you will probably need to
#   adjust process_files() below).
# Then you can start IP accounting again and return to normal operation (which
# is probably, have a cronjob process yesterday's files in /var/lib/ipac before
# running ipacsum on /var/lib/ipac-fixed).

# usage: $0 [-dw] [date]
# -d: print debug output
# -w: wait for <return> after each file (not much use without -d)
# [date]: date (yyymmdd) of logs to process. Default: yesterday.

use strict;
use Time::Local;
use Getopt::Std;

my $IPACDIR = '/var/lib/ipac';
my $FIXDIR = '/var/lib/ipac-fixed';
my $CACHEFILE = '/var/lib/ipacngfix-cache';
# cachefile which will contain numbers from the previous run. 
# you can set this to '' if you do not want this file to be created, the
# script will then get the numbers from the file $FIXDIR/<time of previous run>
# so it will still work the same way. However, you need to be sure that in this
# case, that file will never be removed/replaced by another file, for instance
# by an 'ipacsum --replace' command running from /etc/cron.monthly.

getopts('dw') or die;
my $Debug = $Getopt::Std::opt_d;
my $Wait = $Getopt::Std::opt_w;
my $Date = $ARGV[0];

#========================================================================
#
# Loop through all files in a directory and child directories,
# (optionally: which matches a certain regexp)
# and for each matching file, call a specified subroutine with the filename
# as argument
#
# In: - reference to list of files/directories to process
#     - reference to subroutine to call for each file
#     - regexp which files need to match (optional. NB directories are
#       not checked against this, only the files in them.)
#
# Note: skips symlinks

use Cwd qw(cwd abs_path);

my $process_file_do_zero = 0; # 0: skip files with length 0. 1: process them
my $process_file_verbose = 0;

sub process_files {
  my ($r_files,$sub,$regexp) = @_;

  @$r_files = ('.') if @$r_files == 0;

  foreach my $file (@$r_files) {
    if (-e $file) {
      # mangle name; make into absolute path
      if (-d $file) {
	$file = abs_path($file);
      } else {
	my ($dirpart,$filepart) = $file =~ /^(.*)(\/.*?)$/;
	if ($dirpart) {
	  $file = abs_path($dirpart) . $filepart;
	} else {
	  $file = cwd . '/' . $file;
	}
      }
      process_file($file,$sub,$regexp);
    } else {
      warn "$file does not exist!\n";      next;
    }
  }
  return();
}

sub process_file {
  my ($file,$sub,$regexp) = @_;
  
  if (-f $file and -z $file and !$process_file_do_zero) {
    print STDERR "$file is zero size; not checking\n" if $process_file_verbose;
    return();
  }
  
  if (-l $file) {
    # I need to do this - because the -f switch will hold true
    # for a symlink to a file! (perl5.00404, Linux ext2 fs)
    # and I don't want anything to do with symlinks at the moment.
    print STDERR "$file is a symlink; skipping\n" if $process_file_verbose;
    next;
  } elsif (-f $file) {
    if (!defined($regexp) or $file =~ /$regexp/) {
      print STDERR "processing $file...\n" if $process_file_verbose;
      &$sub($file);
    }
  } elsif (-d $file) {
    # directory? Read all filenames from here and call this routine
    # recursively for each of them
    
    my @dir;
    opendir(DIR,$file) or die "opendir $file: $!\n";
    foreach my $fn (sort readdir DIR) { # sometimes sort does make a difference
                                        # I haven't made it an option yet
      next if $fn =~ /^\.(\.)?$/;
      push @dir,"$file/$fn";
    }
    closedir DIR;
    print STDERR "descending directory $file\n" if $process_file_verbose;
    while (my $fn = shift @dir) {
      process_file($fn,$sub,$regexp);
    }    
  }				# No regular file, no dir: do nothing
   
  return();
}
#
#
#========================================================================

# define routine processing one file

my @prevnr1;
my @prevnr2;
my $lastfile;

my $sub = sub {
  my ($full_filename) = @_;
  my $new_filename = $full_filename;
  $new_filename =~ s/^\Q$IPACDIR/$FIXDIR/;

  open INFILE, $full_filename or die "open $full_filename: $!\n";
  open OUTFILE, ">$new_filename" or die "open $new_filename for output: $!\n";

  print STDERR $full_filename,"\n" if $Debug;

  my $linenr = 0;

  while (! eof(INFILE)) {
    my $line = <INFILE>;
    if (my ($nr1,$nr2) = $line =~ /^(\d+)\s+(\d+)/) {

      # negative number? Then really negate
#      $nr1 -= 18446744073709551616 if $nr1 >= 9223372036854775808;
# above does not work, perl also chokes on large numbers
      $nr1 = substr($nr1,-9) - 709551616 if $nr1 >= 1000000000;

      # previous number was subtracted twice, so add it to here
      my $newnr1 = $nr1 + $prevnr1[$linenr];
      # check which should never be necessary: if number is negative,
      # something went wrong
      if ($newnr1<0) {
	print STDERR "$full_filename:$linenr,1 is negative! ($newnr1)\n";
	$newnr1 = 1;
      }
      # keep old one
      $prevnr1[$linenr] = $nr1;

      # same for nr2
      $nr2 = substr($nr2,-9) - 709551616 if $nr2 >= 1000000000;
      my $newnr2 = $nr2 + $prevnr2[$linenr];
      if ($newnr2<0) {
	print STDERR "$full_filename:$linenr,2 is negative! ($newnr2)\n";
	$newnr2 = 1;
      }
      $prevnr2[$linenr] = $nr2;

      $line = $newnr1 . ' ' . $newnr2 . "\n";
      $linenr++;
    }
    print OUTFILE $line;
    print STDERR $line if $Debug;
  }

  close OUTFILE or die "error closing $new_filename: $!\n";
  close INFILE or die "error closing $full_filename: $!\n";

  $lastfile = $full_filename; # remember because the calling routine
			      # needs the name of the last file processed
  if ($Wait) {
    print STDERR "Press Enter...";  
    my $dummy = <STDIN>;
  }

};

!-d $FIXDIR and system('mkdir', $FIXDIR) and die "creating $FIXDIR: $!\n";

# get date to process
if (!defined($Date)) {
  my @a = localtime(time-3600*24); 
  $Date = ($a[5]+1900) . substr('0'.($a[4]+1),-2) . substr('0'.$a[3],-2);
}

# get numbers from previous run; you need them for computing the real numbers
# in the first file-to-process.
if ($CACHEFILE and -f $CACHEFILE) {
  $lastfile = $CACHEFILE;
} else {
  # get previous date, read numbers from last file on that date
  my $prevdate  = $Date - 1;
  opendir(DIR,$IPACDIR) or die "opendir $IPACDIR: $!\n";
  map { $lastfile = $_ if /^$prevdate/; } sort readdir DIR;
  closedir DIR;
  $lastfile = $IPACDIR.'/'.$lastfile;
}

# get numbers from that file (to add to the first file)
open INFILE, $lastfile and do {
  print STDERR "Previous numbers from $lastfile\n" if $Debug;
  my $linenr = 0;
  while (! eof(INFILE)) {
    my $line = <INFILE>;
    my ($nr1,$nr2) = $line =~ /^(\-?\d+)\s+(\-?\d+)/;
    if (defined($nr2)) {
      $nr1 = substr($nr1,-9) - 709551616 if $nr1 >= 1000000000;
      $prevnr1[$linenr] = $nr1;
      $nr2 = substr($nr2,-9) - 709551616 if $nr2 >= 1000000000;
      $prevnr2[$linenr] = $nr2;
      $linenr++;
      print STDERR "$nr1 $nr2\n" if $Debug;
    }
  }
  close INFILE or die "error closing $lastfile: $!\n"; 
  if ($Wait) {
    print STDERR "Press Enter...";  
    my $dummy = <STDIN>;
  }
};

# process files
process_files([$IPACDIR], $sub, $Date.'-');

if ($CACHEFILE) {
  unlink $CACHEFILE if $CACHEFILE;
  link $lastfile, $CACHEFILE;
}

exit 0;

__END__

fetchipac is run every 10 minutes (without arguments). Here's figures from several files which were created in order: 

tomachine in smtp 
tomachine out smtp 
#-#-#-#-# 
0 0 
0 0 
tomachine in smtp 
tomachine out smtp 
#-#-#-#-# 
0 0 
0 0 
tomachine in smtp 
tomachine out smtp 
#-#-#-#-# 
12 1106 
15 943 
tomachine in smtp 
tomachine out smtp 
#-#-#-#-# 
18446744073709551604 18446744073709550510 
18446744073709551601 18446744073709550673 
tomachine in smtp 
tomachine out smtp 
#-#-#-#-# 
12 1106 
15 943 
tomachine in smtp 
tomachine out smtp 
#-#-#-#-# 
18446744073709551604 18446744073709550510 
18446744073709551601 18446744073709550673 
tomachine in smtp 
tomachine out smtp 
#-#-#-#-# 
12 1106 
15 943 
tomachine in smtp 
tomachine out smtp 
#-#-#-#-# 
18446744073709551604 18446744073709550510 
18446744073709551601 18446744073709550673 

etc etc etc... 


If I look at the difference between these huge numbers, they look to me like they actually represent -12, -1106, -15, -943. 

I don't know how fetchipac works... But it looks to me like something wants to distract 12 from the counter to set it to 0, but somehow the counter is set to -12. Then the next time, something wants to distract -12 from the counter and the counter is set to +12. etc etc etc.