################################################################################
#      writer: Dan Tsafrir (dants AT cs DOT huji DOT ac DOT il)
# description: All the stuff needed to produce the conversion summary (how many
#              fields were successfully converted, what were the problems, etc.)
################################################################################
package ConversionLog;
use warnings;
use strict;
use Data::Dumper;

use lib '.';
use ConversionConfig;


###############################################################################
# THE EXPORTED INTERFACE:
#
# 1) &inc_njobs_started  - one more started job was successfully converted
# 2) &inc_njobs_canceled - one more non-started job was successfully converted
# 3) &log_field          - successful field conversion
# 4) &report_problem     - in conversion
# 5) &print_summary      - of conversion (successes and failures)
# 6) also using the keys of %g_problems as arguments to &report_problem.
#
# for full description see header comment of subroutines.
################################################################################
use Exporter;
our @ISA       = qw(Exporter);
our @EXPORT    = qw(&conversionlog_init
                    &inc_njobs_started
                    &inc_njobs_canceled
                    &log_field
                    &report_problem
                    &print_summary);
our $VERSION   = 1.00;


################################################################################
# internal variables
################################################################################


# number of started jobs
my $g_njobs_started = 0;


# number of jobs that where canceled (before they were started)
my $g_njobs_canceled = 0;


# used within intenal &print_summary_header: how many subsection in the summary
# were printed thus far.
my $g_summary_header_cnt = 0;


# $g_sources{$swf_field}{$pbs_field} = 
#      number of times $swf_field was deduced from $pbs_field
#  [ a $pbs_field=-1 indicates conversion failed ]
my %g_sources = ();


# a summary of all the problem encountered while converting the PBS file.
# use keys as arguments of report_problem().
my %g_problems = 
    (
     #----------------------------------------------------------------------
     'CONVERT_MEMREQ' =>
     {
      'code'         => 1,	# code of problem (orders problems)
      'classify'     => "lost data", # classify this problem
      'description'  =>		# a verbal description of the problem
        "REQUESTED size of job is unknown; failed converting from per_process "
      . "to per-job requested-memory (or vice versa)",
      'count'        => 0,	# number of jobs in which problem occurred
      'examples'     => [],	# at most $g_example_max PBSrecords with problem
				# each example is {'rec'=>$rec, 'inf'=>$inf}
				# where:
				# - $rec is the actual PBS line
				# - $inf may be 'undef' or a string further
				#   describing the error
     },
     #----------------------------------------------------------------------
     'CONVERT_MEMUSED' => 
     {
      'code'         => 2,
      'classify'     => "lost data of one field",
      'description'  =>
        "ACTUAL size of job is unknown; failed converting from per_process "
      . "to per-job used-memory (or vice versa)",
      'count'        => 0,
      'examples'     => [],
     },
     #----------------------------------------------------------------------
     'CONVERT_CPUUSED' => 
     {
      'code'         => 3,
      'classify'     => "lost data of one field",
      'description'  =>
        "size of job is unknown; failed converting from per_process to per-job "
      . "used-CPU (or vice versa)",
      'count'        => 0,
      'examples'     => [],
     },
     #----------------------------------------------------------------------
     'TYPE_MEMREQ' => 
     {
      'code'         => 4,
      'classify'     => "lost data of one field",
      'description'  =>
        "have data about physical requested-memory but using virtual (or vice "
      . "versa)",
      'count'        => 0,
      'examples'     => [],
     },
     #----------------------------------------------------------------------
     'TYPE_MEMUSED' => 
     {
      'code'         => 5,
      'classify'     => "lost data of one field",
      'description'  =>
        "have data about physical used-memory but using virtual (or vice "
      . "versa)",
      'count'        => 0,
      'examples'     => [],
     },
     #----------------------------------------------------------------------
     'PARSE_FORMAT' => 
     {
      'code'         => 6,
      'classify'     => "parse failed; lost record",
      'description'  => "record's format not: date_time;rec_type;id;msg_text",
      'count'        => 0,
      'examples'     => [],
     },
     #----------------------------------------------------------------------
     'PARSE_DATE' => 
     {
      'code'         => 7,
      'classify'     => "parse failed; lost record",
      'description'  => "record's date/time format not:  mm/dd/yyyy hh:mm:ss",
      'count'        => 0,
      'examples'     => [],
     },
     #----------------------------------------------------------------------
     'PARSE_PBSID' => 
     {
      'code'         => 8,
      'classify'     => "parse failed; lost record",
      'description'  => "record's ID format not: <number>.<server>",
      'count'        => 0,
      'examples'     => [],
     },
     #----------------------------------------------------------------------
     'PARSE_RECTYPE' => 
     {
      'code'         => 9,
      'classify'     => "parse failed; lost record",
      'description'  => "record's type not: A|B|C|D|E|F|K|Q|R|S",
      'count'        => 0,
      'examples'     => [],
     },
     #----------------------------------------------------------------------
     'PARSE_PAIR' => 
     {
      'code'         => 10,
      'classify'     => "parse failed; lost record",
      'description'  => "record's text contains token not in key=value format",
      'count'        => 0,
      'examples'     => [],
     },
     #----------------------------------------------------------------------
     'FLOW_ARRIVAL' => 
     {
      'code'         => 11,
      'classify'     => "problem in flow of job's records; lost job",
      'description'  => "job's first record <> Q (1st Q indicates arrival)",
      'count'        => 0,
      'examples'     => [],
     },
     #----------------------------------------------------------------------
     'FLOW_WAIT' => 
     {
      'code'         => 12,
      'classify'     => "problem in flow of job's records; lost job",
      'description'  => "record <> S|D|A for waiting job (S=start, D|A=cancel)",
      'count'        => 0,
      'examples'     => [],
     },
     #----------------------------------------------------------------------
     'FLOW_RUN' => 
     {
      'code'         => 13,
      'classify'     => "problem in flow of job's records; lost job",
      'description'  => "record <> E|D|A for running job (E=end, D|A=cancel)",
      'count'        => 0,
      'examples'     => [],
     },
     #----------------------------------------------------------------------
     'FLOW_CANCEL_BEFORE_RUN' => 
     {
      'code'         => 14,
      'classify'     => "problem in flow of job's records; lost job",
      'description'  => 
        "record <> Q|D|A for a job which was canceled before it started "
      . "(Q=change-queue, D|A=cancel)",
      'count'        => 0,
      'examples'     => [],
     },
     #----------------------------------------------------------------------
     'FLOW_CANCEL_AFTER_RUN' => 
     {
      'code'         => 15,
      'classify'     => "problem in flow of job's records; lost job",
      'description'  => 
        "record <> E|D|A for started job which was canceled (E=end,D|A=cancel)",
      'count'        => 0,
      'examples'     => [],
     },
     #----------------------------------------------------------------------
     'FLOW_UNFINISHED' => 
     {
      'code'         => 16,
      'classify'     => "informative; lost job",
      'description'  => "PBS-log does not include an ending record of the job",
      'count'        => 0,
      'examples'     => [],
     },
     #----------------------------------------------------------------------
     'MANDATORY' => 
     {
      'code'         => 17,
      'classify'     => "mandatory data missing; lost job",
      'description'  => "failed to extract mandatory SWF field from record",
      'count'        => 0,
      'examples'     => [],
     },
     #----------------------------------------------------------------------
     'CANCEL_TIME' => 
     {
      'code'         => 18,
      'classify'     => "conversion from humen time to epoch; lost job",
      'description'  => 
        "canceled job's arrivelTime (Q) > cancelTime (D|A); and it's not a "
      . "daylight saving problem",
      'count'        => 0,
      'examples'     => [],
     },
     #----------------------------------------------------------------------
     'WRAPAROUND' => 
     {
      'code'         => 19,
      'classify'     => "informative; indicates (termination) data is missing",
      'description'  => 
        "wraparound detected: Q (arrival) appearing after job starts (S) "
      . "but before job ends (E|D|A)",
      'count'        => 0,
      'examples'     => [],
     },
);



################################################################################
# implementation:
################################################################################


#-------------------------------------------------------------------------------
# increment started job-count (of jobs that were successfully converted).
#-------------------------------------------------------------------------------
sub inc_njobs_started() { $g_njobs_started++; }


#-------------------------------------------------------------------------------
# increment canceled-before-start job-count (of jobs that were successfully 
# converted).
#-------------------------------------------------------------------------------
sub inc_njobs_canceled() { $g_njobs_canceled++; }


#-------------------------------------------------------------------------------
# problematic $record was encountered as indicated by $problem (a key to 
# %g_problems) further described by $info (string). update %g_problems 
# accordingly.
#-------------------------------------------------------------------------------
sub report_problem($$$) {

    my ($problem, $record, $info) = @_;

    exists $g_problems{$problem} or die "bug";

    my $cnt = $g_problems{$problem}{count};

    if( ( $cnt < $CONFIG{summary}{exmpl_max} ) || 
	( $cnt < $CONFIG{summary}{exmpl_max_unfin} 
	  && $problem eq 'FLOW_UNFINISHED' ) ) {

	push @{$g_problems{$problem}{examples}}
	    , {'rec' => $record, 'inf' => $info};
    }

    $g_problems{$problem}{count}++;
}


#-------------------------------------------------------------------------------
# log the following operation:
# for a certain job, converted the value of the $pbs_field, for the purpose 
# of assigning it to $swf_field (if $pbs_field==-1, no suitable $pbs_field
# was found).
#-------------------------------------------------------------------------------
sub log_field($$) {

    my ($swf_field, $pbs_field) = @_;

    if( exists $g_sources{$swf_field}{$pbs_field} ) {
	$g_sources{$swf_field}{$pbs_field}++;
    }
    else {
	$g_sources{$swf_field}{$pbs_field} = 1;
    }
}


#-------------------------------------------------------------------------------
# print header of a section in the summary.
#-------------------------------------------------------------------------------
sub print_section_header($$) {
    my ($fd, $title) = @_;
    my $sep = "#" x 80;
    $g_summary_header_cnt++;
    print $fd "\n", $sep, "\n", "$g_summary_header_cnt) $title:\n", $sep, "\n";
}


#-------------------------------------------------------------------------------
# -print summary of failures/successes of conversion into $fd. 
# -the conversion resulted in $swf (ref2arr; each row is one SWF job line) 
#  which is needed in order to compute some statistics.
# -conversion $anonymize tables are printed.
#-------------------------------------------------------------------------------
sub print_summary($$$$)  {

    my ($fd, $swf, $anonymize, $header) = @_;
    my @problems = 
	sort {$g_problems{$a}{code}<=>$g_problems{$b}{code}} keys %g_problems;

   
    #---------------------------------------------------------------------------
    # 0- print the header of the SWF file
    #---------------------------------------------------------------------------
    my $title = "The header section of the SWF file";
    print_section_header($fd, $title);
    print $fd $header;


    #---------------------------------------------------------------------------
    # 1- print general summary
    #---------------------------------------------------------------------------
    $title    = "GENERAL";
    my $w     = 22;
    my $spec  = $CONFIG{spec};

    print_section_header($fd, $title);
    printf($fd "%${w}s : %d\n"   , "jobs"                 , scalar(@$swf)    );
    printf($fd "%${w}s : %d\n"   , "started"              , $g_njobs_started );
    printf($fd "%${w}s : %d\n"   , "canceled before start", $g_njobs_canceled);
    printf($fd "%${w}s : %s %s\n", "mem_req"              , 
	   "type=$spec->{mem_req}{type}"                  ,
	   "quantity=$spec->{mem_req}{quantity}"                         );
    printf($fd "%${w}s : %s %s\n", "mem_used"             , 
	   "type=$spec->{mem_used}{type}"                 ,
	   "quantity=$spec->{mem_used}{quantity}"                        );
    printf($fd "%${w}s : %s\n"   , "cpu_used"             , 
	   "quantity=$spec->{cpu_used}{quantity}"                        );


    #---------------------------------------------------------------------------
    # 2- print the fields conversion summary
    #---------------------------------------------------------------------------
    $title = "PBS FIELDS USED FOR CONVERSION (jobs_count[pbsfield])";
    print_section_header($fd, $title);
    printf($fd "%10s | %8s | %s\n", 'SWF FIELD', 'CONVERT#', 'PBS FIELD');

    
    foreach my $swf_field (@SWF_COLS) {

	next if $swf_field =~ m/job|prev_jobs|think_time/;
	
	my $sum     = 0;
	my $default = $CONFIG{defaults}{$swf_field}{value};

	while( my ($pbs_field, $conversions) = each %{$g_sources{$swf_field}} ){
	    $sum += $conversions;
	}
	
	printf($fd "%10s | %8d |", $swf_field, $sum);
	
	my $first = 1;
	while( my ($pbs_field, $conversions) = each %{$g_sources{$swf_field}} ){
	    if( $pbs_field eq "-1" ) { # failed conversions...
		$pbs_field = defined($default)? "default=$default" : "failures";
	    }
	    if( $first ) {
		$first = 0;
	    } else {
		print $fd " +";
	    }
	    print $fd " ${conversions}[$pbs_field]";
	}
	
	if( defined($default) &&  
	    $CONFIG{defaults}{$swf_field}{overwrite} == 1 ) {
	    print $fd " // default=$default overwrites all values";
	}
	print $fd "\n";
    }

    
    #---------------------------------------------------------------------------
    # 3- print per-field most popular values
    #---------------------------------------------------------------------------
    my $maxpop = $CONFIG{summary}{max_popular};
    $title="$maxpop MOST POPULAR VALUES (before applying 'overwrite' defaults)";
    print_section_header($fd, $title);
    printf($fd "%10s | %s\n", 'SWF FIELD', "$maxpop MOST POPULAR VALUES");

    # fill histogram
    my %hist = ();		# $hist{$swf_field}{$value} = occurrence number
    foreach my $job (@$swf) {
	foreach my $swf_field (@SWF_COLS) {
	    next if $swf_field =~ m/job|prev_jobs|think_time/;
	    my $idx   = $SWF_COL2INDEX0{$swf_field};
	    my $value = $job->[$idx];
	    if( exists $hist{$swf_field}{$value} ) {
		$hist{$swf_field}{$value}++;
	    } else {
		$hist{$swf_field}{$value} = 1;
	    }
	}
    }

    # print $maxpop values per field
    my $n = scalar(@$swf);
    foreach my $swf_field (@SWF_COLS) {
	next if $swf_field =~ m/job|prev_jobs|think_time|submit/;
	my @arr = sort( { $hist{$swf_field}{$b}<=>$hist{$swf_field}{$a} } 
			keys %{$hist{$swf_field}} );
	printf($fd "%10s |", $swf_field);
	my $i=0;
	my $sum=0;
	foreach my $value (@arr) {
	    printf($fd " %8d[%2.0f%%]", $arr[$i], 
		   100*$hist{$swf_field}{$value}/$n);
	    $sum += $hist{$swf_field}{$value};
	    $i++;
	    last if $i == $maxpop;
	}
	
	printf($fd "%s (sum=%.0f%%)\n", (" " x (14*($maxpop-$i))), 100*$sum/$n);
    }


    #---------------------------------------------------------------------------
    # 4- print problems summary
    #---------------------------------------------------------------------------
    $title = "PROBLEMS SUMMARY";
    print_section_header($fd, $title);

    foreach my $p (@problems) {
	
	my $h = $g_problems{$p};
	next if $h->{count} == 0;
	
	print($fd ("~" x 80), "\n");
	print $fd "     PROBLEM: $p\n";
	print $fd "     OUTCOME: $h->{classify}\n";
	print $fd "    RECORDS#: $h->{count}\n";
	print $fd " DESCRIPTION: $h->{description}\n";
	print $fd "   EXAMPLES:\n";
	
	my $i=1;
	foreach my $exam (@{$h->{examples}}) {
	    if( $p ne 'FLOW_UNFINISHED' ) {
		print( $fd "$i:\n", "$exam->{rec}" );
		print $fd "[ $exam->{inf} ]\n" 
		    if defined( $exam->{inf} );
	    } 
	    else { # $exam->{rec} == PBSid...
		printf( $fd "%2d: %30s [ %30s ]\n", 
			$i, 
			$exam->{rec},
			$exam->{inf});
	    }
	    $i++;
	}
    }

    #---------------------------------------------------------------------------
    # 5- print anonymize table (the meaning of the SWF codes)
    #---------------------------------------------------------------------------
    $title   = "ANONYMIZATION TABLES (before applying 'overwrite' defaults)";
    my @tbls = qw(partition queue gid uid executable);
    print_section_header($fd, $title);

    foreach my $tbl (@tbls) {

	my $t = $anonymize->{$tbl};
	my $l = 0;

	printf($fd "%10s anonymizing %d %s values %10s\n",
	       "-"x10, scalar(keys %$t), "'$tbl'", "-"x10);

	foreach my $pbsval (sort {$t->{$a} <=> $t->{$b}} keys %$t) {
	    my $swfval = $t->{$pbsval};

	    # can happen if user anonymized non-existent PBS values...
	    next if ! exists $hist{$tbl}{$swfval};

	    printf($fd "%25s -> %5d     [%6d = %5.1f%% of the jobs]\n", 
		   $pbsval, 
		   $swfval, 
		   $hist{$tbl}{$swfval},
		   100 * $hist{$tbl}{$swfval} / $n );
	    last if( ++$l >= $CONFIG{summary}{max_anonymize} );
	    
	}
    }

    #---------------------------------------------------------------------------
    # 6- report which fields are overwritten
    #---------------------------------------------------------------------------
    $title  = 
       "OVERWRITING VALUES (possibly reported above as frequent/anonymized)";
    my $cnt = 0;
    print_section_header($fd, $title);
    while( my ($field, $inf) = each %{$CONFIG{defaults}} ) {
	next if ( ! defined($inf->{overwrite}) || ($inf->{overwrite} != 1) );
	printf($fd
	       "%d) overwriting field %12s with value %s [applied to %s jobs]\n"
	       ,
	       ++$cnt,
	       ("'" . $field        . "'"),
	       ("'" . $inf->{value} . "'"),
	       $inf->{jobs});
    }

}


#-------------------------------------------------------------------------------
# all ended well in load
#-------------------------------------------------------------------------------
1;


################################################################################
#                                   EOF
################################################################################
