#!/usr/bin/perl
#
# scan job execution log files from the Intel Netbatch grid, and use convert2swf.pm
# to convert the data into the standard workload format.
#

use warnings;
use strict;
use Time::Local;
use POSIX;
use convert2swf_v5;


############################
# Read local data

my $tz_str;
my $inst;

open(LOC, "Locale");

while (<LOC>) {
    chomp;
    if (/TimeZone: ([\w\/]+)/) {
	$tz_str = $1;
    }
    if (/Installation: (.+)/) {
	$inst = $1;
    }
}

close(LOC);

# find timestamp shift to compensate for using local time
$ENV{TZ} = $tz_str;
POSIX::tzset();
# for some unknown reason it seems that setting the timezone helps,
# but actually adjusting the timestamps is wrong.
my $timeloc = timelocal( 0, 0, 0, 1, 0, 113 );
my $timegm  = timegm( 0, 0, 0, 1, 0, 113 );
my $timeshift = $timeloc - $timegm;
my $shifthr = $timeshift / 3600;
#print STDERR ">>> input time zone is $tz_str\n";
#print STDERR ">>> time shift is $timeshift ($shifthr hours)\n";
$timeshift = 0;


############################
#
# DO THE CONVERSION:
#

my $max_procs = -1;
my @queues = ();
my $job_cnt = 0;

SWF::init( $tz_str, $max_procs, \@queues );

print STDERR "done init\n";

my $format_bugs;
my $jobs_data = get_jobs();

print STDERR "\ngot jobs\n";

my ($zero, $maxtime) = SWF::convert( $jobs_data, $format_bugs, 1 );

print STDERR "converted\n";

my $header = generate_header( $zero, $maxtime, $max_procs );

print STDERR "got header\n";

SWF::print( $header );



############################
# SPECIFICS for Intel Netbatch workload

# the format of each entry includes the following comma-separated fields:
# 0  job ID (anonymized)
# 1  group (anonymized)
# 2  user (anonymized)
# 3  command (anonymized)
# 4  submit time (secs since epoch)
# 5  start time (secs since epoch)
# 6  end time (secs since epoch)
# 7  exit status
# 8  suspend time (secs)
# 9  wall time (secs)
# 10 utime (secs total for all threads)
# 11 stime (secs total for all threads)
# 12 max RSS (4KB pages)
# 13 max VM (4KB pages)
# 14 machine ID (anonymized)
# 15 iteration (jobs may re-run)
# 16 iteration submit time (secs since epoch)
# 17 memory (GB total for all threads)
# 18 cores (may be non-integer!)

sub get_jobs {
##############
# parse input format and create a hash for each job

    my @list_of_jobs;
    my %prev;
    my %first;

    #
    # scan trace and collect job info
    #
    while (<>) {

	#
	# parse line and extract info
	#

	if (/^\s*$/) {		# empty line
	    next;
	}
	if (/^#/) {		# comment
	    next;
	}

	$_ =~ /\s*(.*)\s*/;	# get rid of white space at ends
	$_ = $1;
	my @line = split(',');	# split into fields

	if ($#line != 18) {
	    $format_bugs .= "format_problem $_\n";
	    warn(">>>Format problem on $_\ngot $#line fields (should be 18)");
	    next;
	}

	my %nj;

	#
	# set the desired fields as specified in convert2swf.pl
	#

	$nj{trace}  = $_;	# used for error messages

	$nj{submit} = $line[4] + $timeshift;
	$nj{start}  = $line[5] + $timeshift;
	$nj{end}    = $line[6] + $timeshift;

	my $procs   = $line[18];
	if ($procs =~ /\./) {
	    my $frac = $procs;
	    $procs = int($frac + 1);
	    $format_bugs .= "fraction_procs_problem [$frac rounded to $procs]\n";
	    warn(">>>Format problem: $frac procs rounded to $procs");
	}
	$nj{procs}  = $procs;

	$nj{memtot} = $line[17] * 1048576;	# GB to KB

	# CPU time is given for all nodes, separated into user and system.
	# we need total per node.
	my $cpu;
	if (($line[18] > 0) && ($line[10] ne "") && ($line[11] ne "")) {
	    $cpu = int( ($line[10] + $line[11]) / $line[18] );
	    $nj{cpu} = $cpu;
	}

	$nj{status} = ($line[7] == 0) ? 1 : 0;

	$nj{user}   = $line[2];

	$nj{group}  = $line[1];

	$nj{app}    = $line[3];

	if(1){
	# note dependencies between iterations of same job
	# reduce hash sizes by keeping singletons separate
	my $jobID = $line[0];
	my $iter  = $line[15];
	if ($iter == 1) {
	    # first and potentially only appearence
	    my %record;
	    $record{end}   = $nj{end};
	    $record{id}    = $job_cnt;
	    $record{iter}  = 1;
	    $first{$jobID} = \%record;
	}
	else {
	    # may have appeared already
	    if (defined($first{$jobID})) {
		# has appeared with iter==1; move to %prev
		$prev{$jobID} = $first{$jobID};
		#delete $first{$jobID};
	    }
	    if (defined($prev{$jobID})) {
		# has appeared, note dependency
		my $prev = $prev{$jobID};
		if ($iter > $prev->{iter}) {
		    if ($iter != $prev->{iter}+1) {
			$format_bugs .= "iter_skip [$iter after $prev->{iter} for ID $jobID]\n";
			warn(">>>Format problem: iter $iter after $prev->{iter} for ID $jobID");
		    }
		    my $submit = $line[16] + $timeshift;
		    my $think  = $submit - $prev->{end};
		    $nj{submit}   = $submit;
		    $nj{prev_job} = $prev->{id};
		    $nj{think}    = $think;
		}
		else {
		    $format_bugs .= "iter_bad_seq [$iter after $prev->{iter} for ID $jobID]\n";
		    warn(">>>Format problem: iter $iter after $prev->{iter} for ID $jobID");
		}
		$prev->{iter} = $iter;
		$prev->{end}  = $nj{end};
		$prev->{id}   = $job_cnt;
	    }
	    else {
		# first appearence but not iter==1
		my %record;
		$record{end}  = $nj{end};
		$record{id}   = $job_cnt;
		$record{iter} = $iter;
		$prev{$jobID} = \%record;
	    }
	}
	}

	$nj{num} = $job_cnt;

	#show_job($job_cnt, \%nj);

	$list_of_jobs[$job_cnt++] = \%nj;

	if (! ($job_cnt % 100000)) {
	    print STDERR ".";
	}
    }

    return \@list_of_jobs;
}


sub generate_header {
    my ($zero, $maxtime, $max_procs) = @_;

    my ($sec, $min, $hr, $mday, $mon, $year, $wday, $yday, $isdst);

    $header  = "; Version: 2.2\n";
    $header .= "; Computer: Intel Netbatch Grid\n";
    $header .= "; Installation: $inst\n";

    $header .= "; Acknowledge: Ohad Shai and Nir Antebi\n";

    $header .= sprintf("; Conversion: Dror Feitelson (feit\@cs.huji.ac.il) %s\n",
		       strftime("%d %b %Y", localtime()));


    $header .= sprintf("; MaxJobs: %d\n", $job_cnt);
    $header .= sprintf("; MaxRecords: %d\n", $job_cnt);

    $header .= sprintf("; UnixStartTime: %d\n", $zero);
    $header .= sprintf("; TimeZoneString: %s\n", $tz_str);

    $header .= sprintf("; StartTime: %s\n",
	   strftime("%a %b %2d %H:%M:%S %Z %Y", localtime($zero)));

    $header .= sprintf("; EndTime:   %s\n",
	   strftime("%a %b %2d %H:%M:%S %Z %Y", localtime($maxtime)));

    $header .= "; MaxNodes: undisclosed\n";
    $header .= "; MaxProcs: undisclosed\n";

    return $header;
}


###
# DEBUG
###
sub show_job {
    my ($ser, $job) = @_;

    print STDERR "\njob no. $ser\n";
    foreach my $field (keys %$job) {
	print STDERR "  $field: $job->{$field}\n"
    }
}
