#!/usr/bin/perl
#
# scan a given SWF (standard workload format) file, and use convert2swf_v4.pm
# to re-convert the data into SWF, including checks and normalizations.
# it is assumed times are OK, only sorting and user/app numbering may be needed.
# VERSION FOR THE RAW CEA CURIE LOG -- ADD HEADER AND SPECIAL CHECKS
#

use warnings;
use strict;
use Time::Local;
use POSIX;
use convert2swf_v4;


############################
#
# DO THE CONVERSION:
#

my $inswf = $ARGV[0];
my ($header, $UTCstart, $tz_str, $max_procs) = get_header();

my ($parts, $part_desc) = get_partitions();
my @noqueue;

SWF::init( $tz_str, $max_procs, \@noqueue, $parts );

my $format_bugs;
my $jobs_data = get_jobs();

my ($zero, $maxtime) = SWF::convert( $jobs_data, $format_bugs );

my $newheader = regenerate_header( $zero, $maxtime );

SWF::print( $newheader, \@noqueue, $part_desc );


sub get_header {
################
# copy given header and return it for reuse

    my $header  = "";
    my $startTS = 0;
    my $TZstr   = "";
    my $procs   = -1;
    my $start_jobs;

    while (<>) {

	if ((/^;/) || (/^\s*$/)) {
	    $header .= $_;
	    $start_jobs = tell(ARGV);

	    if (/^;\s*UnixStartTime:\s*(\d+)\s*$/) {
		$startTS = $1;
	    }
	    if (/^;\s*TimeZoneString:\s*([\w\/]+)\s*$/) {
		$TZstr = "Europe/Paris";
		$ENV{TZ} = $TZstr;
		POSIX::tzset();
	    }
	    if (/^;\s*MaxProcs:\s*(\d+)\s*$/) {
		$procs = $1;
	    }
	    if ((/^;\s*MaxNodes:\s*(\d+)\s*$/) && ($procs == -1)) {
		$procs = $1;
	    }
	}
	else {
	    seek(ARGV, $start_jobs, 0);

	    $header = ";   --- SWF file converted from SACCT ---
; Version: 2.2
; Conversion: sacct-2-swf script - March 2013
;
; Installation: CEA Curie Cluster
; Acknowledge: Joseph Emeras, CEA/CNRS
; Information: http://www-hpc.cea.fr/en/complexe/tgcc-curie.htm
;
; MaxJobs: 773138
; MaxRecords: 773138
;
; UnixStartTime: 1296571124
;
; TimeZoneString: Europe/Paris
; StartTime: 2011-02-01 15:38:44
; EndTime:   2012-10-17 04:25:36
; 
; Note: Curie is composed of 3 types of nodes: thin, fat, and hybrid,
;	see: http://www-hpc.cea.fr/en/complexe/tgcc-curie.htm
; Note: hybrid nodes (16 chassis) are not taken into account in the MaxNodes
;	and MaxProcs fields.
; MaxNodes: 5544
; Note: 5040 thin nodes + 360 fat nodes + 144 hybrid nodes
; 
; MaxProcs: 93312
; Note: 80640 in thin nodes + 11520 in fat nodes + 1152 in hybrid nodes
;
; Note: Scheduler is Slurm (https://computing.llnl.gov/linux/slurm/)
;
";

	    return ($header, $startTS, $TZstr, $procs);
	}
    }
    die "no jobs after header!\n";
}


my $job_cnt;

sub get_jobs {
##############
# parse input format and create a hash for each job

    my @list_of_jobs; 
    my $prevN = 0;
    my $prevT = 0;

    #
    # scan trace and collect job info
    #

    while (<>) {

	#
	# parse line and extract info
	#

	if (/^\s*$/) {		# empty line
	    next;
	}

	$_ =~ /\s*(.*)\s*/;	# get rid of white space at ends
	$_ = $1;


	my @line = split(/\s+/, $_);
	if ($#line != 17) {
	    $format_bugs .= "format_problem $_\n";
	    warn(">>>Format problem on $_\ngot $#line fields (should be 17)");
	    next;
	}

	my $serial = $line[0];
	if ($serial != $prevN + 1) {
	    $format_bugs .= "non_serial prev=$prevN>>$_\n";
	}
	$prevN = $serial;

	my %nj;

	#
	# set the desired fields as specified in convert2swf.pl
	#

	$nj{trace}  = $_;	# used for error messages

	# record submit, start, and end times
	# log has problem of jobs with wait/run of ~15 months, probably bogus
	$nj{submit}    = $UTCstart + $line[1];
	if ($line[1] < $prevT) {
	    $format_bugs .= "bad_sequence prev=$prevT>>$_\n";
	}
	$prevT = $line[1];

	my $wait = $line[2];
	my $run  = $line[3];

	if ($wait >= 30000000) {
	    $format_bugs .= "long_wait $_\n";
	}
	if ($run >= 30000000) {
	    $format_bugs .= "long_run $_\n";
	}

	if (($wait < 30000000) && ($run < 30000000)) {
	    $nj{start} = $nj{submit} + $wait;
	    $nj{end}   = $nj{start}  + $run;
	}
	elsif ($run < 30000000) {
	    $nj{run}   = $run;
	}
	elsif ($wait < 30000000) {
	    $nj{start} = $nj{submit} + $wait;
	}

	$nj{procs}     = $line[4];

	$nj{cpu}       = $line[5];
	$nj{mem}       = $line[6];

	$nj{req_procs} = $line[7];
	$nj{req_time}  = $line[8];
	$nj{req_mem}   = $line[9];

	$nj{status}    = $line[10];

	$nj{user}      = $line[11];
	$nj{group}     = $line[12];
	$nj{app}       = $line[13];

	$nj{queue}     = $line[14];
	$nj{partition} = $line[15];

	$list_of_jobs[$job_cnt++] = \%nj;
    }

    return \@list_of_jobs;
}


sub regenerate_header {
#######################
    my ($zero, $maxtime) = @_;

    my $new;

    my @lines = split(/\n/, $header);

    foreach my $ln (@lines) {

	if ($ln =~ /MaxJobs/) {
	    $new .= "; MaxJobs: $job_cnt\n";
	}
	elsif ($ln =~ /MaxRecords/) {
	    $new .= "; MaxRecords: $job_cnt\n";
	}
	elsif ($ln =~ /UnixStartTime/) {
	    $new .= "; UnixStartTime: $zero\n";
	}
	elsif ($ln =~ /StartTime/) {
	    $new .= sprintf("; StartTime: %s\n",
			    strftime("%a %b %2d %H:%M:%S %Z %Y", localtime($zero)));
	}
	elsif ($ln =~ /EndTime/) {
	    $new .= sprintf("; EndTime:   %s\n",
			    strftime("%a %b %2d %H:%M:%S %Z %Y", localtime($maxtime)));
	}
	else {
	    $new .= "$ln\n";
	}
    }

    return $new;
}


sub get_partitions {
####################
# read partitions file and append to header
# (note: input is already in SWF, so partitions are given as numbers and not strings;
# here we just copy the original strings)

    my $fn = $inswf . ".partitions";

    open(PRT, "<$fn") || die "can't open partitions file";

    my @parts;
    my @part_desc;
    my $cnt = 0;

    while (<PRT>) {
	chomp;
	if (/^([\w\.,:;`'"~!@#$%^&*\-+=\\\/\(\)\[\]\{\}\?\<\>\|]+) (\d+)\s*$/) {
	    $parts[$cnt] = $2;
	    $part_desc[$2] = $1;
	}
	elsif (/(\d+)\s*$/) {
	    $parts[$cnt] = $1;
	    $part_desc[$1] = "UNKNOWN";
	}
	$cnt++;
    }

    return (\@parts, \@part_desc);
}

