#!/usr/local/bin/perl
#
# scan job execution logs from the LANL CM-5 system, and extract
# data about executed jobs
#
# the data about each job is contained in multiple non-contiguous entries.
# these are four SUBMIT entries, two RUN entries, and two FINISHED entries.
# foreign jobs hav a single FOREIGN entry instead of SUBMIT and RUN;
# some jobs may have KILL entries, and others are also possible though rare.
# the corresponding entries of the same type are identified by virtue of
# being on consecutive lines, and with the same timestamp (up to one second
# difference). corresponding entries of different types are identified by
# their job IDs.
#
# this script extracts the data on each job and prints it in a single line.
# the output format includes the following fields:
#   submit date and time (mon/day/hr:min:sec)
#   start date and time
#   end date and time
#   requested processors
#   used processors
#   requested CPU time
#   used CPU time
#   requested memory
#   used memory
#   queue
#   dedicated (y/n)
#   user
#   project
#   executable
#   exit code
#
# if a job is killed, the reason is given at the end of the line.
# in this case the exit code is followed by a "sigXX" field with a
# signal number.
# if certain fields are not known, "unknown" is given, so that the number
# and index of all fields is always the same.


# prepare for month conversions
$mon{"Jan"} = 1;
$mon{"Feb"} = 2;
$mon{"Mar"} = 3;
$mon{"Apr"} = 4;
$mon{"May"} = 5;
$mon{"Jun"} = 6;
$mon{"Jul"} = 7;
$mon{"Aug"} = 8;
$mon{"Sep"} = 9;
$mon{"Oct"} = 10;
$mon{"Nov"} = 11;
$mon{"Dec"} = 12;

#
# scan trace and collect job info
#
while (<>) {

    #
    # parse line and extract info.
    # this depends on which family of lines we are looking at.
    # at the end of each family, the data is collected in associative
    # arrays indexed by the job ID.
    #

    if (/SUBMIT/) {

	if (/^([\w: ]+) SUBMIT: user ([^,]+), .* file (.+)$/) {
	    $subline = $.;
	    $subtime = $1;
	    $user = $2;
	    $file = $3;
	    next;
	}
	if (/^([\w: ]+) SUBMIT: requested_nproc (\d+), requested_cputime (\d+), requested_memory (\d+)/) {
	    if (($1 ne $subtime) && (substr($1,-1) != (substr($subtime,-1)+1)%10)) {
		print STDERR ">>> Format problem in SUBMIT block at line $.\n";
	    }
	    $req_proc = $2;
	    $req_cpu  = $3;
	    $req_mem  = $4;
	    next;
	}
	if (/^([\w: ]+) SUBMIT: jid (\d+).*dedicated (\w)/) {
	    if (($1 ne $subtime) && (substr($1,-1) != (substr($subtime,-1)+1)%10)) {
		print STDERR ">>> Format problem in SUBMIT block at line $.\n";
	    }
	    $job = $2;
	    $ded = $3;
	    
	    # check for corrupt data
	    # use queue field to catch foreign jobs too
	    #if ($queue{$job}) {
	    #	print STDERR ">>> job ID at line $. already used at line $subline{$job}\n";
	    #}
	    #
	    # doesn't work: system reuses job IDs after a year or so...

	    next;
	}
	if (/^([\w: ]+) SUBMIT: queue (.+)$/) {
	    if (($1 ne $subtime) && (substr($1,-1) != (substr($subtime,-1)+1)%10)) {
		print STDERR ">>> Format problem in SUBMIT block at line $.\n";
	    }
	    $queue = $2;

	    # this is the last line in a submit block, so all variables
	    # should be filled by now. record them by job ID.

	    $subline{$job} = $subline;
	    $subtime{$job} = $subtime;
	    $user{$job} = $user;
	    $file{$job} = $file;
	    $req_proc{$job} = $req_proc;
	    $req_cpu{$job} = $req_cpu;
	    $req_mem{$job} = $req_mem;
	    $ded{$job} = $ded;
	    $queue{$job} = $queue;
	    next;
	}

	# if here, didn't match anything
	print STDERR ">>> Problem matching line $.:\n";
	print STDERR $_;
	next;
    }

    if (/RUN/) {

	if (/^([\w: ]+) RUN: job (\d+), user ([^,]+).*nproc (\d+)/) {
	    $startline = $.;
	    $starttime = $1;
	    $job = $2;
	    $user = $3;
	    $proc = $4;
	    if ($user ne $user{$job}) {
		if ($user{$job} eq "") {
		    if ($missing{$job} eq "") {
			print STDERR "*** missing SUBMIT info for job $job\n";
			$missing{$job} = "y";
		    }
		    $user{$job} = $user;
		} else {
		    print STDERR ">>> user at line $. differs from line $subline{$job}\n";
		}
	    }
	    next;
	}
	if (/^([\w: ]+) RUN: .* cputime (\d+), mem (\d+), project (.+)$/) {
	    if (($1 ne $starttime) && (substr($1,-1) != (substr($starttime,-1)+1)%10)) {
		print STDERR ">>> Format problem in RUN block at line $.\n";
	    }
	    if ($2 ne $req_cpu{$job}) {
		if ($req_cpu{$job} eq "") {
		    if ($missing{$job} eq "") {
			print STDERR "*** missing SUBMIT info for job $job\n";
			$missing{$job} = "y";
		    }
		    $req_cpu{$job} = $2;
		} else {
		    print STDERR ">>> cputime at line $. differs from line $subline{$job}\n";
		}
	    }
	    if ($3 ne $req_mem{$job}) {
		if ($req_mem{$job} eq "") {
		    if ($missing{$job} eq "") {
			print STDERR "*** missing SUBMIT info for job $job\n";
			$missing{$job} = "y";
		    }
		    $req_mem{$job} = $3;
		} else {
		    print STDERR ">>> memory at line $. differs from line $subline{$job}\n";
		}
	    }

	    # the only new things to record are
	    $startline{$job} = $startline;
	    $starttime{$job} = $starttime;
	    $proc{$job} = $proc;
	    $project{$job} = $4;
	    next;
	}

	# if here, didn't match anything
	print STDERR ">>> Problem matching line $.:\n";
	print STDERR $_;
	next;
    }

    if (/FOREIGN/) {

	if (/^([\w: ]+) FOREIGN: job (\d+), user ([^,]+).*nproc (\d+), project (.*)/) {
	    $job = $2;

	    # check for corrupt data
	    #if ($queue{$job}) {
	    #	print STDERR ">>> job ID at line $. already used at line $subline{$job}\n";
	    #}

	    $subline{$job} = $.;
	    $startline{$job} = $.;
	    $starttime{$job} = $1;
	    $user{$job} = $3;
	    $proc{$job} = $4;
	    $project{$job} = $5;
	    $queue{$job} = "FOREIGN";
	    $missing{$job} = "f";

	    # make sure data that is not available is clear
	    $subtime{$job} = "";
	    $file{$job} = "";
	    $req_proc{$job} = "";
	    $req_cpu{$job} = "";
	    $req_mem{$job} = "";
	    $ded{$job} = "";

	}
	next;
    }

    if (/KILL/) {

	if (/^([\w: ]+) KILL: User/) {
	    # nothing of interest
	    next;
	}
	if (/^([\w: ]+) KILL: job (\d+) user (\S+).*sig (\d+)/) {
	    $job = $2;
	    if ($3 ne $user{$job}) {
		if ($user{$job} eq "") {
		    if ($missing{$job} eq "") {
			print STDERR "*** missing SUBMIT and RUN info for job $job\n";
			$missing{$job} = "y";
		    }
		    $user{$job} = $3;
		} else {
		    if ($subline{$job} ne "") {
			print STDERR ">>> user at line $. differs from line $subline{$job}\n";
		    } else {
			print STDERR ">>> user at line $. differs from line $startline{$job}\n";
		    }
		}
	    }
	    $killtime = $1;
	    $sig{$job} = "sig" . $4 . " ";
	    next;
	}
	if (/^([\w: ]+) KILL: reason/) {
	    if (($1 ne $killtime) && (substr($1,-1) != (substr($killtime,-1)+1)%10)) {
		print STDERR ">>> Format problem in KILL block at line $.\n";
	    }
	    while (<>) {
		if (/\*\*\*/) {
		    chop;
		    $sig{$job} .= $_;
		} else {
		    last;
		}
	    }
	    $sig{$job} =~ s/\*//g;
	    next;
	}

	# if here, didn't match anything
	# this is typically an in-line reason
	if (/^([\w: ]+) KILL: (.*)/) {
	    if (($1 ne $killtime) && (substr($1,-1) != (substr($killtime,-1)+1)%10)) {
		print STDERR ">>> Format problem in KILL block at line $.\n";
	    }
	    $sig{$job} .= $2;
	}
	next;
    }

    if (/FINISHED/) {

	if (/^([\w: ]+) FINISHED: job (\d+).*exitcode (\w+), cputime (\d+), max_memory (\d+)/) {
	    $endline = $.;
	    $endtime = $1;
	    $job = $2;
	    $exit = $3;
	    $cpu = $4;
	    $mem = $5;
	    next;
	}
	if (/^([\w: ]+) FINISHED: user ([^,]+), file ([^,]+), project ([^,]+)/) {
	    if (($1 ne $endtime) && (substr($1,-1) != (substr($endtime,-1)+1)%10)) {
		print STDERR ">>> Format problem in FINISHED block at line $.\n";
	    }
	    if ($2 ne $user{$job}) {
		if ($user{$job} eq "") {
		    if ($missing{$job} eq "") {
			print STDERR "*** missing SUBMIT and RUN info for job $job\n";
			$missing{$job} = "y";
		    }
		    $user{$job} = $2;
		} else {
		    if ($subline{$job} ne "") {
			print STDERR ">>> user at line $. differs from line $subline{$job}\n";
		    } else {
			print STDERR ">>> user at line $. differs from line $startline{$job}\n";
		    }
		}
	    }
	    if ($3 ne $file{$job}) {
		if ($file{$job} eq "") {
		    if ($missing{$job} eq "") {
			print STDERR "*** missing SUBMIT info for job $job\n";
			$missing{$job} = "y";
		    }
		    $file{$job} = $3;
		} else {
		    print STDERR ">>> file at line $. differs from line $subline{$job}\n";
		}
	    }
	    if ($4 ne $project{$job}) {
		if ($project{$job} eq "") {
		    if ($missing{$job} eq "") {
			print STDERR "*** missing RUN info for job $job\n";
			$missing{$job} = "y";
		    }
		    $project{$job} = $4;
		} else {
		    print STDERR ">>> project at line $. differs from line $startline{$job}\n";
		}
	    }

	    # get year from filename.
	    # this is correct except for end of log files on Dec 31...
	    # corrected by hand in the output.
	    $ARGV =~ /log.(\d\d)/;
	    $year = $1;

	    #
	    # we now have all the data, and can print it out
	    #

	    if ($subtime{$job} =~ /(\w+)\s+(\d+)\s(\d\d:\d\d:\d\d)/) {
		$subtime = $year . "/" . $mon{$1} . "/" . $2 . "-" . $3;
	    } else {
		if ($subtime{$job} eq "") {
		    $subtime = "unknown";
		} else {
		    print STDERR ">>> Format problem in submit time at line $subline{$job}\n";
		}
	    }
	    if ($starttime{$job} =~ /(\w+)\s+(\d+)\s(\d\d:\d\d:\d\d)/) {
		$starttime = $year . "/" . $mon{$1} . "/" . $2 . "-" . $3;
	    } else {
		if ($starttime{$job} eq "") {
		    $starttime = "unknown";
		} else {
		    print STDERR ">>> Format problem in start time at line $startline{$job}\n";
		}
	    }
	    if ($endtime =~ /(\w+)\s+(\d+)\s(\d\d:\d\d:\d\d)/) {
		$endtime = $year . "/" . $mon{$1} . "/" . $2 . "-" . $3;
	    } else {
		print STDERR ">>> Format problem in end time at line $.\n";
	    }
	    if ($req_proc{$job} eq "") {
		$req_proc{$job} = "unknown";
	    }
	    if ($proc{$job} eq "") {
		$proc{$job} = "unknown";
	    }
	    if ($req_cpu{$job} eq "") {
		$req_cpu{$job} = "unknown";
	    }
	    if ($cpu{$job} eq "") {
		$cpu{$job} = "unknown";
	    }
	    if ($req_mem{$job} eq "") {
		$req_mem{$job} = "unknown";
	    }
	    if ($mem{$job} eq "") {
		$mem{$job} = "unknown";
	    }
	    if ($queue{$job} eq "") {
		$queue{$job} = "unknown";
	    }
	    if ($ded{$job} eq "") {
		$ded{$job} = "unknown";
	    }
	    if ($user{$job} eq "") {
		$user{$job} = "unknown";
	    }
	    if ($project{$job} eq "") {
		$project{$job} = "unknown";
	    }
	    if ($file{$job} eq "") {
		$file{$job} = "unknown";
	    }
	    if ($exit eq "") {
		$exit = "unknown";
	    }

	    print "$job $subtime $starttime $endtime $req_proc{$job} $proc{$job} $req_cpu{$job} $cpu $req_mem{$job} $mem $queue{$job} $ded{$job} $user{$job} $project{$job} $file{$job} $exit $sig{$job}\n";

	    #
	    # save memory?
	    #
	    delete $subtime{$job};
	    delete $user{$job};
	    delete $file{$job};
	    delete $req_proc{$job};
	    delete $req_cpu{$job};
	    delete $req_mem{$job};
	    delete $ded{$job};
	    delete $queue{$job};
	    delete $starttime{$job};
	    delete $proc{$job};
	    delete $project{$job};
	    delete $sig{$job};

	    next;
	}

	# if here, didn't match anything
	print STDERR ">>> Problem matching line $.:\n";
	print STDERR $_;
	next;
    }

    # here lines of different types, e.g. DELETE
}
