#!/opt/local/bin/perl

use Env;
use Shell;
use File::Copy;

#
# This script is designed to re-calibrate past mk4 data
# automatically.  It will get raw mk4 data from MSS
# and calibrate it using the latest version of the
# calibration code.
#
# usage:  msscalmk4 <yrddoy> OR <year> <month> <day>
#
# Written by:	Tony Darnell (tdarnell@ucar.edu)   2 Jan 2002
#               16 May 2002 [ALS] replace domk4plot with domk4plots.
#		31 May 2002 [ALS] change fts2jpg scaling: 0 to 1000.
#		18 Jun 2002 [ALS] fts2jpg scaling: -75 to 925.
#				  Exclude Bad & Ugly files from calibration.
#		19 Jun 2002 [ALS] fts2jpg scaline: -100 to 900.
#		20 Jun 2002 [ALS] Only exclude Ugly files from calibration.

# Need this to run fts2jpg

$ENV{LD_LIBRARY_PATH}=
  "/orwell/d/tdarnell/SunOS/ImageMagick/lib:/orwell/d/tdarnell/SunOS/lib";

$domigs  = 1;
$dotar   = 1;
$docal   = 1;
$dojpgs  = 1;
$movem   = 1;
$cleanup = 1;
$user    = "stanger";

sub printtime($);
sub getmss($$$);

$numargs = @ARGV;
@months = ("01_jan","02_feb","03_mar","04_apr","05_may","06_jun","07_jul",
          "08_aug","09_sep","10_oct","11_nov","12_dec");

#--- Command line arguments.

if ($numargs != 1 && $numargs != 3) 
  {
  print "\nUsage:\t\tmsscalmk4 <yrddoy> OR <dataset> <year> <month> <day>\n\n";
  exit(1);
  }

if ($numargs == 1) 
  {
  $yrddoy = $ARGV[0];
  $yrmonday = `ydd $yrddoy`;
  $year = substr($yrmonday,0,4);
  $month = substr($yrmonday,5,2);
  $day = substr($yrmonday,8,2);
  $day = sprintf("%02d",$day);
  } 
else
  {
  $year  = @ARGV[0];
  $month = @ARGV[1];
  $day   = @ARGV[2];
  $yrddoy = `ydd $year $month $day`;
  chop($yrddoy);
  $yrddoy = substr($yrddoy,2,6);
  }

#--- Directory locations.

$webdir  = "/toshi/ftp/$year/@months[$month-1]";
#$datadir = "/puu/d/cordyn/mk4/Data/$yrddoy";
#$datadir = "/puu/f/cordyn/mk4/$yrddoy";
$datadir = "/toshi/ftp/d6/mk4/dp/$yrddoy";
$idxdir  = "/puu/f/cordyn/DATA_BASE";
$gbudir  = "/puu/f/cordyn/mk4_best/$year";
$engdir  = "/toshi/ftp/d5/mlso/log/engdata/mk4"; 

#--- Check Mk4 database file.  Look for "tar" file names.

open(MK4,"<$idxdir/midx/$yrddoy") or warn "No MK4  data for $yrddoy\n";
@all_lines = <MK4>;
close MK4;
@mk4files = grep{/rawmk4.tar/} @all_lines;
@mk4names = ();
@mk4lengths = ();

$num_lines = @all_lines;

print "\nWill retrieve the following files from MSS:\n\n";
print "\tFilename\t\t\tSize\n";
print "========================================================\n";
print "\n";

foreach $f (@mk4files)
  {
  @tmp = split " ", $f;
  print ("\t$tmp[0]\t\t@tmp[1]\n");
  push @mk4names,$tmp[0];
  push @mk4lengths,$tmp[1];
  }
print "\n";

#--- Create working directory.

system("mkdir $datadir");
chdir($datadir);

#--- Copy GBU file.

copy("$gbudir/$yrddoy.gbu","$datadir/$yrddoy.gbu");

$nummk4  = @mk4files;

if (!($nummk4)) 
  {
  print "No files to retrieve from MSS.\n";
  exit(1);
  }

# --- Fetch tar files from MSS.

if ($domigs)
  {
  $num_mk4files = @mk4files;
  foreach $f (@mk4names) 
    {
    $mssfile = "/CORDYN/MK4/Y$year/$f";
    getmss($mssfile, $datadir, $f);
    }

  $num_totalfiles = $num_mk4files;
  print "\n\n\t\t*** All files are back from MSS ***\n\n";
  }


#--- Now that files are here from MSS,
#--- unpack data from tar files.

if ($dotar)
  {
  @tarfiles = <*.tar>;
  foreach $f (@tarfiles) 
    { 
    print "\tUntarring $f\n";
    $tmp=`tar -xf $f`;
    }

#--- Uncompress data files.

  print "\tUncompressing files...\n";
  $tmp=`uncompress *.Z`;
  }

#---  Calibrate the data.

if ($docal)
  {
  open(GBU,"<$datadir/$yrddoy.gbu");
  @all_lines = <GBU>;
  close GBU;

  @calfiles = ();
  @goodfiles = grep{/Good/} @all_lines;
  @tmp = grep{/CALIBRATION/} @all_lines;
  foreach (@tmp) 
    {
    s/(CALIBRATION|Good)//g;
    s/^\s*(.*?)\s*$/$1/; # Trim whitespace
    }

  #--- Only use the first two cal files found

  push @calfiles, @tmp[0];
  push @calfiles, @tmp[1];
  @tmp = grep{/BEST/} @all_lines;
  foreach (@tmp)
    {
    s/(Good|BEST)//g;
    s/^\s*(.*?)\s*$/$1/;	# Trim whitespace
    }
  push @calfiles, @tmp;
  chomp @calfiles;

  #--- Get rid of calibration files from qlist.
  #--- Also, eliminate Ugly images.

  open(FILES, ">qlist");
  foreach (@all_lines)
    {
    if (m/CALIBRATION/){ next; }
    if (m/Ugly/){ next; }
    s/(DARK|DRK|BEST|Best|Good|Bad|Ugly)//g;
    s/^\s*(.*?)\s*$/$1/;	# Trim whitespace
    printf(FILES "$_\n");
    }
  close(FILES);

  #--- Use the first 15 Good files for making the average image.

  $j = 0;
  open(AVGS, ">listpBavg");
  foreach (@goodfiles)
    {
    if (m/CALIBRATION/){ next; }
    if (m/(BEST|DARK|DRK)/){ next; }
    s/Good//g;
    s/q/pB/;
    s/^\s*(.*?)\s*$/$1/; # Trim whitespace
    if ($j <= 15)
      {
      printf(AVGS "$_\n"); 
      }
    $j++;
    }
  close (AVGS);

  #--- This is the perl equivalent to "domk4cal".

  print("\tCalibrating Mk4 data...\n");
  open(CSH, "| /bin/csh") or die "Couldn't fork:  $!\n";
  print CSH "rawt4v2 *.rawmk4 >& sumry.rawt4v2\n";
  print CSH "getcal4v2 -I @calfiles >& sumry.getcal4v2\n";
  print CSH "getcal4v2 @calfiles >>& sumry.getcal4v2\n";
  print CSH "appcal4v2 -nH -nIU *.tscpinv `cat qlist` >& sumry.appcal4v2\n";
  print CSH "avgimg4 `cat listpBavg` >& sumry.avgimg4\n";
  print CSH "ls -1 *.mk4.pB *.mk4.pB.avg > pblist\n";
  print CSH "polvigv2 -M -O 200. `cat pblist` >& sumry.polvigv2\n";
  print CSH "ls -1 *.mk4.pB.vig *.mk4.pB.avg.vig > viglist\n";
  print CSH "polcarv2 `cat viglist` >& sumry.polcarv2\n";
  print CSH "ls -1 *.mk4.rpB.vig > rpBlist\n";
  print CSH "domk4plots $yrddoy\n";
  close(CSH);

  print "\tPost Processing MK4 files...\n";
  $i = 0;
  @cpbfiles = <*.mk4.pB *.mk4.pB.avg>;
  @rpbfiles = <*.mk4.rpB.vig *.mk4.rpB.avg.vig>;

  #--- Convert cpb & rpb files to FITS format.

  foreach $f (@cpbfiles)
    {
    print "Running mk4fits on $f\n";
    $tmp = `mk4fitsv2 $f 1>/dev/null 2>&1`;

    print "Running mk4fits on $rpbfiles[$i]\n";
    $tmp = `mk4fitsv2 $rpbfiles[$i] 1>/dev/null 2>&1`;
    $i++;
    }
  }

#--- Create JPEG files.

if ($dojpgs)
  {
  @jpegs = <*.mk4.rpb.vig.fts *.mk4.rpb.avg.vig.fts>;
  foreach $f (@jpegs)
    {
    print "Running fts2jpg on $f\n";
    $tmp = `/orwell/d/tdarnell/SunOS/bin/fts2jpg -l -100 -h 900 -t 250 $f`;
    }
  }

#--- Need to gzip only those files to be moved to ftp area.
#--- Only deal with jpegs and cpb images.

if ($movem)
  {
  system ("mkdir $webdir") if !(-e $webdir);
  @myfiles = ();
  @fitsfiles = <*.fts>;
  @avgfiles  = <*.pB.avg>;

  foreach (@avgfiles)
    {
    @tmpavg = `cp $_ /puu/f/cordyn/mk4_best/$year/$_.v2`;
    }

  @tscpfiles = <*.tscp>;
  foreach (@tscpfiles)
    {
    @tmp = `cp $_ /puu/f/cordyn/mk4_best/$year/$_.v2`;
    }

  @tmp = grep{/mk4.cpb.fts/} @fitsfiles;
  push @myfiles, @tmp;

  #--- Move JPEG files to web directory.

  print "Gzipping and moving files to $webdir\n";
  $tmp = `mv *.jpg $webdir`;

  #--- Move GIF files to engineering directory.

  $tmp = `mv *.gif $engdir/$year`;

  #--- Compress FITS files.
  #--- Move compressed FITS files to web directory.

  foreach $f (@myfiles)
    { 
    $tmp = `gzip $f`;
    print "Moving $f.gz to $webdir\n";
    $tmp = `mv $f.gz $webdir`; 
    }
  }

#--- Send e-mail message to indicate that processing is done.

$pid = open(MAILX, "| mailx -s 'msscalmk4 is complete for $yrddoy' $user 1>/dev/null 2>&1") or die "Couldn't fork: $!\n";
printf(MAILX "\n\nmsscalmk4 is complete for $yrddoy ($month/$day/$year)\n");
close(MAILX);

#--- Cleanup

if ($cleanup)
  {
  #`/bin/rm -fr $datadir`;
  #rmdir($datadir);
  #unlink(<*.[iuIU]>, <*pB*>, <*rawmk4*>, <*.sk>, <*.ba>, <*.st>, <*.fts>);
  unlink(<*.[iquIU]>, <*rpB*>, <*rawmk4>, <*.sk>, <*.ba>, <*.st>, <*.fts>);
  unlink (<*.pB>, <*vig>, <*jpg>, <*tar>);
  }
print "\n\nDone\n";

# ------------------------------------------------------------------------------
#  --- SUBROUTINES 
# ------------------------------------------------------------------------------

sub getmss ($$$)
  {
  $mssfile = @_[0];
  $dir = @_[1];
  $filename = @_[2];
  $cmnd = "msrcp -async mss:$mssfile $dir";
  $starttime = time();
  $status = system("$cmnd 1>cmnds");
  open(MSSID, "<cmnds");
  while (<MSSID>) {$mssid = $_;}
  close (MSSID);
  chomp($mssid);
  system("/bin/rm cmnds");

  if ($status == 0)
    {
    print "Waiting for $filename to return from MSS.\n";
    $tmp = system("dcswait $mssid");
    print "$filename has returned from MSS.\n";
    }

  $endtime = time();
  $elapsed_time = $endtime - $starttime;
  printtime($elapsed_time);
  }

# ------------------------------------------------------------------------------

sub printtime ($) 
  {
  my $elapsed_time = @_[0];
  my $hours_elapsed = sprintf("%02d",$elapsed_time / 3600);
  my $mins_elapsed = sprintf("%02d",($elapsed_time - 3600 * $hours_elapsed) / 60);
  my $secs_elapsed = sprintf("%02d",($elapsed_time % 60));
  print ("Time elapsed:  $hours_elapsed:$mins_elapsed:$secs_elapsed\n");
  }