#!/opt/local/bin/perl use Env; use Shell; use File::Copy; # # This script is designed to re-calibrate past mk4 data # automatically. It will get raw mk4 data from MSS # and calibrate it using the latest version of the # calibration code. # # usage: msscalmk4 OR # # Written by: Tony Darnell (tdarnell@ucar.edu) 2 Jan 2002 # 16 May 2002 [ALS] replace domk4plot with domk4plots. # 31 May 2002 [ALS] change fts2jpg scaling: 0 to 1000. # 18 Jun 2002 [ALS] fts2jpg scaling: -75 to 925. # Exclude Bad & Ugly files from calibration. # 19 Jun 2002 [ALS] fts2jpg scaline: -100 to 900. # 20 Jun 2002 [ALS] Only exclude Ugly files from calibration. # Need this to run fts2jpg $ENV{LD_LIBRARY_PATH}= "/orwell/d/tdarnell/SunOS/ImageMagick/lib:/orwell/d/tdarnell/SunOS/lib"; $domigs = 1; $dotar = 1; $docal = 1; $dojpgs = 1; $movem = 1; $cleanup = 1; $user = "stanger"; sub printtime($); sub getmss($$$); $numargs = @ARGV; @months = ("01_jan","02_feb","03_mar","04_apr","05_may","06_jun","07_jul", "08_aug","09_sep","10_oct","11_nov","12_dec"); #--- Command line arguments. if ($numargs != 1 && $numargs != 3) { print "\nUsage:\t\tmsscalmk4 OR \n\n"; exit(1); } if ($numargs == 1) { $yrddoy = $ARGV[0]; $yrmonday = `ydd $yrddoy`; $year = substr($yrmonday,0,4); $month = substr($yrmonday,5,2); $day = substr($yrmonday,8,2); $day = sprintf("%02d",$day); } else { $year = @ARGV[0]; $month = @ARGV[1]; $day = @ARGV[2]; $yrddoy = `ydd $year $month $day`; chop($yrddoy); $yrddoy = substr($yrddoy,2,6); } #--- Directory locations. $webdir = "/toshi/ftp/$year/@months[$month-1]"; #$datadir = "/puu/d/cordyn/mk4/Data/$yrddoy"; #$datadir = "/puu/f/cordyn/mk4/$yrddoy"; $datadir = "/toshi/ftp/d6/mk4/dp/$yrddoy"; $idxdir = "/puu/f/cordyn/DATA_BASE"; $gbudir = "/puu/f/cordyn/mk4_best/$year"; $engdir = "/toshi/ftp/d5/mlso/log/engdata/mk4"; #--- Check Mk4 database file. Look for "tar" file names. open(MK4,"<$idxdir/midx/$yrddoy") or warn "No MK4 data for $yrddoy\n"; @all_lines = ; close MK4; @mk4files = grep{/rawmk4.tar/} @all_lines; @mk4names = (); @mk4lengths = (); $num_lines = @all_lines; print "\nWill retrieve the following files from MSS:\n\n"; print "\tFilename\t\t\tSize\n"; print "========================================================\n"; print "\n"; foreach $f (@mk4files) { @tmp = split " ", $f; print ("\t$tmp[0]\t\t@tmp[1]\n"); push @mk4names,$tmp[0]; push @mk4lengths,$tmp[1]; } print "\n"; #--- Create working directory. system("mkdir $datadir"); chdir($datadir); #--- Copy GBU file. copy("$gbudir/$yrddoy.gbu","$datadir/$yrddoy.gbu"); $nummk4 = @mk4files; if (!($nummk4)) { print "No files to retrieve from MSS.\n"; exit(1); } # --- Fetch tar files from MSS. if ($domigs) { $num_mk4files = @mk4files; foreach $f (@mk4names) { $mssfile = "/CORDYN/MK4/Y$year/$f"; getmss($mssfile, $datadir, $f); } $num_totalfiles = $num_mk4files; print "\n\n\t\t*** All files are back from MSS ***\n\n"; } #--- Now that files are here from MSS, #--- unpack data from tar files. if ($dotar) { @tarfiles = <*.tar>; foreach $f (@tarfiles) { print "\tUntarring $f\n"; $tmp=`tar -xf $f`; } #--- Uncompress data files. print "\tUncompressing files...\n"; $tmp=`uncompress *.Z`; } #--- Calibrate the data. if ($docal) { open(GBU,"<$datadir/$yrddoy.gbu"); @all_lines = ; close GBU; @calfiles = (); @goodfiles = grep{/Good/} @all_lines; @tmp = grep{/CALIBRATION/} @all_lines; foreach (@tmp) { s/(CALIBRATION|Good)//g; s/^\s*(.*?)\s*$/$1/; # Trim whitespace } #--- Only use the first two cal files found push @calfiles, @tmp[0]; push @calfiles, @tmp[1]; @tmp = grep{/BEST/} @all_lines; foreach (@tmp) { s/(Good|BEST)//g; s/^\s*(.*?)\s*$/$1/; # Trim whitespace } push @calfiles, @tmp; chomp @calfiles; #--- Get rid of calibration files from qlist. #--- Also, eliminate Ugly images. open(FILES, ">qlist"); foreach (@all_lines) { if (m/CALIBRATION/){ next; } if (m/Ugly/){ next; } s/(DARK|DRK|BEST|Best|Good|Bad|Ugly)//g; s/^\s*(.*?)\s*$/$1/; # Trim whitespace printf(FILES "$_\n"); } close(FILES); #--- Use the first 15 Good files for making the average image. $j = 0; open(AVGS, ">listpBavg"); foreach (@goodfiles) { if (m/CALIBRATION/){ next; } if (m/(BEST|DARK|DRK)/){ next; } s/Good//g; s/q/pB/; s/^\s*(.*?)\s*$/$1/; # Trim whitespace if ($j <= 15) { printf(AVGS "$_\n"); } $j++; } close (AVGS); #--- This is the perl equivalent to "domk4cal". print("\tCalibrating Mk4 data...\n"); open(CSH, "| /bin/csh") or die "Couldn't fork: $!\n"; print CSH "rawt4v2 *.rawmk4 >& sumry.rawt4v2\n"; print CSH "getcal4v2 -I @calfiles >& sumry.getcal4v2\n"; print CSH "getcal4v2 @calfiles >>& sumry.getcal4v2\n"; print CSH "appcal4v2 -nH -nIU *.tscpinv `cat qlist` >& sumry.appcal4v2\n"; print CSH "avgimg4 `cat listpBavg` >& sumry.avgimg4\n"; print CSH "ls -1 *.mk4.pB *.mk4.pB.avg > pblist\n"; print CSH "polvigv2 -M -O 200. `cat pblist` >& sumry.polvigv2\n"; print CSH "ls -1 *.mk4.pB.vig *.mk4.pB.avg.vig > viglist\n"; print CSH "polcarv2 `cat viglist` >& sumry.polcarv2\n"; print CSH "ls -1 *.mk4.rpB.vig > rpBlist\n"; print CSH "domk4plots $yrddoy\n"; close(CSH); print "\tPost Processing MK4 files...\n"; $i = 0; @cpbfiles = <*.mk4.pB *.mk4.pB.avg>; @rpbfiles = <*.mk4.rpB.vig *.mk4.rpB.avg.vig>; #--- Convert cpb & rpb files to FITS format. foreach $f (@cpbfiles) { print "Running mk4fits on $f\n"; $tmp = `mk4fitsv2 $f 1>/dev/null 2>&1`; print "Running mk4fits on $rpbfiles[$i]\n"; $tmp = `mk4fitsv2 $rpbfiles[$i] 1>/dev/null 2>&1`; $i++; } } #--- Create JPEG files. if ($dojpgs) { @jpegs = <*.mk4.rpb.vig.fts *.mk4.rpb.avg.vig.fts>; foreach $f (@jpegs) { print "Running fts2jpg on $f\n"; $tmp = `/orwell/d/tdarnell/SunOS/bin/fts2jpg -l -100 -h 900 -t 250 $f`; } } #--- Need to gzip only those files to be moved to ftp area. #--- Only deal with jpegs and cpb images. if ($movem) { system ("mkdir $webdir") if !(-e $webdir); @myfiles = (); @fitsfiles = <*.fts>; @avgfiles = <*.pB.avg>; foreach (@avgfiles) { @tmpavg = `cp $_ /puu/f/cordyn/mk4_best/$year/$_.v2`; } @tscpfiles = <*.tscp>; foreach (@tscpfiles) { @tmp = `cp $_ /puu/f/cordyn/mk4_best/$year/$_.v2`; } @tmp = grep{/mk4.cpb.fts/} @fitsfiles; push @myfiles, @tmp; #--- Move JPEG files to web directory. print "Gzipping and moving files to $webdir\n"; $tmp = `mv *.jpg $webdir`; #--- Move GIF files to engineering directory. $tmp = `mv *.gif $engdir/$year`; #--- Compress FITS files. #--- Move compressed FITS files to web directory. foreach $f (@myfiles) { $tmp = `gzip $f`; print "Moving $f.gz to $webdir\n"; $tmp = `mv $f.gz $webdir`; } } #--- Send e-mail message to indicate that processing is done. $pid = open(MAILX, "| mailx -s 'msscalmk4 is complete for $yrddoy' $user 1>/dev/null 2>&1") or die "Couldn't fork: $!\n"; printf(MAILX "\n\nmsscalmk4 is complete for $yrddoy ($month/$day/$year)\n"); close(MAILX); #--- Cleanup if ($cleanup) { #`/bin/rm -fr $datadir`; #rmdir($datadir); #unlink(<*.[iuIU]>, <*pB*>, <*rawmk4*>, <*.sk>, <*.ba>, <*.st>, <*.fts>); unlink(<*.[iquIU]>, <*rpB*>, <*rawmk4>, <*.sk>, <*.ba>, <*.st>, <*.fts>); unlink (<*.pB>, <*vig>, <*jpg>, <*tar>); } print "\n\nDone\n"; # ------------------------------------------------------------------------------ # --- SUBROUTINES # ------------------------------------------------------------------------------ sub getmss ($$$) { $mssfile = @_[0]; $dir = @_[1]; $filename = @_[2]; $cmnd = "msrcp -async mss:$mssfile $dir"; $starttime = time(); $status = system("$cmnd 1>cmnds"); open(MSSID, ") {$mssid = $_;} close (MSSID); chomp($mssid); system("/bin/rm cmnds"); if ($status == 0) { print "Waiting for $filename to return from MSS.\n"; $tmp = system("dcswait $mssid"); print "$filename has returned from MSS.\n"; } $endtime = time(); $elapsed_time = $endtime - $starttime; printtime($elapsed_time); } # ------------------------------------------------------------------------------ sub printtime ($) { my $elapsed_time = @_[0]; my $hours_elapsed = sprintf("%02d",$elapsed_time / 3600); my $mins_elapsed = sprintf("%02d",($elapsed_time - 3600 * $hours_elapsed) / 60); my $secs_elapsed = sprintf("%02d",($elapsed_time % 60)); print ("Time elapsed: $hours_elapsed:$mins_elapsed:$secs_elapsed\n"); }