package Subs::SW0WrapUp; ############################################################################## # # DESCRIPTION: This subroutine class handles a number of cleanup tasks which # DESCRIPTION: need to be done at the end of each processing run, # DESCRIPTION: including creating output catalog files and verifying # DESCRIPTION: the product files. Each of these functions is handled # DESCRIPTION: by a separate method to make it easier to write a sub-class # DESCRIPTION: which overriseds the default body method. # DESCRIPTION: # DESCRIPTION: Each processing script should run this subroutine class # DESCRIPTION: (or a decendant of it) last. # # HISTORY: # HISTORY: $Log: SW0WrapUp.pm,v $ # HISTORY: Revision 1.65 2011/01/20 17:45:54 apsop # HISTORY: Added code to determine the value of ATTFlag based on which # HISTORY: attitude file has been used # HISTORY: # HISTORY: Revision 1.64 2008/12/10 13:57:02 apsop # HISTORY: Previous commit was bogus. This is the fix. # HISTORY: # HISTORY: Revision 1.63 2008/12/10 13:51:35 apsop # HISTORY: Fix problem with multiple copies of GTI extensions. # HISTORY: # HISTORY: Revision 1.62 2008/06/23 16:40:34 apsop # HISTORY: Previous checkin was a mistake. Fix. # HISTORY: # HISTORY: Revision 1.61 2008/06/23 14:12:40 apsop # HISTORY: Check for existence of files before applying aspect correction. # HISTORY: # HISTORY: Revision 1.60 2007/11/20 15:55:30 apsop # HISTORY: Fix typo in calling cleanup_files. # HISTORY: # HISTORY: Revision 1.59 2007/09/28 17:14:48 apsop # HISTORY: Change writing of ATTFLAG keyword so that it is a string rather than an integer. # HISTORY: # HISTORY: Revision 1.58 2007/09/11 18:01:26 apsop # HISTORY: Look for and use ATTFLAG value in primary header of files before using default value. # HISTORY: # HISTORY: Revision 1.57 2007/07/25 20:17:57 apsop # HISTORY: Write REPROC keyword if needed. # HISTORY: # HISTORY: Revision 1.56 2007/06/28 20:56:37 apsop # HISTORY: Don;t write standard keywords to scaled map files. # HISTORY: # HISTORY: Revision 1.55 2007/04/02 15:03:47 apsop # HISTORY: Uncomment cleanup_files(), which was accidentally comment out. # HISTORY: # HISTORY: Revision 1.54 2007/04/01 19:11:30 apsop # HISTORY: Move calculation of version numbers to SwiftSub.pm # HISTORY: # HISTORY: Revision 1.53 2007/03/15 21:25:57 apsop # HISTORY: Remove hard coding of cycle number. # HISTORY: # HISTORY: Revision 1.52 2007/02/27 19:36:37 apsop # HISTORY: Get trigger time from jobpar burst_time param. # HISTORY: # HISTORY: Revision 1.51 2007/02/13 15:38:01 apsop # HISTORY: Remove ATTSTATU keyword from *sat.fits file. # HISTORY: # HISTORY: Revision 1.50 2007/02/08 21:54:54 apsop # HISTORY: Fix bug in getting ATTSTATU value. # HISTORY: # HISTORY: Revision 1.49 2007/02/01 15:01:53 apsop # HISTORY: Do not write UTCFINIT to attitdue files. Write attitude status keyword to files. # HISTORY: # HISTORY: Revision 1.48 2006/09/20 20:32:26 apsop # HISTORY: Remove trailing _S from swift version string, if present. # HISTORY: # HISTORY: Revision 1.47 2006/09/10 20:00:38 apsop # HISTORY: Make parsing of ftools version compatible with ape. Call swiftversion as a generic tool. # HISTORY: # HISTORY: Revision 1.46 2006/07/03 01:44:59 apsop # HISTORY: Fix bug in detecting 900+ segments. # HISTORY: # HISTORY: Revision 1.45 2006/06/28 19:05:16 apsop # HISTORY: For 900 sequences, change the root when searching for ack messages. # HISOTRY: Set error condition if any level 2 errors occur on the final processing. # HISTORY: # HISTORY: Revision 1.44 2006/05/02 13:13:21 apsop # HISTORY: Do not try and get EXTNAME from hdu 0. # HISTORY: # HISTORY: Revision 1.43 2006/04/27 16:02:48 apsop # HISTORY: Add message for tracking keyword updates in log file. # HISTORY: # HISTORY: Revision 1.42 2006/01/12 17:05:42 apsop # HISTORY: Remove repository bgaoff and bdetflag files in clean up. # HISTORY: # HISTORY: Revision 1.41 2006/01/05 23:26:06 apsop # HISTORY: Implemented merging of BAT detector enable/disable and gain/offset maps # HISTORY: for the observation and pruned the work-around that saved them with # HISTORY: index numbers. # HISTORY: # HISTORY: Revision 1.40 2005/12/19 16:17:10 apsop # HISTORY: Export bat catalog files to repository. Remove any leftover bat tdrss # HISTORY: attitude data files. # HISTORY: # HISTORY: Revision 1.39 2005/11/20 20:36:11 apsop # HISTORY: Fix bug in renaming of bgaoff type files. # HISTORY: # HISTORY: Revision 1.37 2005/11/08 19:22:28 apsop # HISTORY: Populate the TIMELIST and DATALIST hashes. Used to be an SWCheckInput. # HISTORY: # HISTORY: Revision 1.36 2005/10/28 18:16:49 apsop # HISTORY: Fix bug in renaming of gain/off and detector flag files. # HISTORY: # HISTORY: Revision 1.35 2005/10/26 17:30:08 apsop # HISTORY: New subroutine for cleaning up files that will not be catalogued. Quick fix to # HISTORY: rename gain/offset and det enable bat files. # HISTORY: # HISTORY: Revision 1.34 2005/08/31 16:05:23 apsop # HISTORY: Add TRIGTIME keyword to all non-trdss products if available. # HISTORY: # HISTORY: Revision 1.33 2005/07/29 14:20:10 apsop # HISTORY: Fix bug in running local version of fverify. Switch to ftverify. # HISTORY: # HISTORY: Revision 1.32 2005/04/19 15:34:25 apsop # HISTORY: Add in CALDBVER keyword. Fix up some keyword comments. # HISTORY: # HISTORY: Revision 1.31 2005/03/18 20:17:48 apsop # HISTORY: Put orbit files back into the distribution. # HISTORY: # HISTORY: Revision 1.30 2005/03/17 16:50:06 apsop # HISTORY: Remove TLE file from distribution. # HISTORY: # HISTORY: Revision 1.29 2005/03/15 19:54:54 apsop # HISTORY: Add in code taken from StartEndTimes for writing UTCFINIT keyword. # HISTORY: # HISTORY: # HISTORY: Revision 1.28 2005/03/02 18:19:16 apsop # HISTORY: Delete any sha1 checksum files still around. # HISTORY: # HISTORY: Revision 1.27 2004/12/22 18:11:57 apsop # HISTORY: Fix up setting of TIERRELA TIERABSO values so that they have decimal points. # HISTORY: # HISTORY: Revision 1.26 2004/12/10 03:36:30 apsop # HISTORY: Do not write _PNT keywords to tdrss files. # HISTORY: # HISTORY: Revision 1.25 2004/12/03 13:35:37 apsop # HISTORY: Make compatible with test pipeline: st versus sw # HISTORY: # HISTORY: Revision 1.24 2004/11/29 15:25:03 apsop # HISTORY: Fix format of TARG_ID and SEG_NUM keywords # HISTORY: # HISTORY: Revision 1.22 2004/11/19 21:46:48 apsop # HISTORY: New version of xrt2fits. # HISTORY: # HISTORY: Revision 1.21 2004/11/16 15:35:58 apsop # HISTORY: Call swiftversion as a real headas tool. # HISTORY: # HISTORY: Revision 1.20 2004/11/09 23:42:29 apsop # HISTORY: Remove telemetry files before making catalogues. # HISTORY: # HISTORY: Revision 1.19 2004/11/02 21:20:20 apsop # HISTORY: Write DATE* keywords into tapecat file. # HISTORY: # HISTORY: Revision 1.18 2004/10/25 14:52:07 apsop # HISTORY: Write timing keywords to primary hdu. Some general reformatting. # HISTORY: # HISTORY: Revision 1.17 2004/10/13 19:53:42 apsop # HISTORY: Remove checksum file from future_files call. # HISTORY: # HISTORY: Revision 1.16 2004/10/12 16:22:21 apsop # HISTORY: Override and do not make the checksum file. Override verify method and turn # HISTORY: off testdata option in fverify to speed this up. Add decimal points where # HISTORY: needed in two keyowrd values. # HISTORY: # HISTORY: Revision 1.15 2004/09/14 18:54:32 apsop # HISTORY: Do not write SEQPNUM or (default) OBS_ID into tdrss messages. # HISTORY: # HISTORY: Revision 1.14 2004/09/02 00:06:22 apsop # HISTORY: Store software version and cycle in job.par file. # HISTORY: # HISTORY: Revision 1.13 2004/08/17 13:57:11 apsop # HISTORY: Write keywords for software version and for burst in bat source catalogue. # HISTORY: # HISTORY: Revision 1.12 2004/07/06 20:02:57 apsop # HISTORY: Keep track of calibration files and delete them at the end. # HISTORY: # HISTORY: Revision 1.11 2004/06/10 19:57:06 apsop # HISTORY: Add in writing of OBJ, TARG_ID, SEG_NUM and timing keywords. # HISTORY: # HISTORY: Revision 1.10 2004/06/07 17:03:44 apsop # HISTORY: Switch to using _PNT instead of _NOM for ra, dec and roll parameters. # HISTORY: # HISTORY: Revision 1.9 2004/05/28 19:44:06 apsop # HISTORY: Make sure leading zeroes are included in OBS_ID # HISTORY: # HISTORY: Revision 1.8 2004/05/06 20:02:34 dah # HISTORY: Add version number back into the header comments. # HISTORY: # HISTORY: Revision 1.7 2004/04/28 13:51:10 dah # HISTORY: Change SERQNUM keyword to OBS_ID, and remove leading zeroes from SEQPNUM. # HISTORY: # HISTORY: Revision 1.6 2004/04/16 20:21:18 dah # HISTORY: Begin using embedded history records # HISTORY: # # VERSION: 0.0 # # ############################################################################## use Subs::Sub; use Subs::WrapUp; use Util::SWCatalogue; @ISA = ("Subs::WrapUp"); use strict; ######################################### # constructor ######################################### sub new { my $proto=shift; my $self=$proto->SUPER::new(); $self->{DESCRIPTION}="Doing final wrapup of all swift files"; $self->{FILES}=(); $self->{FITS_FILES}=(); $self->{CHECKSUM_TYPE}="checksum"; $self->{HTML_CAT_TYPE}="fileinfo"; return $self; } ######################################################################### # add standard keywords to the headers of all the FITS files. ######################################################################### sub write_standard_keywords { my $self=shift; my $log = $self->log(); my $filename = $self->filename(); my $jobpar = $self->jobpar(); my $procpar = $self->procpar(); $log->entry("Writing standard keywords to all FITS files"); ####################### # Setup for UTCF value ####################### my $time_file = $filename->get('timedata', 'swift', '', 0); my (@utcf_times, @utcf); if( -f $time_file ){ my $time_fits = Util::FITSfile->new($time_file, 'UTCF'); @utcf_times = $time_fits->cols('TIME')->table(); @utcf = $time_fits->cols('UTCF')->table(); } my $soft_version = $jobpar->read('softver'); my $cal_version = $jobpar->read('caldbver'); my $reprocess = $jobpar->read('reprocess'); ############################ # Trigger time, if relevant ############################ my $trigtime = $jobpar->read('burst_time'); #################################### # loop over all FITS files #################################### my $file; my $sat_attitude = $filename->get('attcorr', 'u'); if (! -e $sat_attitude) { $sat_attitude = $filename->get('attcorr', 'p'); if (! -e $sat_attitude) { $sat_attitude = $filename->get('attitude', 's'); } } my $attflag = '100'; if($sat_attitude =~ /uat\.fits/){ $attflag = '111'; } elsif ($sat_attitude =~ /pat\.fits/){ $attflag = '110'; } foreach $file (Util::SWCatalogue->fits_files() ) { $log->entry("Doing file $file."); my $fits = Util::FITSfile->new($file); my $is_tdrss = $file =~ /s[wt][t\d]\d{10}ms/; my $is_batevt = $file =~ /s[wt][t\d]\d{10}bev/; my $is_eng = $file =~ /s[wt][t\d]\d{10}.*en\.hk$/; my $is_att = $file =~ /s[wt][t\d]\d{10}.at\.fits$/; my $is_sm = $file =~ /s[wt][t\d]\d{10}bsmcb\.fits$/; #################################### # Which attitude file is being used #################################### # my $attflag = '100'; ################################## # loop over HDUs in the FITS file ################################## my $nhdus = $fits->nhdus(); unless ($nhdus) { $log->error(2, "Cannot get number of HDUs for FITS file $file."); next; } my $hdu; for($hdu=0; $hdu<$nhdus; $hdu++) { $fits->ext($hdu); my $extname = $hdu==0 ? '' : $fits->keyword('EXTNAME'); ################################ # write keywords to the file ################################ $fits->begin_many_keywords(); $fits->keyword('PROCVER', $procpar->read('version'), 'Processing script version' ); unless( $is_eng || ($extname && $extname=~/GTI/) ){ $fits->keyword('SOFTVER', $soft_version, 'HEASOFT and Swift versions'); $fits->keyword('CALDBVER', $cal_version, 'CALDB index versions used'); } if($hdu==0){ $fits->keyword('TIMESYS', 'TT', 'time system'); $fits->keyword('MJDREFI', 51910, 'MJD reference day 01 Jan 2001 00:00:00'); $fits->keyword('MJDREFF', 7.428703700000000E-04, 'MJD reference (fraction of day) 01 Jan 2001 00:00:00'); $fits->keyword('CLOCKAPP', 'F', 'If clock correction are applied (F/T)'); $fits->keyword('TIMEUNIT', 's', 'Time unit for timing header keywords'); $fits->keyword('REPROC', 'T', 'Is this from a bulk reprocessing run?') if $reprocess eq 'yes'; # my $att0 = $fits->keyword('ATTFLAG'); # $attflag = $att0 if $att0; }else{ $fits->keyword('TIERRELA', '1.0E-8', '[s/s] relative errors expressed as rate'); $fits->keyword('TIERABSO', '1.0', '[s] timing precision in seconds'); } unless($is_tdrss || $is_sm){ $fits->keyword('OBS_ID', sprintf("'%011d'", $jobpar->read('sequence')), 'Observation ID' ); $fits->keyword('SEQPNUM', int($jobpar->read('seqprocnum')), 'Number of times the dataset processed' ); $fits->keyword('TARG_ID', int(sprintf("%d",$jobpar->read('target'))), 'Target ID'); $fits->keyword('SEG_NUM', int(sprintf("%d",$jobpar->read('obs'))), 'Segment number'); $fits->keyword('OBJECT', $jobpar->read('object'), 'Object name'); $fits->keyword('RA_OBJ', $jobpar->read('burst_ra'), '[deg] R.A. Object'); $fits->keyword('DEC_OBJ', $jobpar->read('burst_dec'), '[deg] Dec Object'); $fits->keyword('RA_PNT', $jobpar->read('ra'), '[deg] RA pointing'); $fits->keyword('DEC_PNT', $jobpar->read('dec'), '[deg] Dec pointing'); $fits->keyword('PA_PNT', $jobpar->read('roll'), '[deg] Position angle (roll)'); $fits->keyword('TRIGTIME', $trigtime, '[s] MET TRIGger Time for Automatic Target') if $trigtime; } if($is_batevt){ $fits->keyword('CATSRC', $jobpar->read('burst_cat_src') eq 'yes' ? 'T' : 'F'); } if( $is_att ){ $fits->keyword('-UTCFINIT', ' '); $fits->keyword('-ATTSTATU', ' '); }else{ $fits->keyword('ATTFLAG', "'$attflag'", 'Orgin of attitude information'); ############################# # Determine and set UTCFINIT ############################# my $tstart = $fits->keyword('TSTART'); unless( $tstart ){ my $date = $fits->keyword('DATE-OBS'); if( $date ){ my $start = Util::Date->new($date); $tstart = $start->seconds(); } } if( $tstart && @utcf ){ my $itime = 0; while( $tstart > $utcf_times[$itime] && $itime < $#utcf_times){ $itime++ } $fits->keyword('UTCFINIT', $utcf[$itime], '[s] UTCF at TSTART'); } } $fits->end_many_keywords(); } } } # end of write_standard_keywords method ########################################################################### # make all FITS catalogs and the HTML file catalog ########################################################################### sub make_catalogs { my $self=shift; my $log = $self->log(); my $jobpar = $self->jobpar(); my $filename=$self->filename(); $filename->export_to_repository('bcatalog', 'bat'); $filename->export_to_repository('rcatalog', 'bat'); $self->cleanup_files(); ###################################################### # get names of catalogs, and assume the HTML catalog # has the same files as the first FITS catalog ###################################################### my @fits_catalogs=$filename->catalog_types(); my $html_cat_is_like=$fits_catalogs[0]; ########################################################## # register the files we have yet to make with the # Catalog class ########################################################## Util::SWCatalogue->filename($filename); Util::SWCatalogue->future_files((map {$_=>'FITS'} @fits_catalogs), 'fileinfo' =>'HTML'); ############################################### # create FITS catalogs ############################################### my $type; foreach $type (@fits_catalogs ) { Util::SWCatalogue->new($type) ->make() ->register_in_parfile(); } my $date = Util::Date->new(); my $attitude = $filename->get('attitude', 's'); my $attstatus; if( -f $attitude ){ $attstatus = Util::FITSfile->new($attitude, 0)->keyword('ATTSTATU'); }else{ $attstatus = '200'; } my $fitscat = Util::FITSfile->new($filename->get('tapecat', 'proc', '', 0)); foreach my $ext (0,1){ $fitscat->ext($ext); $fitscat->keyword('DATE-OBS', $jobpar->read('obsdate') .'T'. $jobpar->read('obstime')); $fitscat->keyword('DATE-END', $jobpar->read('enddate') .'T'. $jobpar->read('endtime')); $fitscat->keyword('DATE', $date->date() .'T'. $date->time()); $fitscat->keyword('ATTSTATU', $attstatus, 'Status of corrected attitude files'); } ######################################## # create the HTML catalog ######################################## Util::HTMLcatalog->new($html_cat_is_like,'fileinfo') ->make(); } # end of make_catalogs method ######################################################## # Don't make a checksum file. This is done by the DTS ######################################################## sub make_checksum_file { my $self=shift; my $log=$self->log(); $log->entry('Not making a checksum file'); } # end of make_checksums method ##################################### # run fverify on all the FITS files ##################################### sub verify { my $self=shift; my $jobpar = $self->jobpar(); my $log=$self->log(); $log->entry('Running ftverify on all FITS files'); ####################################################### # set up fverify. Set testdata='no' to speed things up ####################################################### my $fverify=Util::HEAdas->new('ftverify') ->params({outfile=>'STDOUT', prhead=>'no', testdata=>'no'}) ->verbose(0); ###################################### # loop over all FITS files ###################################### my $file; foreach $file (Util::Catalog->fits_files() ) { $fverify->params({infile => $file}) ->run(); } ############################################# # If this is a final processing, and level 2 # errors are bad ############################################# if( $jobpar->{TIMELIST}->{final} && $jobpar->read('nprocerrors') ){ $jobpar->set({proc_error=>'yes'}); } } # end of verify method sub cleanup_files { my $self=shift; my $log = $self->log(); my $jobpar = $self->jobpar(); my $filename=$self->filename(); $log->entry("Cleaning up temporary files."); unlink @{$filename->{CAL_FILES}}; unlink $filename->any('telemetry'); unlink $filename->get('tdrsslcatt', 'bat', '', '*'); ### unlink $filename->fetch_orbit(); delete $filename->{INFO}->{b}->{bdetflag}->{repository}; unlink $filename->get('bdetflag', 'bat', '*', '*'); delete $filename->{INFO}->{b}->{bgaoff}->{repository}; unlink $filename->get('bgaoff', 'bat', '*', '*'); unlink glob '*.sha1'; unlink glob '*.lock'; unlink glob '*.flag'; unlink glob '*.queue'; unlink glob '*.front_end_id'; unlink $filename->get('hk', 'proc', 'badb', 0); }