/home/production/cvs/JSOC/doc/devlev1_flatfield.txt 29Aug2012 This is the FF associated processing that occurs during the creation of the def lev1 data. ============================================================================ Cronjob on cl1n001 as user jsocprod: 30 18 * * * /home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/lev0a2deflev1_FULL_PZT_FSN_cron.pl hmi 30 20 * * * /home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/lev0a2deflev1_FULL_PZT_FSN_cron.pl aia ============================================================================ Lets look at the run of: /home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/lev0a2deflev1_FULL_PZT_FSN_cron.pl hmi #This is a cron job run by cl1n001 after the start of a new UT day. #For example, after 2010.10.26_17:00:00 PDT is UTC day 2010.300_UTC. #It will call lev0a2deflev1_FULL_PZT_FSN.pl with the date equal n-4 from #the new current date, e.g. after 2010.10.26_17:00:00 PDT, the call #will be: # lev0a2deflev1_FULL_PZT_FSN.pl hmi 2010.296_UC #New 16Feb2011 First do a pzt flat if($instru eq "hmi") { `/home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/pzt_flat_cron.pl`; } #Now make the def lev1 for our given date $cmd = "lev0a2deflev1_FULL_PZT_FSN.pl $instru $ord_date"; system($cmd); ============================================================================ /home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/pzt_flat_cron.pl #Run Richards pzt_flatfield IDL program. $cmd = "cd /home/jsocprod/pztflat; echo \"pzt_flatfield, $pztdate\" | /usr/local/bin/idl 1>> $QSUBDIR/pzt2.$PID.log 2>&1"; ============================================================================ /home/jsoc/cvs/Development/JSOC/scripts/lev0a2deflev1_FULL_PZT_FSN.pl $instru $ord_date #Run the build of the hmi.lev1 and fill in any hmi.lev1 gaps: #lev1_def_gui_called_PZT_FSN -x hmi 2010.298_UTC $cmd = "/home/jsoc/cvs/Development/JSOC/base/sums/scripts/lev1_def_gui_called_PZT_FSN -x $instru $ord_date 1> $statlog 2>&1"; ============================================================================ lev1_def_gui_called_PZT_FSN -x $instru $ord_date #These requirements must be met (hmi.lev1_probe DB consulted): # #1. For the current UTC day# (1-366), there must be a file like so: #/dds/soc2pipe/hmi,aia/xday/Xmit_All.198 #Optionally, the -o[verride] flag can be given for older data before #the Xmit_All.ddd was implemented. # #2. Do a query for the day of interest like so (note TAI): #sdo.hk_dayfile[2010.07.14_00:00:00_TAI][129][moc] #where MERGED=1. #If this exists, then the sdo.lev0_asd_0004 for this day is complete, #and ok to use for the definitive lev1. # #3. Check sdo.fds_orbit_vectors for a record for the given ord date. # #4. Check that the flatfield has a flatfield_version >= 1. #Check for camera 1&2 for hmi, and for the 23 wave_str for aia. # #5. Check that sdo.master_pointing record exists for our date. # #6. Check that hmi,aia.temperature_summary_300s exists for our date. # #7. Check that the manually set No Go file does not exist: #/surge40/jsocprod/lev0/data/NoDefLev1[HMI,AIA] #Now see if flat field has flatfield_version >= 1 (version ck obsolete) for($i=1; $i < 3; $i++) { $query = sprintf("%s[? t_start <= \$(%s) and t_stop > \$(%s) and CAMERA=%d ?]", 'hmi.flatfield', $fulldateFF, $fulldateFF, $i); #print "hmi query= $query\n"; #!!TEMP #print "Must put single quote around the above\n"; $cmd = "show_info key=date,flatfield_version '$query'"; print "$cmd\n"; @result = `$cmd`; #print "Result of flatfield query for $orddate:\n"; print "@result"; $x = shift(@result); if($x =~ /date\tflatfield_version/) { $x = shift(@result); #looks like 2010-07-01T17:28:23Z 1 ($a, $ffver) = split(/\s+/, $x); if($ffver >= 0) { #NOTE: now >= 0 is ok, just find one print "*OK flatfield found for CAMERA=$i\n"; } else { print "**NG flatfield_version not >= 1 for CAMERA=$i\n"; $OKff = 0; } } else { print "**NG flatfield not found for CAMERA=$i\n"; $OKff = 0; } } #back up 96 fsn for the way cosmic rays are made by module_flatfield_daily #See mail: "Re: patching cosmic ray records" Richard Wachter 11/18/10 12:08 $firstfsncosmic = $firstfsn - 96; #new 19Nov2010. Change vrbl 2/28/2011 $lastfsncosmic = $lastfsn - 96; #new 19Nov2010. Change vrbl 2/28/2011 $cmd = "time build_lev1_mgr mode=fsn instru=aia dsin=$DSINAIA dsout=$DSOUTAIA bfsn=$firstfsn efsn=$lastfsn numqsub=8 logfile=/usr/local/logs/lev1/build_lev1_mgr_aia.$utcdate.log"; $gdate = &get_date; $statlog = "$LOGDIR/module_flat_cron_48_PZT_FSN_$gdate.log"; $cmd = "module_flatfield_daily_cron_48_PZT_FSN.pl $instru $orddate $firstfsncosmic $lastfsncosmic 1> $statlog 2>&1"; print "$cmd\n"; if($EXECUTE) { @ffrun = `$cmd`; } ============================================================================ cvs/Development/JSOC/scripts/module_flatfield_daily_cron_48_PZT_FSN.pl $instru $orddate $firstfsncosmic $lastfsncosmic #Called after the start of a new UT day to make the hmi.cosmic_rays #and rotation flatfield and cosmic_ray_post #for the day from the hmi.lev1 data (that is typically made just before #this call). $cmd = "module_flatfield_daily_qsub_48_PZT_FSN.pl $indata $firstfsn $lastfsn $mdate"; ============================================================================ cvs/Development/JSOC/scripts/module_flatfield_daily_qsub_48_PZT_FSN.pl #Here are the 48 command that we must run: @cmds = ( "module_flatfield input_series='$inds' cadence=135 cosmic_rays=1 flatfield=1 fid=10054 camera=1 fsn_first=$firstfsn fsn_last=$lastfsn datum='$datum'", "module_flatfield input_series='$inds' cadence=135 cosmic_rays=1 flatfield=1 fid=10055 camera=1 fsn_first=$firstfsn fsn_last=$lastfsn datum='$datum'", [etc.] #Now run the combine program $cmd = "/home/jsoc/cvs/Development/JSOC/bin/linux_x86_64/module_flatfield_combine camera=2 input_series='su_production.flatfield_fid' datum='$datum' >& $lfile"; $cmd = "/home/jsoc/cvs/Development/JSOC/bin/linux_x86_64/module_flatfield_combine camera=1 input_series='su_production.flatfield_fid' datum='$datum' >& $lfile"; #New: 10Jan2011 Do cosmic_ray post processing. This populates hmi.cosmic_rays. #See mail richard@sun.stanford.edu 01/04/11 10:40 "cosmic ray series" #Here are the 24 command that we must run: @postcmds = ( "cosmic_ray_post input_series='su_production.cosmic_rays' fsn_first=$firstfsn fsn_last=$lastfsn datum='$datum' camera=1 hour=00", "cosmic_ray_post input_series='su_production.cosmic_rays' fsn_first=$firstfsn fsn_last=$lastfsn datum='$datum' camera=1 hour=02", [etc.]