1 production 1.1 /home/production/cvs/JSOC/doc/devlev1_flatfield.txt		29Aug2012
  2                
  3                This is the FF associated processing that occurs during the 
  4                creation of the def lev1 data.
  5                ============================================================================
  6                
  7                Cronjob on cl1n001 as user jsocprod:
  8                30 18 * * * /home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/lev0a2deflev1_FULL_PZT_FSN_cron.pl hmi
  9                30 20 * * * /home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/lev0a2deflev1_FULL_PZT_FSN_cron.pl aia
 10                
 11                ============================================================================
 12                
 13                Lets look at the run of:
 14                /home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/lev0a2deflev1_FULL_PZT_FSN_cron.pl hmi
 15                
 16                #This is a cron job run by cl1n001 after the start of a new UT day.
 17                #For example, after 2010.10.26_17:00:00 PDT is UTC day 2010.300_UTC.
 18                #It will call lev0a2deflev1_FULL_PZT_FSN.pl with the date equal n-4 from
 19                #the new current date, e.g. after  2010.10.26_17:00:00 PDT, the call
 20                #will be:
 21                #  lev0a2deflev1_FULL_PZT_FSN.pl hmi 2010.296_UC
 22 production 1.1 
 23                #New 16Feb2011 First do a pzt flat
 24                if($instru eq "hmi") {
 25                  `/home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/pzt_flat_cron.pl`;
 26                }
 27                
 28                #Now make the def lev1 for our given date
 29                $cmd = "lev0a2deflev1_FULL_PZT_FSN.pl $instru $ord_date";
 30                system($cmd);
 31                
 32                ============================================================================
 33                
 34                /home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/pzt_flat_cron.pl
 35                
 36                #Run Richards pzt_flatfield IDL program.
 37                $cmd = "cd /home/jsocprod/pztflat; echo \"pzt_flatfield, $pztdate\" | /usr/local/bin/idl 1>> $QSUBDIR/pzt2.$PID.log 2>&1";
 38                
 39                ============================================================================
 40                
 41                /home/jsoc/cvs/Development/JSOC/scripts/lev0a2deflev1_FULL_PZT_FSN.pl $instru $ord_date
 42                
 43 production 1.1 #Run the build of the hmi.lev1 and fill in any hmi.lev1 gaps:
 44                #lev1_def_gui_called_PZT_FSN -x hmi 2010.298_UTC
 45                 $cmd = "/home/jsoc/cvs/Development/JSOC/base/sums/scripts/lev1_def_gui_called_PZT_FSN -x $instru $ord_date 1> $statlog 2>&1";
 46                
 47                ============================================================================
 48                
 49                lev1_def_gui_called_PZT_FSN -x $instru $ord_date
 50                
 51                #These requirements must be met (hmi.lev1_probe DB consulted):
 52                #
 53                #1. For the current UTC day# (1-366), there must be a file like so:
 54                #/dds/soc2pipe/hmi,aia/xday/Xmit_All.198
 55                #Optionally, the -o[verride] flag can be given for older data before
 56                #the Xmit_All.ddd was implemented.
 57                #
 58                #2. Do a query for the day of interest like so (note TAI):
 59                #sdo.hk_dayfile[2010.07.14_00:00:00_TAI][129][moc]
 60                #where MERGED=1.
 61                #If this exists, then the sdo.lev0_asd_0004 for this day is complete,
 62                #and ok to use for the definitive lev1.
 63                #
 64 production 1.1 #3. Check sdo.fds_orbit_vectors for a record for the given ord date.
 65                #
 66                #4. Check that the flatfield has a flatfield_version >= 1.
 67                #Check for camera 1&2 for hmi, and for the 23 wave_str for aia.
 68                #
 69                #5. Check that sdo.master_pointing record exists for our date.
 70                #
 71                #6. Check that hmi,aia.temperature_summary_300s exists for our date.
 72                #
 73                #7. Check that the manually set No Go file does not exist:
 74                #/surge40/jsocprod/lev0/data/NoDefLev1[HMI,AIA]
 75                
 76                  #Now see if flat field has flatfield_version >= 1 (version ck obsolete)
 77                  for($i=1; $i < 3; $i++) {
 78                    $query = sprintf("%s[? t_start <= \$(%s) and t_stop > \$(%s) and CAMERA=%d ?]", 'hmi.flatfield', $fulldateFF, $fulldateFF, $i);
 79                
 80                    #print "hmi query= $query\n"; #!!TEMP
 81                    #print "Must put single quote around the above\n";
 82                    $cmd = "show_info key=date,flatfield_version '$query'";
 83                    print "$cmd\n";
 84                    @result = `$cmd`;
 85 production 1.1     #print "Result of flatfield query for $orddate:\n";
 86                    print "@result";
 87                    $x = shift(@result);
 88                    if($x =~ /date\tflatfield_version/) {
 89                      $x = shift(@result);      #looks like 2010-07-01T17:28:23Z   1
 90                      ($a, $ffver) = split(/\s+/, $x);
 91                      if($ffver >= 0) {         #NOTE: now >= 0 is ok, just find one
 92                        print "*OK flatfield found for CAMERA=$i\n";
 93                      } else {
 94                        print "**NG flatfield_version not >= 1  for CAMERA=$i\n";
 95                        $OKff = 0;
 96                      }
 97                    }
 98                    else {
 99                      print "**NG flatfield not found for CAMERA=$i\n";
100                      $OKff = 0;
101                    }
102                  }
103                
104                  #back up 96 fsn for the way cosmic rays are made by module_flatfield_daily
105                  #See mail: "Re: patching cosmic ray records" Richard Wachter 11/18/10 12:08
106 production 1.1   $firstfsncosmic = $firstfsn - 96;     #new 19Nov2010. Change vrbl 2/28/2011
107                  $lastfsncosmic = $lastfsn - 96;       #new 19Nov2010. Change vrbl 2/28/2011
108                
109                    $cmd = "time build_lev1_mgr mode=fsn instru=aia dsin=$DSINAIA dsout=$DSOUTAIA bfsn=$firstfsn efsn=$lastfsn numqsub=8 logfile=/usr/local/logs/lev1/build_lev1_mgr_aia.$utcdate.log";
110                
111                
112                  $gdate = &get_date;
113                  $statlog = "$LOGDIR/module_flat_cron_48_PZT_FSN_$gdate.log";
114                  $cmd = "module_flatfield_daily_cron_48_PZT_FSN.pl $instru $orddate $firstfsncosmic $lastfsncosmic 1> $statlog 2>&1";
115                  print "$cmd\n";
116                  if($EXECUTE) {
117                    @ffrun = `$cmd`;
118                  }
119                
120                ============================================================================
121                
122                cvs/Development/JSOC/scripts/module_flatfield_daily_cron_48_PZT_FSN.pl $instru $orddate $firstfsncosmic $lastfsncosmic 
123                
124                #Called after the start of a new UT day to make the hmi.cosmic_rays
125                #and  rotation flatfield and cosmic_ray_post
126                #for the day from the hmi.lev1 data (that is typically made just before
127 production 1.1 #this call).
128                
129                $cmd = "module_flatfield_daily_qsub_48_PZT_FSN.pl $indata $firstfsn $lastfsn $mdate";
130                
131                ============================================================================
132                
133                cvs/Development/JSOC/scripts/module_flatfield_daily_qsub_48_PZT_FSN.pl
134                
135                #Here are the 48 command that we must run:
136                @cmds = (
137                "module_flatfield input_series='$inds' cadence=135 cosmic_rays=1 flatfield=1 fid=10054 camera=1 fsn_first=$firstfsn fsn_last=$lastfsn datum='$datum'",
138                "module_flatfield input_series='$inds' cadence=135 cosmic_rays=1 flatfield=1 fid=10055 camera=1 fsn_first=$firstfsn fsn_last=$lastfsn datum='$datum'",
139                [etc.]
140                
141                #Now run the combine program
142                $cmd = "/home/jsoc/cvs/Development/JSOC/bin/linux_x86_64/module_flatfield_combine camera=2 input_series='su_production.flatfield_fid' datum='$datum' >& $lfile";
143                
144                $cmd = "/home/jsoc/cvs/Development/JSOC/bin/linux_x86_64/module_flatfield_combine camera=1 input_series='su_production.flatfield_fid' datum='$datum' >& $lfile";
145                
146                
147                #New: 10Jan2011 Do cosmic_ray post processing. This populates hmi.cosmic_rays.
148 production 1.1 #See mail richard@sun.stanford.edu 01/04/11 10:40 "cosmic ray series"
149                #Here are the 24 command that we must run:
150                @postcmds = (
151                "cosmic_ray_post input_series='su_production.cosmic_rays' fsn_first=$firstfsn fsn_last=$lastfsn datum='$datum' camera=1 hour=00",
152                "cosmic_ray_post input_series='su_production.cosmic_rays' fsn_first=$firstfsn fsn_last=$lastfsn datum='$datum' camera=1 hour=02",
153                [etc.]
154