Return to devlev1_flatfield.txt CVS log

Up to [Development] / JSOC / doc

*** empty log message ***

/home/production/cvs/JSOC/doc/devlev1_flatfield.txt		29Aug2012

This is the FF associated processing that occurs during the 
creation of the def lev1 data.
============================================================================

Cronjob on cl1n001 as user jsocprod:
30 18 * * * /home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/lev0a2deflev1_FULL_PZT_FSN_cron.pl hmi
30 20 * * * /home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/lev0a2deflev1_FULL_PZT_FSN_cron.pl aia

============================================================================

Lets look at the run of:
/home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/lev0a2deflev1_FULL_PZT_FSN_cron.pl hmi

#This is a cron job run by cl1n001 after the start of a new UT day.
#For example, after 2010.10.26_17:00:00 PDT is UTC day 2010.300_UTC.
#It will call lev0a2deflev1_FULL_PZT_FSN.pl with the date equal n-4 from
#the new current date, e.g. after  2010.10.26_17:00:00 PDT, the call
#will be:
#  lev0a2deflev1_FULL_PZT_FSN.pl hmi 2010.296_UC

#New 16Feb2011 First do a pzt flat
if($instru eq "hmi") {
  `/home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/pzt_flat_cron.pl`;
}

#Now make the def lev1 for our given date
$cmd = "lev0a2deflev1_FULL_PZT_FSN.pl $instru $ord_date";
system($cmd);

============================================================================

/home/jsoc/cvs/Development/JSOC/proj/lev0/scripts/pzt_flat_cron.pl

#Run Richards pzt_flatfield IDL program.
$cmd = "cd /home/jsocprod/pztflat; echo \"pzt_flatfield, $pztdate\" | /usr/local/bin/idl 1>> $QSUBDIR/pzt2.$PID.log 2>&1";

============================================================================

/home/jsoc/cvs/Development/JSOC/scripts/lev0a2deflev1_FULL_PZT_FSN.pl $instru $ord_date

#Run the build of the hmi.lev1 and fill in any hmi.lev1 gaps:
#lev1_def_gui_called_PZT_FSN -x hmi 2010.298_UTC
 $cmd = "/home/jsoc/cvs/Development/JSOC/base/sums/scripts/lev1_def_gui_called_PZT_FSN -x $instru $ord_date 1> $statlog 2>&1";

============================================================================

lev1_def_gui_called_PZT_FSN -x $instru $ord_date

#These requirements must be met (hmi.lev1_probe DB consulted):
#
#1. For the current UTC day# (1-366), there must be a file like so:
#/dds/soc2pipe/hmi,aia/xday/Xmit_All.198
#Optionally, the -o[verride] flag can be given for older data before
#the Xmit_All.ddd was implemented.
#
#2. Do a query for the day of interest like so (note TAI):
#sdo.hk_dayfile[2010.07.14_00:00:00_TAI][129][moc]
#where MERGED=1.
#If this exists, then the sdo.lev0_asd_0004 for this day is complete,
#and ok to use for the definitive lev1.
#
#3. Check sdo.fds_orbit_vectors for a record for the given ord date.
#
#4. Check that the flatfield has a flatfield_version >= 1.
#Check for camera 1&2 for hmi, and for the 23 wave_str for aia.
#
#5. Check that sdo.master_pointing record exists for our date.
#
#6. Check that hmi,aia.temperature_summary_300s exists for our date.
#
#7. Check that the manually set No Go file does not exist:
#/surge40/jsocprod/lev0/data/NoDefLev1[HMI,AIA]

  #Now see if flat field has flatfield_version >= 1 (version ck obsolete)
  for($i=1; $i < 3; $i++) {
    $query = sprintf("%s[? t_start <= \$(%s) and t_stop > \$(%s) and CAMERA=%d ?]", 'hmi.flatfield', $fulldateFF, $fulldateFF, $i);

    #print "hmi query= $query\n"; #!!TEMP
    #print "Must put single quote around the above\n";
    $cmd = "show_info key=date,flatfield_version '$query'";
    print "$cmd\n";
    @result = `$cmd`;
    #print "Result of flatfield query for $orddate:\n";
    print "@result";
    $x = shift(@result);
    if($x =~ /date\tflatfield_version/) {
      $x = shift(@result);      #looks like 2010-07-01T17:28:23Z   1
      ($a, $ffver) = split(/\s+/, $x);
      if($ffver >= 0) {         #NOTE: now >= 0 is ok, just find one
        print "*OK flatfield found for CAMERA=$i\n";
      } else {
        print "**NG flatfield_version not >= 1  for CAMERA=$i\n";
        $OKff = 0;
      }
    }
    else {
      print "**NG flatfield not found for CAMERA=$i\n";
      $OKff = 0;
    }
  }

  #back up 96 fsn for the way cosmic rays are made by module_flatfield_daily
  #See mail: "Re: patching cosmic ray records" Richard Wachter 11/18/10 12:08
  $firstfsncosmic = $firstfsn - 96;     #new 19Nov2010. Change vrbl 2/28/2011
  $lastfsncosmic = $lastfsn - 96;       #new 19Nov2010. Change vrbl 2/28/2011

    $cmd = "time build_lev1_mgr mode=fsn instru=aia dsin=$DSINAIA dsout=$DSOUTAIA bfsn=$firstfsn efsn=$lastfsn numqsub=8 logfile=/usr/local/logs/lev1/build_lev1_mgr_aia.$utcdate.log";


  $gdate = &get_date;
  $statlog = "$LOGDIR/module_flat_cron_48_PZT_FSN_$gdate.log";
  $cmd = "module_flatfield_daily_cron_48_PZT_FSN.pl $instru $orddate $firstfsncosmic $lastfsncosmic 1> $statlog 2>&1";
  print "$cmd\n";
  if($EXECUTE) {
    @ffrun = `$cmd`;
  }

============================================================================

cvs/Development/JSOC/scripts/module_flatfield_daily_cron_48_PZT_FSN.pl $instru $orddate $firstfsncosmic $lastfsncosmic 

#Called after the start of a new UT day to make the hmi.cosmic_rays
#and  rotation flatfield and cosmic_ray_post
#for the day from the hmi.lev1 data (that is typically made just before
#this call).

$cmd = "module_flatfield_daily_qsub_48_PZT_FSN.pl $indata $firstfsn $lastfsn $mdate";

============================================================================

cvs/Development/JSOC/scripts/module_flatfield_daily_qsub_48_PZT_FSN.pl

#Here are the 48 command that we must run:
@cmds = (
"module_flatfield input_series='$inds' cadence=135 cosmic_rays=1 flatfield=1 fid=10054 camera=1 fsn_first=$firstfsn fsn_last=$lastfsn datum='$datum'",
"module_flatfield input_series='$inds' cadence=135 cosmic_rays=1 flatfield=1 fid=10055 camera=1 fsn_first=$firstfsn fsn_last=$lastfsn datum='$datum'",
[etc.]

#Now run the combine program
$cmd = "/home/jsoc/cvs/Development/JSOC/bin/linux_x86_64/module_flatfield_combine camera=2 input_series='su_production.flatfield_fid' datum='$datum' >& $lfile";

$cmd = "/home/jsoc/cvs/Development/JSOC/bin/linux_x86_64/module_flatfield_combine camera=1 input_series='su_production.flatfield_fid' datum='$datum' >& $lfile";


#New: 10Jan2011 Do cosmic_ray post processing. This populates hmi.cosmic_rays.
#See mail richard@sun.stanford.edu 01/04/11 10:40 "cosmic ray series"
#Here are the 24 command that we must run:
@postcmds = (
"cosmic_ray_post input_series='su_production.cosmic_rays' fsn_first=$firstfsn fsn_last=$lastfsn datum='$datum' camera=1 hour=00",
"cosmic_ray_post input_series='su_production.cosmic_rays' fsn_first=$firstfsn fsn_last=$lastfsn datum='$datum' camera=1 hour=02",
[etc.]