JSOC_Documentation: proj/lev0/apps/build_lev1

00001 /*-----------------------------------------------------------------------------
00002  * cvs/JSOC/proj/lev1/apps/build_lev1.c
00003  *-----------------------------------------------------------------------------
00004  *
00005  * This is a module that runs with DRMS and processes lev0
00006  * filtergrams to lev1.
00007  * It is scheduled by build_lev1_mgr either by qsub to the cluster
00008  * or by fork to run on the local machine. It is called with 
00009  * 12 (NUMRECLEV1 defined in lev0lev1.h) lev0 images at a time.
00010  * See the -h nice_intro() for details of the calling options.
00011  */ 
00012 
00013 #include <jsoc_main.h>
00014 #include <cmdparams.h>
00015 #include <drms.h>
00016 #include <drms_names.h>
00017 #include <stdio.h>
00018 #include <stdlib.h>
00019 #include <ctype.h>
00020 #include <strings.h>
00021 #include <sys/types.h>
00022 #include <sys/time.h>
00023 #include <sys/stat.h>   //for umask(2)
00024 #include <unistd.h> //for alarm(2) among other things...
00025 #include <printk.h>
00026 #include <astro.h>
00027 #include <fresize.h>
00028 #include <gapfill.h>
00029 #include "list.h"
00030 #include </home/jsoc/include/fftw3.h> 
00031 //#include "fftw3.h" 
00032 #include "imgdecode.h"
00033 #include "lev0lev1.h"
00034 #include "quallev1.h"
00035 #include "limb_fit.h"
00036 #include "cosmic_ray.h"
00037 #include "get_pointing_info.c"
00038 
00039 //default in and out data series
00040 #define LEV0SERIESNAMEIRIS "iris_ground.lev0_dc1"
00041 #define LEV1SERIESNAMEIRIS "iris_ground.lev1_dc1"
00042 //#define DSCRSNAME "iris_ground.crs_table"
00043 #define DSCRSNAME "iris.crs_table"
00044 //#define DSCRSNAME "iris_ground.window_table"
00045 #define DSFFNAME "iris.flatfield_prelim"
00046 //#define DSFFNAME "iris_ground.flatfield_test"
00047 
00048 //#define DSFFNAME "su_richard.flatfield"       //temp test case
00049 //#define DSFFNAMEHMI "su_production.hmi_flatfield" //temp test case
00050 //#define DSFFNAMEHMI "hmi.flatfield"
00051 //#define DSFFNAMEAIA "su_production.aia_flatfield" //temp test case
00052 //#define DSFFNAMEAIA "aia_test.flatfield"  //temp test case
00053 //#define DSFFNAMEAIA "aia.flatfield"
00054 
00055 #define LEV1LOG_BASEDIR "/usr/local/logs/lev1"
00056 #define H1LOGFILE "/usr/local/logs/lev1/build_lev1_iris.%s.log"
00057 #define NUMTIMERS 8     //number of seperate timers avail
00058 #define NOTSPECIFIED "***NOTSPECIFIED***"
00059 #define LOGTEST 0
00060 #define CAL_HCFTID 17       //image is cal mode 
00061 #define STOP_FILE "/usr/local/logs/lev1/build_mgr_stop_iris"
00062 
00063 // SAA-HLZ
00064 #define SAA_HLZ_SERIES "iris.saa_hlz"
00065 
00066 int compare_rptr(const void *a, const void *b);
00067 static TIME SDO_to_DRMS_time(int sdo_s, int sdo_ss);
00068 
00069 // List of default parameter values. 
00070 ModuleArgs_t module_args[] = { 
00071   {ARG_STRING, "mode", NOTSPECIFIED, "either recnum or fsn"},
00072   {ARG_STRING, "dsin", NOTSPECIFIED, "dataset of lev0 filtergrams"},
00073   {ARG_STRING, "dsout", NOTSPECIFIED, "dataset of lev1 output"},
00074   {ARG_STRING, "logfile", NOTSPECIFIED, "optional log file name. Will create one if not given"},
00075   {ARG_INTS, "brec", "0", "first lev0 rec# to process. 0=error must be given by build_lev1_mgr"},
00076   {ARG_INTS, "erec", "0", "last lev0 rec# to process. 0=error must be given by build_lev1_mgr"},
00077   {ARG_INTS, "bfsn", "0", "first lev0 fsn# to process. 0=error must be given by build_lev1_mgr"},
00078   {ARG_INTS, "efsn", "0", "last lev0 fsn# to process. 0=error must be given by build_lev1_mgr"},
00079   {ARG_INTS, "quicklook", "1", "1=quick look, 0 = definitive mode"},
00080   {ARG_FLAG, "v", "0", "verbose flag"},
00081   {ARG_FLAG, "h", "0", "help flag"},
00082   {ARG_FLAG, "r", "0", "restart flag"},
00083   {ARG_END}
00084 };
00085 
00086 CmdParams_t cmdparams;
00087 // Module name presented to DRMS. 
00088 char *module_name = "build_lev1_iris_NEW";
00089 
00090 FILE *h1logfp;      // fp for h1 ouput log for this run 
00091 static struct stat stbuf;
00092 static LEV0LEV1 lev0lev1;
00093 static LEV0LEV1 *l0l1 = &lev0lev1;
00094 //static CCSDS_Packet_t *Hk;
00095 static DRMS_Record_t *rs;
00096 static DRMS_Record_t *crsrec;
00097 static DRMS_Record_t *rs0, *rs1, *rsff, *rsbad_pix, *rec_bad_aia, *rt, *rresp;
00098 static DRMS_Record_t *rptr;
00099 static DRMS_Segment_t *segment, *segment0;
00100 static DRMS_Segment_t *segmentff;
00101 static DRMS_Segment_t *darkseg;
00102 static DRMS_Segment_t *badseg;
00103 //static DRMS_Segment_t *badoutpixseg;
00104 //static DRMS_Segment_t *spikeseg;
00105 static DRMS_Array_t *segArray;
00106 static DRMS_RecordSet_t *crsset;
00107 static DRMS_RecordSet_t *rset0, *rset1, *rsetff, *rsbad_aia, *rs_t=NULL,
00108                         *rs_resp=NULL;
00109 static DRMS_Array_t *Array0;
00110 static DRMS_Array_t *Arrayff;
00111 static DRMS_Array_t *ArrayDark;
00112 static DRMS_Array_t *ArrayBad;
00113 static DRMS_Array_t *ArraySpike;
00114 static TIME sdo_epoch;
00115 static PTINFO *ptinfo = NULL;
00116 static PTINFO ptdata;
00117 static char bld_vers[16];
00118 static char datestr[32];
00119 static char open_dsname[256];
00120 static char dsffname[128];
00121 static char path[DRMS_MAXPATHLEN], bad_pix_path[DRMS_MAXPATHLEN];
00122 static char bad_aia_path[DRMS_MAXPATHLEN];
00123 static char rs1_path[DRMS_MAXPATHLEN];
00124 static struct timeval first[NUMTIMERS], second[NUMTIMERS];
00125 //static char *orbseries = "sdo.fds_orbit_vectors";
00126 //static char *orbseries = "su_arta.orbitvectors";
00127 static char *orbseries = "iris.orbit_vectors";
00128 
00129 static int nspikes, respike, fid, aiftsid, *oldvalues, *spikelocs, *newvalues;
00130 static int hcftid, aiagp6;
00131 static short aifcps;
00132 double aiascale = 1.0;
00133 
00134 IORBIT_Info_t *IOinfo = NULL;
00135 
00136 IORBIT_Info_t IOdata;
00137 LIBASTRO_Error_t IOstatus;
00138 unsigned int fsnarray[NUMRECLEV1];
00139 unsigned int fsnx = 0;
00140 //short data1[MAXPIXELS];       //iris lev1 is shorts
00141 //int data1[MAXPIXELS];
00142 float data1[MAXPIXELS];     //floats for HMI
00143 float ftmp;
00144 int data1A[MAXPIXELS];      //ints for AIA
00145 short data1S[MAXPIXELS];    //shorts for iris
00146 int array_cosmic[16777216]; //4096x4096
00147 double tgttimes[NUMRECLEV1];
00148 
00149 long long brec, erec, bfsn, efsn; //begin and end lev0 rec/fsn. must be same data type
00150 long long bnumx, enumx;       //has either the brec/erec of bfsn/efsn pair accord to mode
00151 int verbose;
00152 int modeflg = 0;        //0=fsn, 1=recnum
00153 int imagecnt = 0;       // num of images since last commit 
00154 int restartflg = 0;     // set when build_lev1 is called for a restart
00155 int abortflg = 0;
00156 int sigalrmflg = 0;             // set on signal so prog will know 
00157 int ignoresigalrmflg = 0;       // set after a close_image()
00158 int quicklook;
00159 //global quality flags
00160 int flatmiss[NUMRECLEV1];
00161 int orbmiss[NUMRECLEV1];
00162 int asdmiss[NUMRECLEV1];
00163 int mpdmiss[NUMRECLEV1];
00164 int limbmiss[NUMRECLEV1];
00165 int noimage[NUMRECLEV1];
00166 int missflg[NUMRECLEV1];
00167 
00168 char logname[128];
00169 char argdsin[128], argdsout[128], arglogfile[128];
00170 char argbx[80], argex[80], argquick[80], argmode[80];
00171 char timetag[32];
00172 char tlmseriesname[128];    // e.g. hmi.tlm
00173 char lev0seriesname[128];   // e.g. hmi.lev0
00174 char *username;         // from getenv("USER") 
00175 char *logfile;          // optional log name passed in 
00176 char *mode;         // given mode. recnum or fsn
00177 char *dsin;         // lev0 input dataset
00178 char *dsout;            // lev1 output dataset
00179 char *dsff;                     // flat field dataset
00180 
00182 typedef struct {
00183   float rsun_lf;
00184   float x0_lf;
00185   float y0_lf;
00186 } LIMB_SOMETHING;
00187 
00188 
00189 
00190 //int get_nspikes() { return nspikes; }
00191 //int get_respike(void) { return respike; }
00192 //int *get_spikelocs() { return spikelocs; }
00193 //int *get_oldvalues() { return oldvalues; }
00194 //int *get_newvalues() { return newvalues; }
00195 //void set_nspikes(int new_nspikes) { nspikes = new_nspikes; }
00196 //void set_spikelocs(int *new_spikelocs) { spikelocs = new_spikelocs; }
00197 //void set_oldvalues(int *new_oldvalues) { oldvalues = new_oldvalues; }
00198 //void set_newvalues(int *new_newvalues) { newvalues = new_newvalues; }
00199 
00200 //Set the QUALITY keyword for lev1
00201 void do_quallev1(DRMS_Record_t *rs0, DRMS_Record_t *rs1, int inx, unsigned int fsn)
00202 {
00203   int quallev1 = 0;
00204   int rstatus;
00205   char *pchar;
00206 
00207   quallev1 = missflg[inx];
00208   if(flatmiss[inx]) quallev1 = quallev1 | Q_NOFLAT;
00209   if(orbmiss[inx]) quallev1 = quallev1 | Q_NOORB;
00210   if(limbmiss[inx]) quallev1 = quallev1 | Q_NOLIMB;
00211   if(asdmiss[inx]) {
00212     quallev1 = quallev1 | Q_NOASD;
00213     drms_setkey_string(rs1, "ASD_REC", DRMS_MISSING_STRING);
00214   }
00215   if(mpdmiss[inx]) quallev1 = quallev1 | Q_NOMPD;
00216   if(noimage[inx]) quallev1 = quallev1 | Q_MISSALL;
00217   if(ptinfo) {
00218     ptdata = ptinfo[inx];
00219     if(strcmp(ptdata.acs_mode, "SCIENCE")) { //not in science pointing mode
00220       quallev1 = quallev1 | Q_NOACS_SCI;
00221     }
00222     if(strcmp(ptdata.acs_eclp, "NO")) {      //in eclipse
00223       quallev1 = quallev1 | Q_ACS_ECLP;
00224     }
00225     if(strcmp(ptdata.acs_sunp, "YES")) {     //not in sun presence flag
00226       quallev1 = quallev1 | Q_ACS_SUNP;
00227     }
00228     if(strcmp(ptdata.acs_safe, "NO")) {     //safemode set
00229       quallev1 = quallev1 | Q_ACS_SAFE;
00230     }
00231   }
00232   pchar = drms_getkey_string(rs0, "IMG_TYPE", &rstatus);
00233   if(rstatus) {
00234     printk("ERROR: in drms_getkey_string(IMG_TYPE) fsn=%u\n", fsn);
00235   }
00236   else {
00237     if(strcmp(pchar, "LIGHT")) {    //dark image
00238       quallev1 = quallev1 | Q_IMG_TYPE;
00239     }
00240   }
00241   if(quicklook) {
00242     quallev1 = quallev1 | Q_NRT;
00243   }
00244   drms_setkey_int(rs1, "QUALITY", quallev1);
00245   drms_setkey_string(rs1, "BLD_VERS", bld_vers); //build vers to every record
00246 }
00247 
00248 int nice_intro ()
00249 {
00250   int usage = cmdparams_get_int (&cmdparams, "h", NULL);
00251   if (usage)
00252     {
00253     printf ("Usage:\nbuild_lev1_iris [-vhr] "
00254     "mode=<recnum|fsn> dsin=<lev0> dsout=<lev1>\n"
00255     "brec=<rec#>|bfsn=<fsn#> erec=<rec#>|efsn=<fsn#>\n"
00256     "quicklook=<0|1> [logfile=<file>]\n"
00257     "  -h: help - show this message then exit\n"
00258     "  -v: verbose\n"
00259     "  -r: restart. only used when we restart our selves periodically\n"
00260         "mode= recnum: brec and erec have the record # range to process \n"
00261         "      fsn: bfsn and efsn have the fsn # range to process\n"
00262         "      For safety, the mode and arg name used must be consistent\n"
00263     "dsin= data set name of lev0 input\n"
00264     "      default hmi=hmi.lev0e   aia=aia.lev0e\n"
00265     "dsout= data set name of lev1 output\n"
00266     "      default hmi=su_production.hmi_lev1e   aia=su_production.aia_lev1e\n"
00267     "brec= first lev0 rec# to process. 0=error must be given by build_lev1_mgr\n"
00268     "erec= last lev0 rec# to process. 0=error must be given by build_lev1_mgr\n"
00269     "bfsn= first fsn# to process. 0=error must be given by build_lev1_mgr\n"
00270     "efsn= last fsn# to process. 0=error must be given by build_lev1_mgr\n"
00271     "quicklook= 1 = quicklook mode, 0 = definitive mode\n"
00272     "logfile= optional log file name. If not given uses:\n"
00273         "         /usr/local/logs/lev1/build_lev1_aia.<time_stamp>.log\n");
00274     return(1);
00275     }
00276   verbose = cmdparams_get_int (&cmdparams, "v", NULL);
00277   restartflg = cmdparams_get_int (&cmdparams, "r", NULL);
00278   return (0);
00279 }
00280 
00281 TIME SDO_to_DRMS_time(int sdo_s, int sdo_ss)
00282 {
00283 static int firstcall = 1;
00284 if (firstcall)
00285   {
00286   firstcall = 0;
00287   }
00288 /* XXX fix build 3/18/2008, arta */
00289 return(sdo_epoch + (TIME)sdo_s + (TIME)(sdo_ss & 0xFFFF)/65536.0);
00290 }
00291 
00292 // Setup global datestr[] like: 2008.07.14_08:29:31
00293 char *do_datestr() {
00294   time_t tval;
00295   struct tm *t_ptr;
00296 
00297   tval = time(NULL);
00298   t_ptr = localtime(&tval);
00299   sprintf(datestr, "%d.%02d.%02d_%02d:%02d:%02d", 
00300       (t_ptr->tm_year+1900), (t_ptr->tm_mon+1),
00301       t_ptr->tm_mday, t_ptr->tm_hour, t_ptr->tm_min, t_ptr->tm_sec);
00302   return(datestr);
00303 }
00304 
00305 // Returns a time tag like  yyyy.mm.dd.hhmmss 
00306 char *gettimetag()
00307 {
00308   struct timeval tvalr;
00309   struct tm *t_ptr;
00310 
00311   gettimeofday(&tvalr, NULL);
00312   t_ptr = localtime((const time_t *)&tvalr);
00313   sprintf(timetag, "%04d.%02d.%02d.%02d%02d%02d",
00314         (t_ptr->tm_year+1900), (t_ptr->tm_mon+1), t_ptr->tm_mday, t_ptr->tm_hour, t_ptr->tm_min, t_ptr->tm_sec);
00315   return(timetag);
00316 }
00317 
00318 
00319 void BeginTimer(int n)
00320 {
00321   gettimeofday (&first[n], NULL);
00322 }
00323 
00324 float EndTimer(int n)
00325 {
00326   gettimeofday (&second[n], NULL);
00327   if (first[n].tv_usec > second[n].tv_usec) {
00328     second[n].tv_usec += 1000000;
00329     second[n].tv_sec--;
00330   }
00331   return (float) (second[n].tv_sec-first[n].tv_sec) +
00332     (float) (second[n].tv_usec-first[n].tv_usec)/1000000.0;
00333 }
00334 
00335 // Outputs the variable format message (re: printf) to the log file.
00336 int h1log(const char *fmt, ...)
00337 {
00338   va_list args;
00339   char string[32768];
00340 
00341   va_start(args, fmt);
00342   vsprintf(string, fmt, args);
00343   if(h1logfp) {
00344     fprintf(h1logfp, string);
00345     fflush(h1logfp);
00346   } 
00347   else {            // couldn't open log 
00348     printf(string);     // also print to stdout
00349     fflush(stdout);
00350   }
00351   va_end(args);
00352   return(0);
00353 }
00354 
00355 int send_mail(char *fmt, ...)
00356 {
00357   va_list args;
00358   char string[1024], cmd[1024];
00359 
00360   va_start(args, fmt);
00361   vsprintf(string, fmt, args);
00362   sprintf(cmd, "echo \"%s\" | Mail -s \"build_lev1_iris mail\" lev0_user", string);
00363   system(cmd);
00364   va_end(args);
00365   return(0);
00366 }
00367 
00368 // Got a fatal error. 
00369 void abortit(int stat)
00370 {
00371   printk("***Abort in progress ...\n");
00372   printk("**Exit build_lev1_iris w/ status = %d\n", stat);
00373   if (h1logfp) fclose(h1logfp);
00374   exit(stat);
00375 }
00376 
00378 int rdout_mode_correct()
00379 {
00380   return(0);
00381 }
00382 
00384 int orbit_calc()
00385 {
00386   return(0);
00387 }
00388 
00389 static int GetSaaHlz(DRMS_Env_t *env, DRMS_RecordSet_t *rs, double *tgttimes, int ntimes, int (*myprintkerr)(const char *fmt, ...))
00390 {
00391     IORBIT_SaaHlzInfo_t *saahlzinfo = NULL;
00392     int itime;
00393     LinkedList_t *list = NULL;
00394     LinkedList_t **pList = NULL;
00395     ListNode_t *node = NULL;
00396     char *eventType = NULL;
00397     int nhlz;
00398     int shlz;
00399     int saa;
00400     HContainer_t *colToList = NULL;
00401     HContainer_t **pColToList = NULL;
00402     char timeStr[IORBIT_SAAHLZINFO_TIME_KEY_LEN];
00403     LIBASTRO_Error_t rv;
00404     int ret = 0;
00405     
00406     rv = iorbit_getSaaHlzInfo(env, SAA_HLZ_SERIES, tgttimes, ntimes, &saahlzinfo);
00407     
00408     if (rv == kLIBASTRO_Success)
00409     {
00410         for (itime = 0; itime < ntimes; itime++)
00411         {
00412             nhlz = 0;
00413             shlz = 0;
00414             saa = 0;
00415             
00416             snprintf(timeStr, sizeof(timeStr), "%lf", tgttimes[itime]);
00417             
00418             if ((pColToList = hcon_lookup(saahlzinfo, timeStr)) != NULL)
00419             {
00420                 colToList = *pColToList;
00421                 
00422                 if ((pList = hcon_lookup(colToList, IORBIT_SAAHLZINFO_KW_EVENT_TYPE)) != NULL)
00423                 {
00424                     list = *pList;
00425                     list_llreset(list);
00426                     
00427                     while((node = list_llnext(list)) != NULL)
00428                     {
00429                         eventType = (char *)node->data;
00430                         
00431                         if (strcasecmp(eventType, "NHLZ") == 0)
00432                         {
00433                             nhlz = 1;
00434                         }
00435                         else if (strcasecmp(eventType, "SHLZ") == 0)
00436                         {
00437                             shlz = 1;
00438                         }
00439                         else if (strcasecmp(eventType, "SAA") == 0)
00440                         {
00441                             saa = 1;
00442                         }
00443                     }
00444                     
00445                     /* Set SAA-HLZ keywords */
00446                     if (nhlz && shlz)
00447                     {
00448                         /* ERROR - set HLZ to 0. */
00449                         printkerr("ERROR - For tobs %lf, both event types NHLZ and SHLZ exist.\n", tgttimes[itime]);
00450                         drms_setkey_int(rs->records[itime], "HLZ", 0);
00451                     }
00452                     else if (nhlz)
00453                     {
00454                         printf("  Setting HLZ to 1.\n");
00455                         drms_setkey_int(rs->records[itime], "HLZ", 1);
00456                     }
00457                     else if (shlz)
00458                     {
00459                         printf("  Setting HLZ to 2.\n");
00460                         drms_setkey_int(rs->records[itime], "HLZ", 2);
00461                     }
00462                     else 
00463                     {
00464                         drms_setkey_int(rs->records[itime], "HLZ", 0);                    
00465                     }
00466                     
00467                     if (saa)
00468                     {
00469                         drms_setkey_int(rs->records[itime], "SAA", 1);
00470                     }
00471                     else
00472                     {
00473                         drms_setkey_int(rs->records[itime], "SAA", 0);
00474                     }
00475                 }
00476                 else
00477                 {
00478                     myprintkerr("ERROR - SAA-HLZ info for keyword %s unexecpectedly missing.\n", IORBIT_SAAHLZINFO_KW_EVENT_TYPE);
00479                     ret = 1;
00480                 }
00481             }
00482             else
00483             {
00484                 myprintkerr("ERROR - SAA-HLZ info for tobs %lf unexecpectedly missing.\n", tgttimes[itime]);
00485                 ret = 1;
00486             }
00487         }
00488         
00489         iorbit_cleanSaaHlzInfo(&saahlzinfo);
00490     }
00491     else
00492     {
00493         myprintkerr("ERROR - Couldn't query db properly.\n");
00494         ret = 1;
00495     }
00496     
00497     return ret;
00498 }
00499 
00500 //#include "aia_despike.c"
00501 //#include "do_flat.c"
00502 #include "get_image_location.c"
00503 #include "limb_fit_function.c"
00504 #include "cosmic_ray.c"
00505 #include "heightformation.c"
00506 
00507 //Called with the range to do. The args are either rec#s or fsn#s accord to modeflg
00508 int do_ingest(long long bbrec, long long eerec)
00509 {
00510   //FILE *fwt;
00511   Image_Location *p_imageloc;
00512   Image_Location imageloc[NUMRECLEV1];
00513   TIME t_obs0;
00514   TIME tobs[NUMRECLEV1];
00515   float percentd;
00516   float cdelt1, rsun, crpix1, crpix2, crota2;
00517   double rsun_lf, x0_lf, y0_lf;
00518   int rstatus, dstatus, lstatus, ncnt, fcnt, i, j, k, qualint, nobs;
00519   int hshiexp, hcamid, nbad, n_cosmic;
00520   int *spikedata, status, axes[2], nbytes;
00521   uint32_t missvals, totvals;
00522   long long recnum0, recnum1, recnumff;
00523   char recrange[128], lev0name[128], flatrec[128], temprec[128], pointrec[128];
00524   //char tmpname[80];
00525   double scroll;
00526 
00527   if(modeflg) sprintf(recrange, ":#%lld-#%lld", bbrec, eerec);
00528   else sprintf(recrange, "%lld-%lld", bbrec, eerec);
00529   sprintf(open_dsname, "%s[%s]", dsin, recrange);
00530   printk("open_dsname = %s\n", open_dsname);
00531   printk("#levnum recnum fsn\n");
00532 
00533     t_obs0 = 0;
00534     rset0 = drms_open_records(drms_env, open_dsname, &rstatus); //open lev0
00535     if(!rset0 || (rset0->n == 0) || rstatus) {
00536       printk("Can't do drms_open_records(%s)\n", open_dsname);
00537       printf("Can't do drms_open_records(%s)\n", open_dsname);
00538       //return(1);
00539       return(0);
00540     }
00541     drms_stage_records(rset0, 1, 0);
00542     ncnt = rset0->n;
00543     rptr = (DRMS_Record_t *)malloc(ncnt * sizeof(DRMS_Record_t));
00544     if(rptr == NULL) {
00545       printk("Can't malloc() for DRMS_Record_t sort\n");
00546       return(1);
00547     }
00548     //make opened records sequential in mem for sort
00549     for(i=0; i < ncnt; i++) {
00550       memcpy(&rptr[i], rset0->records[i], sizeof(DRMS_Record_t));
00551     }
00552     //Must sort to get in ascending t_obs order
00553     qsort(rptr, ncnt, sizeof(DRMS_Record_t), &compare_rptr);
00554 
00555     //this loop is for the benefit of the lev1view display to show
00556     //info on all lev0 records opened
00557     for(i=0; i < ncnt; i++) {
00558       flatmiss[i] = 0;      //init quality flags
00559       orbmiss[i] = 0;
00560       limbmiss[i] = 0;
00561       asdmiss[i] = 0;
00562       mpdmiss[i] = 0;
00563       noimage[i] = 0;
00564       missflg[i] = 0;
00565       rs0 = &rptr[i];
00566       recnum0 = rs0->recnum;
00567       //must get fsn in case got record from last lev1 record
00568       fsnx = drms_getkey_int(rs0, "FSN", &rstatus); 
00569       fsnarray[i] = fsnx;
00570       printk("*0 %u %u\n", recnum0, fsnx);
00571       //also set up call for get_pointing_info() and iorbit_getinfo()
00572       tobs[i] = drms_getkey_time(rs0, "t_obs", &rstatus);
00573       if(rstatus) {
00574         printk("Error on drms_getkey_time() fsn=%u. Use DRMS_MISSING_TIME\n", 
00575         fsnx);
00576        tobs[i] = DRMS_MISSING_TIME;
00577       }
00578     }
00579     //New from Art's stuff 07Aug2013
00580     HContainer_t *keymap = NULL;
00581 
00582     keymap = hcon_create(DRMS_MAXKEYNAMELEN, DRMS_MAXKEYNAMELEN, NULL, NULL, NULL, NULL, 0);
00583 
00584     if (!keymap)
00585     {
00586         IOstatus = kLIBASTRO_OutOfMemory;
00587     }
00588     else {
00589         hcon_insert(keymap, "kXGCI", "geixobs");
00590         hcon_insert(keymap, "kYGCI", "geiyobs");
00591         hcon_insert(keymap, "kZGCI", "geizobs");
00592         hcon_insert(keymap, "kXHCI", "heixobs");
00593         hcon_insert(keymap, "kYHCI", "heiyobs");
00594         hcon_insert(keymap, "kZHCI", "heizobs");
00595         hcon_insert(keymap, "kRSUNOBS", "rsunobs");
00596         hcon_insert(keymap, "kOBSVR", "obsvr");
00597         hcon_insert(keymap, "kDSUNOBS", "dsunobs");
00598         hcon_insert(keymap, "kOBSDATE", "obsdate");
00599 
00600         IOstatus = iorbit_getinfo_ext(drms_env,
00601                                 orbseries,
00602                                 NULL,
00603                                 IORBIT_Alg_Quadratic,
00604                                 tobs,
00605                                 ncnt,
00606                                 kIORBIT_CacheAction_DontCache,
00607                                 &IOinfo,
00608                                 keymap);
00609         hcon_destroy(&keymap);
00610     }
00611     if(IOstatus != kLIBASTRO_Success) {
00612       if(IOstatus == kLIBASTRO_InsufficientData) {
00613         printk("***ERROR in iorbit_getinfo_ext: kLIBASTRO_InsufficientData\n");
00614       }
00615       else {
00616        printk("***ERROR in iorbit_getinfo_ext() status=%d\n", IOstatus);
00617       }
00618       for(j=0; j < ncnt; j++) {  //set qual bits
00619         orbmiss[j] = 1;
00620       }
00621       return(1);                //abort. new 2/22/2011
00622     }
00623     
00624     /* IOStatus == kLIBASTRO_Success */
00625     
00626     rset1 = drms_create_records(drms_env, ncnt, dsout, DRMS_PERMANENT,&dstatus);
00627     if(dstatus) {
00628       printk("**ERROR: Can't create records for %s\n", dsout);
00629       for(j=0; j < ncnt; j++) {  //set qual bits
00630         noimage[j] = 1;
00631       }
00632       return(1);    //new 2/22/2011
00633     }
00634     
00635     /*****************************************************************/
00636     /* Obtain SAA-HLZ information, and set the SAA and HLZ keywords. */
00637     
00638     if (GetSaaHlz(drms_env, rset1, tobs, ncnt, printkerr))
00639     {
00640         printk("***Error - Unable to fetch SAA-HLZ information.\n");
00641         return 1;
00642     }
00643     
00644     /* End SAA-HLZ                                                   */
00645     /*****************************************************************/
00646     
00647 
00648     for(i=0; i < ncnt; i++) {   //do for all the sorted lev0 records
00649       //StartTimer(2);  //!!TEMP
00650       rs0 = &rptr[i];
00651       recnum0 = rs0->recnum;
00652       fsnx = fsnarray[i]; 
00653       sprintf(lev0name, "%s[%u]", dsin, fsnx);
00654       if(drms_getkey_int(rs0, "QUALITY", 0) < 0) {
00655         printk("Bad QUALITY for %s, no lev1 made\n", lev0name);
00656         noimage[i] = 1;
00657         //continue; //make an image anyway so the lev1 fsn will exist
00658       }
00659       segment0 = drms_segment_lookupnum(rs0, 0);
00660       int seg0sz = segment0->axis[0]*segment0->axis[1];
00661       Array0 = drms_segment_read(segment0, DRMS_TYPE_SHORT, &rstatus);
00662       if(!Array0) {
00663         printk("Can't do drms_segment_read() %s status=%d\n", 
00664             lev0name, rstatus);
00665         noimage[i] = 1;
00666         return(1);  //return until we learn
00667         continue;
00668       }
00669       l0l1->adata0 = (short *)Array0->data; //free at end
00670       l0l1->dat1.adata1A = &data1A;         //int out
00671       l0l1->rs0 = rs0;
00672       l0l1->recnum0 = recnum0;
00673       l0l1->fsn = fsnx;
00674       l0l1->nx = segment0->axis[0];
00675       l0l1->ny = segment0->axis[1];
00676       l0l1->datavals = drms_getkey_int(rs0, "DATAVALS", &rstatus);
00677       l0l1->missvals = drms_getkey_int(rs0, "MISSVALS", &rstatus);
00678 
00679       sprintf(open_dsname, "%s[%u]", dsout, fsnx);
00680       rs = rset1->records[i]; 
00681 
00682       // 
00683       // find closest iris.pointing_data record and set scroll
00684       //
00685       {
00686         if(!quicklook) {
00687       DRMS_RecordSet_t *rset;
00688       DRMS_Record_t *rt;
00689       int st;
00690       TIME t_obs, time_qbi;
00691       float aeulrbrx, aeulrbry, aeulrbrz, acg_roll, ophase;
00692       t_obs = drms_getkey_time(rs0, "t_obs", &st);
00693       if (st) {
00694           t_obs = DRMS_MISSING_TIME;
00695           }
00696           else {
00697       sprintf(open_dsname, "iris.pointing_data[? date_obs > %f and date_obs <= %f ?]", t_obs-5, t_obs+5); 
00698           printf("%s\n", open_dsname);
00699       rset = drms_open_records(drms_env, open_dsname, &st);
00700       if (!rset || !rset->n || st) {
00701           printk("Error in drms_open_records(%s); setting scroll to zero\n", open_dsname);
00702           scroll = 0.0;
00703       } else {
00704           // There should be only one record returned
00705           rt = rset->records[0];
00706               sprintf(pointrec, "iris.pointing_data[:#%lld]", rt->recnum);
00707               if(dstatus = drms_setkey_string(rs, "POINTREC", pointrec)) {
00708                 printk("**ERROR on drms_setkey_string() for %s\n", pointrec);
00709               }
00710           time_qbi = drms_getkey_time(rt, "TIME_QBI", &st);
00711           aeulrbrx = drms_getkey_float(rt, "A_EULERBR_X", &st);
00712           aeulrbry = drms_getkey_float(rt, "A_EULERBR_Y",  &st);
00713           aeulrbrz = drms_getkey_float(rt, "A_EULERBR_Z", &st);
00714           acg_roll = drms_getkey_float(rt, "A_CG_ROLL_ANGLE", &st);
00715           ophase   = drms_getkey_float(rt, "OPHASE", &st);
00716 
00717           drms_setkey_time(rs, "TIME_QBI", time_qbi);
00718           drms_setkey_float(rs, "AEULRBRX", aeulrbrx);
00719           drms_setkey_float(rs, "AEULRBRY", aeulrbry);
00720           drms_setkey_float(rs, "AEULRBRZ", aeulrbrz);
00721           drms_setkey_float(rs, "SAT_ROT", aeulrbrz);
00722           drms_setkey_float(rs, "ACG_ROLL", acg_roll);
00723           drms_setkey_float(rs, "OPHASE",   ophase);
00724 
00725           scroll = aeulrbrz;
00726       }
00727 
00728       if (rset)
00729           drms_close_records(rset, DRMS_FREE_RECORD);
00730           }
00731         }
00732       }
00733       
00734       drms_record_directory(rs, rs1_path, 0);
00735       if(!*rs1_path) {
00736         printk("***ERROR: No path to segment for %s\n", open_dsname);
00737         noimage[i] = 1;
00738         continue;
00739       }
00740       printf("\npath to lev1 = %s\n", rs1_path);    
00741       if(rstatus = iris_isp2wcs(rs0, rs, scroll)) {
00742         printk("**ERROR: iris_isp2wcs() status = %d\n", rstatus);
00743         printk("Press on after error at fsn=%u...\n", fsnx);
00744         printf("**ERROR: at fsn %u\n", fsnx);
00745         continue;
00746         //printf("**ERROR: Abort on fsn = %u\n", fsnx);
00747         //return(1);
00748       }
00749       dstatus = drms_setkey_int(rs, "FSN", fsnx);
00750       //dstatus = drms_setkey_string(rs, "LEV0SERIES", lev0name); //no such keyword
00751       if(!(segment = drms_segment_lookup(rs, "image_lev1"))) {
00752         printk("No drms_segment_lookup(rs, image_lev1) for %s\n", open_dsname);
00753         noimage[i] = 1;
00754         continue;
00755       }
00756         segArray = drms_array_create(DRMS_TYPE_INT,
00757                                        segment->info->naxis,
00758                                        segment0->axis,
00759                                        &data1A,
00760                                        &dstatus);
00761 //short *adata = (short *)Array0->data;
00762 //short *bdata = &data1S;
00763 //memcpy(bdata, adata, 2*seg0sz); //!!TEMP mv the lev0 in for now
00764 
00765       //transer the lev0 keywords
00766       rstatus = drms_copykeys(rs, rs0, 0, kDRMS_KeyClass_Explicit);
00767       if(rstatus != DRMS_SUCCESS) {
00768         printk("Error %d in drms_copykeys() for fsn %u\n", fsnx);
00769         return(1);      //new 2/22/2011
00770         continue;
00771       }
00772       qualint = drms_getkey_int(rs0, "QUALITY", &rstatus);
00773       drms_setkey_int(rs, "QUALLEV0", qualint);
00774       fid = drms_getkey_int(rs0, "FID", &rstatus);
00775 
00776       short isqisysn = drms_getkey_short(rs0, "ISQISYSN", &rstatus);  
00777       int iimgots1 = drms_getkey_int(rs0, "IIMGOTS1", &rstatus);
00778       int iimgots2 = drms_getkey_int(rs0, "IIMGOTS2", &rstatus);
00779       int iimgots3 = drms_getkey_int(rs0, "IIMGOTS3", &rstatus); 
00780       switch (isqisysn) {
00781           case 0:
00782             drms_setkey_int(rs, "IIMGOTS", iimgots1);
00783             break;
00784           case 1:
00785             drms_setkey_int(rs, "IIMGOTS", iimgots2);
00786             break;
00787           case 2:
00788             drms_setkey_int(rs, "IIMGOTS", iimgots3);
00789             break;
00790       } 
00791 
00792       drms_setkey_time(rs, "T_OBS", tobs[i]);
00793       printk("t_obs for lev0 = %10.5f fsn=%u\n", tobs[i], fsnarray[i]);  
00794       drms_setkey_double(rs, "DATE", CURRENT_SYSTEM_TIME);
00795       if(ptinfo) {
00796         ptdata = ptinfo[i];
00797           drms_setkey_float(rs, "SAT_Y0", ptdata.sat_y0);
00798           drms_setkey_float(rs, "SAT_Z0", ptdata.sat_z0);
00799           drms_setkey_float(rs, "SAT_ROT", ptdata.sat_rot);
00800           drms_setkey_string(rs, "ACS_MODE", ptdata.acs_mode);
00801           drms_setkey_string(rs, "ACS_ECLP", ptdata.acs_eclp);
00802           drms_setkey_string(rs, "ACS_SUNP", ptdata.acs_sunp);
00803           drms_setkey_string(rs, "ACS_SAFE", ptdata.acs_safe);
00804           drms_setkey_string(rs, "ASD_REC", ptdata.asd_rec);
00805           drms_setkey_string(rs, "ACS_CGT", ptdata.acs_cgt);
00806       }
00807       if(IOinfo) {
00808         IOdata = IOinfo[i];
00809            drms_setkey_double(rs, "RSUN_OBS", IOdata.rsun_obs);
00810            drms_setkey_double(rs, "DSUN_OBS", IOdata.dsun_obs);
00811            drms_setkey_double(rs, "OBS_VR", IOdata.obs_vr);
00812            drms_setkey_double(rs, "GEIX_OBS", IOdata.gciX);
00813            drms_setkey_double(rs, "GEIY_OBS", IOdata.gciY);
00814            drms_setkey_double(rs, "GEIZ_OBS", IOdata.gciZ);
00815            drms_setkey_double(rs, "HEIX_OBS", IOdata.hciX);
00816            drms_setkey_double(rs, "HEIY_OBS", IOdata.hciY);
00817            drms_setkey_double(rs, "HEIZ_OBS", IOdata.hciZ);
00818            drms_setkey_string(rs, "ORB_REC", IOdata.orb_rec); 
00819       }
00820         
00821         
00822 
00823         
00824         
00825                 
00826         
00827       drms_setkey_float(rs, "X0_MP", imageloc[i].x);
00828       drms_setkey_float(rs, "Y0_MP", imageloc[i].y);
00829       drms_setkey_float(rs, "INST_ROT", imageloc[i].instrot);
00830       //drms_setkey_float(rs, "INST_ROT", 180.0); //force this for now
00831       drms_setkey_float(rs, "IMSCL_MP", imageloc[i].imscale);
00832       drms_setkey_string(rs, "MPO_REC", imageloc[i].mpo_rec);
00833 
00834       int camera = drms_getkey_int(rs0, "CAMERA", &rstatus);
00835       if(rstatus) {
00836         printk("Can't do drms_getkey_int() for fsn %u\n", fsnx);
00837         noimage[i] = 1;
00838         goto TEMPSKIP;
00839         //return(1);
00840       }
00841       if ( 0 == drms_setkey_time(rs, "T_REC", tobs[i])) {
00842         int status, allstat = 0;
00843         double tr_step;
00844         long long tr_index;
00845         TIME t_rec, tr_epoch;
00846         tr_index = drms_getkey_longlong(rs, "T_REC_index", &status);
00847         allstat += status;
00848         tr_step =  drms_getkey_double(rs, "T_REC_step", &status);
00849         allstat += status;
00850         tr_epoch = drms_getkey_time(rs, "T_REC_epoch", &status);
00851         allstat += status;
00852         if (0 == allstat) {
00853           t_rec = tr_epoch + tr_index*tr_step;
00854           drms_setkey_time(rs, "T_REC", t_rec);
00855         }
00856       }
00857       //now get all the crs_table values according to lev0 IICRSID
00858       int iicrsid = drms_getkey_int(rs0, "IICRSID", &rstatus);
00859       if(rstatus) {
00860         printk("Can't do drms_getkey_int() for IICRSID for fsn %u\n", fsnx);
00861         printk("No CRS_TABLE keywords for lev1 fsn %u\n", fsnx);
00862       }
00863       else {
00864         sprintf(open_dsname, "%s[%d]", DSCRSNAME, iicrsid);
00865         printk("Open: %s\n", open_dsname);
00866         crsset = drms_open_records(drms_env, open_dsname, &rstatus);
00867         if(!crsset || (crsset->n == 0) || rstatus) {
00868           printk("Can't do drms_open_records(%s)\n", open_dsname);
00869         }
00870         else {
00871           crsrec = crsset->records[0];
00872           char *crsstr = drms_getkey_string(crsrec, "CRS_DESC", &status);
00873           drms_setkey_string(rs, "CRS_DESC", crsstr);
00874           crsstr = drms_getkey_string(crsrec, "CRS_TYPE", &status);
00875           drms_setkey_string(rs, "CRS_TYPE", crsstr);
00876           int crsint = drms_getkey_int(crsrec, "CRS_NREG", &status);
00877           drms_setkey_int(rs, "CRS_NREG", crsint);
00878           crsint = drms_getkey_int(crsrec, "TSR1", &status);
00879           drms_setkey_int(rs, "TSR1", crsint);
00880           crsint = drms_getkey_int(crsrec, "TER1", &status);
00881           drms_setkey_int(rs, "TER1", crsint);
00882           crsint = drms_getkey_int(crsrec, "TSC1", &status);
00883           drms_setkey_int(rs, "TSC1", crsint);
00884           crsint = drms_getkey_int(crsrec, "TEC1", &status);
00885           drms_setkey_int(rs, "TEC1", crsint);
00886           crsint = drms_getkey_int(crsrec, "TSR2", &status);
00887           drms_setkey_int(rs, "TSR2", crsint);
00888           crsint = drms_getkey_int(crsrec, "TER2", &status);
00889           drms_setkey_int(rs, "TER2", crsint);
00890           crsint = drms_getkey_int(crsrec, "TSC2", &status);
00891           drms_setkey_int(rs, "TSC2", crsint);
00892           crsint = drms_getkey_int(crsrec, "TEC2", &status);
00893           drms_setkey_int(rs, "TEC2", crsint);
00894           crsint = drms_getkey_int(crsrec, "TSR3", &status);
00895           drms_setkey_int(rs, "TSR3", crsint);
00896           crsint = drms_getkey_int(crsrec, "TER3", &status);
00897           drms_setkey_int(rs, "TER3", crsint);
00898           crsint = drms_getkey_int(crsrec, "TSC3", &status);
00899           drms_setkey_int(rs, "TSC3", crsint);
00900           crsint = drms_getkey_int(crsrec, "TEC3", &status);
00901           drms_setkey_int(rs, "TEC3", crsint);
00902           crsint = drms_getkey_int(crsrec, "TSR4", &status);
00903           drms_setkey_int(rs, "TSR4", crsint);
00904           crsint = drms_getkey_int(crsrec, "TER4", &status);
00905           drms_setkey_int(rs, "TER4", crsint);
00906           crsint = drms_getkey_int(crsrec, "TSC4", &status);
00907           drms_setkey_int(rs, "TSC4", crsint);
00908           crsint = drms_getkey_int(crsrec, "TEC4", &status);
00909           drms_setkey_int(rs, "TEC4", crsint);
00910           crsint = drms_getkey_int(crsrec, "TSR5", &status);
00911           drms_setkey_int(rs, "TSR5", crsint);
00912           crsint = drms_getkey_int(crsrec, "TER5", &status);
00913           drms_setkey_int(rs, "TER5", crsint);
00914           crsint = drms_getkey_int(crsrec, "TSC5", &status);
00915           drms_setkey_int(rs, "TSC5", crsint);
00916           crsint = drms_getkey_int(crsrec, "TEC5", &status);
00917           drms_setkey_int(rs, "TEC5", crsint);
00918           crsint = drms_getkey_int(crsrec, "TSR6", &status);
00919           drms_setkey_int(rs, "TSR6", crsint);
00920           crsint = drms_getkey_int(crsrec, "TER6", &status);
00921           drms_setkey_int(rs, "TER6", crsint);
00922           crsint = drms_getkey_int(crsrec, "TSC6", &status);
00923           drms_setkey_int(rs, "TSC6", crsint);
00924           crsint = drms_getkey_int(crsrec, "TEC6", &status);
00925           drms_setkey_int(rs, "TEC6", crsint);
00926           crsint = drms_getkey_int(crsrec, "TSR7", &status);
00927           drms_setkey_int(rs, "TSR7", crsint);
00928           crsint = drms_getkey_int(crsrec, "TER7", &status);
00929           drms_setkey_int(rs, "TER7", crsint);
00930           crsint = drms_getkey_int(crsrec, "TSC7", &status);
00931           drms_setkey_int(rs, "TSC7", crsint);
00932           crsint = drms_getkey_int(crsrec, "TEC7", &status);
00933           drms_setkey_int(rs, "TEC7", crsint);
00934           crsint = drms_getkey_int(crsrec, "TSR8", &status);
00935           drms_setkey_int(rs, "TSR8", crsint);
00936           crsint = drms_getkey_int(crsrec, "TER8", &status);
00937           drms_setkey_int(rs, "TER8", crsint);
00938           crsint = drms_getkey_int(crsrec, "TSC8", &status);
00939           drms_setkey_int(rs, "TSC8", crsint);
00940           crsint = drms_getkey_int(crsrec, "TEC8", &status);
00941           drms_setkey_int(rs, "TEC8", crsint);
00942 
00943           crsint = drms_getkey_int(crsrec, "WIN_FLIP", &status);
00944           l0l1->winflip = crsint;
00945           drms_setkey_int(rs, "WIN_FLIP", crsint); 
00946           printk("Close: %s\n", open_dsname);
00947           drms_close_records(crsset, DRMS_FREE_RECORD);
00948         }
00949       }
00950 
00951 /****NEW from Rock 10Sep2013************************************************/
00952       if(!quicklook) {
00953 /*  ITF1CCD1  CCD1_FUV1_OPERATING                                    */
00954 /*  ITF2CCD2  CCD2_FUV2_OPERATING                                    */
00955 /*  ITNUCCD3  CCD3_NUV_OPERATING                                     */
00956 /*  ITSJCCD4  CCD4_SJI_OPERATING                                     */
00957 /*  BT06CBPX  CEB_ON_THE_POSX_AXIS                                   */
00958 /*  BT07CBNX  CEB_ON_THE_NEGX_AXIS                                   */
00959 /*  BT15IEB   ELECTRONICS_BOX                                        */
00960 /*  IT08GTWM  GUIDE_TELESCOPE_WEDGE_HC_MOTOR_TS08                    */
00961 /*  IT14SPPX  SPECTROGRAPH_OPTICS_PACKAGE_HOZ5_CONTROL_TS14 (pos_x)  */
00962 /*  IT16SPNX  SPECTROGRAPH_OPTICS_PACKAGE_HOZ7_CONTROL_TS16 (neg_x)  */
00963 
00964         float itf1ccd1, itf2ccd2, itnuccd3, itsjccd4;
00965         float bt06cbpx, bt07cbnx, bt15ieb, it08gtwm;
00966         float it14sppx, it16spnx;
00967 
00968         char *dstemp = "iris.temperatures_60s";
00969         if(fabs(tobs[i] - t_obs0) > 300.0) {
00970           char *selstr = "select max(date_obs) from ";
00971           char *whrstr = "where date_obs <= ";
00972           int nr;
00973           if(rs_t) {
00974             drms_close_records(rs_t, DRMS_FREE_RECORD);
00975             rs_t = NULL;
00976           }
00977           sprintf(open_dsname, "%s[? date_obs=(%s %s %s %f) ?]",
00978                   dstemp, selstr, dstemp, whrstr, tobs[i]);
00979           printf(" %s\n", open_dsname);
00980           rt = NULL;
00981           rs_t = drms_open_records(drms_env, open_dsname, &rstatus);
00982           if(rstatus) printk("Can not open temperature series.\n");
00983           else {
00984             nr = rs_t->n;
00985             if(nr != 1) printk("%d records != 1.\n", nr);
00986             rt = rs_t->records[0];
00987           }
00988           t_obs0 = tobs[i];
00989         }
00990         if (rt) {
00991           int st;
00992           sprintf(temprec, "%s[:#%lld]", dstemp, rt->recnum);
00993           if(dstatus = drms_setkey_string(rs, "TEMP_REC", temprec)) {
00994             printk("**ERROR on drms_setkey_string() for %s\n", temprec);
00995           }
00996           itf1ccd1 = drms_getkey_float(rt, "ITF1CCD1", &st);
00997           itf2ccd2 = drms_getkey_float(rt, "ITF2CCD2", &st);
00998           itnuccd3 = drms_getkey_float(rt, "ITNUCCD3", &st);
00999           itsjccd4 = drms_getkey_float(rt, "ITSJCCD4", &st);
01000           bt06cbpx = drms_getkey_float(rt, "BT06CBPX", &st);
01001           bt07cbnx = drms_getkey_float(rt, "BT07CBNX", &st);
01002           bt15ieb  = drms_getkey_float(rt, "BT15IEB",  &st);
01003           it08gtwm = drms_getkey_float(rt, "IT08GTWM", &st);
01004           it14sppx = drms_getkey_float(rt, "IT14SPPX", &st);
01005           it16spnx = drms_getkey_float(rt, "IT16SPNX", &st);
01006 
01007           drms_setkey_float(rs, "ITF1CCD1", itf1ccd1);
01008           drms_setkey_float(rs, "ITF2CCD2", itf2ccd2);
01009           drms_setkey_float(rs, "ITNUCCD3", itnuccd3);
01010           drms_setkey_float(rs, "ITSJCCD4", itsjccd4);
01011           drms_setkey_float(rs, "BT06CBPX", bt06cbpx);
01012           drms_setkey_float(rs, "BT07CBNX", bt07cbnx);
01013           drms_setkey_float(rs, "BT15IEB",  bt15ieb);
01014           drms_setkey_float(rs, "IT08GTWM", it08gtwm);
01015           drms_setkey_float(rs, "IT14SPPX", it14sppx);
01016           drms_setkey_float(rs, "IT16SPNX", it16spnx);
01017         }
01018       }
01019 /****END NEW from Rock 10Sep2013************************************************/
01020 
01021       //For quicklook, find closest iris.timeline record & set scroll 26Nov2013
01022       if(quicklook) {
01023           DRMS_RecordSet_t *rset;
01024           DRMS_Record_t *rt;
01025           int st;
01026           TIME t_obs, roll_start;
01027           float roll_deg;
01028           t_obs = drms_getkey_time(rs0, "t_obs", &st);
01029         if (st) {
01030            t_obs = DRMS_MISSING_TIME;
01031         }
01032         else {
01033           sprintf(open_dsname, "iris.timeline_roll[? roll_start > %f and roll_start <= %f ?]", t_obs-86401, t_obs+1);
01034           printf("%s\n", open_dsname);
01035           rset = drms_open_records(drms_env, open_dsname, &st);
01036 
01037           if (!rset || !rset->n || st) {
01038               printk("Error in drms_open_records(%s); setting scroll to zero\n", open_dsname);
01039               scroll = 0.0;
01040 
01041           } else {
01042               // Pick last record
01043               //rt = rset->records[0];
01044               rt = rset->records[(rset->n)-1];
01045               sprintf(pointrec, "iris.timeline_roll[:#%lld]", rt->recnum);
01046               if(dstatus = drms_setkey_string(rs, "POINTREC", pointrec)) {
01047                 printk("**ERROR on drms_setkey_string() for %s\n", pointrec);
01048               }
01049               roll_start = drms_getkey_time(rt, "ROLL_START", &st);
01050               roll_deg = drms_getkey_float(rt, "DEGREES", &st);
01051 
01052               drms_setkey_float(rs, "SAT_ROT", roll_deg);
01053 
01054               scroll = roll_deg;
01055           }
01056 
01057           if (rset)
01058               drms_close_records(rset, DRMS_FREE_RECORD);
01059         }
01060       }
01061 
01062       //char *wavstr = drms_getkey_string(rs0, "WAVE_STR", &rstatus);
01063       char *imgpath = drms_getkey_string(rs0, "IMG_PATH", &rstatus);
01064       if(rstatus) {
01065         printk("Can't do drms_getkey_string() for IMG_PATH\n");
01066         return(1);
01067       }
01068       if(quicklook) {
01069         sprintf(open_dsname, "%s[? t_start=(select max(t_start) from %s where t_start <= %10.5f and t_stop > %10.5f and IMG_PATH='%s') and IMG_PATH='%s' ?]",
01070             dsffname, dsffname, tobs[i], tobs[i], imgpath, imgpath);
01071       }
01072       else {
01073         sprintf(open_dsname, "%s[? t_start <= %10.5f and t_stop > %10.5f and IMG_PATH='%s' ?]",
01074         dsffname, tobs[i], tobs[i], imgpath);
01075       }
01076       //printf("!!TEMP Flat field query: %s\n", open_dsname); //!!TEMP
01077       //printk("!!TEMP Flat field query: %s\n", open_dsname); //!!TEMP
01078       rsetff = drms_open_records(drms_env, open_dsname, &rstatus); //open FF 
01079       if(!rsetff || (rsetff->n == 0) || rstatus) {
01080         printk("Can't do drms_open_records(%s)\n", open_dsname);
01081         flatmiss[i] = 1; noimage[i] = 1;
01082         goto TEMPSKIP;
01083         return(1);      //new 2/22/2011
01084       }
01085       fcnt = rsetff->n;
01086       if(fcnt > 1) {
01087         printk("More than one FF found for %s?\n", open_dsname);
01088         printk("Use last one of %d found\n", fcnt); 
01089         //return(1);        //!!TBD
01090       }
01091       //rsff = rsetff->records[0];
01092       rsff = rsetff->records[fcnt-1];
01093       recnumff = rsff->recnum;
01094       sprintf(flatrec, "%s[:#%lld]", dsffname, recnumff);
01095       if(dstatus = drms_setkey_string(rs, "FLAT_REC", flatrec )) {
01096         printk("**ERROR on drms_setkey_string() for %s\n", flatrec);
01097       }
01098       drms_record_directory(rsff, path, 1);
01099       if(!*path) {
01100         printk("***ERROR: No path to segment for %s\n", open_dsname);
01101         //goto TEMPSKIP;    //!!!TEMP until have good flatfield
01102         return(1);
01103       }
01104       //printf("\npath to FF = %s\n", path);    //!!TEMP
01105       segmentff = drms_segment_lookup(rsff, "flatfield");
01106       Arrayff = drms_segment_read(segmentff, DRMS_TYPE_FLOAT, &rstatus);
01107       if(!Arrayff) {
01108         printk("Can't do drms_segment_read() for Flat Field status=%d\n", 
01109             rstatus);
01110         return(1);
01111       }
01112       l0l1->adataff = (float *)Arrayff->data; 
01113 
01114 /**************************No DARK for IRIS?**********************************/
01115       //darkseg = drms_segment_lookup(rsff, "DARK");
01116       darkseg = drms_segment_lookup(rsff, "dark");
01117       ArrayDark = drms_segment_read(darkseg, DRMS_TYPE_FLOAT, &rstatus);
01118       if(!ArrayDark) {
01119         printk("Can't do drms_segment_read() for dark. status=%d\n", rstatus);
01120         return(1);
01121       }
01122       l0l1->adatadark = (float *)ArrayDark->data; //free at end
01123 
01124       badseg = drms_segment_lookup(rsff, "bad_pixel");
01125       ArrayBad = drms_segment_read(badseg, DRMS_TYPE_INT, &rstatus);
01126       nbad = drms_array_size(ArrayBad)/sizeof(int);
01127       if(!ArrayBad) {
01128         printk("Can't do drms_segment_read() for bad_pixel. status=%d\n", 
01129             rstatus);
01130         return(1);
01131       }
01132       l0l1->adatabad = (int *)ArrayBad->data; //free at end
01133       //if(!(badoutpixseg = drms_segment_lookup(rs, "bad_pixel"))) {
01134       //    printk("No drms_segment_lookup(rs, bad_pixel) for lev1\n");
01135       //    return(1);
01136       //}
01137       l0l1->rs1 = rs;
01138       l0l1->rsff = rsff;
01139       l0l1->recnum1 = rs->recnum;  
01140       l0l1->darkflag = 0;
01141       l0l1->sumx = drms_getkey_short(rs0, "SUMSPTRL", &rstatus);
01142       l0l1->sumy = drms_getkey_short(rs0, "SUMSPAT", &rstatus);
01143       hshiexp = drms_getkey_int(rs, "HSHIEXP", &rstatus);
01144       hcamid = drms_getkey_int(rs, "HCAMID", &rstatus);
01145         float sumdc=0.0;
01146         int idc, numdc=0;
01147         int aimgshce = drms_getkey_int(rs, "AIMGSHCE", &rstatus);
01148         //if(aimgshce == 0) l0l1->darkflag = 1;
01149         short iifrmtyp = drms_getkey_short(rs0, "IIFRMTYP", &rstatus);
01150         if((iifrmtyp == 2) || (iifrmtyp == 5)) l0l1->darkflag = 1;
01151         if(rstatus = do_flat_iris(l0l1)) {
01152           printk("***ERROR in do_flat_iris() status=%d\n", rstatus);
01153           printf("***ERROR in do_flat_iris() status=%d\n", rstatus);
01154           flatmiss[i] = 1; noimage[i] = 1;
01155           //return(1);      //!!TBD what to do?
01156           goto FLATERR;
01157         }
01158 
01159         //sprintf(tmpname, "/tmp/data_lev1.%u", fsnx);
01160         //fwt = fopen(tmpname, "w");
01161         //int wsize = 4096 * 4096;
01162         //fwrite(l0l1->adata1, sizeof(float), wsize, fwt);
01163         //fclose(fwt);
01164         drms_setkey_float(rs, "OSCNMEAN", l0l1->oscnmean);
01165         drms_setkey_float(rs, "OSCNRMS", l0l1->oscnrms);
01166         drms_setkey_int(rs, "DATAMIN", l0l1->datamin);
01167         drms_setkey_int(rs, "DATAMAX", l0l1->datamax);
01168         drms_setkey_int(rs, "DATAMEDN", l0l1->datamedn);
01169         drms_setkey_float(rs, "DATAMEAN", l0l1->datamean);
01170         drms_setkey_float(rs, "DATARMS", l0l1->data_rms);
01171         drms_setkey_float(rs, "DATASKEW", l0l1->dataskew);
01172         drms_setkey_float(rs, "DATAKURT", l0l1->datakurt);
01173         drms_setkey_int(rs, "DATAVALS", l0l1->datavals);
01174         drms_setkey_int(rs, "MISSVALS", l0l1->missvals);
01175         missvals = (uint32_t)l0l1->missvals;
01176         totvals = (uint32_t)l0l1->datavals + missvals;
01177         drms_setkey_int(rs, "TOTVALS", (int)totvals);
01178         percentd = (float)((100.0 * (float)l0l1->datavals)/(float)totvals);
01179         drms_setkey_float(rs, "PERCENTD", percentd);
01180         if(missvals > 0) missflg[i] = missflg[i] | Q_1_MISS0;
01181         if(missvals > (uint32_t)(totvals * 0.01))
01182            missflg[i] = missflg[i] | Q_1_MISS1;
01183         if(missvals > (uint32_t)(totvals * 0.05)) 
01184            missflg[i] = missflg[i] | Q_1_MISS2;
01185         if(missvals > (uint32_t)(totvals * 0.25)) 
01186            missflg[i] = missflg[i] | Q_1_MISS3;
01187         if(l0l1->datavals == 0) 
01188            missflg[i] = missflg[i] | Q_MISSALL; //high bit, no data
01189         //if(nspikes) {
01190         //  if (spikeseg = drms_segment_lookup(rs,"spikes") ) {
01191         //    nbytes = nspikes*sizeof(int);
01192         //    axes[0] = nspikes;
01193         //    axes[1] = 3;
01194         //    spikedata = (int *) malloc (3*nspikes*sizeof(int));
01195         //    ArraySpike = drms_array_create(DRMS_TYPE_INT, 2, axes,
01196         //               (void *) spikedata, &status);
01197         //    memcpy((void *)spikedata, (void *)spikelocs, nbytes);
01198         //    memcpy((void *)(spikedata+nspikes), (void *)oldvalues, nbytes);
01199         //    memcpy((void *)(spikedata+2*nspikes), (void *)newvalues, nbytes);
01200         //    status = drms_segment_write(spikeseg, ArraySpike, 0);
01201         //    drms_free_array(ArraySpike);
01202         //  } // else { printf("spikes segment not found\n"); }
01203         //}
01204 
01205 /**************************No DARK for IRIS?**********************************/
01206 
01207 FLATERR:
01208       drms_close_records(rsetff, DRMS_FREE_RECORD);
01209       free(ArrayDark->data);
01210       free(Arrayff->data);
01211       free(ArrayBad->data);
01212 
01213 TEMPSKIP:
01214   x0_lf = DRMS_MISSING_DOUBLE;
01215   y0_lf = DRMS_MISSING_DOUBLE;
01216   rsun_lf = DRMS_MISSING_DOUBLE;
01217   //Don't send calmode or darks to limb_fit()
01218   //calmode: HCFTID=17
01219   //dark: HSHIEXP=0 and HCAMID=0 or 1
01220   int skiplimb = 0;
01221   hcftid = drms_getkey_int(rs, "HCFTID", &rstatus);
01222   if(hcftid == CAL_HCFTID) {    //don't call limb_fit()
01223     printk("Cal mode image fsn=%u\n", fsnx);
01224     skiplimb = 1;
01225   }
01226   else {
01227     if(hshiexp == 0) {
01228       if(hcamid == 0 || hcamid == 1) {
01229         printk("Dark image fsn=%u\n", fsnx);
01230         skiplimb = 1;
01231       }
01232     }
01233   }
01234   lstatus = 1;              //default bad limb_fit
01235   //goto WCSEND;        //skip all WCS stuff for IRIS
01236 
01237 WCSEND:
01238 
01239   //if(hmiaiaflg) {                       //aia
01240   //  int wl = drms_getkey_int(rs, "WAVELNTH", &rstatus);
01241   //}
01242 
01243   do_quallev1(rs0, rs, i, fsnx);
01244   //ftmp = StopTimer(2); //!!!TEMP
01245   //printf( "\nTime sec in inside loop for fsn=%u : %f\n\n", fsnx, ftmp );
01246 
01247 IRISSKIP:
01248       dstatus = drms_segment_writewithkeys(segment, segArray, 0);
01249       //dstatus = drms_segment_write(segment, segArray, 0);
01250       if (dstatus) {
01251         printk("ERROR: drms_segment_write error=%d for fsn=%u\n",
01252 dstatus,fsnx);
01253         noimage[i] = 1;
01254       }
01255       recnum1 = rs->recnum;
01256       printk("*1 %u %u\n", recnum1, fsnx);
01257       free(Array0->data);
01258 //      if (rs_t) drms_close_records(rs_t, DRMS_FREE_RECORD);
01259       if (rs_resp) drms_close_records(rs_resp, DRMS_FREE_RECORD);
01260       rs_resp = NULL;
01261   }             //END do for all the sorted lev0 records
01262 
01263   drms_close_records(rset0, DRMS_FREE_RECORD);   //close lev0 records
01264   drms_close_records(rset1, DRMS_INSERT_RECORD); //close lev1 records
01265   return(0);
01266 }
01267 
01268 int compare_rptr(const void *a, const void *b)
01269 {
01270   TIME t1, t2;
01271   int rstatus;
01272   DRMS_Record_t *x=(DRMS_Record_t *)a, *y=(DRMS_Record_t *)b;
01273 
01274   t1 = drms_getkey_time(x, "t_obs", &rstatus); 
01275   if(rstatus) t1 = DRMS_MISSING_TIME;   //treat error as missing t_obs
01276   t2 = drms_getkey_time(y, "t_obs", &rstatus); 
01277   if(rstatus) t2 = DRMS_MISSING_TIME;
01278   if(t1 < t2) return(-1);
01279   if(t1 > t2) return(1);
01280   return(0);
01281 }
01282 
01283 // Initial setup stuff called when main is first entered.
01284 void setup()
01285 {
01286   FILE *fin;
01287   char string[128], cwdbuf[128], idstr[256], lfile[128];
01288   int tpid;
01289 
01290   sdo_epoch = sscan_time("1958.01.01_00:00:00_TAI");
01291   do_datestr();
01292   printk_set(h1log, h1log); // set for printk calls 
01293   printk("%s\n", datestr);
01294   gethostname(idstr, 256);
01295   printf("Host: %s\n", idstr);
01296   printk("Host: %s\n", idstr);
01297   getcwd(cwdbuf, 126);
01298   sprintf(idstr, "Cwd: %s\nCall: ", cwdbuf);
01299   sprintf(string, "build_lev1_iris started as pid=%d ppid=%d user=%s\n", 
01300         getpid(), getppid(), username);
01301   strcat(idstr, string);
01302   printk("%s", idstr);
01303   printf("%s", idstr);
01304   if(restartflg) printk("-r ");
01305   sprintf(argmode, "mode=%s", mode);
01306   sprintf(argdsin, "dsin=%s", dsin);
01307   sprintf(argdsout, "dsout=%s", dsout);
01308   if(modeflg) {
01309     sprintf(argbx, "brec=%lld", brec);
01310     sprintf(argex, "erec=%lld", erec);
01311   }
01312   else {
01313     sprintf(argbx, "bfsn=%lld", bfsn);
01314     sprintf(argex, "efsn=%lld", efsn);
01315   }
01316   sprintf(argquick, "quicklook=%d", quicklook);
01317   sprintf(arglogfile, "logfile=%s", logname);
01318   printk("%s %s %s %s %s %s %s\n", 
01319     argmode, argdsin, argdsout, argbx, argex, argquick, arglogfile);
01320   printf("%s %s %s %s %s %s %s\n", 
01321     argmode, argdsin, argdsout, argbx, argex, argquick, arglogfile);
01322   if(!restartflg) {
01323     //printk("tlmseriesname=%s\nlev0seriesname=%s\n", 
01324     //      tlmseriesname, lev0seriesname);
01325   }
01326   sprintf(bld_vers, "%s", jsoc_version);
01327   sprintf(idstr,  "ps -ef | grep %s", LEV1VIEWERNAME);
01328   fin = popen(idstr, "r");
01329   while(fgets(string, sizeof string, fin)) {  //get ps line
01330     if(!(strstr(string, "perl"))) continue;
01331     sscanf(string, "%s %d", idstr, &tpid); /* get user name & process id */
01332     sprintf(lfile, "%s/build_lev1_iris_restart_%d.touch", LEV1LOG_BASEDIR, tpid);
01333     sprintf(idstr, "/bin/touch %s", lfile);
01334     printk("%s\n", idstr);
01335     system(idstr);
01336   }
01337   umask(002);           // allow group write 
01338   //Image.initialized = 0;  // init the two image structures 
01339   //ImageOld.initialized = 0;
01340   //Img = &Image;
01341   //ImgO = &ImageOld;
01342   //Img->initialized = 0;
01343   //ImgO->initialized = 0;
01344 }
01345 
01346 // Module main function. 
01347 int DoIt(void)
01348 {
01349   long long numofrecs, frec, lrec;
01350   int numrec, numofchunks, i;
01351   char line[80];
01352 
01353   if (nice_intro())
01354     return (0);
01355   if(!(username = (char *)getenv("USER"))) username = "nouser"; 
01356   mode = cmdparams_get_str(&cmdparams, "mode", NULL);
01357   if(strcmp(mode, "recnum") && strcmp(mode, "fsn")) {
01358     printf("Error: mode= must be given as 'recnum' or 'fsn'\n");
01359     return(0);
01360   }
01361   if(!strcmp(mode, "recnum")) modeflg = 1;
01362   dsin = cmdparams_get_str(&cmdparams, "dsin", NULL);
01363   dsout = cmdparams_get_str(&cmdparams, "dsout", NULL);
01364   brec = cmdparams_get_int(&cmdparams, "brec", NULL);
01365   erec = cmdparams_get_int(&cmdparams, "erec", NULL);
01366   bfsn = cmdparams_get_int(&cmdparams, "bfsn", NULL);
01367   efsn = cmdparams_get_int(&cmdparams, "efsn", NULL);
01368   quicklook = cmdparams_get_int(&cmdparams, "quicklook", NULL);
01369   //quicklook = 1; //!!TEMP for test
01370   if(modeflg) {     //recnum mode
01371     if(brec == 0 || erec == 0) {
01372       fprintf(stderr, "brec and erec must be given for recnum mode. 0 not allowed\n");
01373       return(0);
01374     }
01375     if(brec > erec) {
01376       fprintf(stderr, "brec must be <= erec\n");
01377       return(0);
01378     }
01379     bnumx = brec;
01380     enumx = erec;
01381   }
01382   else {        //fsn mode
01383     if(bfsn == 0 || efsn == 0) {
01384       fprintf(stderr, "bfsn and efsn must be given for fsn mode. 0 not allowed\n");
01385       return(0);
01386     }
01387     if(bfsn > efsn) {
01388       fprintf(stderr, "bfsn must be <= efsn\n");
01389       return(0);
01390     }
01391     bnumx = bfsn;
01392     enumx = efsn;
01393   }
01394   logfile = cmdparams_get_str(&cmdparams, "logfile", NULL);
01395   if (strcmp(dsin, NOTSPECIFIED) == 0) {
01396     dsin = LEV0SERIESNAMEIRIS;
01397   }
01398   if (strcmp(dsout, NOTSPECIFIED) == 0) {
01399     dsout = LEV1SERIESNAMEIRIS;
01400   }
01401   sprintf(dsffname, "%s", DSFFNAME);
01402 
01403   if (strcmp(logfile, NOTSPECIFIED) == 0) {
01404     sprintf(logname, H1LOGFILE, gettimetag());
01405   }
01406   else {
01407     sprintf(logname, "%s", logfile);
01408   }
01409   if(restartflg || LOGTEST) {
01410     if((h1logfp=fopen(logname, "a")) == NULL)
01411       fprintf(stderr, "**Can't open for append the log file %s\n", logname);
01412   }
01413   else {
01414     if((h1logfp=fopen(logname, "w")) == NULL)
01415       fprintf(stderr, "**Can't open the log file %s\n", logname);
01416   }
01417   setup();
01418   numofrecs = (enumx - bnumx) + 1;
01419   numrec = NUMRECLEV1;     //# of records to do at a time
01420   numofchunks = numofrecs/numrec;
01421   if((numofrecs % numrec) != 0) numofchunks++; //extra loop for partial chunk
01422   lrec = bnumx-1;
01423   for(i = 0; i < numofchunks; i++) {
01424     frec = lrec+1; lrec = (frec + numrec)-1;
01425     if(lrec > enumx) lrec=enumx;
01426     if(do_ingest(frec, lrec)) {  //do a chunk to get files from the lev0
01427       printf("build_lev1_iris abort\nSee log: %s\n", logname); 
01428       send_mail("build_lev1_iris abort\nSee log: %s\n", logname); 
01429       return(0);
01430     }
01431     if(modeflg) {               //only do for recnum mode
01432       if(stat(STOP_FILE, &stbuf) == 0) {
01433         printf("Stop file %s seen. Exit build_lev1_iris.\n", STOP_FILE);
01434         break;
01435       }
01436     }
01437   }
01438   printf("build_lev1_iris done last fsn=%u\n", fsnx); 
01439   return(0);
01440 }
proj/lev0/apps/build_lev1_iris.c
