JSOC_Documentation: proj/lev0/apps/build_lev1

00001 /*-----------------------------------------------------------------------------
00002  * cvs/JSOC/proj/lev1/apps/build_lev1_empty.c
00003  *-----------------------------------------------------------------------------
00004  *
00005  * This is called by lev1_def_gui calling lev1_def_gui_called which figures
00006  * out the lev1 FSN that are missing for the Ord date just processed.
00007  * These missing FSNs are sent here and this module makes empty hmi.lev1
00008  * records for them. A missing record just contains the date, fsn, and
00009  * QUALITY keyword with the high bit set.
00010  */ 
00011 
00012 #include <jsoc_main.h>
00013 #include <cmdparams.h>
00014 #include <drms.h>
00015 #include <drms_names.h>
00016 #include <stdio.h>
00017 #include <stdlib.h>
00018 #include <ctype.h>
00019 #include <strings.h>
00020 #include <sys/types.h>
00021 #include <sys/time.h>
00022 #include <sys/stat.h>   //for umask(2)
00023 #include <unistd.h> //for alarm(2) among other things...
00024 #include <printk.h>
00025 #include <astro.h>
00026 #include <fresize.h>
00027 #include <gapfill.h>
00028 #include "fftw3.h" 
00029 #include "imgdecode.h"
00030 #include "lev0lev1.h"
00031 #include "quallev1.h"
00032 
00033 //default in and out data series
00034 #define LEV1SERIESNAMEHMI "hmi.lev1"
00035 #define LEV1SERIESNAMEAIA "su_production.aia_lev1e" //temp test case
00036 #define DSFFNAMEHMI "hmi.flatfield"
00037 //#define DSFFNAMEAIA "su_production.aia_flatfield" //temp test case
00038 //#define DSFFNAMEAIA "aia_test.flatfield"  //temp test case
00039 #define DSFFNAMEAIA "aia.flatfield"
00040 
00041 #define LEV1LOG_BASEDIR "/usr/local/logs/lev1"
00042 #define H1LOGFILE "/usr/local/logs/lev1/build_lev1_empty.%s.log"
00043 #define NUMTIMERS 8     //number of seperate timers avail
00044 #define NOTSPECIFIED "***NOTSPECIFIED***"
00045 #define LOGTEST 0
00046 #define CAL_HCFTID 17       //image is cal mode 
00047 
00048 int compare_rptr(const void *a, const void *b);
00049 static TIME SDO_to_DRMS_time(int sdo_s, int sdo_ss);
00050 
00051 // List of default parameter values. 
00052 ModuleArgs_t module_args[] = { 
00053   {ARG_STRING, "instru", NOTSPECIFIED, "instrument. either hmi or aia"},
00054   {ARG_STRING, "dsout", NOTSPECIFIED, "dataset of lev1 output"},
00055   {ARG_STRING, "argfile", NOTSPECIFIED, "file name with FSN args from lev1_def_gui_called"},
00056   {ARG_STRING, "logfile", NOTSPECIFIED, "optional log file name. Will create one if not given"},
00057   {ARG_INTS, "bfsn", "0", "first lev1 fsn# to process. 0=error"},
00058   {ARG_INTS, "efsn", "0", "last lev1 fsn# to process. 0=error"},
00059   {ARG_FLAG, "v", "0", "verbose flag"},
00060   {ARG_FLAG, "h", "0", "help flag"},
00061   {ARG_END}
00062 };
00063 
00064 CmdParams_t cmdparams;
00065 // Module name presented to DRMS. 
00066 char *module_name = "build_lev1_empty";
00067 
00068 FILE *h1logfp;      // fp for h1 ouput log for this run 
00069 static LEV0LEV1 lev0lev1;
00070 static LEV0LEV1 *l0l1 = &lev0lev1;
00071 static DRMS_Record_t *rs;
00072 static DRMS_Record_t *rs0, *rs1, *rsff, *rsbad_pix, *rec_bad_aia;
00073 static DRMS_Record_t *rptr;
00074 static DRMS_Segment_t *segment;
00075 static DRMS_Segment_t *segmentff;
00076 static DRMS_Segment_t *darkseg;
00077 static DRMS_Segment_t *badseg;
00078 static DRMS_Segment_t *badoutpixseg;
00079 static DRMS_Segment_t *spikeseg;
00080 static DRMS_Array_t *segArray;
00081 static DRMS_RecordSet_t *rset0, *rset1, *rsetff, *rsbad_aia;
00082 static DRMS_Array_t *Array0;
00083 static DRMS_Array_t *Arrayff;
00084 static DRMS_Array_t *ArrayDark;
00085 static DRMS_Array_t *ArrayBad;
00086 static DRMS_Array_t *ArraySpike;
00087 static TIME sdo_epoch;
00088 //static PTINFO *ptinfo = NULL;
00089 //static PTINFO ptdata;
00090 static char bld_vers[16];
00091 static char datestr[32], datestrZ[32];
00092 static char open_dsname[256];
00093 static char dsffname[128];
00094 static char path[DRMS_MAXPATHLEN], bad_pix_path[DRMS_MAXPATHLEN];
00095 static char bad_aia_path[DRMS_MAXPATHLEN];
00096 static char rs1_path[DRMS_MAXPATHLEN];
00097 static struct timeval first[NUMTIMERS], second[NUMTIMERS];
00098 static char *orbseries = "sdo.fds_orbit_vectors";
00099 //static char *orbseries = "sdo_ground.fds_orbit_vectors";
00100 
00101 static int nspikes, respike, fid, aiftsid, *oldvalues, *spikelocs, *newvalues;
00102 static int hcftid, aiagp6;
00103 static short aifcps;
00104 double aiascale = 1.0;
00105 
00106 IORBIT_Info_t *IOinfo = NULL;
00107 IORBIT_Info_t IOdata;
00108 LIBASTRO_Error_t IOstatus;
00109 unsigned int fsnarray[NUMRECLEV1];
00110 unsigned int fsnx = 0;
00111 //short data1[MAXPIXELS];
00112 //int data1[MAXPIXELS];
00113 float data1[MAXPIXELS];     //floats for HMI
00114 float ftmp;
00115 int data1A[MAXPIXELS];      //ints for AIA
00116 int array_cosmic[16777216]; //4096x4096
00117 double tgttimes[NUMRECLEV1];
00118 
00119 unsigned int bfsn, efsn;    //begin and end fsn.
00120 unsigned int bnumx, enumx;  //has the bfsn/efsn pair
00121 int verbose;
00122 int hmiaiaflg = 0;      //0=hmi, 1=aia
00123 int imagecnt = 0;       // num of images since last commit 
00124 int restartflg = 0;     // set when build_lev1 is called for a restart
00125 int abortflg = 0;
00126 int sigalrmflg = 0;             // set on signal so prog will know 
00127 int ignoresigalrmflg = 0;       // set after a close_image()
00128 //global quality flags
00129 int flatmiss[NUMRECLEV1];
00130 int orbmiss[NUMRECLEV1];
00131 int asdmiss[NUMRECLEV1];
00132 int mpdmiss[NUMRECLEV1];
00133 int limbmiss[NUMRECLEV1];
00134 int noimage[NUMRECLEV1];
00135 int missflg[NUMRECLEV1];
00136 
00137 char logname[128];
00138 char argdsout[128], arglogfile[128], arginstru[80], argargfile[128];
00139 char argbx[80], argex[80];
00140 char timetag[32];
00141 char tlmseriesname[128];    // e.g. hmi.tlm
00142 char lev0seriesname[128];   // e.g. hmi.lev0
00143 char *username;         // from getenv("USER") 
00144 char *logfile;          // optional log name passed in 
00145 char *argfile;          // arg file name passed in 
00146 char *instru;           // instument. hmi or aia 
00147 char *dsout;            // lev1 output dataset
00148 char *dsff;                     // flat field dataset
00149 
00150 
00151 int get_nspikes() { return nspikes; }
00152 int get_respike(void) { return respike; }
00153 int *get_spikelocs() { return spikelocs; }
00154 int *get_oldvalues() { return oldvalues; }
00155 int *get_newvalues() { return newvalues; }
00156 void set_nspikes(int new_nspikes) { nspikes = new_nspikes; }
00157 void set_spikelocs(int *new_spikelocs) { spikelocs = new_spikelocs; }
00158 void set_oldvalues(int *new_oldvalues) { oldvalues = new_oldvalues; }
00159 void set_newvalues(int *new_newvalues) { newvalues = new_newvalues; }
00160 
00161 int nice_intro ()
00162 {
00163   int usage = cmdparams_get_int (&cmdparams, "h", NULL);
00164   if (usage)
00165     {
00166     printf ("Usage:\nbuild_lev1_empty [-vh] "
00167     "instru=<hmi|aia> dsout=<lev1>\n"
00168     "bfsn=<fsn#> efsn=<fsn#>\n"
00169         "argfile=<file>\n"
00170     "[logfile=<file>]\n"
00171     "  -h: help - show this message then exit\n"
00172     "  -v: verbose\n"
00173     "instru= instrument. must be 'hmi' or 'aia'\n"
00174     "dsout= data set name of lev1 output\n"
00175     "      default hmi=su_production.hmi_lev1e   aia=su_production.aia_lev1e\n"
00176     "bfsn= first fsn# to process. 0=error must be given by build_lev1_mgr\n"
00177     "efsn= last fsn# to process. 0=error must be given by build_lev1_mgr\n"
00178     "argfile= file with FSN args from lev1_def_gui_called\n"
00179     "logfile= optional log file name. If not given uses:\n"
00180         "         /usr/local/logs/lev1/build_lev1_empty.<time_stamp>.log\n");
00181     return(1);
00182     }
00183   verbose = cmdparams_get_int (&cmdparams, "v", NULL);
00184   //restartflg = cmdparams_get_int (&cmdparams, "r", NULL);
00185   return (0);
00186 }
00187 
00188 TIME SDO_to_DRMS_time(int sdo_s, int sdo_ss)
00189 {
00190 static int firstcall = 1;
00191 if (firstcall)
00192   {
00193   firstcall = 0;
00194   }
00195 /* XXX fix build 3/18/2008, arta */
00196 return(sdo_epoch + (TIME)sdo_s + (TIME)(sdo_ss & 0xFFFF)/65536.0);
00197 }
00198 
00199 // Setup global datestr[] like: 2008.07.14_08:29:31
00200 char *do_datestr() {
00201   time_t tval;
00202   struct tm *t_ptr;
00203 
00204   tval = time(NULL);
00205   t_ptr = localtime(&tval);
00206   sprintf(datestr, "%d.%02d.%02d_%02d:%02d:%02d", 
00207       (t_ptr->tm_year+1900), (t_ptr->tm_mon+1),
00208       t_ptr->tm_mday, t_ptr->tm_hour, t_ptr->tm_min, t_ptr->tm_sec);
00209   return(datestr);
00210 }
00211 
00212 // Setup global datestrZ[] like: 2010-09-28T20:19:38Z
00213 char *do_datestrZ() {
00214   time_t tval;
00215   struct tm *t_ptr;
00216 
00217   tval = time(NULL);
00218   t_ptr = gmtime(&tval);
00219   sprintf(datestrZ, "%d-%02d-%02dT%02d:%02d:%02dZ", 
00220       (t_ptr->tm_year+1900), (t_ptr->tm_mon+1),
00221       t_ptr->tm_mday, t_ptr->tm_hour, t_ptr->tm_min, t_ptr->tm_sec);
00222   return(datestrZ);
00223 }
00224 
00225 // Returns a time tag like  yyyy.mm.dd.hhmmss 
00226 char *gettimetag()
00227 {
00228   struct timeval tvalr;
00229   struct tm *t_ptr;
00230 
00231   gettimeofday(&tvalr, NULL);
00232   t_ptr = localtime((const time_t *)&tvalr);
00233   sprintf(timetag, "%04d.%02d.%02d.%02d%02d%02d",
00234         (t_ptr->tm_year+1900), (t_ptr->tm_mon+1), t_ptr->tm_mday, t_ptr->tm_hour, t_ptr->tm_min, t_ptr->tm_sec);
00235   return(timetag);
00236 }
00237 
00238 
00239 void BeginTimer(int n)
00240 {
00241   gettimeofday (&first[n], NULL);
00242 }
00243 
00244 float EndTimer(int n)
00245 {
00246   gettimeofday (&second[n], NULL);
00247   if (first[n].tv_usec > second[n].tv_usec) {
00248     second[n].tv_usec += 1000000;
00249     second[n].tv_sec--;
00250   }
00251   return (float) (second[n].tv_sec-first[n].tv_sec) +
00252     (float) (second[n].tv_usec-first[n].tv_usec)/1000000.0;
00253 }
00254 
00255 // Outputs the variable format message (re: printf) to the log file.
00256 int h1log(const char *fmt, ...)
00257 {
00258   va_list args;
00259   char string[32768];
00260 
00261   va_start(args, fmt);
00262   vsprintf(string, fmt, args);
00263   if(h1logfp) {
00264     fprintf(h1logfp, string);
00265     fflush(h1logfp);
00266   } 
00267   else {            // couldn't open log 
00268     printf(string);     // also print to stdout
00269     fflush(stdout);
00270   }
00271   va_end(args);
00272   return(0);
00273 }
00274 
00275 int send_mail(char *fmt, ...)
00276 {
00277   va_list args;
00278   char string[1024], cmd[1024];
00279 
00280   va_start(args, fmt);
00281   vsprintf(string, fmt, args);
00282   sprintf(cmd, "echo \"%s\" | Mail -s \"build_lev1_empty mail\" lev0_user", string);
00283   system(cmd);
00284   va_end(args);
00285   return(0);
00286 }
00287 
00288 // Got a fatal error. 
00289 void abortit(int stat)
00290 {
00291   printk("***Abort in progress ...\n");
00292   printk("**Exit build_lev1_empty w/ status = %d\n", stat);
00293   if (h1logfp) fclose(h1logfp);
00294   exit(stat);
00295 }
00296 
00297 //#include "aia_despike.c"
00298 //#include "do_flat.c"
00299 //#include "get_image_location.c"
00300 //#include "limb_fit_function.c"
00301 //#include "cosmic_ray.c"
00302 //#include "heightformation.c"
00303 
00304 //Called with the range to do. The args are fsn#s.
00305 //int do_ingest(long long bbrec, long long eerec)
00306 int do_ingest(unsigned int bbrec, unsigned int eerec)
00307 {
00308   int rstatus, dstatus, lstatus, ncnt, fcnt, i, j, k, qualint, nobs;
00309 
00310   ncnt = (eerec - bbrec) + 1;
00311     rset1 = drms_create_records(drms_env, ncnt, dsout, DRMS_PERMANENT,&dstatus);
00312     if(dstatus) {
00313       printk("**ERROR: Can't create records for %s\n", dsout);
00314       for(j=0; j < ncnt; j++) {  //set qual bits
00315         noimage[j] = 1;
00316       }
00317       return(1);
00318     }
00319   for(i=0; i < ncnt; i++) {
00320     rs = rset1->records[i]; 
00321     dstatus = drms_setkey_int(rs, "FSN", bbrec++);
00322     dstatus = drms_setkey_string(rs, "LEV0SERIES", "N/A");
00323     dstatus = drms_setkey_int(rs, "QUALITY", Q_MISSALL);
00324     dstatus = drms_setkey_string(rs, "BLD_VERS", bld_vers);
00325     dstatus = drms_setkey_time(rs, "DATE", time(0) + UNIX_EPOCH);
00326   }
00327   drms_close_records(rset1, DRMS_INSERT_RECORD); //close lev1 records
00328   return(0);
00329 }
00330 
00331 // Initial setup stuff called when main is first entered.
00332 void setup()
00333 {
00334   FILE *fin;
00335   char string[128], cwdbuf[128], idstr[256], lfile[128];
00336   int tpid;
00337 
00338   sdo_epoch = sscan_time("1958.01.01_00:00:00_TAI");
00339   do_datestr();
00340   printk_set(h1log, h1log); // set for printk calls 
00341   printk("%s\n", datestr);
00342   gethostname(idstr, 256);
00343   printf("Host: %s\n", idstr);
00344   printk("Host: %s\n", idstr);
00345   getcwd(cwdbuf, 126);
00346   sprintf(idstr, "Cwd: %s\nCall: ", cwdbuf);
00347   sprintf(string, "build_lev1_empty started as pid=%d ppid=%d user=%s\n", 
00348         getpid(), getppid(), username);
00349   strcat(idstr, string);
00350   printk("%s", idstr);
00351   printf("%s", idstr);
00352   //if(restartflg) printk("-r ");
00353   sprintf(arginstru, "instru=%s", instru);
00354   sprintf(argdsout, "dsout=%s", dsout);
00355     sprintf(argbx, "bfsn=%u", bfsn);
00356     sprintf(argex, "efsn=%u", efsn);
00357   sprintf(arglogfile, "logfile=%s", logname);
00358   sprintf(argargfile, "argfile=%s", argfile);
00359   printk("%s %s %s %s %s %s\n", 
00360     arginstru, argdsout, argbx, argex, argargfile, arglogfile);
00361   printf("%s %s %s %s %s %s\n", 
00362     arginstru, argdsout, argbx, argex, argargfile, arglogfile);
00363   if(!restartflg) {
00364     //printk("tlmseriesname=%s\nlev0seriesname=%s\n", 
00365     //      tlmseriesname, lev0seriesname);
00366   }
00367   sprintf(bld_vers, "%s", jsoc_version);
00368   /********
00369   sprintf(idstr,  "ps -ef | grep %s", LEV1VIEWERNAME);
00370   fin = popen(idstr, "r");
00371   while(fgets(string, sizeof string, fin)) {  //get ps line
00372     if(!(strstr(string, "perl"))) continue;
00373     sscanf(string, "%s %d", idstr, &tpid); // get user name & process id 
00374     sprintf(lfile, "%s/build_lev1_empty_restart_%d.touch", LEV1LOG_BASEDIR, tpid);
00375     sprintf(idstr, "/bin/touch %s", lfile);
00376     printk("%s\n", idstr);
00377     system(idstr);
00378   }
00379   ********/
00380   umask(002);           // allow group write 
00381 }
00382 
00383 // Module main function. 
00384 int DoIt(void)
00385 {
00386   FILE *fplog;
00387   long long numofrecs, frec, lrec;
00388   int numrec, numofchunks, i;
00389   unsigned int lowfsn, highfsn;
00390   char line[128], scr[80];
00391 
00392   if (nice_intro())
00393     return (0);
00394   if(!(username = (char *)getenv("USER"))) username = "nouser"; 
00395   instru = cmdparams_get_str(&cmdparams, "instru", NULL);
00396   if(strcmp(instru, "hmi") && strcmp(instru, "aia")) {
00397     printf("Error: instru=%s must be given as 'hmi' or 'aia'\n", instru);
00398     printk("Error: instru=%s must be given as 'hmi' or 'aia'\n", instru);
00399     return(0);
00400   }
00401   if(!strcmp(instru, "aia")) hmiaiaflg = 1;
00402   dsout = cmdparams_get_str(&cmdparams, "dsout", NULL);
00403   bfsn = cmdparams_get_int(&cmdparams, "bfsn", NULL);
00404   efsn = cmdparams_get_int(&cmdparams, "efsn", NULL);
00405 /**************************************
00406     if(bfsn == 0 || efsn == 0) {
00407       fprintf(stderr, "bfsn and efsn must be given. 0 not allowed\n");
00408       return(0);
00409     }
00410     if(bfsn > efsn) {
00411       fprintf(stderr, "bfsn must be <= efsn\n");
00412       return(0);
00413     }
00414     bnumx = bfsn;
00415     enumx = efsn;
00416 **************************************/
00417   logfile = cmdparams_get_str(&cmdparams, "logfile", NULL);
00418   argfile = cmdparams_get_str(&cmdparams, "argfile", NULL);
00419   if (strcmp(dsout, NOTSPECIFIED) == 0) {
00420     fprintf(stderr, "You must specify dsout\n");
00421     return(0);
00422   }
00423   if (strcmp(argfile, NOTSPECIFIED) == 0) {
00424     fprintf(stderr, "You must specify argfile\n");
00425     return(0);
00426   }
00427   if (strcmp(logfile, NOTSPECIFIED) == 0) {
00428     sprintf(logname, H1LOGFILE, gettimetag());
00429   }
00430   else {
00431     sprintf(logname, "%s", logfile);
00432   }
00433   if((h1logfp=fopen(logname, "w")) == NULL)
00434     fprintf(stderr, "**Can't open the log file %s\n", logname);
00435   setup();
00436 
00437   if((fplog=fopen(argfile, "r")) == NULL) {
00438     fprintf(stderr, "Can't open %s\n", argfile);
00439     return(0);
00440   }
00441   //argfile lines look like:
00442   //Missing FSN: went from 12207204 to 12207207
00443   //Missing FSN: went from 12207603 to 12207610
00444   while(fgets(line, 128, fplog)) {       /* get ps lines */
00445      if (strstr(line, "Missing FSN:")) {
00446        sscanf(line, "%s %s %s %s %u %s %u", scr, scr, scr, scr, &lowfsn, scr, &highfsn);
00447        printf("lowfsn=%u highfsn=%u\n", lowfsn, highfsn); 
00448        if(do_ingest(lowfsn+1, highfsn-1)) {
00449          printf("build_lev1_empty abort\nSee log: %s\n", logname); 
00450          send_mail("build_lev1_empty abort\nSee log: %s\n", logname); 
00451          return(0);
00452        }
00453      }
00454   }
00455   fclose(fplog);
00456   printf("build_lev1_empty done\n"); 
00457   return(0);
00458 }
proj/lev0/apps/build_lev1_empty.c
