00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00013
00014 #include <jsoc_main.h>
00015 #include <cmdparams.h>
00016 #include <drms.h>
00017 #include <drms_names.h>
00018 #include <stdio.h>
00019 #include <stdlib.h>
00020 #include <ctype.h>
00021 #include <strings.h>
00022 #include <sys/types.h>
00023 #include <sys/time.h>
00024 #include <sys/stat.h>
00025 #include <unistd.h>
00026 #include <printk.h>
00027 #include <astro.h>
00028 #include <fresize.h>
00029 #include <gapfill.h>
00030
00031 #include "fftw3.h"
00032 #include "imgdecode.h"
00033 #include "lev0lev1.h"
00034 #include "quallev1.h"
00035 #include "limb_fit.h"
00036
00037 #include "get_pointing_info.c"
00038
00039
00040
00041
00042 #define LEV0SERIESNAMEHMI "su_production.lev0f_hmi"
00043 #define LEV0SERIESNAMEAIA "su_production.lev0f_aia"
00044
00045 #define LEV1SERIESNAMEHMI "hmi.lev1"
00046 #define LEV1SERIESNAMEAIA "su_production.aia_lev1e" //temp test case
00047
00048 #define DSFFNAMEHMI "su_production.hmi_flatfield" //temp test case
00049 #define DSFFNAMEAIA "su_production.aia_flatfield" //temp test case
00050
00051 #define LEV1LOG_BASEDIR "/usr/local/logs/lev1"
00052 #define H1LOGFILE "/usr/local/logs/lev1/build_lev1_fsn.%s.log"
00053 #define NUMTIMERS 8 //number of seperate timers avail
00054 #define NOTSPECIFIED "***NOTSPECIFIED***"
00055 #define LOGTEST 0
00056 #define CAL_HCFTID 17 //image is cal mode
00057
00058 #define kDpath "dpath"
00059 #define kDpathDef "/home/jsoc/cvs/Development/JSOC"
00060
00061 int compare_rptr(const void *a, const void *b);
00062 static TIME SDO_to_DRMS_time(int sdo_s, int sdo_ss);
00063
00064
00065 ModuleArgs_t module_args[] = {
00066 {ARG_STRING, "instru", NOTSPECIFIED, "instrument. either hmi or aia"},
00067 {ARG_STRING, "dsin", NOTSPECIFIED, "dataset of lev0 filtergrams"},
00068 {ARG_STRING, "dsout", NOTSPECIFIED, "dataset of lev1 output"},
00069 {ARG_STRING, "logfile", NOTSPECIFIED, "optional log file name. Will create one if not given"},
00070 {ARG_INTS, "bfsn", "0", "first lev0 fsn to process. 0=error must be given by build_lev1_mgr_fsn"},
00071 {ARG_INTS, "efsn", "0", "last lev0 fsn to process. 0=error must be given by build_lev1_mgr_fsn"},
00072 {ARG_INTS, "quicklook", "1", "1=quick look, 0 = definitive mode"},
00073 {ARG_FLAG, "v", "0", "verbose flag"},
00074 {ARG_FLAG, "h", "0", "help flag"},
00075 {ARG_FLAG, "r", "0", "restart flag"},
00076 {ARG_STRING, kDpath, kDpathDef, "path to the source code tree (the JSOC root directory)"},
00077 {ARG_END}
00078 };
00079
00080 CmdParams_t cmdparams;
00081
00082 char *module_name = "build_lev1_fsn";
00083
00084 FILE *h1logfp;
00085
00086
00087
00088 static LEV0LEV1 lev0lev1;
00089 static LEV0LEV1 *l0l1 = &lev0lev1;
00090
00091 static DRMS_Record_t *rs;
00092 static DRMS_Record_t *rs0, *rs1, *rsff, *rsbad_pix;
00093 static DRMS_Record_t *rptr;
00094 static DRMS_Segment_t *segment;
00095 static DRMS_Segment_t *segmentff;
00096 static DRMS_Segment_t *darkseg;
00097 static DRMS_Segment_t *badseg;
00098 static DRMS_Segment_t *badoutpixseg;
00099 static DRMS_Segment_t *spikeseg;
00100 static DRMS_Array_t *segArray;
00101 static DRMS_RecordSet_t *rset0, *rset1, *rsetff;
00102 static DRMS_Array_t *Array0;
00103 static DRMS_Array_t *Arrayff;
00104 static DRMS_Array_t *ArrayDark;
00105 static DRMS_Array_t *ArrayBad;
00106 static DRMS_Array_t *ArraySpike;
00107 static TIME sdo_epoch;
00108 static PTINFO *ptinfo = NULL;
00109 static PTINFO ptdata;
00110 static char bld_vers[16];
00111 static char datestr[32];
00112 static char open_dsname[256];
00113 static char dsffname[128];
00114 static char path[DRMS_MAXPATHLEN], bad_pix_path[DRMS_MAXPATHLEN];
00115 static char rs1_path[DRMS_MAXPATHLEN];
00116 static struct timeval first[NUMTIMERS], second[NUMTIMERS];
00117 static char *orbseries = "sdo.fds_orbit_vectors";
00118
00119
00120 static int nspikes, *oldvalues, *spikelocs, *newvalues;
00121
00122 IORBIT_Info_t *IOinfo = NULL;
00123 IORBIT_Info_t IOdata;
00124 LIBASTRO_Error_t IOstatus;
00125 unsigned int fsnarray[NUMRECLEV1];
00126 unsigned int fsnx = 0;
00127
00128
00129 float data1[MAXPIXELS];
00130 int data1A[MAXPIXELS];
00131 double tgttimes[NUMRECLEV1];
00132
00133 unsigned int bfsn, efsn;
00134 int verbose;
00135 int hmiaiaflg = 0;
00136 int imagecnt = 0;
00137 int restartflg = 0;
00138 int abortflg = 0;
00139 int sigalrmflg = 0;
00140 int ignoresigalrmflg = 0;
00141 int quicklook;
00142
00143 int flatmiss[NUMRECLEV1];
00144 int orbmiss[NUMRECLEV1];
00145 int asdmiss[NUMRECLEV1];
00146 int mpdmiss[NUMRECLEV1];
00147 int noimage[NUMRECLEV1];
00148 int missflg[NUMRECLEV1];
00149
00150 char logname[128];
00151 char argdsin[128], argdsout[128], arglogfile[128], arginstru[80];
00152 char argbfsn[80], argefsn[80], argquick[80];
00153 char timetag[32];
00154 char tlmseriesname[128];
00155 char lev0seriesname[128];
00156 char *username;
00157 char *logfile;
00158 char *instru;
00159 char *dsin;
00160 char *dsout;
00161
00163 typedef struct {
00164 float rsun_lf;
00165 float x0_lf;
00166 float y0_lf;
00167 } LIMB_SOMETHING;
00168
00169
00170
00171 int get_nspikes() { return nspikes; }
00172 int *get_spikelocs() { return spikelocs; }
00173 int *get_oldvalues() { return oldvalues; }
00174 int *get_newvalues() { return newvalues; }
00175 void set_nspikes(int new_nspikes) { nspikes = new_nspikes; }
00176 void set_spikelocs(int *new_spikelocs) { spikelocs = new_spikelocs; }
00177 void set_oldvalues(int *new_oldvalues) { oldvalues = new_oldvalues; }
00178 void set_newvalues(int *new_newvalues) { newvalues = new_newvalues; }
00179
00180
00181 void do_quallev1(DRMS_Record_t *rs0, DRMS_Record_t *rs1, int inx, unsigned int fsn)
00182 {
00183 int quallev1 = 0;
00184 int rstatus;
00185 char *pchar;
00186
00187 quallev1 = missflg[inx];
00188 if(flatmiss[inx]) quallev1 = quallev1 | Q_NOFLAT;
00189 if(orbmiss[inx]) quallev1 = quallev1 | Q_NOORB;
00190 if(asdmiss[inx]) quallev1 = quallev1 | Q_NOASD;
00191 if(mpdmiss[inx]) quallev1 = quallev1 | Q_NOMPD;
00192 if(noimage[inx]) quallev1 = quallev1 | Q_MISSALL;
00193 if(ptinfo) {
00194 ptdata = ptinfo[inx];
00195 if(strcmp(ptdata.acs_mode, "SCIENCE")) {
00196 quallev1 = quallev1 | Q_NOACS_SCI;
00197 }
00198 if(strcmp(ptdata.acs_eclp, "NO")) {
00199 quallev1 = quallev1 | Q_ACS_ECLP;
00200 }
00201 if(strcmp(ptdata.acs_sunp, "YES")) {
00202 quallev1 = quallev1 | Q_ACS_SUNP;
00203 }
00204 if(strcmp(ptdata.acs_safe, "NO")) {
00205 quallev1 = quallev1 | Q_ACS_SAFE;
00206 }
00207 }
00208 pchar = drms_getkey_string(rs0, "IMG_TYPE", &rstatus);
00209 if(rstatus) {
00210 printk("ERROR: in drms_getkey_string(IMG_TYPE) fsn=%u\n", fsn);
00211 }
00212 else {
00213 if(strcmp(pchar, "LIGHT")) {
00214 quallev1 = quallev1 | Q_IMG_TYPE;
00215 }
00216 }
00217 if(hmiaiaflg) {
00218 pchar = drms_getkey_string(rs0, "AISTATE", &rstatus);
00219 }
00220 else {
00221 pchar = drms_getkey_string(rs0, "HWLTNSET", &rstatus);
00222 }
00223 if(rstatus) {
00224 printk("ERROR: in drms_getkey_string(HWLTNSET or AISTATE) fsn=%u\n", fsn);
00225 }
00226 else {
00227 if(!strcmp(pchar, "OPEN")) {
00228 quallev1 = quallev1 | Q_LOOP_OPEN;
00229 }
00230 }
00231
00233
00234 if(quicklook) {
00235 quallev1 = quallev1 | Q_NRT;
00236 }
00237 drms_setkey_int(rs1, "QUALITY", quallev1);
00238 drms_setkey_string(rs, "BLD_VERS", bld_vers);
00239 }
00240
00241 int nice_intro ()
00242 {
00243 int usage = cmdparams_get_int (&cmdparams, "h", NULL);
00244 if (usage)
00245 {
00246 printf ("Usage:\nbuild_lev1_fsn [-vhr] "
00247 "instru=<hmi|aia> dsin=<lev0> dsout=<lev1> bfsn=<fsn#>\n"
00248 " efsn=<fsn#> quicklook=<0|1> [logfile=<file>]\n"
00249 " -h: help - show this message then exit\n"
00250 " -v: verbose\n"
00251 " -r: restart. only used when we restart our selves periodically\n"
00252 "instru= instrument. must be 'hmi' or 'aia'\n"
00253 "dsin= data set name of lev0 input\n"
00254 " default hmi=hmi.lev0e aia=aia.lev0e\n"
00255 "dsout= data set name of lev1 output\n"
00256 " default hmi=su_production.hmi_lev1e aia=su_production.aia_lev1e\n"
00257 "bfsn= first lev0 fsn to process. 0=error, must be given by build_lev1_mgr_fsn\n"
00258 "efsn= last lev0 fsn to process. 0=error, must be given by build_lev1_mgr_fsn\n"
00259 "quicklook= 1 = quicklook mode, 0 = definitive mode\n"
00260 "logfile= optional log file name. If not given uses:\n"
00261 " /usr/local/logs/lev1/build_lev1_fsn.<time_stamp>.log\n");
00262 return(1);
00263 }
00264 verbose = cmdparams_get_int (&cmdparams, "v", NULL);
00265 restartflg = cmdparams_get_int (&cmdparams, "r", NULL);
00266 return (0);
00267 }
00268
00269 TIME SDO_to_DRMS_time(int sdo_s, int sdo_ss)
00270 {
00271 static int firstcall = 1;
00272 if (firstcall)
00273 {
00274 firstcall = 0;
00275 }
00276
00277 return(sdo_epoch + (TIME)sdo_s + (TIME)(sdo_ss)/65536.0);
00278 }
00279
00280
00281 char *do_datestr() {
00282 time_t tval;
00283 struct tm *t_ptr;
00284
00285 tval = time(NULL);
00286 t_ptr = localtime(&tval);
00287 sprintf(datestr, "%d.%02d.%02d_%02d:%02d:%02d",
00288 (t_ptr->tm_year+1900), (t_ptr->tm_mon+1),
00289 t_ptr->tm_mday, t_ptr->tm_hour, t_ptr->tm_min, t_ptr->tm_sec);
00290 return(datestr);
00291 }
00292
00293
00294 char *gettimetag()
00295 {
00296 struct timeval tvalr;
00297 struct tm *t_ptr;
00298
00299 gettimeofday(&tvalr, NULL);
00300 t_ptr = localtime((const time_t *)&tvalr);
00301 sprintf(timetag, "%04d.%02d.%02d.%02d%02d%02d",
00302 (t_ptr->tm_year+1900), (t_ptr->tm_mon+1), t_ptr->tm_mday, t_ptr->tm_hour, t_ptr->tm_min, t_ptr->tm_sec);
00303 return(timetag);
00304 }
00305
00306
00307 void BeginTimer(int n)
00308 {
00309 gettimeofday (&first[n], NULL);
00310 }
00311
00312 float EndTimer(int n)
00313 {
00314 gettimeofday (&second[n], NULL);
00315 if (first[n].tv_usec > second[n].tv_usec) {
00316 second[n].tv_usec += 1000000;
00317 second[n].tv_sec--;
00318 }
00319 return (float) (second[n].tv_sec-first[n].tv_sec) +
00320 (float) (second[n].tv_usec-first[n].tv_usec)/1000000.0;
00321 }
00322
00323
00324 int h1log(const char *fmt, ...)
00325 {
00326 va_list args;
00327 char string[32768];
00328
00329 va_start(args, fmt);
00330 vsprintf(string, fmt, args);
00331 if(h1logfp) {
00332 fprintf(h1logfp, string);
00333 fflush(h1logfp);
00334 }
00335 else {
00336 printf(string);
00337 fflush(stdout);
00338 }
00339 va_end(args);
00340 return(0);
00341 }
00342
00343 int send_mail(char *fmt, ...)
00344 {
00345 va_list args;
00346 char string[1024], cmd[1024];
00347
00348 va_start(args, fmt);
00349 vsprintf(string, fmt, args);
00350 sprintf(cmd, "echo \"%s\" | Mail -s \"build_lev1_fsn mail\" lev0_user", string);
00351 system(cmd);
00352 va_end(args);
00353 return(0);
00354 }
00355
00356
00357 void abortit(int stat)
00358 {
00359 printk("***Abort in progress ...\n");
00360 printk("**Exit build_lev1_fsn w/ status = %d\n", stat);
00361 if (h1logfp) fclose(h1logfp);
00362 exit(stat);
00363 }
00364
00366 int rdout_mode_correct()
00367 {
00368 return(0);
00369 }
00370
00372 int orbit_calc()
00373 {
00374 return(0);
00375 }
00376
00377
00378 #include "do_flat.c"
00379 #include "get_image_location.c"
00380 #include "limb_fit_function.c"
00381
00382
00383 int do_ingest(unsigned int bbfsn, unsigned int eefsn, const char *dpath)
00384 {
00385
00386 Image_Location *p_imageloc;
00387 Image_Location imageloc[NUMRECLEV1];
00388 TIME t_obs0;
00389 TIME tobs[NUMRECLEV1];
00390 float percentd;
00391 double rsun_lf, x0_lf, y0_lf;
00392 int rstatus, dstatus, ncnt, fcnt, i, j, qualint, nobs;
00393 int hshiexp, hcamid, skipflat;
00394 uint32_t missvals, totvals;
00395 long long recnum0, recnum1, recnumff;
00396 char recrange[128], lev0name[128], flatrec[128];
00397
00398
00399 sprintf(recrange, "%u-%u", bbfsn, eefsn);
00400 sprintf(open_dsname, "%s[%s]", dsin, recrange);
00401 printk("open_dsname = %s\n", open_dsname);
00402 printk("#levnum recnum fsn\n");
00403
00404 rset0 = drms_open_records(drms_env, open_dsname, &rstatus);
00405 if(!rset0 || (rset0->n == 0) || rstatus) {
00406 printk("Can't do drms_open_records(%s)\n", open_dsname);
00407 return(1);
00408 }
00409 ncnt = rset0->n;
00410 rptr = (DRMS_Record_t *)malloc(ncnt * sizeof(DRMS_Record_t));
00411 if(rptr == NULL) {
00412 printk("Can't malloc() for DRMS_Record_t sort\n");
00413 return(1);
00414 }
00415
00416 for(i=0; i < ncnt; i++) {
00417 memcpy(&rptr[i], rset0->records[i], sizeof(DRMS_Record_t));
00418 }
00419
00420 qsort(rptr, ncnt, sizeof(DRMS_Record_t), &compare_rptr);
00421
00422
00423
00424 for(i=0; i < ncnt; i++) {
00425 rs0 = &rptr[i];
00426 recnum0 = rs0->recnum;
00427
00428 fsnx = drms_getkey_int(rs0, "FSN", &rstatus);
00429 fsnarray[i] = fsnx;
00430 printk("*0 %u %u\n", recnum0, fsnx);
00431
00432 tobs[i] = drms_getkey_time(rs0, "t_obs", &rstatus);
00433 if(rstatus) {
00434 printk("Error on drms_getkey_time() fsn=%u. Use DRMS_MISSING_TIME\n",
00435 fsnx);
00436 tobs[i] = DRMS_MISSING_TIME;
00437 }
00438 }
00439 if(rstatus = get_pointing_info(drms_env, tobs, ncnt, &ptinfo)) {
00440 printk("**ERROR: get_pointing_info() status = %d fsn tobs ASD:\n",
00441 rstatus);
00442 for(j=0; j < ncnt; j++) {
00443 printk("%u %10.5f ", fsnarray[j], tobs[j]);
00444 ptdata = ptinfo[j];
00445 printk("%s\n", ptdata.asd_rec);
00446 }
00447 return(1);
00448 }
00449 if ((IOstatus = iorbit_getinfo(drms_env,
00450 orbseries,
00451 NULL,
00452 IORBIT_Alg_Quadratic,
00453 tobs,
00454 ncnt,
00455 kIORBIT_CacheAction_DontCache,
00456 &IOinfo)) != kLIBASTRO_Success)
00457 {
00458 if(IOstatus == kLIBASTRO_InsufficientData) {
00459 printk("***ERROR in iorbit_getinfo: kLIBASTRO_InsufficientData\n");
00460 }
00461 else {
00462 printk("***ERROR in iorbit_getinfo() status=%d\n", IOstatus);
00463 }
00464 return(1);
00465 }
00466 rset1 = drms_create_records(drms_env, ncnt, dsout, DRMS_PERMANENT,&dstatus);
00467 if(dstatus) {
00468 printk("**ERROR: Can't create records for %s\n", dsout);
00469 return(1);
00470 }
00471
00472 for(i=0; i < ncnt; i++) {
00473 rs0 = &rptr[i];
00474 imageloc[i].tobs = tobs[i];
00475 imageloc[i].camera = drms_getkey_int(rs0, "CAMERA", &rstatus);
00476 if(rstatus) {
00477 printk("ERROR: in drms_getkey_int(CAMERA) fsn=%u\n", fsnarray[i]);
00478 }
00479 imageloc[i].wavelength = drms_getkey_int(rs0, "WAVELNTH", &rstatus);
00480 if(rstatus) {
00481 printk("ERROR: in drms_getkey_int(WAVELNTH) fsn=%u\n", fsnarray[i]);
00482 }
00483 snprintf(imageloc[i].telescope, 10, "%s",
00484 drms_getkey_string(rs0, "TELESCOP", &rstatus));
00485 if(rstatus) {
00486 printk("ERROR: in drms_getkey_string(TELESCOP) fsn=%u\n", fsnarray[i]);
00487 }
00488 }
00489 p_imageloc = imageloc;
00490 rstatus = get_image_location(drms_env, ncnt, &p_imageloc);
00491 if(rstatus) {
00492 printk("ERROR: get_image_location() returns status=%d\n", rstatus);
00493 return(1);
00494 }
00495
00496
00497 for(i=0; i < ncnt; i++) {
00498 flatmiss[i] = 0;
00499 orbmiss[i] = 0;
00500 asdmiss[i] = 0;
00501 mpdmiss[i] = 0;
00502 noimage[i] = 0;
00503 missflg[i] = 0;
00504 rs0 = &rptr[i];
00505 recnum0 = rs0->recnum;
00506 fsnx = fsnarray[i];
00507 sprintf(lev0name, "%s[%u]", dsin, fsnx);
00508 if(drms_getkey_int(rs0, "QUALITY", 0) < 0) {
00509 printk("Bad QUALITY for %s, no lev1 made\n", lev0name);
00510 continue;
00511 }
00512
00513 segment = drms_segment_lookupnum(rs0, 0);
00514 Array0 = drms_segment_read(segment, DRMS_TYPE_SHORT, &rstatus);
00515 if(!Array0) {
00516 printk("Can't do drms_segment_read() %s status=%d\n",
00517 lev0name, rstatus);
00518 return(1);
00519
00520 }
00521 l0l1->adata0 = (short *)Array0->data;
00522 l0l1->rs0 = rs0;
00523 l0l1->recnum0 = recnum0;
00524 l0l1->fsn = fsnx;
00525 if(hmiaiaflg) {
00526 l0l1->dat1.adata1A = &data1A;
00527
00528 l0l1->himgcfid = 90;
00529 }
00530 else {
00531 l0l1->dat1.adata1 = &data1;
00532 l0l1->himgcfid = drms_getkey_int(rs0, "HIMGCFID", &rstatus);
00533 }
00534 if(rstatus) {
00535 printk("Can't do drms_getkey_int(HIMGCFID) for fsn %u\n", fsnx);
00537 l0l1->himgcfid = 104;
00538
00539
00540 }
00541
00542 sprintf(open_dsname, "%s[%u]", dsout, fsnx);
00543 rs = rset1->records[i];
00544 drms_record_directory(rs, rs1_path, 0);
00545 if(!*rs1_path) {
00546 printk("***ERROR: No path to segment for %s\n", open_dsname);
00547 return(1);
00548 }
00549
00550 dstatus = drms_setkey_int(rs, "FSN", fsnx);
00551 dstatus = drms_setkey_string(rs, "LEV0SERIES", lev0name);
00552 if(!(segment = drms_segment_lookup(rs, "image_lev1"))) {
00553 printk("No drms_segment_lookup(rs, image_lev1) for %s\n", open_dsname);
00554 return(1);
00555 }
00556 if(hmiaiaflg) {
00557 segArray = drms_array_create(DRMS_TYPE_INT,
00558 segment->info->naxis,
00559 segment->axis,
00560 &data1A,
00561 &dstatus);
00562 }
00563 else {
00564 segArray = drms_array_create(DRMS_TYPE_FLOAT,
00565 segment->info->naxis,
00566 segment->axis,
00567 &data1,
00568 &dstatus);
00569 }
00570
00571 rstatus = drms_copykeys(rs, rs0, 0, kDRMS_KeyClass_Explicit);
00572 if(rstatus != DRMS_SUCCESS) {
00573 printk("Error %d in drms_copykeys() for fsn %u\n", fsnx);
00574 return(1);
00575 }
00576 qualint = drms_getkey_int(rs0, "QUALITY", &rstatus);
00577 drms_setkey_int(rs, "QUALLEV0", qualint);
00578
00579 drms_setkey_time(rs, "T_OBS", tobs[i]);
00580 printk("t_obs for lev0 = %10.5f\n", tobs[i]);
00581 drms_setkey_double(rs, "DATE", CURRENT_SYSTEM_TIME);
00582 if(ptinfo) {
00583 ptdata = ptinfo[i];
00584 drms_setkey_float(rs, "SAT_Y0", ptdata.sat_y0);
00585 drms_setkey_float(rs, "SAT_Z0", ptdata.sat_z0);
00586 drms_setkey_float(rs, "SAT_ROT", ptdata.sat_rot);
00587 drms_setkey_string(rs, "ACS_MODE", ptdata.acs_mode);
00588 drms_setkey_string(rs, "ACS_ECLP", ptdata.acs_eclp);
00589 drms_setkey_string(rs, "ACS_SUNP", ptdata.acs_sunp);
00590 drms_setkey_string(rs, "ACS_SAFE", ptdata.acs_safe);
00591 drms_setkey_string(rs, "ASD_REC", ptdata.asd_rec);
00592 drms_setkey_string(rs, "ACS_CGT", ptdata.acs_cgt);
00593 }
00594 if(IOinfo) {
00595 IOdata = IOinfo[i];
00596 drms_setkey_double(rs, "HCIEC_X", IOdata.hciX);
00597 drms_setkey_double(rs, "HCIEC_Y", IOdata.hciY);
00598 drms_setkey_double(rs, "HCIEC_Z", IOdata.hciZ);
00599 drms_setkey_double(rs, "GCIEC_X", IOdata.gciX);
00600 drms_setkey_double(rs, "GCIEC_Y", IOdata.gciY);
00601 drms_setkey_double(rs, "GCIEC_Z", IOdata.gciZ);
00602 drms_setkey_float(rs, "DSUN_OBS", (float)IOdata.dsun_obs);
00603 drms_setkey_double(rs, "OBS_VR", IOdata.obs_vr);
00604 drms_setkey_double(rs, "OBS_VW", IOdata.obs_vw);
00605 drms_setkey_double(rs, "OBS_VN", IOdata.obs_vn);
00606 drms_setkey_double(rs, "RSUN_OBS", IOdata.rsun_obs);
00607 drms_setkey_float(rs, "CRLN_OBS", (float)IOdata.crln_obs);
00608 drms_setkey_float(rs, "CRLT_OBS", (float)IOdata.crlt_obs);
00609 drms_setkey_int(rs, "CAR_ROT", (int)IOdata.car_rot);
00610 drms_setkey_string(rs, "ORB_REC", IOdata.orb_rec);
00611 }
00612 drms_setkey_float(rs, "X0_MP", imageloc[i].x);
00613 drms_setkey_float(rs, "Y0_MP", imageloc[i].y);
00614 drms_setkey_float(rs, "INST_ROT", imageloc[i].instrot);
00615
00616 drms_setkey_float(rs, "IMSCL_MP", imageloc[i].imscale);
00617 drms_setkey_string(rs, "MPO_REC", imageloc[i].mpo_rec);
00618
00619 int camera = drms_getkey_int(rs0, "CAMERA", &rstatus);
00620 if(rstatus) {
00621 printk("Can't do drms_getkey_int() for fsn %u\n", fsnx);
00622 return(1);
00623 }
00624
00625 if(drms_ismissing_time(tobs[i])) {
00626 printk("DRMS_MISSING_TIME for fsn=%u. Continue...\n", fsnx);
00627 noimage[i] = 1;
00628 goto TEMPSKIP;
00629 }
00630 if(!hmiaiaflg) {
00631 if(quicklook) {
00632 sprintf(open_dsname, "%s[? t_start=(select max(t_start) from %s where t_start <= %10.5f and t_stop > %10.5f and CAMERA=%d) and CAMERA=%d ?]",
00633 dsffname, dsffname, tobs[i], tobs[i], camera, camera);
00634 }
00635 else {
00636 sprintf(open_dsname, "%s[? t_start <= %10.5f and t_stop > %10.5f and CAMERA=%d and flatfield_version=(select max(flatfield_version) from %s where t_start <= %10.5f and t_stop > %10.5f and CAMERA=%d) ?]",
00637 dsffname, tobs[i], tobs[i], camera, dsffname, tobs[i], tobs[i], camera);
00638 }
00639 }
00640 else {
00641 char *wavstr = drms_getkey_string(rs0, "WAVE_STR", &rstatus);
00642 if(rstatus) {
00643 printk("Can't do drms_getkey_string() for WAVE_STR\n");
00644 return(1);
00645 }
00646 if(quicklook) {
00647 sprintf(open_dsname, "%s[? t_start=(select max(t_start) from %s where t_start <= %10.5f and t_stop > %10.5f and WAVE_STR='%s') and WAVE_STR='%s' ?]",
00648 dsffname, dsffname, tobs[i], tobs[i], wavstr, wavstr);
00649 }
00650 else {
00651 sprintf(open_dsname, "%s[? t_start <= %10.5f and t_stop > %10.5f and WAVE_STR=%s and flatfield_version=(select max(flatfield_version) from %s where t_start <= %10.5f and t_stop > %10.5f and WAVE_STR='%s') ?]",
00652 dsffname, tobs[i], tobs[i], wavstr, dsffname, tobs[i], tobs[i], wavstr);
00653 }
00654 }
00655
00656 rsetff = drms_open_records(drms_env, open_dsname, &rstatus);
00657 if(!rsetff || (rsetff->n == 0) || rstatus) {
00658 printk("Can't do drms_open_records(%s)\n", open_dsname);
00659 flatmiss[i] = 1; noimage[i] = 1;
00660 goto TEMPSKIP;
00661
00662 }
00663 fcnt = rsetff->n;
00664 if(fcnt > 1) {
00665 printk("More than one FF found for %s?\n", open_dsname);
00666 printk("Use last one of %d found\n", fcnt);
00667
00668 }
00669
00670 rsff = rsetff->records[fcnt-1];
00671 recnumff = rsff->recnum;
00672 sprintf(flatrec, "%s[:#%lld]", dsffname, recnumff);
00673 if(dstatus = drms_setkey_string(rs, "FLAT_REC", flatrec )) {
00674 printk("**ERROR on drms_setkey_string() for %s\n", flatrec);
00675 }
00676 drms_record_directory(rsff, path, 1);
00677 if(!*path) {
00678 printk("***ERROR: No path to segment for %s\n", open_dsname);
00679
00680 return(1);
00681 }
00682
00683 segmentff = drms_segment_lookup(rsff, "flatfield");
00684 Arrayff = drms_segment_read(segmentff, DRMS_TYPE_FLOAT, &rstatus);
00685 if(!Arrayff) {
00686 printk("Can't do drms_segment_read() for Flat Field status=%d\n",
00687 rstatus);
00688 return(1);
00689 }
00690 l0l1->adataff = (float *)Arrayff->data;
00691
00692 darkseg = drms_segment_lookup(rsff, "DARK");
00693 ArrayDark = drms_segment_read(darkseg, DRMS_TYPE_FLOAT, &rstatus);
00694 if(!ArrayDark) {
00695 printk("Can't do drms_segment_read() for DARK. status=%d\n", rstatus);
00696 return(1);
00697 }
00698 l0l1->adatadark = (float *)ArrayDark->data;
00699
00700 badseg = drms_segment_lookup(rsff, "BAD_PIXEL");
00701 ArrayBad = drms_segment_read(badseg, DRMS_TYPE_INT, &rstatus);
00702 if(!ArrayBad) {
00703 printk("Can't do drms_DARKsegment_read() for BAD_PIXEL. status=%d\n",
00704 rstatus);
00705 return(1);
00706 }
00707 l0l1->adatabad = (int *)ArrayBad->data;
00709
00710
00711
00712
00713 if(!hmiaiaflg) {
00714 if(!(badoutpixseg = drms_segment_lookup(rs, "bad_pixel_list"))) {
00715 printk("No drms_segment_lookup(rs, bad_pixel_list) for lev1\n");
00716 return(1);
00717 }
00718 }
00719 else {
00720 if(!(badoutpixseg = drms_segment_lookup(rs, "bad_pixel"))) {
00721 printk("No drms_segment_lookup(rs, bad_pixel) for lev1\n");
00722 return(1);
00723 }
00724 }
00725 dstatus = drms_segment_write(badoutpixseg, ArrayBad, 0);
00726 if (dstatus) {
00727 printk("ERROR: drms_segment_write error=%d for lev1 bad_pixel_list\n");
00728
00729 return(1);
00730 }
00731 l0l1->rs1 = rs;
00732 l0l1->rsff = rsff;
00733 l0l1->recnum1 = rs->recnum;
00734 hshiexp = drms_getkey_int(rs, "HSHIEXP", &rstatus);
00735 hcamid = drms_getkey_int(rs, "HCAMID", &rstatus);
00736 skipflat = 0;
00737 if(hshiexp == 0) {
00738 if(hcamid == 0 || hcamid == 1) {
00739 skipflat = 1;
00740 }
00741 }
00742 if(!skipflat) {
00743 if(hmiaiaflg) {
00744 if(rstatus = do_flat_aia(l0l1)) {
00745 printk("***ERROR in do_flat() status=%d\n", rstatus);
00746 printf("***ERROR in do_flat() status=%d\n", rstatus);
00747 flatmiss[i] = 1; noimage[i] = 1;
00748
00749 goto FLATERR;
00750 }
00751 }
00752 else {
00753 if(rstatus = do_flat(l0l1)) {
00754 printk("***ERROR in do_flat() status=%d\n", rstatus);
00755 printf("***ERROR in do_flat() status=%d\n", rstatus);
00756 flatmiss[i] = 1; noimage[i] = 1;
00757
00758 goto FLATERR;
00759 }
00760 }
00761
00762
00763
00764
00765
00766 drms_setkey_float(rs, "OSCNMEAN", l0l1->oscnmean);
00767 drms_setkey_float(rs, "OSCNRMS", l0l1->oscnrms);
00768 drms_setkey_int(rs, "DATAMIN", l0l1->datamin);
00769 drms_setkey_int(rs, "DATAMAX", l0l1->datamax);
00770 drms_setkey_int(rs, "DATAMEDN", l0l1->datamedn);
00771 drms_setkey_float(rs, "DATAMEAN", l0l1->datamean);
00772 drms_setkey_float(rs, "DATARMS", l0l1->data_rms);
00773 drms_setkey_float(rs, "DATASKEW", l0l1->dataskew);
00774 drms_setkey_float(rs, "DATAKURT", l0l1->datakurt);
00775 drms_setkey_int(rs, "DATAVALS", l0l1->datavals);
00776 drms_setkey_int(rs, "MISSVALS", l0l1->missvals);
00777 missvals = (uint32_t)l0l1->missvals;
00778 totvals = (uint32_t)l0l1->datavals + missvals;
00779 drms_setkey_int(rs, "TOTVALS", (int)totvals);
00780 percentd = (float)((100.0 * (float)l0l1->datavals)/(float)totvals);
00781 drms_setkey_float(rs, "PERCENTD", percentd);
00782 if(missvals > 0) missflg[i] = missflg[i] | Q_1_MISS0;
00783 if(missvals > (uint32_t)(totvals * 0.01))
00784 missflg[i] = missflg[i] | Q_1_MISS1;
00785 if(missvals > (uint32_t)(totvals * 0.05))
00786 missflg[i] = missflg[i] | Q_1_MISS2;
00787 if(missvals > (uint32_t)(totvals * 0.25))
00788 missflg[i] = missflg[i] | Q_1_MISS3;
00789 if(l0l1->datavals == 0)
00790 missflg[i] = missflg[i] | Q_MISSALL;
00791 if(hmiaiaflg) {
00792
00793 if (spikeseg = drms_segment_lookup(rs,"spikes") ) {
00794 int *spikedata, status, axes[2], nbytes;
00795 nbytes = nspikes*sizeof(int);
00796
00797
00798
00799
00800
00801
00802
00803
00804
00805
00806 axes[0] = nspikes;
00807 axes[1] = 3;
00808 spikedata = (int *) malloc (3*nspikes*sizeof(int));
00809 ArraySpike = drms_array_create(DRMS_TYPE_INT, 2, axes,
00810 (void *) spikedata, &status);
00811 memcpy((void *)spikedata, (void *)spikelocs, nbytes);
00812 memcpy((void *)(spikedata+nspikes), (void *)oldvalues, nbytes);
00813 memcpy((void *)(spikedata+2*nspikes), (void *)newvalues, nbytes);
00814 status = drms_segment_write(spikeseg, ArraySpike, 0);
00815 drms_free_array(ArraySpike);
00816 } else { printf("spikes segment not found\n"); }
00817 }
00818 }
00819 FLATERR:
00820 drms_close_records(rsetff, DRMS_FREE_RECORD);
00821 free(ArrayDark->data);
00822 free(Arrayff->data);
00823 free(ArrayBad->data);
00824
00825 TEMPSKIP:
00826 if(!hmiaiaflg) {
00827 segArray->type = DRMS_TYPE_FLOAT;
00828 segArray->bscale = 1.0;
00829 segArray->bzero = 0.0;
00830 }
00831 dstatus = drms_segment_write(segment, segArray, 0);
00832 if (dstatus) {
00833 printk("ERROR: drms_segment_write error=%d for fsn=%u\n", dstatus,fsnx);
00834 noimage[i] = 1;
00835 }
00836 recnum1 = rs->recnum;
00837 printk("*1 %u %u\n", recnum1, fsnx);
00838 free(Array0->data);
00839
00840 x0_lf = DRMS_MISSING_DOUBLE;
00841 y0_lf = DRMS_MISSING_DOUBLE;
00842 rsun_lf = DRMS_MISSING_DOUBLE;
00843
00844
00845
00846 int skiplimb = 0;
00847 int hcftid = drms_getkey_int(rs, "HCFTID", &rstatus);
00848 if(hcftid == CAL_HCFTID) {
00849 printk("Cal mode image fsn=%u\n", fsnx);
00850 skiplimb = 1;
00851 }
00852 else {
00853 if(hshiexp == 0) {
00854 if(hcamid == 0 || hcamid == 1) {
00855 printk("Dark image fsn=%u\n", fsnx);
00856 skiplimb = 1;
00857 }
00858 }
00859 }
00860 if(!skiplimb && !hmiaiaflg) {
00861 dstatus = limb_fit(rs,l0l1->dat1.adata1,&rsun_lf,&x0_lf,&y0_lf,4096,4096,1, dpath);
00862 if(dstatus) {
00863 printk("ERROR: limb_fit() %d error for fsn=%u\n", dstatus, fsnx);
00864 noimage[i] = 1;
00865 }
00866 }
00867 drms_setkey_float(rs, "RSUN_LF", (float)rsun_lf);
00868 drms_setkey_float(rs, "X0_LF", (float)x0_lf);
00869 drms_setkey_float(rs, "Y0_LF", (float)y0_lf);
00870 if(drms_ismissing_double(rsun_lf)) {
00871 drms_setkey_float(rs, "CDELT1", imageloc[i].imscale);
00872 drms_setkey_float(rs, "CDELT2", imageloc[i].imscale);
00873 drms_setkey_float(rs, "R_SUN",(float)IOdata.rsun_obs/imageloc[i].imscale);
00874 }
00875 else {
00876 drms_setkey_float(rs, "CDELT1", (float)IOdata.rsun_obs/rsun_lf);
00877 drms_setkey_float(rs, "CDELT2", (float)IOdata.rsun_obs/rsun_lf);
00878 drms_setkey_float(rs, "R_SUN", (float)rsun_lf);
00879 }
00880 if(!drms_ismissing_double(x0_lf) && !drms_ismissing_double(y0_lf)) {
00881 drms_setkey_float(rs, "CRPIX1", (float)x0_lf + 1);
00882 drms_setkey_float(rs, "CRPIX2", (float)y0_lf + 1);
00883 }
00884 else {
00885
00886
00888 drms_setkey_float(rs, "CRPIX1", imageloc[i].x + 1);
00889 drms_setkey_float(rs, "CRPIX2", imageloc[i].y + 1);
00890 }
00891 drms_setkey_float(rs, "CROTA2", imageloc[i].instrot + ptdata.sat_rot);
00892
00893
00894 if(hmiaiaflg) {
00895 int wl = drms_getkey_int(rs, "WAVELNTH", &rstatus);
00896 drms_setkey_float(rs, "CDELT1", 0.609);
00897 drms_setkey_float(rs, "CDELT2", 0.609);
00898 switch (wl) {
00899 case 94:
00900 drms_setkey_float(rs, "CRPIX1", 2048.5);
00901 drms_setkey_float(rs, "CRPIX2", 2048.5);
00902 break;
00903 case 131:
00904 drms_setkey_float(rs, "CRPIX1", 2048.5);
00905 drms_setkey_float(rs, "CRPIX2", 2048.5);
00906 break;
00907 case 171:
00908 drms_setkey_float(rs, "CRPIX1", 2048.5);
00909 drms_setkey_float(rs, "CRPIX2", 2048.5);
00910 break;
00911 case 193:
00912 drms_setkey_float(rs, "CRPIX1", 2048.5);
00913 drms_setkey_float(rs, "CRPIX2", 2048.5);
00914 break;
00915 case 211:
00916 drms_setkey_float(rs, "CRPIX1", 2048.5);
00917 drms_setkey_float(rs, "CRPIX2", 2048.5);
00918 break;
00919 case 304:
00920 drms_setkey_float(rs, "CRPIX1", 2048.5);
00921 drms_setkey_float(rs, "CRPIX2", 2048.5);
00922 break;
00923 case 335:
00924 drms_setkey_float(rs, "CRPIX1", 2048.5);
00925 drms_setkey_float(rs, "CRPIX2", 2048.5);
00926 break;
00927 case 1600:
00928 drms_setkey_float(rs, "CRPIX1", 2048.5);
00929 drms_setkey_float(rs, "CRPIX2", 2048.5);
00930 break;
00931 case 1700:
00932 drms_setkey_float(rs, "CRPIX1", 2048.5);
00933 drms_setkey_float(rs, "CRPIX2", 2048.5);
00934 break;
00935 case 4500:
00936 drms_setkey_float(rs, "CRPIX1", 2048.5);
00937 drms_setkey_float(rs, "CRPIX2", 2048.5);
00938 break;
00939 }
00940 drms_setkey_float(rs, "CRVAL1", 0.0);
00941 drms_setkey_float(rs, "CRVAL2", 0.0);
00942 }
00943
00944 do_quallev1(rs0, rs, i, fsnx);
00945
00946 }
00947
00948 drms_close_records(rset0, DRMS_FREE_RECORD);
00949 drms_close_records(rset1, DRMS_INSERT_RECORD);
00950 return(0);
00951 }
00952
00953 int compare_rptr(const void *a, const void *b)
00954 {
00955 TIME t1, t2;
00956 int rstatus;
00957 DRMS_Record_t *x=(DRMS_Record_t *)a, *y=(DRMS_Record_t *)b;
00958
00959 t1 = drms_getkey_time(x, "t_obs", &rstatus);
00960 if(rstatus) t1 = DRMS_MISSING_TIME;
00961 t2 = drms_getkey_time(y, "t_obs", &rstatus);
00962 if(rstatus) t2 = DRMS_MISSING_TIME;
00963 if(t1 < t2) return(-1);
00964 if(t1 > t2) return(1);
00965 return(0);
00966 }
00967
00968
00969 void setup()
00970 {
00971 FILE *fin;
00972 char string[128], cwdbuf[128], idstr[256], lfile[128];
00973 int tpid;
00974
00975 sdo_epoch = sscan_time("1958.01.01_00:00:00_TAI");
00976 do_datestr();
00977 printk_set(h1log, h1log);
00978 printk("%s\n", datestr);
00979 getcwd(cwdbuf, 126);
00980 sprintf(idstr, "Cwd: %s\nCall: ", cwdbuf);
00981 sprintf(string, "build_lev1_fsn started as pid=%d ppid=%d user=%s\n",
00982 getpid(), getppid(), username);
00983 strcat(idstr, string);
00984 printk("%s", idstr);
00985 printf("%s", idstr);
00986 if(restartflg) printk("-r ");
00987 sprintf(arginstru, "instru=%s", instru);
00988 sprintf(argdsin, "dsin=%s", dsin);
00989 sprintf(argdsout, "dsout=%s", dsout);
00990 sprintf(argbfsn, "bfsn=%u", bfsn);
00991 sprintf(argefsn, "efsn=%u", efsn);
00992 sprintf(argquick, "quicklook=%d", quicklook);
00993 sprintf(arglogfile, "logfile=%s", logname);
00994 printk("%s %s %s %s %s %s %s\n",
00995 arginstru, argdsin, argdsout, argbfsn, argefsn, argquick, arglogfile);
00996 printf("%s %s %s %s %s %s %s\n",
00997 arginstru, argdsin, argdsout, argbfsn, argefsn, argquick, arglogfile);
00998 if(!restartflg) {
00999
01000
01001 }
01002 sprintf(bld_vers, "%s", jsoc_version);
01003 sprintf(idstr, "ps -ef | grep %s", LEV1VIEWERNAME);
01004 fin = popen(idstr, "r");
01005 while(fgets(string, sizeof string, fin)) {
01006 if(!(strstr(string, "perl"))) continue;
01007 sscanf(string, "%s %d", idstr, &tpid);
01008 sprintf(lfile, "%s/build_lev1_restart_fsn_%d.touch", LEV1LOG_BASEDIR, tpid);
01009 sprintf(idstr, "/bin/touch %s", lfile);
01010 printk("%s\n", idstr);
01011 system(idstr);
01012 }
01013 umask(002);
01014
01015
01016
01017
01018
01019
01020 }
01021
01022
01023 int DoIt(void)
01024 {
01025 long long numofrecs, frec, lrec;
01026 int numrec, numofchunks, i;
01027 char line[80];
01028 const char *dpath = NULL;
01029
01030 if (nice_intro())
01031 return (0);
01032 if(!(username = (char *)getenv("USER"))) username = "nouser";
01033 instru = cmdparams_get_str(&cmdparams, "instru", NULL);
01034 if(strcmp(instru, "hmi") && strcmp(instru, "aia")) {
01035 printf("Error: instru= must be given as 'hmi' or 'aia'\n");
01036 return(0);
01037 }
01038 if(!strcmp(instru, "aia")) hmiaiaflg = 1;
01039 dsin = cmdparams_get_str(&cmdparams, "dsin", NULL);
01040 dsout = cmdparams_get_str(&cmdparams, "dsout", NULL);
01041 bfsn = cmdparams_get_int(&cmdparams, "bfsn", NULL);
01042 efsn = cmdparams_get_int(&cmdparams, "efsn", NULL);
01043 quicklook = cmdparams_get_int(&cmdparams, "quicklook", NULL);
01044
01045 dpath = cmdparams_get_str(&cmdparams, kDpath, NULL);
01046
01047 if(bfsn == 0 || efsn == 0) {
01048 fprintf(stderr, "bfsn and/or efsn must be given. 0 not allowed\n");
01049 return(0);
01050 }
01051 if(bfsn > efsn) {
01052 fprintf(stderr, "bfsn must be <= efsn\n");
01053 return(0);
01054 }
01055 logfile = cmdparams_get_str(&cmdparams, "logfile", NULL);
01056 if (strcmp(dsin, NOTSPECIFIED) == 0) {
01057 if(hmiaiaflg == 0) dsin = LEV0SERIESNAMEHMI;
01058 else dsin = LEV0SERIESNAMEAIA;
01059 }
01060 if (strcmp(dsout, NOTSPECIFIED) == 0) {
01061 if(hmiaiaflg == 0) dsout = LEV1SERIESNAMEHMI;
01062 else dsout = LEV1SERIESNAMEAIA;
01063 }
01064 if(hmiaiaflg) {
01065 sprintf(dsffname, "%s", DSFFNAMEAIA);
01066 if(strstr(dsin, "hmi") || strstr(dsout, "hmi")) {
01067 printf("Warning: You said instru=aia but have 'hmi' in ds name?\n");
01068 printf("Do you want to abort this [y/n]? ");
01069 if(gets(line) == NULL) { return(0); }
01070 if(strcmp(line, "n")) { return(0); }
01071 }
01072 }
01073 else {
01074 sprintf(dsffname, "%s", DSFFNAMEHMI);
01075 if(strstr(dsin, "aia") || strstr(dsout, "aia")) {
01076 printf("Warning: You said instru=hmi but have 'aia' in ds name?\n");
01077 printf("Do you want to abort this [y/n]? ");
01078 if(gets(line) == NULL) { return(0); }
01079 if(strcmp(line, "n")) { return(0); }
01080 }
01081 }
01082 if (strcmp(logfile, NOTSPECIFIED) == 0) {
01083 sprintf(logname, H1LOGFILE, gettimetag());
01084 }
01085 else {
01086 sprintf(logname, "%s", logfile);
01087 }
01088 if(restartflg || LOGTEST) {
01089 if((h1logfp=fopen(logname, "a")) == NULL)
01090 fprintf(stderr, "**Can't open for append the log file %s\n", logname);
01091 }
01092 else {
01093 if((h1logfp=fopen(logname, "w")) == NULL)
01094 fprintf(stderr, "**Can't open the log file %s\n", logname);
01095 }
01096 setup();
01097 numofrecs = (efsn - bfsn) + 1;
01098 numrec = NUMRECLEV1;
01099 numofchunks = numofrecs/numrec;
01100 if((numofrecs % numrec) != 0) numofchunks++;
01101 lrec = bfsn-1;
01102 for(i = 0; i < numofchunks; i++) {
01103 frec = lrec+1; lrec = (frec + numrec)-1;
01104 if(lrec > efsn) lrec=efsn;
01105 if(do_ingest(frec, lrec, dpath)) {
01106 printf("build_lev1_fsn abort\nSee log: %s\n", logname);
01107 send_mail("build_lev1_fsn abort\nSee log: %s\n", logname);
01108 return(0);
01109 }
01110 }
01111 printf("build_lev1_fsn done last fsn=%u\n", fsnx);
01112 return(0);
01113 }
01114