JSOC_Documentation: proj/dsdsmigr/apps/ingest_dsds_to

00001 /* MODULE NAME: ingest_dsds_to_drms.c */
00002 /*
00003  * CREATED BY : P. SCHERRER
00004  * MODIFIED BY: M. BOBRA 
00005  * MODIFIED TO: -- include updated scale corrections calculation using mdi.scale_corrections drms series as an input
00006  *              -- populate ephemeris keywords
00007  *              -- populate calibration and roll table keywords
00008  *              -- check for input data availability for a general file  
00009  *   
00010  *  USE:
00011  *  INPUT PARAMETERS:  IN  = <SUMS directory>
00012  *                     MAP = <mapfile>
00013  *                     SCALE_CORRECTIONS = <drms series with scale corrections file>
00014  *              
00015  *  OUTPUT PARAMETERS: OUT = <drms series>
00016  * 
00017  *  FLAGS: M =
00018  *         v =   
00019  *         z =
00020  *         p = 
00021  *
00022  *  EXAMPLE: 
00023  *  ingest_dsds_to_drms in=/SUM17/D231103541/D18885727/S00000 out=su_mbobra.test_vwv map=fd_test.map SCALE_CORRECTIONS=mdi.scale_corrections
00024  */
00025 
00026 #include "jsoc_main.h"
00027 #include "drms_types.h"
00028 #include "dsdsmigr.h"
00029 #include "printk.h"
00030 #include <time.h>
00031 #include <math.h>
00032 #include "limbfuncs.c"
00033 
00034 char *module_name = "opendsrecs";
00035 
00036 #define kNOT_SPEC   "Not Specified"
00037 
00038 #define DIE(msg)    return(fprintf(stderr,"%s",msg),1)
00039 #define DIE_status(msg) return(fprintf(stderr,"%s, status of death=%d\n",msg,status),1)
00040 
00041 #define FDSCALE_DEFAULT (1.97784)
00042 #define CANT_OPEN_FILE  (730)
00043 #define BEYOND_TABLE_RANGE      (1)
00044 #define EPHEMERIS_GAP   (2)
00045 #define TRUNCATED_FIT   (3)
00046 
00047 ModuleArgs_t module_args[] =
00048 {
00049      {ARG_STRING, "in",  kNOT_SPEC, "Input data series."},
00050      {ARG_STRING, "out", kNOT_SPEC, "Output data series."},
00051      {ARG_STRING, "map", kNOT_SPEC, "Name conversion list."},
00052      {ARG_STRING, "SCALE_CORRECTIONS", kNOT_SPEC, "scale correction recordset (1 record)"},
00053      {ARG_FLAG, "M", "0", "SkipMissingFiles - no records if DATAFILE is blank."},
00054      {ARG_FLAG, "v", "0", "verbose - more diagnostics"},
00055      {ARG_FLAG, "z", "0", "Use DPC_SMPL of '60 second' only"},
00056      {ARG_FLAG, "p", "0", "Indicate polarization state in POLSTATE keyword"},
00057      {ARG_FLAG, "s", "0", "skip check for existence of input datafile, use for series with no segments"},
00058      {ARG_FLAG, "l", "0", "use this with the limb figure data ONLY"},
00059      {ARG_END}
00060 };
00061 
00062 double soho_obs_dist (TIME t) {
00063   return 1.0;
00064 }
00065 
00066 static int fgetline (char *line, FILE *tbl) {
00067 /*
00068  *  A local utility for reading correction (or other) tables with the
00069  *    convention of one \n line delimited line per information record,
00070  *    comments with # as first character, and continuations flagged by
00071  *    \ at the end of the line
00072  */
00073   int c, b, cloc, len = 0;
00074 
00075   while ((c = fgetc (tbl)) != EOF) {
00076     if (c == '\n') {
00077       line[len++] = 0;
00078       if (line[0] == '#') line[len = 0] = 0;
00079       else if (b != '\\') return len;
00080       else {
00081     line[len=cloc] = ' ';
00082     line[++len] = 0;
00083       }
00084     } else {
00085       if (c != ' ' && c != '\t') {
00086     b = c;
00087     cloc = len;
00088       }
00089       line[len++] = c;
00090     }
00091   }
00092   return 0;
00093 }
00094 
00095 int MDI_plate_scale_adjustment (char *filepath, double *fdscl, TIME t, int *focus, int *qual,
00096     double obs_dist) {
00097   static struct plist {
00098     int qual;
00099     int focus;
00100     TIME t0;
00101     TIME t1;
00102     TIME tmid;
00103     double a0;
00104     double a1;
00105     double a2;
00106     double a3;
00107     int status;
00108   } *fitp;
00109   FILE *tbl;
00110   TIME t0, t1, t2, tmid;
00111   double a0, a1, a2, a3, rsun_adj;
00112   static int allocd = 32, file_read = 0;
00113   int status = 0;
00114   static int nct;
00115   int c, fields, m, n;
00116   char line[4096], tstr0[128], tstr1[128], tstr2[128];
00117 
00118   if (!file_read) {
00119      //    if (!(tbl = fopen ("/home/soi/CM/tables/calib/geom/scale_corrections", "r"))) {
00120      if (!(tbl = fopen (filepath, "r"))) {
00121       *fdscl = FDSCALE_DEFAULT;
00122       *qual = 0;
00123       *focus = 0;
00124       return CANT_OPEN_FILE;
00125     }
00126     fitp = (struct plist *)malloc (allocd * sizeof (struct plist));
00127     nct = 0;
00128 /*
00129  * Reads a file in the format of multiple lines of the form
00130  *   Qual Focus T1  T2  T0  a0  a1  a2 a3
00131  * e.g.
00132  *   2 1 1995.12.21 1997.11.03 1997.02.09_14:31 1.005 -0.005  5.7e-12 -1.4e-19
00133  * and returns a set of n parameter sets
00134  */
00135     while (fgetline (line, tbl)) {
00136       fields = sscanf (line, "%d %d %s %s %s %lg %lg %lg %lg",
00137           qual, focus, tstr1, tstr2, tstr0, &a0, &a1, &a2, &a3);
00138       if (fields < 4) continue;
00139                 /*  ignore lines without start & stop times  */
00140       fitp[nct].qual = *qual;
00141       fitp[nct].focus = *focus;
00142       fitp[nct].status = (fields < 9) ? TRUNCATED_FIT : 0;
00143       fitp[nct].t0 = sscan_time (tstr1);
00144       fitp[nct].t1 = sscan_time (tstr2);
00145       fitp[nct].tmid = (fields > 4) ? sscan_time (tstr0) : 0.0;
00146       fitp[nct].a0 = (fields > 5) ? a0 : 1.0;
00147       fitp[nct].a1 = (fields > 6) ? a1 : 0.0;
00148       fitp[nct].a2 = (fields > 7) ? a2 : 0.0;
00149       fitp[nct].a3 = (fields > 8) ? a3 : 0.0;
00150       if (++nct >= allocd) {
00151         allocd += 32;
00152     fitp = (struct plist *)realloc (fitp, allocd * sizeof (struct plist));
00153       }
00154     }
00155     fclose (tbl);
00156     file_read = 1;
00157   }
00158 
00159   if (obs_dist == 0.0) obs_dist = soho_obs_dist (t);
00160   if (obs_dist == 0.0) status = EPHEMERIS_GAP;
00161   n = 0;
00162   while (n < nct) {
00163     if (t >= fitp[n].t0 && t <= fitp[n].t1) break;
00164     n++;
00165   }
00166   if (n >= nct) {
00167     *fdscl = FDSCALE_DEFAULT;
00168     *qual = 0;
00169     *focus = 0;
00170     return BEYOND_TABLE_RANGE;
00171   }
00172   t -= fitp[n].tmid;
00173   rsun_adj = fitp[n].a0;
00174   rsun_adj += obs_dist * (fitp[n].a1 + t * (fitp[n].a2 + t * fitp[n].a3));
00175 /*
00176   *fdscl *= FDSCALE_DEFAULT;
00177     WRONG:  FDSCALE_DEFAULT is based on nominal rsun0 and corrected rsun
00178     is rsun0 * rsun_adj
00179 */
00180   *fdscl = FDSCALE_DEFAULT / rsun_adj;
00181   *qual = fitp[n].qual;
00182   *focus = fitp[n].focus;
00183   if (!status) {
00184     status = fitp[n].status;
00185   }
00186   return status;
00187 }
00188 
00189 // #include <sys/time.h>
00190 TIME time_now()
00191   {
00192   TIME now;
00193   TIME UNIX_epoch = -220924792.000; /* 1970.01.01_00:00:00_UTC */
00194   struct timeval tp;
00195   gettimeofday(&tp, NULL);
00196   now = (double)tp.tv_sec + (double)tp.tv_usec/1.0e6;
00197   return(now +  UNIX_epoch);
00198   }
00199 
00200 /* Name check code.  This code uses an external table to drive the mapping of
00201 dsds keywords into drms keywords.  It should contain a row for each keyword in
00202 the target drms series.  Each row should contain 3 fields, drms_name, dsds_name, action.
00203 The action will be a string which matches the action table.  The action table
00204 converts the string to an integer.
00205 Sample file lines like:
00206   cadence CADENCE copy
00207   date DATE copy
00208 Lines begging with "#" are comments and will be ignored.
00209 Blank lines are not allowed.
00210 */
00211 
00212 typedef struct NameListLookup_struct
00213   {
00214   char *drms_name;
00215   char *dsds_name;
00216   int action;
00217   struct NameListLookup_struct *next;
00218   } NameListLookup_t;
00219 
00220 typedef enum {
00221   ACT_NOP, ACT_COPY, ACT_ANGLE, ACT_CENTER, ACT_TIME, ACT_AU, ACT_RESCALE, ACT_NOT_FOUND
00222   } Action_t;
00223 
00224 Action_t actions[] = {
00225   ACT_NOP, ACT_COPY, ACT_ANGLE, ACT_CENTER, ACT_TIME, ACT_AU, ACT_RESCALE, ACT_NOT_FOUND
00226   };
00227 
00228 char *action_names[] = {
00229   "nop",   "copy",   "pangle",   "center",   "time",  "au",   "scale", "done"
00230   };
00231 
00232 int keyNameCheck(char *name, char **fromname)
00233   {
00234   static int first_call = 1;
00235   static NameListLookup_t *actionlist;
00236   NameListLookup_t *this;
00237   if (first_call)
00238     {
00239     char *actionlistname;
00240     FILE *flist;
00241     char drms_name[100], dsds_name[100], action_name[100];
00242     char line[1024];
00243     // int action;
00244     NameListLookup_t *last;
00245 
00246     first_call = 0;
00247     actionlistname = strdup(cmdparams_get_str(&cmdparams, "map", NULL));
00248     if (strcmp(actionlistname, kNOT_SPEC) == 0)
00249       {
00250       fprintf(stderr, "Name mapping list must be specified\n");
00251       exit(1);
00252       }
00253     flist = fopen(actionlistname, "r");
00254     if (!flist)
00255       {
00256       fprintf(stderr, "Name mapping file not found\n");
00257       exit(1);
00258       }
00259     last = actionlist = (NameListLookup_t *)malloc(sizeof(struct NameListLookup_struct));
00260     last->next = NULL;
00261     while (fgets(line, 1024, flist))
00262       {
00263       if (line[0] == '#')
00264         continue;                 
00265       else if (sscanf(line,"%s %s %s\n", drms_name, dsds_name, action_name)==3)
00266         {
00267         int iname;
00268     this = last;
00269         last = this->next = (NameListLookup_t *)malloc(sizeof(struct NameListLookup_struct));
00270         last->next = NULL;
00271         this->drms_name = strdup(drms_name);
00272         this->dsds_name = strdup(dsds_name);
00273         this->action = ACT_NOT_FOUND;
00274         for (iname=0; actions[iname] != ACT_NOT_FOUND; iname++)
00275           if (strcmp(action_name, action_names[iname]) == 0)
00276             {
00277             this->action = actions[iname];
00278             break;
00279             }
00280         }
00281       else fprintf(stderr, "Name map read error on line containing %s, ignoring this line\n",line);
00282       }
00283     }
00284   /* lookup name and return action */
00285   for (this=actionlist; this->next; this = this->next)
00286     {
00287     if (strcmp(this->drms_name, name) == 0)
00288       {
00289       *fromname = this->dsds_name;
00290       return(this->action);
00291       }
00292     }
00293   *fromname = name;
00294   return(ACT_NOT_FOUND);
00295   }
00296 
00297 int DoIt(void) 
00298    {
00299    int errbufstat=setvbuf(stderr, NULL, _IONBF, BUFSIZ);
00300    int outbufstat=setvbuf(stdout, NULL, _IONBF, BUFSIZ);
00301    int status = DRMS_SUCCESS;
00302    int SkipMissingFiles;
00303    int verbose;
00304    int qualnodata=0x80000000;
00305    int nRecs, iRec;
00306    int qualstat, dpcflag, polflag, skipflag, limbflag;
00307    double val;
00308    char *val1 = NULL;
00309 # define AU_m (149597870691.0)
00310    char *inRecQuery, *outRecQuery;
00311    DRMS_RecordSet_t *inRecSet, *outRecSet; 
00312    unsigned int quality;
00313    inRecQuery = strdup(cmdparams_get_str(&cmdparams, "in", NULL));
00314    outRecQuery = strdup(cmdparams_get_str(&cmdparams, "out", NULL));
00315    SkipMissingFiles = cmdparams_get_int(&cmdparams, "M", NULL) != 0;
00316    verbose = cmdparams_get_int(&cmdparams, "v", NULL) != 0;
00317    char scfilepath[DRMS_MAXPATHLEN];
00318    char infilepath[DRMS_MAXPATHLEN];
00319        
00320        DRMS_RecordSet_t *screcset = NULL;
00321        
00322    dpcflag = params_isflagset(&cmdparams, "z");
00323    polflag = params_isflagset(&cmdparams, "p");
00324 //   printf("dpcflag=%d\n",dpcflag);
00325    skipflag = params_isflagset(&cmdparams, "s");
00326    limbflag = params_isflagset(&cmdparams, "l");
00327    if (strcmp(inRecQuery, kNOT_SPEC) == 0 || strcmp(outRecQuery, kNOT_SPEC) == 0)
00328       DIE("Both the in and out dataseries must be specified.\n");
00329 
00330    inRecSet = drms_open_records(drms_env, inRecQuery, &status);
00331    if (!inRecSet)
00332       DIE_status("Input dataseries not found\n");
00333    if ((nRecs = inRecSet->n) == 0)
00334       DIE("No input records found\n");
00335    if (verbose) printf("%d input records found\n", nRecs);
00336    char *screcquery = (char *)cmdparams_get_str(&cmdparams, "SCALE_CORRECTIONS", NULL);
00337    if (strcmp(screcquery, kNOT_SPEC) != 0)
00338    {
00339       screcset = drms_open_records(drms_env, screcquery, &status);
00340       if (status != DRMS_SUCCESS || screcset == NULL)
00341       {
00342          fprintf(stderr, "ERROR: problem reading scale corrections recordset: query = %s, status = %d\n", screcquery, status);
00343          return 1;
00344       }
00345 
00346       if (screcset->n != 1)
00347       {
00348          fprintf(stderr, "ERROR: scale corrections recordset contains more than one record: query = %s, nrecs = %d\n", screcquery, screcset->n);
00349          return 1;
00350       }
00351 
00352       DRMS_Segment_t *scseg = drms_segment_lookupnum(screcset->records[0], 0);
00353       drms_segment_filename(scseg,scfilepath);
00354       printf("scale corrections filepath = %s\n", scfilepath);
00355    }
00356 
00357    for (iRec=0; iRec<nRecs; iRec++)
00358       {
00359       char *DataFile;
00360       char timebuf[1024];
00361       float UNIX_epoch = -220924792.000; /* 1970.01.01_00:00:00_UTC */
00362       float now,cropmin,cropmax;
00363       float result;
00364       //double result;
00365       int Record_OK = 1;
00366       DRMS_Record_t *inRec, *outRec;
00367       DRMS_Keyword_t *outKey;
00368       DRMS_Segment_t *inSeg, *outSeg;
00369       HIterator_t *outKey_last = NULL;
00370 //      DRMS_Link_t *outLink;
00371       /* create output series rec prototype */
00372       inRec = inRecSet->records[iRec];
00373       outRecSet = drms_create_records(drms_env, 1, outRecQuery, DRMS_PERMANENT, &status);
00374       if (!outRecSet || outRecSet->n != 1)
00375          DIE_status("Output dataseries not found or can't create records\n");
00376 
00377       outRec = outRecSet->records[0];
00378 
00379       val = drms_getkey_time(inRec, "T_OBS",&status);
00380       val1= drms_getkey_string(inRec,"DPC_SMPL",&status);
00381   
00382     
00383 /* Check the polarisation status of the data. If the data has the string "_Vm_", then it is circularly polarized*/ 
00384 
00385 
00386       if (polflag && time_is_invalid(val) == 0 )
00387       {
00388          char *val2     = drms_getkey_string(inRec,"DATAFILE",&status); 
00389          char *val2test = strstr(val2,"_Vm_"); 
00390          if (val2test == NULL) drms_setkey_string(outRec,"POLSTATE","LP"); 
00391          else drms_setkey_string(outRec,"POLSTATE","CP");
00392           
00393           if (val2)
00394           {
00395               free(val2);
00396               val2 = NULL;
00397           }
00398       }
00399 
00400   // set cvs commit version into keyword HEADER
00401       char *cvsinfo = strdup("$Header: /home/cvsuser/cvsroot/JSOC/proj/dsdsmigr/apps/ingest_dsds_to_drms.c,v 1.18 2015/01/15 23:00:19 mbobra Exp $");
00402       status = drms_setkey_string(outRec, "HEADER", cvsinfo);
00403       if (cvsinfo)
00404       {
00405           free(cvsinfo);
00406           cvsinfo = NULL;
00407       }
00408 
00409       if (skipflag)
00410          goto skip;
00411 
00412       /* assume only one segment */
00413       DataFile = drms_getkey_string(inRec,"DATAFILE",&status);
00414       if (status && verbose)fprintf(stderr,"*** Segment Read DATAFILE status=%d\n",status);
00415       char filepath[DRMS_MAXPATHLEN];
00416       inSeg = drms_segment_lookupnum(inRec, 0);
00417       if (inSeg)
00418          drms_segment_filename(inSeg, filepath);
00419       else 
00420         filepath[0] = '\0';
00421       if (verbose) printf("incoming filepath = %s\n", filepath);
00422 
00423       if (dpcflag)
00424       {
00425          if (*DataFile && access(filepath, R_OK | F_OK)  == 0 && time_is_invalid(val) == 0 && strcmp(val1,"60 second") == 0)
00426      {
00427             outSeg = drms_segment_lookupnum(outRec, 0);
00428             if (inSeg && outSeg)
00429             {
00430                 if (limbflag)
00431                 {
00432                    status = embed_limbpixels(filepath,outRec,&cropmin,&cropmax); 
00433                    drms_setkey_double(outRec, "CROPMIN", cropmin);
00434                    drms_setkey_double(outRec, "CROPMAX", cropmax);
00435                    if (status != 0)                                          
00436                    DIE("Problem in function embed_limbpixels. \n");              
00437                 }
00438                 else
00439                 {
00440                    DRMS_Array_t *data;
00441                 /* read the data ad doubles so allow rescaling on output */
00442                    data = drms_segment_read(inSeg, DRMS_TYPE_DOUBLE, &status);
00443                    if (!data)
00444                    {
00445                       fprintf(stderr, "Bad data record %lld, status=%d\n",inRec->recnum, status);
00446                       DIE_status("giveup\n");
00447                    }
00448                 /* use the zero and offset values in the JSD for the new record segment */
00449                    data->bscale = outSeg->bscale;
00450                    data->bzero = outSeg->bzero;
00451                    drms_segment_write(outSeg, data, 0);
00452                    drms_free_array(data);
00453                 }
00454               Record_OK = 1;    
00455           quality = drms_getkey_int(inRec, "QUALITY", &qualstat);
00456               quality = quality & (~qualnodata);
00457               drms_setkey_int(outRec,"QUALITY",quality);
00458         }
00459             else
00460                DIE("Bad data segment lookup, in or out\n");
00461          }
00462          else
00463          { /* record is missing, copy t_rec and soho ephemeris keywords anyway*/
00464             drms_copykey(outRec, inRec, "T_REC");
00465             drms_copykey(outRec, inRec, "OBS_VW");
00466             drms_copykey(outRec, inRec, "OBS_VR");
00467             drms_copykey(outRec, inRec, "OBS_VN");
00468             sprint_time(timebuf, (double)time(NULL) + UNIX_epoch, "ISO", 0);
00469             drms_setkey_string(outRec, "DATE", timebuf);
00470             val = drms_getkey_double(inRec, "OBS_DIST", &status);
00471             drms_setkey_double(outRec, "DSUN_OBS", val*AU_m);
00472             val = drms_getkey_double(inRec, "OBS_B0", &status);
00473             drms_setkey_double(outRec, "CRLT_OBS", val);    
00474             val = drms_getkey_double(inRec, "OBS_CR", &status);
00475             drms_setkey_double(outRec, "CAR_ROT", val); 
00476             val = drms_getkey_double(inRec, "OBS_R0", &status);
00477             drms_setkey_double(outRec, "RSUN_OBS", val);
00478             val = drms_getkey_double(inRec, "OBS_L0", &status);
00479             drms_setkey_double(outRec, "CRLN_OBS", val); 
00480             drms_setkey_int(outRec, "QUALITY", 0X80000000);
00481             val = drms_getkey_int(inRec, "DPC", &status);
00482             drms_setkey_int(outRec, "DPC", val);
00483 
00484             if (drms_keyword_lookup(outRec, "DATAVALS", 0))
00485            drms_setkey_int(outRec, "DATAVALS", 0);
00486             if (SkipMissingFiles)
00487             {
00488                Record_OK = 0;
00489                if (verbose)
00490                {
00491                    char *strtodel = drms_getkey_string(outRec,"T_REC",NULL);
00492                    if (strtodel)
00493                    {
00494                        fprintf(stderr,"DSDS Record %d has no datafile, T_REC=%s, set missing.\n", iRec, strtodel);
00495                        free(strtodel);
00496                        strtodel = NULL;
00497                    }
00498                }
00499             }
00500             else
00501                Record_OK = 1;
00502             drms_close_records(outRecSet,(Record_OK ? DRMS_INSERT_RECORD : DRMS_FREE_RECORD));
00503              
00504              if (DataFile)
00505              {
00506                  free(DataFile);
00507                  DataFile = NULL;
00508              }
00509             continue;
00510          }
00511           
00512           if (DataFile)
00513           {
00514               free(DataFile);
00515               DataFile = NULL;
00516           }
00517           
00518           if (val1)
00519           {
00520               free(val1);
00521               val1 = NULL;
00522           }
00523       }
00524       else
00525       {
00526          if (*DataFile && access(filepath, R_OK | F_OK)  == 0 && time_is_invalid(val) == 0)
00527      {
00528             outSeg = drms_segment_lookupnum(outRec, 0);
00529             if (inSeg && outSeg)   
00530             {
00531                if (limbflag)
00532                 {
00533                    status = embed_limbpixels(filepath,outRec,&cropmin,&cropmax);
00534                    drms_setkey_double(outRec, "CROPMIN", cropmin);
00535                    drms_setkey_double(outRec, "CROPMAX", cropmax);
00536                    if (status != 0) 
00537                    DIE("Problem in function embed_limbpixels. \n"); 
00538               }
00539                 else
00540                 {
00541                    DRMS_Array_t *data;
00542                    /* read the data ad doubles so allow rescaling on output */
00543                    data = drms_segment_read(inSeg, DRMS_TYPE_DOUBLE, &status);
00544                    //data = drms_segment_read(inSeg, DRMS_TYPE_SHORT, &status);
00545                    if (!data)
00546                    {
00547                       fprintf(stderr, "Bad data record %lld, status=%d\n",inRec->recnum, status);
00548                       DIE_status("giveup\n");
00549                    }
00550                    /* use the zero and offset values in the JSD for the new record segment */
00551                    data->bscale = outSeg->bscale;
00552                    data->bzero  = outSeg->bzero;
00553                    //printf("cropmin from outside the function =%d\n",cropmin);
00554                    drms_segment_write(outSeg, data, 0);
00555                    drms_free_array(data);
00556                 }
00557                Record_OK = 1;    
00558            quality = drms_getkey_int(inRec, "QUALITY", &qualstat);
00559                quality = quality & (~qualnodata);
00560                drms_setkey_int(outRec,"QUALITY",quality);
00561             }
00562             else
00563                DIE("Bad data segment lookup, in or out\n");
00564          }
00565          else
00566          { /* record is missing, copy t_rec and soho ephemeris keywords anyway*/
00567             drms_copykey(outRec, inRec, "T_REC");
00568             drms_copykey(outRec, inRec, "OBS_VW");
00569             drms_copykey(outRec, inRec, "OBS_VR");
00570             drms_copykey(outRec, inRec, "OBS_VN");
00571             sprint_time(timebuf, (double)time(NULL) + UNIX_epoch, "ISO", 0);
00572             drms_setkey_string(outRec, "DATE", timebuf);
00573             val = drms_getkey_double(inRec, "OBS_DIST", &status);
00574             drms_setkey_double(outRec, "DSUN_OBS", val*AU_m);
00575             val = drms_getkey_double(inRec, "OBS_B0", &status);
00576             drms_setkey_double(outRec, "CRLT_OBS", val);    
00577             val = drms_getkey_double(inRec, "OBS_CR", &status);
00578             drms_setkey_double(outRec, "CAR_ROT", val); 
00579             val = drms_getkey_double(inRec, "OBS_R0", &status);
00580             drms_setkey_double(outRec, "RSUN_OBS", val);
00581             val = drms_getkey_double(inRec, "OBS_L0", &status);
00582             drms_setkey_double(outRec, "CRLN_OBS", val); 
00583             drms_setkey_int(outRec, "QUALITY", 0X80000000);
00584             val = drms_getkey_int(inRec, "DPC", &status);
00585             drms_setkey_int(outRec, "DPC", val);
00586             if (drms_keyword_lookup(outRec, "DATAVALS", 0))
00587                drms_setkey_int(outRec, "DATAVALS", 0);
00588             if (SkipMissingFiles)
00589             {
00590                Record_OK = 0;
00591                if (verbose)
00592                {
00593                    char *strtodel = drms_getkey_string(outRec,"T_REC",NULL);
00594                    if (strtodel)
00595                    {
00596                        fprintf(stderr,"DSDS Record %d has no datafile, T_REC=%s, set missing.\n", iRec, strtodel);
00597                        free(strtodel);
00598                        strtodel = NULL;
00599                    }
00600                }
00601             }
00602             else
00603                Record_OK = 1;
00604             drms_close_records(outRecSet,(Record_OK ? DRMS_INSERT_RECORD : DRMS_FREE_RECORD));
00605              
00606              if (DataFile)
00607              {
00608                  free(DataFile);
00609                  DataFile = NULL;
00610              }
00611             continue;
00612          }
00613 
00614           if (DataFile)
00615           {
00616               free(DataFile);
00617               DataFile = NULL;
00618           }
00619       }
00620 
00621       skip:
00622 
00623       /* loop through all target keywords */
00624       outKey_last = NULL;
00625       while (outKey = drms_record_nextkey(outRec, &outKey_last, 1))
00626     {
00627     char *wantKey, *keyName = outKey->info->name;
00628         int action = keyNameCheck(keyName, &wantKey);
00629         if (!drms_keyword_inclass(outKey, kDRMS_KeyClass_Explicit) || outKey->info->recscope == 1)
00630         continue;  // skip implicit and constant keywords.
00631         switch (action)
00632           {
00633       case ACT_NOP:
00634         break;
00635       case ACT_COPY:
00636         {
00637         DRMS_Value_t inValue = {DRMS_TYPE_STRING, '\0'};
00638         inValue = drms_getkey_p(inRec, wantKey, &status);
00639         if (status == DRMS_ERROR_UNKNOWNKEYWORD)
00640             break;
00641             if (status && verbose)fprintf(stderr,"*** ACT_COPY drms_getkey_p %s status=%d\n",wantKey,status);
00642         drms_setkey_p(outRec, keyName, &inValue);
00643         if ((inValue.type == DRMS_TYPE_STRING) && inValue.value.string_val)
00644           free(inValue.value.string_val);
00645         inValue.value.string_val = NULL;
00646         break;
00647                 }
00648       case ACT_ANGLE:
00649         { /* on CROTA2 set CROTA2, SAT_ROT, INST_ROT */
00650         double pangle, sat_rot;
00651         pangle = drms_getkey_double(inRec, "SOLAR_P", &status);
00652         if (status && verbose)fprintf(stderr,"*** ACT_ANGLE drms_getkey_double SOLAR_P status=%d, pangle=%f\n",status,pangle);
00653         sat_rot = -pangle;
00654         drms_setkey_double(outRec, "CROTA2", sat_rot);
00655         drms_setkey_double(outRec, "SAT_ROT", sat_rot);
00656         drms_setkey_double(outRec, "INST_ROT", 0.0);
00657         break;
00658         }
00659       case ACT_RESCALE:
00660             { /* on X_SCALE, Y_SCALE, MAGNIFY, FD_SCALE set CDELT1, CDELT2 */
00661                 double X_SCALE, Y_SCALE, fdscl, MAGNIFY, rsunobs;
00662         int qual, focus;
00663         double obs_dist;
00664         TIME t_obs;
00665                 drms_setkey_string(outRec, "BLD_VERS", jsoc_version);
00666         obs_dist = drms_getkey_double(inRec, "OBS_DIST", &status);
00667                 X_SCALE = drms_getkey_double(inRec, "X_SCALE", &status);
00668         if (status && verbose)fprintf(stderr,"*** ACT_RESCALE drms_getkey_double X_SCALE status=%d, x0=%f\n",status,X_SCALE);
00669         Y_SCALE = drms_getkey_double(inRec, "Y_SCALE", &status);
00670         if (status && verbose)fprintf(stderr,"*** ACT_RESCALE drms_getkey_double Y_SCALE status=%d, x0=%f\n",status,Y_SCALE);
00671         MAGNIFY = drms_getkey_double(inRec, "MAGNIFY", &status);
00672         if (status && verbose)fprintf(stderr,"*** ACT_RESCALE drms_getkey_double MAGNIFY status=%d, x0=%f\n",status,MAGNIFY);
00673         /* it is critical that the status of the t_obs is what is fed into the two subsequent if statements */
00674                 t_obs = drms_getkey_time(inRec, "T_OBS", &status);
00675                 sprint_time(timebuf, t_obs, "ISO",0);
00676                 //printf("t_obs=%s\n",timebuf);
00677                 if (status && verbose)fprintf(stderr,"*** ACT_RESCALE drms_getkey_time t_obs status=%d, x0=%f\n",status,t_obs);
00678                 //printf("qual=%d,focus=%d,obs_dist=%f,fdscl=%f,X_SCALE=%f,MAGNIFY=%f\n",qual,focus,obs_dist,fdscl,X_SCALE,MAGNIFY);
00679                 if ( status == DRMS_SUCCESS)
00680           {
00681                      status = MDI_plate_scale_adjustment(scfilepath, &fdscl, t_obs, &focus, &qual, obs_dist);
00682                         if (status != 0)
00683                           {
00684                              fprintf(stderr,"status of plate scale adjustment=%d\n",status);
00685                              fprintf(stderr,"incoming filepath with error = %s\n", infilepath);
00686                              return status;
00687                           }
00688           } 
00689         drms_setkey_double(outRec, "CDELT1", X_SCALE*(fdscl/MAGNIFY));
00690         drms_setkey_double(outRec, "CDELT2", Y_SCALE*(fdscl/MAGNIFY));
00691                 rsunobs=drms_getkey_double(inRec,"OBS_R0",&status);
00692                 result=(rsunobs)/(X_SCALE*(fdscl/MAGNIFY));
00693                 status=drms_setkey_float(outRec, "R_SUN", result);  
00694                 //printf("result=%f,status=%d,qual=%d,focus=%d,obs_dist=%f,fdscl=%f,rsunobs=%f,X_SCALE=%f,MAGNIFY=%f\n",result,status,qual,focus,obs_dist,fdscl,rsunobs,X_SCALE,MAGNIFY);
00695             }
00696       case ACT_CENTER:
00697         { /* on CRPIX1 set CRPIX1, CRPIX2, CRVAL1, CRVAL2 */
00698         double x0, y0;
00699         x0 = drms_getkey_double(inRec, "X0", &status);
00700         if (status && verbose)fprintf(stderr,"*** ACT_CENTER drms_getkey_double X0 status=%d, x0=%f\n",status,x0);
00701         y0 = drms_getkey_double(inRec, "Y0", &status);
00702         if (status && verbose)fprintf(stderr,"*** ACT_CENTER drms_getkey_double Y0 status=%d, y0=%f\n",status,y0);
00703         drms_setkey_double(outRec, "CRPIX1", x0+1.0);
00704         drms_setkey_double(outRec, "CRPIX2", y0+1.0);
00705         drms_setkey_double(outRec, "CRVAL1", 0.0);
00706         drms_setkey_double(outRec, "CRVAL2", 0.0);
00707         break;
00708         }
00709       case ACT_TIME:
00710         { /* on T_OBS set T_OBS, DATE-OBS, EXPTIME, CADENCE, TIME, MJD */
00711         char timebuf[1024];
00712         TIME MJD_epoch = -3727641600.000; /* 1858.11.17_00:00:00_UT  */
00713         TIME UNIX_epoch = -220924792.000; /* 1970.01.01_00:00:00_UTC */
00714                 TIME t_obs, t_rec, date__obs, mjd, now; 
00715         double t_step;
00716         double exptime, mjd_day, mjd_time;
00717         t_rec = drms_getkey_time(inRec, "T_REC", &status);
00718         if (status && verbose)fprintf(stderr,"*** ACT_TIME drms_getkey_time T_REC status=%d, t_rec=%f\n",status,t_rec);
00719         t_step = drms_getkey_double(outRec, "T_REC_step", &status); /* note from outRec */
00720         if (status && verbose)fprintf(stderr,"*** ACT_TIME drms_getkey_double T_REC_step status=%d, t_step=%f\n",status,t_step);
00721         // exptime = t_step; /* note - for lev1.5 */
00722                 exptime = drms_getkey_double(inRec, "INTERVAL", &status);
00723         t_obs = drms_getkey_time(inRec, "T_OBS", &status);
00724         if (status && verbose)fprintf(stderr,"*** ACT_TIME drms_getkey_time T_OBS status=%d, t_obs=%f\n",status,t_obs);
00725                 if (status == DRMS_ERROR_UNKNOWNKEYWORD) // T_OBS is not present, skip this record.
00726         date__obs = t_obs - exptime/2.0;
00727                 mjd = date__obs - MJD_epoch; /* sign error corrected by tplarson 2008.05.29 */
00728         mjd_day = floor(mjd / 86400.0);
00729         mjd_time = mjd - 86400.0 * mjd_day;
00730         now = (double)time(NULL) + UNIX_epoch;
00731         drms_setkey_time(outRec, "T_REC", t_rec);
00732         drms_setkey_time(outRec, "T_OBS", t_obs);
00733         drms_setkey_double(outRec, "EXPTIME", exptime);
00734         drms_setkey_double(outRec, "CADENCE", t_step);
00735         drms_setkey_double(outRec, "MJD", mjd_day);
00736         drms_setkey_double(outRec, "TIME", mjd_time);
00737                 // allow either string or time types for DATE and DATE_OBS
00738                 if (drms_keyword_type(drms_keyword_lookup(outRec, "DATE__OBS", 1)) == DRMS_TYPE_STRING)
00739                   {
00740           sprint_time(timebuf, date__obs, "ISO", 0);
00741           drms_setkey_string(outRec, "DATE__OBS", timebuf);
00742                   }
00743                 else {
00744                   date__obs = t_obs - exptime/2.0;
00745           drms_setkey_time(outRec, "DATE__OBS", date__obs);}
00746         if (drms_keyword_type(drms_keyword_lookup(outRec, "DATE", 1)) == DRMS_TYPE_STRING)
00747                   {
00748           sprint_time(timebuf, now, "ISO", 0);
00749           drms_setkey_string(outRec, "DATE", timebuf);
00750           }
00751                 else
00752                   drms_setkey_time(outRec, "DATE", now);
00753 
00754         break;
00755         }
00756       case ACT_AU:
00757         {
00758 # define AU_m (149597870691.0)
00759 //#define AU_m (1.49597892e11) bad
00760         double au;
00761         au = drms_getkey_double(inRec, "OBS_DIST", &status);
00762         if (status && verbose)fprintf(stderr,"*** ACT_AU drms_getkey_double OBS_DIST status=%d,au=%f\n",status,au);
00763         if (status != DRMS_ERROR_UNKNOWNKEYWORD)
00764             drms_setkey_double(outRec, "DSUN_OBS", au * AU_m);
00765         break;
00766         }
00767           case ACT_NOT_FOUND:
00768           default:
00769                 /* name not in table, just take same name from input series */
00770                 {
00771         DRMS_Value_t inValue = {DRMS_TYPE_STRING, '\0'};
00772                 DRMS_Keyword_t *outKey = drms_keyword_lookup(outRec, keyName, 0);
00773                 if (drms_keyword_isconstant(outKey))
00774             break;
00775                 inValue = drms_getkey_p(inRec, keyName, &status);
00776         if (status == DRMS_ERROR_UNKNOWNKEYWORD)
00777             break;
00778         if (status && verbose)fprintf(stderr,"*** DEFAULT drms_getkey_p %s status=%d\n",keyName, status);
00779                 drms_setkey_p(outRec, keyName, &inValue);
00780         if ((inValue.type == DRMS_TYPE_STRING) && inValue.value.string_val)
00781           free(inValue.value.string_val);
00782         inValue.value.string_val = NULL;
00783                 break;
00784                 }
00785           }
00786         }
00787           
00788           if (outKey_last)
00789           {
00790               hiter_destroy(&outKey_last);
00791           }
00792 
00793         drms_setkey_time(outRec, "DATE", time_now());
00794 
00795 /* populate the ROLL_TBL keyword */
00796 
00797 char *rollrecstring;
00798 char *rolltbls=drms_getkey_string(inRec, "ROLL_TBL", &status);
00799 
00800 if ( status == DRMS_SUCCESS)
00801     {
00802         if (!strcmp("", rolltbls)) 
00803         rollrecstring="";
00804     if (!strcmp("not used", rolltbls)) 
00805         rollrecstring="not used";
00806     if (!strcmp("/CM/tables/ephemeris/mdi_roll.smooth", rolltbls)) 
00807         rollrecstring="mdi.roll_table";
00808     if (!strcmp("/CM/tables/ephemeris/mdi_roll.ascii", rolltbls)) 
00809         rollrecstring="mdi.roll_table_ascii";
00810         drms_setkey_string(outRec, "ROLL_TBL", rollrecstring);
00811     }
00812           
00813 if (rolltbls)
00814 {
00815     free(rolltbls);
00816     rolltbls = NULL;
00817 }
00818 
00819 /* populate the CALTBLS keyword */
00820 
00821 char *calrecstring;
00822 char *caltbls=drms_getkey_string(inRec, "CALTBLS", &status);
00823 
00824 if ( status == DRMS_SUCCESS)
00825 {
00826 if (!strcmp("", caltbls)) 
00827   calrecstring="";
00828 if (!strcmp("/home/soi/CM/tables/calib/flat/fd/vers_0/", caltbls)) 
00829   calrecstring="mdi.caltables_intensity[fd_vers_0]";
00830 if (!strcmp("/home/soi/CM/tables/calib/flat/fd/vers_1/", caltbls))
00831   calrecstring="mdi.caltables_intensity[fd_vers_1]";
00832 if (!strcmp("/home/soi/CM/tables/calib/flat/hr/", caltbls))
00833   calrecstring="mdi.caltables_intensity[hr_vers_1]";
00834 if (!strcmp("/home/soi/CM/tables/calib/flat/hr/vers_0/", caltbls))
00835   calrecstring="mdi.caltables_intensity[hr_vers_0]";
00836 if (!strcmp("/home/soi/CM/tables/calib/obflat/vers_0/", caltbls))
00837   calrecstring="mdi.caltables_intensity[obflat_vers_0]";
00838 if (!strcmp("/home/soi/CM/tables/calib/obflat/vers_1/", caltbls))
00839   calrecstring="mdi.caltables_intensity[obflat_vers_1]";
00840 if (!strcmp("/home/soi/CM/tables/calib/obflat/vers_2/", caltbls))
00841   calrecstring="mdi.caltables_intensity[obflat_vers_2]";
00842 if (!strcmp("/home/soi/CM/tables/calib/dop/fd/tune_0/", caltbls))
00843   calrecstring="mdi.caltables_doppler[fd_tune_0]";
00844 if (!strcmp("/home/soi/CM/tables/calib/dop/fd/tune_1/", caltbls))
00845   calrecstring="mdi.caltables_doppler[fd_tune_1]";
00846 if (!strcmp("/home/soi/CM/tables/calib/dop/fd/tune_2/", caltbls))
00847   calrecstring="mdi.caltables_doppler[fd_tune_2]";
00848 if (!strcmp("/home/soi/CM/tables/calib/dop/fd/tune_4/", caltbls))
00849   calrecstring="mdi.caltables_doppler[fd_tune_4]";
00850 if (!strcmp("/home/soi/CM/tables/calib/dop/fd/tune_5/", caltbls))
00851   calrecstring="mdi.caltables_doppler[fd_tune_5]";
00852 if (!strcmp("/home/soi/CM/tables/calib/dop/fd/tune_6/", caltbls))
00853   calrecstring="mdi.caltables_doppler[fd_tune_6]";
00854 if (!strcmp("/home/soi/CM/tables/calib/dop/fd_spec/tune_0/", caltbls))
00855   calrecstring="mdi.caltables_doppler[fd_spec_tune_0]";
00856 if (!strcmp("/home/soi/CM/tables/calib/dop/hr/tune_1/", caltbls))
00857   calrecstring="mdi.caltables_doppler[hr_tune_1]";
00858 if (!strcmp("/home/soi/CM/tables/calib/dop/hr/tune_2/", caltbls))
00859   calrecstring="mdi.caltables_doppler[hr_tune_2]";
00860 if (!strcmp("/home/soi/CM/tables/calib/dop/hr/tune_3/", caltbls))
00861   calrecstring="mdi.caltables_doppler[hr_tune_3]";
00862 if (!strcmp("/home/soi/CM/tables/calib/dop/hr/tune_4/", caltbls))
00863   calrecstring="mdi.caltables_doppler[hr_tune_4]";
00864 if (!strcmp("/home/soi/CM/tables/calib/dop/hr/tune_5/", caltbls))
00865   calrecstring="mdi.caltables_doppler[hr_tune_5]";
00866 if (!strcmp("/home/soi/CM/tables/calib/dop/hr/tune_6/", caltbls))
00867   calrecstring="mdi.caltables_doppler[hr_tune_6]";
00868 if (!strcmp("/home/soi/CM/tables/calib/dop/hr_spec/tune_0/", caltbls))
00869   calrecstring="mdi.caltables_doppler[hr_spec_tune_0]";
00870 if (!strcmp("/home/soi/CM/tables/calib/dop/orig/tune_1/", caltbls))
00871   calrecstring="mdi.caltables_doppler_orig[orig_tune_1]";
00872 if (!strcmp("/home/soi/CM/tables/calib/dop/orig/tune_2/", caltbls))
00873   calrecstring="mdi.caltables_doppler_orig[orig_tune_2]";
00874 if (!strcmp("/home/soi/CM/tables/calib/dop/orig/tune_4/", caltbls))
00875   calrecstring="mdi.caltables_doppler_orig[orig_tune_4]";
00876 if (!strcmp("/home/soi/CM/tables/calib/dop/orig/tune_5/", caltbls))
00877   calrecstring="mdi.caltables_doppler_orig[orig_tune_5]";
00878 if (!strcmp("/home/soi/CM/tables/calib/dop/orig/tune_6/", caltbls))
00879   calrecstring="mdi.caltables_doppler_orig[orig_tune_6]";
00880 if (!strcmp("NONE", caltbls))
00881   calrecstring="NONE";
00882 
00883 drms_setkey_string(outRec, "CALTBLS", calrecstring);
00884     
00885     if (caltbls)
00886     {
00887         free(caltbls);
00888         caltbls = NULL;
00889     }
00890 }
00891         
00892       /* loop through all target links */
00893       //while ( (outLink=(DRMS_Keyword_t *)hiter_getnext(&link_list)) )
00894         //{
00896         //}
00897       drms_close_records(outRecSet,(Record_OK ? DRMS_INSERT_RECORD : DRMS_FREE_RECORD));
00898       }
00899        
00900        /* Close no-longer needed record set. 
00901         * -ART
00902         */
00903        drms_close_records(screcset, DRMS_FREE_RECORD);
00904    drms_close_records(inRecSet,DRMS_FREE_RECORD);
00905    return(DRMS_SUCCESS);
00906    }