JSOC_Documentation: proj/dsdsmigr/apps/ingest_dsds

00001 /* ingest_dsds_a.c */
00002 
00003 #include "jsoc_main.h"
00004 #include "drms_types.h"
00005 #include <time.h>
00006 #include <math.h>
00007 
00008 char *module_name = "opendsrecs";
00009 
00010 #define kRecSetIn   "in"
00011 #define kRecSetOut  "out"
00012 #define kNameList   "map"
00013 #define kNOT_SPEC   "Not Specified"
00014 
00015 #define DIE(msg)    return(fprintf(stderr,"%s",msg),1)
00016 #define DIE_status(msg) return(fprintf(stderr,"%s, status=%d",msg,status),1)
00017 
00018 ModuleArgs_t module_args[] =
00019 {
00020      {ARG_STRING, kRecSetIn, kNOT_SPEC, "Input data series."},
00021      {ARG_STRING, kRecSetOut, kNOT_SPEC, "Output data series."},
00022      {ARG_STRING, kNameList, kNOT_SPEC, "Name conversion list."},
00023      {ARG_FLAG, "M", "0", "SkipMissingFiles - no records if DATAFILE is blank."},
00024      {ARG_FLAG, "v", "0", "verbose - more diagnostics"},
00025      {ARG_END}
00026 };
00027 
00028 // #include <sys/time.h>
00029 TIME time_now()
00030   {
00031   TIME now;
00032   TIME UNIX_epoch = -220924792.000; /* 1970.01.01_00:00:00_UTC */
00033   struct timeval tp;
00034   gettimeofday(&tp, NULL);
00035   now = (double)tp.tv_sec + (double)tp.tv_usec/1.0e6;
00036   return(now +  UNIX_epoch);
00037   }
00038   
00039 
00040 /* Name check code.  This code uses an external table to drive the mapping of
00041 dsds keywords into drms keywords.  It should contain a row for each keyword in
00042 the target drms series.  Each row should contain 3 fields, drms_name, dsds_name, action.
00043 The action will be a string which matches the action table.  The action table
00044 converts the string to an integer.
00045 Sample file lines like:
00046   cadence CADENCE copy
00047   date DATE copy
00048 Lines begging with "#" are comments and will be ignored.
00049 Blank lines are not allowed.
00050 */
00051 
00052 typedef struct NameListLookup_struct
00053   {
00054   char *drms_name;
00055   char *dsds_name;
00056   int action;
00057   struct NameListLookup_struct *next;
00058   } NameListLookup_t;
00059 
00060 typedef enum {
00061   ACT_NOP, ACT_COPY, ACT_ANGLE, ACT_CENTER, ACT_TIME, ACT_AU, ACT_NOT_FOUND
00062   } Action_t;
00063 
00064 Action_t actions[] = {
00065   ACT_NOP, ACT_COPY, ACT_ANGLE, ACT_CENTER, ACT_TIME, ACT_AU, ACT_NOT_FOUND
00066   };
00067 
00068 char *action_names[] = {
00069   "nop",   "copy",   "pangle",   "center",   "time",  "au",   "done"
00070   };
00071 
00072 int keyNameCheck(char *name, char **fromname)
00073   {
00074   static int first_call = 1;
00075   static NameListLookup_t *actionlist;
00076   NameListLookup_t *this;
00077   if (first_call)
00078     {
00079     char *actionlistname;
00080     FILE *flist;
00081     char drms_name[100], dsds_name[100], action_name[100];
00082     char line[1024];
00083     // int action;
00084     NameListLookup_t *last;
00085 
00086     first_call = 0;
00087     actionlistname = cmdparams_get_str(&cmdparams, kNameList, NULL);
00088     if (strcmp(actionlistname, kNOT_SPEC) == 0)
00089       {
00090       fprintf(stderr, "Name mapping list must be specified\n");
00091       exit(1);
00092       }
00093     flist = fopen(actionlistname, "r");
00094     if (!flist)
00095       {
00096       fprintf(stderr, "Name mapping file not found\n");
00097       exit(1);
00098       }
00099     last = actionlist = (NameListLookup_t *)malloc(sizeof(struct NameListLookup_struct));
00100     last->next = NULL;
00101     while (fgets(line, 1024, flist))
00102       {
00103       if (line[0] == '#')
00104         continue;
00105       else if (sscanf(line,"%s %s %s\n", drms_name, dsds_name, action_name)==3)
00106         {
00107         int iname;
00108     this = last;
00109         last = this->next = (NameListLookup_t *)malloc(sizeof(struct NameListLookup_struct));
00110         last->next = NULL;
00111         this->drms_name = strdup(drms_name);
00112         this->dsds_name = strdup(dsds_name);
00113         this->action = ACT_NOT_FOUND;
00114         for (iname=0; actions[iname] != ACT_NOT_FOUND; iname++)
00115           if (strcmp(action_name, action_names[iname]) == 0)
00116             {
00117             this->action = actions[iname];
00118             break;
00119             }
00120         }
00121       else fprintf(stderr, "Name map read error on line containing %s, ignoring this line\n",line);
00122       }
00123     }
00124   /* lookup name and return action */
00125   for (this=actionlist; this->next; this = this->next)
00126     {
00127     if (strcmp(this->drms_name, name) == 0)
00128       {
00129       *fromname = this->dsds_name;
00130       return(this->action);
00131       }
00132     }
00133   *fromname = name;
00134   return(ACT_NOT_FOUND);
00135   }
00136 
00137 int DoIt(void) 
00138    {
00139    int status = DRMS_SUCCESS;
00140    int SkipMissingFiles;
00141    int verbose;
00142    int nRecs, iRec;
00143    char *inRecQuery, *outRecQuery;
00144    DRMS_RecordSet_t *inRecSet, *outRecSet; 
00145 
00146    inRecQuery = cmdparams_get_str(&cmdparams, kRecSetIn, NULL);
00147    outRecQuery = cmdparams_get_str(&cmdparams, kRecSetOut, NULL);
00148    SkipMissingFiles = cmdparams_get_int(&cmdparams, "M", NULL) != 0;
00149    verbose = cmdparams_get_int(&cmdparams, "v", NULL) != 0;
00150 
00151    if (strcmp(inRecQuery, kNOT_SPEC) == 0 || strcmp(outRecQuery, kNOT_SPEC) == 0)
00152       DIE("Both the "kRecSetIn" and "kRecSetOut" dataseries must be specified.\n");
00153 
00154    inRecSet = drms_open_records(drms_env, inRecQuery, &status);
00155    if (!inRecSet)
00156       DIE_status("Input dataseries not found\n");
00157    if ((nRecs = inRecSet->n) == 0)
00158       DIE("No input records found\n");
00159    printf("%d input records found\n", nRecs);
00160 
00161    for (iRec=0; iRec<nRecs; iRec++)
00162       {
00163       char *DataFile;
00164       int Record_OK = 1;
00165       DRMS_Record_t *inRec, *outRec;
00166       DRMS_Keyword_t *outKey;
00167       DRMS_Segment_t *inSeg, *outSeg;
00168       HIterator_t *outKey_last = NULL;
00169 //      DRMS_Link_t *outLink;
00170 
00171       /* create output series rec prototype */
00172       inRec = inRecSet->records[iRec];
00173       outRecSet = drms_create_records(drms_env, 1, outRecQuery, DRMS_PERMANENT, &status);
00174       if (!outRecSet || outRecSet->n != 1)
00175          DIE_status("Output dataseries not found or can't create records\n");
00176 
00177       outRec = outRecSet->records[0];
00178 
00179       /* loop through all target keywords */
00180       outKey_last = NULL;
00181       while (outKey = drms_record_nextkey(outRec, &outKey_last, 1))
00182     {
00183     char *wantKey, *keyName = outKey->info->name;
00184         int action = keyNameCheck(keyName, &wantKey);
00185         if (!drms_keyword_inclass(outKey, kDRMS_KeyClass_Explicit))
00186         continue;  // skip implicit keywords.
00187         switch (action)
00188           {
00189       case ACT_NOP:
00190         break;
00191       case ACT_COPY:
00192         {
00193         DRMS_Value_t inValue = {DRMS_TYPE_STRING, NULL};
00194         inValue = drms_getkey_p(inRec, wantKey, &status);
00195         if (status == DRMS_ERROR_UNKNOWNKEYWORD)
00196             break;
00197             if (status && verbose)fprintf(stderr,"*** ACT_COPY drms_getkey_p %s status=%d\n",wantKey,status);
00198         drms_setkey_p(outRec, keyName, &inValue);
00199         if ((inValue.type == DRMS_TYPE_STRING) && inValue.value.string_val)
00200           free(inValue.value.string_val);
00201         inValue.value.string_val = NULL;
00202                 break;
00203                 }
00204       case ACT_ANGLE:
00205         { /* on CROTA2 set CROTA2, SAT_ROT, INST_ROT */
00206         double pangle, sat_rot;
00207         pangle = drms_getkey_double(inRec, "SOLAR_P", &status);
00208         if (status && verbose)fprintf(stderr,"*** ACT_ANGLE drms_getkey_double SOLAR_P status=%d, pangle=%f\n",status,pangle);
00209         sat_rot = -pangle;
00210         drms_setkey_double(outRec, "CROTA2", sat_rot);
00211         drms_setkey_double(outRec, "SAT_ROT", sat_rot);
00212         drms_setkey_double(outRec, "INST_ROT", 0.0);
00213         break;
00214         }
00215       case ACT_CENTER:
00216         { /* on CRPIX1 set CRPIX1, CRPIX2, CRVAL1, CRVAL2 */
00217         double x0, y0;
00218         x0 = drms_getkey_double(inRec, "X0", &status);
00219         if (status && verbose)fprintf(stderr,"*** ACT_CENTER drms_getkey_double X0 status=%d, x0=%f\n",status,x0);
00220         y0 = drms_getkey_double(inRec, "Y0", &status);
00221         if (status && verbose)fprintf(stderr,"*** ACT_CENTER drms_getkey_double Y0 status=%d, y0=%f\n",status,y0);
00222         drms_setkey_double(outRec, "CRPIX1", x0+1.0);
00223         drms_setkey_double(outRec, "CRPIX2", y0+1.0);
00224         drms_setkey_double(outRec, "CRVAL1", 0.0);
00225         drms_setkey_double(outRec, "CRVAL2", 0.0);
00226         break;
00227         }
00228       case ACT_TIME:
00229         { /* on T_OBS set T_OBS, DATE-OBS, EXPTIME, CADENCE, TIME, MJD */
00230         char timebuf[1024];
00231         TIME MJD_epoch = -3727641600.000; /* 1858.11.17_00:00:00_UT  */
00232         TIME UNIX_epoch = -220924792.000; /* 1970.01.01_00:00:00_UTC */
00233         TIME t_obs, t_rec, date__obs, mjd, now;
00234         double t_step;
00235         double exptime, mjd_day, mjd_time;
00236         t_rec = drms_getkey_time(inRec, "T_REC", &status);
00237         if (status && verbose)fprintf(stderr,"*** ACT_TIME drms_getkey_time T_REC status=%d, t_rec=%f\n",status,t_rec);
00238         t_step = drms_getkey_double(outRec, "T_REC_step", &status); /* note from outRec */
00239         if (status && verbose)fprintf(stderr,"*** ACT_TIME drms_getkey_double T_REC_step status=%d, t_step=%f\n",status,t_step);
00240         // exptime = t_step; /* note - for lev1.5 */
00241                 exptime = drms_getkey_double(inRec, "INTERVAL", &status);
00242         t_obs = drms_getkey_time(inRec, "T_OBS", &status);
00243         if (status && verbose)fprintf(stderr,"*** ACT_TIME drms_getkey_time T_OBS status=%d, t_obs=%f\n",status,t_obs);
00244                 if (status == DRMS_ERROR_UNKNOWNKEYWORD) // T_OBS is not present, skip this record.
00245         date__obs = t_obs - exptime/2.0;
00246         mjd = date__obs - MJD_epoch; /* sign error corrected by tplarson 2008.05.29 */
00247         mjd_day = floor(mjd / 86400.0);
00248         mjd_time = mjd - 86400.0 * mjd_day;
00249         now = (double)time(NULL) + UNIX_epoch;
00250         drms_setkey_time(outRec, "T_REC", t_rec);
00251         drms_setkey_time(outRec, "T_OBS", t_obs);
00252         drms_setkey_double(outRec, "EXPTIME", exptime);
00253         drms_setkey_double(outRec, "CADENCE", t_step);
00254         drms_setkey_double(outRec, "MJD", mjd_day);
00255         drms_setkey_double(outRec, "TIME", mjd_time);
00256                 // allow either string or time types for DATE and DATE_OBS
00257                 if (drms_keyword_type(drms_keyword_lookup(outRec, "DATE__OBS", 1)) == DRMS_TYPE_STRING)
00258                   {
00259           sprint_time(timebuf, date__obs, "ISO", 0);
00260           drms_setkey_string(outRec, "DATE__OBS", timebuf);
00261                   }
00262                 else
00263                   drms_setkey_time(outRec, "DATE__OBS", date__obs);
00264                 if (drms_keyword_type(drms_keyword_lookup(outRec, "DATE", 1)) == DRMS_TYPE_STRING)
00265                   {
00266           sprint_time(timebuf, now, "ISO", 0);
00267           drms_setkey_string(outRec, "DATE", timebuf);
00268           }
00269                 else
00270                   drms_setkey_time(outRec, "DATE", now);
00271 
00272         break;
00273         }
00274       case ACT_AU:
00275         {
00276 # define AU_m (149597870691.0)
00277 //#define AU_m (1.49597892e11) bad
00278         double au;
00279         au = drms_getkey_double(inRec, "OBS_DIST", &status);
00280         if (status && verbose)fprintf(stderr,"*** ACT_AU drms_getkey_double OBS_DIST status=%d,au=%f\n",status,au);
00281         if (status != DRMS_ERROR_UNKNOWNKEYWORD)
00282             drms_setkey_double(outRec, "DSUN_OBS", au * AU_m);
00283         break;
00284         }
00285           case ACT_NOT_FOUND:
00286           default:
00287                 /* name not in table, just take same name from input series */
00288                 {
00289         DRMS_Value_t inValue = {DRMS_TYPE_STRING, NULL};
00290                 DRMS_Keyword_t *outKey = drms_keyword_lookup(outRec, keyName, 0);
00291                 if (drms_keyword_isconstant(outKey))
00292             break;
00293                 inValue = drms_getkey_p(inRec, keyName, &status);
00294         if (status == DRMS_ERROR_UNKNOWNKEYWORD)
00295             break;
00296         if (status && verbose)fprintf(stderr,"*** DEFAULT drms_getkey_p %s status=%d\n",keyName, status);
00297                 drms_setkey_p(outRec, keyName, &inValue);
00298         if ((inValue.type == DRMS_TYPE_STRING) && inValue.value.string_val)
00299           free(inValue.value.string_val);
00300         inValue.value.string_val = NULL;
00301                 break;
00302                 }
00303           }
00304         }
00305 
00306       /* assume only one segment */
00307     DataFile = drms_getkey_string(inRec,"DATAFILE",&status);
00308     if (status && verbose)fprintf(stderr,"*** Segment Read DATAFILE status=%d\n",status);
00309     char filepath[DRMS_MAXPATHLEN];
00310     inSeg = drms_segment_lookupnum(inRec, 0);
00311         if (inSeg)
00312       drms_segment_filename(inSeg, filepath);
00313         else 
00314           filepath[0] = '\0';
00315     if (*DataFile && access(filepath, R_OK | F_OK) == 0)
00316       {
00317           outSeg = drms_segment_lookupnum(outRec, 0);
00318           if (inSeg && outSeg)
00319             {
00320             DRMS_Array_t *data;
00321             /* read the data ad doubles so allow rescaling on output */
00322             data = drms_segment_read(inSeg, DRMS_TYPE_DOUBLE, &status);
00323             if (!data)
00324                   {
00325                   fprintf(stderr, "Bad data record %d\n",iRec);
00326                   DIE_status("giveup\n");
00327                   }
00328             /* use the zero and offset values in the JSD for the new record segment */
00329             data->bscale = outSeg->bscale;
00330             data->bzero = outSeg->bzero;
00331             drms_segment_write(outSeg, data, 0);
00332             drms_free_array(data);
00333             Record_OK = 1;
00334             }
00335           else
00336             DIE("Bad data segment lookup, in or out\n");
00337       }
00338         else
00339       { /* record is missing */
00340       int qualstat = 0;
00341           int quality = drms_getkey_int(outRec, "QUALITY", &qualstat);
00342       if (!qualstat)
00343         drms_setkey_int(outRec, "QUALITY", 0X80000000 | quality); 
00344       if (drms_keyword_lookup(outRec, "DATAVALS", 0))
00345         drms_setkey_int(outRec, "DATAVALS", 0);
00346           if (SkipMissingFiles)
00347              {
00348              Record_OK = 0;
00349              if (verbose) 
00350                fprintf(stderr,"DSDS Record %d has no datafile, T_REC=%s, set missing.\n", iRec, drms_getkey_string(outRec,"T_REC",NULL));
00351              }
00352           else
00353              Record_OK = 1;
00354       }
00355 
00356       /* loop through all target links */
00357       //while ( (outLink=(DRMS_Keyword_t *)hiter_getnext(&link_list)) )
00358         //{
00360         //}
00361       drms_close_records(outRecSet,(Record_OK ? DRMS_INSERT_RECORD : DRMS_FREE_RECORD));
00362       }
00363 
00364    drms_close_records(inRecSet,DRMS_FREE_RECORD);
00365 
00366    return(DRMS_SUCCESS);
00367    }
00368
proj/dsdsmigr/apps/ingest_dsds_a.c
