00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011 #include <stdio.h>
00012 #include <stdlib.h>
00013
00014 #include "jsoc_main.h"
00015 #include "cmdparams.h"
00016 #include "drms_env.h"
00017
00018 #define DEBUG 0
00019
00020 #define kDefaultNamespace "sdo"
00021 #define kDefaultSeriesIn "fds"
00022 #define kDefaultSeriesOut "fds_orbit_vectors"
00023 #define kSeriesHistory "fds_orbit_ingesthist"
00024 #define kObsDateKey "OBS_DATE"
00025 #define kFdsDataProductKey "FDS_DATA_PRODUCT"
00026 #define kFdsProductCompKey "FDS_PRODUCT_COMP"
00027 #define kFdsDataFormatKey "DATA_FORMAT"
00028 #define kFileVersionKey "FILE_VERSION"
00029 #define kFdsProductCompHELIO "ORBIT_HELIO"
00030 #define kFdsProductCompGEO "ORBIT_GEO"
00031 #define kNotSpecified "NOT SPECIFIED"
00032
00033 #define kSVFileSegName "FILENAME"
00034
00035 #define kSVLineMax 1024
00036 #define kMaxHashKey 1024
00037
00038
00039 #define kSVKeyPrimary "OBS_DATE"
00040 #define kSVKeyXHELIO "HCIEC_X"
00041 #define kSVKeyYHELIO "HCIEC_Y"
00042 #define kSVKeyZHELIO "HCIEC_Z"
00043 #define kSVKeyVxHELIO "HCIEC_VX"
00044 #define kSVKeyVyHELIO "HCIEC_VY"
00045 #define kSVKeyVzHELIO "HCIEC_VZ"
00046 #define kSVKeyXGEO "GCIEC_X"
00047 #define kSVKeyYGEO "GCIEC_Y"
00048 #define kSVKeyZGEO "GCIEC_Z"
00049 #define kSVKeyVxGEO "GCIEC_VX"
00050 #define kSVKeyVyGEO "GCIEC_VY"
00051 #define kSVKeyVzGEO "GCIEC_VZ"
00052 #define kSVKeyDATE "DATE"
00053
00054 #define kSVKey_idHELIO "id_HELIO"
00055 #define kSVKey_idGEO "id_GEO"
00056
00057 #define gChunkSize 16384
00058 #define gCacheKeySize 128
00059
00060 #define kTOBSMaxLen 64
00061 #define kIDMaxLen 256
00062
00063 struct VectorNode_struct
00064 {
00065 char tobs[kTOBSMaxLen];
00066 double hcix;
00067 double hciy;
00068 double hciz;
00069 double hcivx;
00070 double hcivy;
00071 double hcivz;
00072 char hciID[kIDMaxLen];
00073 double gcix;
00074 double gciy;
00075 double gciz;
00076 double gcivx;
00077 double gcivy;
00078 double gcivz;
00079 char gciID[kIDMaxLen];
00080 };
00081
00082 typedef struct VectorNode_struct VectorNode_t;
00083
00084 typedef HContainer_t VectorCache_t;
00085
00086 ModuleArgs_t module_args[] =
00087 {
00088 {ARG_STRING, "ns", kDefaultNamespace, "working namespace (sdo, sdo_ground, sdo_dev)"},
00089 {ARG_STRING, "seriesIn", kDefaultSeriesIn, "name of series containing FDS data"},
00090 {ARG_STRING, "timeRange", kNotSpecified, "DRMS time interval encompassing data file dates"},
00091 {ARG_STRING, "seriesOut", kDefaultSeriesOut, "name of series in which to save extracted data"},
00092 {ARG_STRING, "owner", kNotSpecified, "name of database user who owns output series"},
00093 {ARG_FLAG, "c", "0", "create the series only"},
00094 {ARG_END}
00095 };
00096
00097 char *module_name = "extractFdsStateV";
00098
00099 int nice_intro ()
00100 {
00101 int usage = cmdparams_get_int(&cmdparams, "h", NULL);
00102 if (usage)
00103 {
00104 printf ("Usage:\n\textractFdsStateV [-h] "
00105 "[seriesIn=<seriesname>] [timeRange=<timerange>] [seriesOut=<seriesname>]\n"
00106 " details are:\n"
00107 " -h: help - show this message then exit\n"
00108 " <seriesname> - fully qualified series name.\n"
00109 " <timerange> - time value range set.\n"
00110 " seriesIn defaults to sdo.fds.\n"
00111 " timeRange defaults to all records in seriesIn.\n"
00112 " seriesOut defaults to sdo.fdsStateVectors.\n"
00113 " example - extractFdsStateV seriesIn=su_arta.TestFDSData timeRange=2006.11.20_22:38:00-2006.11.20_22:45:00,2006.11.20_22:52:00. seriesOut=su_arta.TestFDSHelio\n");
00114 return(1);
00115 }
00116 return (0);
00117 }
00118
00119 static VectorCache_t *CreateVectorCache()
00120 {
00121 VectorCache_t *cache = (VectorCache_t *)malloc(sizeof(VectorCache_t));
00122 hcon_init_ext((HContainer_t *)cache, 49999, sizeof(VectorNode_t), kTOBSMaxLen, NULL, NULL);
00123 return cache;
00124 }
00125
00126 static void DestroyVectorCache(VectorCache_t **cache)
00127 {
00128 if (cache)
00129 {
00130 if (*cache)
00131 {
00132 hcon_destroy((HContainer_t **)cache);
00133 }
00134 }
00135 }
00136
00137 static void VCacheCache(VectorCache_t *cache,
00138 char *tobs,
00139 double *hcix,
00140 double *hciy,
00141 double *hciz,
00142 double *hcivx,
00143 double *hcivy,
00144 double *hcivz,
00145 char *hciID,
00146 double *gcix,
00147 double *gciy,
00148 double *gciz,
00149 double *gcivx,
00150 double *gcivy,
00151 double *gcivz,
00152 char *gciID)
00153 {
00154 if (cache && tobs && *tobs)
00155 {
00156 #if DEBUG
00157
00158 TIMER_t *timer = CreateTimer();
00159 VectorNode_t *node = (VectorNode_t *)hcon_allocslot_lower(cache, tobs);
00160 fprintf(stdout, "alloc slot: %f seconds.\n", GetElapsedTime(timer));
00161 DestroyTimer(&timer);
00162 #else
00163 VectorNode_t *node = (VectorNode_t *)hcon_allocslot_lower(cache, tobs);
00164 #endif
00165 snprintf(node->tobs, kTOBSMaxLen, "%s", tobs);
00166
00167 if (hcix)
00168 {
00169 node->hcix = *hcix;
00170 }
00171 else
00172 {
00173 node->hcix = DRMS_MISSING_DOUBLE;
00174 }
00175 if (hciy)
00176 {
00177 node->hciy = *hciy;
00178 }
00179 else
00180 {
00181 node->hciy = DRMS_MISSING_DOUBLE;
00182 }
00183 if (hciz)
00184 {
00185 node->hciz = *hciz;
00186 }
00187 else
00188 {
00189 node->hciz = DRMS_MISSING_DOUBLE;
00190 }
00191 if (hcivx)
00192 {
00193 node->hcivx = *hcivx;
00194 }
00195 else
00196 {
00197 node->hcivx = DRMS_MISSING_DOUBLE;
00198 }
00199 if (hcivy)
00200 {
00201 node->hcivy = *hcivy;
00202 }
00203 else
00204 {
00205 node->hcivy = DRMS_MISSING_DOUBLE;
00206 }
00207 if (hcivz)
00208 {
00209 node->hcivz = *hcivz;
00210 }
00211 else
00212 {
00213 node->hcivz = DRMS_MISSING_DOUBLE;
00214 }
00215
00216 if (hciID && *hciID)
00217 {
00218 snprintf(node->hciID, kIDMaxLen, "%s", hciID);
00219 }
00220 else
00221 {
00222 snprintf(node->hciID, kIDMaxLen, "%s", "unknown");
00223 }
00224
00225 if (gcix)
00226 {
00227 node->gcix = *gcix;
00228 }
00229 else
00230 {
00231 node->gcix = DRMS_MISSING_DOUBLE;
00232 }
00233 if (gciy)
00234 {
00235 node->gciy = *gciy;
00236 }
00237 else
00238 {
00239 node->gciy = DRMS_MISSING_DOUBLE;
00240 }
00241 if (gciz)
00242 {
00243 node->gciz = *gciz;
00244 }
00245 else
00246 {
00247 node->gciz = DRMS_MISSING_DOUBLE;
00248 }
00249 if (gcivx)
00250 {
00251 node->gcivx = *gcivx;
00252 }
00253 else
00254 {
00255 node->gcivx = DRMS_MISSING_DOUBLE;
00256 }
00257 if (gcivy)
00258 {
00259 node->gcivy = *gcivy;
00260 }
00261 else
00262 {
00263 node->gcivy = DRMS_MISSING_DOUBLE;
00264 }
00265 if (gcivz)
00266 {
00267 node->gcivz = *gcivz;
00268 }
00269 else
00270 {
00271 node->gcivz = DRMS_MISSING_DOUBLE;
00272 }
00273
00274 if (gciID && *gciID)
00275 {
00276 snprintf(node->gciID, kIDMaxLen, "%s", gciID);
00277 }
00278 else
00279 {
00280 snprintf(node->gciID, kIDMaxLen, "%s", "unknown");
00281 }
00282 }
00283 }
00284
00285 static VectorNode_t *VCacheLookup(VectorCache_t *cache, const char *tstr)
00286 {
00287 VectorNode_t *node = NULL;
00288
00289 if (tstr && *tstr)
00290 {
00291 node = (VectorNode_t *)hcon_lookup_lower(cache, tstr);
00292 }
00293
00294 return node;
00295 }
00296
00297 static DRMS_Keyword_t *AddKey(DRMS_Record_t *prototype,
00298 DRMS_Type_t type,
00299 const char *name,
00300 const char *format,
00301 const char *unit,
00302 DRMS_RecScopeType_t scope,
00303 const char *desc,
00304 int intprime,
00305 int extprime,
00306 int *rank)
00307 {
00308 DRMS_Keyword_t *tKey = NULL;
00309
00310 tKey = hcon_allocslot_lower(&(prototype->keywords), name);
00311 XASSERT(tKey);
00312 memset(tKey, 0, sizeof(DRMS_Keyword_t));
00313 tKey->info = malloc(sizeof(DRMS_KeywordInfo_t));
00314 XASSERT(tKey->info);
00315 memset(tKey->info, 0, sizeof(DRMS_KeywordInfo_t));
00316
00317
00318
00319 if (tKey && tKey->info)
00320 {
00321
00322 tKey->record = prototype;
00323
00324
00325 snprintf(tKey->info->name,
00326 DRMS_MAXKEYNAMELEN,
00327 "%s",
00328 name);
00329 tKey->info->type = type;
00330 snprintf(tKey->info->format, DRMS_MAXFORMATLEN, "%s", format);
00331 snprintf(tKey->info->unit, DRMS_MAXUNITLEN, "%s", unit);
00332 tKey->info->recscope = scope;
00333
00334 if (intprime)
00335 {
00336 drms_keyword_setintprime(tKey);
00337 }
00338 else
00339 {
00340 drms_keyword_unsetintprime(tKey);
00341 }
00342
00343 if (extprime)
00344 {
00345 drms_keyword_setextprime(tKey);
00346 }
00347 else
00348 {
00349 drms_keyword_unsetextprime(tKey);
00350 }
00351
00352 snprintf(tKey->info->description, DRMS_MAXCOMMENTLEN, "%s", desc);
00353
00354
00355 drms_missing(type, &(tKey->value));
00356
00357
00358 tKey->info->rank = (*rank)++;
00359 tKey->info->kwflags |= (tKey->info->rank + 1) << 16;
00360 }
00361
00362 return tKey;
00363 }
00364
00365 static void CreateOutSeries(DRMS_Env_t *drmsEnv, char *outSeries, const char *owner, int *status)
00366 {
00367 int stat = DRMS_SUCCESS;
00368 int rank;
00369
00370 if (!drms_series_exists(drmsEnv, outSeries, &stat))
00371 {
00372 DRMS_Record_t *prototype = (DRMS_Record_t *)calloc(1, sizeof(DRMS_Record_t));
00373
00374 if (prototype)
00375 {
00376 prototype->seriesinfo = calloc(1, sizeof(DRMS_SeriesInfo_t));
00377
00378 if (prototype->seriesinfo)
00379 {
00380 DRMS_Keyword_t *pkey = NULL;
00381 DRMS_Keyword_t *akey = NULL;
00382 char keyname[DRMS_MAXKEYNAMELEN];
00383
00384 prototype->env = drmsEnv;
00385 prototype->recnum = 0;
00386 prototype->sunum = -1;
00387 prototype->init = 1;
00388 prototype->sessionid = 0;
00389 prototype->sessionns = NULL;
00390 prototype->su = NULL;
00391
00392
00393 hcon_init(&prototype->segments, sizeof(DRMS_Segment_t), DRMS_MAXHASHKEYLEN,
00394 (void (*)(const void *)) drms_free_segment_struct,
00395 (void (*)(const void *, const void *)) drms_copy_segment_struct);
00396
00397 hcon_init(&prototype->links, sizeof(DRMS_Link_t), DRMS_MAXHASHKEYLEN,
00398 (void (*)(const void *)) drms_free_link_struct,
00399 (void (*)(const void *, const void *)) drms_copy_link_struct);
00400
00401 hcon_init(&prototype->keywords, sizeof(DRMS_Keyword_t), DRMS_MAXHASHKEYLEN,
00402 (void (*)(const void *)) drms_free_keyword_struct,
00403 (void (*)(const void *, const void *)) drms_copy_keyword_struct);
00404
00405
00406 snprintf(prototype->seriesinfo->seriesname,
00407 DRMS_MAXSERIESNAMELEN,
00408 "%s",
00409 outSeries);
00410
00411 strcpy(prototype->seriesinfo->author, "Art Amezcua");
00412
00413
00414 if (drmsEnv->session->db_direct)
00415 {
00416 strcpy(prototype->seriesinfo->owner, drmsEnv->session->db_handle->dbuser);
00417 }
00418 else
00419 {
00420 if (owner && strlen(owner) < DRMS_MAXCOMMENTLEN)
00421 {
00422 strcpy(prototype->seriesinfo->owner, owner);
00423 }
00424 else
00425 {
00426 strcpy(prototype->seriesinfo->owner, "unknown");
00427 }
00428 }
00429
00430 prototype->seriesinfo->unitsize = 0;
00431 prototype->seriesinfo->archive = 0;
00432 prototype->seriesinfo->retention = 0;
00433 prototype->seriesinfo->tapegroup = 0;
00434
00435 snprintf(prototype->seriesinfo->description,
00436 DRMS_MAXCOMMENTLEN,
00437 "%s",
00438 "Helio- and geo-centric orbit position and velocity vectors.");
00439
00440
00441 rank = 0;
00442 akey = AddKey(prototype, DRMS_TYPE_TIME, kSVKeyPrimary, "0", "UTC", kRecScopeType_TS_EQ, "Date of prediction", 0, 1, &rank);
00443
00444 snprintf(keyname, sizeof(keyname), "%s_%s", kSVKeyPrimary, "index");
00445 pkey = AddKey(prototype, kIndexKWType, keyname, kIndexKWFormat, "none", kRecScopeType_Index, "Slotted-key index for OBS_DATE", 1, 0, &rank);
00446 drms_keyword_setimplicit(pkey);
00447
00448
00449 snprintf(keyname, sizeof(keyname), "%s_%s", kSVKeyPrimary, "epoch");
00450 akey = AddKey(prototype, DRMS_TYPE_TIME, keyname, "0", "UTC", kRecScopeType_Constant, "MDI epoch - adjusted by 30 seconds to center slots on minutes", 0, 0, &rank);
00451 akey->value.time_val = sscan_time("1993.01.01_00:00:30_UT");
00452
00453
00454 snprintf(keyname, sizeof(keyname), "%s_%s", kSVKeyPrimary, "step");
00455 akey = AddKey(prototype, DRMS_TYPE_TIME, keyname, "%f", "secs", kRecScopeType_Constant, "Slots are 60 seconds wide", 0, 0, &rank);
00456 akey->value.time_val = 60.0;
00457
00458 AddKey(prototype, DRMS_TYPE_DOUBLE, kSVKeyXHELIO, "%f", "km", kRecScopeType_Variable, "X position", 0, 0, &rank);
00459 AddKey(prototype, DRMS_TYPE_DOUBLE, kSVKeyYHELIO, "%f", "km", kRecScopeType_Variable, "Y position", 0, 0, &rank);
00460 AddKey(prototype, DRMS_TYPE_DOUBLE, kSVKeyZHELIO, "%f", "km", kRecScopeType_Variable, "Z position", 0, 0, &rank);
00461 AddKey(prototype, DRMS_TYPE_DOUBLE, kSVKeyVxHELIO, "%f", "km/sec", kRecScopeType_Variable, "Velocity in the X direction", 0, 0, &rank);
00462 AddKey(prototype, DRMS_TYPE_DOUBLE, kSVKeyVyHELIO, "%f", "km/sec", kRecScopeType_Variable, "Velocity in the Y direction", 0, 0, &rank);
00463 AddKey(prototype, DRMS_TYPE_DOUBLE, kSVKeyVzHELIO, "%f", "km/sec", kRecScopeType_Variable, "Velocity in the Z direction", 0, 0, &rank);
00464
00465 akey = AddKey(prototype, DRMS_TYPE_STRING, kSVKey_idHELIO, "%s", "NA", kRecScopeType_Variable, "Record query to identify source file containing heliocentric data", 0, 0, &rank);
00466 copy_string(&(akey->value.string_val), "Unknown");
00467
00468 AddKey(prototype, DRMS_TYPE_DOUBLE, kSVKeyXGEO, "%f", "km", kRecScopeType_Variable, "X position", 0, 0, &rank);
00469 AddKey(prototype, DRMS_TYPE_DOUBLE, kSVKeyYGEO, "%f", "km", kRecScopeType_Variable, "Y position", 0, 0, &rank);
00470 AddKey(prototype, DRMS_TYPE_DOUBLE, kSVKeyZGEO, "%f", "km", kRecScopeType_Variable, "Z position", 0, 0, &rank);
00471 AddKey(prototype, DRMS_TYPE_DOUBLE, kSVKeyVxGEO, "%f", "km/sec", kRecScopeType_Variable, "Velocity in the X direction", 0, 0, &rank);
00472 AddKey(prototype, DRMS_TYPE_DOUBLE, kSVKeyVyGEO, "%f", "km/sec", kRecScopeType_Variable, "Velocity in the Y direction", 0, 0, &rank);
00473 AddKey(prototype, DRMS_TYPE_DOUBLE, kSVKeyVzGEO, "%f", "km/sec", kRecScopeType_Variable, "Velocity in the Z direction", 0, 0, &rank);
00474
00475 akey = AddKey(prototype, DRMS_TYPE_STRING, kSVKey_idGEO, "%s", "NA", kRecScopeType_Variable, "Record query to identify source file containing geocentric data", 0, 0, &rank);
00476 copy_string(&(akey->value.string_val), "Unknown");
00477
00478
00479 AddKey(prototype, DRMS_TYPE_TIME, kSVKeyDATE, "0", "ISO", kRecScopeType_Variable, "Date of orbit-data ingestion; ISO 86\
00480 01", 0, 0, &rank);
00481
00482 prototype->seriesinfo->pidx_keywords[0] = pkey;
00483 prototype->seriesinfo->pidx_num = 1;
00484 }
00485
00486 stat = drms_create_series_fromprototype(&prototype, outSeries, 0);
00487 }
00488 }
00489
00490 if (status)
00491 {
00492 *status = stat;
00493 }
00494 }
00495
00496 static void CreateHistSeries(DRMS_Env_t *drmsEnv, char *histSeries, const char *owner, int *status)
00497 {
00498 int stat = DRMS_SUCCESS;
00499 int rank;
00500
00501 if (!drms_series_exists(drmsEnv, histSeries, &stat))
00502 {
00503 DRMS_Record_t *prototype = (DRMS_Record_t *)calloc(1, sizeof(DRMS_Record_t));
00504
00505 if (prototype)
00506 {
00507 prototype->seriesinfo = calloc(1, sizeof(DRMS_SeriesInfo_t));
00508
00509 if (prototype->seriesinfo)
00510 {
00511 DRMS_Keyword_t *pkey1 = NULL;
00512 DRMS_Keyword_t *pkey2 = NULL;
00513 DRMS_Keyword_t *pkey3 = NULL;
00514 DRMS_Keyword_t *pkey4 = NULL;
00515 DRMS_Keyword_t *akey = NULL;
00516 char keyname[DRMS_MAXKEYNAMELEN];
00517
00518 prototype->env = drmsEnv;
00519 prototype->recnum = 0;
00520 prototype->sunum = -1;
00521 prototype->init = 1;
00522 prototype->sessionid = 0;
00523 prototype->sessionns = NULL;
00524 prototype->su = NULL;
00525
00526
00527 hcon_init(&prototype->segments, sizeof(DRMS_Segment_t), DRMS_MAXHASHKEYLEN,
00528 (void (*)(const void *)) drms_free_segment_struct,
00529 (void (*)(const void *, const void *)) drms_copy_segment_struct);
00530
00531 hcon_init(&prototype->links, sizeof(DRMS_Link_t), DRMS_MAXHASHKEYLEN,
00532 (void (*)(const void *)) drms_free_link_struct,
00533 (void (*)(const void *, const void *)) drms_copy_link_struct);
00534
00535 hcon_init(&prototype->keywords, sizeof(DRMS_Keyword_t), DRMS_MAXHASHKEYLEN,
00536 (void (*)(const void *)) drms_free_keyword_struct,
00537 (void (*)(const void *, const void *)) drms_copy_keyword_struct);
00538
00539
00540 snprintf(prototype->seriesinfo->seriesname,
00541 DRMS_MAXSERIESNAMELEN,
00542 "%s",
00543 histSeries);
00544
00545 strcpy(prototype->seriesinfo->author, "Art Amezcua");
00546
00547
00548 if (drmsEnv->session->db_direct)
00549 {
00550 strcpy(prototype->seriesinfo->owner, drmsEnv->session->db_handle->dbuser);
00551 }
00552 else
00553 {
00554 if (owner && strlen(owner) < DRMS_MAXCOMMENTLEN)
00555 {
00556 strcpy(prototype->seriesinfo->owner, owner);
00557 }
00558 else
00559 {
00560 strcpy(prototype->seriesinfo->owner, "unknown");
00561 }
00562 }
00563
00564 prototype->seriesinfo->unitsize = 0;
00565 prototype->seriesinfo->archive = 0;
00566 prototype->seriesinfo->retention = 0;
00567 prototype->seriesinfo->tapegroup = 0;
00568
00569 snprintf(prototype->seriesinfo->description,
00570 DRMS_MAXCOMMENTLEN,
00571 "%s",
00572 "Ingestion history of helio- and geo-centric orbit prediction data.");
00573
00574
00575 rank = 0;
00576 akey = AddKey(prototype, DRMS_TYPE_TIME, kObsDateKey, "0", "UTC", kRecScopeType_TS_EQ, "Date embedded in orbit file name", 0, 1, &rank);
00577
00578 snprintf(keyname, sizeof(keyname), "%s_%s", kSVKeyPrimary, "index");
00579 pkey1 = AddKey(prototype, kIndexKWType, keyname, kIndexKWFormat, "none", kRecScopeType_Index, "Slotted-key index for OBS_DATE", 1, 0, &rank);
00580 drms_keyword_setimplicit(pkey1);
00581
00582
00583 snprintf(keyname, sizeof(keyname), "%s_%s", kSVKeyPrimary, "epoch");
00584 akey = AddKey(prototype, DRMS_TYPE_TIME, keyname, "0", "UTC", kRecScopeType_Constant, "MDI epoch - adjusted by 12 hours to center slots on days.", 0, 0, &rank);
00585 akey->value.time_val = sscan_time("1993.01.01_12:00:00_UT");
00586
00587
00588 snprintf(keyname, sizeof(keyname), "%s_%s", kSVKeyPrimary, "step");
00589 akey = AddKey(prototype, DRMS_TYPE_TIME, keyname, "%f", "day", kRecScopeType_Constant, "Slots are 1 day wide", 0, 0, &rank);
00590 akey->value.time_val = 86400.0;
00591
00592 pkey2 = AddKey(prototype, DRMS_TYPE_STRING, kFdsProductCompKey, "%s", "none", kRecScopeType_Variable, "FDS data-product component", 1, 1, &rank);
00593 pkey3 = AddKey(prototype, DRMS_TYPE_STRING, kFdsDataFormatKey, "%s", "none", kRecScopeType_Variable, "Format of data file contained in segment", 1, 1, &rank);
00594 pkey4 = AddKey(prototype, DRMS_TYPE_INT, kFileVersionKey, "%d", "none", kRecScopeType_Variable, "FDS data product", 1, 1, &rank);
00595
00596 prototype->seriesinfo->pidx_keywords[0] = pkey1;
00597 prototype->seriesinfo->pidx_keywords[1] = pkey2;
00598 prototype->seriesinfo->pidx_keywords[2] = pkey3;
00599 prototype->seriesinfo->pidx_keywords[3] = pkey4;
00600 prototype->seriesinfo->pidx_num = 4;
00601 }
00602
00603 stat = drms_create_series_fromprototype(&prototype, histSeries, 0);
00604 }
00605 }
00606
00607 if (status)
00608 {
00609 *status = stat;
00610 }
00611 }
00612
00613 static void CloseCachedRecords(HContainer_t **cache)
00614 {
00615 if (cache && *cache)
00616 {
00617 DRMS_RecordSet_t *rs = *((DRMS_RecordSet_t **)hcon_lookup(*cache, "bobmould"));
00618 drms_close_records(rs, DRMS_FREE_RECORD);
00619 rs = NULL;
00620
00621
00622 hcon_destroy(cache);
00623 }
00624 }
00625
00626
00627
00628
00629 static int FetchCachedRecord(DRMS_Env_t *drmsEnv,
00630 char *series,
00631 char *tbuf,
00632 HContainer_t **cache,
00633 DRMS_Record_t **recout)
00634 {
00635 int error = 0;
00636 DRMS_RecordSet_t *rs = NULL;
00637 int stop;
00638 int rehydrated;
00639 DRMS_Record_t **prec = NULL;
00640 DRMS_Record_t *rec = NULL;
00641 char *timestr = NULL;
00642 int status = DRMS_SUCCESS;
00643 DRMS_RecChunking_t cstat = kRecChunking_None;
00644 int nrecs;
00645
00646 if (!cache || !recout)
00647 {
00648 error = 1;
00649 }
00650 else
00651 {
00652 *recout = NULL;
00653
00654 if (*cache == NULL)
00655 {
00656 drms_recordset_setchunksize(gChunkSize);
00657 rs = drms_open_recordset(drmsEnv, series, &status);
00658
00659 if (status != DRMS_SUCCESS || rs == NULL)
00660 {
00661 fprintf(stderr, "drms_open_recordset() failed, query=%s, error=%d. Aborting.\n", series, status);
00662 }
00663 else
00664 {
00665 nrecs = drms_count_records(drmsEnv, series, &status);
00666
00667 if (status != DRMS_SUCCESS)
00668 {
00669 fprintf(stderr, "drms_count_records() failed, query=%s, error=%d. Aborting.\n", series, status);
00670 }
00671 }
00672 }
00673
00674 else
00675 {
00676
00677 rs = *((DRMS_RecordSet_t **)hcon_lookup(*cache, "bobmould"));
00678
00679 if (nrecs > 0)
00680 {
00681 if ((prec = (DRMS_Record_t **)hcon_lookup(*cache, tbuf)) != NULL)
00682 {
00683 *recout = *prec;
00684 }
00685 }
00686 }
00687
00688 if (nrecs > 0)
00689 {
00690
00691 int morerecs = 1;
00692 int newchunk;
00693
00694 while (!prec && !error && morerecs)
00695 {
00696
00697 stop = 0;
00698 rehydrated = 0;
00699
00700
00701
00702
00703 newchunk = 0;
00704 while (!stop && (rec = drms_recordset_fetchnext(drmsEnv, rs, &status, &cstat, &newchunk)) != NULL)
00705 {
00706 if (newchunk)
00707 {
00708 if (*cache)
00709 {
00710 hcon_destroy(cache);
00711 }
00712
00713 *cache = hcon_create(sizeof(DRMS_Record_t *), gCacheKeySize, NULL, NULL, NULL, NULL, 0);
00714
00715
00716 hcon_insert(*cache, "bobmould", &rs);
00717 }
00718
00719 if (cstat == kRecChunking_LastInChunk || cstat == kRecChunking_LastInRS)
00720 {
00721
00722 stop = 1;
00723 }
00724
00725
00726 timestr = drms_getkey_string(rec, kObsDateKey, &status);
00727 if (timestr)
00728 {
00729 hcon_insert(*cache, timestr, &rec);
00730 rehydrated = 1;
00731 }
00732 else
00733 {
00734 error = 1;
00735 break;
00736 }
00737 }
00738
00739 if (cstat == kRecChunking_NoMoreRecs)
00740 {
00741 morerecs = 0;
00742 }
00743
00744
00745 if (rehydrated)
00746 {
00747 if ((prec = (DRMS_Record_t **)hcon_lookup(*cache, tbuf)) != NULL)
00748 {
00749 *recout = *prec;
00750 }
00751 }
00752 }
00753 }
00754 else if (*cache == NULL)
00755 {
00756 *cache = hcon_create(sizeof(DRMS_Record_t *), gCacheKeySize, NULL, NULL, NULL, NULL, 0);
00757
00758
00759 hcon_insert(*cache, "bobmould", &rs);
00760 }
00761 }
00762
00763 #if DEBUG
00764 if (!prec)
00765 {
00766 fprintf(stderr, "Record for time '%s' not found in series '%s'.\n", tbuf, series);
00767 }
00768 #endif
00769
00770 return error;
00771 }
00772
00773 static int ParseSVRecFields(char *recBuf, char **date, double *xVal, double *yVal, double *zVal,
00774 double *vxVal, double *vyVal, double *vzVal)
00775 {
00776 int error = 0;
00777 char *token;
00778 char *line = strdup(recBuf);
00779
00780 if (line != NULL)
00781 {
00782 token = strtok(line, " ");
00783
00784 if (token != NULL)
00785 {
00786
00787 char year[8];
00788 char day[8];
00789 char hour[8];
00790 char minute[8];
00791 char second[8];
00792
00793 strncpy(year, token, 4);
00794 year[4] = '\0';
00795 strncpy(day, &token[4], 3);
00796 day[3] = '\0';
00797 strncpy(hour, &token[8], 2);
00798 hour[2] = '\0';
00799 strncpy(minute, &token[10], 2);
00800 minute[2] = '\0';
00801 strncpy(second, &token[12], 2);
00802 second[2] = '\0';
00803
00804 char timeBuf[64];
00805 snprintf(timeBuf, sizeof(timeBuf), "%s.01.%s_%s:%s:%s_UT", year, day, hour, minute, second);
00806
00807 *date = strdup(timeBuf);
00808
00809 if ((token = strtok(NULL, " ")) != NULL)
00810 {
00811 sscanf(token, "%lf", xVal);
00812 }
00813 else
00814 {
00815 error = 1;
00816 }
00817
00818 if (error == 0)
00819 {
00820 if ((token = strtok(NULL, " ")) != NULL)
00821 {
00822 sscanf(token, "%lf", yVal);
00823 }
00824 else
00825 {
00826 error = 1;
00827 }
00828 }
00829
00830 if (error == 0)
00831 {
00832 if((token = strtok(NULL, " ")) != NULL)
00833 {
00834 sscanf(token, "%lf", zVal);
00835 }
00836 else
00837 {
00838 error = 1;
00839 }
00840 }
00841
00842 if (error == 0)
00843 {
00844 if ((token = strtok(NULL, " ")) != NULL)
00845 {
00846 sscanf(token, "%lf", vxVal);
00847 }
00848 else
00849 {
00850 error = 1;
00851 }
00852 }
00853
00854 if (error == 0)
00855 {
00856 if ((token = strtok(NULL, " ")) != NULL)
00857 {
00858 sscanf(token, "%lf", vyVal);
00859 }
00860 else
00861 {
00862 error = 1;
00863 }
00864 }
00865
00866 if (error == 0)
00867 {
00868 if ((token = strtok(NULL, " ")) != NULL)
00869 {
00870 sscanf(token, "%lf", vzVal);
00871 }
00872 else
00873 {
00874 error = 1;
00875 }
00876 }
00877 }
00878 }
00879 else
00880 {
00881 error = 1;
00882 }
00883
00884 return error;
00885 }
00886
00887 static int IsDiff(double v1, double v2)
00888 {
00889 if (!drms_ismissing_double(v1) && !drms_ismissing_double(v2))
00890 {
00891 return (fabs(v1 - v2) > 1.0e-11 * (fabs(v1) + fabs(v2)));
00892 }
00893 else if (!drms_ismissing_double(v1) || !drms_ismissing_double(v2))
00894 {
00895 return 1;
00896 }
00897
00898 return 0;
00899 }
00900
00901 static int ExtractStateVectors(DRMS_Env_t *drmsEnv,
00902 char *filePathHELIO,
00903 char *idHELIO,
00904 char *filePathGEO,
00905 char *idGEO,
00906 char *outSeries,
00907 const char *owner)
00908 {
00909 int stat = DRMS_SUCCESS;
00910 int error = 0;
00911 char *obsDate = NULL;
00912 double xValHel;
00913 double yValHel;
00914 double zValHel;
00915 double vxValHel;
00916 double vyValHel;
00917 double vzValHel;
00918 double xValGeo;
00919 double yValGeo;
00920 double zValGeo;
00921 double vxValGeo;
00922 double vyValGeo;
00923 double vzValGeo;
00924
00925 int addedRecsHELIO = 0;
00926 int addedRecsGEO = 0;
00927
00928 HContainer_t *existRCache = NULL;
00929 DRMS_Record_t *existRec = NULL;
00930 VectorCache_t *helioOutVCache = NULL;
00931 VectorCache_t *outVCache = NULL;
00932
00933 outVCache = CreateVectorCache();
00934
00935 #if DEBUG
00936 Hash_Table_t hash;
00937 hash_init(&hash, 49999, 0, (int (*)(const void *, const void *))strcmp, hash_universal_hash);
00938 #endif
00939
00940 #if DEBUG
00941 int throttle = 0;
00942 #endif
00943
00944
00945 FILE *datafp = NULL;
00946
00947 if (filePathHELIO);
00948 {
00949 datafp = fopen(filePathHELIO, "r");
00950 if (datafp == NULL)
00951 {
00952 error = 1;
00953 fprintf(stderr, "Could not open %s for reading\n", filePathHELIO);
00954 }
00955 }
00956
00957 if (datafp)
00958 {
00959 char lineBuf[kSVLineMax];
00960 int oneMore = -1;
00961
00962 CreateOutSeries(drmsEnv, outSeries, owner, &stat);
00963
00964 if (filePathGEO)
00965 {
00966
00967
00968
00969
00970
00971
00972
00973 helioOutVCache = CreateVectorCache();
00974 }
00975
00976
00977 #if DEBUG
00978 TIMER_t *timer = CreateTimer();
00979 int nitems = 0;
00980 #endif
00981
00982 while (!error && fgets(lineBuf, kSVLineMax, datafp) != NULL)
00983 {
00984 if (oneMore == -1)
00985 {
00986 if (strlen(lineBuf) >= 4)
00987 {
00988 if (strncmp(lineBuf, "Time", 4) == 0)
00989 {
00990 oneMore = 1;
00991 }
00992 }
00993
00994 continue;
00995 }
00996
00997 if (oneMore > 0)
00998 {
00999 oneMore--;
01000 continue;
01001 }
01002
01003 ParseSVRecFields(lineBuf,
01004 &obsDate,
01005 &xValHel,
01006 &yValHel,
01007 &zValHel,
01008 &vxValHel,
01009 &vyValHel,
01010 &vzValHel);
01011
01012 TIME od = sscan_time(obsDate);
01013 char tbuf[128];
01014 sprint_time(tbuf, od, "UTC", 0);
01015
01016 if (helioOutVCache)
01017 {
01018 #if DEBUG
01019 if (nitems < 10000)
01020 {
01021 #endif
01022
01023 #if DEBUG
01024
01025 char *buff = malloc(64);
01026 snprintf(buff, 64, "test%d", nitems);
01027 TIMER_t *timer10 = CreateTimer();
01028 hash_insert(&hash, buff, (void *)(buff));
01029 hash_lookup(&hash, buff);
01030 fprintf(stdout, "hash insert + lookup %f seconds.\n", GetElapsedTime(timer10));
01031 DestroyTimer(&timer10);
01032 #else
01033 VCacheCache(helioOutVCache,
01034 tbuf,
01035 &xValHel,
01036 &yValHel,
01037 &zValHel,
01038 &vxValHel,
01039 &vyValHel,
01040 &vzValHel,
01041 idHELIO,
01042 NULL,
01043 NULL,
01044 NULL,
01045 NULL,
01046 NULL,
01047 NULL,
01048 NULL);
01049 #endif
01050
01051 #if DEBUG
01052 }
01053 #endif
01054 }
01055 else
01056 {
01057
01058
01059
01060
01061
01062
01063
01064
01065
01066
01067
01068
01069 char filtered[DRMS_MAXQUERYLEN];
01070 snprintf(filtered, sizeof(filtered), "%s[? %s >= $(%s) ?]", outSeries, kObsDateKey, tbuf);
01071
01072 if (FetchCachedRecord(drmsEnv, filtered, tbuf, &existRCache, &existRec))
01073 {
01074 double xValHelSav = drms_getkey_double(existRec, kSVKeyXHELIO, &stat);
01075 double yValHelSav = drms_getkey_double(existRec, kSVKeyYHELIO, &stat);
01076 double zValHelSav = drms_getkey_double(existRec, kSVKeyZHELIO, &stat);
01077 double vxValHelSav = drms_getkey_double(existRec, kSVKeyVxHELIO, &stat);
01078 double vyValHelSav = drms_getkey_double(existRec, kSVKeyVyHELIO, &stat);
01079 double vzValHelSav = drms_getkey_double(existRec, kSVKeyVzHELIO, &stat);
01080
01081 char *idh = drms_getkey_string(existRec, kSVKey_idHELIO, &stat);
01082
01083 double xValGeoSav = drms_getkey_double(existRec, kSVKeyXGEO, &stat);
01084 double yValGeoSav = drms_getkey_double(existRec, kSVKeyYGEO, &stat);
01085 double zValGeoSav = drms_getkey_double(existRec, kSVKeyZGEO, &stat);
01086 double vxValGeoSav = drms_getkey_double(existRec, kSVKeyVxGEO, &stat);
01087 double vyValGeoSav = drms_getkey_double(existRec, kSVKeyVyGEO, &stat);
01088 double vzValGeoSav = drms_getkey_double(existRec, kSVKeyVzGEO, &stat);
01089
01090 char *id = drms_getkey_string(existRec, kSVKey_idGEO, &stat);
01091
01092 if (IsDiff(xValHelSav, xValHel) || IsDiff(yValHelSav, yValHel) ||
01093 IsDiff(zValHelSav, zValHel) || IsDiff(vxValHelSav, vxValHel) ||
01094 IsDiff(vyValHelSav, vyValHel) || IsDiff(vzValHelSav, vzValHel))
01095 {
01096
01097
01098 VCacheCache(outVCache,
01099 tbuf,
01100 &xValHel,
01101 &yValHel,
01102 &zValHel,
01103 &vxValHel,
01104 &vyValHel,
01105 &vzValHel,
01106 idHELIO,
01107 &xValGeoSav,
01108 &yValGeoSav,
01109 &zValGeoSav,
01110 &vxValGeoSav,
01111 &vyValGeoSav,
01112 &vzValGeoSav,
01113 id);
01114 }
01115
01116 if (idh)
01117 {
01118 free(idh);
01119 }
01120
01121 if (id)
01122 {
01123 free(id);
01124 }
01125 }
01126 else
01127 {
01128
01129
01130 VCacheCache(outVCache,
01131 tbuf,
01132 &xValHel,
01133 &yValHel,
01134 &zValHel,
01135 &vxValHel,
01136 &vyValHel,
01137 &vzValHel,
01138 idHELIO,
01139 NULL,
01140 NULL,
01141 NULL,
01142 NULL,
01143 NULL,
01144 NULL,
01145 NULL);
01146 }
01147 }
01148
01149 if (!filePathGEO)
01150 {
01151 addedRecsHELIO++;
01152 }
01153
01154 #if DEBUG
01155 nitems++;
01156 #endif
01157
01158 #if DEBUG
01159 throttle++;
01160 #endif
01161
01162 #if DEBUG
01163
01164 if (throttle == 10)
01165 {
01166 break;
01167 }
01168 #endif
01169
01170 }
01171
01172 #if DEBUG
01173 fprintf(stdout, "Helio ingest time: %f seconds.\n", GetElapsedTime(timer));
01174 DestroyTimer(&timer);
01175 #endif
01176
01177 fclose(datafp);
01178 datafp = NULL;
01179 }
01180
01181
01182
01183
01184 CloseCachedRecords(&existRCache);
01185 existRec = NULL;
01186
01187
01188
01189 if (error == 0);
01190 {
01191 if (filePathGEO)
01192 {
01193 datafp = fopen(filePathGEO, "r");
01194 if (datafp == NULL)
01195 {
01196 error = 1;
01197 fprintf(stderr, "Could not open %s for reading\n", filePathGEO);
01198 }
01199 }
01200 }
01201
01202 if (datafp)
01203 {
01204 char lineBuf[kSVLineMax];
01205 int oneMore = -1;
01206 int helioexist;
01207
01208 #if DEBUG
01209 TIMER_t *timer = CreateTimer();
01210 #endif
01211
01212 while (!error && fgets(lineBuf, kSVLineMax, datafp) != NULL)
01213 {
01214 if (oneMore == -1)
01215 {
01216 if (strlen(lineBuf) >= 4)
01217 {
01218 if (strncmp(lineBuf, "Time", 4) == 0)
01219 {
01220 oneMore = 1;
01221 }
01222 }
01223
01224 continue;
01225 }
01226
01227 if (oneMore > 0)
01228 {
01229 oneMore--;
01230 continue;
01231 }
01232
01233 ParseSVRecFields(lineBuf,
01234 &obsDate,
01235 &xValGeo,
01236 &yValGeo,
01237 &zValGeo,
01238 &vxValGeo,
01239 &vyValGeo,
01240 &vzValGeo);
01241
01242
01243 helioexist = 0;
01244 char *idHELIOtmp = NULL;
01245 VectorNode_t *hrec = NULL;
01246
01247 TIME od = sscan_time(obsDate);
01248 char tbuf[128];
01249 sprint_time(tbuf, od, "UTC", 0);
01250
01251 if (helioOutVCache)
01252 {
01253 hrec = VCacheLookup(helioOutVCache, tbuf);
01254 }
01255
01256
01257 if (hrec)
01258 {
01259 xValHel = hrec->hcix;
01260 yValHel = hrec->hciy;
01261 zValHel = hrec->hciz;
01262 vxValHel = hrec->hcivx;
01263 vyValHel = hrec->hcivy;
01264 vzValHel = hrec->hcivz;
01265 idHELIOtmp = hrec->hciID;
01266 helioexist = 1;
01267 }
01268
01269
01270
01271
01272
01273
01274
01275
01276
01277
01278
01279
01280
01281
01282
01283 char filtered[DRMS_MAXQUERYLEN];
01284 snprintf(filtered, sizeof(filtered), "%s[? %s >= $(%s) ?]", outSeries, kObsDateKey, tbuf);
01285
01286 if (FetchCachedRecord(drmsEnv, filtered, tbuf, &existRCache, &existRec))
01287 {
01288 int heliodiff = 0;
01289 int geodiff = 0;
01290
01291 double xValHelSav = drms_getkey_double(existRec, kSVKeyXHELIO, &stat);
01292 double yValHelSav = drms_getkey_double(existRec, kSVKeyYHELIO, &stat);
01293 double zValHelSav = drms_getkey_double(existRec, kSVKeyZHELIO, &stat);
01294 double vxValHelSav = drms_getkey_double(existRec, kSVKeyVxHELIO, &stat);
01295 double vyValHelSav = drms_getkey_double(existRec, kSVKeyVyHELIO, &stat);
01296 double vzValHelSav = drms_getkey_double(existRec, kSVKeyVzHELIO, &stat);
01297
01298 char *id = drms_getkey_string(existRec, kSVKey_idHELIO, &stat);
01299
01300 double xValGeoSav = drms_getkey_double(existRec, kSVKeyXGEO, &stat);
01301 double yValGeoSav = drms_getkey_double(existRec, kSVKeyYGEO, &stat);
01302 double zValGeoSav = drms_getkey_double(existRec, kSVKeyZGEO, &stat);
01303 double vxValGeoSav = drms_getkey_double(existRec, kSVKeyVxGEO, &stat);
01304 double vyValGeoSav = drms_getkey_double(existRec, kSVKeyVyGEO, &stat);
01305 double vzValGeoSav = drms_getkey_double(existRec, kSVKeyVzGEO, &stat);
01306
01307 char *idg = drms_getkey_string(existRec, kSVKey_idGEO, &stat);
01308
01309
01310 geodiff = (IsDiff(xValGeoSav, xValGeo) || IsDiff(yValGeoSav, yValGeo) ||
01311 IsDiff(zValGeoSav, zValGeo) || IsDiff(vxValGeoSav, vxValGeo) ||
01312 IsDiff(vyValGeoSav, vyValGeo) || IsDiff(vzValGeoSav, vzValGeo));
01313
01314 if (helioexist)
01315 {
01316
01317 heliodiff = (IsDiff(xValHelSav, xValHel) || IsDiff(yValHelSav, yValHel) ||
01318 IsDiff(zValHelSav, zValHel) || IsDiff(vxValHelSav, vxValHel) ||
01319 IsDiff(vyValHelSav, vyValHel) || IsDiff(vzValHelSav, vzValHel));
01320 }
01321
01322 if (geodiff)
01323 {
01324
01325 if (!helioexist || !heliodiff)
01326 {
01327
01328 VCacheCache(outVCache,
01329 tbuf,
01330 &xValHelSav,
01331 &yValHelSav,
01332 &zValHelSav,
01333 &vxValHelSav,
01334 &vyValHelSav,
01335 &vzValHelSav,
01336 id,
01337 &xValGeo,
01338 &yValGeo,
01339 &zValGeo,
01340 &vxValGeo,
01341 &vyValGeo,
01342 &vzValGeo,
01343 idGEO);
01344 }
01345 else
01346 {
01347
01348
01349 VCacheCache(outVCache,
01350 tbuf,
01351 &xValHel,
01352 &yValHel,
01353 &zValHel,
01354 &vxValHel,
01355 &vyValHel,
01356 &vzValHel,
01357 idHELIOtmp,
01358 &xValGeo,
01359 &yValGeo,
01360 &zValGeo,
01361 &vxValGeo,
01362 &vyValGeo,
01363 &vzValGeo,
01364 idGEO);
01365 }
01366 }
01367 else
01368 {
01369
01370
01371 if (helioexist && heliodiff)
01372 {
01373 VCacheCache(outVCache,
01374 tbuf,
01375 &xValHel,
01376 &yValHel,
01377 &zValHel,
01378 &vxValHel,
01379 &vyValHel,
01380 &vzValHel,
01381 idHELIOtmp,
01382 &xValGeoSav,
01383 &yValGeoSav,
01384 &zValGeoSav,
01385 &vxValGeoSav,
01386 &vyValGeoSav,
01387 &vzValGeoSav,
01388 idg);
01389 }
01390 }
01391
01392 if (id)
01393 {
01394 free(id);
01395 }
01396
01397 if (idg)
01398 {
01399 free(idg);
01400 }
01401 }
01402 else
01403 {
01404
01405 if (helioexist)
01406 {
01407 VCacheCache(outVCache,
01408 tbuf,
01409 &xValHel,
01410 &yValHel,
01411 &zValHel,
01412 &vxValHel,
01413 &vyValHel,
01414 &vzValHel,
01415 idHELIOtmp,
01416 &xValGeo,
01417 &yValGeo,
01418 &zValGeo,
01419 &vxValGeo,
01420 &vyValGeo,
01421 &vzValGeo,
01422 idGEO);
01423 }
01424 else
01425 {
01426 VCacheCache(outVCache,
01427 tbuf,
01428 NULL,
01429 NULL,
01430 NULL,
01431 NULL,
01432 NULL,
01433 NULL,
01434 NULL,
01435 &xValGeo,
01436 &yValGeo,
01437 &zValGeo,
01438 &vxValGeo,
01439 &vyValGeo,
01440 &vzValGeo,
01441 idGEO);
01442 }
01443 }
01444
01445 addedRecsGEO++;
01446
01447 if (helioexist)
01448 {
01449 addedRecsHELIO++;
01450 }
01451
01452 #if DEBUG
01453 throttle--;
01454 if (throttle == 0)
01455 {
01456 break;
01457 }
01458 #endif
01459 }
01460
01461 #if DEBUG
01462 fprintf(stdout, "Geo ingest time: %f seconds.\n", GetElapsedTime(timer));
01463 DestroyTimer(&timer);
01464 #endif
01465
01466 fclose(datafp);
01467 datafp = NULL;
01468 }
01469
01470 CloseCachedRecords(&existRCache);
01471
01472
01473 HIterator_t *cachehit = hiter_create((HContainer_t *)outVCache);
01474 DRMS_RecordSet_t *rsout = drms_create_records(drmsEnv,
01475 outVCache->num_total,
01476 outSeries,
01477 DRMS_PERMANENT,
01478 &stat);
01479 DRMS_Record_t *recout = NULL;
01480 int irec = 0;
01481 VectorNode_t *node = NULL;
01482
01483 if (rsout && rsout->n > 0)
01484 {
01485 while ((node = (VectorNode_t *)hiter_getnext(cachehit)) != NULL)
01486 {
01487 recout = rsout->records[irec];
01488
01489 drms_setkey_string(recout, kSVKeyPrimary, node->tobs);
01490 drms_setkey_double(recout, kSVKeyXHELIO, node->hcix);
01491 drms_setkey_double(recout, kSVKeyYHELIO, node->hciy);
01492 drms_setkey_double(recout, kSVKeyZHELIO, node->hciz);
01493 drms_setkey_double(recout, kSVKeyVxHELIO, node->hcivx);
01494 drms_setkey_double(recout, kSVKeyVyHELIO, node->hcivy);
01495 drms_setkey_double(recout, kSVKeyVzHELIO, node->hcivz);
01496 drms_setkey_string(recout, kSVKey_idHELIO, node->hciID);
01497
01498 drms_setkey_double(recout, kSVKeyXGEO, node->gcix);
01499 drms_setkey_double(recout, kSVKeyYGEO, node->gciy);
01500 drms_setkey_double(recout, kSVKeyZGEO, node->gciz);
01501 drms_setkey_double(recout, kSVKeyVxGEO, node->gcivx);
01502 drms_setkey_double(recout, kSVKeyVyGEO, node->gcivy);
01503 drms_setkey_double(recout, kSVKeyVzGEO, node->gcivz);
01504 drms_setkey_string(recout, kSVKey_idGEO, node->gciID);
01505
01506
01507 drms_keyword_setdate(recout);
01508
01509 irec++;
01510 }
01511
01512 error = (drms_close_records(rsout, DRMS_INSERT_RECORD) != DRMS_SUCCESS);
01513 }
01514 else
01515 {
01516 fprintf(stderr, "Failed to create records in series '%s'.\n", outSeries);
01517 error = 1;
01518 }
01519
01520 DestroyVectorCache(&outVCache);
01521 DestroyVectorCache(&helioOutVCache);
01522
01523 if (!error)
01524 {
01525 if (filePathHELIO && filePathGEO)
01526 {
01527 fprintf(stdout,
01528 "Ingested '%s' (%d recs) and '%s' (%d recs).\n",
01529 filePathHELIO,
01530 addedRecsHELIO,
01531 filePathGEO,
01532 addedRecsGEO);
01533 }
01534 else if (filePathHELIO)
01535 {
01536 fprintf(stdout, "Ingested '%s' (%d recs).\n", filePathHELIO, addedRecsHELIO);
01537 }
01538 else
01539 {
01540 fprintf(stdout, "Ingested '%s' (%d recs).\n", filePathGEO, addedRecsGEO);
01541 }
01542 }
01543
01544 return error;
01545 }
01546
01547 int DoIt(void)
01548 {
01549 int error = 0;
01550 int stat = DRMS_SUCCESS;
01551
01552 if (drms_env == NULL)
01553 {
01554 error = 1;
01555 }
01556 else
01557 {
01558 char seriesin[DRMS_MAXSERIESNAMELEN];
01559 char seriesout[DRMS_MAXSERIESNAMELEN];
01560 char serieshist[DRMS_MAXSERIESNAMELEN];
01561 char *fdsTRange = NULL;
01562
01563 const char *ns = cmdparams_get_str(&cmdparams, "ns", NULL);
01564 const char *si = cmdparams_get_str(&cmdparams, "seriesIn", NULL);
01565 const char *so = cmdparams_get_str(&cmdparams, "seriesOut", NULL);
01566 const char *timeRange = cmdparams_get_str(&cmdparams, "timeRange", NULL);
01567 const char *owner = cmdparams_get_str(&cmdparams, "owner", NULL);
01568
01569 if (strcmp(owner, kNotSpecified) == 0)
01570 {
01571 owner = NULL;
01572 }
01573
01574 snprintf(seriesin, sizeof(seriesin), "%s.%s", ns, si);
01575 snprintf(seriesout, sizeof(seriesout), "%s.%s", ns, so);
01576 snprintf(serieshist, sizeof(serieshist), "%s.%s", ns, kSeriesHistory);
01577
01578 if (cmdparams_isflagset(&cmdparams, "c"))
01579 {
01580 CreateOutSeries(drms_env, seriesout, owner, &stat);
01581 return 0;
01582 }
01583
01584
01585 if (error == 0)
01586 {
01587 if (strcmp(timeRange, "NOT SPECIFIED") != 0)
01588 {
01589 size_t len = strlen(kObsDateKey) + strlen(timeRange) + 3;
01590 fdsTRange = malloc(sizeof(char) * len + 1);
01591 if (fdsTRange != NULL)
01592 {
01593 snprintf(fdsTRange, len + 1, "[%s=%s]", kObsDateKey, timeRange);
01594 }
01595 else
01596 {
01597 error = 1;
01598 }
01599 }
01600 }
01601
01602 if (error == 0)
01603 {
01604
01605 DRMS_RecordSet_t *rsInput = NULL;
01606 DRMS_RecordSet_t *rsHistory = NULL;
01607 DRMS_RecordSet_t *rsHELIO = NULL;
01608 DRMS_RecordSet_t *rsGEO = NULL;
01609 char queryInput[DRMS_MAXQUERYLEN];
01610 char queryHistory[DRMS_MAXQUERYLEN];
01611 char *filePathHELIO = NULL;
01612 char *filePathGEO = NULL;
01613 char queryCORE[DRMS_MAXQUERYLEN];
01614 char queryHELIO[DRMS_MAXQUERYLEN];
01615 char queryGEO[DRMS_MAXQUERYLEN];
01616 char hashbuf[kMaxHashKey];
01617 char path[DRMS_MAXPATHLEN];
01618
01619 char *obs = NULL;
01620 char *prodcomp = NULL;
01621 char *format = NULL;
01622 int filevers;
01623
01624 int history = 0;
01625
01626 HContainer_t *proclist = NULL;
01627
01628
01629 if (drms_series_exists(drms_env, serieshist, &stat))
01630 {
01631
01632
01633
01634
01635
01636
01637
01638
01639
01640 history = 1;
01641 }
01642
01643 if (fdsTRange)
01644 {
01645 snprintf(queryInput,
01646 sizeof(queryInput),
01647 "%s[%s=%s]%s,%s[%s=%s]%s",
01648 seriesin,
01649 kFdsProductCompKey,
01650 kFdsProductCompHELIO,
01651 fdsTRange,
01652 seriesin,
01653 kFdsProductCompKey,
01654 kFdsProductCompGEO,
01655 fdsTRange);
01656 }
01657 else
01658 {
01659 snprintf(queryInput,
01660 sizeof(queryInput),
01661 "%s[%s=%s],%s[%s=%s]",
01662 seriesin,
01663 kFdsProductCompKey,
01664 kFdsProductCompHELIO,
01665 seriesin,
01666 kFdsProductCompKey,
01667 kFdsProductCompGEO);
01668 }
01669
01670
01671 proclist = hcon_create(kMaxHashKey,
01672 kMaxHashKey,
01673 NULL,
01674 NULL,
01675 NULL,
01676 NULL,
01677 0);
01678
01679 rsInput = drms_open_records(drms_env, queryInput, &stat);
01680 if (rsInput && rsInput->n > 0)
01681 {
01682 DRMS_Record_t *recin = NULL;
01683 int irec;
01684
01685 for (irec = 0; irec < rsInput->n && !error; irec++)
01686 {
01687 recin = rsInput->records[irec];
01688
01689 if (recin)
01690 {
01691 obs = drms_getkey_string(recin, kObsDateKey, &stat);
01692 prodcomp = drms_getkey_string(recin, kFdsProductCompKey, &stat);
01693 format = drms_getkey_string(recin, kFdsDataFormatKey, &stat);
01694 filevers = drms_getkey_int(recin, kFileVersionKey, &stat);
01695
01696
01697 if (history)
01698 {
01699 snprintf(queryHistory,
01700 sizeof(queryHistory),
01701 "%s[%s=%s][%s=%s][%s=%s][%s=%d]",
01702 serieshist,
01703 kObsDateKey,
01704 obs,
01705 kFdsProductCompKey,
01706 prodcomp,
01707 kFdsDataFormatKey,
01708 format,
01709 kFileVersionKey,
01710 filevers);
01711
01712 rsHistory = drms_open_records(drms_env, queryHistory, &stat);
01713 if (rsHistory && rsHistory->n > 0)
01714 {
01715
01716 drms_close_records(rsHistory, DRMS_FREE_RECORD);
01717 continue;
01718 }
01719 else
01720 {
01721 drms_close_records(rsHistory, DRMS_FREE_RECORD);
01722 }
01723 }
01724
01725
01726 snprintf(hashbuf,
01727 sizeof(hashbuf),
01728 "%s[%s=%s][%s=%s][%s=%d]",
01729 seriesin,
01730 kObsDateKey,
01731 obs,
01732 kFdsDataFormatKey,
01733 format,
01734 kFileVersionKey,
01735 filevers);
01736
01737 if (hcon_member(proclist, hashbuf))
01738 {
01739 continue;
01740 }
01741
01742
01743
01744 snprintf(queryCORE,
01745 sizeof(queryCORE),
01746 "%s[%s=%s][%s=%s][%s=%d]",
01747 seriesin,
01748 kObsDateKey,
01749 obs,
01750 kFdsDataFormatKey,
01751 format,
01752 kFileVersionKey,
01753 filevers);
01754
01755 hcon_insert(proclist, queryCORE, queryCORE);
01756 }
01757
01758 if (obs)
01759 {
01760 free(obs);
01761 }
01762
01763 if (prodcomp)
01764 {
01765 free(prodcomp);
01766 }
01767
01768 if (format)
01769 {
01770 free(format);
01771 }
01772
01773 }
01774
01775 drms_close_records(rsInput, DRMS_FREE_RECORD);
01776 }
01777
01778
01779
01780 HIterator_t *hit = hiter_create(proclist);
01781 char *iquery = NULL;
01782
01783 while ((iquery = (char *)hiter_getnext(hit)) != NULL && !error)
01784 {
01785 #if 0
01786
01787
01788
01789 if (strncmp(iquery, "###", 3) == 0)
01790 {
01791
01792
01793
01794
01795 if (hcon_member(proclist, iquery[3]))
01796 {
01797
01798
01799
01800 continue;
01801 }
01802 else
01803 {
01804
01805
01806
01807 oktoadd = 0;
01808 iquery = iquery[3];
01809 }
01810 }
01811 #endif
01812 snprintf(queryHELIO,
01813 sizeof(queryHELIO),
01814 "%s[%s=%s]",
01815 iquery,
01816 kFdsProductCompKey,
01817 kFdsProductCompHELIO);
01818
01819 snprintf(queryGEO,
01820 sizeof(queryGEO),
01821 "%s[%s=%s]",
01822 iquery,
01823 kFdsProductCompKey,
01824 kFdsProductCompGEO);
01825
01826 DRMS_Record_t *recsv = NULL;
01827 TIME obsvalHELIO = 0;
01828 char *dprodvalHELIO = NULL;
01829 char *prodcompvalHELIO = NULL;
01830 char *formatvalHELIO = NULL;
01831 int fileversvalHELIO = 0;
01832 TIME obsvalGEO = 0;
01833 char *dprodvalGEO = NULL;
01834 char *prodcompvalGEO = NULL;
01835 char *formatvalGEO = NULL;
01836 int fileversvalGEO = 0;
01837
01838 rsHELIO = drms_open_records(drms_env, queryHELIO, &stat);
01839
01840 if (rsHELIO && stat != DRMS_ERROR_UNKNOWNSERIES)
01841 {
01842 if (rsHELIO->n != 1)
01843 {
01844 fprintf(stderr, "Expected only one record with query '%s'\n", queryHELIO);
01845 error = 1;
01846 }
01847 else
01848 {
01849
01850 DRMS_Record_t *recHELIO = rsHELIO->records[0];
01851
01852 DRMS_Segment_t *helioseg = drms_segment_lookup(recHELIO, kSVFileSegName);
01853
01854 if (helioseg != NULL)
01855 {
01856 drms_record_directory(recHELIO, path, 1);
01857
01858 if (*path)
01859 {
01860 size_t len = strlen(path) + strlen(helioseg->filename) + 1;
01861 filePathHELIO = malloc(sizeof(char) * len + 1);
01862 if (filePathHELIO)
01863 {
01864 snprintf(filePathHELIO, len + 1, "%s/%s", path, helioseg->filename);
01865 }
01866 }
01867 }
01868
01869 if (filePathHELIO)
01870 {
01871 recsv = rsHELIO->records[0];
01872 obsvalHELIO = drms_getkey_time(recsv, kObsDateKey, &stat);
01873 dprodvalHELIO = drms_getkey_string(recsv, kFdsDataProductKey, &stat);
01874 prodcompvalHELIO = drms_getkey_string(recsv, kFdsProductCompKey, &stat);
01875 formatvalHELIO = drms_getkey_string(recsv, kFdsDataFormatKey, &stat);
01876 fileversvalHELIO = drms_getkey_int(recsv, kFileVersionKey, &stat);
01877 }
01878 }
01879 }
01880
01881 if (!error)
01882 {
01883 rsGEO = drms_open_records(drms_env, queryGEO, &stat);
01884 }
01885
01886 if (rsGEO && stat != DRMS_ERROR_UNKNOWNSERIES)
01887 {
01888 if (rsGEO->n != 1)
01889 {
01890 fprintf(stderr, "Expected only one record with query '%s'\n", queryGEO);
01891 error = 1;
01892 }
01893 else
01894 {
01895
01896 DRMS_Record_t *recGEO = rsGEO->records[0];
01897
01898 DRMS_Segment_t *geoseg = drms_segment_lookup(recGEO, kSVFileSegName);
01899
01900 if (geoseg != NULL)
01901 {
01902 drms_record_directory(recGEO, path, 1);
01903
01904 if (*path)
01905 {
01906 size_t len = strlen(path) + strlen(geoseg->filename) + 1;
01907 filePathGEO = malloc(sizeof(char) * len + 1);
01908 if (filePathGEO)
01909 {
01910 snprintf(filePathGEO, len + 1, "%s/%s", path, geoseg->filename);
01911 }
01912 }
01913 }
01914
01915 if (filePathGEO)
01916 {
01917 recsv = rsGEO->records[0];
01918 obsvalGEO = drms_getkey_time(recsv, kObsDateKey, &stat);
01919 dprodvalGEO = drms_getkey_string(recsv, kFdsDataProductKey, &stat);
01920 prodcompvalGEO = drms_getkey_string(recsv, kFdsProductCompKey, &stat);
01921 formatvalGEO = drms_getkey_string(recsv, kFdsDataFormatKey, &stat);
01922 fileversvalGEO = drms_getkey_int(recsv, kFileVersionKey, &stat);
01923 }
01924 }
01925 }
01926
01927 if (!error)
01928 {
01929
01930 char idHELIO[256];
01931 char idGEO[256];
01932
01933 if (rsHELIO && filePathHELIO)
01934 {
01935 char *datestr = drms_getkey_string(rsHELIO->records[0],
01936 kObsDateKey,
01937 &stat);
01938
01939
01940 snprintf(idHELIO,
01941 sizeof(idHELIO),
01942 "%s[%s][%s][%s][%d]",
01943 rsHELIO->records[0]->seriesinfo->seriesname,
01944 datestr,
01945 prodcompvalHELIO,
01946 formatvalHELIO,
01947 fileversvalHELIO);
01948
01949 if (datestr)
01950 {
01951 free(datestr);
01952 }
01953 }
01954
01955 if (rsGEO && filePathGEO)
01956 {
01957 char *datestr = drms_getkey_string(rsGEO->records[0],
01958 kObsDateKey,
01959 &stat);
01960
01961
01962 snprintf(idGEO,
01963 sizeof(idGEO),
01964 "%s[%s][%s][%s][%d]",
01965 rsGEO->records[0]->seriesinfo->seriesname,
01966 datestr,
01967 prodcompvalGEO,
01968 formatvalGEO,
01969 fileversvalGEO);
01970
01971 if (datestr)
01972 {
01973 free(datestr);
01974 }
01975 }
01976
01977 if (filePathHELIO || filePathGEO)
01978 {
01979 error = ExtractStateVectors(drms_env,
01980 filePathHELIO,
01981 idHELIO,
01982 filePathGEO,
01983 idGEO,
01984 seriesout,
01985 owner);
01986 }
01987 }
01988
01989 if (!error)
01990 {
01991
01992
01993
01994 if (!history)
01995 {
01996
01997 CreateHistSeries(drms_env, serieshist, owner, &stat);
01998 if (!stat)
01999 {
02000 history = 1;
02001 }
02002 }
02003
02004 int nhist = 0;
02005 int ihist = 0;
02006
02007 if (filePathHELIO)
02008 {
02009 nhist++;
02010 }
02011
02012 if (filePathGEO)
02013 {
02014 nhist++;
02015 }
02016
02017 if (nhist > 0)
02018 {
02019 DRMS_RecordSet_t *rshist = drms_create_records(drms_env,
02020 nhist,
02021 serieshist,
02022 DRMS_PERMANENT,
02023 &stat);
02024 DRMS_Record_t *rechist = NULL;
02025
02026 if (rshist)
02027 {
02028 if (filePathHELIO)
02029 {
02030 rechist = rshist->records[ihist++];
02031 drms_setkey_time(rechist, kObsDateKey, obsvalHELIO);
02032 drms_setkey_string(rechist, kFdsDataProductKey, dprodvalHELIO);
02033 drms_setkey_string(rechist, kFdsProductCompKey, prodcompvalHELIO);
02034 drms_setkey_string(rechist, kFdsDataFormatKey, formatvalHELIO);
02035 drms_setkey_int(rechist, kFileVersionKey, fileversvalHELIO);
02036 }
02037
02038 if (filePathGEO)
02039 {
02040 rechist = rshist->records[ihist++];
02041 drms_setkey_time(rechist, kObsDateKey, obsvalGEO);
02042 drms_setkey_string(rechist, kFdsDataProductKey, dprodvalGEO);
02043 drms_setkey_string(rechist, kFdsProductCompKey, prodcompvalGEO);
02044 drms_setkey_string(rechist, kFdsDataFormatKey, formatvalGEO);
02045 drms_setkey_int(rechist, kFileVersionKey, fileversvalGEO);
02046 }
02047
02048 drms_close_records(rshist, DRMS_INSERT_RECORD);
02049 }
02050 }
02051 }
02052
02053 if (rsHELIO)
02054 {
02055 drms_close_records(rsHELIO, DRMS_FREE_RECORD);
02056 rsHELIO = NULL;
02057 }
02058
02059 if (rsGEO)
02060 {
02061 drms_close_records(rsGEO, DRMS_FREE_RECORD);
02062 rsGEO = NULL;
02063 }
02064
02065 if (dprodvalHELIO)
02066 {
02067 free(dprodvalHELIO);
02068 }
02069
02070 if (prodcompvalHELIO)
02071 {
02072 free(prodcompvalHELIO);
02073 }
02074
02075 if (formatvalHELIO)
02076 {
02077 free(formatvalHELIO);
02078 }
02079
02080 if (dprodvalGEO)
02081 {
02082 free(dprodvalGEO);
02083 }
02084
02085 if (prodcompvalGEO)
02086 {
02087 free(prodcompvalGEO);
02088 }
02089
02090 if (formatvalGEO)
02091 {
02092 free(formatvalGEO);
02093 }
02094
02095 }
02096
02097 hiter_destroy(&hit);
02098 hcon_destroy(&proclist);
02099 }
02100
02101 if (fdsTRange)
02102 {
02103 free(fdsTRange);
02104 }
02105 }
02106
02107 return error;
02108 }