2 tplarson 1.1
3 int jretile_maxmem(void)
4 {
5 int newstat = 0;
6 int status = DRMS_SUCCESS;
7
8 char *inrecquery = NULL;
9 char *outseries = NULL;
10 char *segnamein = NULL;
11 char *segnameout = NULL;
12 DRMS_RecordSet_t *inrecset = NULL;
13 DRMS_RecordSet_t *outrecset = NULL;
14 int irecin, irecout, nrecsin=0, nrecsout=0, nlchunks;
15 DRMS_Record_t *inrec = NULL;
16 DRMS_Record_t *outrec = NULL;
17 DRMS_Segment_t *segin = NULL;
18 DRMS_Segment_t *segout = NULL;
19 DRMS_Array_t *inarr = NULL;
20 DRMS_Array_t *outarr = NULL;
21 DRMS_RecLifetime_t lifetime;
22 DRMS_Type_t usetype = DRMS_TYPE_FLOAT;
23 tplarson 1.1 int length[2], startind[2], endind[2];
24 float *inptr, *outptr;
25 long long histrecnum=-1;
26 int quality;
27 int mapmmax=-1;
28 int sinbdivs=-1;
29 double cadence=0;
30 int i;
31
32 TIME tnow, UNIX_epoch = -220924792.000; /* 1970.01.01_00:00:00_UTC */
33 char tstartstr[100];
34
35 double tstart, tepoch, tstep, tround, tstop, tstartin, tstopin, tstepin, tstartuse, tstopuse, nseconds, chunksecs;
36 char *ttotal, *tchunk;
37 int ndt, ndtin;
38 int lmin, lmax, lminin, lmaxin, lminuse, lmaxuse, lchunk, lchunksize, lchunkfirst, lchunklast;
39 int ntimechunks, nmodes, npts, imode, itime;
40 int out_time_offset, out_modes_offset, out_offset, in_time_offset, in_modes_offset, in_offset, out_index, in_index;
41 int iset, lminout, lmaxout;
42 double tstartout, tstopout;
43 float *arrptr;
44 tplarson 1.1 int nrecords, nsegments, nskip;
45
46 int errbufstat=setvbuf(stderr, NULL, _IONBF, BUFSIZ);
47 int outbufstat=setvbuf(stdout, NULL, _IONBF, BUFSIZ);
48
49 double wt0, wt1, wt2, wt3, wt;
50 double ut0, ut1, ut2, ut3, ut;
51 double st0, st1, st2, st3, st;
52 double ct0, ct1, ct2, ct3, ct;
53
54 wt0=getwalltime();
55 ct0=getcputime(&ut0, &st0);
56
57 inrecquery = (char *)cmdparams_save_str(&cmdparams, "in", &newstat);
58 outseries = (char *)cmdparams_save_str(&cmdparams, "out", &newstat);
59 segnamein = (char *)cmdparams_save_str(&cmdparams, "segin", &newstat);
60 segnameout = (char *)cmdparams_save_str(&cmdparams, "segout", &newstat);
61 int seginflag = strcmp(kNOTSPECIFIED, segnamein);
62 int segoutflag = strcmp(kNOTSPECIFIED, segnameout);
63 int verbflag = cmdparams_save_int(&cmdparams, "VERB", &newstat);
64 int permflag = cmdparams_save_int(&cmdparams, "PERM", &newstat);
65 tplarson 1.1 if (permflag)
66 lifetime = DRMS_PERMANENT;
67 else
68 lifetime = DRMS_TRANSIENT;
69 unsigned short calverkey = (unsigned short)cmdparams_save_int(&cmdparams, "CALVERKEY", &newstat);
70
71 char *histlinkname = (char *)cmdparams_save_str(&cmdparams, "histlink", &newstat);
72
73 tstart=cmdparams_save_time(&cmdparams, "TSTART", &newstat);
74 ttotal=(char *)cmdparams_save_str(&cmdparams, "TTOTAL", &newstat);
75 status=drms_names_parseduration(&ttotal, &nseconds, 1);
76 if (status != DRMS_SUCCESS)
77 {
78 // newstat = newstat | CPSAVE_UNKNOWN_ERROR;
79 fprintf(stderr, "ERROR: problem parsing TTOTAL, = %s\n", ttotal);
80 return 1;
81 }
82 tchunk=(char *)cmdparams_save_str(&cmdparams, "TCHUNK", &newstat);
83 if (strcmp(kNOTSPECIFIED, tchunk))
84 {
85 status=drms_names_parseduration(&tchunk, &chunksecs, 1);
86 tplarson 1.1 if (status != DRMS_SUCCESS)
87 newstat = newstat | CPSAVE_UNKNOWN_ERROR;
88 }
89 else
90 chunksecs=0;
91
92 lmin=cmdparams_save_int(&cmdparams, "LMIN", &newstat);
93 lmax=cmdparams_save_int(&cmdparams, "LMAX", &newstat);
94 lchunksize=cmdparams_save_int(&cmdparams, "LCHUNK", &newstat);
95 if (lchunksize == 0)
96 lchunksize=lmax+1;
97
98 if (newstat)
99 {
100 fprintf(stderr, "ERROR: problem with input arguments, status = %d, diagnosis follows\n", newstat);
101 cpsave_decode_error(newstat);
102 return 1;
103 }
104 else if (savestrlen != strlen(savestr))
105 {
106 fprintf(stderr, "ERROR: problem with savestr, savestrlen = %d, strlen(savestr) = %d\n", savestrlen, (int)strlen(savestr));
107 tplarson 1.1 return 1;
108 }
109
110 DRMS_Record_t *tempoutrec = drms_create_record(drms_env,
111 outseries,
112 DRMS_TRANSIENT,
113 &status);
114
115 if (status != DRMS_SUCCESS)
116 {
117 fprintf(stderr,"ERROR: couldn't open a record in output dataseries %s, status = %d\n", outseries, status);
118 return 1;
119 }
120
121 // set up ancillary dataseries for processing metadata
122 DRMS_Link_t *histlink = hcon_lookup_lower(&tempoutrec->links, histlinkname);
123 if (histlink != NULL)
124 {
125 histrecnum=set_history(histlink);
126 if (histrecnum < 0)
127 {
128 tplarson 1.1 drms_close_record(tempoutrec, DRMS_FREE_RECORD);
129 return 1;
130 }
131 }
132 else
133 {
134 fprintf(stderr,"WARNING: could not find history link in output dataseries\n");
135 }
136
137 // these must be present in the output dataseries and variable, not links or constants
138 // now done in DoIt() that calls this function
139 /*
140 char *outchecklist[] = {"T_START", "QUALITY", "LMIN", "LMAX", "NDT"};
141 int itest;
142 for (itest=0; itest < ARRLENGTH(outchecklist); itest++)
143 {
144 DRMS_Keyword_t *outkeytest = hcon_lookup_lower(&tempoutrec->keywords, outchecklist[itest]);
145 if (outkeytest == NULL || outkeytest->info->islink || outkeytest->info->recscope == 1)
146 {
147 fprintf(stderr, "ERROR: output keyword %s is either missing, constant, or a link\n", outchecklist[itest]);
148 drms_close_record(tempoutrec, DRMS_FREE_RECORD);
149 tplarson 1.1 return 1;
150 }
151 }
152 */
153
154 tepoch=drms_getkey_time(tempoutrec, "T_START_epoch", &status);
155 tstep=drms_getkey_float(tempoutrec, "T_START_step", &status);
156 tround=drms_getkey_float(tempoutrec, "T_START_round", &status);
157 cadence=drms_getkey_float(tempoutrec, "T_STEP", &status);
158 if (fmod(tstart-tepoch,tstep) > tround/2)
159 {
160 fprintf(stderr, "ERROR: output dataseries seems incompatible with input parameters (tstep must divide tstart-tepoch): tstart = %f, tepoch = %f, tstep = %f\n", tstart, tepoch, tstep);
161 drms_close_record(tempoutrec, DRMS_FREE_RECORD);
162 return 1;
163 }
164 if (chunksecs == 0.0)
165 chunksecs = tstep;
166 else if (fmod(chunksecs,tstep))
167 {
168 fprintf(stderr, "ERROR: output dataseries seems incompatible with input parameters (tstep must divide chunksecs): chunksecs = %f, tstep = %f\n", chunksecs, tstep);
169 drms_close_record(tempoutrec, DRMS_FREE_RECORD);
170 tplarson 1.1 return 1;
171 }
172 if (fmod(nseconds,chunksecs) != 0.0)
173 {
174 fprintf(stderr, "ERROR: input parameters seem incompatible (chunksecs must divide nseconds): nseconds = %f, chunksecs = %f\n", nseconds, chunksecs);
175 drms_close_record(tempoutrec, DRMS_FREE_RECORD);
176 return 1;
177 }
178 ntimechunks=nseconds/chunksecs;
179 ndt=chunksecs/cadence;
180 if (verbflag)
181 {
182 printf("%d timechunks, %.1f seconds per chunk\n", ntimechunks, chunksecs);
183 }
184
185 int mapmmaxout=-1;
186 int sinbdivsout=-1;
187 DRMS_Keyword_t *outkeytest = hcon_lookup_lower(&tempoutrec->keywords, "MAPMMAX");
188 if (outkeytest != NULL && outkeytest->info->recscope == 1)
189 mapmmaxout=drms_getkey_int(tempoutrec, "MAPMMAX", &status);
190 outkeytest = hcon_lookup_lower(&tempoutrec->keywords, "SINBDIVS");
191 tplarson 1.1 if (outkeytest != NULL && outkeytest->info->recscope == 1)
192 sinbdivsout=drms_getkey_int(tempoutrec, "SINBDIVS", &status);
193
194 drms_close_record(tempoutrec, DRMS_FREE_RECORD);
195
196
197 char *inchecklist[] = {"T_START", "QUALITY", "LMIN", "LMAX", "T_STEP"};
198 DRMS_Keyword_t *inkeytest;
199 int itest;
200 inrecset = drms_open_records(drms_env, inrecquery, &status);
201 // inrecset = drms_open_recordset(drms_env, inrecquery, &status);
202
203 if (status != DRMS_SUCCESS || inrecset == NULL)
204 {
205 fprintf(stderr, "ERROR: problem opening input recordset: status = %d\n", status);
206 return 1;
207 }
208 nrecsin = inrecset->n;
209
210 if (verbflag)
211 printf("input recordset opened, nrecs = %d\n", nrecsin);
212 tplarson 1.1
213 int noinput=0;
214 if (nrecsin == 0)
215 {
216 printf("WARNING: input recordset contains no records\n");
217 noinput=1;
218 goto skip1;
219 // return 1;
220 }
221
222 inrec = inrecset->records[0];
223
224 for (itest=0; itest < ARRLENGTH(inchecklist); itest++)
225 {
226 inkeytest = hcon_lookup_lower(&inrec->keywords, inchecklist[itest]);
227 if (inkeytest == NULL)
228 {
229 fprintf(stderr, "ERROR: required input keyword %s is missing\n", inchecklist[itest]);
230 drms_close_records(inrecset, DRMS_FREE_RECORD);
231 return 1;
232 }
233 tplarson 1.1 }
234
235 if (cadence != drms_getkey_float(inrec, "T_STEP", &status))
236 {
237 fprintf(stderr, "ERROR: input T_STEP does not equal output T_STEP\n");
238 drms_close_records(inrecset, DRMS_FREE_RECORD);
239 return 1;
240 }
241
242 inkeytest = hcon_lookup_lower(&inrec->keywords, "MAPMMAX");
243 if (inkeytest != NULL)
244 mapmmax=drms_getkey_int(inrec, "MAPMMAX", &status);
245 if (mapmmaxout != -1 && mapmmaxout != mapmmax)
246 {
247 fprintf(stderr, "ERROR: input MAPMMAX does not equal output MAPMMAX, in=%d, out=%d\n", mapmmax, mapmmaxout);
248 drms_close_records(inrecset, DRMS_FREE_RECORD);
249 return 1;
250 }
251
252 inkeytest = hcon_lookup_lower(&inrec->keywords, "SINBDIVS");
253 if (outkeytest != NULL)
254 tplarson 1.1 sinbdivs=drms_getkey_int(inrec, "SINBDIVS", &status);
255 if (sinbdivsout != -1 && sinbdivsout != sinbdivs)
256 {
257 fprintf(stderr, "ERROR: input SINBDIVS does not equal output SINBDIVS, in=%d, out=%d\n", sinbdivs, sinbdivsout);
258 drms_close_records(inrecset, DRMS_FREE_RECORD);
259 return 1;
260 }
261
262
263 status=drms_stage_records(inrecset, 1, 0);
264 if (status != DRMS_SUCCESS)
265 {
266 fprintf(stderr, "ERROR: drms_stage_records returned status = %d\n", status);
267 return 1;
268 }
269
270 skip1:
271
272 nrecords=0;
273 nsegments=0;
274 nskip=0;
275 tplarson 1.1
276 lchunkfirst = lmin/lchunksize;
277 lchunklast = lmax/lchunksize;
278
279 nlchunks = (lchunklast - lchunkfirst) + 1;
280 nrecsout = nlchunks*ntimechunks;
281 outrecset = drms_create_records(drms_env, nrecsout, outseries, lifetime, &status);
282 if (status != DRMS_SUCCESS || outrecset == NULL)
283 {
284 fprintf(stderr,"ERROR: unable to create records record in output dataseries %s, status = %d\n", outseries, status);
285 drms_close_records(inrecset, DRMS_FREE_RECORD);
286 return 1;
287 }
288
289 unsigned long long calversout, calvers;
290 int calversunset=1;
291 int mixflag=0;
292 unsigned int nybblearrout[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
293 int fixflagarr[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
294 for (i=0;i<16;i++)
295 {
296 tplarson 1.1 if (getbits(calverkey,i,1))
297 fixflagarr[i]=1;
298 }
299
300 int *firstflagarr=(int *)malloc(nrecsin*sizeof(int));
301 for (irecin=0; irecin < nrecsin; irecin++)
302 firstflagarr[irecin]=1;
303 DRMS_Array_t **inarrarr=(DRMS_Array_t **)malloc(nrecsin*sizeof(DRMS_Array_t *));
304
305 irecout=0;
306 for (iset=0; iset < ntimechunks; iset++)
307 {
308 tstartout=tstart + iset * chunksecs;
309 tstopout=tstartout+chunksecs;
310 sprint_time(tstartstr, tstartout, "TAI", 0);
311
312 if (verbflag)
313 {
314 wt1=getwalltime();
315 ct1=getcputime(&ut1, &st1);
316 printf("processing timechunk %d, tstart = %s\n", iset, tstartstr);
317 tplarson 1.1 }
318
319 for (lchunk = lchunkfirst; lchunk <= lchunklast; lchunk++)
320 {
321 lminout = lchunk * lchunksize;
322 lmaxout = lminout + lchunksize - 1;
323 lminout = MAXIMUM(lminout,lmin);
324 lmaxout = MINIMUM(lmaxout,lmax);
325
326 outrec = outrecset->records[irecout++];
327 if (histlink != NULL)
328 drms_setlink_static(outrec, histlinkname, histrecnum);
329
330 if (verbflag > 1)
331 {
332 wt2=getwalltime();
333 ct2=getcputime(&ut2, &st2);
334 printf(" processing lchunk %d, min = %d, max = %d\n", lchunk-lchunkfirst, lminout, lmaxout);
335 }
336
337 if (noinput)
338 tplarson 1.1 {
339 goto skip2;
340 }
341
342 if (segoutflag)
343 segout = drms_segment_lookup(outrec, segnameout);
344 else
345 segout = drms_segment_lookupnum(outrec, 0);
346
347 length[0]=2*ndt;
348 length[1]=lmaxout*(lmaxout+1)/2+lmaxout - lminout*(lminout+1)/2 + 1;
349 arrptr=(float *)(calloc(length[0]*length[1],sizeof(float)));
350 outarr = drms_array_create(usetype, 2, length, arrptr, &status);
351 if (status != DRMS_SUCCESS || outarr == NULL || segout == NULL)
352 {
353 fprintf(stderr,"ERROR: problem creating output array or segment: length = [%d, %d], status = %d, histrecnum = %lld", length[0], length[1], status, histrecnum);
354 drms_close_records(inrecset, DRMS_FREE_RECORD);
355 drms_close_records(outrecset, DRMS_FREE_RECORD);
356 return 0;
357 }
358 outptr = (float *)(outarr->data);
359 tplarson 1.1
360 nskip=0;
361 mixflag=0;
362 calversunset=1;
363 for (irecin=0; irecin < nrecsin; irecin++)
364 {
365 inrec = inrecset->records[irecin];
366 quality=drms_getkey_int(inrec, "QUALITY", &status);
367 if (status != DRMS_SUCCESS || (quality & QUAL_NODATA)) //may want stricter test on quality here
368 {
|
370 tplarson 1.1 nskip++;
371 continue;
372 }
373
374 tstartin=drms_getkey_time(inrec, "T_START", &status);
375 tstopin=drms_getkey_time(inrec, "T_STOP", &status);
376 lminin=drms_getkey_int(inrec, "LMIN", &status);
377 lmaxin=drms_getkey_int(inrec, "LMAX", &status);
378 ndtin=(tstopin-tstartin)/cadence;
379
380 if (tstartin >= tstopout || tstopin <= tstartout || lminin > lmaxout || lmaxin < lminout)
381 {
382 nskip++;
383 continue;
384 }
385
386 if (calversunset)
387 {
388 calversout=drms_getkey_longlong(inrec, "CALVER64", &status);
389 if (status != DRMS_SUCCESS)
390 calversout = 0;
391 tplarson 1.1 calversunset=0;
392 for (i=0;i<16;i++)
393 nybblearrout[i]=getbits(calversout,4*i+3,4);
394 }
395
396 if (firstflagarr[irecin])
397 {
398 calvers=drms_getkey_longlong(inrec, "CALVER64", &status);
399 if (status != DRMS_SUCCESS)
400 calvers = 0;
401
402 for (i=0;i<16;i++)
403 {
404 int nybble=getbits(calvers,4*i+3,4);
405 if (fixflagarr[i])
406 {
407 if (nybble != nybblearrout[i])
408 {
409 fprintf(stderr, "ERROR: input data has mixed values for field %d of CALVER64: %d and %d, recnum = %lld, histrecnum = %lld\n", i, nybblearrout[i], nybble, inrec->recnum, histrecnum);
410 return 0;
411 }
412 tplarson 1.1 }
413 else
414 {
415 if (nybble < nybblearrout[i])
416 nybblearrout[i]=nybble;
417 }
418 }
419
420 if (!mixflag && (quality & QUAL_MIXEDCALVER || calvers != calversout))
421 mixflag=1;
422
423 if (seginflag)
424 segin = drms_segment_lookup(inrec, segnamein);
425 else
426 segin = drms_segment_lookupnum(inrec, 0);
427 if (segin != NULL)
428 inarr = drms_segment_read(segin, usetype, &status);
429 if (status != DRMS_SUCCESS || inarr == NULL || segin == NULL)
430 {
431 fprintf(stderr, "ERROR: problem reading input segment: tstart = %f, lmin = %d, lmax = %d, irecin = %d, status = %d, histrecnum = %lld\n", tstartin, lminin, lmaxin, irecin, status, histrecnum);
432 drms_close_records(inrecset, DRMS_FREE_RECORD);
433 tplarson 1.1 drms_close_records(outrecset, DRMS_FREE_RECORD);
434 return 0;
435 }
436 else
437 {
438 inarrarr[irecin]=inarr;
439 }
440 firstflagarr[irecin]=0;
441 }
442
443 inptr = (float *)(inarrarr[irecin])->data;
444
445 tstartuse=MAXIMUM(tstartout, tstartin);
446 tstopuse= MINIMUM(tstopout, tstopin);
447 lminuse=MAXIMUM(lminout, lminin);
448 lmaxuse=MINIMUM(lmaxout, lmaxin);
449 nmodes=MODES(lmaxuse+1)-MODES(lminuse);
450 npts=(tstopuse - tstartuse)/cadence;
451
452 out_time_offset = (tstartuse - tstartout)/cadence;
453 out_modes_offset = MODES(lminuse) - MODES(lminout);
454 tplarson 1.1 out_offset = 2 * (out_modes_offset * ndt + out_time_offset);
455 in_time_offset = (tstartuse - tstartin)/cadence;
456 in_modes_offset = MODES(lminuse) - MODES(lminin);
457 in_offset = 2 * (in_modes_offset * ndtin + in_time_offset);
458
459 for (imode=0; imode<nmodes; imode++)
460 {
461 for (itime=0; itime<npts; itime++)
462 {
463 in_index=in_offset + 2*itime;
464 out_index=out_offset + 2*itime;
465 outptr[out_index] = inptr[in_index];
466 outptr[out_index+1] = inptr[in_index+1];
467 }
468 out_offset+=2*ndt;
469 in_offset+=2*ndtin;
470 }
471
472 }
473
474 if (nskip == nrecsin)
475 tplarson 1.1 {
476 fprintf(stderr, "WARNING: no inputs for this output, no segment written\n");
477 }
478 else
479 {
480 outarr->bzero=segout->bzero;
481 outarr->bscale=segout->bscale;
482 status=drms_segment_write(segout, outarr, 0);
483 nsegments++;
484 if (status != DRMS_SUCCESS)
485 {
486 fprintf(stderr, "ERROR: problem writing output segment: status = %d, T_START = %s, LMIN = %d, LMAX = %d, histrecnum = %lld\n", status, tstartstr, lminout, lmaxout, histrecnum);
487 drms_close_records(inrecset, DRMS_FREE_RECORD);
488 drms_close_records(outrecset, DRMS_FREE_RECORD);
489 return 0;
490 }
491 }
492 drms_free_array(outarr);
493
494 drms_copykeys(outrec, inrec, 0, kDRMS_KeyClass_Explicit);
495 // copykeys takes care of MAPMMAX, SINBDIVS, etc. that should be constant across the input
496 tplarson 1.1 skip2:
497
498 drms_setkey_int(outrec, "LMIN", lminout);
499 drms_setkey_int(outrec, "LMAX", lmaxout);
500 drms_setkey_time(outrec, "T_START", tstartout);
501 drms_setkey_time(outrec, "T_STOP", tstopout);
502 drms_setkey_time(outrec, "T_OBS", tstartout+chunksecs/2);
503 drms_setkey_int(outrec, "NDT", ndt);
504
505 for (i=0;i<16;i++)
506 calversout=setbits(calversout,4*i+3,4,nybblearrout[i]);
507 drms_setkey_longlong(outrec, "CALVER64", calversout);
508
509 if (noinput || nskip == nrecsin)
510 drms_setkey_int(outrec, "QUALITY", QUAL_NODATA);
511 else if (mixflag)
512 drms_setkey_int(outrec, "QUALITY", QUAL_MIXEDCALVER);
513 else
514 drms_setkey_int(outrec, "QUALITY", 0);
515
516 tnow = (double)time(NULL);
517 tplarson 1.1 tnow += UNIX_epoch;
518 drms_setkey_time(outrec, "DATE", tnow);
519
520 nrecords++;
521
522 if (verbflag > 1)
523 {
524 wt=getwalltime();
525 ct=getcputime(&ut, &st);
526 fprintf(stdout, " lchunk %d done: %.2f ms wall time, %.2f ms cpu time\n", lchunk-lchunkfirst,
527 wt-wt2,
528 ct-ct2);
529 }
530
531 }
532
533 if (verbflag)
534 {
535 wt=getwalltime();
536 ct=getcputime(&ut, &st);
537 fprintf(stdout, "timechunk %d done: %.2f ms wall time, %.2f ms cpu time\n", iset,
538 tplarson 1.1 wt-wt1,
539 ct-ct1);
540
541 }
542
543 }
544
545 for (irecin=0; irecin < nrecsin; irecin++)
546 drms_free_array(inarrarr[irecin]);
547 free(inarrarr);
548 free(firstflagarr);
549
550 drms_close_records(inrecset, DRMS_FREE_RECORD);
551 drms_close_records(outrecset, DRMS_INSERT_RECORD);
552
553 wt=getwalltime();
554 ct=getcputime(&ut, &st);
555 if (verbflag)
556 {
557 printf("number of records created = %d\n", nrecords);
558 printf("number of segments created = %d\n", nsegments);
559 tplarson 1.1 fprintf(stdout, "total time spent: %.2f ms wall time, %.2f ms cpu time\n",
560 wt-wt0, ct-ct0);
561 }
562
563 printf("module %s successful completion\n", cmdparams.argv[0]);
564
565 return 0;
566
567 }
|
Return to
jretile_maxmem.c
CVS log
Up to [Development] / JSOC / proj / globalhs / apps