2 tplarson 1.1
3 int jretile_maxmem(void)
4 {
5 int newstat = 0;
6 int status = DRMS_SUCCESS;
7
8 char *inrecquery = NULL;
9 char *outseries = NULL;
10 char *segnamein = NULL;
11 char *segnameout = NULL;
12 DRMS_RecordSet_t *inrecset = NULL;
13 DRMS_RecordSet_t *outrecset = NULL;
14 int irecin, irecout, nrecsin=0, nrecsout=0, nlchunks;
15 DRMS_Record_t *inrec = NULL;
16 DRMS_Record_t *outrec = NULL;
17 DRMS_Segment_t *segin = NULL;
18 DRMS_Segment_t *segout = NULL;
19 DRMS_Array_t *inarr = NULL;
20 DRMS_Array_t *outarr = NULL;
21 DRMS_RecLifetime_t lifetime;
22 DRMS_Type_t usetype = DRMS_TYPE_FLOAT;
23 tplarson 1.1 int length[2], startind[2], endind[2];
24 float *inptr, *outptr;
25 long long histrecnum=-1;
26 int quality;
27 int mapmmax=-1;
28 int sinbdivs=-1;
29 double cadence=0;
30 int i;
31
32 TIME tnow, UNIX_epoch = -220924792.000; /* 1970.01.01_00:00:00_UTC */
33 char tstartstr[100];
34
35 double tstart, tepoch, tstep, tround, tstop, tstartin, tstopin, tstepin, tstartuse, tstopuse, nseconds, chunksecs;
36 char *ttotal, *tchunk;
37 int ndt, ndtin;
38 int lmin, lmax, lminin, lmaxin, lminuse, lmaxuse, lchunk, lchunksize, lchunkfirst, lchunklast;
39 int ntimechunks, nmodes, npts, imode, itime;
40 int out_time_offset, out_modes_offset, out_offset, in_time_offset, in_modes_offset, in_offset, out_index, in_index;
41 int iset, lminout, lmaxout;
42 double tstartout, tstopout;
43 float *arrptr;
44 tplarson 1.1 int nrecords, nsegments, nskip;
45
46 int errbufstat=setvbuf(stderr, NULL, _IONBF, BUFSIZ);
47 int outbufstat=setvbuf(stdout, NULL, _IONBF, BUFSIZ);
48
49 double wt0, wt1, wt2, wt3, wt;
50 double ut0, ut1, ut2, ut3, ut;
51 double st0, st1, st2, st3, st;
52 double ct0, ct1, ct2, ct3, ct;
53
54 wt0=getwalltime();
55 ct0=getcputime(&ut0, &st0);
56
57 inrecquery = (char *)cmdparams_save_str(&cmdparams, "in", &newstat);
58 outseries = (char *)cmdparams_save_str(&cmdparams, "out", &newstat);
59 segnamein = (char *)cmdparams_save_str(&cmdparams, "segin", &newstat);
60 segnameout = (char *)cmdparams_save_str(&cmdparams, "segout", &newstat);
61 int seginflag = strcmp(kNOTSPECIFIED, segnamein);
62 int segoutflag = strcmp(kNOTSPECIFIED, segnameout);
63 int verbflag = cmdparams_save_int(&cmdparams, "VERB", &newstat);
64 int permflag = cmdparams_save_int(&cmdparams, "PERM", &newstat);
65 tplarson 1.1 if (permflag)
66 lifetime = DRMS_PERMANENT;
67 else
68 lifetime = DRMS_TRANSIENT;
69 unsigned short calverkey = (unsigned short)cmdparams_save_int(&cmdparams, "CALVERKEY", &newstat);
70
71 char *histlinkname = (char *)cmdparams_save_str(&cmdparams, "histlink", &newstat);
72
73 tstart=cmdparams_save_time(&cmdparams, "TSTART", &newstat);
74 ttotal=(char *)cmdparams_save_str(&cmdparams, "TTOTAL", &newstat);
75 status=drms_names_parseduration(&ttotal, &nseconds, 1);
76 if (status != DRMS_SUCCESS)
77 {
78 // newstat = newstat | CPSAVE_UNKNOWN_ERROR;
79 fprintf(stderr, "ERROR: problem parsing TTOTAL, = %s\n", ttotal);
80 return 1;
81 }
82 tchunk=(char *)cmdparams_save_str(&cmdparams, "TCHUNK", &newstat);
83 if (strcmp(kNOTSPECIFIED, tchunk))
84 {
85 status=drms_names_parseduration(&tchunk, &chunksecs, 1);
86 tplarson 1.1 if (status != DRMS_SUCCESS)
87 newstat = newstat | CPSAVE_UNKNOWN_ERROR;
88 }
89 else
90 chunksecs=0;
91
92 lmin=cmdparams_save_int(&cmdparams, "LMIN", &newstat);
93 lmax=cmdparams_save_int(&cmdparams, "LMAX", &newstat);
94 lchunksize=cmdparams_save_int(&cmdparams, "LCHUNK", &newstat);
95 if (lchunksize == 0)
96 lchunksize=lmax+1;
97
98 if (newstat)
99 {
100 fprintf(stderr, "ERROR: problem with input arguments, status = %d, diagnosis follows\n", newstat);
101 cpsave_decode_error(newstat);
102 return 1;
103 }
104 else if (savestrlen != strlen(savestr))
105 {
106 fprintf(stderr, "ERROR: problem with savestr, savestrlen = %d, strlen(savestr) = %d\n", savestrlen, (int)strlen(savestr));
107 tplarson 1.1 return 1;
108 }
109
110 DRMS_Record_t *tempoutrec = drms_create_record(drms_env,
111 outseries,
112 DRMS_TRANSIENT,
113 &status);
114
115 if (status != DRMS_SUCCESS)
116 {
117 fprintf(stderr,"ERROR: couldn't open a record in output dataseries %s, status = %d\n", outseries, status);
118 return 1;
119 }
120
121 // set up ancillary dataseries for processing metadata
122 DRMS_Link_t *histlink = hcon_lookup_lower(&tempoutrec->links, histlinkname);
123 if (histlink != NULL)
124 {
125 histrecnum=set_history(histlink);
126 if (histrecnum < 0)
127 {
128 tplarson 1.1 drms_close_record(tempoutrec, DRMS_FREE_RECORD);
129 return 1;
130 }
131 }
132 else
133 {
134 fprintf(stderr,"WARNING: could not find history link in output dataseries\n");
135 }
136
137 // these must be present in the output dataseries and variable, not links or constants
138 // now done in DoIt() that calls this function
139 /*
140 char *outchecklist[] = {"T_START", "QUALITY", "LMIN", "LMAX", "NDT"};
141 int itest;
142 for (itest=0; itest < ARRLENGTH(outchecklist); itest++)
143 {
144 DRMS_Keyword_t *outkeytest = hcon_lookup_lower(&tempoutrec->keywords, outchecklist[itest]);
145 if (outkeytest == NULL || outkeytest->info->islink || outkeytest->info->recscope == 1)
146 {
147 fprintf(stderr, "ERROR: output keyword %s is either missing, constant, or a link\n", outchecklist[itest]);
148 drms_close_record(tempoutrec, DRMS_FREE_RECORD);
149 tplarson 1.1 return 1;
150 }
151 }
152 */
153
154 tepoch=drms_getkey_time(tempoutrec, "T_START_epoch", &status);
155 tstep=drms_getkey_float(tempoutrec, "T_START_step", &status);
156 tround=drms_getkey_float(tempoutrec, "T_START_round", &status);
157 cadence=drms_getkey_float(tempoutrec, "T_STEP", &status);
158 if (fmod(tstart-tepoch,tstep) > tround/2)
159 {
160 fprintf(stderr, "ERROR: output dataseries seems incompatible with input parameters (tstep must divide tstart-tepoch): tstart = %f, tepoch = %f, tstep = %f\n", tstart, tepoch, tstep);
161 drms_close_record(tempoutrec, DRMS_FREE_RECORD);
162 return 1;
163 }
164 if (chunksecs == 0.0)
165 chunksecs = tstep;
166 else if (fmod(chunksecs,tstep))
167 {
168 fprintf(stderr, "ERROR: output dataseries seems incompatible with input parameters (tstep must divide chunksecs): chunksecs = %f, tstep = %f\n", chunksecs, tstep);
169 drms_close_record(tempoutrec, DRMS_FREE_RECORD);
170 tplarson 1.1 return 1;
171 }
172 if (fmod(nseconds,chunksecs) != 0.0)
173 {
174 fprintf(stderr, "ERROR: input parameters seem incompatible (chunksecs must divide nseconds): nseconds = %f, chunksecs = %f\n", nseconds, chunksecs);
175 drms_close_record(tempoutrec, DRMS_FREE_RECORD);
176 return 1;
177 }
178 ntimechunks=nseconds/chunksecs;
179 ndt=chunksecs/cadence;
180 if (verbflag)
181 {
182 printf("%d timechunks, %.1f seconds per chunk\n", ntimechunks, chunksecs);
183 }
184
185 int mapmmaxout=-1;
186 int sinbdivsout=-1;
187 DRMS_Keyword_t *outkeytest = hcon_lookup_lower(&tempoutrec->keywords, "MAPMMAX");
188 if (outkeytest != NULL && outkeytest->info->recscope == 1)
189 mapmmaxout=drms_getkey_int(tempoutrec, "MAPMMAX", &status);
190 outkeytest = hcon_lookup_lower(&tempoutrec->keywords, "SINBDIVS");
191 tplarson 1.1 if (outkeytest != NULL && outkeytest->info->recscope == 1)
192 sinbdivsout=drms_getkey_int(tempoutrec, "SINBDIVS", &status);
193
194 drms_close_record(tempoutrec, DRMS_FREE_RECORD);
195
196
197 char *inchecklist[] = {"T_START", "QUALITY", "LMIN", "LMAX", "T_STEP"};
198 DRMS_Keyword_t *inkeytest;
199 int itest;
200 inrecset = drms_open_records(drms_env, inrecquery, &status);
201 // inrecset = drms_open_recordset(drms_env, inrecquery, &status);
202
203 if (status != DRMS_SUCCESS || inrecset == NULL)
204 {
205 fprintf(stderr, "ERROR: problem opening input recordset: status = %d\n", status);
206 return 1;
207 }
208 nrecsin = inrecset->n;
209
210 if (verbflag)
211 printf("input recordset opened, nrecs = %d\n", nrecsin);
212 tplarson 1.1
213 int noinput=0;
214 if (nrecsin == 0)
215 {
216 printf("WARNING: input recordset contains no records\n");
217 noinput=1;
218 goto skip1;
219 // return 1;
220 }
221
222 inrec = inrecset->records[0];
223
224 for (itest=0; itest < ARRLENGTH(inchecklist); itest++)
225 {
226 inkeytest = hcon_lookup_lower(&inrec->keywords, inchecklist[itest]);
227 if (inkeytest == NULL)
228 {
229 fprintf(stderr, "ERROR: required input keyword %s is missing\n", inchecklist[itest]);
230 drms_close_records(inrecset, DRMS_FREE_RECORD);
231 return 1;
232 }
233 tplarson 1.1 }
234
235 if (cadence != drms_getkey_float(inrec, "T_STEP", &status))
236 {
237 fprintf(stderr, "ERROR: input T_STEP does not equal output T_STEP\n");
238 drms_close_records(inrecset, DRMS_FREE_RECORD);
239 return 1;
240 }
241
242 inkeytest = hcon_lookup_lower(&inrec->keywords, "MAPMMAX");
243 if (inkeytest != NULL)
244 mapmmax=drms_getkey_int(inrec, "MAPMMAX", &status);
245 if (mapmmaxout != -1 && mapmmaxout != mapmmax)
246 {
247 fprintf(stderr, "ERROR: input MAPMMAX does not equal output MAPMMAX, in=%d, out=%d\n", mapmmax, mapmmaxout);
248 drms_close_records(inrecset, DRMS_FREE_RECORD);
249 return 1;
250 }
251
252 inkeytest = hcon_lookup_lower(&inrec->keywords, "SINBDIVS");
253 if (outkeytest != NULL)
254 tplarson 1.1 sinbdivs=drms_getkey_int(inrec, "SINBDIVS", &status);
255 if (sinbdivsout != -1 && sinbdivsout != sinbdivs)
256 {
257 fprintf(stderr, "ERROR: input SINBDIVS does not equal output SINBDIVS, in=%d, out=%d\n", sinbdivs, sinbdivsout);
258 drms_close_records(inrecset, DRMS_FREE_RECORD);
259 return 1;
260 }
261
262
263 status=drms_stage_records(inrecset, 1, 0);
264 if (status != DRMS_SUCCESS)
265 {
266 fprintf(stderr, "ERROR: drms_stage_records returned status = %d\n", status);
267 return 1;
268 }
269
270 skip1:
271
272 nrecords=0;
273 nsegments=0;
274 nskip=0;
275 tplarson 1.1
276 lchunkfirst = lmin/lchunksize;
277 lchunklast = lmax/lchunksize;
278
279 nlchunks = (lchunklast - lchunkfirst) + 1;
280 nrecsout = nlchunks*ntimechunks;
281 outrecset = drms_create_records(drms_env, nrecsout, outseries, lifetime, &status);
282 if (status != DRMS_SUCCESS || outrecset == NULL)
283 {
284 fprintf(stderr,"ERROR: unable to create records record in output dataseries %s, status = %d\n", outseries, status);
285 drms_close_records(inrecset, DRMS_FREE_RECORD);
286 return 1;
287 }
288
289 unsigned long long calversout, calvers;
290 int calversunset=1;
291 int mixflag=0;
292 unsigned int nybblearrout[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
293 int fixflagarr[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
294 for (i=0;i<16;i++)
295 {
296 tplarson 1.1 if (getbits(calverkey,i,1))
297 fixflagarr[i]=1;
298 }
299
300 int *firstflagarr=(int *)malloc(nrecsin*sizeof(int));
|
373 tplarson 1.1 nskip++;
374 continue;
375 }
376
377 tstartin=drms_getkey_time(inrec, "T_START", &status);
378 tstopin=drms_getkey_time(inrec, "T_STOP", &status);
379 lminin=drms_getkey_int(inrec, "LMIN", &status);
380 lmaxin=drms_getkey_int(inrec, "LMAX", &status);
381 ndtin=(tstopin-tstartin)/cadence;
382
383 if (tstartin >= tstopout || tstopin <= tstartout || lminin > lmaxout || lmaxin < lminout)
384 {
385 nskip++;
386 continue;
387 }
388
389 if (calversunset)
390 {
391 calversout=drms_getkey_longlong(inrec, "CALVER64", &status);
392 if (status != DRMS_SUCCESS)
393 calversout = 0;
394 tplarson 1.1 calversunset=0;
395 for (i=0;i<16;i++)
396 nybblearrout[i]=getbits(calversout,4*i+3,4);
397 }
398
399 if (firstflagarr[irecin])
400 {
401 calvers=drms_getkey_longlong(inrec, "CALVER64", &status);
402 if (status != DRMS_SUCCESS)
403 calvers = 0;
404
405 for (i=0;i<16;i++)
406 {
407 int nybble=getbits(calvers,4*i+3,4);
408 if (fixflagarr[i])
409 {
410 if (nybble != nybblearrout[i])
411 {
412 fprintf(stderr, "ERROR: input data has mixed values for field %d of CALVER64: %d and %d, recnum = %lld, histrecnum = %lld\n", i, nybblearrout[i], nybble, inrec->recnum, histrecnum);
413 return 0;
414 }
415 tplarson 1.1 }
416 else
417 {
418 if (nybble < nybblearrout[i])
419 nybblearrout[i]=nybble;
420 }
421 }
422
423 if (!mixflag && (quality & QUAL_MIXEDCALVER || calvers != calversout))
424 mixflag=1;
425
426 if (seginflag)
427 segin = drms_segment_lookup(inrec, segnamein);
428 else
429 segin = drms_segment_lookupnum(inrec, 0);
430 if (segin != NULL)
431 inarr = drms_segment_read(segin, usetype, &status);
432 if (status != DRMS_SUCCESS || inarr == NULL || segin == NULL)
433 {
434 fprintf(stderr, "ERROR: problem reading input segment: tstart = %f, lmin = %d, lmax = %d, irecin = %d, status = %d, histrecnum = %lld\n", tstartin, lminin, lmaxin, irecin, status, histrecnum);
435 drms_close_records(inrecset, DRMS_FREE_RECORD);
436 tplarson 1.1 drms_close_records(outrecset, DRMS_FREE_RECORD);
437 return 0;
438 }
439 else
440 {
441 inarrarr[irecin]=inarr;
442 }
443 firstflagarr[irecin]=0;
444 }
445
446 inptr = (float *)(inarrarr[irecin])->data;
447
448 tstartuse=MAXIMUM(tstartout, tstartin);
449 tstopuse= MINIMUM(tstopout, tstopin);
450 lminuse=MAXIMUM(lminout, lminin);
451 lmaxuse=MINIMUM(lmaxout, lmaxin);
452 nmodes=MODES(lmaxuse+1)-MODES(lminuse);
453 npts=(tstopuse - tstartuse)/cadence;
454
455 out_time_offset = (tstartuse - tstartout)/cadence;
456 out_modes_offset = MODES(lminuse) - MODES(lminout);
457 tplarson 1.1 out_offset = 2 * (out_modes_offset * ndt + out_time_offset);
458 in_time_offset = (tstartuse - tstartin)/cadence;
459 in_modes_offset = MODES(lminuse) - MODES(lminin);
460 in_offset = 2 * (in_modes_offset * ndtin + in_time_offset);
461
462 for (imode=0; imode<nmodes; imode++)
463 {
464 for (itime=0; itime<npts; itime++)
465 {
466 in_index=in_offset + 2*itime;
467 out_index=out_offset + 2*itime;
468 outptr[out_index] = inptr[in_index];
469 outptr[out_index+1] = inptr[in_index+1];
470 }
471 out_offset+=2*ndt;
472 in_offset+=2*ndtin;
473 }
474
475 }
476
477 if (nskip == nrecsin)
478 tplarson 1.1 {
479 fprintf(stderr, "WARNING: no inputs for this output, no segment written\n");
480 }
481 else
482 {
483 outarr->bzero=segout->bzero;
484 outarr->bscale=segout->bscale;
485 status=drms_segment_write(segout, outarr, 0);
486 nsegments++;
487 if (status != DRMS_SUCCESS)
488 {
489 fprintf(stderr, "ERROR: problem writing output segment: status = %d, T_START = %s, LMIN = %d, LMAX = %d, histrecnum = %lld\n", status, tstartstr, lminout, lmaxout, histrecnum);
490 drms_close_records(inrecset, DRMS_FREE_RECORD);
491 drms_close_records(outrecset, DRMS_FREE_RECORD);
492 return 0;
493 }
494 }
495 drms_free_array(outarr);
496
497 drms_copykeys(outrec, inrec, 0, kDRMS_KeyClass_Explicit);
498 // copykeys takes care of MAPMMAX, SINBDIVS, etc. that should be constant across the input
499 tplarson 1.1 skip2:
500
501 drms_setkey_int(outrec, "LMIN", lminout);
502 drms_setkey_int(outrec, "LMAX", lmaxout);
503 drms_setkey_time(outrec, "T_START", tstartout);
504 drms_setkey_time(outrec, "T_STOP", tstopout);
505 drms_setkey_time(outrec, "T_OBS", tstartout+chunksecs/2);
506 drms_setkey_int(outrec, "NDT", ndt);
507
508 for (i=0;i<16;i++)
509 calversout=setbits(calversout,4*i+3,4,nybblearrout[i]);
510 drms_setkey_longlong(outrec, "CALVER64", calversout);
511
512 if (noinput || nskip == nrecsin)
513 drms_setkey_int(outrec, "QUALITY", QUAL_NODATA);
514 else if (mixflag)
515 drms_setkey_int(outrec, "QUALITY", QUAL_MIXEDCALVER);
516 else
517 drms_setkey_int(outrec, "QUALITY", 0);
518
519 tnow = (double)time(NULL);
520 tplarson 1.1 tnow += UNIX_epoch;
521 drms_setkey_time(outrec, "DATE", tnow);
522
523 nrecords++;
524
525 if (verbflag > 1)
526 {
527 wt=getwalltime();
528 ct=getcputime(&ut, &st);
529 fprintf(stdout, " lchunk %d done: %.2f ms wall time, %.2f ms cpu time\n", lchunk-lchunkfirst,
530 wt-wt2,
531 ct-ct2);
532 }
533
534 }
535
536 if (verbflag)
537 {
538 wt=getwalltime();
539 ct=getcputime(&ut, &st);
540 fprintf(stdout, "timechunk %d done: %.2f ms wall time, %.2f ms cpu time\n", iset,
541 tplarson 1.1 wt-wt1,
542 ct-ct1);
543
544 }
545
546 }
547
548 for (irecin=0; irecin < nrecsin; irecin++)
|
Return to
jretile_maxmem.c
CVS log
Up to [Development] / JSOC / proj / globalhs / apps