Return to doglobalhs CVS log

Up to [Development] / JSOC / proj / globalhs / scripts

fix goto's

#! /bin/tcsh -f

setenv JSOCROOT /home/jsoc/cvs/Development/JSOC
setenv PATH     $JSOCROOT/bin/$JSOC_MACHINE':'$JSOCROOT/proj/globalhs/scripts':'$JSOCROOT/proj/globalhs/scripts/pkbgn':'$PATH
setenv IDL_PATH $JSOCROOT/proj/globalhs/scripts/idl:/home/jsoc/idl:'<IDL_DEFAULT>'

checkglobalhsargs $argv
if ($status) then
  echo parameter check fails
  exit 1
endif

@ i = 1
while ($i <= $#argv )
  set $argv[$i]
@ i++
end

if (! $?starttime) then
  echo must specify parameter starttime
  exit 1
endif

if (! $?totaltime) then
  echo must specify parameter totaltime
  exit 1
endif

if (! $?label) then
  echo must specify parameter label
  exit 1
endif
setenv GLOBALHS_LABEL $label

date
echo $PWD
echo $0 $argv

if (! $?lmin) set lmin = 0
if (! $?lmax) set lmax = 300
if (! $?lchunk) set lchunk = 40
if (! $?lchunk1) set lchunk1 = $lchunk
if (! $?lchunk2) set lchunk2 = $lchunk
if (! $?lchunk3) set lchunk3 = 1
if (! $?lchunk4) set lchunk4 = 10
if (! $?lchunk5) set lchunk5 = $lchunk4
if (! $?timechunk) set timechunk = 1d
if (! $?timechunk1) set timechunk1 = $timechunk
if (! $?timechunk2) set timechunk2 = 12d
if (! $?powtotaltime) set powtotaltime = $totaltime
if (! $?slicestep) set slicestep = 3d
if (! $?powstartoffset) set powstartoffset = 0d
if (! $?firststep) set firststep = sht
if (! $?laststep) set laststep = pkbgn

# optionally set njobs, jobthresholds, and queues for the various processing steps
# for dosht and doretilen, njobs=totaltime/timechunk
if ($?retnjobs) then
  setenv GLOBALHS_RETNJOBS $retnjobs
endif
if ($?tsfnjobs) then
  setenv GLOBALHS_TSFNJOBS $tsfnjobs
endif
if ($?pownjobs) then
  setenv GLOBALHS_POWNJOBS $pownjobs
endif
if ($?pkbnjobs) then
  setenv GLOBALHS_PKBNJOBS $pkbnjobs
endif

if ($?v2tjobthreshold) then
  setenv GLOBALHS_V2TJOBTHRESHOLD $v2tjobthreshold
endif
if ($?retjobthreshold) then
  setenv GLOBALHS_RETJOBTHRESHOLD $retjobthreshold
endif
if ($?tsfjobthreshold) then
  setenv GLOBALHS_TSFJOBTHRESHOLD $tsfjobthreshold
endif
if ($?powjobthreshold) then
  setenv GLOBALHS_POWJOBTHRESHOLD $powjobthreshold
endif
if ($?pkbjobthreshold) then
  setenv GLOBALHS_PKBJOBTHRESHOLD $pkbjobthreshold
endif

if ($?v2tqueue) then
  setenv GLOBALHS_V2TQUEUE $v2tqueue
endif
if ($?retqueue) then
  setenv GLOBALHS_RETQUEUE $retqueue
endif
if ($?tsfqueue) then
  setenv GLOBALHS_TSFQUEUE $tsfqueue
endif
if ($?powqueue) then
  setenv GLOBALHS_POWQUEUE $powqueue
endif
if ($?pkbqueue) then
  setenv GLOBALHS_PKBQUEUE $pkbqueue
endif

if (! $?topdir) set topdir = /tmp28
set workdir = /$topdir/$USER/globalhs_work/$label/$starttime'_'$totaltime/$lmin-$lmax

goto $firststep

sht:
if (! -es $JSOCROOT/proj/globalhs/scripts/parmtemplates/v2t.parms.$label) goto retilen
mkdir -p $workdir/sht
cd $workdir/sht
cp $JSOCROOT/proj/globalhs/scripts/parmtemplates/v2t.parms.$label v2t.parms.blank
set nonomatch
set inrecset = `cat v2t.parms.blank | grep -v '^[[:space:]]*#' | sed s@"[[:space:]]"@"\n"@g | grep ^in= | cut -d= -f2- | sed s@XXXX@$starttime/$totaltime@ | sed s/\"//g | sed s/\'//g`

#below is necessary in case any input records must be read from tape
if (! -es ../inrecsetonline.txt) then
  show_info -ApiR "$inrecset" >& ../inrecsetonline.txt
endif

dosht starttime=$starttime totaltime=$totaltime timechunk=$timechunk1 lchunk=$lchunk1 lmin=$lmin lmax=$lmax >>& shtlog
if ($?) then
  echo failure of sht for starttime = $starttime, totaltime = $totaltime, check shtlog
  exit 1
endif
if ($laststep == sht) goto end


shtcheck:
if (! -es $JSOCROOT/proj/globalhs/scripts/parmtemplates/v2t.parms.$label) goto retilen
mkdir -p $workdir/sht
cd $workdir/sht
cp $JSOCROOT/proj/globalhs/scripts/parmtemplates/v2t.parms.$label v2t.parms.blank
set sourcetype = `cat v2t.parms.blank | grep "^[[:space:]]*##" | awk -F '##' '{print $2}' | sed s@"[[:space:]]"@"\n"@g | grep sourcetype`
set $sourcetype
#quality.txt will be used by doshtcheck.
if ($sourcetype == hmi) then
  set nonomatch
  set inrecset = `cat v2t.parms.blank | grep -v '^[[:space:]]*#' | sed s@"[[:space:]]"@"\n"@g | grep ^in= | cut -d= -f2- | sed s@XXXX@$starttime/$totaltime@ | sed s/\"//g | sed s/\'//g`
  if (! -es ../quality.txt) then
   show_info -q "$inrecset" key=quality > ../quality.txt
  endif
endif

doshtcheck starttime=$starttime totaltime=$totaltime lchunk=$lchunk1 lmin=$lmin lmax=$lmax >>& shtchecklog
if ($?) then
  echo failure of sht check for starttime = $starttime, totaltime = $totaltime, check shtchecklog
  exit 1
endif
rm subt.* v2t.parms.[0-9]* v2t.log.* jv2ts.exitstatus.*
if ($laststep == shtcheck) goto end


retilen:
if (! -es $JSOCROOT/proj/globalhs/scripts/parmtemplates/retn.parms.$label) goto retile1
mkdir -p $workdir/retilen
cd $workdir/retilen
cp $JSOCROOT/proj/globalhs/scripts/parmtemplates/retn.parms.$label ret.parms.blank

doretilen starttime=$starttime totaltime=$totaltime timechunk=$timechunk2 lchunkin=$lchunk1 lchunkout=$lchunk2 lmin=$lmin lmax=$lmax >>& retilenlog
if ($?) then
  echo failure of retilen for starttime = $starttime, totaltime = $totaltime, check retilenlog
  exit 1
endif

set tagclause = `cat ret.parms.blank  | grep -v '^[[:space:]]*#' | sed s@"[[:space:]]"@"\n"@g | grep ^in= | cut -d= -f2- | sed s@"\["@"\n"@g | sed s@]@"\n"@g | grep TAG`
set totalsecs = `durcon $totaltime`
set out = `cat ret.parms.blank | grep -v "^[[:space:]]*#" | sed s@"[[:space:]]"@"\n"@g | grep ^out= | cut -d= -f2`
set cadence = `show_info -j $out | grep -i '^keyword:t_step' | cut -d, -f5`
set epoch = `show_info -j $out | grep -i '^keyword:t_start_epoch' | cut -d, -f5`
set epochsecs = `time_convert o=jsoc time=$epoch`
set tchunk = `cat ret.parms.blank | grep -v "^[[:space:]]*#" | sed s@"[[:space:]]"@"\n"@g | grep TCHUNK= | cut -d= -f2`
set chunksecs = `durcon $tchunk`
set ndt   = `echo "$chunksecs / $cadence" | bc`
set ntimechunks = `echo "$totalsecs / $chunksecs" | bc`
durcon $starttime >& /dev/null
if ($status) then
  set startsecs = `time_convert o=jsoc time=$starttime`
else
  set startsecs = `durcon $starttime` 
  set startsecs = `echo "$startsecs + $epochsecs" | bc`
endif
set start = `time_convert o=cal zone=tai s=$startsecs`
set endsecs = `echo "$startsecs + $totalsecs - $cadence" | bc`
set end = `time_convert o=cal zone=tai s=$endsecs`
set in = `cat ret.parms.blank | grep -v "^[[:space:]]*#" | sed s@"[[:space:]]"@"\n"@g | grep ^in= | sed s/\'//g | sed s/\"//g | awk -F '[=[]' '{print $2}'`
set clauselist = `cat ret.parms.blank | grep -v "^[[:space:]]*#" | sed s@"[[:space:]]"@"\n"@g | grep ^in= | cut -d= -f2- | sed s@"\["@"\n"@g | sed s@]@"\n"@g | grep =`
set nmiss = `show_coverage -iq ds=$in low=$start high=$end block=$tchunk $clauselist | awk -v total=0 '$2 == 0 {total+=1} END {print total}'`
@ nexpected = $ntimechunks - $nmiss

@ lchunkfirst = $lmin / $lchunk2
@ lchunklast  = $lmax / $lchunk2
@ nlc = ($lchunklast - $lchunkfirst) + 1

show_info $out\[$starttime/$totaltime]\[$lmin-$lmax]\[]\[$ndt]\[$tagclause] DRMS_RETENTION=200 -APiR >& retilenout.tmp
if (`grep -c "/SUM.*/data\.fits" retilenout.tmp` != $nexpected * $nlc) then
  echo failure of retilen for starttime = $starttime, totaltime = $totaltime : some outputs missing
  exit 1
endif
rm subr.* ret.parms.[0-9]* ret.log.* jretile.exitstatus.*
if ($laststep == retilen) goto end


dscopyn:
if (! -es $JSOCROOT/proj/globalhs/scripts/parmtemplates/retn.parms.$label) goto retile1
mkdir -p $workdir/retilen
cd $workdir/retilen
cp $JSOCROOT/proj/globalhs/scripts/parmtemplates/retn.parms.$label ret.parms.blank

set copyto = `cat ret.parms.blank | grep "^[[:space:]]*##" | awk -F '##' '{print $2}' | sed s@"[[:space:]]"@"\n"@g | grep copyto`
if ($#copyto) then
  set $copyto
  dodscopyn starttime=$starttime totaltime=$totaltime timechunk=$timechunk2 lchunkin=$lchunk1 lchunkout=$lchunk2 lmin=$lmin lmax=$lmax >>& dscopynlog
  if ($?) then
    echo failure of dscopyn for starttime = $starttime, totaltime = $totaltime, check dscopynlog
    exit 1
  endif

  set tagclause = `cat ret.parms.blank  | grep -v '^[[:space:]]*#' | sed s@"[[:space:]]"@"\n"@g | grep ^in= | cut -d= -f2- | sed s@"\["@"\n"@g | sed s@]@"\n"@g | grep TAG`
  set totalsecs = `durcon $totaltime`
  set out = $copyto
  set cadence = `show_info -j $out | grep -i '^keyword:t_step' | cut -d, -f5`
  set epoch = `show_info -j $out | grep -i '^keyword:t_start_epoch' | cut -d, -f5`
  set epochsecs = `time_convert o=jsoc time=$epoch`
  set tchunk = `cat ret.parms.blank | grep -v "^[[:space:]]*#" | sed s@"[[:space:]]"@"\n"@g | grep TCHUNK= | cut -d= -f2`
  set chunksecs = `durcon $tchunk`
  set ndt   = `echo "$chunksecs / $cadence" | bc`
  set ntimechunks = `echo "$totalsecs / $chunksecs" | bc`
  durcon $starttime >& /dev/null
  if ($status) then
    set startsecs = `time_convert o=jsoc time=$starttime`
  else
    set startsecs = `durcon $starttime` 
    set startsecs = `echo "$startsecs + $epochsecs" | bc`
  endif
  set start = `time_convert o=cal zone=tai s=$startsecs`
  set endsecs = `echo "$startsecs + $totalsecs - $cadence" | bc`
  set end = `time_convert o=cal zone=tai s=$endsecs`
  set in = `cat ret.parms.blank | grep -v "^[[:space:]]*#" | sed s@"[[:space:]]"@"\n"@g | grep ^in= | sed s/\'//g | sed s/\"//g | awk -F '[=[]' '{print $2}'`
  set clauselist = `cat ret.parms.blank | grep -v "^[[:space:]]*#" | sed s@"[[:space:]]"@"\n"@g | grep ^in= | cut -d= -f2- | sed s@"\["@"\n"@g | sed s@]@"\n"@g | grep =`
  set nmiss = `show_coverage -iq ds=$in low=$start high=$end block=$tchunk $clauselist | awk -v total=0 '$2 == 0 {total+=1} END {print total}'`
  @ nexpected = $ntimechunks - $nmiss

  @ lchunkfirst = $lmin / $lchunk2
  @ lchunklast  = $lmax / $lchunk2
  @ nlc = ($lchunklast - $lchunkfirst) + 1

  show_info $copyto\[$starttime/$totaltime]\[$lmin-$lmax]\[]\[$ndt]\[$tagclause] DRMS_RETENTION=200 -APiR >& dscopynout.tmp
  if (`grep -c "/SUM.*/data\.fits" dscopynout.tmp` != $nexpected * $nlc) then
    echo failure of dscopyn for starttime = $starttime, totaltime = $totaltime : some outputs missing
    exit 1
  endif
  rm subr.* dscp.log.* dscp.exitstatus.*
endif
if ($laststep == dscopyn) goto end


retile:
retile1:
if ((! -es $JSOCROOT/proj/globalhs/scripts/parmtemplates/ret.parms.$label) && (! -es $JSOCROOT/proj/globalhs/scripts/parmtemplates/ret1.parms.$label)) goto gapfill
mkdir -p $workdir/retile1
cd $workdir/retile1
if (-es $JSOCROOT/proj/globalhs/scripts/parmtemplates/ret1.parms.$label) then
  cp $JSOCROOT/proj/globalhs/scripts/parmtemplates/ret1.parms.$label ret.parms.blank
else
  cp $JSOCROOT/proj/globalhs/scripts/parmtemplates/ret.parms.$label ret.parms.blank
endif

doretile1 starttime=$starttime totaltime=$totaltime lchunkin=$lchunk2 lchunkout=$lchunk3 lmin=$lmin lmax=$lmax >>& retilelog
if ($?) then
  echo failure of retile1 for starttime = $starttime, totaltime = $totaltime, check retilelog
  exit 1
endif

set tagclause = `cat ret.parms.blank  | grep -v '^[[:space:]]*#' | sed s@"[[:space:]]"@"\n"@g | grep ^in= | cut -d= -f2- | sed s@"\["@"\n"@g | sed s@]@"\n"@g | grep TAG`
set totalsecs = `durcon $totaltime`
set out = `cat ret.parms.blank | grep -v "^[[:space:]]*#"  | sed s@"[[:space:]]"@"\n"@g | grep ^out= | cut -d= -f2`
set cadence = `show_info -j $out | grep -i '^keyword:t_step' | cut -d, -f5`
set ndt   = `echo "$totalsecs / $cadence" | bc`
@ lchunkfirst = $lmin / $lchunk3
@ lchunklast  = $lmax / $lchunk3
@ nlc = ($lchunklast - $lchunkfirst) + 1

show_info $out\[$starttime]\[$lmin-$lmax]\[]\[$ndt]\[$tagclause] DRMS_RETENTION=100 -APiR >& retileout.tmp
if (`grep -c "/SUM.*/data\.fits" retileout.tmp` != $nlc) then
  echo failure of retile1 for starttime = $starttime, totaltime = $totaltime : some outputs missing
  exit 1
endif
rm subr.* ret.parms.[0-9]* ret.log.* jretile.exitstatus.*
if ($laststep == retile || $laststep == retile1) goto end


dscopy1:
if ((! -es $JSOCROOT/proj/globalhs/scripts/parmtemplates/ret.parms.$label) && (! -es $JSOCROOT/proj/globalhs/scripts/parmtemplates/ret1.parms.$label)) goto gapfill
mkdir -p $workdir/retile1
cd $workdir/retile1
if (-es $JSOCROOT/proj/globalhs/scripts/parmtemplates/ret1.parms.$label) then
  cp $JSOCROOT/proj/globalhs/scripts/parmtemplates/ret1.parms.$label ret.parms.blank
else
  cp $JSOCROOT/proj/globalhs/scripts/parmtemplates/ret.parms.$label ret.parms.blank
endif

set copyto = `cat ret.parms.blank | grep "^[[:space:]]*##" | awk -F '##' '{print $2}' | sed s@"[[:space:]]"@"\n"@g | grep copyto`
if ($#copyto) then
  set $copyto
  dodscopy1 starttime=$starttime totaltime=$totaltime lchunkin=$lchunk2 lchunkout=$lchunk3 lmin=$lmin lmax=$lmax >>& dscopy1log
  if ($?) then
    echo failure of dscopy1 for starttime = $starttime, totaltime = $totaltime, check dscopy1log
    exit 1
  endif

  set tagclause = `cat ret.parms.blank  | grep -v '^[[:space:]]*#' | sed s@"[[:space:]]"@"\n"@g | grep ^in= | cut -d= -f2- | sed s@"\["@"\n"@g | sed s@]@"\n"@g | grep TAG`
  set totalsecs = `durcon $totaltime`
  set out = $copyto
  set cadence = `show_info -j $out | grep -i '^keyword:t_step' | cut -d, -f5`
  set ndt   = `echo "$totalsecs / $cadence" | bc`
  @ lchunkfirst = $lmin / $lchunk3
  @ lchunklast  = $lmax / $lchunk3
  @ nlc = ($lchunklast - $lchunkfirst) + 1

  show_info $copyto\[$starttime]\[$lmin-$lmax]\[]\[$ndt]\[$tagclause] DRMS_RETENTION=100 -APiR >& dscopy1out.tmp
  if (`grep -c "/SUM.*/data\.fits" dscopy1out.tmp` != $nlc) then
    echo failure of dscopy1 for starttime = $starttime, totaltime = $totaltime : some outputs missing
    exit 1
  endif
  rm subr.* dscp.log.* dscp.exitstatus.*
endif
if ($laststep == dscopy1) goto end


mkgaps:
if ($lmin > 50) goto gapfill
if ((! -es $JSOCROOT/proj/globalhs/scripts/parmtemplates/ret.parms.$label) && (! -es $JSOCROOT/proj/globalhs/scripts/parmtemplates/ret1.parms.$label)) goto gapfill
mkdir -p $workdir/retile1
cd $workdir/retile1
if (-es $JSOCROOT/proj/globalhs/scripts/parmtemplates/ret1.parms.$label) then
  cp $JSOCROOT/proj/globalhs/scripts/parmtemplates/ret1.parms.$label ret.parms.blank
else
  cp $JSOCROOT/proj/globalhs/scripts/parmtemplates/ret.parms.$label ret.parms.blank
endif
set sourcetype = `cat ret.parms.blank | grep "^[[:space:]]*##" | awk -F '##' '{print $2}' | sed s@"[[:space:]]"@"\n"@g | grep sourcetype`
set $sourcetype
#quality.txt and seckeys.txt will be used by domkgaps.
if ($sourcetype == hmi) then
  set nonomatch
  set inrecset = `cat $JSOCROOT/proj/globalhs/scripts/parmtemplates/v2t.parms.$label | grep -v '^[[:space:]]*#' | sed s@"[[:space:]]"@"\n"@g | grep ^in= | cut -d= -f2- | sed s@XXXX@$starttime/$totaltime@ | sed s/\"//g | sed s/\'//g`
  if (! -es ../quality.txt) then
    show_info -q "$inrecset" key=quality > ../quality.txt
  endif
  if (! -es ../seckeys.txt) then
    show_info -q "$inrecset" key=t_rec_index,CAL_FSN,HCFTID,CALVER64 > ../seckeys.txt
  endif
endif

if ($sourcetype == concatenated) then
  docatgaps starttime=$starttime totaltime=$totaltime >>& catgapslog
  set stat = $status
else
  domkgaps starttime=$starttime totaltime=$totaltime >>& mkgapslog
  set stat = $status
endif
if ($stat) then
  echo failure : unable to make gap/section files for starttime = $starttime, totaltime = $totaltime
  exit 1
endif
if ($laststep == mkgaps) goto end


gapfill:
if (! -es $JSOCROOT/proj/globalhs/scripts/parmtemplates/tsf.parms.$label) goto pow
mkdir -p $workdir/gapfill
cd $workdir/gapfill
cp $JSOCROOT/proj/globalhs/scripts/parmtemplates/tsf.parms.$label tsf.parms.blank

dogapfill starttime=$starttime totaltime=$totaltime lmin=$lmin lmax=$lmax lchunk=$lchunk4 >>& gapfilllog
if ($?) then
  echo failure of gapfill for starttime = $starttime, totaltime = $totaltime, check gapfilllog
  exit 1
endif

set tagclause = `cat tsf.parms.blank | grep -v '^[[:space:]]*#' | sed s@"[[:space:]]"@"\n"@g | grep ^TAG=`
if ($#tagclause == 0) set tagclause = `cat tsf.parms.blank | grep -v '^[[:space:]]*#' | sed s@"[[:space:]]"@"\n"@g | grep ^in= | cut -d= -f2- | sed s@"\["@"\n"@g | sed s@]@"\n"@g | grep TAG`
set totalsecs = `durcon $totaltime`
set out = `cat tsf.parms.blank | grep -v "^[[:space:]]*#" | sed s@"[[:space:]]"@"\n"@g | grep ^tsout= | cut -d= -f2`
set cadence = `show_info -j $out | grep -i '^keyword:t_step' | cut -d, -f5`
set ndt   = `echo "$totalsecs / $cadence" | bc`
@ nts = ($lmax - $lmin) + 1

show_info $out\[$starttime]\[$lmin-$lmax]\[]\[$ndt]\[$tagclause] DRMS_RETENTION=360 -APiR >& gapfillout.tmp
if (`grep -c "/SUM.*/data\.fits" gapfillout.tmp` != $nts) then
  echo failure of gapfill for starttime = $starttime, totaltime = $totaltime : some timeseries missing
  exit 1
endif

#add one of the following sections for each type of possible output
set out = `cat tsf.parms.blank | grep -v "^[[:space:]]*#" | sed s@"[[:space:]]"@"\n"@g | grep ^powout= | cut -d= -f2`
if ($#out != 0) then
  show_info $out\[$starttime]\[$lmin-$lmax]\[]\[$ndt]\[$tagclause] DRMS_RETENTION=100 -APiR >& powout.tmp
  if (`grep -c "/SUM.*/data\.fits" powout.tmp` != $nts) then
    echo failure of gapfill for starttime = $starttime, totaltime = $totaltime : some power spectra missing
    exit 1
  endif
endif

set out = `cat tsf.parms.blank | grep -v "^[[:space:]]*#" | sed s@"[[:space:]]"@"\n"@g | grep ^fftout= | cut -d= -f2`
if ($#out != 0) then
  show_info $out\[$starttime]\[$lmin-$lmax]\[]\[$ndt]\[$tagclause] DRMS_RETENTION=100 -APiR >& fftout.tmp
  if (`grep -c "/SUM.*/data\.fits" fftout.tmp` != $nts) then
    echo failure of gapfill for starttime = $starttime, totaltime = $totaltime : some ffts missing
    exit 1
  endif
endif

set out = `cat tsf.parms.blank | grep -v "^[[:space:]]*#" | sed s@"[[:space:]]"@"\n"@g | grep ^fft1out= | cut -d= -f2`
if ($#out != 0) then
  show_info $out\[$starttime]\[$lmin-$lmax]\[]\[$ndt]\[$tagclause] DRMS_RETENTION=100 -APiR >& fft1out.tmp
  if (`grep -c "/SUM.*/data\.fits" fft1out.tmp` != $nts) then
    echo failure of gapfill for starttime = $starttime, totaltime = $totaltime : some fft1s missing
    exit 1
  endif
endif

set out = `cat tsf.parms.blank | grep -v "^[[:space:]]*#" | sed s@"[[:space:]]"@"\n"@g | grep ^mavgout= | cut -d= -f2`
if ($#out != 0) then
  show_info $out\[$starttime]\[$lmin-$lmax]\[]\[$ndt]\[$tagclause] DRMS_RETENTION=100 -APiR >& mavgout.tmp
  if (`grep -c "/SUM.*/data\.fits" mavgout.tmp` != $nts) then
    echo failure of gapfill for starttime = $starttime, totaltime = $totaltime : some m-averaged spectra missing
    exit 1
  endif
endif

rm subg.* tsf.parms.[0-9]* tsf.log.* jtsfiddle.exitstatus.*
if ($laststep == gapfill) goto end

gfgaps:
if ($lmin > 50) goto pow
if (! -es $JSOCROOT/proj/globalhs/scripts/parmtemplates/tsf.parms.$label) goto pow
mkdir -p $workdir/gapfill
cd $workdir/gapfill
cp $JSOCROOT/proj/globalhs/scripts/parmtemplates/tsf.parms.$label tsf.parms.blank
dogfgaps starttime=$starttime totaltime=$totaltime lmin=$lmin lmax=$lmax >>& gfgapslog
if ($?) then
  echo failure : unable to make filled gap files for starttime = $starttime, totaltime = $totaltime
  exit 1
endif
if ($laststep == gfgaps) goto end

#pkbgn is independent of the next two steps, so they could be skipped, but for now test for existence of their parameter file templates


# the dopow script is now only needed if you want its startoffset feature (if you want a power spectrum of only part of a timeseries)
# full power spectra are output at the same time as gapfilled timeseries by an upgraded jtsfiddle
# it's also currently needed if a different FLIPM is desired for timeseries and fft's
pow:
if (! -es $JSOCROOT/proj/globalhs/scripts/parmtemplates/pow.parms.$label && $laststep != pow) goto slice
mkdir -p $workdir/pow
cd $workdir/pow
cp $JSOCROOT/proj/globalhs/scripts/parmtemplates/pow.parms.$label pow.parms.blank

dopow starttime=$starttime totaltimein=$totaltime totaltimeout=$powtotaltime startoffset=$powstartoffset lmin=$lmin lmax=$lmax lchunk=$lchunk5 >>& powlog
if ($?) then
  echo failure of dopow for starttime = $starttime, totaltime = $totaltime, check powlog
  exit 1
endif

set tagclause = `cat pow.parms.blank | grep -v '^[[:space:]]*#' | sed s@"[[:space:]]"@"\n"@g | grep ^in= | cut -d= -f2- | sed s@"\["@"\n"@g | sed s@]@"\n"@g | grep TAG`
set totalsecs = `durcon $powtotaltime`
@ nts = ($lmax - $lmin) + 1
set out = `cat pow.parms.blank | grep -v "^[[:space:]]*#" | sed s@"[[:space:]]"@"\n"@g | grep ^powout= | cut -d= -f2`
if ($#out != 0) then
  set cadence = `show_info -j $out | grep -i '^keyword:t_step' | cut -d, -f5`
  set ndt = `echo "$totalsecs / $cadence" | bc`
  show_info $out\[$starttime/$totaltime]\[$lmin-$lmax]\[]\[$ndt]\[$tagclause] DRMS_RETENTION=100 -APiR >& powout.tmp
  if (`grep -c "/SUM.*/data\.fits" powout.tmp` != $nts) then
    echo failure of dopow for starttime = $starttime, totaltime = $totaltime : some power spectra missing
    exit 1
  endif
endif

set out = `cat pow.parms.blank | grep -v "^[[:space:]]*#" | sed s@"[[:space:]]"@"\n"@g | grep ^fftout= | cut -d= -f2`
if ($#out != 0) then
  set cadence = `show_info -j $out | grep -i '^keyword:t_step' | cut -d, -f5`
  set ndt = `echo "$totalsecs / $cadence" | bc`
  show_info $out\[$starttime/$totaltime]\[$lmin-$lmax]\[]\[$ndt]\[$tagclause] DRMS_RETENTION=100 -APiR >& fftout.tmp
  if (`grep -c "/SUM.*/data\.fits" fftout.tmp` != $nts) then
    echo failure of dopow for starttime = $starttime, totaltime = $totaltime : some ffts missing
    exit 1
  endif
endif

set out = `cat pow.parms.blank | grep -v "^[[:space:]]*#" | sed s@"[[:space:]]"@"\n"@g | grep ^fft1out= | cut -d= -f2`
if ($#out != 0) then
  set cadence = `show_info -j $out | grep -i '^keyword:t_step' | cut -d, -f5`
  set ndt = `echo "$totalsecs / $cadence" | bc`
  show_info $out\[$starttime/$totaltime]\[$lmin-$lmax]\[]\[$ndt]\[$tagclause] DRMS_RETENTION=100 -APiR >& fft1out.tmp
  if (`grep -c "/SUM.*/data\.fits" fft1out.tmp` != $nts) then
    echo failure of dopow for starttime = $starttime, totaltime = $totaltime : some fft1s missing
    exit 1
  endif
endif

set out = `cat pow.parms.blank | grep -v "^[[:space:]]*#" | sed s@"[[:space:]]"@"\n"@g | grep ^mavgout= | cut -d= -f2`
if ($#out != 0) then
  set cadence = `show_info -j $out | grep -i '^keyword:t_step' | cut -d, -f5`
  set ndt = `echo "$totalsecs / $cadence" | bc`
  show_info $out\[$starttime/$totaltime]\[$lmin-$lmax]\[]\[$ndt]\[$tagclause] DRMS_RETENTION=100 -APiR >& mavgout.tmp
  if (`grep -c "/SUM.*/data\.fits" mavgout.tmp` != $nts) then
    echo failure of dopow for starttime = $starttime, totaltime = $totaltime : some m-averaged spectra missing
    exit 1
  endif
endif

rm subpow.* pow.parms.[0-9]* pow.log.* jtsfiddle.exitstatus.*
if ($laststep == pow) goto end


slice:
if (! -es $JSOCROOT/proj/globalhs/scripts/parmtemplates/slc.parms.$label && $laststep != slice) goto pkbgn
mkdir -p $workdir/slice
cd $workdir/slice
cp $JSOCROOT/proj/globalhs/scripts/parmtemplates/slc.parms.$label slc.parms.blank

dopowslice starttime=$starttime totaltimein=$totaltime totaltimeout=$powtotaltime startoffset=$powstartoffset slicestep=$slicestep lmin=$lmin lmax=$lmax lchunk=$lchunk5 >>& slicelog
if ($?) then
  echo failure to make $stepdays day power spectra for starttime = $starttime, totaltime = $totaltime, check slicelog
  exit 1
endif

set tagclause = `cat slc.parms.blank | grep -v '^[[:space:]]*#' | sed s@"[[:space:]]"@"\n"@g | grep ^in= | cut -d= -f2- | sed s@"\["@"\n"@g | sed s@]@"\n"@g | grep TAG`
set stepsecs  = `durcon $slicestep`
set stepdays = `echo "$stepsecs/86400" | bc`
set totalsecs = `durcon $powtotaltime`
set nslice = `echo "$totalsecs / $stepsecs" | bc`
set out = `cat slc.parms.blank | grep -v "^[[:space:]]*#" | sed s@"[[:space:]]"@"\n"@g | grep ^out= | cut -d= -f2`
set cadence = `show_info -j $out | grep -i '^keyword:t_step' | cut -d, -f5`
set dtstep = `echo "$stepsecs / $cadence" | bc`
@ nts = ($lmax - $lmin) + 1

show_info $out\[$starttime/$totaltime]\[$lmin-$lmax]\[]\[$dtstep]\[$tagclause] DRMS_RETENTION=100 -APiR >& sliceout.$stepdays.tmp
if (`grep -c "/SUM.*/data\.fits" sliceout.$stepdays.tmp` != $nts * $nslice) then
  echo failure to make $stepdays day power spectra for starttime = $starttime, totaltime = $totaltime : some outputs missing
  exit 1
endif
rm subslc.* slc.parms.[0-9]* slc.log.* jtsslice.exitstatus.*
if ($laststep == slice) goto end


pkbgn:
if ($lmin > 300) goto end
if (! -es $JSOCROOT/proj/globalhs/scripts/parmtemplates/pkb.parms.$label) goto end
mkdir -p $workdir/pkbgn
cd $workdir/pkbgn
cp $JSOCROOT/proj/globalhs/scripts/parmtemplates/pkb.parms.$label pkb.parms.blank

dopkbgn starttime=$starttime totaltime=$totaltime lmin=$lmin lmax=$lmax >>& pkbgnlog
if ($?) then
  echo failure of pkbgn for starttime = $starttime, totaltime = $totaltime, check pkbgnlog
  exit 1
endif

set tagclause = `cat pkb.parms.blank | grep "^[[:space:]]*##" | awk -F '##' '{print $2}' | sed s@"[[:space:]]"@"\n"@g | grep ^TAG=`
if ($#tagclause == 0) set tagclause = `cat pkb.parms.blank | grep -v '^[[:space:]]*#' | sed s@"[[:space:]]"@"\n"@g | grep ^in= | cut -d= -f2- | sed s@"\["@"\n"@g | sed s@]@"\n"@g | grep TAG`
set totalsecs = `durcon $totaltime`
set out = `cat pkb.parms.blank | grep "^[[:space:]]*##" | awk -F '##' '{print $2}' | sed s@"[[:space:]]"@"\n"@g | grep ^archseries= | cut -d= -f2`
set cadence = `show_info -j $out | grep -i '^keyword:t_step' | cut -d, -f5`
set ndt = `echo "$totalsecs / $cadence" | bc`

set tarfile = `show_info $out\[$starttime]\[$lmin]\[$lmax]\[$ndt]\[$tagclause] -qAP`
set test = `tar -dzf $tarfile`
if ($status != 0 || $#test != 0) then
  echo failure of pkbgn for starttime = $starttime, totaltime = $totaltime : problem with tar file
  exit 1
endif
rm subp.* pkb.parms.[0-9]* pkb.log.* result.*
rm -f par[0-9]* {m,res}[0-9]*{q,qc,qp,qx,qw,q.18,q.36} tmp*.idl
rm archive.*
if ($laststep == pkbgn) goto end

invert:
if (! -es $JSOCROOT/proj/globalhs/scripts/parmtemplates/inv.parms.$label) goto end
if (! -es $workdir/pkbgn/split*.18.2d) then
  echo input for inversion not found
  exit 1
endif
cd $workdir/pkbgn
cp $JSOCROOT/proj/globalhs/scripts/parmtemplates/inv.parms.$label inv.parms.blank

doinvert starttime=$starttime totaltime=$totaltime lmin=$lmin lmax=$lmax >>& invertlog
if ($?) then
  echo failure of inversion for starttime = $starttime, totaltime = $totaltime, check invertlog
  exit 1
endif

set tagclause = `cat inv.parms.blank | grep "^[[:space:]]*##" | awk -F '##' '{print $2}' | sed s@"[[:space:]]"@"\n"@g | grep ^TAG=`
if ($#tagclause == 0) set tagclause = `cat inv.parms.blank | grep '^[[:space:]]*##' | awk -F '##' '{print $2}' | sed s@"[[:space:]]"@"\n"@g | grep ^input= | cut -d= -f2- | sed s@"\["@"\n"@g | sed s@]@"\n"@g | grep TAG`
set totalsecs = `durcon $totaltime`
set out = `cat inv.parms.blank | grep "^[[:space:]]*##" | awk -F '##' '{print $2}' | sed s@"[[:space:]]"@"\n"@g | grep ^invseries= | cut -d= -f2`
set cadence = `show_info -j $out | grep -i '^keyword:t_step' | cut -d, -f5`
set ndt = `echo "$totalsecs / $cadence" | bc`

set test = `show_info $out\[$starttime]\[$lmin]\[$lmax]\[$ndt]\[$tagclause] seg=split -qP | grep -c "/SUM.*/splittings.out"`
if ($test != 1) then
  echo failure of inversion for starttime = $starttime, totaltime = $totaltime : splittings.out file missing
  exit 1
endif

end:
cd $workdir
rm -f quality.txt seckeys.txt inrecsetonline.txt
echo successful completion for starttime = $starttime, totaltime = $totaltime
exit 0