JSOC_Documentation: proj/lev0/apps/ingest

00001 /*-----------------------------------------------------------------------------
00002  * cvs/JSOC/proj/lev0/apps/ingest_lev0.c
00003  * NOTE: This originally came from hmi_lev0.c on hmi0
00004  *-----------------------------------------------------------------------------
00005  *
00006  * This is a module that runs with DRMS and continuously extracts images and HK
00007  * data from .tlm files that appear in the given input dir. It puts the .tlm
00008  * and .qac files in the DRMS dataset TLMSERIESNAME and also in the outdir if
00009  * given.  It extracts images from the .tlm files an puts them in
00010  * the DRMS dataset LEV0SERIESNAME and extracts hk data to appropriate hk datasets.
00011  *
00012  * Call for testing (set TLMSERIESNAME and LEV0SERIESNAME to test datasets): 
00013  * ingest_lev0 vc=VC05 indir=/scr20/jim/tlm outdir=/scr20/jim/tlm/out [logfile=name]
00014  *
00015  * OPERATION:
00016  * This is normally run on the dedicated lev0 host. Four instances are run.
00017  * No outdir= is given, i.e. the .tlm and .qac files only end up in SUMS.
00018  *
00019  * ingest_lev0 vc=VC02 indir=/dds/socdc/hmi
00020  * ingest_lev0 vc=VC05 indir=/dds/socdc/hmi
00021  * ingest_lev0 vc=VC10 indir=/dds/socdc/aia
00022  * ingest_lev0 vc=VC13 indir=/dds/socdc/aia
00023  */ 
00024 
00025 #include <jsoc_main.h>
00026 #include <cmdparams.h>
00027 #include <drms.h>
00028 #include <drms_names.h>
00029 #include <stdio.h>
00030 #include <stdlib.h>
00031 #include <ctype.h>
00032 #include <strings.h>
00033 #include <errno.h>
00034 #include <sys/types.h>
00035 #include <sys/time.h>
00036 #include <sys/stat.h>   //for umask(2)
00037 #include <dirent.h>
00038 #include <unistd.h> //for alarm(2) among other things...
00039 #include <printk.h>
00040 #include <math.h>
00041 #include "packets.h"
00042 #include "imgdecode.h"
00043 #include "decode_hk_vcdu.h"
00044 #include "decode_hk.h"
00045 #include "load_hk_config_files.h"
00046 #include "add_small_image.c"
00047 #include "mypng.h"
00048 #include "tdsignals.h"
00049 #include "quallev0.h"
00050 
00051 #define RESTART_CNT 2   //#of tlm files to process before restart
00052 
00053 //#define LEV0SERIESNAMEHMI "su_production.lev0d_test"
00054 //#define LEV0SERIESNAMEHMI "su_production.lev0f_test"
00055 //#define LEV0SERIESNAMEHMI "su_production.lev0f_hmi"
00057 //For test with DDS on Jan 19, 2010
00058 //#define LEV0SERIESNAMEHMI "su_production.lev0f_hmi_JAN2010"
00059 //#define LEV0SERIESNAMEHMI "su_production.lev0f_hmi_junk"
00060 //#define LEV0SERIESNAMEHMI "hmi.lev0f"
00062 //For test with DDS on Jan 19, 2010
00063 //#define TLMSERIESNAMEHMI "su_production.tlm_hmi_JAN2010"
00064 //#define TLMSERIESNAMEHMI "su_production.tlm_hmi_junk"
00065 
00066 #define LEV0SERIESNAMEHMIGND "hmi_ground.lev0_dds"
00067 #define TLMSERIESNAMEHMIGND "hmi_ground.tlm_dds"
00068 //#define TLMSERIESNAMEHMI "hmi.tlm_reingest"
00069 
00070 //#define LEV0SERIESNAMEAIA "su_production.lev0d_test_aia"
00071 //#define LEV0SERIESNAMEAIA "su_production.lev0f_aia"
00073 //For test with DDS on Jan 19, 2010
00074 //#define LEV0SERIESNAMEAIA "su_production.lev0f_aia_JAN2010"
00075 //#define LEV0SERIESNAMEAIA "su_production.lev0f_aia_junk"
00077 //For test with DDS on Jan 19, 2010
00078 //#define TLMSERIESNAMEAIA "su_production.tlm_aia_JAN2010"
00079 //#define TLMSERIESNAMEAIA "su_production.tlm_aia_junk"
00080 
00081 #define LEV0SERIESNAMEAIAGND "aia_ground.lev0_dds"
00082 #define TLMSERIESNAMEAIAGND "aia_ground.tlm_dds"
00083 //#define TLMSERIESNAMEAIA "aia.tlm_reingest"
00084 
00085 //#define LEV0SERIESNAMEHMI "hmi.lev0_60d"
00086 //#define TLMSERIESNAMEHMI "hmi.tlm_60d"
00087 //#define LEV0SERIESNAMEAIA "aia.lev0_60d"
00088 //#define TLMSERIESNAMEAIA "aia.tlm_60d"
00089 
00090 //When change to these data series below to save real data.
00091 #define TLMSERIESNAMEHMI "hmi.tlm"
00092 #define LEV0SERIESNAMEHMI "hmi.lev0a"
00093 #define LEV0SERIESNAMEAIA "aia.lev0"
00094 #define TLMSERIESNAMEAIA "aia.tlm"
00095 
00096 //#define LEV0SERIESNAMEAIA "aia.lev0d"
00097 //#define TLMSERIESNAMEAIA "aia.tlmd"
00098 //#define LEV0SERIESNAMEHMI "hmi.lev0d"
00099 //#define TLMSERIESNAMEHMI "hmi.tlmd"
00100 
00101 //#define LEV0SERIESNAMEAIA "aia.lev0e"
00102 //#define TLMSERIESNAMEAIA "aia.tlme"
00103 //#define LEV0SERIESNAMEHMI "hmi.lev0e"
00104 //#define TLMSERIESNAMEHMI "hmi.tlme"
00105 
00106 //Also, change setting in $JSOCROOT/proj/lev0/apps/SOURCE_ENV_HK_DECODE file to:
00107 //setenv HK_LEV0_BY_APID_DATA_ID_NAME      lev0
00108 //setenv HK_DF_HSB_DIRECTORY               /tmp21/production/lev0/hk_hsb_dayfile
00109 //setenv HK_JSVNMAP_DIRECTORY              /home/production/cvs/TBL_JSOC/lev0/hk_jsn_map_file/prod
00110 
00111 
00112 #define H0LOGFILE "/usr/local/logs/lev0/ingest_lev0.%s.%s.%s.log"
00113 #define PKTSZ 1788      //size of VCDU pkt
00114 #define MAXFILES 65535      //max # of file can handle in tlmdir
00115 #define NUMTIMERS 8     //number of seperate timers avail
00116 #define IMAGE_NUM_COMMIT 12 //number of complete images until commit
00117 //#define IMAGE_NUM_COMMIT 2    //!!TEMP number of complete images until commit
00118 #define TESTAPPID 0x199     //appid of test pattern packet
00119 #define TESTVALUE 0xc0b     //first value in test pattern packet
00120 #define MAXERRMSGCNT 10     //max # of err msg before skip the tlm file
00121 #define NOTSPECIFIED "***NOTSPECIFIED***"
00122 #define ENVFILE "/home/production/cvs/JSOC/proj/lev0/apps/SOURCE_ENV_FOR_HK_DECODE"
00123 #define ENVFILE_GND "/home/production/cvs/JSOC/proj/lev0/apps/SOURCE_ENV_FOR_HK_DECODE_GROUND"
00124 
00125 extern int decode_next_hk_vcdu(unsigned short *tbuf, CCSDS_Packet_t **hk, unsigned int *Fsn);
00126 extern int write_hk_to_drms();
00127 extern void HMI_compute_exposure_times(DRMS_Record_t *rec, HK_Keyword_t *isp, int flg);
00128 extern int set_HMI_mech_values(DRMS_Record_t *rec);
00129 static TIME SDO_to_DRMS_time(int sdo_s, int sdo_ss);
00130 
00131 // List of default parameter values. 
00132 ModuleArgs_t module_args[] = { 
00133   {ARG_STRING, "vc", NOTSPECIFIED, "Primary virt channel to listen to"},
00134   {ARG_STRING, "indir", NOTSPECIFIED, "directory containing the files to ingest"},
00135   {ARG_STRING, "outdir", NOTSPECIFIED, "directory to move the files to after the ingest"}, 
00136   {ARG_STRING, "logfile", NOTSPECIFIED, "optional log file name. Will create one if not given"},
00137   {ARG_FLAG, "v", "0", "verbose flag"},
00138   {ARG_FLAG, "h", "0", "help flag"},
00139   {ARG_FLAG, "r", "0", "restart flag"},
00140   {ARG_FLAG, "g", "0", "ground data flag"},
00141   {ARG_END}
00142 };
00143 
00144 CmdParams_t cmdparams;
00145 // Module name presented to DRMS. 
00146 char *module_name = "ingest_lev0";
00147 
00148 typedef enum
00149 {
00150    kThreadSigErr_Success
00151 } ThreadSigErr_t;
00152 static int gLoop = 1;
00153 static pthread_mutex_t mutex;
00154 static struct timeval tv0;
00155 static td_alarm_t talarm = 0;
00156 
00157 FILE *h0logfp;      // fp for h0 ouput log for this run 
00158 static IMG Image, ImageOld;
00159 static IMG *Img, *ImgO, *ImgC;
00160 static CCSDS_Packet_t *Hk;
00161 static DRMS_Record_t *rs;
00162 static DRMS_Segment_t *segment;
00163 static DRMS_Array_t *segArray;
00164 static DRMS_RecordSet_t *rset;
00165 static DRMS_Record_t *rs_old, *rsc;
00166 static DRMS_Segment_t *segmentc;
00167 static DRMS_Array_t *cArray, *oldArray;
00168 static TIME sdo_epoch;
00169 static char datestr[32];
00170 static char bld_vers[16];
00171 static struct timeval first[NUMTIMERS], second[NUMTIMERS];
00172 
00173 int INVALtime;
00174 unsigned int fsn = 0;
00175 unsigned int fsnx = 0;
00176 unsigned int fsn_prev = 0;
00177 unsigned int fsn_pre_rexmit = 0;
00178 unsigned int fid = 0;
00179 
00180 short *rdat;
00181 long long vcdu_seq_num;
00182 long long vcdu_seq_num_next;
00183 long long total_missing_im_pdu;
00184 unsigned int vcdu_24_cnt, vcdu_24_cnt_next;
00185 int verbose;
00186 int grounddata;
00187 int appid;
00188 int testid1, testid2;
00189 int hmiaiaflg;          //0=hmi, 1=aia
00190 int whk_status;
00191 int total_tlm_vcdu;
00192 int total_missing_vcdu;
00193 int errmsgcnt, fileimgcnt;
00194 int cntsleeps = 0;
00195 int paused = 0;
00196 int imagecnt = 0;       // num of images since last commit 
00197 int restartflg = 0;     // set when ingest_lev0 is called for a restart
00198 int abortflg = 0;
00199 int sigalrmflg = 0;             // set on signal so prog will know 
00200 int ignoresigalrmflg = 0;       // set after a close_image()
00201 int firstfound = 0;     // set if see any file after startup
00202 int rexmitmode = 0;     // set if this process is doing /rexmit dir
00203 int ALRMSEC = 60;               // must get 2 in a row for no image timeout
00204 int sleep_interval = 2;     // #of sec to sleep after do_ingest() calls
00205 char logname[128];
00206 char argvc[32], argindir[96], arglogfile[96], argoutdir[96];
00207 char timetag[32];
00208 char pchan[8];          // primary channel to listen to e.g. VC02 
00209 char rchan[8];          // redundant channel to listen to e.g. VC10 
00210 char stopfile[80];      // e.g. /usr/local/logs/lev0/VC04_stop
00211 char tlmseriesname[128];    // e.g. hmi.tlm
00212 char lev0seriesname[128];   // e.g. hmi.lev0
00213 char tlmnamekey[128];       // shortened tlm file name for TLMDSNAM keyword
00214 char tlmnamekeyfirst[128];  // for TLMDSNAM keyword for 1st file of image
00215 char oldtlmdsnam[128];      // TLMDSNAM keyword from prev rec in db
00216 char *username;         // from getenv("USER") 
00217 char *tlmdir;           // tlm dir name passed in 
00218 char *outdir;           // output dir for .tlm file (can be /dev/null)
00219 char *logfile;          // optional log name passed in 
00220 char *vc;           // virtual channel to process, e.g. VC02 
00221 struct stat stbuf;
00222 struct p_r_chans {
00223   char *pchan;
00224   char *rchan;
00225   int instru;       //0=hmi, 1=aia
00226 };
00227 typedef struct p_r_chans P_R_CHANS;
00228 
00229 P_R_CHANS p_r_chan_pairs[] = {
00230 {"VC01", "VC09", 1},        // AIA 
00231 {"VC04", "VC12", 1},        // AIA 
00232 {"VC02", "VC10", 0},        // HMI 
00233 {"VC05", "VC13", 0},        // HMI 
00234 {"n/a", "n/a"}
00235 };
00236 
00237 struct namesort {       // sorted file names in tlmdir 
00238   char *name;
00239 };
00240 typedef struct namesort NAMESORT;
00241 
00242 
00243 //Function callback that gets called when alarm 'rings'.  This runs in the signal
00244 //thread that receives, the alarm signal, not the main thread. */
00245 void shandler(int sig, pthread_mutex_t *mtx)
00246 {
00247    float elapsed;
00248    struct timeval tv1;
00249    gettimeofday(&tv1, NULL);
00250 
00251    elapsed = (float)((tv1.tv_sec * 1000000.0 + tv1.tv_usec -
00252                       (tv0.tv_sec * 1000000.0 + tv0.tv_usec)) / 1000000.0);
00253 
00254    printk("Thread '%lld' received alarm signal '%d'.\n", 
00255         (long long )pthread_self(), sig);
00256    printk("Elapsed time is %f seconds.\n", elapsed);
00257 
00258    /* This isn't in the main thread, so put globals in a critical region. */
00259    pthread_mutex_lock(mtx);
00260    sigalrmflg = 1;
00261    pthread_mutex_unlock(mtx);
00262 }
00263 
00264 int nice_intro ()
00265 {
00266   int usage = cmdparams_get_int (&cmdparams, "h", NULL);
00267   if (usage)
00268     {
00269     printf ("Usage:\ningest_lev0 [-vh] "
00270     "vc=<virt chan> indir=</dir> [outdir=</dir>] [logfile=<file>]\n"
00271     "  -h: help - show this message then exit\n"
00272     "  -v: verbose\n"
00273         "  -g: output to hmi_ground/aia_ground\n"
00274     "  -r: restart. only used when we restart our selves periodically\n"
00275     "vc= primary virt channel to listen to e.g. VC02\n"
00276     "indir= directory containing the files to ingest\n"
00277     "outdir= optional dir to copy the files to after the ingest\n"
00278     "logfile= optional log file name. Will create one if not given\n");
00279     return(1);
00280     }
00281   verbose = cmdparams_get_int (&cmdparams, "v", NULL);
00282   grounddata = cmdparams_get_int (&cmdparams, "g", NULL);
00283   restartflg = cmdparams_get_int (&cmdparams, "r", NULL);
00284   return (0);
00285 }
00286 
00287 TIME SDO_to_DRMS_time(int sdo_s, int sdo_ss)
00288 {
00289 static int firstcall = 1;
00290 if (firstcall)
00291   {
00292   firstcall = 0;
00293   }
00294 /* XXX fix build 3/18/2008, arta */
00295 return(sdo_epoch + (TIME)sdo_s + (TIME)(sdo_ss)/65536.0);
00296 }
00297 
00298 // Setup global datestr[] like: 2008.07.14_08:29:31
00299 char *do_datestr() {
00300   time_t tval;
00301   struct tm *t_ptr;
00302 
00303   tval = time(NULL);
00304   t_ptr = localtime(&tval);
00305   sprintf(datestr, "%d.%02d.%02d_%02d:%02d:%02d", 
00306       (t_ptr->tm_year+1900), (t_ptr->tm_mon+1),
00307       t_ptr->tm_mday, t_ptr->tm_hour, t_ptr->tm_min, t_ptr->tm_sec);
00308   return(datestr);
00309 }
00310 
00311 // Returns a time tag like  yyyy.mm.dd.hhmmss 
00312 char *gettimetag()
00313 {
00314   struct timeval tvalr;
00315   struct tm *t_ptr;
00316 
00317   gettimeofday(&tvalr, NULL);
00318   t_ptr = localtime((const time_t *)&tvalr);
00319   sprintf(timetag, "%04d.%02d.%02d.%02d%02d%02d",
00320         (t_ptr->tm_year+1900), (t_ptr->tm_mon+1), t_ptr->tm_mday, t_ptr->tm_hour, t_ptr->tm_min, t_ptr->tm_sec);
00321   return(timetag);
00322 }
00323 
00324 
00325 void BeginTimer(int n)
00326 {
00327   gettimeofday (&first[n], NULL);
00328 }
00329 
00330 float EndTimer(int n)
00331 {
00332   gettimeofday (&second[n], NULL);
00333   if (first[n].tv_usec > second[n].tv_usec) {
00334     second[n].tv_usec += 1000000;
00335     second[n].tv_sec--;
00336   }
00337   return (float) (second[n].tv_sec-first[n].tv_sec) +
00338     (float) (second[n].tv_usec-first[n].tv_usec)/1000000.0;
00339 }
00340 
00341 // Outputs the variable format message (re: printf) to the log file.
00342 int h0log(const char *fmt, ...)
00343 {
00344   va_list args;
00345   char string[32768];
00346 
00347   va_start(args, fmt);
00348   vsprintf(string, fmt, args);
00349   if(h0logfp) {
00350     fprintf(h0logfp, string);
00351     fflush(h0logfp);
00352   } else {          // couldn't open log 
00353     printf(string);
00354     fflush(stdout);
00355   }
00356   va_end(args);
00357   return(0);
00358 }
00359 
00360 int send_mail(char *fmt, ...)
00361 {
00362   va_list args;
00363   char string[1024], cmd[1024];
00364 
00365   va_start(args, fmt);
00366   vsprintf(string, fmt, args);
00367   sprintf(cmd, "echo \"%s\" | Mail -s \"ingest_lev0 mail\" jsoc_ops", string);
00368   system(cmd);
00369   va_end(args);
00370   return(0);
00371 }
00372 
00373 //Set the QUALLEV0 keyword
00374 void do_quallev0(DRMS_Record_t *rs, IMG *img, int fsn) 
00375 {
00376   char *hwltnset, *cdark;
00377   char *hseqerr, *aistate;
00378   char wave_str[16];
00379   int status, hsqfgsn, asqfsn, hgp1rgst;
00380   float percentd;
00381   uint32_t missvals, datav;
00382   uint32_t hcf1encd, hcf2encd, hps1encd, hps2encd, hps3encd; 
00383   uint32_t hwt1encd, hwt2encd, hwt3encd, hwt4encd;
00384   uint32_t hcf1pos, hcf2pos, hpl1pos, hpl2pos, hpl3pos, hwl1pos, hwl2pos;
00385   uint32_t hwl3pos, hwl4pos;
00386   uint32_t aiawvlen, aifwen, aiasen;
00387   short aifiltyp;
00388   uint32_t quallev0 = 0;
00389   uint32_t qmiss = 0;
00390 
00391   if(img->overflow) quallev0 = quallev0 | Q_OVFL;
00392   if(img->headerr) quallev0 = quallev0 | Q_HDRERR;
00393   if(img->nerrors) quallev0 = quallev0 | Q_CMPERR;
00394   if(img->last_pix_err) quallev0 = quallev0 | Q_LPXERR;
00395   if(img->reopened) quallev0 = quallev0 | Q_REOPENED;
00396   missvals = img->totalvals - img->datavals;
00397   if(missvals > 0) quallev0 = quallev0 | Q_MISS0;
00398   datav = img->totalvals;
00399   if(missvals > (uint32_t)(datav * 0.01)) quallev0 = quallev0 | Q_MISS1;
00400   if(missvals > (uint32_t)(datav * 0.05)) quallev0 = quallev0 | Q_MISS2;
00401   if(missvals > (uint32_t)(datav * 0.25)) quallev0 = quallev0 | Q_MISS3;
00402   if(missvals == datav) quallev0 = quallev0 | Q_MISSI;
00403   if(img->datavals == 0) quallev0 = quallev0 | Q_MISSALL; //high bit, no data
00404 
00405 //  if(missvals != 0) {
00406 //    qmiss = (uint32_t)(0.84 * log(missvals/img->totalvals)) + 15;
00407 //    qmiss = qmiss & 0x0f;     //need this?
00408 //    qmiss = qmiss * 256;      //bits 8-11
00409 //    quallev0 = quallev0 | qmiss;
00410 //  }
00411 
00412 if(fsn == 469769216) quallev0 = quallev0 | Q_CORRUPT; //corrupt image
00413 if(INVALtime) quallev0 = quallev0 | Q_INVALTIME; 
00414 if(cdark = drms_getkey_string(rs, "IMG_TYPE", &status)) {
00415   if(!strcmp(cdark, "DARK")) quallev0 = quallev0 | Q_DARK;
00416   free(cdark);
00417 }
00418 if(!hmiaiaflg) {        //HMI specific qual bits
00419   hsqfgsn = drms_getkey_int(rs, "HSQFGSN", &status);
00420   if(status || (fsn != hsqfgsn)) quallev0 = quallev0 | Q_NOISP;
00421   //Removed per Rock's email Re: lev0 quality updates 9Jun2010 10:22
00422   //if(hseqerr = drms_getkey_string(rs, "HSEQERR", &status)) {
00423   //  if(strcmp(hseqerr, "SUCCESS")) quallev0 = quallev0 | Q_SEQERR;
00424   //  free(hseqerr);
00425   //}
00426   if(hwltnset = drms_getkey_string(rs, "HWLTNSET", &status)) {
00427     if(!strcmp(hwltnset, "OPEN")) quallev0 = quallev0 | Q_ISSOPEN;
00428     free(hwltnset);
00429   }
00430   hcf1encd = drms_getkey_int(rs, "HCF1ENCD", &status);
00431   hcf1pos = drms_getkey_int(rs, "HCF1POS", &status);
00432   //if(!((hcf1encd == hcf1pos) || (hcf1encd == hcf1pos+1) || (hcf1encd == hcf1pos-1)))
00433   if(!((hcf1encd == hcf1pos) || (hcf1encd == hcf1pos-1)))
00434     quallev0 = quallev0 | Q_HCF1ENCD;
00435   hcf2encd = drms_getkey_int(rs, "HCF2ENCD", &status);
00436   hcf2pos = drms_getkey_int(rs, "HCF2POS", &status);
00437   //if(!((hcf2encd == hcf2pos) || (hcf2encd == hcf2pos+1) || (hcf2encd == hcf2pos-1)))
00438   if(!((hcf2encd == hcf2pos) || (hcf2encd == hcf2pos-1)))
00439     quallev0 = quallev0 | Q_HCF2ENCD;
00440   hps1encd = drms_getkey_int(rs, "HPS1ENCD", &status);
00441   hpl1pos = drms_getkey_int(rs, "HPL1POS", &status);
00442   if(!((hpl1pos == hps1encd) || (hpl1pos == (hps1encd+1) % 240)))
00443     quallev0 = quallev0 | Q_HPS1ENCD;
00444   hps2encd = drms_getkey_int(rs, "HPS2ENCD", &status);
00445   hpl2pos = drms_getkey_int(rs, "HPL2POS", &status);
00446   if(!((hpl2pos == hps2encd) || (hpl2pos == (hps2encd+1) % 240)))
00447     quallev0 = quallev0 | Q_HPS2ENCD;
00448   hps3encd = drms_getkey_int(rs, "HPS3ENCD", &status);
00449   hpl3pos = drms_getkey_int(rs, "HPL3POS", &status);
00450   if(!((hpl3pos == hps3encd) || (hpl3pos == (hps3encd+1) % 240)))
00451     quallev0 = quallev0 | Q_HPS3ENCD;
00452   hwt1encd = drms_getkey_int(rs, "HWT1ENCD", &status);
00453   hwl1pos = drms_getkey_int(rs, "HWL1POS", &status);
00454   if(!((hwl1pos == hwt1encd) || (hwl1pos == (hwt1encd+1) % 240)))
00455     quallev0 = quallev0 | Q_HWT1ENCD;
00456   hwt2encd = drms_getkey_int(rs, "HWT2ENCD", &status);
00457   hwl2pos = drms_getkey_int(rs, "HWL2POS", &status);
00458   if(!((hwl2pos == hwt2encd) || (hwl2pos == (hwt2encd+1) % 240)))
00459     quallev0 = quallev0 | Q_HWT2ENCD;
00460   hwt3encd = drms_getkey_int(rs, "HWT3ENCD", &status);
00461   hwl3pos = drms_getkey_int(rs, "HWL3POS", &status);
00462   if(!((hwl3pos == hwt3encd) || (hwl3pos == (hwt3encd+1) % 240)))
00463     quallev0 = quallev0 | Q_HWT3ENCD;
00464   hwt4encd = drms_getkey_int(rs, "HWT4ENCD", &status);
00465   hwl4pos = drms_getkey_int(rs, "HWL4POS", &status);
00466   if(!((hwl4pos == hwt4encd) || (hwl4pos == (hwt4encd+1) % 240)))
00467     quallev0 = quallev0 | Q_HWT4ENCD;
00468   //New 14Aug2013. Set Q_GPREGBIT0/1 
00469   hgp1rgst = drms_getkey_int(rs, "HGP1RGST", &status);
00470   if(hgp1rgst != DRMS_MISSING_INT) {
00471     hgp1rgst = (hgp1rgst << 28) & 0x30000000;
00472     quallev0 = quallev0 | hgp1rgst;
00473   }
00474 }
00475 else {              //AIA specific qual bits
00476   asqfsn = drms_getkey_int(rs, "ASQFSN", &status);
00477   if(fsn != asqfsn) quallev0 = quallev0 | Q_NOISP;
00478   if(aistate = drms_getkey_string(rs, "AISTATE", &status)) {
00479     if(!strcmp(aistate, "OPEN")) quallev0 = quallev0 | AQ_ISSOPEN;
00480     free(aistate);
00481   }
00482   strcpy(wave_str, "UNKNOWN");
00483   aiawvlen = drms_getkey_int(rs, "AIAWVLEN", &status);
00484   aifiltyp = drms_getkey_short(rs, "AIFILTYP", &status);
00485   aifwen = drms_getkey_int(rs, "AIFWEN", &status);
00486   aiasen = drms_getkey_int(rs, "AIASEN", &status);
00487   switch(aiawvlen) {
00488   case 9:           //9.4
00489     if(aifiltyp == 0) {
00490       if((aifwen != 269) && (aifwen != 270)) {
00491         quallev0 = quallev0 | A94Mech_Err;
00492       }
00493     }
00494     else if(aifiltyp == 1) {
00495       if((aifwen != 11) && (aifwen != 12)) {
00496         quallev0 = quallev0 | A94Mech_Err;
00497       }
00498     }
00499     else if(aifiltyp == 2) {
00500       if((aifwen != 74) && (aifwen != 75)) {
00501         quallev0 = quallev0 | A94Mech_Err;
00502       }
00503     }
00504     //All the wave_str code can be compacted, but I'm keeping it 
00505     //symetric with Rock's notes
00506     if((aifwen == 269) || (aifwen == 270)) {
00507       strcpy(wave_str, "94_THIN");
00508     }
00509     else if((aifwen == 11) || (aifwen == 12)) {
00510       strcpy(wave_str, "94_THICK");
00511     }
00512     else if((aifwen == 74) || (aifwen == 75)) {
00513       strcpy(wave_str, "94_OPEN");
00514     }
00515     break;
00516   case 1:           //13.1
00517     if(aifiltyp == 0) {
00518       if((aifwen != 269) && (aifwen != 270)) {
00519         quallev0 = quallev0 | A131Mech_Err;
00520       }
00521     }
00522     else if(aifiltyp == 1) {
00523       if((aifwen != 11) && (aifwen != 12)) {
00524         quallev0 = quallev0 | A131Mech_Err;
00525       }
00526     }
00527     else if(aifiltyp == 2) {
00528       if((aifwen != 74) && (aifwen != 75)) {
00529         quallev0 = quallev0 | A131Mech_Err;
00530       }
00531     }
00532     if((aifwen == 269) || (aifwen == 270)) {
00533       strcpy(wave_str, "131_THIN");
00534     }
00535     else if((aifwen == 11) || (aifwen == 12)) {
00536       strcpy(wave_str, "131_THICK");
00537     }
00538     else if((aifwen == 74) || (aifwen == 75)) {
00539       strcpy(wave_str, "131_OPEN");
00540     }
00541     break;
00542   case 7:           //17.1
00543     if(aifiltyp == 0) {
00544       if((aifwen != 203) && (aifwen != 204)) {
00545         quallev0 = quallev0 | A171Mech_Err;
00546       }
00547     }
00548     else if(aifiltyp == 1) {
00549       if((aifwen != 11) && (aifwen != 12)) {
00550         quallev0 = quallev0 | A171Mech_Err;
00551       }
00552     }
00553     else if(aifiltyp == 2) {
00554       //NOTE: no 171_OPEN
00555       if((aifwen != 74) && (aifwen != 75)) {
00556         quallev0 = quallev0 | A171Mech_Err;
00557       }
00558     }
00559     if((aifwen == 203) || (aifwen == 204)) {
00560       strcpy(wave_str, "171_THIN");
00561     }
00562     else if((aifwen == 11) || (aifwen == 12)) {
00563       strcpy(wave_str, "171_THICK");
00564     }
00565     break;
00566 /* !!!TBD fix below like did above after resolve MIX_THIN stuff w/Rock */
00567   case 3:           //19.3
00568     if(aifiltyp == 0) {
00569       if(aiasen != 6) {
00570         quallev0 = quallev0 | A193Mech_Err;
00571       }
00572       if((aifwen != 269) && (aifwen != 270)) {
00573         quallev0 = quallev0 | A193Mech_Err;
00574       }
00575     }
00576     else if(aifiltyp == 1) {
00577       if(aiasen != 6) {
00578         quallev0 = quallev0 | A193Mech_Err;
00579       }
00580       if((aifwen != 11) && (aifwen != 12)) {
00581         quallev0 = quallev0 | A193Mech_Err;
00582       }
00583     }
00584     else if(aifiltyp == 2) {
00585       if(aiasen != 6) {
00586         quallev0 = quallev0 | A193Mech_Err;
00587       }
00588       if((aifwen != 74) && (aifwen != 75)) {
00589         quallev0 = quallev0 | A193Mech_Err;
00590       }
00591     }
00592     if(aiasen == 6) {
00593       if((aifwen == 269) || (aifwen == 270)) {
00594         strcpy(wave_str, "193_THIN");
00595       } else if((aifwen == 11) || (aifwen == 12)) {
00596         strcpy(wave_str, "193_THICK");
00597       } else if((aifwen == 74) || (aifwen == 75)) {
00598         strcpy(wave_str, "193_OPEN");
00599       }
00600     }
00601     else {
00602       if((aifwen == 269) || (aifwen == 270)) {
00603         strcpy(wave_str, "MIX_THIN");
00604       } else if((aifwen == 11) || (aifwen == 12)) {
00605         strcpy(wave_str, "MIX_THICK");
00606       } else if((aifwen == 74) || (aifwen == 75)) {
00607         strcpy(wave_str, "MIX_OPEN");
00608       }
00609     }
00610     break;
00611   case 2:           //21.1
00612     if(aifiltyp == 0) {
00613       if(aiasen != 24) {
00614         quallev0 = quallev0 | A211Mech_Err;
00615       }
00616       if((aifwen != 203) && (aifwen != 204)) {
00617         quallev0 = quallev0 | A211Mech_Err;
00618       }
00619     }
00620     else if(aifiltyp == 1) {
00621       if(aiasen != 24) {
00622         quallev0 = quallev0 | A211Mech_Err;
00623       }
00624       if((aifwen != 137) && (aifwen != 138)) {
00625         quallev0 = quallev0 | A211Mech_Err;
00626       }
00627     }
00628     else if(aifiltyp == 2) {
00629       if(aiasen != 24) {
00630         quallev0 = quallev0 | A211Mech_Err;
00631       }
00632       if((aifwen != 74) && (aifwen != 75)) {
00633         quallev0 = quallev0 | A211Mech_Err;
00634       }
00635     }
00636     if(aiasen == 24) {
00637       if((aifwen == 203) || (aifwen == 204)) {
00638         strcpy(wave_str, "211_THIN");
00639       } else if((aifwen == 137) || (aifwen == 138)) {
00640         strcpy(wave_str, "211_THICK");
00641       } else if((aifwen == 74) || (aifwen == 75)) {
00642         strcpy(wave_str, "211_OPEN");
00643       }
00644     }
00645     else {
00646       if((aifwen == 203) || (aifwen == 204)) {
00647         strcpy(wave_str, "MIX_THIN");
00648       } else if((aifwen == 137) || (aifwen == 138)) {
00649         strcpy(wave_str, "MIX_THICK");
00650       } else if((aifwen == 74) || (aifwen == 75)) {
00651         strcpy(wave_str, "MIX_OPEN");
00652       }
00653     }
00654     break;
00655   case 8:           //30.4
00656     if(aifiltyp == 0) {
00657       if((aifwen != 203) && (aifwen != 204)) {
00658         quallev0 = quallev0 | A304Mech_Err;
00659       }
00660     }
00661     else if(aifiltyp == 1) {
00662       if((aifwen != 137) && (aifwen != 138)) {
00663         quallev0 = quallev0 | A304Mech_Err;
00664       }
00665     }
00666     else if(aifiltyp == 2) {
00667       if((aifwen != 74) && (aifwen != 75)) {
00668         quallev0 = quallev0 | A304Mech_Err;
00669       }
00670     }
00671     if((aifwen == 203) || (aifwen == 204)) {
00672       strcpy(wave_str, "304_THIN");
00673     } else if((aifwen == 137) || (aifwen == 138)) {
00674       strcpy(wave_str, "304_THICK");
00675     } else if((aifwen == 74) || (aifwen == 75)) {
00676       strcpy(wave_str, "304_OPEN");
00677     }
00678     break;
00679   case 0:           //33.5
00680     if(aifiltyp == 0) {
00681       if((aifwen != 203) && (aifwen != 204)) {
00682         quallev0 = quallev0 | A335Mech_Err;
00683       }
00684     }
00685     else if(aifiltyp == 1) {
00686       if((aifwen != 137) && (aifwen != 138)) {
00687         quallev0 = quallev0 | A335Mech_Err;
00688       }
00689     }
00690     else if(aifiltyp == 2) {
00691       if((aifwen != 74) && (aifwen != 75)) {
00692         quallev0 = quallev0 | A335Mech_Err;
00693       }
00694     }
00695     if((aifwen == 203) || (aifwen == 204)) {
00696       strcpy(wave_str, "335_THIN");
00697     } else if((aifwen == 137) || (aifwen == 138)) {
00698       strcpy(wave_str, "335_THICK");
00699     } else if((aifwen == 74) || (aifwen == 75)) {
00700       strcpy(wave_str, "335_OPEN");
00701     }
00702     break;
00703   case 4:           //160.0
00704     if((aifwen != 269) && (aifwen != 270)) {
00705       quallev0 = quallev0 | A160Mech_Err;
00706     }
00707     else strcpy(wave_str, "1600");
00708     break;
00709   case 5:           //170.0
00710     if((aifwen != 137) && (aifwen != 138)) {
00711       quallev0 = quallev0 | A170Mech_Err;
00712     }
00713     else strcpy(wave_str, "1700");
00714     break;
00715   case 6:           //450.0
00716     if((aifwen != 74) && (aifwen != 75)) {
00717       quallev0 = quallev0 | A450Mech_Err;
00718     }
00719     else strcpy(wave_str, "4500");
00720     break;
00721   }
00722   if(!strcmp(wave_str, "UNKNOWN")) quallev0 = quallev0 | AQ_INVAL_WL;
00723 }
00724   //drms_setkey_int(rs, "QUALLEV0", quallev0);  //don't use this anymore
00725   drms_setkey_int(rs, "QUALITY", quallev0);
00726   drms_setkey_string(rs, "BLD_VERS", bld_vers); //build vers to every record
00727   drms_setkey_string(rs, "WAVE_STR", wave_str);
00728   percentd = (float)((100.0 * (float)img->datavals)/(float)img->totalvals);
00729   drms_setkey_float(rs, "PERCENTD", percentd);
00730 }
00731 
00732 // Close out an image.
00733 void close_image(DRMS_Record_t *rs, DRMS_Segment_t *seg, DRMS_Array_t *array,
00734         IMG *img, int fsn)
00735 {
00736   STAT stat;
00737   int status, n, k;
00738   uint32_t missvals;
00739   long long cmdx;
00740   char tlmdsname[128];
00741 
00742   printk("*Closing image for fsn = %u\n", fsn);
00743   if(imgstat(img, &stat)) {
00744     printk("**Error on imgstat() for fsn = %u\n", fsn);
00745   }
00746   else {
00747     drms_setkey_short(rs, "DATAMIN", stat.min);
00748     drms_setkey_short(rs, "DATAMAX", stat.max);
00749     drms_setkey_short(rs, "DATAMEDN", stat.median);
00750     drms_setkey_float(rs, "DATAMEAN", stat.mean);
00751     drms_setkey_float(rs, "DATARMS", stat.rms);
00752     drms_setkey_float(rs, "DATASKEW", stat.skew);
00753     drms_setkey_float(rs, "DATAKURT", stat.kurt);
00754   }
00755   // CAMERA set by HMI_compute_exposure_times()
00756   if(hmiaiaflg) {       // except for AIA use telnum 
00757     drms_setkey_int(rs, "CAMERA", (img->telnum)+1);
00758     cmdx = drms_getkey_longlong(rs, "AIMGSHCE", &status);
00759     if (0 == cmdx) drms_setkey_int(rs, "AIMGTYP", 1);
00760   }
00761   drms_setkey_int(rs, "IMGAPID", img->apid);
00762   drms_setkey_int(rs, "CROPID", img->cropid);
00763   if(drms_setkey_int(rs, "LUTID", img->luid)) {
00764     printk("ERROR on setkey_int for LUTID\n");
00765   }
00766   drms_setkey_int(rs, "TAPCODE", img->tap);
00767   n = (img->N) & 0x1F;
00768   n = n << 3;
00769   k = (img->K) & 0x07;
00770   if(img->N == 16) { 
00771     drms_setkey_int(rs, "COMPID", 0);
00772     drms_setkey_int(rs, "BITSELID", 0);
00773   }
00774   else { 
00775     drms_setkey_int(rs, "COMPID", n+k);
00776     drms_setkey_int(rs, "BITSELID", img->R);
00777   }
00778   drms_setkey_int(rs, "FID", img->fid);
00779   drms_setkey_int(rs, "TOTVALS", img->totalvals);
00780   drms_setkey_int(rs, "DATAVALS", img->datavals);
00781   drms_setkey_int(rs, "NPACKETS", img->npackets);
00782   drms_setkey_int(rs, "NERRORS", img->nerrors);
00783   drms_setkey_short(rs, "EOIERROR", img->last_pix_err);
00784   drms_setkey_short(rs, "HEADRERR", img->headerr);
00785   drms_setkey_short(rs, "OVERFLOW", img->overflow);
00786   missvals = img->totalvals - img->datavals;
00787   drms_setkey_int(rs, "MISSVALS", missvals);
00788   snprintf(tlmdsname, 128, "%s[%s]", tlmseriesname, tlmnamekeyfirst);
00789   drms_setkey_string(rs, "TLMDSNAM", tlmdsname);
00790   unsigned int pts = img->first_packet_time >> 16;
00791   int ptss = img->first_packet_time & 0xffff;
00792   TIME fpt = SDO_to_DRMS_time(pts, ptss);
00793   drms_setkey_double(rs, "IMGFPT", fpt);
00794   drms_setkey_double(rs, "DATE", CURRENT_SYSTEM_TIME);
00795   do_quallev0(rs, img, fsn);        //set the QUALLEV0 keyword
00796 
00797   status = drms_segment_write(seg, array, 0);
00798   if (status) {
00799     printk("ERROR: drms_segment_write error=%d for fsn=%u\n", status, fsn);
00800   }
00801   add_small_array(rs, array, 8, 16); //add Phil's png and small fits
00802   array->data = NULL;        // must do before free 
00803   drms_free_array(array);
00804   if((status = drms_close_record(rs, DRMS_INSERT_RECORD))) {
00805     printk("**ERROR: drms_close_record failed for %s fsn=%u\n",
00806                         lev0seriesname, fsn);
00807   }
00808   img->initialized = 0;     //indicate image is ready for use again
00809   //imgdecode_init_hack(img);
00810 }
00811 
00812 // Got a fatal error. 
00813 void abortit(int stat)
00814 {
00815   printk("***Abort in progress ...\n");
00816   printk("**Exit ingest_lev0 w/ status = %d\n", stat);
00817   if (h0logfp) fclose(h0logfp);
00818   exit(stat);
00819 }
00820 
00821 
00822 int compare_names(const void *a, const void *b)
00823 {
00824   NAMESORT *x=(NAMESORT *)a, *y=(NAMESORT *)b;
00825   return(strcmp(x->name+4, y->name+4)); // skip VC02/VC05 in compare 
00826 }
00827 
00828 unsigned short MDI_getshort (unsigned char *c)    //  machine independent  
00829 {
00830   unsigned short s = 0;
00831 
00832   s = (unsigned short) *c++ << 8;
00833   s |= (unsigned short) *c;
00834   return s;
00835 }
00836 
00837 //fsn_normal_new_image()
00838 int fsn_normal_new_image() 
00839 {
00840   int dstatus, i;
00841 
00842   // start a new image 
00843   Img->initialized = 0;
00844   Img->reopened = 0;
00845   errmsgcnt = 0;
00846   fsn_prev = fsnx;
00847   sprintf(tlmnamekeyfirst, "%s", tlmnamekey);   //save for TLMDSNAM
00848   rs = drms_create_record(drms_env, lev0seriesname, DRMS_PERMANENT, &dstatus);
00849   if(dstatus) {
00850     if(dstatus == DRMS_ERROR_SUMOPEN) {
00851       printk("**ERROR: DRMS can't open w/SUMS. Aborting...\n");
00852       abortit(4);
00853     }
00854     printk("**ERROR: Can't create record for %s fsn=%u\n", lev0seriesname, fsnx);
00855     return(1);
00856   }
00857   dstatus = drms_setkey_int(rs, "FSN", fsnx);
00858   if(!(segment = drms_segment_lookup(rs, "image"))) {
00859     printk("No drms_segment_lookup(rs, image)\n");
00860     return(1);
00861   }
00862   //must initialize Img in case no data pkt comes in for this image
00863 #if 0
00864     Img->datavals = 0;
00865     Img->npackets = 0;
00866     Img->nerrors = 0;
00867     Img->last_pix_err = 0;
00868     Img->first_packet_time = UINT64_MAX;
00869     for (i = 0; i < MAXPIXELS; ++i)
00870         Img->dat[i] = BLANK;
00871     for (i = 0; i < MAXHIST; ++i)
00872         Img->hist[i] = 0;
00873 #endif
00874     imgdecode_init_hack(Img);
00875 
00876   rdat = Img->dat;
00877   segArray = drms_array_create(DRMS_TYPE_SHORT,
00878                                        segment->info->naxis,
00879                                        segment->axis,
00880                                        rdat,
00881                                        &dstatus);
00882   ImgC = Img;       //set current image pointer
00883   return(0);
00884 } 
00885 
00886 // An fsn has changed in the context of a normal stream of
00887 // tlm files. Returns 0 on success.
00888 int fsn_change_normal()
00889 {
00890   int dstatus, rstatus, compid, n, k;
00891   char reopen_dsname[256];
00892   char *cptr;
00893 
00894   printk("*FSN has changed from %u to %u\n", fsn_prev, fsnx);
00895   if(fsn_prev == 0) {   //startup mode. restore any prev image
00896     errmsgcnt = 0;
00897     fsn_prev = fsnx;
00898     if(hmiaiaflg)
00899       sprintf(reopen_dsname, "%s[%u]", LEV0SERIESNAMEAIA, fsnx);
00900     else
00901       sprintf(reopen_dsname, "%s[%u]", LEV0SERIESNAMEHMI, fsnx);
00902     printk("Open normal prev ds: %s\n", reopen_dsname);
00903     rset = drms_open_records(drms_env, reopen_dsname, &rstatus);
00904     if(rstatus) {
00905       printk("Can't do drms_open_records(%s)\n", reopen_dsname);
00906       return(1);          // !!!TBD
00907     }
00908     if(!rset || (rset->n == 0) || rstatus) {
00909       printk("No prev ds\n");     // start a new image
00910       fsn_normal_new_image();
00911     }
00912     else {
00913       Img->initialized = 1;
00914       Img->reopened = 1;
00915       Img->fsn = fsnx;
00916       Img->apid = appid;
00917       rs_old = rset->records[0];
00918       rs = drms_clone_record(rs_old, DRMS_PERMANENT, DRMS_COPY_SEGMENTS, &rstatus);
00919       if(rstatus) {
00920         printk("Can't do drms_clone_record()\n");
00921         return(1);      // !!!TBD ck 
00922       }
00923       drms_close_records(rset, DRMS_FREE_RECORD);
00924       rstatus = drms_setkey_int(rs, "FSN", fsnx);
00925       Img->telnum = drms_getkey_int(rs, "CAMERA", &rstatus);
00926       Img->telnum--;
00927       Img->cropid = drms_getkey_int(rs, "CROPID", &rstatus);
00928       Img->fid = drms_getkey_int(rs, "FID", &rstatus); 
00929       Img->luid = drms_getkey_int(rs, "LUTID", &rstatus);
00930       if(rstatus) {
00931         Img->luid = 0;  
00932         printk("ERROR on getkey_int for LUTID\n");
00933       }
00934       Img->tap = drms_getkey_int(rs, "TAPCODE", &rstatus);
00935       compid = drms_getkey_int(rs, "COMPID", &rstatus);
00936       if(compid == 0) {
00937         n = 16; k = 0;
00938       }
00939       else {
00940         k = compid & 0x07;      //low 3 bits
00941         n = compid >> 3;            //next 5 bits
00942         n = n & 0x1F;
00943       }
00944       Img->N = n;
00945       Img->K = k;
00946       Img->R = drms_getkey_int(rs, "BITSELID", &rstatus);
00947       Img->overflow = drms_getkey_int(rs, "OVERFLOW", &rstatus);
00948       Img->headerr = drms_getkey_int(rs, "HEADRERR", &rstatus);
00949       Img->totalvals = drms_getkey_int(rs, "TOTVALS", &rstatus);
00950       Img->datavals = drms_getkey_int(rs, "DATAVALS", &rstatus);
00951       Img->npackets = drms_getkey_int(rs, "NPACKETS", &rstatus);
00952       Img->nerrors = drms_getkey_int(rs, "NERRORS", &rstatus);
00953       Img->last_pix_err = drms_getkey_int(rs, "EOIERROR", &rstatus);
00954 
00955       TIME fpt = drms_getkey_double(rs, "IMGFPT", &rstatus);
00956       Img->first_packet_time = round(65536.0*(fpt - sdo_epoch));
00957       if(Img->first_packet_time == 0) 
00958         Img->first_packet_time = UINT64_MAX; //fix for stuck 1958 value
00959       snprintf(oldtlmdsnam, 128, "%s", drms_getkey_string(rs, "TLMDSNAM",&rstatus));
00960       cptr = strstr(oldtlmdsnam, "VC");
00961       if(cptr) 
00962         snprintf(tlmnamekeyfirst, 23, "%s", cptr);
00963       else
00964         sprintf(tlmnamekeyfirst, "%s", "UNK");
00965       segment = drms_segment_lookupnum(rs, 0);
00966       cArray = drms_segment_read(segment, DRMS_TYPE_SHORT, &rstatus);
00967       if(rstatus) {
00968         printk("Can't do drms_segment_read()\n");
00969         return(1);      // !!!!TBD ck 
00970       }
00971       short *adata = (short *)cArray->data;
00972       memcpy(Img->dat, adata, 2*MAXPIXELS);
00973       rdat = Img->dat;
00974       segArray = drms_array_create(DRMS_TYPE_SHORT, 
00975                                          segment->info->naxis,
00976                                          segment->axis,
00977                                          rdat,
00978                                          &dstatus);
00979       ImgC = Img;           //set current image pointer
00980       drms_free_array(cArray); //must free from drms_segment_read()
00981     }
00982   }
00983   else {
00984     if(rs) {        // make sure have a created record
00985       close_image(rs, segment, segArray, Img, fsn_prev);
00986       //force a commit to DB on an image boundry & not during a rexmit
00987       if(imagecnt++ >= IMAGE_NUM_COMMIT) {
00988         printk("drms_server_end_transaction()\n");
00989         drms_server_end_transaction(drms_env, 0 , 0);
00990         printk("drms_server_begin_transaction()\n");
00991         drms_server_begin_transaction(drms_env); //start another cycle
00992         imagecnt = 0;
00993       }
00994       fileimgcnt++;
00995     }
00996     else {
00997       printk("**ERROR: Null record ptr for an opened image fsn=%u\n",fsn_prev);
00998     }
00999     if(fsn_normal_new_image()) {
01000       printk("**ERROR: Can't create new image\n");
01001       return(1);
01002     }
01003   }
01004   return(0);
01005 }
01006 
01007 // An fsn has changed in the context of a retransmitted or higher version
01008 // tlm file. Returns 0 on success.
01009 int fsn_change_rexmit() 
01010 {
01011   char rexmit_dsname[256];
01012   int rstatus, dstatus, compid, n, k;
01013   char *cptr;
01014 
01015   if(fsn_prev != 0) {   // close image of prev fsn if not 0 
01016     if(rsc) {       //make sure have created record
01017       close_image(rsc, segmentc, oldArray, ImgO, fsn_prev);
01018       imagecnt++;
01019       fileimgcnt++;
01020     }
01021     else {
01022       printk("**ERROR: Null record ptr for an rexmit image fsn=%u\n",fsn_prev);
01023     }
01024   }
01025   errmsgcnt = 0;
01026   fsn_prev = fsnx;
01027   if(hmiaiaflg) 
01028     sprintf(rexmit_dsname, "%s[%u]", LEV0SERIESNAMEAIA, fsnx);
01029   else 
01030     sprintf(rexmit_dsname, "%s[%u]", LEV0SERIESNAMEHMI, fsnx);
01031   printk("Open prev ds: %s\n", rexmit_dsname);
01032   rset = drms_open_records(drms_env, rexmit_dsname, &rstatus); 
01033   if(rstatus) {
01034     printk("Can't do drms_open_records(%s)\n", rexmit_dsname);
01035     return(1);      // !!!TBD 
01036   }
01037   if(!rset || (rset->n == 0) || rstatus) {
01038 startnew:
01039     printk("No prev ds\n"); // start a new image 
01040     ImgO->initialized = 0;
01041     ImgO->reopened = 0;
01042     imgdecode_init_hack(ImgO);
01043     sprintf(tlmnamekeyfirst, "%s", tlmnamekey); //save for TLMDSNAM
01044     rsc = drms_create_record(drms_env, lev0seriesname, DRMS_PERMANENT, &rstatus);
01045     if(rstatus) {
01046       if(rstatus == DRMS_ERROR_SUMOPEN) {
01047         printk("**ERROR: DRMS can't open w/SUMS. Aborting...\n");
01048         abortit(4);
01049       }
01050       printk("Can't create record for %s fsn=%u\n", lev0seriesname, fsnx);
01051       return(1);                     // !!!TBD ck this 
01052     }
01053     rstatus = drms_setkey_int(rsc, "FSN", fsnx);
01054     if(!(segmentc = drms_segment_lookup(rsc, "image"))) {
01055       printk("No drms_segment_lookup(rsc, image)\n");
01056       return(1);
01057     }
01058     rdat = ImgO->dat;
01059     oldArray = drms_array_create(DRMS_TYPE_SHORT,
01060                                        segmentc->info->naxis,
01061                                        segmentc->axis,
01062                                        rdat,
01063                                        &dstatus);
01064   }
01065   else {
01066     ImgO->initialized = 1;
01067     ImgO->reopened = 1;
01068     ImgO->fsn = fsnx;
01069     ImgO->apid = appid;
01070     rs_old = rset->records[0];
01071     rsc = drms_clone_record(rs_old, DRMS_PERMANENT, DRMS_COPY_SEGMENTS, &rstatus);
01072     if(rstatus) {
01073       printk("Can't do drms_clone_record()\n");
01074       printk("Assume this was a temp ds and the segments are now gone...\n");
01075       goto startnew;
01076       //return(1);      // !!!TBD ck 
01077     }
01078     drms_close_records(rset, DRMS_FREE_RECORD);
01079     rstatus = drms_setkey_int(rsc, "FSN", fsnx);
01080     ImgO->telnum = drms_getkey_int(rsc, "CAMERA", &rstatus);
01081     ImgO->telnum--;
01082     ImgO->cropid = drms_getkey_int(rsc, "CROPID", &rstatus);
01083     ImgO->fid = drms_getkey_int(rsc, "FID", &rstatus); 
01084     ImgO->luid = drms_getkey_int(rsc, "LUTID", &rstatus);
01085     ImgO->tap = drms_getkey_int(rsc, "TAPCODE", &rstatus);
01086     compid = drms_getkey_int(rsc, "COMPID", &rstatus);
01087     if(compid == 0) {
01088       n = 16; k = 0;
01089     }
01090     else {
01091       k = compid & 0x07;              //low 3 bits
01092       n = compid >> 3;                        //next 5 bits
01093       n = n & 0x1F;
01094     }
01095     ImgO->N = n;
01096     ImgO->K = k;
01097     ImgO->R = drms_getkey_int(rsc, "BITSELID", &rstatus);
01098     ImgO->overflow = drms_getkey_int(rsc, "OVERFLOW", &rstatus);
01099     ImgO->headerr = drms_getkey_int(rsc, "HEADRERR", &rstatus);
01100     ImgO->totalvals = drms_getkey_int(rsc, "TOTVALS", &rstatus);
01101     ImgO->datavals = drms_getkey_int(rsc, "DATAVALS", &rstatus);
01102     ImgO->npackets = drms_getkey_int(rsc, "NPACKETS", &rstatus);
01103     ImgO->nerrors = drms_getkey_int(rsc, "NERRORS", &rstatus);
01104     ImgO->last_pix_err = drms_getkey_int(rsc, "EOIERROR", &rstatus);
01105 
01106     TIME fpt = drms_getkey_double(rsc, "IMGFPT", &rstatus);
01107     ImgO->first_packet_time = round(65536.0*(fpt - sdo_epoch));
01108     if(ImgO->first_packet_time == 0) 
01109       ImgO->first_packet_time = UINT64_MAX; //fix for stuck 1958 value
01110     snprintf(oldtlmdsnam, 128, "%s", drms_getkey_string(rsc, "TLMDSNAM",&rstatus));
01111     cptr = strstr(oldtlmdsnam, "VC");
01112     if(cptr) 
01113       snprintf(tlmnamekeyfirst, 23, "%s", cptr);
01114     else
01115       sprintf(tlmnamekeyfirst, "%s", "UNK");
01116     segmentc = drms_segment_lookupnum(rsc, 0);
01117     cArray = drms_segment_read(segmentc, DRMS_TYPE_SHORT, &rstatus);
01118     if(rstatus) {
01119       printk("Can't do drms_segment_read()\n");
01120       return(1);        // !!!!TBD ck 
01121     }
01122     short *adata = (short *)cArray->data;
01123     memcpy(ImgO->dat, adata, 2*MAXPIXELS);
01124     rdat = ImgO->dat;
01125     oldArray = drms_array_create(DRMS_TYPE_SHORT, 
01126                                        segmentc->info->naxis,
01127                                        segmentc->axis,
01128                                        rdat,
01129                                        &dstatus);
01130     drms_free_array(cArray); //must free from drms_segment_read()
01131   }
01132   ImgC = ImgO;      //set current image pointer
01133   return(0);
01134 }
01135 
01136 // Process the tlm file to validate and to extract the lev0 image.
01137 int get_tlm(char *file, int rexmit, int higherver)
01138 {
01139   FILE *fpin;
01140   unsigned char cbuf[PKTSZ];
01141   long long gap_42_cnt;
01142   int status, rstatus, fpkt_cnt, i, j, sync_bad_cnt;
01143   int datval, eflg, firstflg;
01144   unsigned int cnt1, cnt2, cnt3, gap_24_cnt;
01145   int zero_pn;
01146   unsigned short pksync1, pksync2;
01147   float ftmp;
01148   int decode_status=0;
01149   unsigned int Fsn;
01150 
01151   if(!(fpin = fopen(file, "r"))) {  // open the tlm input 
01152     printk("*Can't open tlm file %s\n", file);
01153     return(1);
01154   }
01155   BeginTimer(1);            //time tlm file processing
01156   printk("*Processing tlm file %s\n", file);
01157   fpkt_cnt = sync_bad_cnt = eflg = errmsgcnt = fileimgcnt = 0;
01158   zero_pn = gap_24_cnt = gap_42_cnt = 0;
01159   firstflg = 1; 
01160   if(rexmit || higherver) {
01161     if(Img->initialized) {      // close normal mode image
01162       if(rs) {
01163         close_image(rs, segment, segArray, Img, fsn_prev);
01164         printk("drms_server_end_transaction()\n");
01165         drms_server_end_transaction(drms_env, 0 , 0);
01166         printk("drms_server_begin_transaction()\n");
01167         drms_server_begin_transaction(drms_env); //start another cycle
01168         imagecnt = 0;
01169       }
01170       else
01171         printk("**ERROR: Null record ptr for an opened image fsn=%u\n",fsn_prev);
01172     }
01173     fsn_pre_rexmit = fsn_prev;      // restore this at end of rexmit tlm file
01174     fsn_prev = 0;           // cause a new image 
01175   }
01176   // read a VCDU packet 
01177   while((status = fread(cbuf,sizeof(char),PKTSZ,fpin) ) == PKTSZ) {
01178     pksync1 = MDI_getshort(cbuf);
01179     pksync2 = MDI_getshort(cbuf+2);
01180     if((pksync1 == 0) && (pksync2 == 0)) { // skip 0 pn code 
01181       if(!zero_pn) {            // give msg for 1st one only 
01182         printk("*0 PN code at pkt# %d\n", fpkt_cnt);
01183         printk("*Subsequent ones will be ignored until non-0 again\n");
01184         zero_pn = 1;
01185       }
01186       fpkt_cnt++;           // count # of pkts found 
01187       continue;
01188     }
01189     if((pksync1 != 0x1acf) || (pksync2 != 0xfc1d)) {
01190       printk("*Lost sync at VCDU pkt# %d. pksync1=%x pksync2=%x\n", 
01191         fpkt_cnt, pksync1, pksync2);
01192       fpkt_cnt++;           // count # of pkts found 
01193       eflg++;
01194       if(sync_bad_cnt++ > 4) {
01195         printk("**Too many out of sync packets.\n");
01196         //return(1);
01197         return(0);      //changed on 01Oct2010
01198       }
01199       printk("  Will attempt to press on...\n");
01200       zero_pn = 0;
01201       continue;
01202     }
01203     if(firstflg) {      // print first good sync found 
01204       printk("*VCDU pkt# %d sync = %x %x\n", fpkt_cnt, pksync1, pksync2);
01205     }
01206     fpkt_cnt++;         // count # of pkts found 
01207     // get 24 bit VCDU counter 
01208     cnt1 = MDI_getshort(cbuf+6);
01209     cnt2 = MDI_getshort(cbuf+8);
01210     cnt2 = (cnt2 >> 8)& 0xFF;
01211     cnt2 = ((cnt1 << 8)& 0xFF00) + cnt2;
01212     cnt1 = (cnt1 >> 8)& 0xFF;
01213     vcdu_24_cnt = (cnt1*65536) + cnt2;
01214     if(vcdu_24_cnt_next != vcdu_24_cnt) {
01215       printk("*VCDU 24bit seq num out of sequence. exp: %u  rec: %u\n", 
01216         vcdu_24_cnt_next, vcdu_24_cnt);
01217       if(vcdu_24_cnt_next > vcdu_24_cnt) {
01218         printk("*NOTE: VCDU 24 bit counter retarded\n"); //cntr does go thru 0
01219         printk("*NOTE: gap report will be inaccurate (tbd)\n");
01220       }
01221       if(!firstflg) {       // don't count gap across .tlm files 
01222         gap_24_cnt += vcdu_24_cnt - vcdu_24_cnt_next;
01223       }
01224     }
01225     vcdu_24_cnt_next = vcdu_24_cnt + 1;
01226     // now get the 42bit IM_PDU counter 
01227     cnt1 = MDI_getshort(cbuf+10);
01228     cnt2 = MDI_getshort(cbuf+12);
01229     cnt3 = MDI_getshort(cbuf+14);
01230     cnt1 = cnt1 & 0x03ff;
01231     vcdu_seq_num = (cnt1*4294967296) + (cnt2*65536) + cnt3;
01232     // printk("vcdu_seq_num = %lld\n", vcdu_seq_num); 
01233     if(vcdu_seq_num_next != vcdu_seq_num) {
01234       printk("*IM_PDU seq num out of sequence. exp: %lld  rec: %lld\n", 
01235         vcdu_seq_num_next, vcdu_seq_num);
01236       if(vcdu_seq_num_next > vcdu_seq_num) {
01237         printk("*NOTE: IM_PDU 42 bit counter retarded\n");
01238         printk("*NOTE: gap report will be inaccurate\n");
01239       }
01240       if(!firstflg) {       // don't count gap across .tlm files 
01241         gap_42_cnt += vcdu_seq_num - vcdu_seq_num_next;
01242       }
01243       eflg++;
01244     }
01245     firstflg = 0;
01246     vcdu_seq_num_next = vcdu_seq_num + 1;
01247     // get the App ID. Low 11 bit of short at buf+18 
01248     appid = MDI_getshort(cbuf+18);
01249     appid = appid & 0x07ff;
01250     if(hmiaiaflg) {     //aia
01251       //testid1 = 509; testid2 = 519;
01252       testid1 = 507; testid2 = 517;
01253     }
01254     else {
01255       //testid1 = 409; testid2 = 419;
01256       testid1 = 407; testid2 = 417;
01257     }
01258     //if(appid == TESTAPPID) {  // appid of test pattern 
01259     if((appid == testid1) || (appid == testid2)) {
01260       if(errmsgcnt++ < MAXERRMSGCNT) {
01261         printk("*Test ApID of %0x found for IM_PDU Cntr = %lld\n", 
01262             appid, vcdu_seq_num);
01263         for(i=0, j=TESTVALUE; i < 877; i=i+2, j++) {
01264           datval = MDI_getshort(cbuf+32+i); // next data value 
01265           if(datval != j) {
01266             printk("*Test data value=%0x, expected=%0x for IM_PDU Cntr=%lld\n", 
01267         datval, j, vcdu_seq_num);
01268             printk("*File = %s\n", file);
01269             eflg++;
01270             break;      // skip the rest of this packet 
01271           }
01272         }
01273       }
01274       continue;         // go on to next packet 
01275     }
01276 
01277     //printk("$$$$$ appid found =  0x%x %d\n", appid, appid); //!!TEMP
01278     // Parse tlm packet headers. 
01279     if(appid == APID_HMI_SCIENCE_1 || appid == APID_HMI_SCIENCE_2 || 
01280     appid == APID_AIA_SCIENCE_1 || appid == APID_AIA_SCIENCE_2)
01281     {
01282       cnt1 = MDI_getshort(cbuf+32);
01283       cnt2 = MDI_getshort(cbuf+34);
01284       fsnx = (unsigned int)(cnt1<<16)+(unsigned int)(cnt2);
01285       fsnx = fsnx & 0x3fffffff;     //low 30bits for fsn */
01286       if(fsnx == 0) continue;       //a 0 fsn is not acceptable
01287       if(rexmit || higherver) {
01288         if(fsnx != fsn_prev) {          // the fsn has changed
01289           if(fsn_change_rexmit()) { //handle old & new images
01290             printk("***FATAL ERROR in fsn_change_rexmit()\n");
01291             return(1);
01292           }
01293         }
01294         else {              //prev could be hk w/diff apid
01295           ImgC->apid = appid;
01296         }
01297       }
01298       else {            // continuing normal stream
01299         if(fsnx != fsn_prev) {          // the fsn has changed 
01300           if(fsn_change_normal()) { //handle old & new images
01301             printk("***FATAL ERROR in fsn_change_normal()\n");
01302             return(1);
01303           }
01304         }
01305       }
01306       // send the sci data to Keh-Cheng. call with pointer to M_PDU_Header 
01307       rstatus = imgdecode((unsigned short *)(cbuf+10), ImgC);
01308       switch(rstatus) {
01309       case 0:
01310         // A science data VCDU was successfully decoded 
01311         break;
01312       case IMGDECODE_DECOMPRESS_ERROR:
01313         if(errmsgcnt++ < MAXERRMSGCNT)
01314           printk("*imgdecode() ret: IMGDECODE_DECOMPRESS_ERROR\n");
01315         break;
01316       case IMGDECODE_TOO_MANY_PIXELS:
01317         if(errmsgcnt++ < MAXERRMSGCNT)
01318           printk("*imgdecode() ret: IMGDECODE_TOO_MANY_PIXELS\n");
01319         break;
01320       case IMGDECODE_BAD_N:
01321         if(errmsgcnt++ < MAXERRMSGCNT)
01322           printk("*imgdecode() ret: IMGDECODE_BAD_N\n");
01323         break;
01324       case IMGDECODE_BAD_APID:
01325         if(errmsgcnt++ < MAXERRMSGCNT)
01326           printk("*imgdecode() ret: IMGDECODE_BAD_APID\n");
01327         break;
01328       case IMGDECODE_NO_LOOKUP_TABLE:
01329         if(errmsgcnt++ < MAXERRMSGCNT)
01330           printk("*imgdecode() ret: IMGDECODE_NO_LOOKUP_TABLE\n");
01331         break;
01332       case IMGDECODE_LOOKUP_ID_MISMATCH:
01333         if(errmsgcnt++ < MAXERRMSGCNT)
01334           printk("*imgdecode() ret: IMGDECODE_LOOKUP_ID_MISMATCH\n");
01335         break;
01336       case IMGDECODE_BAD_LOOKUP_TABLE:
01337         if(errmsgcnt++ < MAXERRMSGCNT)
01338           printk("*imgdecode() ret: IMGDECODE_BAD_LOOKUP_TABLE\n");
01339         break;
01340       case IMGDECODE_NO_CROP_TABLE:
01341         if(errmsgcnt++ < MAXERRMSGCNT)
01342           printk("*imgdecode() ret: IMGDECODE_NO_CROP_TABLE\n");
01343         break;
01344       case IMGDECODE_CROP_ID_MISMATCH:
01345         if(errmsgcnt++ < MAXERRMSGCNT)
01346           printk("*imgdecode() ret: IMGDECODE_CROP_ID_MISMATCH\n");
01347         break;
01348       case IMGDECODE_BAD_CROP_GEOMETRY:
01349         if(errmsgcnt++ < MAXERRMSGCNT)
01350           printk("*imgdecode() ret: IMGDECODE_BAD_CROP_GEOMETRY\n");
01351         break;
01352       case IMGDECODE_BAD_CROP_TABLE:
01353         if(errmsgcnt++ < MAXERRMSGCNT)
01354           printk("*imgdecode() ret: IMGDECODE_BAD_CROP_TABLE\n");
01355         break;
01356       case IMGDECODE_BAD_CROP_SKIP_TAKE:
01357         if(errmsgcnt++ < MAXERRMSGCNT)
01358           printk("*imgdecode() ret: IMGDECODE_BAD_CROP_SKIP_TAKE\n");
01359         break;
01360       case IMGDECODE_BAD_OFFSET:
01361         if(errmsgcnt++ < MAXERRMSGCNT)
01362           printk("*imgdecode() ret: IMGDECODE_BAD_OFFSET\n");
01363         break;
01364       case IMGDECODE_OUT_OF_MEMORY:
01365         if(errmsgcnt++ < MAXERRMSGCNT)
01366           printk("*imgdecode() ret: IMGDECODE_OUT_OF_MEMORY\n");
01367         break;
01368       default:
01369         if(errmsgcnt++ < MAXERRMSGCNT)
01370           printk("*imgdecode() ret: unknown err status = %d:\n", rstatus);
01371         break;
01372       }
01373     }
01374     else {          // send the HK data to Carl 
01375       //printk("$$$ appid assumed hk =  0x%x %d\n", appid, appid); //!!TEMP
01376       decode_status = decode_next_hk_vcdu((unsigned short *)(cbuf+10), &Hk, &Fsn);
01377       switch (decode_status) {
01378         case SUCCESS_HK_NEED_TO_WTD_CTD:
01379           printk("*ISP found for fsn = %u\n", Fsn);
01380           fsnx = Fsn;
01381           if(rexmit || higherver) {
01382             if(fsnx != fsn_prev) {          // the fsn has changed
01383               if(fsn_change_rexmit()) { //handle old & new images
01384                 printk("***FATAL ERROR in fsn_change_rexmit()\n");
01385                 return(1);
01386               }
01387             }
01388             //calculate and setkey some values from the keywords returned
01389             HMI_compute_exposure_times(rsc, Hk->keywords, hmiaiaflg);
01390             whk_status = write_hk_to_drms(rsc, &Hk); //ISP keywords to drms
01391             if(!hmiaiaflg) {
01392               if(whk_status = set_HMI_mech_values(rsc)) {
01393                 printk("***ERROR: mechanism position keywords are wrong!\n");
01394               }
01395             }
01396           }
01397           else {                    //normal stream
01398             if(fsnx != fsn_prev) {          // the fsn has changed 
01399               if(fsn_change_normal()) { //handle old & new images
01400                 printk("***FATAL ERROR in fsn_change_normal()\n");
01401                 return(1);
01402               }
01403             }
01404             //calculate and setkey some values from the keywords returned
01405             HMI_compute_exposure_times(rs, Hk->keywords, hmiaiaflg);
01406             whk_status = write_hk_to_drms(rs, &Hk); //ISP keywords to drms
01407             if(!hmiaiaflg) {
01408               if(whk_status = set_HMI_mech_values(rs)) {
01409                 printk("***ERROR: mechanism position keywords are wrong!\n");
01410               }
01411             }
01412           }
01413           hk_ccsds_free(&Hk);
01414           rstatus=0;
01415           break;
01416         case SUCCESS_HK_NEED_TO_CTD:
01417           /*NO need to write  keywords's to drms for level0 data series 
01418             BUT need to commit to drms for level0 by APID data series*/
01419           break;
01420         case SUCCESS_SKIP_IMAGE:
01421           printk("decode_next_hk_vcdu() ret:Warning:SUCCESS_SKIP_IMAGE\n");
01422           break;
01423         case SUCCESS_SKIP_PROCESSING_APID:
01424           printk("decode_next_hk_vcdu() ret:Warning:SUCCESS_SKIP_PROCESSING_APID:\n");
01425           break;
01426         case ERROR_HK_FAILED_GETTING_FSN:
01427           printk("decode_next_hk_vcdu() ret:Warning:ERROR_HK_FAILED_GETTING_FSN:\n");
01428           break;
01429         case ERROR_NODATA:
01430           printk("decode_next_hk_vcdu() ret:Warning:ERROR_NODATA\n");
01431           break;
01432         case ERROR_HK_ENVIRONMENT_VARS_NOT_SET:
01433           printk("decode_next_hk_vcdu() ret:ERROR:ERROR_HK_ENVIRONMENT_VARS_NOT_SET\n");
01434            break;
01435         case ERROR_HK_FAILED_OPEN_DAYFILE:
01436           printk("decode_next_hk_vcdu() ret:ERROR:ERROR_HK_FAILED_OPEN_DAYFILE\n");
01437            break;
01438         default:
01439           printk("decode_next_hk_vcdu() ret:Warning:Unexpected return code for decode_status:<%d>:\n", decode_status);
01440           break;
01441       }
01442 
01443     }
01444   }             // end of rd of vcdu pkts 
01445   fclose(fpin);
01446   if(!eflg) {
01447     printk("*No errors in tlm file\n");
01448   }
01449   if(rexmit || higherver) { // close the opened record 
01450     if(rsc) {           // make sure have created record
01451       close_image(rsc, segmentc, oldArray, ImgO, fsnx);
01452       printk("drms_server_end_transaction()\n");  //commit at end of file
01453       drms_server_end_transaction(drms_env, 0 , 0);
01454       printk("drms_server_begin_transaction()\n\n");
01455       drms_server_begin_transaction(drms_env); //start another cycle
01456       imagecnt = 0;
01457       fileimgcnt++;
01458       //fsn_prev = fsn_pre_rexmit; // restore orig for next normal .tlm file
01459       fsn_prev = 0;     //need for restart of normal mode
01460       rsc = 0;
01461     }
01462   }
01463   ftmp = EndTimer(1);
01464   printk("**Processed %s\n**with %d images and %d VCDUs in %f sec\n\n",
01465     file, fileimgcnt, fpkt_cnt, ftmp);
01466 
01467   //now commit to DB in case another file doesn't come in
01468 /***************************TEMP***********************************
01469 NOTE: Can't do this unless close current image and then set fsn=0
01470       so that the next tlm file will start by opening the image again.
01471       I don't think that we want this. Instead find a way to get the ^C
01472       instead of DRMS, and then do a close and commit.
01473   printk("drms_server_end_transaction()\n");
01474   drms_server_end_transaction(drms_env, 0 , 0);
01475   printk("drms_server_begin_transaction()\n\n");
01476   drms_server_begin_transaction(drms_env); //start another cycle
01477   imagecnt = 0;
01478 ***************************TEMP***********************************/
01479 
01480   if(fpkt_cnt != total_tlm_vcdu) {
01481     printk("**WARNING: Found #vcdu=%d; expected=%d\n", fpkt_cnt, total_tlm_vcdu);
01482   }
01483   //if(gap_24_cnt != total_missing_vcdu) {
01484   //  printk("**WARNING: VCDU 24bit cntr gaps=%d; expected=%d\n",
01485   //      gap_24_cnt, total_missing_vcdu);
01486   //}
01487   if(gap_42_cnt != total_missing_im_pdu) {
01488     printk("**WARNING: IM_PDU 42bit cntr gaps=%lld; expected=%lld\n",
01489      gap_42_cnt, total_missing_im_pdu);
01490   }
01491   ignoresigalrmflg = 1;     //got a file. noop next alarm signal timeout
01492   return(0);
01493 }
01494 
01495 // This is the main loop that gets the .qac and .tlm files and 
01496 // puts them into ds tlmseriesname and extracts the lev0 and puts it in 
01497 // lev0seriesname in DRMS.
01498 void do_ingest()
01499 {
01500   FILE *fp;
01501   DRMS_Record_t *rs_tlm;
01502   DIR *dfd;
01503   NAMESORT *nameptr;
01504   struct dirent *dp;
01505   int i, j, status, mvstat;
01506   int rexmit, higherversion;
01507   char path[DRMS_MAXPATHLEN];
01508   char name[128], line[128], tlmfile[128], tlmname[96];
01509   char cmd[256], xxname[128], vername[16];
01510   char *token, *filename;
01511 
01512   if((dfd=opendir(tlmdir)) == NULL) {
01513     printk("**Can't opendir(%s) to find files\n", tlmdir);
01514     abortit(3);
01515   }
01516   i = 0;
01517   if((nameptr = (NAMESORT *)malloc(MAXFILES * sizeof(NAMESORT))) == NULL) {
01518     printk("***Can't alloc memory for file name sort\n");
01519     abortit(3);
01520   }
01521 
01522   while((dp=readdir(dfd)) != NULL) {
01523     // printk("%s\n", dp->d_name) ; continue; // !!TEMP 
01524     // Only accept our files. 
01525     if(strstr(dp->d_name, pchan) || strstr(dp->d_name, rchan) || strstr(dp->d_name, ".dsf")) {
01526       nameptr[i++].name = strdup(dp->d_name);
01527       if(i >= MAXFILES) {
01528         printk("***Fatal error. Too many (%d) files in %s\n", MAXFILES, tlmdir);
01529         printf("***Fatal error. Too many (%d) files in %s\n", MAXFILES, tlmdir);
01530         abortit(3);
01531       }
01532       if(!strstr(dp->d_name, ".dsf")) cntsleeps = 0;    //we saw a file
01533       if((stat(stopfile, &stbuf) == 0) || abortflg) { break; } //signal to stop
01534     }
01535   }
01536   closedir(dfd);
01537   qsort(nameptr, i, sizeof(NAMESORT), &compare_names);
01538 
01539   for(j=0; j < i; j++) {
01540     //printk("####QSORT FILES: %s\n", nameptr[j].name); // !!TEMP 
01541     // NOTE: the dsf files is moved to indir/dsf e.g. (/dds/soc2pipe/aia/dsf)
01542     // Currently the cron job pipefe_rm does this:
01543     // `/bin/mv $dsfname /dds/socdc/hmi/dsf`
01544     //Only do for V01/VC02. VC04/VC05 won't mv the .dsf
01545     if(!strcmp(pchan, "VC01") || !strcmp(pchan, "VC02")) { 
01546       if(strstr(nameptr[j].name, ".dsf")) {
01547         //sprintf(cmd, "/bin/mv %s/%s %s", tlmdir, nameptr[j].name, outdir);
01548         sprintf(cmd, "/bin/mv -f %s/%s %s/dsf/",tlmdir,nameptr[j].name,tlmdir);
01549         printk("*mv dsf file to %s/dsf/\n", tlmdir);
01550         printk("%s\n", cmd);
01551         if(system(cmd)) {
01552           printk("***Error on: %s\n", cmd);
01553         }
01554       }
01555     }
01556     if(!strstr(nameptr[j].name, ".qac")) {  // can be .qac or .qacx 
01557       free(nameptr[j].name);
01558       continue;
01559     }
01560     rexmit = higherversion = 0;
01561     if(strstr(nameptr[j].name, ".qacx")) {      // this is a rexmit file 
01562       rexmit = 1;
01563     }
01564     sprintf(name, "%s/%s", tlmdir, nameptr[j].name);
01565     printk("%s\n*Found qac file:\n* %s\n", do_datestr(), name);
01566     if(!(fp=fopen(name, "r"))) {
01567       printk("***Can't open %s\n", name);
01568       free(nameptr[j].name);
01569       continue;
01570     }
01571     // NOTE: the qac file is already verified by the caller of ingest_lev0 
01572     while(fgets(line, 256, fp)) {   // get qac file lines 
01573       if(line[0] == '#' || line[0] == '\n') continue;
01574       if(strstr(line, "TLM_FILE_NAME=")) {
01575         token = (char *)strtok(line, "=");
01576         token = (char *)strtok(NULL, "\n");
01577         printk("tlm file is %s\n", token);
01578         sprintf(tlmfile, "%s/%s", tlmdir, token);
01579         sprintf(tlmname, "%s", token);
01580       }
01581       else if(strstr(line, "TLM_FILE_SIZE=")) {
01582         token = (char *)strtok(line, "=");
01583         token = (char *)strtok(NULL, "=");
01584         printk("*tlm file size is %s", token);
01585       }
01586       else if((strstr(line, "TOTAL_TLM_IM_PDU=")) || (strstr(line, "TOTAL_VCDU="))) {
01587         token = (char *)strtok(line, "=");
01588         token = (char *)strtok(NULL, "\n");
01589         total_tlm_vcdu = atoi(token);
01590       }
01591       else if(strstr(line, "TOTAL_MISSING_VCDU=")) {
01592         token = (char *)strtok(line, "=");
01593         token = (char *)strtok(NULL, "\n");
01594         total_missing_vcdu = atoi(token);
01595       }
01596       else if(strstr(line, "TOTAL_MISSING_IM_PDU=")) {
01597         token = (char *)strtok(line, "=");
01598         token = (char *)strtok(NULL, "\n");
01599         total_missing_im_pdu = atol(token);
01600         break;
01601       }
01602     }
01603     fclose(fp);
01604     //alarm(ALRMSEC);       //restart alarm if no more files come
01605     strcpy((char *)tlmnamekey, (char *)tlmname);
01606     filename = (char *)rindex(tlmname, '.');
01607     if(!filename) {
01608       printk("**ERROR: Bad formed tlm name in qac file\n");
01609       printk("  tlm name = %s\n", tlmname);
01610       continue;
01611     }
01612     *filename = 0;          // elim .tlm 
01613     *((char *)tlmnamekey+22) = 0;   // elim after hh_mm_ss
01614     rs_tlm = drms_create_record(drms_env, tlmseriesname, 
01615                 DRMS_PERMANENT, &status);
01616     if(status) {
01617       if(status == DRMS_ERROR_SUMOPEN) {
01618         printk("**ERROR: DRMS can't open w/SUMS. Aborting...\n");
01619         abortit(4);
01620       }
01621       printk("***Can't create record for %s. Status=%d\n", tlmseriesname, status);
01622       continue;
01623     }
01624     if((status = drms_setkey_string(rs_tlm, "filename", tlmnamekey))) {
01625       printk("**ERROR: drms_setkey_string failed for 'filename'\n");
01626     }
01627     drms_record_directory(rs_tlm, path, 0);
01628     if(!*path) {
01629       printk("***ERROR: No path to segment for %s\n", tlmseriesname);
01630       continue;
01631     }
01632     if(outdir) {
01633       sprintf(cmd, "cp -p %s %s", name, outdir);
01634       printk("*cp qac file to %s\n", outdir);
01635       printk("%s\n", cmd);
01636       if(system(cmd)) {
01637         printk("***Error on: %s\n", cmd);
01638       }
01639       sprintf(cmd, "cp -p %s %s", tlmfile, outdir);
01640       printk("*cp tlm file to %s\n", outdir);
01641       printk("%s\n", cmd);
01642       if(system(cmd)) {
01643         printk("***Error on: %s\n", cmd);
01644       }
01645     }
01646     sprintf(cmd, "/bin/mv %s %s", name, path);
01647     printk("*mv qac to %s\n", path);
01648     printk("%s\n", cmd);
01649     if(status = system(cmd)) {
01650       printk("**ERROR: %d errno=%d on: %s\n", status, errno, cmd);
01651       if(WIFEXITED(status)) {
01652         if(mvstat = WEXITSTATUS(status)) {  //status ret by mv
01653           printk("**ERROR: mv exit status = %d\n", mvstat);
01654         }
01655       }
01656     }
01657     sprintf(cmd, "/bin/mv %s %s", tlmfile, path);
01658     printk("*mv tlm to %s\n", path);
01659     printk("%s\n", cmd);
01660     if(status = system(cmd)) {
01661       printk("**ERROR: %d errno=%d on: %s\n", status, errno, cmd);
01662       if(WIFEXITED(status)) {
01663         if(mvstat = WEXITSTATUS(status)) {  //status ret by mv
01664           printk("**ERROR: mv exit status = %d\n", mvstat);
01665         }
01666       }
01667       printk("**Continue after ERROR on mv of tlm file\n");
01668       continue;
01669     }
01670     if((status = drms_close_record(rs_tlm, DRMS_INSERT_RECORD))) {
01671       printk("**ERROR: drms_close_record failed for %s\n", tlmseriesname);
01672     }
01673 
01674     sprintf(xxname, "%s/%s.tlm", path, tlmname);
01675     filename = (char *)rindex(tlmname, '_');
01676     sprintf(vername, "%s", filename);   // e.g. _00 or _01 etc. 
01677     if(strcmp(vername, "_00")) {    // this is a higher vers # file 
01678       higherversion = 1;
01679       printk("Higher version tlm found: %s.tlm\n", tlmname);
01680     }
01681     firstfound = 1;         //a file has been seen
01682     if(get_tlm(xxname, rexmit, higherversion)) { // lev0 extraction of image 
01683       printk("***Error in lev0 extraction for %s\n", xxname);
01684       //printk("***Going to abort\n");
01685       //abortflg = 1;
01686     }
01687     if((stat(stopfile, &stbuf) == 0) || abortflg) { break; } //signal to stop
01688   }
01689   free(nameptr);
01690 }
01691 
01692 // Initial setup stuff called when main is first entered.
01693 void setup()
01694 {
01695   FILE *fp;
01696   int i;
01697   char string[128], cwdbuf[128], idstr[256];
01698   char envfile[100], s1[256],s2[256],s3[256], line[256];
01699   ThreadSigErr_t error = kThreadSigErr_Success;
01700 
01701   sdo_epoch = sscan_time("1958.01.01_00:00:00_TAI");
01702   do_datestr();
01703   printk_set(h0log, h0log); // set for printk calls 
01704   printk("%s\n", datestr);
01705   getcwd(cwdbuf, 126);
01706   sprintf(idstr, "Cwd: %s\nCall: ", cwdbuf);
01707   if(grounddata)
01708     sprintf(string, "ingest_lev0 -g started as pid=%d user=%s\n", getpid(), username);
01709   else
01710     sprintf(string, "ingest_lev0 started as pid=%d user=%s\n", getpid(), username);
01711   strcat(idstr, string);
01712   printk("*%s", idstr);
01713   if(restartflg) printk("-r ");
01714   sprintf(argvc, "vc=%s", vc);
01715   sprintf(argindir, "indir=%s", tlmdir);
01716   sprintf(arglogfile, "logfile=%s", logname);
01717   if(outdir) {
01718     sprintf(argoutdir, "outdir=%s", outdir);
01719     printk("%s %s %s %s\n", argvc, argindir, argoutdir, arglogfile);
01720   } else {
01721     printk("%s %s %s\n", argvc, argindir, arglogfile);
01722   }
01723   strcpy(pchan, vc);        // virtual channel primary 
01724   if(rexmitmode)        //new 'stopX' 11/2/2012
01725     sprintf(stopfile, "/usr/local/logs/lev0/%s_stopX", pchan);
01726   else
01727     sprintf(stopfile, "/usr/local/logs/lev0/%s_stop", pchan);
01728   //Dont rm stopfile any more (1/6/2012). 
01729   //With the new rexmit dir there is a second
01730   //ingest_lev0 running on a VC and it needs the stop file too.
01731   //The stop file is removed when ingest_lev0 is started by
01732   //doingestlev0_[HMI,AIA].pl
01733   //sprintf(string, "/bin/rm -f %s", stopfile); //remove any stop file
01734   //system(string);
01735   for(i=0; ; i++) {     // ck for valid and get redundant chan 
01736     if(!strcmp(p_r_chan_pairs[i].pchan, pchan)) {
01737       strcpy(rchan, p_r_chan_pairs[i].rchan);
01738       hmiaiaflg = p_r_chan_pairs[i].instru;
01739       break;
01740     }
01741     if(!strcmp(p_r_chan_pairs[i].pchan, "n/a")) {
01742       printk("!!ERROR: Invalid VCid (%s) specified\n", pchan);
01743       fprintf(stderr, "!!ERROR: Invalid VCid (%s) specified. Abort\n", pchan);
01744       abortit(1);
01745     }
01746   }
01747   if(hmiaiaflg) {
01748     if(grounddata) {
01749       sprintf(tlmseriesname, "%s", TLMSERIESNAMEAIAGND);
01750       sprintf(lev0seriesname, "%s", LEV0SERIESNAMEAIAGND);
01751     }
01752     else {
01753       sprintf(tlmseriesname, "%s", TLMSERIESNAMEAIA);
01754       sprintf(lev0seriesname, "%s", LEV0SERIESNAMEAIA);
01755     }
01756   }
01757   else {
01758     if(grounddata) {
01759       sprintf(tlmseriesname, "%s", TLMSERIESNAMEHMIGND);
01760       sprintf(lev0seriesname, "%s", LEV0SERIESNAMEHMIGND);
01761     }
01762     else {
01763       sprintf(tlmseriesname, "%s", TLMSERIESNAMEHMI);
01764       sprintf(lev0seriesname, "%s", LEV0SERIESNAMEHMI);
01765     }
01766   }
01767   if(!restartflg) {
01768     printk("tlmseriesname=%s\nlev0seriesname=%s\n", 
01769         tlmseriesname, lev0seriesname);
01770   }
01771   sprintf(bld_vers, "%s", jsoc_version);
01772   umask(002);           // allow group write 
01773   Image.initialized = 0;    // init the two image structures 
01774   ImageOld.initialized = 0;
01775   Img = &Image;
01776   ImgO = &ImageOld;
01777   imgdecode_init_hack(Img);
01778   imgdecode_init_hack(ImgO);
01779 
01780   //set environment variables for hk code
01781   //create filename and path
01782   if(grounddata) strcpy(envfile, ENVFILE_GND );
01783   else strcpy(envfile, ENVFILE );
01784   //fopen file
01785   if(!(fp=fopen(envfile, "r"))) {
01786       printk("***Can't open %s. Check setting is correct\n", envfile);
01787       exit(0);
01788   }
01789   //read in lines
01790   while( fgets(line, MAXLINE_IN_FILE, fp) != NULL )
01791   {
01792     if (!strncmp(line, "#", 1)) {
01793       continue; /*skip comments */
01794     }
01795     else  {
01796       sscanf(line, "%s %s %s ", s1,s2,s3);
01797       // set each line to setenv();
01798       setenv(s2, s3, 1);
01799     }
01800   }
01801   fclose(fp);
01802 
01803 /***************************!!TBD ck this out later*************************
01804   //setup a sigalrm for threads
01805   //pthread_mutex_init(&mutex, NULL);
01806 
01807   // Set alarm
01808   gettimeofday(&tv0, NULL);
01809   if (td_createalarm(ALRMSEC, shandler, &mutex, &talarm)) { //to shandler in ALRMSEC
01810     pthread_mutex_destroy(&mutex); 
01811     printk("Can't set an alarm signal for %dsec of no data\n", ALRMSEC);
01812   }
01813 *****************************************************************************/
01814 }
01815 
01816 // Module main function. 
01817 int DoIt(void)
01818 {
01819   pid_t pid;
01820   int wflg = 1;
01821   char *args[6];
01822   char callcmd[256];
01823 
01824   if (nice_intro())
01825     return (0);
01826   if(!(username = (char *)getenv("USER"))) username = "nouser"; 
01827   vc = cmdparams_get_str(&cmdparams, "vc", NULL);
01828   tlmdir = cmdparams_get_str(&cmdparams, "indir", NULL);
01829   outdir = cmdparams_get_str(&cmdparams, "outdir", NULL);
01830   logfile = cmdparams_get_str(&cmdparams, "logfile", NULL);
01831   if(strstr(tlmdir, "rexmit")) { rexmitmode = 1; }
01832   if (strcmp(vc, NOTSPECIFIED) == 0) {
01833     fprintf(stderr, "'vc' virt channel must be specified.  Abort\n");
01834     return(1);
01835   }
01836   if (strcmp(tlmdir, NOTSPECIFIED) == 0) {
01837     fprintf(stderr, "'indir' must be specified.  Abort\n");
01838     return(1);
01839   }
01840   if (strcmp(outdir, NOTSPECIFIED) == 0) {
01841     outdir = NULL;
01842   }
01843   if (strcmp(logfile, NOTSPECIFIED) == 0) {
01844     sprintf(logname, H0LOGFILE, username, vc, gettimetag());
01845   }
01846   else {
01847     sprintf(logname, "%s", logfile);
01848   }
01849   if(restartflg) {
01850     //sleep(30);        //make sure old ingest_lev0 is done
01851     if((h0logfp=fopen(logname, "a")) == NULL)
01852       fprintf(stderr, "**Can't open for append the log file %s\n", logname);
01853   }
01854   else {
01855     if((h0logfp=fopen(logname, "w")) == NULL)
01856       fprintf(stderr, "**Can't open the log file %s\n", logname);
01857   }
01858   setup();
01859   while(wflg) {
01860     do_ingest();                // loop to get files from the input dir 
01861 /************************NOOP this out for now********************************
01862     if(sigalrmflg) {        // process an alarm timout for no data in
01863       sigalrmflg = 0;
01864       td_destroyalarm(&talarm, &mutex);
01865       gettimeofday(&tv0, NULL);
01866       if (td_createalarm(ALRMSEC, shandler, &mutex, &talarm)) { 
01867         pthread_mutex_destroy(&mutex);
01868         printk("Can't set an alarm signal for %dsec of no data\n", ALRMSEC);
01869       }
01870       if(!ignoresigalrmflg && firstfound) {
01871         if(Image.initialized) {
01872           if(rs && fsn_prev) {  //make sure have a created record
01873             close_image(rs, segment, segArray, &Image, fsn_prev);
01874             printk("*Closed image on timeout FSN=%u\n", fsn_prev);
01875           }
01876         }
01877         else {
01878           printk("*No image to close on timeout\n");
01879         }
01880         printk("alrm_sig: drms_server_end_transaction()\n");
01881         drms_server_end_transaction(drms_env, 0 , 0); //commit
01882         printk("alrm_sig: drms_server_begin_transaction()\n");
01883         drms_server_begin_transaction(drms_env); //start another cycle
01884         fsn_prev = 0;   //make sure don't try to flush this image
01885         imagecnt = 0;   //and start a new transaction interval
01886       }
01887       ignoresigalrmflg = 0;
01888     }
01889 *****************************************************************************/
01890 
01891     if((stat(stopfile, &stbuf) == 0) || abortflg) {
01892       printk("Abort or Found file: %s. Terminate.\n", stopfile);
01893       if(abortflg) send_mail("Abort for ingest_lev0 for %s\n", pchan);
01894       //now close any open image
01895       if(Image.initialized) {
01896         if(rs) {        //make sure have a created record
01897           close_image(rs, segment, segArray, &Image, fsn_prev);
01898         }
01899       }
01900       else {
01901         printk("*No image to close on exit\n");
01902       }
01903       //No need to commit here. the exit will do it
01904       //printk("restart: drms_server_end_transaction()\n"); 
01905       //drms_server_end_transaction(drms_env, 0 , 0); //commit
01906       //NEW 1Jun2010: leave the stopfile. 
01907       //Must use doingestlev0_HMI(AIA).pl to start ingest_lev0
01908       //sprintf(callcmd, "/bin/rm -f %s", stopfile);
01909       //system(callcmd);
01910       if(rexmitmode)        //new 11/02/21012
01911         sprintf(callcmd, "touch /usr/local/logs/lev0/%s_exitX", pchan);
01912       else
01913         sprintf(callcmd, "touch /usr/local/logs/lev0/%s_exit", pchan);
01914       system(callcmd);      //let the world know we're gone
01915       wflg = 0; //leave DoIt()
01916       continue;
01917     }
01918     sleep(sleep_interval);  //normally 2 sec
01919     if(cntsleeps == 0) {    //file was seen
01920       if(paused) {      //send resume data flow msg
01921         paused = 0;
01922         if(!rexmitmode) {
01923           send_mail("tlm files seen again for ingest_lev0 for %s\n", pchan);
01924         }
01925       }
01926     }
01927     cntsleeps++;        //#of 2sec sleeps w/o any files in do_ingest()
01928     if(cntsleeps > 300) {   // >600sec w/o any files
01929       if(!paused) {
01930         if(!rexmitmode) {
01931           send_mail("No files seen for ingest_lev0 for %s for 600sec\n", pchan);
01932         }
01933         paused = 1;
01934       }
01935     }
01936   }
01937   /*************!!TBD noop this out for now
01938   td_destroyalarm(&talarm);
01939   //pthread_mutex_destroy(&mutex);  //can't destroy here. used in shandler()
01940   pthread_mutex_destroy(&mutex);  //if don't detroy here can end up w/2 ingest_lev0
01941                   //on a restart by doingestlev0.pl ??
01942   ***************************************************/
01943   return(0);
01944 }
01945