JSOC_Documentation: proj/lev0/apps/xingest

00001 /*-----------------------------------------------------------------------------
00002  * cvs/JSOC/proj/lev0/apps/ingest_lev0.c
00003  * NOTE: This originally came from hmi_lev0.c on hmi0
00004  *-----------------------------------------------------------------------------
00005  *
00006  * This is a module that runs with DRMS and continuously extracts images and HK
00007  * data from .tlm files that appear in the given input dir. It puts the .tlm
00008  * and .qac files in the DRMS dataset TLMSERIESNAME and also in the outdir if
00009  * given.  It extracts images from the .tlm files an puts them in
00010  * the DRMS dataset LEV0SERIESNAME and extracts hk data to appropriate hk datasets.
00011  *
00012  * Call for testing (set TLMSERIESNAME and LEV0SERIESNAME to test datasets): 
00013  * ingest_lev0 vc=VC05 indir=/scr20/jim/tlm outdir=/scr20/jim/tlm/out [logfile=name]
00014  *
00015  * OPERATION:
00016  * This is normally run on the dedicated lev0 host. Four instances are run.
00017  * No outdir= is given, i.e. the .tlm and .qac files only end up in SUMS.
00018  *
00019  * ingest_lev0 vc=VC02 indir=/dds/socdc/hmi
00020  * ingest_lev0 vc=VC05 indir=/dds/socdc/hmi
00021  * ingest_lev0 vc=VC10 indir=/dds/socdc/aia
00022  * ingest_lev0 vc=VC13 indir=/dds/socdc/aia
00023  */ 
00024 
00025 #include <jsoc_main.h>
00026 #include <cmdparams.h>
00027 #include <drms.h>
00028 #include <drms_names.h>
00029 #include <stdio.h>
00030 #include <stdlib.h>
00031 #include <ctype.h>
00032 #include <strings.h>
00033 #include <errno.h>
00034 #include <sys/types.h>
00035 #include <sys/time.h>
00036 #include <sys/stat.h>   //for umask(2)
00037 #include <dirent.h>
00038 #include <unistd.h> //for alarm(2) among other things...
00039 #include <printk.h>
00040 #include <math.h>
00041 #include "packets.h"
00042 #include "imgdecode.h"
00043 #include "decode_hk_vcdu.h"
00044 #include "decode_hk.h"
00045 #include "load_hk_config_files.h"
00046 #include "add_small_image.c"
00047 #include "mypng.h"
00048 #include "tdsignals.h"
00049 #include "quallev0.h"
00050 
00051 #define RESTART_CNT 2   //#of tlm files to process before restart
00052 
00053 //#define LEV0SERIESNAMEHMI "su_production.lev0d_test"
00054 //#define LEV0SERIESNAMEHMI "su_production.lev0f_test"
00055 //#define LEV0SERIESNAMEHMI "su_production.lev0f_hmi"
00057 //For test with DDS on Jan 19, 2010
00058 //#define LEV0SERIESNAMEHMI "su_production.lev0f_hmi_JAN2010"
00059 //#define LEV0SERIESNAMEHMI "su_production.lev0f_hmi_junk"
00060 //#define LEV0SERIESNAMEHMI "hmi.lev0f"
00062 //For test with DDS on Jan 19, 2010
00063 //#define TLMSERIESNAMEHMI "su_production.tlm_hmi_JAN2010"
00064 //#define TLMSERIESNAMEHMI "su_production.tlm_hmi_junk"
00065 
00066 #define LEV0SERIESNAMEHMIGND "hmi_ground.lev0_dds"
00067 #define TLMSERIESNAMEHMIGND "hmi_ground.tlm_dds"
00068 //#define TLMSERIESNAMEHMI "hmi.tlm_reingest"
00069 
00070 //#define LEV0SERIESNAMEAIA "su_production.lev0d_test_aia"
00071 //#define LEV0SERIESNAMEAIA "su_production.lev0f_aia"
00073 //For test with DDS on Jan 19, 2010
00074 //#define LEV0SERIESNAMEAIA "su_production.lev0f_aia_JAN2010"
00075 //#define LEV0SERIESNAMEAIA "su_production.lev0f_aia_junk"
00077 //For test with DDS on Jan 19, 2010
00078 //#define TLMSERIESNAMEAIA "su_production.tlm_aia_JAN2010"
00079 //#define TLMSERIESNAMEAIA "su_production.tlm_aia_junk"
00080 
00081 #define LEV0SERIESNAMEAIAGND "aia_ground.lev0_dds"
00082 #define TLMSERIESNAMEAIAGND "aia_ground.tlm_dds"
00083 //#define TLMSERIESNAMEAIA "aia.tlm_reingest"
00084 
00085 //#define LEV0SERIESNAMEHMI "hmi.lev0_60d"
00086 //#define TLMSERIESNAMEHMI "hmi.tlm_60d"
00087 //#define LEV0SERIESNAMEAIA "aia.lev0_60d"
00088 //#define TLMSERIESNAMEAIA "aia.tlm_60d"
00089 
00090 //When change to these data series below to save real data.
00091 //#define TLMSERIESNAMEHMI "hmi.tlm"
00092 #define TLMSERIESNAMEHMI "su_production.tlm"
00093 //#define LEV0SERIESNAMEHMI "hmi.lev0a"
00094 #define LEV0SERIESNAMEHMI "su_production.lev0a"
00095 #define LEV0SERIESNAMEAIA "aia.lev0"
00096 #define TLMSERIESNAMEAIA "aia.tlm"
00097 
00098 //#define LEV0SERIESNAMEAIA "aia.lev0d"
00099 //#define TLMSERIESNAMEAIA "aia.tlmd"
00100 //#define LEV0SERIESNAMEHMI "hmi.lev0d"
00101 //#define TLMSERIESNAMEHMI "hmi.tlmd"
00102 
00103 //#define LEV0SERIESNAMEAIA "aia.lev0e"
00104 //#define TLMSERIESNAMEAIA "aia.tlme"
00105 //#define LEV0SERIESNAMEHMI "hmi.lev0e"
00106 //#define TLMSERIESNAMEHMI "hmi.tlme"
00107 
00108 //Also, change setting in $JSOCROOT/proj/lev0/apps/SOURCE_ENV_HK_DECODE file to:
00109 //setenv HK_LEV0_BY_APID_DATA_ID_NAME      lev0
00110 //setenv HK_DF_HSB_DIRECTORY               /tmp21/production/lev0/hk_hsb_dayfile
00111 //setenv HK_JSVNMAP_DIRECTORY              /home/production/cvs/TBL_JSOC/lev0/hk_jsn_map_file/prod
00112 
00113 
00114 #define H0LOGFILE "/usr/local/logs/lev0/ingest_lev0.%s.%s.%s.log"
00115 #define PKTSZ 1788      //size of VCDU pkt
00116 #define MAXFILES 65535      //max # of file can handle in tlmdir
00117 #define NUMTIMERS 8     //number of seperate timers avail
00118 #define IMAGE_NUM_COMMIT 12 //number of complete images until commit
00119 //#define IMAGE_NUM_COMMIT 2    //!!TEMP number of complete images until commit
00120 #define TESTAPPID 0x199     //appid of test pattern packet
00121 #define TESTVALUE 0xc0b     //first value in test pattern packet
00122 #define MAXERRMSGCNT 10     //max # of err msg before skip the tlm file
00123 #define NOTSPECIFIED "***NOTSPECIFIED***"
00124 #define ENVFILE "/home/production/cvs/JSOC/proj/lev0/apps/SOURCE_ENV_FOR_HK_DECODE"
00125 #define ENVFILE_GND "/home/production/cvs/JSOC/proj/lev0/apps/SOURCE_ENV_FOR_HK_DECODE_GROUND"
00126 
00127 extern int decode_next_hk_vcdu(unsigned short *tbuf, CCSDS_Packet_t **hk, unsigned int *Fsn);
00128 extern int write_hk_to_drms();
00129 extern void HMI_compute_exposure_times(DRMS_Record_t *rec, HK_Keyword_t *isp, int flg);
00130 extern int set_HMI_mech_values(DRMS_Record_t *rec);
00131 static TIME SDO_to_DRMS_time(int sdo_s, int sdo_ss);
00132 
00133 // List of default parameter values. 
00134 ModuleArgs_t module_args[] = { 
00135   {ARG_STRING, "vc", NOTSPECIFIED, "Primary virt channel to listen to"},
00136   {ARG_STRING, "indir", NOTSPECIFIED, "directory containing the files to ingest"},
00137   {ARG_STRING, "outdir", NOTSPECIFIED, "directory to move the files to after the ingest"}, 
00138   {ARG_STRING, "logfile", NOTSPECIFIED, "optional log file name. Will create one if not given"},
00139   {ARG_FLAG, "v", "0", "verbose flag"},
00140   {ARG_FLAG, "h", "0", "help flag"},
00141   {ARG_FLAG, "r", "0", "restart flag"},
00142   {ARG_FLAG, "g", "0", "ground data flag"},
00143   {ARG_END}
00144 };
00145 
00146 CmdParams_t cmdparams;
00147 // Module name presented to DRMS. 
00148 char *module_name = "ingest_lev0";
00149 
00150 typedef enum
00151 {
00152    kThreadSigErr_Success
00153 } ThreadSigErr_t;
00154 static int gLoop = 1;
00155 static pthread_mutex_t mutex;
00156 static struct timeval tv0;
00157 static td_alarm_t talarm = 0;
00158 
00159 FILE *h0logfp;      // fp for h0 ouput log for this run 
00160 static IMG Image, ImageOld;
00161 static IMG *Img, *ImgO, *ImgC;
00162 static CCSDS_Packet_t *Hk;
00163 static DRMS_Record_t *rs;
00164 static DRMS_Segment_t *segment;
00165 static DRMS_Array_t *segArray;
00166 static DRMS_RecordSet_t *rset;
00167 static DRMS_Record_t *rs_old, *rsc;
00168 static DRMS_Segment_t *segmentc;
00169 static DRMS_Array_t *cArray, *oldArray;
00170 static TIME sdo_epoch;
00171 static char datestr[32];
00172 static char bld_vers[16];
00173 static struct timeval first[NUMTIMERS], second[NUMTIMERS];
00174 
00175 int INVALtime;
00176 unsigned int fsn = 0;
00177 unsigned int fsnx = 0;
00178 unsigned int fsn_prev = 0;
00179 unsigned int fsn_pre_rexmit = 0;
00180 unsigned int fid = 0;
00181 
00182 short *rdat;
00183 long long vcdu_seq_num;
00184 long long vcdu_seq_num_next;
00185 long long total_missing_im_pdu;
00186 unsigned int vcdu_24_cnt, vcdu_24_cnt_next;
00187 int verbose;
00188 int grounddata;
00189 int appid;
00190 int testid1, testid2;
00191 int hmiaiaflg;          //0=hmi, 1=aia
00192 int whk_status;
00193 int total_tlm_vcdu;
00194 int total_missing_vcdu;
00195 int errmsgcnt, fileimgcnt;
00196 int cntsleeps = 0;
00197 int paused = 0;
00198 int imagecnt = 0;       // num of images since last commit 
00199 int restartflg = 0;     // set when ingest_lev0 is called for a restart
00200 int abortflg = 0;
00201 int sigalrmflg = 0;             // set on signal so prog will know 
00202 int ignoresigalrmflg = 0;       // set after a close_image()
00203 int firstfound = 0;     // set if see any file after startup
00204 int rexmitmode = 0;     // set if this process is doing /rexmit dir
00205 int ALRMSEC = 60;               // must get 2 in a row for no image timeout
00206 int sleep_interval = 2;     // #of sec to sleep after do_ingest() calls
00207 char logname[128];
00208 char argvc[32], argindir[96], arglogfile[96], argoutdir[96];
00209 char timetag[32];
00210 char pchan[8];          // primary channel to listen to e.g. VC02 
00211 char rchan[8];          // redundant channel to listen to e.g. VC10 
00212 char stopfile[80];      // e.g. /usr/local/logs/lev0/VC04_stop
00213 char tlmseriesname[128];    // e.g. hmi.tlm
00214 char lev0seriesname[128];   // e.g. hmi.lev0
00215 char tlmnamekey[128];       // shortened tlm file name for TLMDSNAM keyword
00216 char tlmnamekeyfirst[128];  // for TLMDSNAM keyword for 1st file of image
00217 char oldtlmdsnam[128];      // TLMDSNAM keyword from prev rec in db
00218 char *username;         // from getenv("USER") 
00219 char *tlmdir;           // tlm dir name passed in 
00220 char *outdir;           // output dir for .tlm file (can be /dev/null)
00221 char *logfile;          // optional log name passed in 
00222 char *vc;           // virtual channel to process, e.g. VC02 
00223 struct stat stbuf;
00224 struct p_r_chans {
00225   char *pchan;
00226   char *rchan;
00227   int instru;       //0=hmi, 1=aia
00228 };
00229 typedef struct p_r_chans P_R_CHANS;
00230 
00231 P_R_CHANS p_r_chan_pairs[] = {
00232 {"VC01", "VC09", 1},        // AIA 
00233 {"VC04", "VC12", 1},        // AIA 
00234 {"VC02", "VC10", 0},        // HMI 
00235 {"VC05", "VC13", 0},        // HMI 
00236 {"n/a", "n/a"}
00237 };
00238 
00239 struct namesort {       // sorted file names in tlmdir 
00240   char *name;
00241 };
00242 typedef struct namesort NAMESORT;
00243 
00244 
00245 //Function callback that gets called when alarm 'rings'.  This runs in the signal
00246 //thread that receives, the alarm signal, not the main thread. */
00247 void shandler(int sig, pthread_mutex_t *mtx)
00248 {
00249    float elapsed;
00250    struct timeval tv1;
00251    gettimeofday(&tv1, NULL);
00252 
00253    elapsed = (float)((tv1.tv_sec * 1000000.0 + tv1.tv_usec -
00254                       (tv0.tv_sec * 1000000.0 + tv0.tv_usec)) / 1000000.0);
00255 
00256    printk("Thread '%lld' received alarm signal '%d'.\n", 
00257         (long long )pthread_self(), sig);
00258    printk("Elapsed time is %f seconds.\n", elapsed);
00259 
00260    /* This isn't in the main thread, so put globals in a critical region. */
00261    pthread_mutex_lock(mtx);
00262    sigalrmflg = 1;
00263    pthread_mutex_unlock(mtx);
00264 }
00265 
00266 int nice_intro ()
00267 {
00268   int usage = cmdparams_get_int (&cmdparams, "h", NULL);
00269   if (usage)
00270     {
00271     printf ("Usage:\ningest_lev0 [-vh] "
00272     "vc=<virt chan> indir=</dir> [outdir=</dir>] [logfile=<file>]\n"
00273     "  -h: help - show this message then exit\n"
00274     "  -v: verbose\n"
00275         "  -g: output to hmi_ground/aia_ground\n"
00276     "  -r: restart. only used when we restart our selves periodically\n"
00277     "vc= primary virt channel to listen to e.g. VC02\n"
00278     "indir= directory containing the files to ingest\n"
00279     "outdir= optional dir to copy the files to after the ingest\n"
00280     "logfile= optional log file name. Will create one if not given\n");
00281     return(1);
00282     }
00283   verbose = cmdparams_get_int (&cmdparams, "v", NULL);
00284   grounddata = cmdparams_get_int (&cmdparams, "g", NULL);
00285   restartflg = cmdparams_get_int (&cmdparams, "r", NULL);
00286   return (0);
00287 }
00288 
00289 TIME SDO_to_DRMS_time(int sdo_s, int sdo_ss)
00290 {
00291 static int firstcall = 1;
00292 if (firstcall)
00293   {
00294   firstcall = 0;
00295   }
00296 /* XXX fix build 3/18/2008, arta */
00297 return(sdo_epoch + (TIME)sdo_s + (TIME)(sdo_ss)/65536.0);
00298 }
00299 
00300 // Setup global datestr[] like: 2008.07.14_08:29:31
00301 char *do_datestr() {
00302   time_t tval;
00303   struct tm *t_ptr;
00304 
00305   tval = time(NULL);
00306   t_ptr = localtime(&tval);
00307   sprintf(datestr, "%d.%02d.%02d_%02d:%02d:%02d", 
00308       (t_ptr->tm_year+1900), (t_ptr->tm_mon+1),
00309       t_ptr->tm_mday, t_ptr->tm_hour, t_ptr->tm_min, t_ptr->tm_sec);
00310   return(datestr);
00311 }
00312 
00313 // Returns a time tag like  yyyy.mm.dd.hhmmss 
00314 char *gettimetag()
00315 {
00316   struct timeval tvalr;
00317   struct tm *t_ptr;
00318 
00319   gettimeofday(&tvalr, NULL);
00320   t_ptr = localtime((const time_t *)&tvalr);
00321   sprintf(timetag, "%04d.%02d.%02d.%02d%02d%02d",
00322         (t_ptr->tm_year+1900), (t_ptr->tm_mon+1), t_ptr->tm_mday, t_ptr->tm_hour, t_ptr->tm_min, t_ptr->tm_sec);
00323   return(timetag);
00324 }
00325 
00326 
00327 void BeginTimer(int n)
00328 {
00329   gettimeofday (&first[n], NULL);
00330 }
00331 
00332 float EndTimer(int n)
00333 {
00334   gettimeofday (&second[n], NULL);
00335   if (first[n].tv_usec > second[n].tv_usec) {
00336     second[n].tv_usec += 1000000;
00337     second[n].tv_sec--;
00338   }
00339   return (float) (second[n].tv_sec-first[n].tv_sec) +
00340     (float) (second[n].tv_usec-first[n].tv_usec)/1000000.0;
00341 }
00342 
00343 // Outputs the variable format message (re: printf) to the log file.
00344 int h0log(const char *fmt, ...)
00345 {
00346   va_list args;
00347   char string[32768];
00348 
00349   va_start(args, fmt);
00350   vsprintf(string, fmt, args);
00351   if(h0logfp) {
00352     fprintf(h0logfp, string);
00353     fflush(h0logfp);
00354   } else {          // couldn't open log 
00355     printf(string);
00356     fflush(stdout);
00357   }
00358   va_end(args);
00359   return(0);
00360 }
00361 
00362 int send_mail(char *fmt, ...)
00363 {
00364   va_list args;
00365   char string[1024], cmd[1024];
00366 
00367   va_start(args, fmt);
00368   vsprintf(string, fmt, args);
00369   sprintf(cmd, "echo \"%s\" | Mail -s \"ingest_lev0 mail\" jsoc_ops", string);
00370   system(cmd);
00371   va_end(args);
00372   return(0);
00373 }
00374 
00375 //Set the QUALLEV0 keyword
00376 void do_quallev0(DRMS_Record_t *rs, IMG *img, int fsn) 
00377 {
00378   char *hwltnset, *cdark;
00379   char *hseqerr, *aistate;
00380   char wave_str[16];
00381   int status, hsqfgsn, asqfsn, hgp1rgst;
00382   float percentd;
00383   uint32_t missvals, datav;
00384   uint32_t hcf1encd, hcf2encd, hps1encd, hps2encd, hps3encd; 
00385   uint32_t hwt1encd, hwt2encd, hwt3encd, hwt4encd;
00386   uint32_t hcf1pos, hcf2pos, hpl1pos, hpl2pos, hpl3pos, hwl1pos, hwl2pos;
00387   uint32_t hwl3pos, hwl4pos;
00388   uint32_t aiawvlen, aifwen, aiasen;
00389   short aifiltyp;
00390   uint32_t quallev0 = 0;
00391   uint32_t qmiss = 0;
00392 
00393   if(img->overflow) quallev0 = quallev0 | Q_OVFL;
00394   if(img->headerr) quallev0 = quallev0 | Q_HDRERR;
00395   if(img->nerrors) quallev0 = quallev0 | Q_CMPERR;
00396   if(img->last_pix_err) quallev0 = quallev0 | Q_LPXERR;
00397   if(img->reopened) quallev0 = quallev0 | Q_REOPENED;
00398   missvals = img->totalvals - img->datavals;
00399   if(missvals > 0) quallev0 = quallev0 | Q_MISS0;
00400   datav = img->totalvals;
00401   if(missvals > (uint32_t)(datav * 0.01)) quallev0 = quallev0 | Q_MISS1;
00402   if(missvals > (uint32_t)(datav * 0.05)) quallev0 = quallev0 | Q_MISS2;
00403   if(missvals > (uint32_t)(datav * 0.25)) quallev0 = quallev0 | Q_MISS3;
00404   if(missvals == datav) quallev0 = quallev0 | Q_MISSI;
00405   if(img->datavals == 0) quallev0 = quallev0 | Q_MISSALL; //high bit, no data
00406 
00407 //  if(missvals != 0) {
00408 //    qmiss = (uint32_t)(0.84 * log(missvals/img->totalvals)) + 15;
00409 //    qmiss = qmiss & 0x0f;     //need this?
00410 //    qmiss = qmiss * 256;      //bits 8-11
00411 //    quallev0 = quallev0 | qmiss;
00412 //  }
00413 
00414 if(fsn == 469769216) quallev0 = quallev0 | Q_CORRUPT; //corrupt image
00415 if(INVALtime) quallev0 = quallev0 | Q_INVALTIME; 
00416 if(cdark = drms_getkey_string(rs, "IMG_TYPE", &status)) {
00417   if(!strcmp(cdark, "DARK")) quallev0 = quallev0 | Q_DARK;
00418   free(cdark);
00419 }
00420 if(!hmiaiaflg) {        //HMI specific qual bits
00421   hsqfgsn = drms_getkey_int(rs, "HSQFGSN", &status);
00422   if(status || (fsn != hsqfgsn)) quallev0 = quallev0 | Q_NOISP;
00423   //Removed per Rock's email Re: lev0 quality updates 9Jun2010 10:22
00424   //if(hseqerr = drms_getkey_string(rs, "HSEQERR", &status)) {
00425   //  if(strcmp(hseqerr, "SUCCESS")) quallev0 = quallev0 | Q_SEQERR;
00426   //  free(hseqerr);
00427   //}
00428   if(hwltnset = drms_getkey_string(rs, "HWLTNSET", &status)) {
00429     if(!strcmp(hwltnset, "OPEN")) quallev0 = quallev0 | Q_ISSOPEN;
00430     free(hwltnset);
00431   }
00432   hcf1encd = drms_getkey_int(rs, "HCF1ENCD", &status);
00433   hcf1pos = drms_getkey_int(rs, "HCF1POS", &status);
00434   //if(!((hcf1encd == hcf1pos) || (hcf1encd == hcf1pos+1) || (hcf1encd == hcf1pos-1)))
00435   if(!((hcf1encd == hcf1pos) || (hcf1encd == hcf1pos-1)))
00436     quallev0 = quallev0 | Q_HCF1ENCD;
00437   hcf2encd = drms_getkey_int(rs, "HCF2ENCD", &status);
00438   hcf2pos = drms_getkey_int(rs, "HCF2POS", &status);
00439   //if(!((hcf2encd == hcf2pos) || (hcf2encd == hcf2pos+1) || (hcf2encd == hcf2pos-1)))
00440   if(!((hcf2encd == hcf2pos) || (hcf2encd == hcf2pos-1)))
00441     quallev0 = quallev0 | Q_HCF2ENCD;
00442   hps1encd = drms_getkey_int(rs, "HPS1ENCD", &status);
00443   hpl1pos = drms_getkey_int(rs, "HPL1POS", &status);
00444   if(!((hpl1pos == hps1encd) || (hpl1pos == (hps1encd+1) % 240)))
00445     quallev0 = quallev0 | Q_HPS1ENCD;
00446   hps2encd = drms_getkey_int(rs, "HPS2ENCD", &status);
00447   hpl2pos = drms_getkey_int(rs, "HPL2POS", &status);
00448   if(!((hpl2pos == hps2encd) || (hpl2pos == (hps2encd+1) % 240)))
00449     quallev0 = quallev0 | Q_HPS2ENCD;
00450   hps3encd = drms_getkey_int(rs, "HPS3ENCD", &status);
00451   hpl3pos = drms_getkey_int(rs, "HPL3POS", &status);
00452   if(!((hpl3pos == hps3encd) || (hpl3pos == (hps3encd+1) % 240)))
00453     quallev0 = quallev0 | Q_HPS3ENCD;
00454   hwt1encd = drms_getkey_int(rs, "HWT1ENCD", &status);
00455   hwl1pos = drms_getkey_int(rs, "HWL1POS", &status);
00456   if(!((hwl1pos == hwt1encd) || (hwl1pos == (hwt1encd+1) % 240)))
00457     quallev0 = quallev0 | Q_HWT1ENCD;
00458   hwt2encd = drms_getkey_int(rs, "HWT2ENCD", &status);
00459   hwl2pos = drms_getkey_int(rs, "HWL2POS", &status);
00460   if(!((hwl2pos == hwt2encd) || (hwl2pos == (hwt2encd+1) % 240)))
00461     quallev0 = quallev0 | Q_HWT2ENCD;
00462   hwt3encd = drms_getkey_int(rs, "HWT3ENCD", &status);
00463   hwl3pos = drms_getkey_int(rs, "HWL3POS", &status);
00464   if(!((hwl3pos == hwt3encd) || (hwl3pos == (hwt3encd+1) % 240)))
00465     quallev0 = quallev0 | Q_HWT3ENCD;
00466   hwt4encd = drms_getkey_int(rs, "HWT4ENCD", &status);
00467   hwl4pos = drms_getkey_int(rs, "HWL4POS", &status);
00468   if(!((hwl4pos == hwt4encd) || (hwl4pos == (hwt4encd+1) % 240)))
00469     quallev0 = quallev0 | Q_HWT4ENCD;
00470   //New 14Aug2013 
00471   hgp1rgst = drms_getkey_int(rs, "HGP1RGST", &status);
00472   if(hgp1rgst != DRMS_MISSING_INT) {
00473     hgp1rgst = (hgp1rgst << 28) & 0x30000000;
00474     quallev0 = quallev0 | hgp1rgst;
00475   }
00476 }
00477 else {              //AIA specific qual bits
00478   asqfsn = drms_getkey_int(rs, "ASQFSN", &status);
00479   if(fsn != asqfsn) quallev0 = quallev0 | Q_NOISP;
00480   if(aistate = drms_getkey_string(rs, "AISTATE", &status)) {
00481     if(!strcmp(aistate, "OPEN")) quallev0 = quallev0 | AQ_ISSOPEN;
00482     free(aistate);
00483   }
00484   strcpy(wave_str, "UNKNOWN");
00485   aiawvlen = drms_getkey_int(rs, "AIAWVLEN", &status);
00486   aifiltyp = drms_getkey_short(rs, "AIFILTYP", &status);
00487   aifwen = drms_getkey_int(rs, "AIFWEN", &status);
00488   aiasen = drms_getkey_int(rs, "AIASEN", &status);
00489   switch(aiawvlen) {
00490   case 9:           //9.4
00491     if(aifiltyp == 0) {
00492       if((aifwen != 269) && (aifwen != 270)) {
00493         quallev0 = quallev0 | A94Mech_Err;
00494       }
00495     }
00496     else if(aifiltyp == 1) {
00497       if((aifwen != 11) && (aifwen != 12)) {
00498         quallev0 = quallev0 | A94Mech_Err;
00499       }
00500     }
00501     else if(aifiltyp == 2) {
00502       if((aifwen != 74) && (aifwen != 75)) {
00503         quallev0 = quallev0 | A94Mech_Err;
00504       }
00505     }
00506     //All the wave_str code can be compacted, but I'm keeping it 
00507     //symetric with Rock's notes
00508     if((aifwen == 269) || (aifwen == 270)) {
00509       strcpy(wave_str, "94_THIN");
00510     }
00511     else if((aifwen == 11) || (aifwen == 12)) {
00512       strcpy(wave_str, "94_THICK");
00513     }
00514     else if((aifwen == 74) || (aifwen == 75)) {
00515       strcpy(wave_str, "94_OPEN");
00516     }
00517     break;
00518   case 1:           //13.1
00519     if(aifiltyp == 0) {
00520       if((aifwen != 269) && (aifwen != 270)) {
00521         quallev0 = quallev0 | A131Mech_Err;
00522       }
00523     }
00524     else if(aifiltyp == 1) {
00525       if((aifwen != 11) && (aifwen != 12)) {
00526         quallev0 = quallev0 | A131Mech_Err;
00527       }
00528     }
00529     else if(aifiltyp == 2) {
00530       if((aifwen != 74) && (aifwen != 75)) {
00531         quallev0 = quallev0 | A131Mech_Err;
00532       }
00533     }
00534     if((aifwen == 269) || (aifwen == 270)) {
00535       strcpy(wave_str, "131_THIN");
00536     }
00537     else if((aifwen == 11) || (aifwen == 12)) {
00538       strcpy(wave_str, "131_THICK");
00539     }
00540     else if((aifwen == 74) || (aifwen == 75)) {
00541       strcpy(wave_str, "131_OPEN");
00542     }
00543     break;
00544   case 7:           //17.1
00545     if(aifiltyp == 0) {
00546       if((aifwen != 203) && (aifwen != 204)) {
00547         quallev0 = quallev0 | A171Mech_Err;
00548       }
00549     }
00550     else if(aifiltyp == 1) {
00551       if((aifwen != 11) && (aifwen != 12)) {
00552         quallev0 = quallev0 | A171Mech_Err;
00553       }
00554     }
00555     else if(aifiltyp == 2) {
00556       //NOTE: no 171_OPEN
00557       if((aifwen != 74) && (aifwen != 75)) {
00558         quallev0 = quallev0 | A171Mech_Err;
00559       }
00560     }
00561     if((aifwen == 203) || (aifwen == 204)) {
00562       strcpy(wave_str, "171_THIN");
00563     }
00564     else if((aifwen == 11) || (aifwen == 12)) {
00565       strcpy(wave_str, "171_THICK");
00566     }
00567     break;
00568 /* !!!TBD fix below like did above after resolve MIX_THIN stuff w/Rock */
00569   case 3:           //19.3
00570     if(aifiltyp == 0) {
00571       if(aiasen != 6) {
00572         quallev0 = quallev0 | A193Mech_Err;
00573       }
00574       if((aifwen != 269) && (aifwen != 270)) {
00575         quallev0 = quallev0 | A193Mech_Err;
00576       }
00577     }
00578     else if(aifiltyp == 1) {
00579       if(aiasen != 6) {
00580         quallev0 = quallev0 | A193Mech_Err;
00581       }
00582       if((aifwen != 11) && (aifwen != 12)) {
00583         quallev0 = quallev0 | A193Mech_Err;
00584       }
00585     }
00586     else if(aifiltyp == 2) {
00587       if(aiasen != 6) {
00588         quallev0 = quallev0 | A193Mech_Err;
00589       }
00590       if((aifwen != 74) && (aifwen != 75)) {
00591         quallev0 = quallev0 | A193Mech_Err;
00592       }
00593     }
00594     if(aiasen == 6) {
00595       if((aifwen == 269) || (aifwen == 270)) {
00596         strcpy(wave_str, "193_THIN");
00597       } else if((aifwen == 11) || (aifwen == 12)) {
00598         strcpy(wave_str, "193_THICK");
00599       } else if((aifwen == 74) || (aifwen == 75)) {
00600         strcpy(wave_str, "193_OPEN");
00601       }
00602     }
00603     else {
00604       if((aifwen == 269) || (aifwen == 270)) {
00605         strcpy(wave_str, "MIX_THIN");
00606       } else if((aifwen == 11) || (aifwen == 12)) {
00607         strcpy(wave_str, "MIX_THICK");
00608       } else if((aifwen == 74) || (aifwen == 75)) {
00609         strcpy(wave_str, "MIX_OPEN");
00610       }
00611     }
00612     break;
00613   case 2:           //21.1
00614     if(aifiltyp == 0) {
00615       if(aiasen != 24) {
00616         quallev0 = quallev0 | A211Mech_Err;
00617       }
00618       if((aifwen != 203) && (aifwen != 204)) {
00619         quallev0 = quallev0 | A211Mech_Err;
00620       }
00621     }
00622     else if(aifiltyp == 1) {
00623       if(aiasen != 24) {
00624         quallev0 = quallev0 | A211Mech_Err;
00625       }
00626       if((aifwen != 137) && (aifwen != 138)) {
00627         quallev0 = quallev0 | A211Mech_Err;
00628       }
00629     }
00630     else if(aifiltyp == 2) {
00631       if(aiasen != 24) {
00632         quallev0 = quallev0 | A211Mech_Err;
00633       }
00634       if((aifwen != 74) && (aifwen != 75)) {
00635         quallev0 = quallev0 | A211Mech_Err;
00636       }
00637     }
00638     if(aiasen == 24) {
00639       if((aifwen == 203) || (aifwen == 204)) {
00640         strcpy(wave_str, "211_THIN");
00641       } else if((aifwen == 137) || (aifwen == 138)) {
00642         strcpy(wave_str, "211_THICK");
00643       } else if((aifwen == 74) || (aifwen == 75)) {
00644         strcpy(wave_str, "211_OPEN");
00645       }
00646     }
00647     else {
00648       if((aifwen == 203) || (aifwen == 204)) {
00649         strcpy(wave_str, "MIX_THIN");
00650       } else if((aifwen == 137) || (aifwen == 138)) {
00651         strcpy(wave_str, "MIX_THICK");
00652       } else if((aifwen == 74) || (aifwen == 75)) {
00653         strcpy(wave_str, "MIX_OPEN");
00654       }
00655     }
00656     break;
00657   case 8:           //30.4
00658     if(aifiltyp == 0) {
00659       if((aifwen != 203) && (aifwen != 204)) {
00660         quallev0 = quallev0 | A304Mech_Err;
00661       }
00662     }
00663     else if(aifiltyp == 1) {
00664       if((aifwen != 137) && (aifwen != 138)) {
00665         quallev0 = quallev0 | A304Mech_Err;
00666       }
00667     }
00668     else if(aifiltyp == 2) {
00669       if((aifwen != 74) && (aifwen != 75)) {
00670         quallev0 = quallev0 | A304Mech_Err;
00671       }
00672     }
00673     if((aifwen == 203) || (aifwen == 204)) {
00674       strcpy(wave_str, "304_THIN");
00675     } else if((aifwen == 137) || (aifwen == 138)) {
00676       strcpy(wave_str, "304_THICK");
00677     } else if((aifwen == 74) || (aifwen == 75)) {
00678       strcpy(wave_str, "304_OPEN");
00679     }
00680     break;
00681   case 0:           //33.5
00682     if(aifiltyp == 0) {
00683       if((aifwen != 203) && (aifwen != 204)) {
00684         quallev0 = quallev0 | A335Mech_Err;
00685       }
00686     }
00687     else if(aifiltyp == 1) {
00688       if((aifwen != 137) && (aifwen != 138)) {
00689         quallev0 = quallev0 | A335Mech_Err;
00690       }
00691     }
00692     else if(aifiltyp == 2) {
00693       if((aifwen != 74) && (aifwen != 75)) {
00694         quallev0 = quallev0 | A335Mech_Err;
00695       }
00696     }
00697     if((aifwen == 203) || (aifwen == 204)) {
00698       strcpy(wave_str, "335_THIN");
00699     } else if((aifwen == 137) || (aifwen == 138)) {
00700       strcpy(wave_str, "335_THICK");
00701     } else if((aifwen == 74) || (aifwen == 75)) {
00702       strcpy(wave_str, "335_OPEN");
00703     }
00704     break;
00705   case 4:           //160.0
00706     if((aifwen != 269) && (aifwen != 270)) {
00707       quallev0 = quallev0 | A160Mech_Err;
00708     }
00709     else strcpy(wave_str, "1600");
00710     break;
00711   case 5:           //170.0
00712     if((aifwen != 137) && (aifwen != 138)) {
00713       quallev0 = quallev0 | A170Mech_Err;
00714     }
00715     else strcpy(wave_str, "1700");
00716     break;
00717   case 6:           //450.0
00718     if((aifwen != 74) && (aifwen != 75)) {
00719       quallev0 = quallev0 | A450Mech_Err;
00720     }
00721     else strcpy(wave_str, "4500");
00722     break;
00723   }
00724   if(!strcmp(wave_str, "UNKNOWN")) quallev0 = quallev0 | AQ_INVAL_WL;
00725 }
00726   //drms_setkey_int(rs, "QUALLEV0", quallev0);  //don't use this anymore
00727   drms_setkey_int(rs, "QUALITY", quallev0);
00728   drms_setkey_string(rs, "BLD_VERS", bld_vers); //build vers to every record
00729   drms_setkey_string(rs, "WAVE_STR", wave_str);
00730   percentd = (float)((100.0 * (float)img->datavals)/(float)img->totalvals);
00731   drms_setkey_float(rs, "PERCENTD", percentd);
00732 }
00733 
00734 // Close out an image.
00735 void close_image(DRMS_Record_t *rs, DRMS_Segment_t *seg, DRMS_Array_t *array,
00736         IMG *img, int fsn)
00737 {
00738   STAT stat;
00739   int status, n, k;
00740   uint32_t missvals;
00741   long long cmdx;
00742   char tlmdsname[128];
00743 
00744   printk("*Closing image for fsn = %u\n", fsn);
00745   if(imgstat(img, &stat)) {
00746     printk("**Error on imgstat() for fsn = %u\n", fsn);
00747   }
00748   else {
00749     drms_setkey_short(rs, "DATAMIN", stat.min);
00750     drms_setkey_short(rs, "DATAMAX", stat.max);
00751     drms_setkey_short(rs, "DATAMEDN", stat.median);
00752     drms_setkey_float(rs, "DATAMEAN", stat.mean);
00753     drms_setkey_float(rs, "DATARMS", stat.rms);
00754     drms_setkey_float(rs, "DATASKEW", stat.skew);
00755     drms_setkey_float(rs, "DATAKURT", stat.kurt);
00756   }
00757   // CAMERA set by HMI_compute_exposure_times()
00758   if(hmiaiaflg) {       // except for AIA use telnum 
00759     drms_setkey_int(rs, "CAMERA", (img->telnum)+1);
00760     cmdx = drms_getkey_longlong(rs, "AIMGSHCE", &status);
00761     if (0 == cmdx) drms_setkey_int(rs, "AIMGTYP", 1);
00762   }
00763   drms_setkey_int(rs, "IMGAPID", img->apid);
00764   drms_setkey_int(rs, "CROPID", img->cropid);
00765   if(drms_setkey_int(rs, "LUTID", img->luid)) {
00766     printk("ERROR on setkey_int for LUTID\n");
00767   }
00768   drms_setkey_int(rs, "TAPCODE", img->tap);
00769   n = (img->N) & 0x1F;
00770   n = n << 3;
00771   k = (img->K) & 0x07;
00772   if(img->N == 16) { 
00773     drms_setkey_int(rs, "COMPID", 0);
00774     drms_setkey_int(rs, "BITSELID", 0);
00775   }
00776   else { 
00777     drms_setkey_int(rs, "COMPID", n+k);
00778     drms_setkey_int(rs, "BITSELID", img->R);
00779   }
00780   drms_setkey_int(rs, "FID", img->fid);
00781   drms_setkey_int(rs, "TOTVALS", img->totalvals);
00782   drms_setkey_int(rs, "DATAVALS", img->datavals);
00783   drms_setkey_int(rs, "NPACKETS", img->npackets);
00784   drms_setkey_int(rs, "NERRORS", img->nerrors);
00785   drms_setkey_short(rs, "EOIERROR", img->last_pix_err);
00786   drms_setkey_short(rs, "HEADRERR", img->headerr);
00787   drms_setkey_short(rs, "OVERFLOW", img->overflow);
00788   missvals = img->totalvals - img->datavals;
00789   drms_setkey_int(rs, "MISSVALS", missvals);
00790   snprintf(tlmdsname, 128, "%s[%s]", tlmseriesname, tlmnamekeyfirst);
00791   drms_setkey_string(rs, "TLMDSNAM", tlmdsname);
00792   unsigned int pts = img->first_packet_time >> 16;
00793   int ptss = img->first_packet_time & 0xffff;
00794   TIME fpt = SDO_to_DRMS_time(pts, ptss);
00795   drms_setkey_double(rs, "IMGFPT", fpt);
00796   drms_setkey_double(rs, "DATE", CURRENT_SYSTEM_TIME);
00797   do_quallev0(rs, img, fsn);        //set the QUALLEV0 keyword
00798 
00799   status = drms_segment_write(seg, array, 0);
00800   if (status) {
00801     printk("ERROR: drms_segment_write error=%d for fsn=%u\n", status, fsn);
00802   }
00803   add_small_array(rs, array, 8, 16); //add Phil's png and small fits
00804   array->data = NULL;        // must do before free 
00805   drms_free_array(array);
00806   if((status = drms_close_record(rs, DRMS_INSERT_RECORD))) {
00807     printk("**ERROR: drms_close_record failed for %s fsn=%u\n",
00808                         lev0seriesname, fsn);
00809   }
00810   img->initialized = 0;     //indicate image is ready for use again
00811   //imgdecode_init_hack(img);
00812 }
00813 
00814 // Got a fatal error. 
00815 void abortit(int stat)
00816 {
00817   printk("***Abort in progress ...\n");
00818   printk("**Exit ingest_lev0 w/ status = %d\n", stat);
00819   if (h0logfp) fclose(h0logfp);
00820   exit(stat);
00821 }
00822 
00823 
00824 int compare_names(const void *a, const void *b)
00825 {
00826   NAMESORT *x=(NAMESORT *)a, *y=(NAMESORT *)b;
00827   return(strcmp(x->name+4, y->name+4)); // skip VC02/VC05 in compare 
00828 }
00829 
00830 unsigned short MDI_getshort (unsigned char *c)    //  machine independent  
00831 {
00832   unsigned short s = 0;
00833 
00834   s = (unsigned short) *c++ << 8;
00835   s |= (unsigned short) *c;
00836   return s;
00837 }
00838 
00839 //fsn_normal_new_image()
00840 int fsn_normal_new_image() 
00841 {
00842   int dstatus, i;
00843 
00844   // start a new image 
00845   Img->initialized = 0;
00846   Img->reopened = 0;
00847   errmsgcnt = 0;
00848   fsn_prev = fsnx;
00849   sprintf(tlmnamekeyfirst, "%s", tlmnamekey);   //save for TLMDSNAM
00850   rs = drms_create_record(drms_env, lev0seriesname, DRMS_PERMANENT, &dstatus);
00851   if(dstatus) {
00852     if(dstatus == DRMS_ERROR_SUMOPEN) {
00853       printk("**ERROR: DRMS can't open w/SUMS. Aborting...\n");
00854       abortit(4);
00855     }
00856     printk("**ERROR: Can't create record for %s fsn=%u\n", lev0seriesname, fsnx);
00857     return(1);
00858   }
00859   dstatus = drms_setkey_int(rs, "FSN", fsnx);
00860   if(!(segment = drms_segment_lookup(rs, "image"))) {
00861     printk("No drms_segment_lookup(rs, image)\n");
00862     return(1);
00863   }
00864   //must initialize Img in case no data pkt comes in for this image
00865 #if 0
00866     Img->datavals = 0;
00867     Img->npackets = 0;
00868     Img->nerrors = 0;
00869     Img->last_pix_err = 0;
00870     Img->first_packet_time = UINT64_MAX;
00871     for (i = 0; i < MAXPIXELS; ++i)
00872         Img->dat[i] = BLANK;
00873     for (i = 0; i < MAXHIST; ++i)
00874         Img->hist[i] = 0;
00875 #endif
00876     imgdecode_init_hack(Img);
00877 
00878   rdat = Img->dat;
00879   segArray = drms_array_create(DRMS_TYPE_SHORT,
00880                                        segment->info->naxis,
00881                                        segment->axis,
00882                                        rdat,
00883                                        &dstatus);
00884   ImgC = Img;       //set current image pointer
00885   return(0);
00886 } 
00887 
00888 // An fsn has changed in the context of a normal stream of
00889 // tlm files. Returns 0 on success.
00890 int fsn_change_normal()
00891 {
00892   int dstatus, rstatus, compid, n, k;
00893   char reopen_dsname[256];
00894   char *cptr;
00895 
00896   printk("*FSN has changed from %u to %u\n", fsn_prev, fsnx);
00897   if(fsn_prev == 0) {   //startup mode. restore any prev image
00898     errmsgcnt = 0;
00899     fsn_prev = fsnx;
00900     if(hmiaiaflg)
00901       sprintf(reopen_dsname, "%s[%u]", LEV0SERIESNAMEAIA, fsnx);
00902     else
00903       sprintf(reopen_dsname, "%s[%u]", LEV0SERIESNAMEHMI, fsnx);
00904     printk("Open normal prev ds: %s\n", reopen_dsname);
00905     rset = drms_open_records(drms_env, reopen_dsname, &rstatus);
00906     if(rstatus) {
00907       printk("Can't do drms_open_records(%s)\n", reopen_dsname);
00908       return(1);          // !!!TBD
00909     }
00910     if(!rset || (rset->n == 0) || rstatus) {
00911       printk("No prev ds\n");     // start a new image
00912       fsn_normal_new_image();
00913     }
00914     else {
00915       Img->initialized = 1;
00916       Img->reopened = 1;
00917       Img->fsn = fsnx;
00918       Img->apid = appid;
00919       rs_old = rset->records[0];
00920       rs = drms_clone_record(rs_old, DRMS_PERMANENT, DRMS_COPY_SEGMENTS, &rstatus);
00921       if(rstatus) {
00922         printk("Can't do drms_clone_record()\n");
00923         return(1);      // !!!TBD ck 
00924       }
00925       drms_close_records(rset, DRMS_FREE_RECORD);
00926       rstatus = drms_setkey_int(rs, "FSN", fsnx);
00927       Img->telnum = drms_getkey_int(rs, "CAMERA", &rstatus);
00928       Img->telnum--;
00929       Img->cropid = drms_getkey_int(rs, "CROPID", &rstatus);
00930       Img->fid = drms_getkey_int(rs, "FID", &rstatus); 
00931       Img->luid = drms_getkey_int(rs, "LUTID", &rstatus);
00932       if(rstatus) {
00933         Img->luid = 0;  
00934         printk("ERROR on getkey_int for LUTID\n");
00935       }
00936       Img->tap = drms_getkey_int(rs, "TAPCODE", &rstatus);
00937       compid = drms_getkey_int(rs, "COMPID", &rstatus);
00938       if(compid == 0) {
00939         n = 16; k = 0;
00940       }
00941       else {
00942         k = compid & 0x07;      //low 3 bits
00943         n = compid >> 3;            //next 5 bits
00944         n = n & 0x1F;
00945       }
00946       Img->N = n;
00947       Img->K = k;
00948       Img->R = drms_getkey_int(rs, "BITSELID", &rstatus);
00949       Img->overflow = drms_getkey_int(rs, "OVERFLOW", &rstatus);
00950       Img->headerr = drms_getkey_int(rs, "HEADRERR", &rstatus);
00951       Img->totalvals = drms_getkey_int(rs, "TOTVALS", &rstatus);
00952       Img->datavals = drms_getkey_int(rs, "DATAVALS", &rstatus);
00953       Img->npackets = drms_getkey_int(rs, "NPACKETS", &rstatus);
00954       Img->nerrors = drms_getkey_int(rs, "NERRORS", &rstatus);
00955       Img->last_pix_err = drms_getkey_int(rs, "EOIERROR", &rstatus);
00956 
00957       TIME fpt = drms_getkey_double(rs, "IMGFPT", &rstatus);
00958       Img->first_packet_time = round(65536.0*(fpt - sdo_epoch));
00959       if(Img->first_packet_time == 0) 
00960         Img->first_packet_time = UINT64_MAX; //fix for stuck 1958 value
00961       snprintf(oldtlmdsnam, 128, "%s", drms_getkey_string(rs, "TLMDSNAM",&rstatus));
00962       cptr = strstr(oldtlmdsnam, "VC");
00963       if(cptr) 
00964         snprintf(tlmnamekeyfirst, 23, "%s", cptr);
00965       else
00966         sprintf(tlmnamekeyfirst, "%s", "UNK");
00967       segment = drms_segment_lookupnum(rs, 0);
00968       cArray = drms_segment_read(segment, DRMS_TYPE_SHORT, &rstatus);
00969       if(rstatus) {
00970         printk("Can't do drms_segment_read()\n");
00971         return(1);      // !!!!TBD ck 
00972       }
00973       short *adata = (short *)cArray->data;
00974       memcpy(Img->dat, adata, 2*MAXPIXELS);
00975       rdat = Img->dat;
00976       segArray = drms_array_create(DRMS_TYPE_SHORT, 
00977                                          segment->info->naxis,
00978                                          segment->axis,
00979                                          rdat,
00980                                          &dstatus);
00981       ImgC = Img;           //set current image pointer
00982       drms_free_array(cArray); //must free from drms_segment_read()
00983     }
00984   }
00985   else {
00986     if(rs) {        // make sure have a created record
00987       close_image(rs, segment, segArray, Img, fsn_prev);
00988       //force a commit to DB on an image boundry & not during a rexmit
00989       if(imagecnt++ >= IMAGE_NUM_COMMIT) {
00990         printk("drms_server_end_transaction()\n");
00991         drms_server_end_transaction(drms_env, 0 , 0);
00992         printk("drms_server_begin_transaction()\n");
00993         drms_server_begin_transaction(drms_env); //start another cycle
00994         imagecnt = 0;
00995       }
00996       fileimgcnt++;
00997     }
00998     else {
00999       printk("**ERROR: Null record ptr for an opened image fsn=%u\n",fsn_prev);
01000     }
01001     if(fsn_normal_new_image()) {
01002       printk("**ERROR: Can't create new image\n");
01003       return(1);
01004     }
01005   }
01006   return(0);
01007 }
01008 
01009 // An fsn has changed in the context of a retransmitted or higher version
01010 // tlm file. Returns 0 on success.
01011 int fsn_change_rexmit() 
01012 {
01013   char rexmit_dsname[256];
01014   int rstatus, dstatus, compid, n, k;
01015   char *cptr;
01016 
01017   if(fsn_prev != 0) {   // close image of prev fsn if not 0 
01018     if(rsc) {       //make sure have created record
01019       close_image(rsc, segmentc, oldArray, ImgO, fsn_prev);
01020       imagecnt++;
01021       fileimgcnt++;
01022     }
01023     else {
01024       printk("**ERROR: Null record ptr for an rexmit image fsn=%u\n",fsn_prev);
01025     }
01026   }
01027   errmsgcnt = 0;
01028   fsn_prev = fsnx;
01029   if(hmiaiaflg) 
01030     sprintf(rexmit_dsname, "%s[%u]", LEV0SERIESNAMEAIA, fsnx);
01031   else 
01032     sprintf(rexmit_dsname, "%s[%u]", LEV0SERIESNAMEHMI, fsnx);
01033   printk("Open prev ds: %s\n", rexmit_dsname);
01034   rset = drms_open_records(drms_env, rexmit_dsname, &rstatus); 
01035   if(rstatus) {
01036     printk("Can't do drms_open_records(%s)\n", rexmit_dsname);
01037     return(1);      // !!!TBD 
01038   }
01039   if(!rset || (rset->n == 0) || rstatus) {
01040 startnew:
01041     printk("No prev ds\n"); // start a new image 
01042     ImgO->initialized = 0;
01043     ImgO->reopened = 0;
01044     imgdecode_init_hack(ImgO);
01045     sprintf(tlmnamekeyfirst, "%s", tlmnamekey); //save for TLMDSNAM
01046     rsc = drms_create_record(drms_env, lev0seriesname, DRMS_PERMANENT, &rstatus);
01047     if(rstatus) {
01048       if(rstatus == DRMS_ERROR_SUMOPEN) {
01049         printk("**ERROR: DRMS can't open w/SUMS. Aborting...\n");
01050         abortit(4);
01051       }
01052       printk("Can't create record for %s fsn=%u\n", lev0seriesname, fsnx);
01053       return(1);                     // !!!TBD ck this 
01054     }
01055     rstatus = drms_setkey_int(rsc, "FSN", fsnx);
01056     if(!(segmentc = drms_segment_lookup(rsc, "image"))) {
01057       printk("No drms_segment_lookup(rsc, image)\n");
01058       return(1);
01059     }
01060     rdat = ImgO->dat;
01061     oldArray = drms_array_create(DRMS_TYPE_SHORT,
01062                                        segmentc->info->naxis,
01063                                        segmentc->axis,
01064                                        rdat,
01065                                        &dstatus);
01066   }
01067   else {
01068     ImgO->initialized = 1;
01069     ImgO->reopened = 1;
01070     ImgO->fsn = fsnx;
01071     ImgO->apid = appid;
01072     rs_old = rset->records[0];
01073     rsc = drms_clone_record(rs_old, DRMS_PERMANENT, DRMS_COPY_SEGMENTS, &rstatus);
01074     if(rstatus) {
01075       printk("Can't do drms_clone_record()\n");
01076       printk("Assume this was a temp ds and the segments are now gone...\n");
01077       goto startnew;
01078       //return(1);      // !!!TBD ck 
01079     }
01080     drms_close_records(rset, DRMS_FREE_RECORD);
01081     rstatus = drms_setkey_int(rsc, "FSN", fsnx);
01082     ImgO->telnum = drms_getkey_int(rsc, "CAMERA", &rstatus);
01083     ImgO->telnum--;
01084     ImgO->cropid = drms_getkey_int(rsc, "CROPID", &rstatus);
01085     ImgO->fid = drms_getkey_int(rsc, "FID", &rstatus); 
01086     ImgO->luid = drms_getkey_int(rsc, "LUTID", &rstatus);
01087     ImgO->tap = drms_getkey_int(rsc, "TAPCODE", &rstatus);
01088     compid = drms_getkey_int(rsc, "COMPID", &rstatus);
01089     if(compid == 0) {
01090       n = 16; k = 0;
01091     }
01092     else {
01093       k = compid & 0x07;              //low 3 bits
01094       n = compid >> 3;                        //next 5 bits
01095       n = n & 0x1F;
01096     }
01097     ImgO->N = n;
01098     ImgO->K = k;
01099     ImgO->R = drms_getkey_int(rsc, "BITSELID", &rstatus);
01100     ImgO->overflow = drms_getkey_int(rsc, "OVERFLOW", &rstatus);
01101     ImgO->headerr = drms_getkey_int(rsc, "HEADRERR", &rstatus);
01102     ImgO->totalvals = drms_getkey_int(rsc, "TOTVALS", &rstatus);
01103     ImgO->datavals = drms_getkey_int(rsc, "DATAVALS", &rstatus);
01104     ImgO->npackets = drms_getkey_int(rsc, "NPACKETS", &rstatus);
01105     ImgO->nerrors = drms_getkey_int(rsc, "NERRORS", &rstatus);
01106     ImgO->last_pix_err = drms_getkey_int(rsc, "EOIERROR", &rstatus);
01107 
01108     TIME fpt = drms_getkey_double(rsc, "IMGFPT", &rstatus);
01109     ImgO->first_packet_time = round(65536.0*(fpt - sdo_epoch));
01110     if(ImgO->first_packet_time == 0) 
01111       ImgO->first_packet_time = UINT64_MAX; //fix for stuck 1958 value
01112     snprintf(oldtlmdsnam, 128, "%s", drms_getkey_string(rsc, "TLMDSNAM",&rstatus));
01113     cptr = strstr(oldtlmdsnam, "VC");
01114     if(cptr) 
01115       snprintf(tlmnamekeyfirst, 23, "%s", cptr);
01116     else
01117       sprintf(tlmnamekeyfirst, "%s", "UNK");
01118     segmentc = drms_segment_lookupnum(rsc, 0);
01119     cArray = drms_segment_read(segmentc, DRMS_TYPE_SHORT, &rstatus);
01120     if(rstatus) {
01121       printk("Can't do drms_segment_read()\n");
01122       return(1);        // !!!!TBD ck 
01123     }
01124     short *adata = (short *)cArray->data;
01125     memcpy(ImgO->dat, adata, 2*MAXPIXELS);
01126     rdat = ImgO->dat;
01127     oldArray = drms_array_create(DRMS_TYPE_SHORT, 
01128                                        segmentc->info->naxis,
01129                                        segmentc->axis,
01130                                        rdat,
01131                                        &dstatus);
01132     drms_free_array(cArray); //must free from drms_segment_read()
01133   }
01134   ImgC = ImgO;      //set current image pointer
01135   return(0);
01136 }
01137 
01138 // Process the tlm file to validate and to extract the lev0 image.
01139 int get_tlm(char *file, int rexmit, int higherver)
01140 {
01141   FILE *fpin;
01142   unsigned char cbuf[PKTSZ];
01143   long long gap_42_cnt;
01144   int status, rstatus, fpkt_cnt, i, j, sync_bad_cnt;
01145   int datval, eflg, firstflg;
01146   unsigned int cnt1, cnt2, cnt3, gap_24_cnt;
01147   int zero_pn;
01148   unsigned short pksync1, pksync2;
01149   float ftmp;
01150   int decode_status=0;
01151   unsigned int Fsn;
01152 
01153   if(!(fpin = fopen(file, "r"))) {  // open the tlm input 
01154     printk("*Can't open tlm file %s\n", file);
01155     return(1);
01156   }
01157   BeginTimer(1);            //time tlm file processing
01158   printk("*Processing tlm file %s\n", file);
01159   fpkt_cnt = sync_bad_cnt = eflg = errmsgcnt = fileimgcnt = 0;
01160   zero_pn = gap_24_cnt = gap_42_cnt = 0;
01161   firstflg = 1; 
01162   if(rexmit || higherver) {
01163     if(Img->initialized) {      // close normal mode image
01164       if(rs) {
01165         close_image(rs, segment, segArray, Img, fsn_prev);
01166         printk("drms_server_end_transaction()\n");
01167         drms_server_end_transaction(drms_env, 0 , 0);
01168         printk("drms_server_begin_transaction()\n");
01169         drms_server_begin_transaction(drms_env); //start another cycle
01170         imagecnt = 0;
01171       }
01172       else
01173         printk("**ERROR: Null record ptr for an opened image fsn=%u\n",fsn_prev);
01174     }
01175     fsn_pre_rexmit = fsn_prev;      // restore this at end of rexmit tlm file
01176     fsn_prev = 0;           // cause a new image 
01177   }
01178   // read a VCDU packet 
01179   while((status = fread(cbuf,sizeof(char),PKTSZ,fpin) ) == PKTSZ) {
01180     pksync1 = MDI_getshort(cbuf);
01181     pksync2 = MDI_getshort(cbuf+2);
01182     if((pksync1 == 0) && (pksync2 == 0)) { // skip 0 pn code 
01183       if(!zero_pn) {            // give msg for 1st one only 
01184         printk("*0 PN code at pkt# %d\n", fpkt_cnt);
01185         printk("*Subsequent ones will be ignored until non-0 again\n");
01186         zero_pn = 1;
01187       }
01188       fpkt_cnt++;           // count # of pkts found 
01189       continue;
01190     }
01191     if((pksync1 != 0x1acf) || (pksync2 != 0xfc1d)) {
01192       printk("*Lost sync at VCDU pkt# %d. pksync1=%x pksync2=%x\n", 
01193         fpkt_cnt, pksync1, pksync2);
01194       fpkt_cnt++;           // count # of pkts found 
01195       eflg++;
01196       if(sync_bad_cnt++ > 4) {
01197         printk("**Too many out of sync packets.\n");
01198         //return(1);
01199         return(0);      //changed on 01Oct2010
01200       }
01201       printk("  Will attempt to press on...\n");
01202       zero_pn = 0;
01203       continue;
01204     }
01205     if(firstflg) {      // print first good sync found 
01206       printk("*VCDU pkt# %d sync = %x %x\n", fpkt_cnt, pksync1, pksync2);
01207     }
01208     fpkt_cnt++;         // count # of pkts found 
01209     // get 24 bit VCDU counter 
01210     cnt1 = MDI_getshort(cbuf+6);
01211     cnt2 = MDI_getshort(cbuf+8);
01212     cnt2 = (cnt2 >> 8)& 0xFF;
01213     cnt2 = ((cnt1 << 8)& 0xFF00) + cnt2;
01214     cnt1 = (cnt1 >> 8)& 0xFF;
01215     vcdu_24_cnt = (cnt1*65536) + cnt2;
01216     if(vcdu_24_cnt_next != vcdu_24_cnt) {
01217       printk("*VCDU 24bit seq num out of sequence. exp: %u  rec: %u\n", 
01218         vcdu_24_cnt_next, vcdu_24_cnt);
01219       if(vcdu_24_cnt_next > vcdu_24_cnt) {
01220         printk("*NOTE: VCDU 24 bit counter retarded\n"); //cntr does go thru 0
01221         printk("*NOTE: gap report will be inaccurate (tbd)\n");
01222       }
01223       if(!firstflg) {       // don't count gap across .tlm files 
01224         gap_24_cnt += vcdu_24_cnt - vcdu_24_cnt_next;
01225       }
01226     }
01227     vcdu_24_cnt_next = vcdu_24_cnt + 1;
01228     // now get the 42bit IM_PDU counter 
01229     cnt1 = MDI_getshort(cbuf+10);
01230     cnt2 = MDI_getshort(cbuf+12);
01231     cnt3 = MDI_getshort(cbuf+14);
01232     cnt1 = cnt1 & 0x03ff;
01233     vcdu_seq_num = (cnt1*4294967296) + (cnt2*65536) + cnt3;
01234     // printk("vcdu_seq_num = %lld\n", vcdu_seq_num); 
01235     if(vcdu_seq_num_next != vcdu_seq_num) {
01236       printk("*IM_PDU seq num out of sequence. exp: %lld  rec: %lld\n", 
01237         vcdu_seq_num_next, vcdu_seq_num);
01238       if(vcdu_seq_num_next > vcdu_seq_num) {
01239         printk("*NOTE: IM_PDU 42 bit counter retarded\n");
01240         printk("*NOTE: gap report will be inaccurate\n");
01241       }
01242       if(!firstflg) {       // don't count gap across .tlm files 
01243         gap_42_cnt += vcdu_seq_num - vcdu_seq_num_next;
01244       }
01245       eflg++;
01246     }
01247     firstflg = 0;
01248     vcdu_seq_num_next = vcdu_seq_num + 1;
01249     // get the App ID. Low 11 bit of short at buf+18 
01250     appid = MDI_getshort(cbuf+18);
01251     appid = appid & 0x07ff;
01252     if(hmiaiaflg) {     //aia
01253       //testid1 = 509; testid2 = 519;
01254       testid1 = 507; testid2 = 517;
01255     }
01256     else {
01257       //testid1 = 409; testid2 = 419;
01258       testid1 = 407; testid2 = 417;
01259     }
01260     //if(appid == TESTAPPID) {  // appid of test pattern 
01261     if((appid == testid1) || (appid == testid2)) {
01262       if(errmsgcnt++ < MAXERRMSGCNT) {
01263         printk("*Test ApID of %0x found for IM_PDU Cntr = %lld\n", 
01264             appid, vcdu_seq_num);
01265         for(i=0, j=TESTVALUE; i < 877; i=i+2, j++) {
01266           datval = MDI_getshort(cbuf+32+i); // next data value 
01267           if(datval != j) {
01268             printk("*Test data value=%0x, expected=%0x for IM_PDU Cntr=%lld\n", 
01269         datval, j, vcdu_seq_num);
01270             printk("*File = %s\n", file);
01271             eflg++;
01272             break;      // skip the rest of this packet 
01273           }
01274         }
01275       }
01276       continue;         // go on to next packet 
01277     }
01278 
01279     //printk("$$$$$ appid found =  0x%x %d\n", appid, appid); //!!TEMP
01280     // Parse tlm packet headers. 
01281     if(appid == APID_HMI_SCIENCE_1 || appid == APID_HMI_SCIENCE_2 || 
01282     appid == APID_AIA_SCIENCE_1 || appid == APID_AIA_SCIENCE_2)
01283     {
01284       cnt1 = MDI_getshort(cbuf+32);
01285       cnt2 = MDI_getshort(cbuf+34);
01286       fsnx = (unsigned int)(cnt1<<16)+(unsigned int)(cnt2);
01287       fsnx = fsnx & 0x3fffffff;     //low 30bits for fsn */
01288       if(fsnx == 0) continue;       //a 0 fsn is not acceptable
01289       if(rexmit || higherver) {
01290         if(fsnx != fsn_prev) {          // the fsn has changed
01291           if(fsn_change_rexmit()) { //handle old & new images
01292             printk("***FATAL ERROR in fsn_change_rexmit()\n");
01293             return(1);
01294           }
01295         }
01296         else {              //prev could be hk w/diff apid
01297           ImgC->apid = appid;
01298         }
01299       }
01300       else {            // continuing normal stream
01301         if(fsnx != fsn_prev) {          // the fsn has changed 
01302           if(fsn_change_normal()) { //handle old & new images
01303             printk("***FATAL ERROR in fsn_change_normal()\n");
01304             return(1);
01305           }
01306         }
01307       }
01308       // send the sci data to Keh-Cheng. call with pointer to M_PDU_Header 
01309       rstatus = imgdecode((unsigned short *)(cbuf+10), ImgC);
01310       switch(rstatus) {
01311       case 0:
01312         // A science data VCDU was successfully decoded 
01313         break;
01314       case IMGDECODE_DECOMPRESS_ERROR:
01315         if(errmsgcnt++ < MAXERRMSGCNT)
01316           printk("*imgdecode() ret: IMGDECODE_DECOMPRESS_ERROR\n");
01317         break;
01318       case IMGDECODE_TOO_MANY_PIXELS:
01319         if(errmsgcnt++ < MAXERRMSGCNT)
01320           printk("*imgdecode() ret: IMGDECODE_TOO_MANY_PIXELS\n");
01321         break;
01322       case IMGDECODE_BAD_N:
01323         if(errmsgcnt++ < MAXERRMSGCNT)
01324           printk("*imgdecode() ret: IMGDECODE_BAD_N\n");
01325         break;
01326       case IMGDECODE_BAD_APID:
01327         if(errmsgcnt++ < MAXERRMSGCNT)
01328           printk("*imgdecode() ret: IMGDECODE_BAD_APID\n");
01329         break;
01330       case IMGDECODE_NO_LOOKUP_TABLE:
01331         if(errmsgcnt++ < MAXERRMSGCNT)
01332           printk("*imgdecode() ret: IMGDECODE_NO_LOOKUP_TABLE\n");
01333         break;
01334       case IMGDECODE_LOOKUP_ID_MISMATCH:
01335         if(errmsgcnt++ < MAXERRMSGCNT)
01336           printk("*imgdecode() ret: IMGDECODE_LOOKUP_ID_MISMATCH\n");
01337         break;
01338       case IMGDECODE_BAD_LOOKUP_TABLE:
01339         if(errmsgcnt++ < MAXERRMSGCNT)
01340           printk("*imgdecode() ret: IMGDECODE_BAD_LOOKUP_TABLE\n");
01341         break;
01342       case IMGDECODE_NO_CROP_TABLE:
01343         if(errmsgcnt++ < MAXERRMSGCNT)
01344           printk("*imgdecode() ret: IMGDECODE_NO_CROP_TABLE\n");
01345         break;
01346       case IMGDECODE_CROP_ID_MISMATCH:
01347         if(errmsgcnt++ < MAXERRMSGCNT)
01348           printk("*imgdecode() ret: IMGDECODE_CROP_ID_MISMATCH\n");
01349         break;
01350       case IMGDECODE_BAD_CROP_GEOMETRY:
01351         if(errmsgcnt++ < MAXERRMSGCNT)
01352           printk("*imgdecode() ret: IMGDECODE_BAD_CROP_GEOMETRY\n");
01353         break;
01354       case IMGDECODE_BAD_CROP_TABLE:
01355         if(errmsgcnt++ < MAXERRMSGCNT)
01356           printk("*imgdecode() ret: IMGDECODE_BAD_CROP_TABLE\n");
01357         break;
01358       case IMGDECODE_BAD_CROP_SKIP_TAKE:
01359         if(errmsgcnt++ < MAXERRMSGCNT)
01360           printk("*imgdecode() ret: IMGDECODE_BAD_CROP_SKIP_TAKE\n");
01361         break;
01362       case IMGDECODE_BAD_OFFSET:
01363         if(errmsgcnt++ < MAXERRMSGCNT)
01364           printk("*imgdecode() ret: IMGDECODE_BAD_OFFSET\n");
01365         break;
01366       case IMGDECODE_OUT_OF_MEMORY:
01367         if(errmsgcnt++ < MAXERRMSGCNT)
01368           printk("*imgdecode() ret: IMGDECODE_OUT_OF_MEMORY\n");
01369         break;
01370       default:
01371         if(errmsgcnt++ < MAXERRMSGCNT)
01372           printk("*imgdecode() ret: unknown err status = %d:\n", rstatus);
01373         break;
01374       }
01375     }
01376     else {          // send the HK data to Carl 
01377       //printk("$$$ appid assumed hk =  0x%x %d\n", appid, appid); //!!TEMP
01378       decode_status = decode_next_hk_vcdu((unsigned short *)(cbuf+10), &Hk, &Fsn);
01379       switch (decode_status) {
01380         case SUCCESS_HK_NEED_TO_WTD_CTD:
01381           printk("*ISP found for fsn = %u\n", Fsn);
01382           fsnx = Fsn;
01383           if(rexmit || higherver) {
01384             if(fsnx != fsn_prev) {          // the fsn has changed
01385               if(fsn_change_rexmit()) { //handle old & new images
01386                 printk("***FATAL ERROR in fsn_change_rexmit()\n");
01387                 return(1);
01388               }
01389             }
01390             //calculate and setkey some values from the keywords returned
01391             HMI_compute_exposure_times(rsc, Hk->keywords, hmiaiaflg);
01392             whk_status = write_hk_to_drms(rsc, &Hk); //ISP keywords to drms
01393             if(!hmiaiaflg) {
01394               if(whk_status = set_HMI_mech_values(rsc)) {
01395                 printk("***ERROR: mechanism position keywords are wrong!\n");
01396               }
01397             }
01398           }
01399           else {                    //normal stream
01400             if(fsnx != fsn_prev) {          // the fsn has changed 
01401               if(fsn_change_normal()) { //handle old & new images
01402                 printk("***FATAL ERROR in fsn_change_normal()\n");
01403                 return(1);
01404               }
01405             }
01406             //calculate and setkey some values from the keywords returned
01407             HMI_compute_exposure_times(rs, Hk->keywords, hmiaiaflg);
01408             whk_status = write_hk_to_drms(rs, &Hk); //ISP keywords to drms
01409             if(!hmiaiaflg) {
01410               if(whk_status = set_HMI_mech_values(rs)) {
01411                 printk("***ERROR: mechanism position keywords are wrong!\n");
01412               }
01413             }
01414           }
01415           hk_ccsds_free(&Hk);
01416           rstatus=0;
01417           break;
01418         case SUCCESS_HK_NEED_TO_CTD:
01419           /*NO need to write  keywords's to drms for level0 data series 
01420             BUT need to commit to drms for level0 by APID data series*/
01421           break;
01422         case SUCCESS_SKIP_IMAGE:
01423           printk("decode_next_hk_vcdu() ret:Warning:SUCCESS_SKIP_IMAGE\n");
01424           break;
01425         case SUCCESS_SKIP_PROCESSING_APID:
01426           printk("decode_next_hk_vcdu() ret:Warning:SUCCESS_SKIP_PROCESSING_APID:\n");
01427           break;
01428         case ERROR_HK_FAILED_GETTING_FSN:
01429           printk("decode_next_hk_vcdu() ret:Warning:ERROR_HK_FAILED_GETTING_FSN:\n");
01430           break;
01431         case ERROR_NODATA:
01432           printk("decode_next_hk_vcdu() ret:Warning:ERROR_NODATA\n");
01433           break;
01434         case ERROR_HK_ENVIRONMENT_VARS_NOT_SET:
01435           printk("decode_next_hk_vcdu() ret:ERROR:ERROR_HK_ENVIRONMENT_VARS_NOT_SET\n");
01436            break;
01437         case ERROR_HK_FAILED_OPEN_DAYFILE:
01438           printk("decode_next_hk_vcdu() ret:ERROR:ERROR_HK_FAILED_OPEN_DAYFILE\n");
01439            break;
01440         default:
01441           printk("decode_next_hk_vcdu() ret:Warning:Unexpected return code for decode_status:<%d>:\n", decode_status);
01442           break;
01443       }
01444 
01445     }
01446   }             // end of rd of vcdu pkts 
01447   fclose(fpin);
01448   if(!eflg) {
01449     printk("*No errors in tlm file\n");
01450   }
01451   if(rexmit || higherver) { // close the opened record 
01452     if(rsc) {           // make sure have created record
01453       close_image(rsc, segmentc, oldArray, ImgO, fsnx);
01454       printk("drms_server_end_transaction()\n");  //commit at end of file
01455       drms_server_end_transaction(drms_env, 0 , 0);
01456       printk("drms_server_begin_transaction()\n\n");
01457       drms_server_begin_transaction(drms_env); //start another cycle
01458       imagecnt = 0;
01459       fileimgcnt++;
01460       //fsn_prev = fsn_pre_rexmit; // restore orig for next normal .tlm file
01461       fsn_prev = 0;     //need for restart of normal mode
01462       rsc = 0;
01463     }
01464   }
01465   ftmp = EndTimer(1);
01466   printk("**Processed %s\n**with %d images and %d VCDUs in %f sec\n\n",
01467     file, fileimgcnt, fpkt_cnt, ftmp);
01468 
01469   //now commit to DB in case another file doesn't come in
01470 /***************************TEMP***********************************
01471 NOTE: Can't do this unless close current image and then set fsn=0
01472       so that the next tlm file will start by opening the image again.
01473       I don't think that we want this. Instead find a way to get the ^C
01474       instead of DRMS, and then do a close and commit.
01475   printk("drms_server_end_transaction()\n");
01476   drms_server_end_transaction(drms_env, 0 , 0);
01477   printk("drms_server_begin_transaction()\n\n");
01478   drms_server_begin_transaction(drms_env); //start another cycle
01479   imagecnt = 0;
01480 ***************************TEMP***********************************/
01481 
01482   if(fpkt_cnt != total_tlm_vcdu) {
01483     printk("**WARNING: Found #vcdu=%d; expected=%d\n", fpkt_cnt, total_tlm_vcdu);
01484   }
01485   //if(gap_24_cnt != total_missing_vcdu) {
01486   //  printk("**WARNING: VCDU 24bit cntr gaps=%d; expected=%d\n",
01487   //      gap_24_cnt, total_missing_vcdu);
01488   //}
01489   if(gap_42_cnt != total_missing_im_pdu) {
01490     printk("**WARNING: IM_PDU 42bit cntr gaps=%lld; expected=%lld\n",
01491      gap_42_cnt, total_missing_im_pdu);
01492   }
01493   ignoresigalrmflg = 1;     //got a file. noop next alarm signal timeout
01494   return(0);
01495 }
01496 
01497 // This is the main loop that gets the .qac and .tlm files and 
01498 // puts them into ds tlmseriesname and extracts the lev0 and puts it in 
01499 // lev0seriesname in DRMS.
01500 void do_ingest()
01501 {
01502   FILE *fp;
01503   DRMS_Record_t *rs_tlm;
01504   DIR *dfd;
01505   NAMESORT *nameptr;
01506   struct dirent *dp;
01507   int i, j, status, mvstat;
01508   int rexmit, higherversion;
01509   char path[DRMS_MAXPATHLEN];
01510   char name[128], line[128], tlmfile[128], tlmname[96];
01511   char cmd[256], xxname[128], vername[16];
01512   char *token, *filename;
01513 
01514   if((dfd=opendir(tlmdir)) == NULL) {
01515     printk("**Can't opendir(%s) to find files\n", tlmdir);
01516     abortit(3);
01517   }
01518   i = 0;
01519   if((nameptr = (NAMESORT *)malloc(MAXFILES * sizeof(NAMESORT))) == NULL) {
01520     printk("***Can't alloc memory for file name sort\n");
01521     abortit(3);
01522   }
01523 
01524   while((dp=readdir(dfd)) != NULL) {
01525     // printk("%s\n", dp->d_name) ; continue; // !!TEMP 
01526     // Only accept our files. 
01527     if(strstr(dp->d_name, pchan) || strstr(dp->d_name, rchan) || strstr(dp->d_name, ".dsf")) {
01528       nameptr[i++].name = strdup(dp->d_name);
01529       if(i >= MAXFILES) {
01530         printk("***Fatal error. Too many (%d) files in %s\n", MAXFILES, tlmdir);
01531         printf("***Fatal error. Too many (%d) files in %s\n", MAXFILES, tlmdir);
01532         abortit(3);
01533       }
01534       if(!strstr(dp->d_name, ".dsf")) cntsleeps = 0;    //we saw a file
01535     }
01536   }
01537   closedir(dfd);
01538   qsort(nameptr, i, sizeof(NAMESORT), &compare_names);
01539 
01540   for(j=0; j < i; j++) {
01541     //printk("####QSORT FILES: %s\n", nameptr[j].name); // !!TEMP 
01542     // NOTE: the dsf files is moved to indir/dsf e.g. (/dds/soc2pipe/aia/dsf)
01543     // Currently the cron job pipefe_rm does this:
01544     // `/bin/mv $dsfname /dds/socdc/hmi/dsf`
01545     //Only do for V01/VC02. VC04/VC05 won't mv the .dsf
01546     if(!strcmp(pchan, "VC01") || !strcmp(pchan, "VC02")) { 
01547       if(strstr(nameptr[j].name, ".dsf")) {
01548         //sprintf(cmd, "/bin/mv %s/%s %s", tlmdir, nameptr[j].name, outdir);
01549         sprintf(cmd, "/bin/mv -f %s/%s %s/dsf/",tlmdir,nameptr[j].name,tlmdir);
01550         printk("*mv dsf file to %s/dsf/\n", tlmdir);
01551         printk("%s\n", cmd);
01552         if(system(cmd)) {
01553           printk("***Error on: %s\n", cmd);
01554         }
01555       }
01556     }
01557     if(!strstr(nameptr[j].name, ".qac")) {  // can be .qac or .qacx 
01558       free(nameptr[j].name);
01559       continue;
01560     }
01561     rexmit = higherversion = 0;
01562     if(strstr(nameptr[j].name, ".qacx")) {      // this is a rexmit file 
01563       rexmit = 1;
01564     }
01565     sprintf(name, "%s/%s", tlmdir, nameptr[j].name);
01566     printk("%s\n*Found qac file:\n* %s\n", do_datestr(), name);
01567     if(!(fp=fopen(name, "r"))) {
01568       printk("***Can't open %s\n", name);
01569       free(nameptr[j].name);
01570       continue;
01571     }
01572     // NOTE: the qac file is already verified by the caller of ingest_lev0 
01573     while(fgets(line, 256, fp)) {   // get qac file lines 
01574       if(line[0] == '#' || line[0] == '\n') continue;
01575       if(strstr(line, "TLM_FILE_NAME=")) {
01576         token = (char *)strtok(line, "=");
01577         token = (char *)strtok(NULL, "\n");
01578         printk("tlm file is %s\n", token);
01579         sprintf(tlmfile, "%s/%s", tlmdir, token);
01580         sprintf(tlmname, "%s", token);
01581       }
01582       else if(strstr(line, "TLM_FILE_SIZE=")) {
01583         token = (char *)strtok(line, "=");
01584         token = (char *)strtok(NULL, "=");
01585         printk("*tlm file size is %s", token);
01586       }
01587       else if((strstr(line, "TOTAL_TLM_IM_PDU=")) || (strstr(line, "TOTAL_VCDU="))) {
01588         token = (char *)strtok(line, "=");
01589         token = (char *)strtok(NULL, "\n");
01590         total_tlm_vcdu = atoi(token);
01591       }
01592       else if(strstr(line, "TOTAL_MISSING_VCDU=")) {
01593         token = (char *)strtok(line, "=");
01594         token = (char *)strtok(NULL, "\n");
01595         total_missing_vcdu = atoi(token);
01596       }
01597       else if(strstr(line, "TOTAL_MISSING_IM_PDU=")) {
01598         token = (char *)strtok(line, "=");
01599         token = (char *)strtok(NULL, "\n");
01600         total_missing_im_pdu = atol(token);
01601         break;
01602       }
01603     }
01604     fclose(fp);
01605     //alarm(ALRMSEC);       //restart alarm if no more files come
01606     strcpy((char *)tlmnamekey, (char *)tlmname);
01607     filename = (char *)rindex(tlmname, '.');
01608     if(!filename) {
01609       printk("**ERROR: Bad formed tlm name in qac file\n");
01610       printk("  tlm name = %s\n", tlmname);
01611       continue;
01612     }
01613     *filename = 0;          // elim .tlm 
01614     *((char *)tlmnamekey+22) = 0;   // elim after hh_mm_ss
01615     rs_tlm = drms_create_record(drms_env, tlmseriesname, 
01616                 DRMS_PERMANENT, &status);
01617     if(status) {
01618       if(status == DRMS_ERROR_SUMOPEN) {
01619         printk("**ERROR: DRMS can't open w/SUMS. Aborting...\n");
01620         abortit(4);
01621       }
01622       printk("***Can't create record for %s. Status=%d\n", tlmseriesname, status);
01623       continue;
01624     }
01625     if((status = drms_setkey_string(rs_tlm, "filename", tlmnamekey))) {
01626       printk("**ERROR: drms_setkey_string failed for 'filename'\n");
01627     }
01628     drms_record_directory(rs_tlm, path, 0);
01629     if(!*path) {
01630       printk("***ERROR: No path to segment for %s\n", tlmseriesname);
01631       continue;
01632     }
01633     if(outdir) {
01634       sprintf(cmd, "cp -p %s %s", name, outdir);
01635       printk("*cp qac file to %s\n", outdir);
01636       printk("%s\n", cmd);
01637       if(system(cmd)) {
01638         printk("***Error on: %s\n", cmd);
01639       }
01640       sprintf(cmd, "cp -p %s %s", tlmfile, outdir);
01641       printk("*cp tlm file to %s\n", outdir);
01642       printk("%s\n", cmd);
01643       if(system(cmd)) {
01644         printk("***Error on: %s\n", cmd);
01645       }
01646     }
01647     sprintf(cmd, "/bin/mv %s %s", name, path);
01648     printk("*mv qac to %s\n", path);
01649     printk("%s\n", cmd);
01650     if(status = system(cmd)) {
01651       printk("**ERROR: %d errno=%d on: %s\n", status, errno, cmd);
01652       if(WIFEXITED(status)) {
01653         if(mvstat = WEXITSTATUS(status)) {  //status ret by mv
01654           printk("**ERROR: mv exit status = %d\n", mvstat);
01655         }
01656       }
01657     }
01658     sprintf(cmd, "/bin/mv %s %s", tlmfile, path);
01659     printk("*mv tlm to %s\n", path);
01660     printk("%s\n", cmd);
01661     if(status = system(cmd)) {
01662       printk("**ERROR: %d errno=%d on: %s\n", status, errno, cmd);
01663       if(WIFEXITED(status)) {
01664         if(mvstat = WEXITSTATUS(status)) {  //status ret by mv
01665           printk("**ERROR: mv exit status = %d\n", mvstat);
01666         }
01667       }
01668       printk("**Continue after ERROR on mv of tlm file\n");
01669       continue;
01670     }
01671     if((status = drms_close_record(rs_tlm, DRMS_INSERT_RECORD))) {
01672       printk("**ERROR: drms_close_record failed for %s\n", tlmseriesname);
01673     }
01674 
01675     sprintf(xxname, "%s/%s.tlm", path, tlmname);
01676     filename = (char *)rindex(tlmname, '_');
01677     sprintf(vername, "%s", filename);   // e.g. _00 or _01 etc. 
01678     if(strcmp(vername, "_00")) {    // this is a higher vers # file 
01679       higherversion = 1;
01680       printk("Higher version tlm found: %s.tlm\n", tlmname);
01681     }
01682     firstfound = 1;         //a file has been seen
01683     if(get_tlm(xxname, rexmit, higherversion)) { // lev0 extraction of image 
01684       printk("***Error in lev0 extraction for %s\n", xxname);
01685       //printk("***Going to abort\n");
01686       //abortflg = 1;
01687     }
01688     if((stat(stopfile, &stbuf) == 0) || abortflg) { break; } //signal to stop
01689   }
01690   free(nameptr);
01691 }
01692 
01693 // Initial setup stuff called when main is first entered.
01694 void setup()
01695 {
01696   FILE *fp;
01697   int i;
01698   char string[128], cwdbuf[128], idstr[256];
01699   char envfile[100], s1[256],s2[256],s3[256], line[256];
01700   ThreadSigErr_t error = kThreadSigErr_Success;
01701 
01702   sdo_epoch = sscan_time("1958.01.01_00:00:00_TAI");
01703   do_datestr();
01704   printk_set(h0log, h0log); // set for printk calls 
01705   printk("%s\n", datestr);
01706   getcwd(cwdbuf, 126);
01707   sprintf(idstr, "Cwd: %s\nCall: ", cwdbuf);
01708   if(grounddata)
01709     sprintf(string, "ingest_lev0 -g started as pid=%d user=%s\n", getpid(), username);
01710   else
01711     sprintf(string, "ingest_lev0 started as pid=%d user=%s\n", getpid(), username);
01712   strcat(idstr, string);
01713   printk("*%s", idstr);
01714   if(restartflg) printk("-r ");
01715   sprintf(argvc, "vc=%s", vc);
01716   sprintf(argindir, "indir=%s", tlmdir);
01717   sprintf(arglogfile, "logfile=%s", logname);
01718   if(outdir) {
01719     sprintf(argoutdir, "outdir=%s", outdir);
01720     printk("%s %s %s %s\n", argvc, argindir, argoutdir, arglogfile);
01721   } else {
01722     printk("%s %s %s\n", argvc, argindir, arglogfile);
01723   }
01724   strcpy(pchan, vc);        // virtual channel primary 
01725   if(rexmitmode)        //new 'stopX' 11/2/2012
01726     sprintf(stopfile, "/usr/local/logs/lev0/%s_stopX", pchan);
01727   else
01728     sprintf(stopfile, "/usr/local/logs/lev0/%s_stop", pchan);
01729   //Dont rm stopfile any more (1/6/2012). 
01730   //With the new rexmit dir there is a second
01731   //ingest_lev0 running on a VC and it needs the stop file too.
01732   //The stop file is removed when ingest_lev0 is started by
01733   //doingestlev0_[HMI,AIA].pl
01734   //sprintf(string, "/bin/rm -f %s", stopfile); //remove any stop file
01735   //system(string);
01736   for(i=0; ; i++) {     // ck for valid and get redundant chan 
01737     if(!strcmp(p_r_chan_pairs[i].pchan, pchan)) {
01738       strcpy(rchan, p_r_chan_pairs[i].rchan);
01739       hmiaiaflg = p_r_chan_pairs[i].instru;
01740       break;
01741     }
01742     if(!strcmp(p_r_chan_pairs[i].pchan, "n/a")) {
01743       printk("!!ERROR: Invalid VCid (%s) specified\n", pchan);
01744       fprintf(stderr, "!!ERROR: Invalid VCid (%s) specified. Abort\n", pchan);
01745       abortit(1);
01746     }
01747   }
01748   if(hmiaiaflg) {
01749     if(grounddata) {
01750       sprintf(tlmseriesname, "%s", TLMSERIESNAMEAIAGND);
01751       sprintf(lev0seriesname, "%s", LEV0SERIESNAMEAIAGND);
01752     }
01753     else {
01754       sprintf(tlmseriesname, "%s", TLMSERIESNAMEAIA);
01755       sprintf(lev0seriesname, "%s", LEV0SERIESNAMEAIA);
01756     }
01757   }
01758   else {
01759     if(grounddata) {
01760       sprintf(tlmseriesname, "%s", TLMSERIESNAMEHMIGND);
01761       sprintf(lev0seriesname, "%s", LEV0SERIESNAMEHMIGND);
01762     }
01763     else {
01764       sprintf(tlmseriesname, "%s", TLMSERIESNAMEHMI);
01765       sprintf(lev0seriesname, "%s", LEV0SERIESNAMEHMI);
01766     }
01767   }
01768   if(!restartflg) {
01769     printk("tlmseriesname=%s\nlev0seriesname=%s\n", 
01770         tlmseriesname, lev0seriesname);
01771   }
01772   sprintf(bld_vers, "%s", jsoc_version);
01773   umask(002);           // allow group write 
01774   Image.initialized = 0;    // init the two image structures 
01775   ImageOld.initialized = 0;
01776   Img = &Image;
01777   ImgO = &ImageOld;
01778   imgdecode_init_hack(Img);
01779   imgdecode_init_hack(ImgO);
01780 
01781   //set environment variables for hk code
01782   //create filename and path
01783   if(grounddata) strcpy(envfile, ENVFILE_GND );
01784   else strcpy(envfile, ENVFILE );
01785   //fopen file
01786   if(!(fp=fopen(envfile, "r"))) {
01787       printk("***Can't open %s. Check setting is correct\n", envfile);
01788       exit(0);
01789   }
01790   //read in lines
01791   while( fgets(line, MAXLINE_IN_FILE, fp) != NULL )
01792   {
01793     if (!strncmp(line, "#", 1)) {
01794       continue; /*skip comments */
01795     }
01796     else  {
01797       sscanf(line, "%s %s %s ", s1,s2,s3);
01798       // set each line to setenv();
01799       setenv(s2, s3, 1);
01800     }
01801   }
01802   fclose(fp);
01803 
01804 /***************************!!TBD ck this out later*************************
01805   //setup a sigalrm for threads
01806   //pthread_mutex_init(&mutex, NULL);
01807 
01808   // Set alarm
01809   gettimeofday(&tv0, NULL);
01810   if (td_createalarm(ALRMSEC, shandler, &mutex, &talarm)) { //to shandler in ALRMSEC
01811     pthread_mutex_destroy(&mutex); 
01812     printk("Can't set an alarm signal for %dsec of no data\n", ALRMSEC);
01813   }
01814 *****************************************************************************/
01815 }
01816 
01817 // Module main function. 
01818 int DoIt(void)
01819 {
01820   pid_t pid;
01821   int wflg = 1;
01822   char *args[6];
01823   char callcmd[256];
01824 
01825   if (nice_intro())
01826     return (0);
01827   if(!(username = (char *)getenv("USER"))) username = "nouser"; 
01828   vc = cmdparams_get_str(&cmdparams, "vc", NULL);
01829   tlmdir = cmdparams_get_str(&cmdparams, "indir", NULL);
01830   outdir = cmdparams_get_str(&cmdparams, "outdir", NULL);
01831   logfile = cmdparams_get_str(&cmdparams, "logfile", NULL);
01832   if(strstr(tlmdir, "rexmit")) { rexmitmode = 1; }
01833   if (strcmp(vc, NOTSPECIFIED) == 0) {
01834     fprintf(stderr, "'vc' virt channel must be specified.  Abort\n");
01835     return(1);
01836   }
01837   if (strcmp(tlmdir, NOTSPECIFIED) == 0) {
01838     fprintf(stderr, "'indir' must be specified.  Abort\n");
01839     return(1);
01840   }
01841   if (strcmp(outdir, NOTSPECIFIED) == 0) {
01842     outdir = NULL;
01843   }
01844   if (strcmp(logfile, NOTSPECIFIED) == 0) {
01845     sprintf(logname, H0LOGFILE, username, vc, gettimetag());
01846   }
01847   else {
01848     sprintf(logname, "%s", logfile);
01849   }
01850   if(restartflg) {
01851     //sleep(30);        //make sure old ingest_lev0 is done
01852     if((h0logfp=fopen(logname, "a")) == NULL)
01853       fprintf(stderr, "**Can't open for append the log file %s\n", logname);
01854   }
01855   else {
01856     if((h0logfp=fopen(logname, "w")) == NULL)
01857       fprintf(stderr, "**Can't open the log file %s\n", logname);
01858   }
01859   setup();
01860   while(wflg) {
01861     do_ingest();                // loop to get files from the input dir 
01862 /************************NOOP this out for now********************************
01863     if(sigalrmflg) {        // process an alarm timout for no data in
01864       sigalrmflg = 0;
01865       td_destroyalarm(&talarm, &mutex);
01866       gettimeofday(&tv0, NULL);
01867       if (td_createalarm(ALRMSEC, shandler, &mutex, &talarm)) { 
01868         pthread_mutex_destroy(&mutex);
01869         printk("Can't set an alarm signal for %dsec of no data\n", ALRMSEC);
01870       }
01871       if(!ignoresigalrmflg && firstfound) {
01872         if(Image.initialized) {
01873           if(rs && fsn_prev) {  //make sure have a created record
01874             close_image(rs, segment, segArray, &Image, fsn_prev);
01875             printk("*Closed image on timeout FSN=%u\n", fsn_prev);
01876           }
01877         }
01878         else {
01879           printk("*No image to close on timeout\n");
01880         }
01881         printk("alrm_sig: drms_server_end_transaction()\n");
01882         drms_server_end_transaction(drms_env, 0 , 0); //commit
01883         printk("alrm_sig: drms_server_begin_transaction()\n");
01884         drms_server_begin_transaction(drms_env); //start another cycle
01885         fsn_prev = 0;   //make sure don't try to flush this image
01886         imagecnt = 0;   //and start a new transaction interval
01887       }
01888       ignoresigalrmflg = 0;
01889     }
01890 *****************************************************************************/
01891 
01892     if((stat(stopfile, &stbuf) == 0) || abortflg) {
01893       printk("Abort or Found file: %s. Terminate.\n", stopfile);
01894       if(abortflg) send_mail("Abort for ingest_lev0 for %s\n", pchan);
01895       //now close any open image
01896       if(Image.initialized) {
01897         if(rs) {        //make sure have a created record
01898           close_image(rs, segment, segArray, &Image, fsn_prev);
01899         }
01900       }
01901       else {
01902         printk("*No image to close on exit\n");
01903       }
01904       //No need to commit here. the exit will do it
01905       //printk("restart: drms_server_end_transaction()\n"); 
01906       //drms_server_end_transaction(drms_env, 0 , 0); //commit
01907       //NEW 1Jun2010: leave the stopfile. 
01908       //Must use doingestlev0_HMI(AIA).pl to start ingest_lev0
01909       //sprintf(callcmd, "/bin/rm -f %s", stopfile);
01910       //system(callcmd);
01911       if(rexmitmode)        //new 11/02/21012
01912         sprintf(callcmd, "touch /usr/local/logs/lev0/%s_exitX", pchan);
01913       else
01914         sprintf(callcmd, "touch /usr/local/logs/lev0/%s_exit", pchan);
01915       system(callcmd);      //let the world know we're gone
01916       wflg = 0; //leave DoIt()
01917       continue;
01918     }
01919     sleep(sleep_interval);  //normally 2 sec
01920     if(cntsleeps == 0) {    //file was seen
01921       if(paused) {      //send resume data flow msg
01922         paused = 0;
01923         if(!rexmitmode) {
01924           send_mail("tlm files seen again for ingest_lev0 for %s\n", pchan);
01925         }
01926       }
01927     }
01928     cntsleeps++;        //#of 2sec sleeps w/o any files in do_ingest()
01929     if(cntsleeps > 300) {   // >600sec w/o any files
01930       if(!paused) {
01931         if(!rexmitmode) {
01932           send_mail("No files seen for ingest_lev0 for %s for 600sec\n", pchan);
01933         }
01934         paused = 1;
01935       }
01936     }
01937   }
01938   /*************!!TBD noop this out for now
01939   td_destroyalarm(&talarm);
01940   //pthread_mutex_destroy(&mutex);  //can't destroy here. used in shandler()
01941   pthread_mutex_destroy(&mutex);  //if don't detroy here can end up w/2 ingest_lev0
01942                   //on a restart by doingestlev0.pl ??
01943   ***************************************************/
01944   return(0);
01945 }
01946
proj/lev0/apps/xingest_lev0.c
