base/export/apps/jsoc_export_as_fits.c

#include "jsoc_main.h"
#include "drms_types.h"
#include "drms_storageunit.h"
#include "exputil.h"
#include "fitsexport.h"

#include "defs.h"
REGISTERSTRINGSPREFIX
#include "data/export.defs"
REGISTERSTRINGSSUFFIX

char *module_name = "export";

typedef enum
{
   kMymodErr_Success = 0,
   kMymodErr_CantRegisterDefs,
   kMymodErr_MissingArg,
   kMymodErr_BadRequestID, 
   kMymodErr_BadRecSetQuery,
   kMymodErr_BadFilenameFmt,
   kMymodErr_ExportFailed,
   kMymodErr_CantOpenPackfile,
   kMymodErr_PackfileFailure,
   kMymodErr_UnsupportedPLRecType,
   kMymodErr_DRMS,
   kMymodErr_MissingSegFile
} MymodError_t;

typedef enum
{
   kPL_metadata,
   kPL_content
} PLRecType_t;

#define kNotSpecified    "NOT SPECIFIED"
#define kArg_reqid       "reqid"
#define kArg_version     "expversion"
#define kArg_method      "method"
#define kArg_protocol    "protocol"
#define kArg_rsquery     "rsquery"
#define kArg_n           "n"
#define kArg_expSeries   "expseries"
#define kArg_ffmt        "ffmt"
#define kArg_path        "path"
#define kArg_clname      "kmclass"
#define kArg_kmfile      "kmfile"
#define kArg_cparms      "cparms"


#define kDef_expSeries   "jsoc.export"

#define kPWD             "PWD"

#define kNoCompression   "**NONE**"

#define kMB              (1048576)

/* If rsquery is provided as a cmd-line argument, then jsoc_export does not 
 * save the output data files to an export series.  Instead the caller
 * MUST provide a filename format string (which is a template that 
 * describes how to create the output file names).  The caller may also
 * provide the path argument to specify the directory into which the 
 * output files are to be written.  If the path argument is not specified,
 * then the output files are written to the current working directory.
 * If requery is omitted from the cmd-line, then an export series
 * must either be provided on the cmd-line, or the default series is used.
 * The query that determines what files are to be exported is found
 * in the export series by searching for the record with a matching
 * reqid.
 */
ModuleArgs_t module_args[] =
{
     {ARG_STRING, kArg_version, kNotSpecified, "jsoc export version."},
     {ARG_STRING, kArg_reqid, kNotSpecified, 
        "Export series primary key value that identifies the output record."},
     {ARG_STRING, kArg_method, kNotSpecified, "jsoc export method (eg, url or ftp)."},
     {ARG_STRING, kArg_protocol, kNotSpecified, "file conversion method (eg, convert to fits)."},
     {ARG_STRING, kArg_rsquery, kNotSpecified, 
        "Record-set query that specifies data to be exported."},
     {ARG_STRING, kArg_expSeries, kDef_expSeries, "Series to which exported data are saved."},
     {ARG_STRING, kArg_ffmt, kNotSpecified, "Export filename template."},
     {ARG_STRING, kArg_path, kNotSpecified, "Path to which fits files are output."},
     {ARG_STRING, kArg_clname, kNotSpecified, "Export key map class."},
     {ARG_STRING, kArg_kmfile, kNotSpecified, "Export key map file."},
     {ARG_STRING, kArg_cparms, kNotSpecified, "FITS-stanford compression string used to compress exported image."},
     {ARG_INT, kArg_n, "0", "Record count limit."},
     {ARG_END}
};

char gDefBuf[PATH_MAX] = {0};

static void JEAFPrintLocalTime(FILE *stm, const char *msg)
{
    char tbuf[64];
    time_t sounnecessarilycomplicated;
    struct tm *ltime = NULL;
    
    time(&sounnecessarilycomplicated);
    ltime = localtime(&sounnecessarilycomplicated);
    
    *tbuf = '\0';
    if (ltime)
    {
        snprintf(tbuf, sizeof(tbuf), "%s", asctime(ltime));
        fprintf(stm, "%s - %s\n", tbuf, msg);
    }
}

/* Convert number of bytes to number of MB. If number of bytes < 1MB, then return 1. */
static long long ToMB(long long nbytes)
{
   if (nbytes <= kMB && nbytes > 0)
   {
      return 1;
   }

   return (long long)((nbytes + 0.5 * kMB) / kMB);
}

MymodError_t WritePListRecord(PLRecType_t rectype, FILE *pkfile, const char *f1, const char *f2)
{
   MymodError_t err = kMymodErr_Success;

   switch (rectype)
   {
      case kPL_metadata:
        fprintf(pkfile, "%s=%s\n", f1, f2);
        break;
      case kPL_content:
        fprintf(pkfile, "%s\t%s\n", f1, f2);
        break;
      default:
        fprintf(stderr, "Unsupported packing-list record type '%d'.\n", (int)rectype);
        err = kMymodErr_UnsupportedPLRecType;
   }

   return err;
}

/* Assumes tcount is zero on the first call.  This function adds 
 * the number of files exported to tcount on each call. */
static unsigned long long MapexportRecordToDir(DRMS_Record_t *recin,
                                               const char *ffmt,
                                               const char *outpath,
                                               FILE *pklist,
                                               const char *classname, 
                                               const char *mapfile,
                                               int *tcount, 
                                               const char **cparms,
                                               MymodError_t *status,
                                               char **errmsg)
{
   int drmsstat = DRMS_SUCCESS;
   MymodError_t modstat = kMymodErr_Success;
   DRMS_Segment_t *segin = NULL;
   DRMS_Segment_t *tgtseg = NULL; /* If segin is a linked segment, then tgtset is the segment in the target series. */
   unsigned long long tsize = 0;
    unsigned long long expsize = 0;
    char *actualfname = NULL;
   char dir[DRMS_MAXPATHLEN];
   char fmtname[DRMS_MAXPATHLEN];
   char fullfname[DRMS_MAXPATHLEN];
   char query[DRMS_MAXQUERYLEN];
   HIterator_t *last = NULL;
   int iseg;
   int lastcparms;
   int count;
    char buf[256];
   ExpUtlStat_t expfn = kExpUtlStat_Success;

   //    JEAFPrintLocalTime(stdout, "Calling drms_record_directory() from MapexportRecordToDir().");
   drms_record_directory(recin, dir, 1); /* This fetches the input data from SUMS. */
   //    JEAFPrintLocalTime(stdout, "Done calling drms_record_directory() from MapexportRecordToDir().");

   /* Must create query from series name and prime keyword values */
   drms_sprint_rec_query(query, recin);

   /* The input rs query can specify a subset of all the series' segments - 
    * this is encapsulated in recin. */

   iseg = 0;
   lastcparms = 0;
   count = 0;

   while ((segin = drms_record_nextseg(recin, &last, 0)) != NULL)
   {
      if (segin->info->islink)
      {
         if ((tgtseg = drms_segment_lookup(recin, segin->info->name)) == NULL)
         {
            fprintf(stderr, "Unable to locate target segment %s.\n", segin->info->name);
            iseg++;
            continue;
         }
      }
      else
      {
         tgtseg = segin;
      }

      if ((expfn = exputl_mk_expfilename(segin, tgtseg, ffmt, fmtname)) == kExpUtlStat_Success)
      {
         snprintf(fullfname, sizeof(fullfname), "%s/%s", outpath, fmtname);
      }
      else
      {
         if (expfn == kExpUtlStat_InvalidFmt)
         {
            fprintf(stderr, "Invalid file-name format template '%s'.\n", ffmt);
         }
         else if (expfn == kExpUtlStat_UnknownKey)
         {
            fprintf(stderr, "One or more keywords in the file-name format template '%s' do not exist in series '%s'.\n", ffmt, recin->seriesinfo->seriesname);
         }

         modstat = kMymodErr_BadFilenameFmt;
         break;
      }

      if (!cparms || !cparms[iseg])
      {
         lastcparms = 1;
      }

      /* Must pass source segment if the segment is a linked segment. */
       // snprintf(buf, sizeof(buf), "Calling fitsexport_mapexport_tofile() from MapexportRecordToDir() on file %s", fullfname);
       //       JEAFPrintLocalTime(stdout, buf);
      drmsstat = fitsexport_mapexport_tofile(segin, 
                                             !lastcparms ? cparms[iseg] : NULL, 
                                             classname, 
                                             mapfile, 
                                             fullfname, 
                                             &actualfname,
                                             &expsize);
      //       JEAFPrintLocalTime(stdout, "Done calling fitsexport_mapexport_tofile() from MapexportRecordToDir().");
      if (drmsstat == DRMS_ERROR_INVALIDFILE)
      {
         /* No input segment file. */
         fprintf(stderr, "Requested input file %s is missing or invalid.\n", segin->filename);
      }
      else if (drmsstat != DRMS_SUCCESS)
      {
         /* There was an input segment file, but for some reason the export failed. */
         modstat = kMymodErr_ExportFailed;
         fprintf(stderr, "Failure exporting segment '%s'.\n", segin->info->name);

         if (errmsg)
         {
            if (drmsstat == DRMS_ERROR_CANTCOMPRESSFLOAT)
            {
               *errmsg = strdup("Cannot export Rice-compressed floating-point images.\n");
            }
         }
         break;
      }
      else
      {
         count++;
         tsize += expsize;
         WritePListRecord(kPL_content, pklist, query, actualfname);
      }

      iseg++;
   }

   /* If NO file exported, this is an error. But if there is more than one segment, it is okay
    * for some input segment files to be missing. */
   if (count == 0)
   {
      modstat = kMymodErr_ExportFailed;
   }

   if (tcount)
   {
      *tcount = count;
   }

   if (last)
   {
      hiter_destroy(&last);
   }

    if (actualfname)
    {
        free(actualfname);
    }
    
   if (status)
   {
      *status = modstat;
   }

   return tsize;
}

static unsigned long long MapexportToDir(DRMS_Env_t *env, 
                                         const char *rsinquery, 
                                         const char *ffmt,
                                         const char *outpath, 
                                         FILE *pklist, 
                                         const char *classname, 
                                         const char *mapfile,
                                         int *tcount,
                                         TIME *exptime,
                                         const char **cparms, 
                                         MymodError_t *status)
{
   int stat = DRMS_SUCCESS;
   MymodError_t modstat = kMymodErr_Success;
   DRMS_RecordSet_t *rsin = NULL;
   DRMS_Record_t *recin = NULL;
   int iRec = 0;
   int nSets = 0;
   int iSet = 0;
   int nRecs = 0;
   int RecordLimit = *tcount;
   unsigned long long tsize = 0;
   int errorCount = 0;
   int okayCount = 0;
   int count;
   int itcount;


   itcount = 0;
   //    JEAFPrintLocalTime(stdout, "Calling drms_open_records().");
    
   if (RecordLimit == 0)
     //     rsin = drms_open_recordset(env, rsinquery, &stat);
     // temporarily reverting to drms_open_records until I can fix the problem with
     // not passing a segment-list ot drms_open_recordset().
     rsin = drms_open_records(env, rsinquery, &stat);
   else
     rsin = drms_open_nrecords(env, rsinquery, RecordLimit, &stat);
   //        JEAFPrintLocalTime(stdout, "Done calling drms_open_records().");

   if (rsin)
   {
      /* stage records to reduce number of calls to SUMS. */
      drms_stage_records(rsin, 1, 0);
      nSets = rsin->ss_n;

      for (iSet = 0; iSet < nSets; iSet++)
      {
         nRecs = drms_recordset_getssnrecs(rsin, iSet, &stat);

         if (stat != DRMS_SUCCESS)
         {
            fprintf(stderr, "Failure calling drms_recordset_getssnrecs(), skipping subset '%d'.\n", iSet);
         }
         else
         {
            for (iRec = 0; iRec < nRecs; iRec++)
            {
               recin = drms_recordset_fetchnext(env, rsin, &stat, NULL, NULL);

               if (!recin)
               {
                  /* Exit rec loop - last record was fetched last time. */
                  break;
               }

               count = 0;
               tsize += MapexportRecordToDir(recin,
                                             ffmt,
                                             outpath,
                                             pklist, 
                                             classname, 
                                             mapfile, 
                                             &count, 
                                             cparms, 
                                             &modstat,
                                             NULL);
               if (modstat == kMymodErr_Success)
               {
                  okayCount++;
                  itcount += count;
               }
               else
               {
                  errorCount++;
               }
            }
         }
      }

      modstat = kMymodErr_Success; /* Could have been set to BAD in loop above, but okayCount and errorCount
                                    * account for fatal errors in the loop, not modstat. */

      if (errorCount > 0)
      {
         fprintf(stderr,"Export failed for %d segments of %d attempted.\n", errorCount, errorCount + okayCount);
      }

      if (exptime)
      {
         *exptime = CURRENT_SYSTEM_TIME;
      }
      if (tcount)
      {
         *tcount = itcount;
      }
   }
   else
   {
      fprintf(stderr, "Record-set query '%s' is not valid.\n", rsinquery);
      modstat = kMymodErr_BadRecSetQuery; 
   }

   if (rsin)
   {
      drms_close_records(rsin, DRMS_FREE_RECORD);
   }

   if (status)
   {
      *status = modstat;
   }
   
   return tsize;
}

static MymodError_t CallExportToFile(DRMS_Segment_t *segout, 
                                     DRMS_Segment_t *segin, 
                                     DRMS_Segment_t *tgtseg, 
                                     const char *clname,
                                     const char *mapfile,
                                     const char *ffmt,
                                     unsigned long long *szout,
                                     char *filewritten,
                                     const char *cparms,
                                     char **errmsg)
{
   int status = DRMS_SUCCESS;
   MymodError_t err = kMymodErr_Success;
   char fileout[DRMS_MAXPATHLEN];
   char filein[DRMS_MAXPATHLEN];
   char basename[DRMS_MAXPATHLEN];
   unsigned long long size = 0;
    unsigned long long expsize = 0;
    char *actualfname = NULL;
   struct stat filestat;

   if (segout)
   {
      if (segin->info->islink)
      {
         drms_segment_filename(tgtseg, filein);
      }
      else
      {
         drms_segment_filename(segin, filein); /* input seg file name */
      }

      if (!stat(filein, &filestat))
      {
         ExpUtlStat_t expfn = kExpUtlStat_Success;
     size = filestat.st_size;

         if ((expfn = exputl_mk_expfilename(segin, tgtseg, ffmt, basename)) == kExpUtlStat_Success)
         {
            CHECKSNPRINTF(snprintf(segout->filename, DRMS_MAXSEGFILENAME, "%s", basename), DRMS_MAXSEGFILENAME);
            drms_segment_filename(segout, fileout);

            status = fitsexport_mapexport_tofile(segin, cparms, clname, mapfile, fileout, &actualfname, &expsize);
            if (status == DRMS_ERROR_INVALIDFILE)
            {
               /* No input file for segment - not necessarily an error. */
               err = kMymodErr_MissingSegFile;
            }
            else if (status != DRMS_SUCCESS)
            {
               err = kMymodErr_ExportFailed;
               fprintf(stderr, "Failed to export segment '%s' to '%s'.\n", segin->info->name, fileout);
               if (errmsg)
               {
                  if (status == DRMS_ERROR_CANTCOMPRESSFLOAT)
                  {
                     *errmsg = strdup("Cannot export Rice-compressed floating-point images.\n");
                  }
               }
            }
         }
         else
         {
            if (expfn == kExpUtlStat_InvalidFmt)
            {
               fprintf(stderr, "Invalid file-name format template '%s'.\n", ffmt);
            }
            else if (expfn == kExpUtlStat_UnknownKey)
            {
               fprintf(stderr, "One or more keywords in the file-name format template '%s' do not exist in series '%s'.\n", ffmt, segin->record->seriesinfo->seriesname);
            }

            err = kMymodErr_BadFilenameFmt;
         }
      }
      else
      {
     fprintf(stderr, "Unable to open source file '%s'.\n", filein);
         err = kMymodErr_ExportFailed;
      }
   }

   *szout = 0;
   if (err == kMymodErr_Success)
   {
      *szout = size;
      snprintf(filewritten, DRMS_MAXPATHLEN, "%s", basename);
   }
    
    if (actualfname)
    {
        free(actualfname);
    }

   return err;
}

/* recout is the export series' record to which the exported data will be saved. */
static int MapexportRecord(DRMS_Record_t *recout,
                           DRMS_Record_t *recin,
                           const char *classname, 
                           const char *mapfile,
                           const char *pklfilename, 
                           int *tcount,
                           const char *ffmt, 
                           char **outpath,
                           FILE **pklist,
                           const char **cparms, 
                           MymodError_t *status)
{
   MymodError_t err = kMymodErr_Success;
   HIterator_t *last = NULL;
   DRMS_Segment_t *segout = NULL;
   DRMS_Segment_t *segin = NULL;
   DRMS_Segment_t *tgtseg = NULL;
   unsigned long long size = 0;
   unsigned long long tsize = 0;
   char dir[DRMS_MAXPATHLEN];
   int iseg;
   int lastcparms;
   int gotone;

   segout = drms_segment_lookupnum(recout, 0);

   drms_record_directory(recin, dir, 1); /* This fetches the input data from SUMS. */

   if (segout)
   {
      char glom[128];
      snprintf(glom, sizeof(glom), "%%s/%s", DRMS_SLOTDIR_FORMAT);
      char path[DRMS_MAXPATHLEN];
      snprintf(path, sizeof(path), glom, segout->record->su->sudir, segout->record->slotnum);

      if (outpath)
      {
         *outpath = strdup(path);
      }

      if (pklist)
      {
         char pkpath[DRMS_MAXPATHLEN];
         snprintf(pkpath, sizeof(pkpath), "%s/%s", path, pklfilename);
         *pklist = fopen(pkpath, "w+");
      }

      char fname[DRMS_MAXPATHLEN];
      char query[DRMS_MAXQUERYLEN];

      /* Must create query from series name and prime keyword values */
      char buf[1028];
      int nkeys = recin->seriesinfo->dbidx_num;
      int snext = 0;
      int ikey = 0;

      snprintf(query, sizeof(query), "%s", recin->seriesinfo->seriesname);
      snext = strlen(query);

      for (ikey = 0; ikey < nkeys; ikey++)
      {
         drms_keyword_snprintfval(recin->seriesinfo->dbidx_keywords[ikey], buf, sizeof(buf));
         snprintf(query + snext, sizeof(query) - snext, "[%s]", buf);
         snext += strlen(buf) + 2;
      }

      /* The input rs query can specify a subset of all the series' segemnts - 
       * this is encapsulated in recin. */
      iseg = 0;
      lastcparms = 0;
      gotone = 0;

      while ((segin = drms_record_nextseg(recin, &last, 0)) != NULL)
      {
         if (segin->info->islink)
         {
            if ((tgtseg = drms_segment_lookup(recin, segin->info->name)) == NULL)
            {
               fprintf(stderr, "Unable to locate target segment %s.\n", segin->info->name);
               iseg++;
               continue;
            }
         }
         else
         {
            tgtseg = segin;
         }

         size = 0;

         if (!cparms || !cparms[iseg])
         {
            lastcparms = 1;
         }

         err = CallExportToFile(segout, 
                                segin, 
                                tgtseg, 
                                classname, 
                                mapfile, 
                                ffmt, 
                                &size, 
                                fname, 
                                !lastcparms ? cparms[iseg] : NULL,
                                NULL);

         if (err == kMymodErr_MissingSegFile)
         {
            /* No input file for this segment */
         }
         else if (err != kMymodErr_Success)
     {
        fprintf(stderr, "Failure exporting segment '%s'.\n", segin->info->name);
        break;
     }
     else
     {
            gotone = 1;

            if (tcount)
            {
               ++*tcount;
            }

            tsize += size;
            if (pklist && *pklist)
            {
               WritePListRecord(kPL_content, *pklist, query, fname);
            }
     }

         iseg++;
      }

      /* If NO file exported, this is an error. But if there is more than one segments, it is okay
       * for some input segment files to be missing. */
      if (!gotone)
      {
         err = kMymodErr_ExportFailed;
         fprintf(stderr, "Failure exporting record number %lld.\n", recin->recnum);
      }

      if (last)
      {
         hiter_destroy(&last);
      }
   }
   else
   {
      fprintf(stderr, "Export series contains no segment!\n");
      err = kMymodErr_ExportFailed;
   }

   if (status)
   {
      *status = err;
   }

   return tsize;
}       

/* recout is the export series' record to which the exported data will be saved. 
 * reqid is the primary key in the export series 
 */
static int Mapexport(DRMS_Env_t *env,
                     const char *reqid,
                     const char *classname, 
                     const char *mapfile,
                     const char *expseries,
                     const char *pklfilename, 
                     int *tcount, 
                     char **outpath, 
                     TIME *exptime, 
                     FILE **pklist, 
                     const char **cparms,
                     MymodError_t *status)
{
   int stat;
   MymodError_t err = kMymodErr_Success;
   int nSets = 0;
   int iSet = 0;
   int nRecs = 0;
   int iRec = 0;
   unsigned long long tsize = 0;
   DRMS_Record_t *recout = NULL;
   DRMS_Record_t *recin = NULL;
   DRMS_RecordSet_t *rs = NULL;
   DRMS_RecordSet_t *rsout = NULL;
   DRMS_RecordSet_t *rsin = NULL;
   char rsoutquery[DRMS_MAXQUERYLEN];
   char *rsinquery = NULL;

   snprintf(rsoutquery, sizeof(rsoutquery), "%s[%s]", expseries, reqid);

   rs = drms_open_records(env, rsoutquery, &stat);

   if (rs)
   {
      /* XXX Change to DRMS_REPLACE_SEGMENTS (which isn't implemented yet). */
      rsout = drms_clone_records(rs, DRMS_PERMANENT, DRMS_COPY_SEGMENTS, &stat);
      drms_close_records(rs, DRMS_FREE_RECORD);
   }
   
   if (rsout && rsout->n == 1)
   {
      /* Filename Format comes from export series */
      char *ffmt = NULL;
      char *kval = NULL;

      recout = rsout->records[0];
      rsinquery = drms_getkey_string(recout, drms_defs_getval("kExportKW_Request"), &stat);
      
      kval = drms_getkey_string(recout, drms_defs_getval("kExportKW_FileNameFormat"), &stat);
      if (kval)
      {
         if (*kval == '\0')
         {
            ffmt = NULL;
         }
         else
         {
            ffmt = strdup(kval);
         }

         free(kval);
      }

      //      if (rsinquery && (rsin = drms_open_recordset(env, rsinquery, &stat)))
      // temporarily reverting to drms_open_records until I can fix the problem with
      // not passing a segment-list ot drms_open_recordset().
      if (rsinquery && (rsin = drms_open_records(env, rsinquery, &stat)))
      {
         /* stage records to reduce number of calls to SUMS. */
         drms_stage_records(rsin, 1, 0);
     nSets = rsin->ss_n;

     for (iSet = 0; 
              stat == DRMS_SUCCESS && err == kMymodErr_Success && iSet < nSets; 
              iSet++)
     {
        /* Perhaps we will need this in the future?
         * request = rsin->ss_queries[iSet];
         */
        nRecs = drms_recordset_getssnrecs(rsin, iSet, &stat);

        for (iRec = 0; 
                 stat == DRMS_SUCCESS && err == kMymodErr_Success &&iRec < nRecs; 
                 iRec++)
        {
           recin = drms_recordset_fetchnext(env, rsin, &stat, NULL, NULL);

               if (!recin)
               {
                  /* Exit rec loop - last record was fetched last time. */
                  break;
               }
               
           tsize += MapexportRecord(recout, 
                                        recin,
                                        classname, 
                                        mapfile, 
                                        pklfilename,
                                        tcount, 
                                        ffmt, 
                                        outpath, 
                                        pklist, 
                                        cparms, 
                                        &err);
        }
     }
     
     if (stat != DRMS_SUCCESS || err != kMymodErr_Success)
     {
        fprintf(stderr, "Export halted due to DRMS failure.\n");
     }
         else if (exptime)
         {
            *exptime = CURRENT_SYSTEM_TIME;
         }
      }
      else
      {
     fprintf(stderr, 
         "Export series keyword '%s' did not contain a valid recordset query.\n", 
                 drms_defs_getval("kExportKW_Request"));
         err = kMymodErr_BadRecSetQuery; 
      }

      if (ffmt)
      {
         free(ffmt);
      }
   }
   else
   {
      fprintf(stderr, "Could not open export destination record set with request id '%s'.\n", reqid);
      err = kMymodErr_BadRequestID; 
   }

   /* Set output record keywords. */
   if (err == kMymodErr_Success)
   {
      drms_setkey_time(recout, drms_defs_getval("kExportKW_ExpTime"), CURRENT_SYSTEM_TIME);
      drms_setkey_int(recout, drms_defs_getval("kExportKW_DataSize"), (int)ToMB(tsize));
   }

   if (rsout)
   {
      drms_close_records(rsout, DRMS_INSERT_RECORD);
   }

   if (rsinquery)
   {
      free(rsinquery);
   }

   if (status)
   {
      *status = err;
   }
   
   return tsize;
}

static char *GenErrMsg(const char *fmt, ...)
{
   char *msgout = NULL;
   char errmsg[4096];

   va_list ap;
   va_start(ap, fmt);
   vsnprintf(errmsg, sizeof(errmsg), fmt, ap);

   msgout = strdup(errmsg);
   fprintf(stderr, errmsg);
   
   va_end (ap);

   return msgout;
}

static MymodError_t AppendContent(FILE *dst, FILE *src)
{
   MymodError_t err = kMymodErr_Success;
   char buf[32768];
   int nread = 0;
   int dstfd;
   int srcfd;

   if (dst && src)
   {
      dstfd = fileno(dst);
      srcfd = fileno(src);

      while(1)
      {
         nread = read(srcfd, buf, sizeof(buf));
         if (nread > 0)
         {
            if (write(dstfd, buf, nread) == -1)
            {
               err = kMymodErr_PackfileFailure;
               fprintf(stderr, "Failure writing packing-list file.\n");
               break;
            }
         }
         else
         {
             if (nread == -1)
             {
                err = kMymodErr_PackfileFailure;
                fprintf(stderr, "Failure reading packing-list file.\n");
             }

             break;
         }
      }
   }

   return err;
}

//#undef DEFS_MKPATH
//#define FRED(X, Y) X##Y
//#define DEFS_MKPATH(Y) FRED(CDIR, Y)
//#define DEFS_MKPATH(Y) CDIR##Y

int DoIt(void) 
{
   MymodError_t err = kMymodErr_Success;
   int drmsstat = DRMS_SUCCESS;
   long long tsize = 0; /* total size of export payload in bytes */
   long long tsizeMB = 0; /* total size of export payload in Mbytes */
   int tcount = 0;
   TIME exptime = DRMS_MISSING_TIME;
   FILE *pklist = NULL;
   FILE *pklistTMP = NULL;
   char pklistfname[PATH_MAX];
   char pklistfnameTMP[PATH_MAX];
   char pklistpath[PATH_MAX];
   char pklistpathTMP[PATH_MAX];

   const char *version = NULL;
   const char *reqid = NULL;
   const char *method = NULL;
   const char *protocol = NULL;
   const char *rsquery = NULL;
   const char *clname = NULL;
   const char *mapfile = NULL;
   const char *cparmsarg = NULL;
   const char **cparms = NULL;
   int RecordLimit = 0;

   /* "packing list" header/metadata */
   char *md_version = NULL;
   char *md_reqid = NULL;
   char *md_method = NULL;
   char *md_protocol = NULL;
   char *md_count = NULL; /* number of files exported */
   char *md_size = NULL; /* total Mbytes exported */
   char *md_exptime = NULL;
   char *md_dir = NULL; /* same as outpath, the /SUMX path that contains the exported files
                         * before they are downloaded by the user */
   char *md_status = NULL; 
   char *md_error = NULL;

   RecordLimit = cmdparams_get_int(&cmdparams, kArg_n, &drmsstat);

   /* This will add strings to the global string container. */
   defs_init();

   snprintf(pklistfname, sizeof(pklistfname), "%s", drms_defs_getval("kPackListFileName"));
   snprintf(pklistfnameTMP, sizeof(pklistfnameTMP), "%s.tmp", pklistfname);

   version = cmdparams_get_str(&cmdparams, kArg_version, &drmsstat);
   reqid = cmdparams_get_str(&cmdparams, kArg_reqid, &drmsstat);
   method = cmdparams_get_str(&cmdparams, kArg_method, &drmsstat);
   protocol = cmdparams_get_str(&cmdparams, kArg_protocol, &drmsstat);

   rsquery = cmdparams_get_str(&cmdparams, kArg_rsquery, &drmsstat);


   clname = cmdparams_get_str(&cmdparams, kArg_clname, &drmsstat);
   if (drmsstat != DRMS_SUCCESS || !strcmp(clname, kNotSpecified))
   {
      clname = NULL;
   }

   mapfile = cmdparams_get_str(&cmdparams, kArg_kmfile, &drmsstat);
   if (drmsstat != DRMS_SUCCESS || !strcmp(mapfile, kNotSpecified))
   {
      mapfile = NULL;
   }

   cparmsarg = cmdparams_get_str(&cmdparams, kArg_cparms, &drmsstat);
   if (strcmp(cparmsarg, kNotSpecified))
   {
      char *dup = strdup(cparmsarg);
      char *pc = NULL;
      char *pend = NULL;
      int nstr;
      int istr;
         
      /* count number of compression strings (one for each segment being exported) */
      pc = dup;
      nstr = 1;
      while ((pc = strchr(pc, ',')) != NULL)
      {
         pc++;
         ++nstr;
      }

      cparms = (const char **)malloc((nstr + 1) * sizeof(char *));
         
      pc = dup;
      for (istr = 0; istr < nstr; istr++)
      {
         pend = strchr(pc, ',');
         if (pend)
         {
            *pend = '\0';
         }
         cparms[istr] = (strcmp(pc, kNoCompression) == 0) ? strdup("") : strdup(pc);
         if (pend)
         {
            pc = pend + 1;
         }
      }

      /* Empty string to indicate end */
      cparms[nstr] = NULL;

      if (dup)
      {
         free(dup);
      }
   }
 
   md_version = strdup(version);   /* Could be "NOT SPECIFIED". */
   md_reqid = strdup(reqid);       /* Could be "NOT SPECIFIED". */
   md_method = strdup(method);     /* Could be "NOT SPECIFIED". */
   md_protocol = strdup(protocol); /* Could be "NOT SPECIFIED". */

   if (strcmp(rsquery, kNotSpecified) == 0)
   {
       /* The record-set query that identifies records to export comes from 
        * the export series, not from the command-line. A dbase query for the
        * reqid in the export series returns a record which contains the
        * record-set query. This export-series record is cloned, and the 
        * output files are written to the clone's SUDIR. 
        *
        * The output filename format is obtained from the export series record */
       char *outpath = NULL;
       
       /* No record-set query provided - must use series to get rsquery */
       const char *expseries = NULL;
       
       if (strcmp(reqid, kNotSpecified) == 0)
       {
           /* No rsquery, and no reqid - cannot find an rsquery, error.*/
           md_error = GenErrMsg("Invalid arguments - missing reqid.\n");
           err = kMymodErr_MissingArg;
       }
       else
       {
           /* reqid is defined in the block. */
           expseries = cmdparams_get_str(&cmdparams, kArg_expSeries, &drmsstat);
           
           /* Open packing-list file */         
           tsize = Mapexport(drms_env, 
                             reqid, 
                             clname, 
                             mapfile, 
                             expseries, 
                             pklistfnameTMP, 
                             &tcount, 
                             &outpath, 
                             &exptime, 
                             &pklistTMP,
                             cparms, 
                             &err);
           
           if (err != kMymodErr_Success)
           {
               md_error = GenErrMsg("Failure occurred while processing export Request ID '%s'.\n", reqid);
               err = kMymodErr_ExportFailed;
           }
           else
           {
               snprintf(pklistpathTMP, sizeof(pklistpathTMP), "%s/%s", outpath, pklistfnameTMP);
               
               /* Set packing list info */
               md_dir = strdup(outpath);
           }
           
           if (outpath)
           {
               free(outpath);
           }
       }
   }
   else
   {
       /* The export system uses this branch of code - the record-set specification comes 
        * from the cmd-line, not from the export DRMS record. */
       
       /* Packing list items that come from the cmd-line arguments. */
       const char *outpath = NULL;
       
       /* Filename Format comes from cmd-line argument */
       const char *ffmt = NULL;
       
       outpath = cmdparams_get_str(&cmdparams, kArg_path, &drmsstat);
       if (strcmp(outpath, kNotSpecified) == 0)
       {
           /* Use current working directory by default */
           outpath = getenv(kPWD);
       }
       
       ffmt = cmdparams_get_str(&cmdparams, kArg_ffmt, &drmsstat);
       if (strcmp(ffmt, kNotSpecified) == 0 || *ffmt == '\0')
       {
           /* Assume the user wants the default filename format - set to NULL */
           ffmt = NULL;
       }
       
       /* Open tmp packing-list file */
       snprintf(pklistpathTMP, sizeof(pklistpathTMP), "%s/%s", outpath, pklistfnameTMP); 
       
       pklistTMP = fopen(pklistpathTMP, "w+");
       if (pklistTMP)
       {
           /* Call export code, filling in tsize, tcount, and exptime */
           tcount = RecordLimit;
           tsize = MapexportToDir(drms_env, 
                                  rsquery, 
                                  ffmt, 
                                  outpath, 
                                  pklistTMP, 
                                  clname, 
                                  mapfile, 
                                  &tcount, 
                                  &exptime, 
                                  cparms, 
                                  &err);
           
       }
       else
       {
           err = kMymodErr_CantOpenPackfile;
           md_error = GenErrMsg("Couldn't open temporary packing-list file '%s'.\n",  pklistpathTMP);
       }
       
       if (err != kMymodErr_Success)
       {
           md_error = GenErrMsg("Failure occurred while processing export Request ID '%s'.\n", reqid);
           err = kMymodErr_ExportFailed;
       }
       else
       {
           /* Set packing list info */
           md_dir = strdup(outpath);
       }
   }

   tsizeMB = ToMB(tsize);

   if (err == kMymodErr_Success)
   {
      char tstr[64];
      int strsize = 0;
      sprint_time(tstr, exptime, "UT", 0);

      /* Set packing list info not already set */
      strsize = 64;
      md_size = malloc(strsize);
      snprintf(md_size, strsize, "%lld", tsizeMB);
      md_count = malloc(strsize);
      snprintf(md_count, strsize, "%d", tcount);
      md_exptime = strdup(tstr);
   }

   fprintf(stdout, "%lld megabytes exported.\n", tsizeMB); 
   
    /* open 'real' pack list */
    if (strcmp(reqid, kNotSpecified) != 0)
    {
        if (md_dir)
        {
            snprintf(pklistpath, sizeof(pklistpath), "%s/%s", md_dir, pklistfname);
            pklist = fopen(pklistpath, "w+");
            
            if (pklist)
            {
                if (fseek(pklistTMP, 0, SEEK_SET))
                {
                    md_error = GenErrMsg("Failure accessing packing-list file '%s'.\n", pklistfnameTMP);
                    err = kMymodErr_PackfileFailure;
                }
            }
            else
            {
                md_error = GenErrMsg("Failure opening packing-list file '%s'.\n", pklistfname);
                err = kMymodErr_PackfileFailure;
            }
        }
        
        /* For now, there is just success/failure in the packing-list file */
        if (err == kMymodErr_Success)
        {
            md_status = strdup(drms_defs_getval("kMDStatus_Good"));
        }
        else
        {
            md_status = strdup(drms_defs_getval("kMDStatus_Bad"));
        }
        
        if (pklist)
        {
            char procSteps[PATH_MAX];
            FILE *fpProcSteps = NULL;
            struct stat stBuf;
            char *lineBuf = NULL;
            size_t lineLen = 0;
            ssize_t nRead = 0;
            
            /* write packing list */
            fprintf(pklist, "# JSOC \n");
            WritePListRecord(kPL_metadata, pklist, drms_defs_getval("kMD_Version"), md_version);
            WritePListRecord(kPL_metadata, pklist, drms_defs_getval("kMD_RequestID"), md_reqid);
            WritePListRecord(kPL_metadata, pklist, drms_defs_getval("kMD_Method"), md_method);
            WritePListRecord(kPL_metadata, pklist, drms_defs_getval("kMD_Protocol"), md_protocol);
            WritePListRecord(kPL_metadata, pklist, drms_defs_getval("kMD_Count"), md_count ? md_count : "-1");
            WritePListRecord(kPL_metadata, pklist, drms_defs_getval("kMD_Size"), md_size ? md_size : "-1");
            WritePListRecord(kPL_metadata, pklist, drms_defs_getval("kMD_ExpTime"), md_exptime ? md_exptime : "");
            WritePListRecord(kPL_metadata, pklist, drms_defs_getval("kMD_Dir"), md_dir ? md_dir : "");
            WritePListRecord(kPL_metadata, pklist, drms_defs_getval("kMD_Status"), md_status);
            
            /* If the proc-steps.txt file exists, put the contents in the packing list. */
            snprintf(procSteps, sizeof(procSteps), "%s/proc-steps.txt", md_dir);
            if (stat(procSteps, &stBuf) == 0)
            {
                fprintf(pklist, "# PROCESSING \n");
                
                fpProcSteps = fopen(procSteps, "r");
                if (fpProcSteps)
                {
                    while ((nRead = getline(&lineBuf, &lineLen, fpProcSteps)) != -1)
                    {
                        /* Put these in comments, otherwise jsoc_export_make_index will have issues. */
                        fprintf(pklist, lineBuf);
                    }
                    
                    if (lineBuf)
                    {
                        free(lineBuf);
                    }
                }
            }
            
            fflush(pklist);
            
            if (err == kMymodErr_Success)
            {
                /* Copy the content from the temporary pklist into the real pklist */
                fprintf(pklist, "# DATA \n");
                fflush(pklist);
                if (tsizeMB > 0)
                {
                    err = AppendContent(pklist, pklistTMP);
                }
                else
                {
                    WritePListRecord(kPL_metadata, pklist, drms_defs_getval("kMD_Warning"), "No FITS files were exported. The requested FITS files no longer exist.");  
                }
            }
            
            if (err != kMymodErr_Success)
            {
                fflush(pklist);
                
                /* If there is an error, write out the error message. */
                WritePListRecord(kPL_metadata, pklist, drms_defs_getval("kMD_Error"), md_error ? md_error : "");
            }
        }
    }

   if (pklistTMP)
   {
      /* close tmp packing-list file */
      fclose(pklistTMP);
      pklistTMP = NULL;

      /* delete temporary file */
      unlink(pklistpathTMP);
   }

   if (pklist)
   {
      /* close actual packing-list file */
      fclose(pklist);
      pklist = NULL;
   }

   if (md_version)
   {
      free(md_version);
   }
   if (md_reqid)
   {
      free(md_reqid);
   }
   if (md_method)
   {
      free(md_method);
   }
   if (md_protocol)
   {
      free(md_protocol);
   }
   if (md_count)
   {
      free(md_count);
   }
   if (md_size)
   {
      free(md_size);
   }
   if (md_exptime)
   {
      free(md_exptime);
   }
   if (md_dir)
   {
      free(md_dir);
   }
   if (md_status)
   {
      free(md_status);
   }
   if (md_error)
   {
      free(md_error);
   }

   if (cparms)
   {
      int iseg = 0;
      while (cparms[iseg])
      {
         free((void *)cparms[iseg]);
         iseg++;
      }

      free(cparms);
   }
   
   return err;
}

---