JSOC_Documentation: base/export/apps/jsoc_export_make

00001 // #define DEBUG 1
00002 #define DEBUG 0
00003 
00004 /*
00005  *  jsoc_export_make_index - Generates index.XXX files for dataset export.
00006  *  Should be run in the directory containing a jsoc export index.txt file.
00007  *  Will read the index.txt and generate index.json and index.html
00008  *
00009 */
00010 
00011 #include <stdio.h>
00012 #include "json.h"
00013 #include <sys/types.h>
00014 #include <sys/stat.h>
00015 #include <unistd.h>
00016 #include <strings.h>
00017 #include <string.h>
00018 #include <ctype.h>
00019 #include <stdlib.h>
00020 
00021 #define kPROTOCOL_as_is 0
00022 #define kPROTOCOL_fits 1
00023 
00024 // #include "json.h"
00025 
00026 static char x2c (char *what) {
00027   register char digit;
00028 
00029   digit = (what[0] >= 'A' ? ((what[0] & 0xdf) - 'A')+10 : (what[0] - '0'));
00030   digit *= 16;
00031   digit += (what[1] >= 'A' ? ((what[1] & 0xdf) - 'A')+10 : (what[1] - '0'));
00032   return (digit);
00033 }
00034 
00035 static void CGI_unescape_url (char *url) {
00036   register int x, y;
00037 
00038   for (x = 0, y = 0; url[y]; ++x, ++y) {
00039     if ((url[x] = url[y]) == '%') {
00040       url[x] = x2c (&url[y+1]);
00041       y += 2;
00042     }
00043   }
00044   url[x] = '\0';
00045 }
00046 
00047 char * string_to_json(char *in)
00048   {
00049   char *new;
00050   new = json_escape(in);
00051   return(new);
00052   }
00053 
00054 #define HTML_INTRO "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\" \"http://www.w3.org/TR/html4/loose.dtd\">" \
00055         "<html><head title=\"JSOC Data Export Request Results Index\">\n"  \
00056         "</head><body>\n"  \
00057         "<h2>JSOC Data Request Summary</h2>\n"  \
00058         "<table>\n"
00059 
00060 #define HTTP_SERVER "http://jsoc.stanford.edu"
00061 #define FTP_SERVER  "ftp://pail.stanford.edu/export"
00062         
00063 
00064 /* Module main function. */
00065 int main(int argc, char **argv)
00066   {
00067   FILE *index_txt, *index_html, *index_json;
00068   char buf[1000];
00069   char dir[1000];
00070   char tarfile[1000]; *tarfile = '\0';
00071   char requestid[1000]; *requestid = '\0';
00072   json_t *jroot;
00073   json_t *recinfo;
00074   json_t *fileinfo;
00075   char *json, *final_json;
00076   int state = 0;
00077   int method_ftp = 0;
00078   int method_tar = 0;
00079   int protocol = kPROTOCOL_as_is;
00080   tarfile[0] = '\0';
00081   requestid[0] = '\0';
00082 
00083   if (argc && strcmp(*argv, "-h") == 0)
00084     {
00085     printf ("Usage:\njsoc_export_make_index {-h}\n");
00086     return(0);
00087     }
00088 
00089   index_txt = fopen("index.txt","r");
00090   if (!index_txt)
00091     {
00092     fprintf(stderr,"XX jsoc_export_make_index - failed to open index.txt.\n");
00093     return(1);
00094     }
00095 
00096   index_html = fopen("index.html","w");
00097   if (!index_html)
00098     {
00099     fprintf(stderr,"XX jsoc_export_make_index - failed to create index.html.\n");
00100     return(1);
00101     }
00102   fprintf(index_html, HTML_INTRO);
00103 
00104   index_json = fopen("index.json","w");
00105   if (!index_json)
00106     {
00107     fprintf(stderr,"XX jsoc_export_make_index - failed to create index.json.\n");
00108     return(1);
00109     }
00110   jroot = json_new_object();
00111   recinfo = json_new_array();
00112   
00113   int doHeader = 0;
00114 
00115   while (fgets(buf, 1000, index_txt))
00116     {
00117     char *name, *val, *sustr;
00118     char *namestr, *valstr;
00119     char *sustatus = NULL;
00120     char *susize = NULL;
00121     char linkbuf[2048];
00122     char protocolbuf[1000];
00123     char *p = buf + strlen(buf) - 1;
00124     char *c;
00125     char *procArg = NULL;
00126     char *procVal = NULL;
00127     
00128     if (p >= buf && *p == '\n')
00129       *p = '\0'; 
00130     p = buf;
00131     switch (state)
00132     {
00133       case 0: // Initial read expect standard header line
00134         if (strncmp(buf, "# JSOC ",7) != 0)
00135         {
00136             fprintf(stderr, "XX jsoc_export_make_index - incorrect index.txt file.\n");
00137             return(1);
00138         }
00139         state = 1;
00140         break;
00141       case 10: // Processing section            
00142         // Skip header separators.
00143         if (strncmp(buf, "  --", 4) == 0)
00144         {
00145             break;
00146         }
00147 
00148         if (strncmp(buf, "# DATA",6) == 0)
00149         {
00150             if (strncmp(buf, "# DATA SU", 9) == 0)
00151                 state = 3;
00152             else
00153                 state = 2;
00154                 
00155             fprintf(index_html, "</table>\n");
00156             fprintf(index_html, "<p><h2><b>Selected Data</b></h2><p>\n");
00157             fprintf(index_html, "<table>\n");
00158             break;
00159         }
00160         
00161         // skip blank and comment lines
00162         if (!*p || *p == '#') 
00163         {
00164             break;
00165         }
00166         
00167         if (strncmp(buf, "Processing", 10) == 0)
00168         {
00169             c = strchr(buf, ':');
00170             procVal = strtok(++c, " ");
00171             
00172             fprintf(index_html, "<tr><td><b>%s</b></td></tr>\n", procVal);
00173             doHeader = 1;
00174         }
00175         else
00176         {
00177             /* Gotta split on whitespace, tab. */
00178             procArg = strtok(buf, " \t");
00179             procVal = strtok(NULL, " \t");
00180             if (doHeader)
00181             {
00182                 fprintf(index_html, "<tr><td><b>%s</b></td><td><b>%s</b></td></tr>\n", procArg, procVal);
00183             }
00184             else
00185             {
00186                 fprintf(index_html, "<tr><td><b>%s</b></td><td>%s</td></tr>\n", procArg, procVal);
00187             }
00188             doHeader = 0;
00189         }
00190         break;
00191         
00192       case 1:  // In header section, take name=val pairs.
00193         if (strncmp(buf, "# DATA",6) == 0 || strncmp(buf, "# PROCESSING", 12) == 0) // done with header ?
00194         {  // Now at end of header section, write special information
00195             if (strncmp(buf, "# PROCESSING", 12) == 0)
00196             {
00197                 state = 10;
00198             }
00199             else if (strncmp(buf, "# DATA SU", 9) == 0)
00200                 state = 3;
00201             else
00202                 state = 2;
00203                 
00204             // special line for keywords
00205             if (protocol == kPROTOCOL_as_is)
00206                 sprintf(protocolbuf, "%s/%s.keywords.txt", dir, requestid);
00207             else
00208                 sprintf(protocolbuf, "**IN FITS FILES**");
00209                 
00210             namestr = string_to_json("keywords");
00211             valstr = string_to_json(protocolbuf);
00212             json_insert_pair_into_object(jroot, namestr, json_new_string(valstr));
00213             free(namestr);
00214             free(valstr);
00215             fprintf(index_html, "<tr><td><b>keywords</b></td><td>%s</td></tr>\n", protocolbuf);
00216               
00217             // special line for tarfiles
00218             if (method_tar)
00219             {
00220                 sprintf(tarfile, "%s/%s.tar", dir, requestid);
00221                 namestr = string_to_json("tarfile");
00222                 valstr = string_to_json(tarfile);
00223                 json_insert_pair_into_object(jroot, namestr, json_new_string(valstr));
00224                 free(namestr);
00225                 free(valstr);
00226                 // put name=value pair into index.html
00227                 fprintf(index_html, "<tr><td>tarfile</td><td><a href=\"%s%s\">%s</a></td></tr>\n", (method_ftp ? FTP_SERVER : HTTP_SERVER), tarfile, tarfile);
00228             }
00229             
00230             if (state == 10)
00231             {
00232                 fprintf(index_html, "</table>\n");
00233                 fprintf(index_html, "<p><h2><b>Processing Steps</b></h2><p>\n");
00234                 fprintf(index_html, "<table>\n");
00235             }
00236             else
00237             {
00238                 fprintf(index_html, "</table><p><h2><b>Selected Data</b></h2><p><table>\n");
00239             }
00240             break;
00241         }
00242         
00243         if (*p == '#' || !*p) // skip blank and comment lines
00244           break;
00245         if ((val=index(p, '='))==NULL)
00246           {
00247           fprintf(stderr, "XX jsoc_export_make_index - ignore unexpected line in header, \n    %s\n", buf);
00248           break;
00249           }
00250         p = val++;
00251         name = buf;
00252         while (isblank(*name))
00253           name++;
00254             *p-- = '\0';
00255         while (isblank(*p) && p >= buf)
00256               *p-- = '\0';
00257             while (isblank(*val))
00258               val++;
00259         p = val + strlen(val);
00260             p--;
00261         while (isblank(*p) && p >= val)
00262               *p-- = '\0';
00263 
00264             // Convert names to lower case.
00265         for (c=name; *c; c++)
00266                *c = tolower(*c);
00267 
00268             // check for special actions on some keywords
00269         // save dir for use in data section
00270         if (strcmp(name, "dir") == 0)
00271             {
00272                strncpy(dir, val, 1000);
00273 
00274                // linux/unix only!
00275                if (dir[strlen(dir) - 1] == '/')
00276                {
00277                   dir[strlen(dir) - 1] = '\0';
00278                }
00279             }
00280 
00281         // save requestid 
00282         if (strcmp(name, "requestid") == 0)
00283           strncpy(requestid, val, 1000);
00284             // Check for method==ftp
00285             if (strncmp(name, "protocol", 6) == 0)
00286               {
00287               if (strcmp(val, "as-is") == 0)
00288                 protocol = kPROTOCOL_as_is;
00289               else if (strcmp(val, "fits") == 0)
00290                 protocol = kPROTOCOL_fits;
00291               }
00292             if (strncmp(name, "method", 6) == 0)
00293               {
00294               char *dash = index(val, '-');
00295               if (dash && strncmp(dash+1, "tar", 3) == 0)
00296                 method_tar = 1;
00297               if (strncmp(val, "ftp", 3) == 0)
00298                 method_ftp = 1;
00299               }
00300 
00301         // put name=value pair into index.json
00302         namestr = string_to_json(name);
00303         valstr = string_to_json(val);
00304         json_insert_pair_into_object(jroot, namestr, json_new_string(valstr));
00305         free(namestr);
00306         free(valstr);
00307         // put name=value pair into index.html
00308         fprintf(index_html, "<tr><td><b>%s</b></td><td>%s</td></tr>\n", name, val);
00309         break;
00310       case 2: // Data section contains pairs of record query and filenames
00311         if (*p == '#' || !*p) // skip blank and comment lines
00312           break;
00313         name = buf;
00314         while (isblank(*name)) // skip leading blanks
00315           name++;
00316             p = name;
00317 
00318             /* record query might have spaces in it - can't use space as a delimiter;
00319              * but it appears that jsoc_export_as_is separates the two fields with a \t */
00320         while (*p && *p != '\t') // skip past query
00321           p++;
00322         if (*p)
00323           *p++ = '\0'; // mark end of query
00324         val = p;
00325         while (isblank(*val)) // skip leading blanks
00326           val++;
00327         p = val + strlen(val);
00328             p--;
00329         while (isblank(*p) && p >= val) // omit trailing blanks
00330               *p-- = '\0';
00331         // put query : filename pair into index.json
00332         fileinfo = json_new_object();
00333         namestr = string_to_json(name);
00334         json_insert_pair_into_object(fileinfo, "record", json_new_string(namestr));
00335         free(namestr);
00336             valstr = string_to_json(val);
00337         json_insert_pair_into_object(fileinfo, "filename", json_new_string(valstr));
00338         free(valstr);
00339         json_insert_child(recinfo, fileinfo);
00340         // put name=value pair into index.html
00341             if (method_tar)
00342           fprintf(index_html, "<tr><td>%s</td><td>%s</td></tr>\n", name, val);
00343             else
00344               {
00345           fprintf(index_html, "<tr><td>%s</td><td><A HREF=\"%s%s/%s\">%s</A></td></tr>\n",
00346                 name, (method_ftp ? FTP_SERVER : HTTP_SERVER), dir, val, val);
00347               }
00348         break;
00349       case 3: // Data section for Storage Units contains triples of sunum, seriesname, path, online status, file size
00350         if (*p == '#' || !*p) // skip blank and comment lines
00351           break;
00352         sustr = buf;
00353         while (isblank(*sustr)) // skip leading blanks
00354           sustr++;
00355         p = sustr;
00356         while (*p && !isblank(*p)) // skip past sunum
00357           p++;
00358         if (*p)
00359           *p++ = '\0'; // mark end of sunum
00360         name = p;
00361         while (isblank(*name)) // skip leading blanks
00362           name++;
00363         p = name;
00364         while (*p && !isblank(*p)) // skip past seriesname
00365           p++;
00366         if (*p)
00367           *p++ = '\0'; // mark end of seriesname
00368         val = p;
00369         while (isblank(*val)) // skip leading blanks
00370           val++;
00371 
00372         p = val;
00373         while (*p && !isblank(*p)) // skip past directory
00374           p++;
00375         if (*p)
00376           *p++ = '\0'; // mark end of directory
00377 
00378         sustatus = p;
00379         while (isblank(*sustatus)) // skip leading blanks
00380           sustatus++;
00381         p = sustatus;
00382         while (*p && !isblank(*p)) // skip past sustatus
00383           p++;
00384         if (*p)
00385           *p++ = '\0'; // mark end of sustatus
00386 
00387         susize = p;
00388         while (isblank(*susize)) // skip leading blanks
00389           susize++;
00390         p = susize;
00391         while (*p && !isblank(*p)) // skip past susize
00392           p++;
00393         if (*p)
00394           *p++ = '\0'; // mark end of susize
00395 
00396         // put sunum, seriesname, and path into json
00397         fileinfo = json_new_object();
00398         namestr = string_to_json(sustr);
00399         json_insert_pair_into_object(fileinfo, "sunum", json_new_string(sustr));
00400         free(namestr);
00401         namestr = string_to_json(name);
00402         json_insert_pair_into_object(fileinfo, "series", json_new_string(namestr));
00403         free(namestr);
00404         valstr = string_to_json(val);
00405         json_insert_pair_into_object(fileinfo, "path", json_new_string(valstr));
00406         free(valstr);
00407 
00408         // online status
00409         json_insert_pair_into_object(fileinfo, "sustatus", json_new_string(sustatus));
00410 
00411         // SU size
00412         json_insert_pair_into_object(fileinfo, "susize", json_new_string(susize));
00413 
00414         json_insert_child(recinfo, fileinfo);
00415 
00416         if (strcasecmp(val, "NA") == 0)
00417         {
00418            snprintf(linkbuf, sizeof(linkbuf), "NA");
00419         }
00420         else
00421         {
00422            /* fill in with SU path */
00423            snprintf(linkbuf, sizeof(linkbuf), "<A HREF=\"%s/%s\">%s</A>", HTTP_SERVER, val, val);
00424         }
00425 
00426         // put name=value pair into index.html
00427         fprintf(index_html, 
00428                 "<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>\n", 
00429                 sustr, /* sunum */
00430                 name, /* owning series */
00431                 linkbuf, /* link or NA */
00432                 sustatus, /* SU status - Y, N, X, I */
00433 // probably in MB
00434                 susize /* SU size in bytes */ );
00435         break;
00436       default:
00437         fprintf(stderr, "Unsupported case '%d'.\n", state);
00438       }
00439     }
00440   fclose(index_txt);
00441 
00442   // finish json
00443   json_insert_pair_into_object(jroot, "data", recinfo);
00444   json_tree_to_string(jroot,&json);
00445   final_json = json_format_string(json);
00446   fprintf(index_json, "%s\n",final_json);
00447   fclose(index_json);
00448 
00449   // finish html
00450   fprintf(index_html, "</table></body></html>\n");
00451   fclose(index_html);
00452 
00453   return(0);
00454   }
base/export/apps/jsoc_export_make_index.c
