DRMS API
PRELIMINARY - some calls are wrong
NOTE - The best place to get detailed usage information is in the JSOC Programming Documentation
DRMS Datatypes
- 1.1 Summary 1.2 Definitions of user visible types.
Record and RecordSet functions
Keyword functions
Link functions
Segment functions
Array functions
1. DRMS data types and structures
1.1 ----- Summary -----
For full definitions see jsoc/src/base/libdrms/drms_types.h Main DRMS types visible to a module: DRMS_Type_t : Basic scalar and string type enumerator. DRMS_Type_Value_t: Basic scalar and string values. DRMS_Array_t : An n-dimensional array stored consecutively in memory. Typically used to store all or part of a data segment. DRMS_RecordSet_t : A simple array container of records. DRMS_Record_t : A single data record. DRMS_Keyword_t : A record keyword (meta-data, headers) DRMS_Link_t : A record link (links to other records) DRMS_Segment_t : A record data segment representing bulk data stored in a file. DMRS types used internally: DRMS_Env_t : The main DRMS environment. Contains information about the DRMS session as well as caches of records, storage units and series template records. In the DRMS server it contains mutexes (locks), signal masks and other data structures synchronize the server threads and allowing them to communicate. DRMS_Session_t : A DRMS session handle. Holds information about the connection to a DRMS server. DRMS_ThreadInfo_t: Information passed to a thread in the DRMS server when it is spawned to service a new client. DRMS_SumRequest_t: Structure used to pass requests regarding storage units to the designated SUMS communications thread in the server.
1.2 ----- Definitions -----
/* Basic type enumeration. DRMS_TYPE_RAW is a special value used to indicate that the original/internal type of an object is desired. For example, specifying DRMS_TYPE_RAW in drms_segment_read will read the raw data values as stored on disk without type conversion and without shifting and scaling according to the bzero and bscale keywords for the segment. */ typedef enum {DRMS_TYPE_CHAR, DRMS_TYPE_SHORT, DRMS_TYPE_INT, DRMS_TYPE_LONGLONG, DRMS_TYPE_FLOAT, DRMS_TYPE_DOUBLE, DRMS_TYPE_TIME, DRMS_TYPE_STRING, DRMS_TYPE_RAW} DRMS_Type_t; /* Default "missing" values for standard types. */ #define DRMS_MISSING_CHAR (SCHAR_MIN) #define DRMS_MISSING_SHORT (SHRT_MIN) #define DRMS_MISSING_INT (INT_MIN) #define DRMS_MISSING_LONGLONG (LLONG_MIN) #define DRMS_MISSING_FLOAT (NAN) #define DRMS_MISSING_DOUBLE (NAN) #define DRMS_MISSING_STRING ("") #define DRMS_MISSING_TIME (-211087684800.0) /* equal to '-4712.01.01_12:00:00.000_UT' which is the time value used for missing in the MDI/SOI system. */ typedef union DRMS_Type_Value { char char_val; short short_val; int int_val; long long longlong_val; float float_val; double double_val; double time_val; char *string_val; } DRMS_Type_Value_t; typedef struct DRMS_Array_struct { /* Array info: */ DRMS_Type_t type; /* Datatype of the data elements. */ DRMS_Type_Value_t missing; /* Missing value. */ int naxis; /* Number of dimensions. */ int axis[DRMS_MAXRANK]; /* Size of each dimension. */ void *data; /* Data stored in column major order. */ /* Fields relating to scaling and slicing. */ struct DRMS_Segment_struct *parent_segment; /* Parent segment. */ int israw; /* Is this read in with type=DRMS_TYPE_RAW? If israw==0 then shift and scaling have been applied to the data and they represent the "true" values. If israw==1 then no shift and scaling have been applied to the data. */ double bzero; /* Zero point for parent->child mapping. */ double bscale; /* Slope for parent->child. */ int start[DRMS_MAXRANK]; /* Start offset of slice in parent. */ /* Private fields used for packed string arrays. */ char *strbuf; /* String buffer used for packed string arrays. */ long long buflen; /* Size of string buffer. */ } DRMS_Array_t; /* Simple container for a set of records. */ typedef struct DRMS_RecordSet_struct { int n; struct DRMS_Record_struct **records; } DRMS_RecordSet_t; /* Global series info. */ typedef struct DRMS_SeriesInfo_struct { char seriesname[DRMS_MAXNAMELEN]; char description[DRMS_MAXCOMMENTLEN]; char author[DRMS_MAXCOMMENTLEN]; char owner[DRMS_MAXNAMELEN]; /* Who is allowed to modify the series definition. */ int unitsize; /* How many records to a storage unit. */ int archive; /* Should records from this series be archived? */ int retention; /* Default retention time in seconds. */ int tapegroup; /* Tapegroup of the series. */ /* Primary index information. */ int pidx_num; /* Number of keywords in primary index. */ struct DRMS_Keyword_struct *pidx_keywords[DRMS_MAXPRIMIDX]; /* Pointers to keyword structs for keywords that make up the primary key.*/ } DRMS_SeriesInfo_t; /* Datastructure holding a single data record. */ typedef struct DRMS_Record_struct { struct DRMS_Env_struct *env; /* Pointer to global DRMS environment. */ long long recnum; /*** Unique record identifier. ***/ long long sunum; /* Unique index of the storage unit associated with this record. */ int init; /* Flag used internally by the series cache. */ int readonly; /* Flag indicating if record is read-only. */ DRMS_RecLifetime_t lifetime; /* Flag indicating if record is session-temporary or permanent. */ struct DRMS_StorageUnit_struct *su; /* Holds sudir (Storage unit directory). Until the storage unit has been requested from SUMS this pointer is NULL. */ int slotnum; /* Number of the slot assigned within storage unit. */ long long sessionid; /* ID of session that created this record.*/ char *sessionns; /* namespace of the session that created this record. */ DRMS_SeriesInfo_t *seriesinfo; /* Series to which this record belongs. */ HContainer_t keywords; /* Container of named keywords. */ HContainer_t links; /* Container of named links. */ HContainer_t segments; /* Container of named data segments. */ } DRMS_Record_t; typedef struct DRMS_Keyword_struct { struct DRMS_Record_struct *record; /* The record this keyword belongs to. */ char name[DRMS_MAXNAMELEN]; /* Keyword name. */ /************ Link keywords ***********/ /* If this is an inherited keyword, islink is non-zero, and linkname holds the name of the link which points to the record holding the actual keyword value. */ int islink; char linkname[DRMS_MAXNAMELEN]; /* Link pointing to a record to inherit from. */ char target_key[DRMS_MAXNAMELEN]; /* Keyword to inherit. */ /************ Regular keywords ********/ DRMS_Type_t type; /* Keyword type. */ int maxsize; /* Maximum size of keyword data in bytes.*/ DRMS_Type_Value_t value; /* Keyword data. If the Keyword is used as part of a series template then value contains the default value. */ char format[DRMS_MAXFORMATLEN]; /* Format string for formatted input and output. */ char unit[DRMS_MAXUNITLEN]; /* Physical unit. */ char description[DRMS_MAXCOMMENTLEN]; int isconstant; /* If isconstant=1 then this keyword has the same value for all records from the series. This value is stored in the master keyword table and assigned to the series template when the DRMS env is initialized. */ int per_segment; /* If per_segment=1 then this keyword has the has a different value for each segment belonging to the record. If the keyword name is "blah" then keywords belonging to specific segments should be referred to as"blah[0]", "blah[1]", etc. when calling functions in the drms_getkey and drms_setkey families. */ } DRMS_Keyword_t; /* Link datatypes. */ typedef enum { STATIC_LINK, DYNAMIC_LINK } DRMS_Link_Type_t; typedef struct DRMS_Link_struct { struct DRMS_Record_struct *record; /* The record this link belongs to. */ char name[DRMS_MAXNAMELEN]; /* Link name. */ char target_series[DRMS_MAXNAMELEN]; /* Series pointed to. */ char description[DRMS_MAXCOMMENTLEN]; /* Comment field. */ DRMS_Link_Type_t type; /* Static or dynamic? */ /*** Static link info ***/ int recnum; /* Unique record number of the record pointed to. */ /*** Dynamic link info ***/ int pidx_num; /* Number of keywords in primary index of target series. */ DRMS_Type_t pidx_type[DRMS_MAXPRIMIDX]; /* Type of primary index values. */ DRMS_Type_Value_t pidx_value[DRMS_MAXPRIMIDX]; /* Primary index values of target record(s). */ char *pidx_name[DRMS_MAXPRIMIDX]; /* Names of primary keywords in the target series. Used internally for constructing DB queries.*/ } DRMS_Link_t; /* Data segment definitions. A data segment is typically an n-dimensional array of a simple type. It can also be a "generic" segment which is just a file (structure-less as far as DRMS is concerned). */ typedef enum {DRMS_CONSTANT, DRMS_VARIABLE, DRMS_VARDIM} DRMS_Segment_Scope_t; typedef enum {DRMS_BINARY, DRMS_BINZIP, DRMS_FITZ, DRMS_FITS, DRMS_MSI, DRMS_TAS} DRMS_Protocol_t; typedef struct DRMS_SegmentInfo_struct { /* Context information: */ char name[DRMS_MAXNAMELEN]; /* Segment name */ int segnum; /* Segment number within the record */ char description[DRMS_MAXCOMMENTLEN]; /* Description string */ /************ Link segments ***********/ /* If this is an inherited segment, islink is non-zero, and linkname holds the name of the link which points to the record holding the actual segment. */ int islink; char linkname[DRMS_MAXNAMELEN]; /* Link to inherit from */ char target_seg[DRMS_MAXNAMELEN]; /* Segment to inherit. */ DRMS_Type_t type; /* Datatype of the data elements. */ int naxis; /* Number of dimensions. */ char unit[DRMS_MAXUNITLEN]; /* Physical unit. */ DRMS_Protocol_t protocol; /* Storage protocol. */ DRMS_Segment_Scope_t scope; /* Does the segment have a a) constant value for all records? b) varying value with fixed dimensions? c) varying value with varying dimensions? */ } DRMS_SegmentInfo_t; typedef struct DRMS_Segment_struct { struct DRMS_Record_struct *record; /* The record this segment belongs to. */ DRMS_SegmentInfo_t *info; /* see above */ char filename[DRMS_MAXSEGFILENAME]; /* Storage file name */ int axis[DRMS_MAXRANK]; /* Size of each dimension. */ int blocksize[DRMS_MAXRANK]; /* block sizes for tiled/blocked storage protocols. */ } DRMS_Segment_t;
2. Record and RecordSet functions
For a full list of functions see jsoc/src/base/libdrms/drms_record.{c,h}.
2.1 ----- RecordSet functions -----
DRMS_RecordSet_t *drms_open_records(DRMS_Env_t *env, char *recordsetquery, int *status) Retrieve a recordset specified by the DRMS dataset name string given in the argument "datasetname". The records are inserted into the record cache and marked read-only. DRMS_RecordSet_t *drms_create_records(DRMS_Env_t *env, int n, char *seriesname, DRMS_RecLifetime_t lifetime, int *status) Create a new set of n records from the series specified. Fill keywords, links and segments with their default values from the series definition. Each record will be assigned a new storage unit slot to store its segment files in. DRMS_RecordSet_t *drms_clone_records(DRMS_RecordSet_t *recset_in, DRMS_RecLifetime_t lifetime, DRMS_CloneAction_t mode, int *status) Clone a set of records, i.e. create a new set of records and copy the value of keywords, links and segments from the pre-existing records given in "rs". If mode=DRMS_SHARE_SEGMENTS the new segments will share segment files with the old records, i.e. it will occupy the same storage unit slot, and only keyword and link data will be replicated. If mode=DRMS_COPY_SEGMENTS the segment files for the old records will be copied to a new storage unit slots and assigned to the new records. int drms_close_records(DRMS_RecordSet_t *rs, int action); Close a set of records and free them from the record cache. a) If action=DRMS_INSERT_RECORD the record meta-data (storage unit info, keywords, links, segment meta-data) will be inserted into the database and the data segments will be left in the storage unit directory for later archiving by SUMS. NOTICE: The records will only be comitted permanently to the database if the session finishes without error or a DRMS_COMMIT (or signal USR1) command is sent to the DRMS server process acting as session master. If this happens, the storage units are also handed over to SUMS (with a SUM_put command). Otherwise the SUMS cleanup process will eventually delete all files in the If any record in the set is marked readonly an error code of DRMS_ERROR_COMMITREADONLY is returned. b) If action=DRMS_FREE_RECORD the storage unit slots belonging to records in the set that are not marked readonly are marked as free and will not be submitted to SUMS for archiving. int drms_closeall_records(DRMS_Env_t *env, int action); Execute drms_close_record for all records in the record cache that are not marked read-only, i.e. which were created by the present program. The value action = DRMS_INSERT_RECORD or DRMS_FREE_RECORD applies to all these records as for drms_close_records. Typically issued at the end of a module.
2.2 ---- Record functions -----
DRMS_Record_t *drms_create_record(DRMS_Env_t *env, char *series, int *status) DRMS_Record_t *drms_clone_record(DRMS_Record_t *oldrec, int mode, int *status) int drms_close_record(DRMS_Record_t *rec, int action) Single record versions of the RecordSet functions listed above. void drms_record_print(DRMS_Record_t *rec); "Pretty" print the contents of a record data structure to stdout. DRMS_Record_t *drms_retrieve_record(DRMS_Env_t *env, const char *seriesname, int recnum, int *status) Retrieve meta-data for a data records with known series and record number. If it is already in the dataset cache, simply return a pointer to its data structure, otherwise retrieve it from the database. In the latter case, add it to the record cache for fast future retrieval. Mostly used internally. long long drms_record_size(DRMS_Record_t *rec); Calculate size of a record and its segment arrays in bytes. FILE *drms_record_fopen(DRMS_Record_t *rec, char *filename, const char *mode) Asks DRMS to open a file in the storage unit slot directory associated with a data record. Mode can take the same values as in a regular fopen call. If mode="w" or mode="a" and the record has not been assigned a storage unit slot, one is allocated. It is an error to call with mode="r" if the record has not been assigned a storage unit slot, e.g. if it is a newly created record whose data segments have not yet been written. In this case a NULL pointer is returned. void drms_record_directory(DRMS_Record_t *rec, char *path) Returns the full path of the storage unit slot directory assigned to this record. If no storage unit slot has been assigned to the record yet, an empty string is returned. This should not be used for creating new files in the directory. Use drms_record_fopen for that.
3. Keyword functions
For full definitions see jsoc/src/base/libdrms/ drms_keyword.{c,h}, drms_statuscodes.h DRMS_Keyword_t *drms_keyword_lookup(DRMS_Record_t *rec, const char *key); Look up keyword by name. If no keyword by the given name exists for the record NULL is returned. If the named keyword exists and is a linked keyword, the link is followed (often resulting in the record pointed to by the link being retrieved from the DRM database) and the keyword pointed to is returned. Since linked keywords are allowed to point to linked keywords in other series, drms_keyword_lookup checks the recursion depth to avoid circular links causing infinite loops. A maximum recursion depth of DRMS_MAXLINKDEPTH (20) is allowed. drms_keyword_lookup is used internally by the setkey and getkey family of functions. char drms_getkey_char(DRMS_Record_t *rec, const char *key,int *status); short drms_getkey_short(DRMS_Record_t *rec, const char *key, int *status); int drms_getkey_int(DRMS_Record_t *rec, const char *key, int *status); long long drms_getkey_longlong(DRMS_Record_t *rec, const char *key, int *status) ; float drms_getkey_float(DRMS_Record_t *rec, const char *key, int *status); double drms_getkey_double(DRMS_Record_t *rec, const char *key, int *status); char *drms_getkey_string(DRMS_Record_t *rec, const char *key, int *status); Get the value of a named keyword. The value (which internally is stored as the type specified for the keyword in the series definition) is cast to the target type indicated by return type and the last part of the function name. Conversion of floating point to integer types is done with rounding. If status is not NULL, the following status codes are returned in *status: DRMS_SUCCESS=0: The type conversion was successful with no loss of information. DRMS_INEXACT=1: The keyword value was within the range of the target, but some loss of information occured due to rounding. DRMS_RANGE=-1: The keyword value was outside the range of the target type. The standard missing value for the target type is returned. DRMS_BADSTRING=-2: When converting from a string, the contents of the string did not match a valid constant of the target type. The list above represents the ranking of the four status codes in degree of severity. If more than one condition holds, the most severe status code is returned. For example: If the drms_getkey_char is called for a keyword with value 200.23, rounding occurs and the value is out of range of the target type. In this case *status=DRMS_RANGE is returned. If the record has no keyword by the name given in "key" *status=DRMS_ERROR_UNKNOWNKEYWORD=-10006 is returned. DRMS_Type_Value_t drms_getkey(DRMS_Record_t *rec, const char *key, DRMS_Type_t *type, int *status); This is a generic version of the type specific functions above. The value is returned as it is stored in the keyword structure. The type of the returned value is returned in *type. The value can subsequently be accessed by reading the appropriate field in the DRMS_Type_Value_t union returned. int drms_setkey_char(DRMS_Record_t *rec, const char *key, char value); int drms_setkey_short(DRMS_Record_t *rec, const char *key, short value); int drms_setkey_int(DRMS_Record_t *rec, const char *key, int value); int drms_setkey_longlong(DRMS_Record_t *rec, const char *key, long long value); int drms_setkey_float(DRMS_Record_t *rec, const char *key, float value); int drms_setkey_double(DRMS_Record_t *rec, const char *key, double value); int drms_setkey_string(DRMS_Record_t *rec, const char *key, char *value); Set the value of a named keyword. The value is converted to the keyword type specified in the series definition from the source type indicated by the type of the "value" parameter and the last part of the function name. Conversion of floating point to integer types is done with rounding. A status code is returned. The status codes are the same used for the getkey family and have identical meanings (see above). int drms_setkey(DRMS_Record_t *rec, const char *key, DRMS_Type_t type, DRMS_Type_Value_t *value) This is a generic version of the type specific functions above. The type of the input value be be given in "type". The input value given in "value" is converted to the keyword type specified in the series definition and stored in the relevant keyword structure. A status code is returned. The status codes are the same used for the getkey family and have identical meanings (see above). void drms_keyword_print(DRMS_Keyword_t *key); Print all fields of the DRMS_Keyword_t struct void drms_keyword_printval(DRMS_Keyword_t *key); Print formatted keyword value to stdout. int drms_convert(DRMS_Type_t dsttype, DRMS_Type_Value_t *dst, DRMS_Type_t srctype, DRMS_Type_Value_t *src) Utility function that converts a scalar value from one DRMS type to another. A status code is returned. The status codes are the same used for the getkey family and have identical meanings (see above).
4. Link functions
For full definitions see jsoc/src/base/libdrms/drms_link.{c,h}. DRMS_Record_t *drms_link_follow(DRMS_Record_t *rec, const char *linkname, int *status); Follow a link to its destination record, retrieve it and return a pointer to it. If the link is dynamic the record with the highest record number out of those with values of the primary index keywords matching those in the link is returned. DRMS_RecordSet_t *drms_link_followall(DRMS_Record_t *rec, const char *linkname, int *status); Follow a link to its destination records, retrieve them and return them in a RecordSet_t structure. If the link is dynamic the functions returns all records with values of the primary index keywords matching those in the link. If the link is static only a single record is contained in the RecordSet. int drms_setlink_static(DRMS_Record_t *rec, const char *linkname, int recnum); Set a static link to point to the record with absolute record number "recnum" in the target series of link "linkname" associated with record "rec". int drms_setlink_dynamic(DRMS_Record_t *rec, const char *linkname, DRMS_Type_t *types, DRMS_Type_Value_t *values); Set a dynamic link to point to the record(s) with primary index values mathing those given in the "types" and "values" arrays. When a dynamic link is resolved, records matching these primary index values are selected. void drms_link_print(DRMS_Link_t *link); Print the contents of a link structure to stdout.
5. Segment functions
For full definitions see jsoc/src/base/libdrms/drms_segment.{c,h}. DRMS_Array_t *drms_segment_read(DRMS_Segment_t *seg, DRMS_Type_t type, int *status) Read the contents of a data segment into an array structure, converting it to the specified type. a) If the corresponding data file exists and type!=DRMS_TYPE_RAW, then read the entire data array into memory. Convert it to the type given as argument and transform the data according to bzero and bscale. The array struct will have israw==0. b) If the corresponding data file exists and type=DRMS_TYPE_RAW then the data is read into an array of the same type it is stored as on disk without scaling. The array struct will have israw==1. c) If the data file does not exist, then return a data array filed with the MISSING value for the given type. /* FIXME should this be the missing value specified for the segment? */ The bzero and bscale fields of the array struct are set to the values that apply to the segment. If type=DRMS_TYPE_RAW then bzero and bscale are still set to the values that would have been used for scaling the stored values to their "true" value. DRMS_Array_t *drms_segment_readslice(DRMS_Segment_t *seg, DRMS_Type_t type, int *start, int *end, int *status) Read a slice from a data segment array. If seg->naxis=n then the first n entries in the start and stop arrays indicate the first and last (inclusive) element along that axis belong to the slice. The data is returned as an array with dimensions axis[i] = (end[i]-start[i]+1) stored consecutively in memory. The arr->start array will contain the starting indices given, such that information needed for mapping back into the parent segment array is preserved. Type conversion and scaling is performed as in drms_segment_read. int drms_segment_write(DRMS_Segment_t *seg, DRMS_Array_t *arr) Write the array argument to the file occupied by the segment argument. The number and size of dimensions of the array must match those of the segment. The data values are scaled and converted to the representation determined by seg->bzero seg->bscale and seg->type. The values of arr->bzero, arr->bscale, arr->israw are used to determine how to properly scale the data. Three distinct cases arise: 1. arr->israw==0: The values in the array struct represent their "true" values. Compute data to store in file as x = (1.0 / seg->bscale) * y - seg->bzero / seg->bscale (y = value in array, x = value written to file) 2. arr->israw==1 and arr->bscale==seg->bscale and arr->bzero==seg->bzero: The raw values in the array struct match the scaling of the raw values for the segment. Write without scaling. 3. arr->israw==1 and (arr->bscale!=seg->bscale or arr->bzero==seg->bzero): Compute data to store in file as x = (arr->bscale/seg->bscale)*y + (arr->bzero-seg->bzero)/seg->bscale (y = value in array, x = value written to file) long long drms_segment_size(DRMS_Segment_t *seg) Return segment size in bytes. The size of generic segments is determined by stat'ing the associated file. If no file is created yet a value of zero is returned. void drms_segment_print(DRMS_Segment_t *seg) Print the fields of a segment struct to stdout. void drms_segment_filename(DRMS_Segment_t *seg, char *filename) Return absolute path to segment file in filename. filename must be able the hold at least DRMS_MAXPATHLEN bytes. int drms_segment_setscaling(DRMS_Segment_t *seg, double bzero, double bscale) Set segment scaling. Can only be done for an array segment and only when creating a new record. Otherwise the error codes DRMS_ERROR_INVALIDACTION and DRMS_ERROR_RECORDREADONLY are returned respectively. int drms_segment_getscaling(DRMS_Segment_t *seg, double *bzero, double *bscale) Get scaling for an array segment. If the segment is not an array segment an error of DRMS_ERROR_INVALIDACTION is returned. DRMS_Segment_t *drms_segment_lookup(DRMS_Record_t *rec, const char *segname) Look up segment by name. DRMS_Segment_t *drms_segment_lookupnum(DRMS_Record_t *rec, int segnum) Look up segment by number. void drms_segment_setblocksize(DRMS_Segment_t *seg, int blksz[DRMS_MAXRANK]) Set tile/block sizes for tiled storage. void drms_segment_getblocksize(DRMS_Segment_t *seg, int blksz[DRMS_MAXRANK]) Get tile/block sizes for tiled storage.
6. Array functions
DRMS_Array_t *drms_array_convert(DRMS_Type_t dsttype, double bzero, double bscale, DRMS_Array_t *src) Convert array from one DRMS type to another with scaling. DRMS_Array_t *drms_array_slice(int *start, int *end, DRMS_Array_t *src) Take out a hyperslab of an array. void drms_array2missing(DRMS_Array_t *arr) Set to MISSING according to arr->type. long long drms_array_count(DRMS_Array_t *arr) Calculate the total number of entries in an n-dimensional array. long long drms_array_size(DRMS_Array_t *arr) Calculate the size in bytes of an n-dimensional array. DRMS_Array_t *drms_array_create(DRMS_Type_t type, int naxis, int *axis, void *data, int *status) Assemble an array struct from its constituent parts. void drms_free_array(DRMS_Array_t *arr) Free array and data.