core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
eccodes/src/grib_index.c

1749 lines
46 KiB
C
Raw Blame History

/*
* Copyright 2005-2018 ECMWF.
*
* This software is licensed under the terms of the Apache Licence Version 2.0
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
*
* In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
* virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
*/
/*
*
* Description: grib index
*
*/
#include "grib_api_internal.h"
#define UNDEF_LONG -99999
#define UNDEF_DOUBLE -99999
#define NULL_MARKER 0
#define NOT_NULL_MARKER 255
/* #if GRIB_PTHREADS */
#if 0
static pthread_once_t once = PTHREAD_ONCE_INIT;
static pthread_mutex_t mutex1 = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t mutex2 = PTHREAD_MUTEX_INITIALIZER;
static void init() {
pthread_mutexattr_t attr;
pthread_mutexattr_init(&attr);
pthread_mutexattr_settype(&attr,PTHREAD_MUTEX_RECURSIVE);
pthread_mutex_init(&mutex1,&attr);
pthread_mutex_init(&mutex2,&attr);
pthread_mutexattr_destroy(&attr);
}
/* #elif GRIB_OMP_THREADS */
static int once = 0;
static omp_nest_lock_t mutex1;
static omp_nest_lock_t mutex2;
static void init()
{
GRIB_OMP_CRITICAL(lock_grib_index_c)
{
if (once == 0)
{
omp_init_nest_lock(&mutex1);
omp_init_nest_lock(&mutex2);
once = 1;
}
}
}
#endif
static const char* mars_keys =
"mars.date,mars.time,mars.expver,mars.stream,mars.class,mars.type,"
"mars.step,mars.param,mars.levtype,mars.levelist,mars.number,mars.iteration,"
"mars.domain,mars.fcmonth,mars.fcperiod,mars.hdate,mars.method,"
"mars.model,mars.origin,mars.quantile,mars.range,mars.refdate,mars.direction,mars.frequency";
/* See GRIB-32: start off ID with -1 as it is incremented before being used */
static int grib_filesid=-1;
static int index_count;
static char* get_key(char** keys,int *type)
{
char* key=NULL;
char* p=NULL;
if (keys==NULL || *keys == 0) return NULL;
*type=GRIB_TYPE_UNDEFINED;
p=*keys;
while (*p == ' ') p++;
while (*p != 0 && *p != ':' && *p != ',') p++;
if ( *p == ':' ) {
*type=grib_type_to_int(*(p+1));
*p=0;
p++;
while (*p != 0 && *p != ',') {*(p++)=0;}
} else *type=GRIB_TYPE_UNDEFINED;
if (*p) {*p=0;p++;}
key=*keys;
*keys= *p==0 ? NULL : p;
return key;
}
static int compare_long(const void* a,const void* b)
{
long* arg1 = (long*) a;
long* arg2 = (long*) b;
if( *arg1 == *arg2 ) return 0;
return *arg1 < *arg2 ? -1 : 1;
}
static int compare_double(const void* a,const void* b)
{
double* arg1 = (double*) a;
double* arg2 = (double*) b;
if( *arg1 == *arg2 ) return 0;
return *arg1 < *arg2 ? -1 : 1;
}
static int compare_string(const void* a,const void* b)
{
char* arg1 = *(char* const*) a;
char* arg2 = *(char* const*) b;
while (*arg1 != 0 && *arg2 != 0 && *arg1 == *arg2 ) {arg1++;arg2++;}
if( *arg1 == *arg2 ) return 0;
return *arg1 < *arg2 ? -1 : 1;
}
static int grib_index_keys_compress(grib_context* c,grib_index* index,int* compress)
{
grib_index_key *keys=index->keys->next;
grib_index_key *prev=index->keys;
int level=0;
if (!keys) return 0;
level=1;
while (keys) {
if (keys->values_count==1) {
prev->next=keys->next;
grib_context_free(c,keys->name);
grib_context_free(c,keys);
keys=prev->next;
compress[level]=1;
level++;
} else {
prev=keys;
keys=keys->next;
compress[level]=0;
level++;
}
}
if (index->keys->values_count==1) {
keys=index->keys;
index->keys=index->keys->next;
grib_context_free(c,keys->name);
grib_context_free(c,keys);
compress[0]=1;
} else compress[0]=0;
return 0;
}
static int grib_index_fields_compress(grib_context* c,
grib_field_tree* fields,grib_field_tree* prev,int level,int* compress)
{
if (!fields) return 0;
if (!prev) {
if (fields->next)
grib_index_fields_compress(c,fields->next,0,level,compress);
level++;
return grib_index_fields_compress(c,fields->next_level,fields,level,compress);
}
if (compress[level]) {
if (!fields->next_level) prev->field=fields->field;
prev->next_level=fields->next_level;
grib_context_free(c,fields->value);
grib_context_free(c,fields);
level++;
grib_index_fields_compress(c,prev->next_level,prev,level,compress);
} else {
grib_field_tree* next;
next=fields->next;
level++;
while (next) {
grib_index_fields_compress(c,next->next_level,next,level,compress);
next=next->next;
}
grib_index_fields_compress(c,fields->next_level,fields,level,compress);
}
return 0;
}
int grib_index_compress(grib_index* index)
{
int err=0;
grib_context* c=index->context;
int compress[200]={0,};
if (!index->keys->next) return 0;
err=grib_index_keys_compress(c,index,compress);
if (err) return err;
grib_index_fields_compress(c,index->fields,0,0,compress);
if (!index->fields->next) {
grib_field_tree* next_level=index->fields->next_level;
grib_context_free(c,index->fields->value);
grib_context_free(c,index->fields);
index->fields=next_level;
}
return 0;
}
static grib_index_key* grib_index_new_key(grib_context* c,grib_index_key* keys,
const char* key,int type,int *err)
{
grib_index_key *next=NULL,*current=NULL;
grib_string_list* values=NULL;
next=(grib_index_key*)grib_context_malloc_clear(c,sizeof(grib_index_key));
if (!next) {
grib_context_log(c,GRIB_LOG_ERROR,
"unable to allocate %d bytes",
sizeof(grib_index_key));
*err=GRIB_OUT_OF_MEMORY;
return NULL;
}
values=(grib_string_list*)grib_context_malloc_clear(c,sizeof(grib_string_list));
if (!values) {
grib_context_log(c,GRIB_LOG_ERROR,
"unable to allocate %d bytes",
sizeof(grib_string_list));
*err=GRIB_OUT_OF_MEMORY;
return NULL;
}
next->values=values;
if (!keys) {
keys=next;
current=keys;
} else {
current=keys;
while (current->next) current=current->next;
current->next=next;
current=current->next;
}
current->type=type;
current->name=grib_context_strdup(c,key);
return keys;
}
int grib_read_uchar( FILE* fh,unsigned char *val)
{
if (fread(val,sizeof(unsigned char),1,fh) <1) {
if (feof(fh)) return GRIB_END_OF_FILE;
else return GRIB_IO_PROBLEM;
}
return GRIB_SUCCESS;
}
int grib_read_short( FILE* fh,short *val)
{
if (fread(val,sizeof(short),1,fh) <1) {
if (feof(fh)) return GRIB_END_OF_FILE;
else return GRIB_IO_PROBLEM;
}
return GRIB_SUCCESS;
}
int grib_read_long( FILE* fh,long *val)
{
if (fread(val,sizeof(long),1,fh) <1) {
if (feof(fh)) return GRIB_END_OF_FILE;
else return GRIB_IO_PROBLEM;
}
return GRIB_SUCCESS;
}
int grib_read_unsigned_long( FILE* fh,unsigned long *val)
{
if (fread(val,sizeof(long),1,fh) <1) {
if (feof(fh)) return GRIB_END_OF_FILE;
else return GRIB_IO_PROBLEM;
}
return GRIB_SUCCESS;
}
int grib_write_uchar(FILE* fh,unsigned char val)
{
if (fwrite(&val,sizeof(unsigned char),1,fh)<1)
return GRIB_IO_PROBLEM;
return GRIB_SUCCESS;
}
int grib_write_short(FILE* fh,short val)
{
if (fwrite(&val,sizeof(short),1,fh)<1)
return GRIB_IO_PROBLEM;
return GRIB_SUCCESS;
}
int grib_write_long(FILE* fh,long val)
{
if (fwrite(&val,sizeof(long),1,fh)<1)
return GRIB_IO_PROBLEM;
return GRIB_SUCCESS;
}
int grib_write_unsigned_long(FILE* fh,unsigned long val)
{
if (fwrite(&val,sizeof(long),1,fh)<1)
return GRIB_IO_PROBLEM;
return GRIB_SUCCESS;
}
int grib_write_string(FILE* fh,const char* s)
{
size_t len = 0;
if (s == NULL)
return GRIB_IO_PROBLEM;
len = strlen(s);
grib_write_uchar(fh,(unsigned char)len);
if (fwrite(s,1,len,fh)<len)
return GRIB_IO_PROBLEM;
return GRIB_SUCCESS;
}
int grib_write_identifier(FILE* fh)
{
return grib_write_string(fh,"GRBIDX1");
}
int grib_write_null_marker(FILE* fh)
{
return grib_write_uchar(fh,NULL_MARKER);
}
int grib_write_not_null_marker(FILE* fh)
{
return grib_write_uchar(fh,NOT_NULL_MARKER);
}
char *grib_read_string(grib_context* c,FILE* fh,int *err)
{
unsigned char len=0;
char* s=NULL;
*err = grib_read_uchar(fh,&len);
if (*err) return NULL;
s = (char*)grib_context_malloc_clear(c,len+1);
if (fread(s,len,1,fh) < 1)
{
if (feof(fh)) *err=GRIB_END_OF_FILE;
else *err=GRIB_IO_PROBLEM;
return NULL;
}
s[len] = 0;
return s;
}
static int grib_write_field(FILE* fh,grib_field* field)
{
int err;
if (!field)
return grib_write_null_marker(fh);
err=grib_write_not_null_marker(fh);
if (err) return err;
err=grib_write_short(fh,field->file->id);
if (err) return err;
err=grib_write_unsigned_long(fh,field->offset);
if (err) return err;
err=grib_write_unsigned_long(fh,field->length);
if (err) return err;
err=grib_write_field(fh,field->next);
if (err) return err;
return GRIB_SUCCESS;
}
static grib_field* grib_read_field(grib_context* c, FILE* fh, grib_file** files, int *err)
{
grib_field* field=NULL;
short file_id;
unsigned char marker ;
unsigned long offset;
unsigned long length;
*err = grib_read_uchar(fh,&marker);
if(marker == NULL_MARKER) return NULL;
if(marker != NOT_NULL_MARKER) {*err=GRIB_CORRUPTED_INDEX;return NULL;}
index_count++;
field=(grib_field*)grib_context_malloc(c,sizeof(grib_field));
*err=grib_read_short(fh,&file_id);
if (*err) return NULL;
field->file=files[file_id];
*err=grib_read_unsigned_long(fh,&offset);
field->offset=offset;
if (*err) return NULL;
*err=grib_read_unsigned_long(fh,&length);
field->length=length;
if (*err) return NULL;
field->next=grib_read_field(c,fh,files,err);
return field;
}
static int grib_write_field_tree(FILE* fh,grib_field_tree* tree)
{
int err=0;
if(!tree)
return grib_write_null_marker(fh);
err=grib_write_not_null_marker(fh);
if (err) return err;
err=grib_write_field(fh,tree->field);
if (err) return err;
err=grib_write_string(fh,tree->value);
if (err) return err;
err=grib_write_field_tree(fh,tree->next_level);
if (err) return err;
err=grib_write_field_tree(fh,tree->next);
if (err) return err;
return GRIB_SUCCESS;
}
grib_field_tree* grib_read_field_tree(grib_context* c, FILE* fh, grib_file** files, int *err)
{
grib_field_tree* tree=NULL;
unsigned char marker=0;
*err = grib_read_uchar(fh,&marker);
if(marker == NULL_MARKER) return NULL;
if(marker != NOT_NULL_MARKER) {*err=GRIB_CORRUPTED_INDEX;return NULL;}
tree = (grib_field_tree*)grib_context_malloc(c, sizeof(grib_field_tree));
tree->field = grib_read_field(c,fh,files,err);
if (*err) return NULL;
tree->value = grib_read_string(c,fh,err);
if (*err) return NULL;
tree->next_level = grib_read_field_tree(c,fh,files,err);
if (*err) return NULL;
tree->next = grib_read_field_tree(c,fh,files,err);
if (*err) return NULL;
return tree;
}
grib_index* grib_index_new(grib_context* c,const char* key,int *err)
{
grib_index* index;
grib_index_key* keys=NULL;
char* q;
int type;
char* p;
if (!strcmp(key,"mars"))
return grib_index_new(c,mars_keys,err);
p= grib_context_strdup(c,key);
q=p;
*err=0;
if (!c) c=grib_context_get_default();
index=(grib_index*)grib_context_malloc_clear(c,sizeof(grib_index));
if (!index) {
grib_context_log(c,GRIB_LOG_ERROR,"unable to create index");
*err=GRIB_OUT_OF_MEMORY;
return NULL;
}
index->context=c;
while ((key=get_key(&p,&type))!=NULL) {
keys=grib_index_new_key(c,keys,key,type,err);
if (*err) return NULL;
}
index->keys=keys;
index->fields=(grib_field_tree*)grib_context_malloc_clear(c,
sizeof(grib_field_tree));
if (!index->fields) {*err=GRIB_OUT_OF_MEMORY;return NULL;}
grib_context_free(c,q);
return index;
}
static void grib_index_values_delete(grib_context* c,grib_string_list* values)
{
if (!values) return;
grib_index_values_delete(c,values->next);
grib_context_free(c,values->value);
grib_context_free(c,values);
return ;
}
static void grib_index_key_delete(grib_context* c,grib_index_key* keys)
{
if (!keys) return;
grib_index_key_delete(c,keys->next);
grib_index_values_delete(c,keys->values);
grib_index_values_delete(c,keys->current);
grib_context_free(c,keys->name);
grib_context_free(c,keys);
}
static long values_count=0;
static grib_string_list* grib_read_key_values(grib_context* c,FILE* fh,int *err)
{
grib_string_list* values;
unsigned char marker=0;
*err = grib_read_uchar(fh,&marker);
if(marker == NULL_MARKER) return NULL;
if(marker != NOT_NULL_MARKER) {*err=GRIB_CORRUPTED_INDEX;return NULL;}
values_count++;
values=(grib_string_list*)grib_context_malloc_clear(c,sizeof(grib_string_list));
values->value=grib_read_string(c,fh,err);
if (*err) return NULL;
values->next=grib_read_key_values(c,fh,err);
if (*err) return NULL;
return values;
}
static int grib_write_key_values(FILE* fh,grib_string_list* values)
{
int err=0;
if (!values)
return grib_write_null_marker(fh);
err=grib_write_not_null_marker(fh);
if (err) return err;
err=grib_write_string(fh,values->value);
if (err) return err;
err=grib_write_key_values(fh,values->next);
if (err) return err;
return GRIB_SUCCESS;
}
static grib_index_key* grib_read_index_keys(grib_context* c,FILE* fh,int *err)
{
grib_index_key* keys=NULL;
unsigned char marker=0;
unsigned char type=0;
if (!c) c=grib_context_get_default();
*err = grib_read_uchar(fh,&marker);
if(marker == NULL_MARKER) return NULL;
if(marker != NOT_NULL_MARKER) {*err=GRIB_CORRUPTED_INDEX;return NULL;}
keys= (grib_index_key*)grib_context_malloc_clear(c,sizeof(grib_index_key));
keys->name = grib_read_string(c,fh,err);
if (*err) return NULL;
*err = grib_read_uchar(fh,&type);
keys->type=type;
if (*err) return NULL;
values_count=0;
keys->values=grib_read_key_values(c,fh,err);
if (*err) return NULL;
keys->values_count=values_count;
keys->next=grib_read_index_keys(c,fh,err);
if (*err) return NULL;
return keys;
}
static int grib_write_index_keys(FILE* fh,grib_index_key* keys)
{
int err=0;
if (!keys)
return grib_write_null_marker(fh);
err=grib_write_not_null_marker(fh);
if (err) return err;
err=grib_write_string(fh,keys->name);
if (err) return err;
err=grib_write_uchar(fh,(unsigned char)keys->type);
if (err) return err;
err=grib_write_key_values(fh,keys->values);
if (err) return err;
err=grib_write_index_keys(fh,keys->next);
if (err) return err;
return GRIB_SUCCESS;
}
static void grib_field_delete(grib_context* c,grib_field* field)
{
int err=0;
if (!field) return;
grib_field_delete(c,field->next);
if (field->file) {
grib_file_close(field->file->name, 0, &err);
field->file=NULL;
}
grib_context_free(c,field);
}
static void grib_field_tree_delete(grib_context* c,grib_field_tree* tree)
{
if(!tree) return;
grib_field_delete(c,tree->field);
grib_context_free(c,tree->value);
grib_field_tree_delete(c,tree->next_level);
grib_field_tree_delete(c,tree->next);
grib_context_free(c,tree);
}
static void grib_field_list_delete(grib_context* c, grib_field_list* field_list)
{
if (!field_list) return;
grib_context_free(c,field_list);
}
void grib_index_delete(grib_index* index)
{
grib_file* file=index->files;
grib_index_key_delete(index->context,index->keys);
grib_field_tree_delete(index->context,index->fields);
grib_field_list_delete(index->context,index->fieldset);
while (file) {
grib_file* f=file;
file=file->next;
grib_file_delete(f);
}
grib_context_free(index->context,index);
}
static int grib_write_files(FILE* fh,grib_file* files)
{
int err;
if (!files)
return grib_write_null_marker(fh);
err=grib_write_not_null_marker(fh);
if (err) return err;
err=grib_write_string(fh,files->name);
if (err) return err;
err=grib_write_short(fh,(short)files->id);
if (err) return err;
return grib_write_files(fh,files->next);
}
static grib_file* grib_read_files(grib_context *c, FILE* fh, int *err)
{
unsigned char marker=0;
short id=0;
grib_file* file;
*err = grib_read_uchar(fh,&marker);
if(marker == NULL_MARKER) return NULL;
if(marker != NOT_NULL_MARKER) {*err=GRIB_CORRUPTED_INDEX;return NULL;}
file=(grib_file*)grib_context_malloc(c,sizeof(grib_file));
file->name=grib_read_string(c,fh,err);
if (*err) return NULL;
*err=grib_read_short(fh,&id);
file->id=id;
if (*err) return NULL;
file->next=grib_read_files(c,fh,err);
if (*err) return NULL;
return file;
}
int grib_index_write(grib_index* index,const char* filename)
{
int err=0;
FILE* fh;
grib_file* files;
fh=fopen(filename,"w");
if (!fh) {
grib_context_log(index->context,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
"Unable to write in file %s",filename);
perror(filename);
return GRIB_IO_PROBLEM;
}
err=grib_write_identifier(fh);
if (err) {
grib_context_log(index->context,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
"Unable to write in file %s",filename);
perror(filename);
return err;
}
if (!index) return grib_write_null_marker(fh);
err=grib_write_not_null_marker(fh);
if (err) return err;
/* See GRIB-32: Do not use the file pool */
/* files=grib_file_pool_get_files(); */
files=index->files;
err=grib_write_files(fh,files);
if (err) {
grib_context_log(index->context,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
"Unable to write in file %s",filename);
perror(filename);
return err;
}
err=grib_write_index_keys(fh,index->keys);
if (err) {
grib_context_log(index->context,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
"Unable to write in file %s",filename);
perror(filename);
return err;
}
err=grib_write_field_tree(fh,index->fields);
if (err) {
grib_context_log(index->context,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
"Unable to write in file %s",filename);
perror(filename);
return err;
}
if (fclose(fh) != 0) {
grib_context_log(index->context,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
"Unable to write in file %s",filename);
perror(filename);
return GRIB_IO_PROBLEM;
}
return err;
}
grib_index* grib_index_read(grib_context* c, const char* filename, int *err)
{
grib_file *file,*f;
grib_file** files;
grib_index* index=NULL;
unsigned char marker=0;
char* identifier=NULL;
int max=0;
FILE* fh = NULL;
if (!c) c=grib_context_get_default();
fh=fopen(filename,"r");
if (!fh) {
grib_context_log(c,(GRIB_LOG_ERROR)|(GRIB_LOG_PERROR),
"Unable to read file %s",filename);
perror(filename);
*err=GRIB_IO_PROBLEM;
return NULL;
}
identifier=grib_read_string(c,fh,err);
if (!identifier) {
fclose(fh);
return NULL;
}
grib_context_free(c,identifier);
*err = grib_read_uchar(fh,&marker);
if(marker == NULL_MARKER) {
fclose(fh);
return NULL;
}
if(marker != NOT_NULL_MARKER) {
*err=GRIB_CORRUPTED_INDEX;
fclose(fh);
return NULL;
}
file = grib_read_files(c,fh,err);
if (*err) return NULL;
f=file;
while (f) {
if (max<f->id) max=f->id;
f=f->next;
}
files=(grib_file**)grib_context_malloc_clear(c,sizeof(grib_file)*(max+1));
f=file;
while (f) {
grib_file_open(f->name,"r",err);
if (*err) return NULL;
files[f->id]=grib_get_file(f->name,err); /* fetch from pool */
f=f->next;
}
while (file) {
f=file;
file=file->next;
grib_context_free(c,f->name);
grib_context_free(c,f);
}
index=(grib_index*)grib_context_malloc_clear(c,sizeof(grib_index));
index->context=c;
index->keys=grib_read_index_keys(c,fh,err);
if (*err) return NULL;
index_count=0;
index->fields=grib_read_field_tree(c,fh,files,err);
if (*err) return NULL;
index->count=index_count;
fclose(fh);
return index;
}
int grib_index_search_same(grib_index* index, grib_handle* h)
{
int err=0;
char buf[1024]={0,};
size_t buflen=1024;
grib_index_key* keys;
long lval=0;
double dval=0.0;
grib_context* c;
if (!index) return GRIB_NULL_INDEX;
c=index->context;
keys=index->keys;
while (keys) {
if (keys->type==GRIB_TYPE_UNDEFINED) {
err=grib_get_native_type(h,keys->name,&(keys->type));
if (err) keys->type=GRIB_TYPE_STRING;
}
buflen=1024;
switch (keys->type) {
case GRIB_TYPE_STRING:
err=grib_get_string(h,keys->name,buf,&buflen);
if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
break;
case GRIB_TYPE_LONG:
err=grib_get_long(h,keys->name,&lval);
if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
else sprintf(buf,"%ld",lval);
break;
case GRIB_TYPE_DOUBLE:
err=grib_get_double(h,keys->name,&dval);
if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
else sprintf(buf,"%g",dval);
break;
default :
err=GRIB_WRONG_TYPE;
return err;
}
if (err && err != GRIB_NOT_FOUND) {
grib_context_log(c,GRIB_LOG_ERROR,
"unable to create index. \"%s\": %s",
keys->name,grib_get_error_message(err));
return err;
}
sprintf(keys->value,"%s",buf);
keys=keys->next;
}
grib_index_rewind(index);
return 0;
}
int grib_index_add_file(grib_index* index,const char* filename) {
return _codes_index_add_file(index,filename,CODES_GRIB);
}
grib_handle* new_message_from_file(int message_type, grib_context *c, FILE *f, int *error)
{
if (message_type == CODES_GRIB)
return grib_new_from_file(c, f, 0, error); /* headers_only=0 */
if (message_type == CODES_BUFR)
return bufr_new_from_file(c, f, error);
Assert(0);
return NULL;
}
int _codes_index_add_file(grib_index* index,const char* filename,int message_type)
{
double dval;
size_t svallen;
size_t message_count = 0;
long length,lval;
char buf[1024]={0,};
int err=0;
grib_file* indfile;
grib_file* newfile;
grib_index_key* index_key=NULL;
grib_handle* h=NULL;
grib_field* field;
grib_field_tree* field_tree;
grib_file* file=NULL;
grib_context* c;
if (!index) return GRIB_NULL_INDEX;
c=index->context;
file=grib_file_open(filename,"r",&err);
if (!file || !file->handle) return err;
if (!index->files) {
grib_filesid++;
newfile=(grib_file*)grib_context_malloc_clear(c,sizeof(grib_file));
newfile->id=grib_filesid;
newfile->name=strdup(file->name);
newfile->handle = file->handle;
index->files=newfile;
} else {
indfile=index->files;
while(indfile) {
if (!strcmp(indfile->name,file->name)) return 0;
indfile=indfile->next;
}
indfile=index->files;
while(indfile->next) indfile=indfile->next;
grib_filesid++;
newfile=(grib_file*)grib_context_malloc_clear(c,sizeof(grib_file));
newfile->id=grib_filesid;
newfile->name=strdup(file->name);
newfile->handle = file->handle;
indfile->next=newfile;
}
fseeko(file->handle,0,SEEK_SET);
while ((h=new_message_from_file(message_type, c, file->handle, &err))!=NULL) {
grib_string_list* v=0;
index_key=index->keys;
field_tree=index->fields;
index_key->value[0]=0;
message_count++;
while (index_key) {
if (index_key->type==GRIB_TYPE_UNDEFINED) {
err=grib_get_native_type(h,index_key->name,&(index_key->type));
if (err) index_key->type=GRIB_TYPE_STRING;
}
svallen=1024;
switch (index_key->type) {
case GRIB_TYPE_STRING:
err=grib_get_string(h,index_key->name,buf,&svallen);
if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
break;
case GRIB_TYPE_LONG:
err=grib_get_long(h,index_key->name,&lval);
if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
else sprintf(buf,"%ld",lval);
break;
case GRIB_TYPE_DOUBLE:
err=grib_get_double(h,index_key->name,&dval);
if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
else sprintf(buf,"%g",dval);
break;
default :
err=GRIB_WRONG_TYPE;
return err;
}
if (err && err != GRIB_NOT_FOUND) {
grib_context_log(c,GRIB_LOG_ERROR,"unable to create index. \"%s\": %s",index_key->name,grib_get_error_message(err));
return err;
}
if (!index_key->values->value) {
index_key->values->value=grib_context_strdup(c,buf);
index_key->values_count++;
} else {
v=index_key->values;
while (v->next && strcmp(v->value,buf)) v=v->next;
if (strcmp(v->value,buf)) {
index_key->values_count++;
if (v->next) v=v->next;
v->next=(grib_string_list*)grib_context_malloc_clear(c,sizeof(grib_string_list));
v->next->value=grib_context_strdup(c,buf);
}
}
if (!field_tree->value) {
field_tree->value=grib_context_strdup(c,buf);
} else {
while (field_tree->next &&
(field_tree->value==NULL ||
strcmp(field_tree->value,buf)))
field_tree=field_tree->next;
if (!field_tree->value || strcmp(field_tree->value,buf)){
field_tree->next=
(grib_field_tree*)grib_context_malloc_clear(c,
sizeof(grib_field_tree));
field_tree=field_tree->next;
field_tree->value=grib_context_strdup(c,buf);
}
}
if (index_key->next) {
if (!field_tree->next_level) {
field_tree->next_level=
(grib_field_tree*)grib_context_malloc_clear(c,sizeof(grib_field_tree));
}
field_tree=field_tree->next_level;
}
index_key=index_key->next;
}
field=(grib_field*)grib_context_malloc_clear(c,sizeof(grib_field));
field->file=file;
index->count++;
field->offset=h->offset;;
err=grib_get_long(h,"totalLength",&length);
if (err) return err;
field->length=length;
if (field_tree->field) {
grib_field* pfield=field_tree->field;
while (pfield->next) pfield=pfield->next;
pfield->next=field;
} else
field_tree->field=field;
grib_handle_delete(h);
}
grib_file_close(file->name, 0, &err);
if (err) return err;
index->rewind=1;
if (message_count == 0) {
grib_context_log(c,GRIB_LOG_ERROR,"File %s contains no messages", filename);
return GRIB_END_OF_FILE;
}
return GRIB_SUCCESS;
}
#if 0
int grib_index_add_file(grib_index* index, const char* filename)
{
double dval;
size_t svallen;
long length,lval;
char buf[1024]={0,};
int err=0;
grib_file* indfile;
grib_file* newfile;
grib_index_key* index_key=NULL;
grib_handle* h=NULL;
grib_field* field;
grib_field_tree* field_tree;
grib_file* file=NULL;
grib_context* c;
if (!index) return GRIB_NULL_INDEX;
c=index->context;
file=grib_file_open(filename,"r",&err);
if (!file || !file->handle) return err;
if (!index->files) {
grib_filesid++;
newfile=(grib_file*)grib_context_malloc_clear(c,sizeof(grib_file));
newfile->id=grib_filesid;
newfile->name=strdup(file->name);
newfile->handle = file->handle;
index->files=newfile;
} else {
indfile=index->files;
while(indfile) {
if (!strcmp(indfile->name,file->name)) return 0;
indfile=indfile->next;
}
indfile=index->files;
while(indfile->next) indfile=indfile->next;
grib_filesid++;
newfile=(grib_file*)grib_context_malloc_clear(c,sizeof(grib_file));
newfile->id=grib_filesid;
newfile->name=strdup(file->name);
newfile->handle = file->handle;
indfile->next=newfile;
}
fseeko(file->handle,0,SEEK_SET);
while ((h=grib_handle_new_from_file(c,file->handle,&err))!=NULL) {
grib_string_list* v=0;
index_key=index->keys;
field_tree=index->fields;
index_key->value[0]=0;
/* process only GRIB for the moment*/
svallen=1024;
grib_get_string(h,"identifier",buf,&svallen);
if (strcmp(buf,"GRIB")) {
grib_handle_delete(h);
return 0;
}
while (index_key) {
if (index_key->type==GRIB_TYPE_UNDEFINED) {
err=grib_get_native_type(h,index_key->name,&(index_key->type));
if (err) index_key->type=GRIB_TYPE_STRING;
}
svallen=1024;
switch (index_key->type) {
case GRIB_TYPE_STRING:
err=grib_get_string(h,index_key->name,buf,&svallen);
if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
break;
case GRIB_TYPE_LONG:
err=grib_get_long(h,index_key->name,&lval);
if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
else sprintf(buf,"%ld",lval);
break;
case GRIB_TYPE_DOUBLE:
err=grib_get_double(h,index_key->name,&dval);
if (err==GRIB_NOT_FOUND) sprintf(buf,GRIB_KEY_UNDEF);
else sprintf(buf,"%g",dval);
break;
default :
err=GRIB_WRONG_TYPE;
return err;
}
if (err && err != GRIB_NOT_FOUND) {
grib_context_log(c,GRIB_LOG_ERROR,"unable to create index. \"%s\": %s",index_key->name,grib_get_error_message(err));
return err;
}
if (!index_key->values->value) {
index_key->values->value=grib_context_strdup(c,buf);
index_key->values_count++;
} else {
v=index_key->values;
while (v->next && strcmp(v->value,buf)) v=v->next;
if (strcmp(v->value,buf)) {
index_key->values_count++;
if (v->next) v=v->next;
v->next=(grib_string_list*)grib_context_malloc_clear(c,sizeof(grib_string_list));
v->next->value=grib_context_strdup(c,buf);
}
}
if (!field_tree->value) {
field_tree->value=grib_context_strdup(c,buf);
} else {
while (field_tree->next &&
(field_tree->value==NULL ||
strcmp(field_tree->value,buf)))
field_tree=field_tree->next;
if (!field_tree->value || strcmp(field_tree->value,buf)){
field_tree->next=
(grib_field_tree*)grib_context_malloc_clear(c,
sizeof(grib_field_tree));
field_tree=field_tree->next;
field_tree->value=grib_context_strdup(c,buf);
}
}
if (index_key->next) {
if (!field_tree->next_level) {
field_tree->next_level=
(grib_field_tree*)grib_context_malloc_clear(c,sizeof(grib_field_tree));
}
field_tree=field_tree->next_level;
}
index_key=index_key->next;
}
field=(grib_field*)grib_context_malloc_clear(c,sizeof(grib_field));
field->file=file;
index->count++;
field->offset=h->offset;;
err=grib_get_long(h,"totalLength",&length);
if (err) return err;
field->length=length;
if (field_tree->field) {
grib_field* pfield=field_tree->field;
while (pfield->next) pfield=pfield->next;
pfield->next=field;
} else
field_tree->field=field;
if (h) grib_handle_delete(h);
}
grib_file_close(file->name, 0, &err);
if (err) return err;
index->rewind=1;
return GRIB_SUCCESS;
}
#endif
grib_index* grib_index_new_from_file(grib_context* c, char* filename, const char* keys, int *err)
{
grib_index* index=NULL;
if(!c) c=grib_context_get_default();
index=grib_index_new(c,keys,err);
*err=grib_index_add_file(index,filename);
if (*err) {
grib_index_delete(index);
return NULL;
}
return index;
}
int grib_index_get_size(grib_index* index,const char* key,size_t* size)
{
grib_index_key* k=index->keys;
while (k && strcmp(k->name,key)) k=k->next;
if (!k) return GRIB_NOT_FOUND;
*size=k->values_count;
return 0;
}
int grib_index_get_string(grib_index* index, const char* key, char** values, size_t *size)
{
grib_index_key* k=index->keys;
grib_string_list* kv;
int i=0;
while (k && strcmp(k->name,key)) k=k->next;
if (!k) return GRIB_NOT_FOUND;
if (k->values_count>*size) return GRIB_ARRAY_TOO_SMALL;
kv=k->values;
while (kv) {
if (kv->value == NULL)
return GRIB_IO_PROBLEM;
values[i++]=grib_context_strdup(index->context,kv->value);
kv=kv->next;
}
*size=k->values_count;
qsort(values,*size,sizeof(char*),&compare_string);
return GRIB_SUCCESS;
}
int grib_index_get_long(grib_index* index, const char* key, long* values, size_t *size)
{
grib_index_key* k=index->keys;
grib_string_list* kv;
int i=0;
while (k && strcmp(k->name,key)) k=k->next;
if (!k) return GRIB_NOT_FOUND;
if (k->type != GRIB_TYPE_LONG) {
grib_context_log(index->context,GRIB_LOG_ERROR, "unable to get index %s as long", key);
return GRIB_WRONG_TYPE;
}
if (k->values_count > *size) return GRIB_ARRAY_TOO_SMALL;
kv=k->values;
while (kv) {
if (strcmp(kv->value,GRIB_KEY_UNDEF) )
values[i++]=atol(kv->value);
else
values[i++]=UNDEF_LONG;
kv=kv->next;
}
*size=k->values_count;
qsort(values,*size,sizeof(long),&compare_long);
return GRIB_SUCCESS;
}
int grib_index_get_double(grib_index* index,const char* key, double* values,size_t *size)
{
grib_index_key* k=index->keys;
grib_string_list* kv;
int i=0;
while (k && strcmp(k->name,key)) k=k->next;
if (!k) return GRIB_NOT_FOUND;
if (k->type != GRIB_TYPE_DOUBLE) {
grib_context_log(index->context,GRIB_LOG_ERROR, "unable to get index %s as double", key);
return GRIB_WRONG_TYPE;
}
if (k->values_count>*size) return GRIB_ARRAY_TOO_SMALL;
kv=k->values;
while (kv) {
if (strcmp(kv->value,GRIB_KEY_UNDEF) )
values[i++]=atof(kv->value);
else
values[i++]=UNDEF_DOUBLE;
kv=kv->next;
}
*size=k->values_count;
qsort(values,*size,sizeof(double),&compare_double);
return GRIB_SUCCESS;
}
int grib_index_select_long(grib_index* index,const char* skey,long value)
{
grib_index_key* key=NULL;
int err=GRIB_NOT_FOUND;
if (!index) {
grib_context* c=grib_context_get_default();
grib_context_log(c,GRIB_LOG_ERROR,"null index pointer");
return GRIB_INTERNAL_ERROR;
}
index->orderby=0;
key=index->keys;
while (key) {
if (!strcmp(key->name,skey)) {
err=0;
break;
}
key=key->next;
}
if (err) {
grib_context_log(index->context,GRIB_LOG_ERROR,
"key \"%s\" not found in index",skey);
return err;
}
Assert(key);
sprintf(key->value,"%ld",value);
grib_index_rewind(index);
return 0;
}
int grib_index_select_double(grib_index* index,const char* skey,double value)
{
grib_index_key* key=NULL;
int err=GRIB_NOT_FOUND;
if (!index) {
grib_context* c=grib_context_get_default();
grib_context_log(c,GRIB_LOG_ERROR,"null index pointer");
return GRIB_INTERNAL_ERROR;
}
index->orderby=0;
key=index->keys;
while (key ) {
if (!strcmp(key->name,skey)) {
err=0;
break;
}
key=key->next;
}
if (err) {
grib_context_log(index->context,GRIB_LOG_ERROR,
"key \"%s\" not found in index",skey);
return err;
}
Assert(key);
sprintf(key->value,"%g",value);
grib_index_rewind(index);
return 0;
}
int grib_index_select_string(grib_index* index,const char* skey,char* value)
{
grib_index_key* key=NULL;
int err=GRIB_NOT_FOUND;
if (!index) {
grib_context* c=grib_context_get_default();
grib_context_log(c,GRIB_LOG_ERROR,"null index pointer");
return GRIB_INTERNAL_ERROR;
}
index->orderby=0;
key=index->keys;
while (key ) {
if (!strcmp(key->name,skey)) {
err=0;
break;
}
key=key->next;
}
if (err) {
grib_context_log(index->context,GRIB_LOG_ERROR,
"key \"%s\" not found in index",skey);
return err;
}
Assert(key);
sprintf(key->value,"%s",value);
grib_index_rewind(index);
return 0;
}
grib_handle* grib_index_get_handle(grib_field* field,int *err)
{
return codes_index_get_handle(field,CODES_GRIB,err);
}
grib_handle* codes_index_get_handle(grib_field* field,int message_type,int *err)
{
grib_handle* h=NULL;
typedef grib_handle* (*message_new_proc) ( grib_context*,FILE*,int,int* );
message_new_proc message_new=NULL;
Assert(field->file);
grib_file_open(field->file->name,"r",err);
if (*err!=GRIB_SUCCESS) return NULL;
switch (message_type) {
case CODES_GRIB:
message_new=grib_new_from_file;
break;
case CODES_BUFR:
Assert(!"_codes_index_add_file for BUFR: not yet implemented");
/* message_new=bufr_new_from_file; */
break;
default :
Assert(0);
}
fseeko(field->file->handle,field->offset,SEEK_SET);
h=message_new(0,field->file->handle,0,err);
if (*err!=GRIB_SUCCESS) return NULL;
grib_file_close(field->file->name, 0, err);
return h;
}
static int grib_index_execute(grib_index* index)
{
grib_index_key* keys = NULL;
grib_field_tree* fields;
if (!index) return GRIB_INTERNAL_ERROR;
keys = index->keys;
fields=index->fields;
index->rewind=0;
while (keys) {
char* value;
if (keys->value[0]) value=keys->value;
else {
grib_context_log(index->context,GRIB_LOG_ERROR,
"please select a value for index key \"%s\"",
keys->name);
return GRIB_NOT_FOUND;
}
while (fields && strcmp(fields->value,value))
fields=fields->next;
if (fields && !strcmp(fields->value,value)) {
if (fields->next_level) {
keys=keys->next;
fields=fields->next_level;
} else {
index->current=index->fieldset;
while(index->current->next) index->current=index->current->next;
index->current->field=fields->field;
return 0;
}
} else return GRIB_END_OF_INDEX;
}
return 0;
}
static void grib_dump_key_values(FILE* fout, grib_string_list* values)
{
grib_string_list* sl = values;
int first = 1; /* boolean for commas */
fprintf(fout, "values = ");
while (sl) {
if (!first) {
fprintf(fout, ", ");
}
fprintf(fout, "%s", sl->value);
first = 0;
sl = sl->next;
}
fprintf(fout, "\n");
}
static void grib_dump_index_keys(FILE* fout, grib_index_key* keys)
{
if (!keys) return;
fprintf(fout, "key name = %s\n", keys->name);
/* fprintf(fout, "key type = %d\n", keys->type); */
grib_dump_key_values(fout, keys->values);
grib_dump_index_keys(fout, keys->next);
}
#if 0
static void grib_dump_files(FILE* fout, grib_file* files)
{
if (!files) return;
fprintf(fout, "file = %s\n", files->name);
fprintf(fout, "ID = %d\n", files->id);
grib_dump_files(fout, files->next);
}
static void grib_dump_field(FILE* fout, grib_field* field)
{
if (!field) return;
fprintf(fout, "field name = %s\n", field->file->name);
/*fprintf(fout, "field FID = %d\n", field->file->id);
* fprintf(fout, "field offset = %ld\n", field->offset);
* fprintf(fout, "field length = %ld\n", field->length);
*/
grib_dump_field(fout, field->next);
}
static void grib_dump_field_tree(FILE* fout, grib_field_tree* tree)
{
if(!tree) return;
grib_dump_field(fout,tree->field);
fprintf(fout, "tree value = %s\n", tree->value);
grib_dump_field_tree(fout,tree->next_level);
grib_dump_field_tree(fout,tree->next);
}
#endif
int grib_index_dump_file(FILE* fout, const char* filename)
{
int err = 0;
grib_index* index = NULL;
grib_context* c = grib_context_get_default();
FILE* fh = NULL;
Assert(fout);
Assert(filename);
index = grib_index_read(c, filename, &err);
if (err) return err;
/* To get the GRIB files referenced we have */
/* to resort to low level reading of the index file! */
fh=fopen(filename,"r");
if (fh) {
grib_file *file,*f;
char* identifier=NULL;
unsigned char marker=0;
identifier = grib_read_string(c,fh,&err);
if (err) return err;
grib_context_free(c,identifier);
err = grib_read_uchar(fh,&marker);
if (err) return err;
file = grib_read_files(c,fh,&err);
if (err) return err;
f=file;
while (f) {
fprintf(fout, "GRIB File: %s\n", f->name);
f=f->next;
}
fclose(fh);
}
grib_index_dump(fout, index);
grib_index_delete(index);
return GRIB_SUCCESS;
}
void grib_index_dump(FILE* fout, grib_index* index)
{
if (!index) return;
Assert(fout);
/* The grib_dump_files does not print anything as */
/* the index object does not store the file names! */
/* grib_dump_files(fout, index->files); */
fprintf(fout, "Index keys:\n");
grib_dump_index_keys(fout, index->keys);
/*
* fprintf(fout, "Index field tree:\n");
* grib_dump_field_tree(fout, index->fields);
*/
fprintf(fout, "Index count = %d\n", index->count);
}
char* grib_get_field_file(grib_index* index,off_t *offset)
{
char* file=NULL;
if (index && index->current && index->current->field) {
file=index->current->field->file->name;
*offset=index->current->field->offset;
}
return file;
}
grib_handle* grib_handle_new_from_index(grib_index* index,int *err)
{
return codes_new_from_index(index,CODES_GRIB,err);
}
grib_handle* codes_new_from_index(grib_index* index,int message_type,int *err)
{
/*grib_index_key* keys;*/
grib_field_list *fieldset,*next;
grib_handle* h=NULL;
grib_context* c=NULL;
if (!index) return NULL;
c=index->context;
if (!index->rewind) {
if (!index->current) {
*err=GRIB_END_OF_INDEX;
return NULL;
}
if (index->current->field->next)
index->current->field=index->current->field->next;
else if(index->current->next)
index->current=index->current->next;
else {*err=GRIB_END_OF_INDEX;return NULL;}
h=codes_index_get_handle(index->current->field,message_type,err);
return h;
}
if (!index->fieldset) {
index->fieldset=(grib_field_list*)grib_context_malloc_clear(index->context,
sizeof(grib_field_list));
if (!index->fieldset) {
grib_context_log(index->context,GRIB_LOG_ERROR,
"unable to allocate %d bytes",
sizeof(grib_field_list));
return NULL;
}
index->current=index->fieldset;
} else {
fieldset=index->fieldset;
while(fieldset->next) {
next=fieldset->next;
grib_context_free(c,fieldset);
fieldset=next;
}
fieldset->field=NULL;
fieldset->next=NULL;
index->fieldset=fieldset;
index->current=fieldset;
}
*err=GRIB_END_OF_INDEX;
h=NULL;
/*keys=index->keys;*/
if ((*err=grib_index_execute(index))==GRIB_SUCCESS) {
if (!index->fieldset) {*err=GRIB_END_OF_INDEX;return NULL;}
index->current=index->fieldset;
h=codes_index_get_handle(index->current->field,message_type,err);
}
return h;
}
void grib_index_rewind(grib_index* index)
{
index->rewind=1;
}
static grib_index_key* search_key(grib_index_key* keys,grib_index_key* to_search)
{
if (!keys || !strcmp(keys->name,to_search->name)) return keys;
return search_key(keys->next,to_search);
}
int grib_index_search(grib_index* index,grib_index_key* keys)
{
grib_index_key* ki=index->keys;
grib_index_key* ks=keys;
while (ks) {
ki=search_key(ki,ks);
if (!ki) {
ki=index->keys;
ki=search_key(ki,ks);
}
if (ki) sprintf(ki->value,"%s",ks->value);
ks=ks->next;
}
grib_index_rewind(index);
return 0;
}
Reference in New Issue View Git Blame Copy Permalink