core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Tidy up use of memfs

This commit is contained in:
Baudouin Raoult 2016-06-19 09:22:15 +01:00
parent 8896007a30
commit 3330f4c634
16 changed files with 87 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
memfs.py
View File

@ -178,7 +178,7 @@ FILE* codes_memfs_open(const char* path) {
size_t size;
const unsigned char* mem = find(path, &size);
if(!mem) {
return fopen(path, "r");
return NULL;
}
return fmemopen((void*)mem, size, "r");
}

1
src/CMakeLists.txt
View File

@ -55,6 +55,7 @@ list( APPEND grib_api_srcs
grib_vsarray.c
grib_iarray.c
grib_viarray.c
codes_memfs.c
grib_accessor_class_array.c
grib_accessor_class_assert.c
grib_accessor_class_ascii.c

55
src/codes_memfs.c Normal file
View File

@ -0,0 +1,55 @@
/*
* Copyright 2005-2016 ECMWF.
*
* This software is licensed under the terms of the Apache Licence Version 2.0
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
*
* In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
* virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
*/
#include "grib_api_internal.h"
#ifdef HAVE_MEMFS
int codes_memfs_exists(const char* path);
FILE* codes_memfs_open(const char* path);
FILE* codes_fopen(const char* name, const char *mode) {
FILE *f;
if (strcmp(mode, "r") != 0) {
return fopen(name, mode);
}
f = codes_memfs_open(name);
if (f) {
return f;
}
return fopen(name, mode);
}
int codes_access(const char* name, int mode) {
if (mode != F_OK) {
return access(name, mode);
}
if (codes_memfs_exists(name)) {
return 0;
}
return access(name, mode);
}
#else
FILE* codes_fopen(const char* name, const char* mode) {
return fopen(name, mode);
}
int codes_access(const char* name, int mode) {
return access(name, mode);
}
#endif

4
src/grib_accessor_class_bufr_elements_table.c
View File

@ -260,7 +260,7 @@ static grib_trie* load_bufr_elements_table(grib_accessor* a, int* err)
grib_context_log(c,GRIB_LOG_DEBUG,"using dictionary %s from file %s",self->dictionary,filename);
}
f=fopen(filename,"r");
f=codes_fopen(filename,"r");
if (!f) {*err=GRIB_IO_PROBLEM; return NULL;}
dictionary=grib_trie_new(c);
@ -273,7 +273,7 @@ static grib_trie* load_bufr_elements_table(grib_accessor* a, int* err)
fclose(f);
if (localFilename!=0) {
f=fopen(localFilename,"r");
f=codes_fopen(localFilename,"r");
if (!f) {*err=GRIB_IO_PROBLEM; return NULL;}
while(fgets(line,sizeof(line)-1,f)) {

7
src/grib_accessor_class_codeflag.c
View File

@ -170,12 +170,7 @@ static int grib_get_codeflag(grib_accessor* a, long code, char* codename)
return GRIB_FILE_NOT_FOUND;
}
#ifdef HAVE_MEMFS
f = codes_memfs_open(filename);
#else
f=fopen(filename, "r");
#endif
f = codes_fopen(filename, "r");
if (!f)
{
grib_context_log(a->context,(GRIB_LOG_WARNING)|(GRIB_LOG_PERROR),"Cannot open flag table %s",filename);

6
src/grib_accessor_class_codetable.c
View File

@ -375,11 +375,7 @@ static int grib_load_codetable(grib_context* c,const char* filename,
int lineNumber = 0;
grib_context_log(c,GRIB_LOG_DEBUG,"Loading code table from %s",filename);
#ifdef HAVE_MEMFS
f = codes_memfs_open(filename);
#else
f=fopen(filename, "r");
#endif
f = codes_fopen(filename, "r");
if (!f) return GRIB_IO_PROBLEM;
Assert(t!=NULL);

4
src/grib_accessor_class_dictionary.c
View File

@ -220,7 +220,7 @@ static grib_trie* load_dictionary(grib_context* c,grib_accessor* a, int* err)
grib_context_log(c,GRIB_LOG_DEBUG,"using dictionary %s from file %s",self->dictionary,filename);
}
f=fopen(filename,"r");
f=codes_fopen(filename,"r");
if (!f) {*err=GRIB_IO_PROBLEM; return NULL;}
dictionary=grib_trie_new(c);
@ -240,7 +240,7 @@ static grib_trie* load_dictionary(grib_context* c,grib_accessor* a, int* err)
fclose(f);
if (localFilename!=0) {
f=fopen(localFilename,"r");
f=codes_fopen(localFilename,"r");
if (!f) {*err=GRIB_IO_PROBLEM; return NULL;}
while(fgets(line,sizeof(line)-1,f)) {

4
src/grib_accessor_class_smart_table.c
View File

@ -189,7 +189,7 @@ static void thread_init()
#endif
static int grib_load_smart_table(grib_context* c,const char* filename,
const char* recomposed_name,size_t size,grib_smart_table* t);
const char* recomposed_name,size_t size,grib_smart_table* t);
static void init(grib_accessor* a, const long len, grib_arguments* params) {
int n=0;
@ -311,7 +311,7 @@ static int grib_load_smart_table(grib_context* c,const char* filename,
grib_context_log(c,GRIB_LOG_DEBUG,"Loading code table form %s",filename);
f=fopen(filename, "r");
f=codes_fopen(filename, "r");
if (!f) return GRIB_IO_PROBLEM;
Assert(t!=NULL);

5
src/grib_api_prototypes.h
View File

@ -1449,3 +1449,8 @@ int grib_decode_double_array_complex(const unsigned char *p, long *bitp, long nb
int grib_encode_long_array(size_t n_vals, const long *val, long bits_per_value, unsigned char *p, long *off);
int grib_encode_double_array(size_t n_vals, const double *val, long bits_per_value, double reference_value, double d, double divisor, unsigned char *p, long *off);
int grib_encode_double_array_complex(size_t n_vals, double *val, long nbits, double reference_value, double *scal, double d, double divisor, unsigned char *p, long *bitp);
/* codes_memfs */
FILE* codes_fopen(const char* name, const char* mode);
int codes_access(const char* name, int mode);

19
src/grib_context.c
View File

@ -629,11 +629,7 @@ char *grib_context_full_defs_path(grib_context* c,const char* basename)
while (dir) {
sprintf(full,"%s/%s",dir->value,basename);
#ifdef HAVE_MEMFS
if(access(full,F_OK) == 0 || codes_memfs_exists(full)) {
#else
if (!access(full,F_OK)) {
#endif
if (!codes_access(full,F_OK)) {
fullpath=(grib_string_list*)grib_context_malloc_clear_persistent(c,sizeof(grib_string_list));
Assert(fullpath);
fullpath->value=grib_context_strdup(c,full);
@ -645,19 +641,6 @@ char *grib_context_full_defs_path(grib_context* c,const char* basename)
}
dir=dir->next;
}
// #ifdef HAVE_MEMFS
// sprintf(full,"/definitions/%s",basename);
// if(codes_memfs_exists(full)) {
// fullpath=(grib_string_list*)grib_context_malloc_clear_persistent(c,sizeof(grib_string_list));
// Assert(fullpath);
// fullpath->value=grib_context_strdup(c,full);
// GRIB_MUTEX_LOCK(&mutex_c);
// grib_trie_insert(c->def_files,basename,fullpath);
// grib_context_log(c,GRIB_LOG_DEBUG,"Found def file %s in MEMFS",full);
// GRIB_MUTEX_UNLOCK(&mutex_c);
// return fullpath->value;
// }
// #endif
}
GRIB_MUTEX_LOCK(&mutex_c);

8
src/grib_expression_class_is_in_dict.c
View File

@ -72,9 +72,9 @@ static grib_expression_class _grib_expression_class_is_in_dict = {
&init_class, /* init_class */
0, /* constructor */
0, /* destructor */
&print,
&compile,
&add_dependency,
&print,
&compile,
&add_dependency,
&native_type,
&get_name,
@ -123,7 +123,7 @@ static grib_trie* load_dictionary(grib_context* c,grib_expression* e, int* err)
grib_context_log(c,GRIB_LOG_DEBUG,"using dictionary %s from file %s",self->dictionary,filename);
}
f=fopen(filename,"r");
f=codes_fopen(filename,"r");
if (!f) {*err=GRIB_IO_PROBLEM; return NULL;}
dictionary=grib_trie_new(c);

8
src/grib_expression_class_is_in_list.c
View File

@ -74,9 +74,9 @@ static grib_expression_class _grib_expression_class_is_in_list = {
&init_class, /* init_class */
0, /* constructor */
&destroy, /* destructor */
&print,
&compile,
&add_dependency,
&print,
&compile,
&add_dependency,
&native_type,
&get_name,
@ -122,7 +122,7 @@ static grib_trie* load_list(grib_context* c,grib_expression* e, int* err) {
grib_context_log(c,GRIB_LOG_DEBUG,"using list %s from file %s",self->list,filename);
}
f=fopen(filename,"r");
f=codes_fopen(filename,"r");
if (!f) {*err=GRIB_IO_PROBLEM; return NULL;}
list=grib_trie_new(c);

6
src/grib_parse_utils.c
View File

@ -604,11 +604,7 @@ void grib_parser_include(const char* included_fname)
f = stdin; /* read from std input */
} else {
grib_context_log(grib_parser_context,GRIB_LOG_DEBUG,"parsing include file %s",parse_file);
#ifdef HAVE_MEMFS
f = codes_memfs_open(parse_file);
#else
f = fopen(parse_file,"r");
#endif
f = codes_fopen(parse_file,"r");
}
/* for(i = 0; i < top ; i++) printf(" "); */
/* printf("PARSING %s\n",parse_file); */

26
src/grib_templates.c
View File

@ -40,7 +40,7 @@ grib_handle* grib_internal_template(grib_context* c,const char* name)
}
#endif
static grib_handle* try_template(grib_context* c,const char* dir,const char* name, int last)
static grib_handle* try_template(grib_context* c,const char* dir,const char* name)
{
char path[1024];
grib_handle *g = NULL;
@ -52,23 +52,9 @@ static grib_handle* try_template(grib_context* c,const char* dir,const char* nam
printf("ECCODES DEBUG: try_template path='%s'\n", path);
}
#ifdef HAVE_MEMFS
if(last && codes_memfs_exists(path)) {
FILE* f = codes_memfs_open(path);
if(f) {
g = grib_handle_new_from_file(c,f,&err);
if (!g) {
grib_context_log(c,GRIB_LOG_ERROR,"cannot create GRIB handle from %s",path);
}
fclose(f);
return g;
}
}
#endif
if(access(path,F_OK) == 0)
if(codes_access(path,F_OK) == 0)
{
FILE* f = fopen(path,"r");
FILE* f = codes_fopen(path,"r");
if(!f)
{
grib_context_log(c,GRIB_LOG_PERROR,"cannot open %s",path);
@ -90,7 +76,7 @@ static char* try_template_path(grib_context* c,const char* dir,const char* name)
sprintf(path,"%s/%s.tmpl",dir,name);
if(access(path,R_OK) == 0)
if(codes_access(path,R_OK) == 0)
{
return grib_context_strdup(c,path);
}
@ -112,7 +98,7 @@ grib_handle* grib_external_template(grib_context* c,const char* name)
if(*base == ':')
{
*p = 0;
g = try_template(c,buffer,name, 0);
g = try_template(c,buffer,name);
if(g) return g;
p = buffer;
base++; /*advance past delimiter*/
@ -121,7 +107,7 @@ grib_handle* grib_external_template(grib_context* c,const char* name)
}
*p = 0;
return g = try_template(c,buffer,name, 1);
return g = try_template(c,buffer,name);
}
char* grib_external_template_path(grib_context* c,const char* name)

2
src/grib_util.c
View File

@ -297,7 +297,7 @@ static grib_trie* init_list(const char* name)
grib_context* c=grib_context_get_default();
full_path=grib_context_full_defs_path(c,name);
fh=fopen(full_path,"r");
fh=codes_fopen(full_path,"r");
if (!fh) {
grib_context_log(c,GRIB_LOG_PERROR,"unable to read %s",full_path);
return NULL;

137
src/hash.c
View File

@ -1,137 +0,0 @@
/*
* Copyright 2005-2016 ECMWF.
*
* This software is licensed under the terms of the Apache Licence Version 2.0
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
*
* In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
* virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
*/
#include <stdio.h>
#include "mars.h"
#define SIZE ((int)4097)
typedef struct hashrec {
struct hashrec *next;
char *name;
int cnt;
} hashrec;
static hashrec *table[SIZE] = {
NULL, };
static int hash(const char *name)
{
int n = 0;
while(*name)
n += (*name++ - 'A') + (n << 5);
#ifdef CRAY
if(n<0) n = -n;
#else
if(n<0) {
int m = -n/SIZE;
n += (m+1)*SIZE;
}
#endif
return n % SIZE;
}
void hash_stat()
{
int i;
int a=0,b=0,c=0;
hashrec *h;
printf("Table size is %d. ",SIZE);
for(i=0;i<SIZE;i++)
if(h = table[i])
{
a++;
while(h) { b++; c += h->cnt; h = h->next; };
}
printf("Used: %d, Total: %d, Strings: %d\n",a,b,c);
}
char *strcache(const char *name)
{
hashrec *h;
int n;
if(name == NULL) return NULL;
n = hash(name);
h = table[n];
while(h)
{
if(EQ(h->name,name))
{
h->cnt++;
return h->name;
}
h = h->next;
}
h = NEW(hashrec);
h->name = NEW_STRING(name);
h->next = table[n];
h->cnt = 1;
table[n] = h;
return h->name;
}
void strfree(char *name)
{
hashrec *h;
hashrec *p = NULL;
int n;
if(name == NULL) return;
n = hash(name);
h = table[n];
while(h)
{
if(h->name == name)
{
h->cnt--;
break;
}
p = h;
h = h->next;
}
if(h == NULL)
{
marslog(LOG_WARN,"%s was not in hash table",name);
abort();
printf("n=%d\n",n);
h = table[n];
while(h)
{
printf("%s %d\n",h->name,h->cnt);
h = h->next;
}
return;
}
if(h->cnt == 0)
{
if(p) p->next = h->next;
else table[n] = h->next;
FREE(h->name);
FREE(h);
}
}