core
/
eccodes
mirror of https://github.com/ecmwf/eccodes.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

EFAS development: (ECC-701) latest MARS implementation

This commit is contained in:
Shahram Najm 2018-07-13 12:49:34 +01:00
parent c31ca69534 cc04daf388
commit e165fb9bda
33 changed files with 112 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
definitions/Makefile.am
View File

@ -16306,12 +16306,12 @@ dist_definitionsmars_DATA = \
mars/grib.edmo.cl.def\
mars/grib.edmo.fc.def\
mars/grib.edmo.ssd.def\
mars/grib.efas.an.def\
mars/grib.efas.cf.def\
mars/grib.efas.fc.def\
mars/grib.efas.fu.def\
mars/grib.efas.go.def\
mars/grib.efas.pf.def\
mars/grib.efas.sfo.def\
mars/grib.efhc.cf.def\
mars/grib.efhc.icp.def\
mars/grib.efhc.pf.def\

1
definitions/grib2/local.98.41.def
View File

@ -43,6 +43,7 @@ unsigned[2] anoffsetFrequency = missing(): dump, can_be_missing;
# Boolean
transient is_efas = 1;
transient lsdate_bug = 1: hidden; # See ECC-707
# Note: the key typeOfPostProcessing is in the PDTNs 70, 71, 72 and 73
concept efas_post_proc {

19
definitions/mars/grib.efas.an.def
View File

@ -1,19 +0,0 @@
# EFAS Analysis (Water Balance)
alias mars.step = startStep;
# See ECC-683
#alias mars.date = dateOfForecast;
#alias mars.time = timeOfForecast;
meta valDate validity_date(dataDate,dataTime,endStep,stepUnits): no_copy;
meta valTime validity_time(dataDate,dataTime,endStep,stepUnits): no_copy;
alias mars.date = valDate;
alias mars.time = valTime;
#meta efas_suite_name sprintf("%s_%s_%s", inputOriginatingCentre, efas_forecast, efas_post_proc) : no_copy;
alias mars.origin = inputOriginatingCentre;
alias mars.anoffset=anoffset;
# We need this because 'efas_post_proc' is defined later
meta efas_model sprintf("%s", efas_post_proc) : no_copy;
alias mars.model = efas_model;

6
definitions/mars/grib.efas.fu.def
View File

@ -1,5 +1,7 @@
# EFAS Fullup
alias mars.step = startStep;
# EFAS Fill-up
# ECC-701
alias mars.step = endStep;
alias mars.date = dateOfForecast;
alias mars.time = timeOfForecast;

17
definitions/mars/grib.efas.go.def
View File

@ -1,13 +1,14 @@
# EFAS Gridded observations
alias mars.step = startStep;
# See ECC-683
#alias mars.date = dateOfForecast;
#alias mars.time = timeOfForecast;
meta valDate validity_date(dataDate,dataTime,endStep,stepUnits): no_copy;
meta valTime validity_time(dataDate,dataTime,endStep,stepUnits): no_copy;
alias mars.date = valDate;
alias mars.time = valTime;
# ECC-701
alias mars.step = endStep;
alias mars.date = dateOfForecast;
alias mars.time = timeOfForecast;
#meta valDate validity_date(dataDate,dataTime,endStep,stepUnits): no_copy;
#meta valTime validity_time(dataDate,dataTime,endStep,stepUnits): no_copy;
#alias mars.date = valDate;
#alias mars.time = valTime;
alias mars.origin = inputOriginatingCentre;

21
definitions/mars/grib.efas.sfo.def Normal file
View File

@ -0,0 +1,21 @@
# Simulation forced with observations (EFAS)
# Water Balance
# ECC-701
alias mars.step = endStep;
alias mars.date = dateOfForecast;
alias mars.time = timeOfForecast;
#meta valDate validity_date(dataDate,dataTime,endStep,stepUnits): no_copy;
#meta valTime validity_time(dataDate,dataTime,endStep,stepUnits): no_copy;
#alias mars.date = valDate;
#alias mars.time = valTime;
#meta efas_suite_name sprintf("%s_%s_%s", inputOriginatingCentre, efas_forecast, efas_post_proc) : no_copy;
alias mars.origin = inputOriginatingCentre;
alias mars.anoffset=anoffset;
# We need this because 'efas_post_proc' is defined later
meta efas_model sprintf("%s", efas_post_proc) : no_copy;
alias mars.model = efas_model;

3
definitions/mars/type.table
View File

@ -55,7 +55,8 @@
65 cv Calibration/Validation forecast
70 or Ocean reanalysis
71 fx Flux forcing
72 fu EFAS Fillup
72 fu Fill-up
73 sfo Simulation forced with observations
80 fcmean Forecast mean
81 fcmax Forecast maximum
82 fcmin Forecast minimum

2
src/grib_accessor_class_count_missing.c
View File

@ -185,7 +185,7 @@ static void init(grib_accessor* a, const long len , grib_arguments* arg )
self->numberOfDataPoints = grib_arguments_get_name(grib_handle_of_accessor(a),arg,n++);
}
static int used[] ={ 0,1,3,7,15,31,63,127,255};
static const int used[] ={ 0,1,3,7,15,31,63,127,255};
static int unpack_long (grib_accessor* a, long* val, size_t *len)
{

4
src/grib_accessor_class_data_g1second_order_general_extended_packing.c
View File

@ -204,7 +204,7 @@ static void init_class(grib_accessor_class* c)
#define MAX_NUMBER_OF_GROUPS 65534
#define EFDEBUG 0
static unsigned long nbits[64]={
static const unsigned long nbits[64]={
0x1, 0x2, 0x4, 0x8,
0x10, 0x20, 0x40, 0x80,
0x100, 0x200, 0x400, 0x800,
@ -225,7 +225,7 @@ static unsigned long nbits[64]={
static long number_of_bits(grib_handle*h, unsigned long x)
{
unsigned long *n=nbits;
const unsigned long *n=nbits;
const int count = sizeof(nbits)/sizeof(nbits[0]);
long i=0;
while (x>=*n) {

4
src/grib_accessor_class_data_simple_packing.c
View File

@ -181,7 +181,7 @@ static void init(grib_accessor* a,const long v, grib_arguments* args)
self->dirty=1;
}
static unsigned long nbits[32]={
static const unsigned long nbits[32]={
0x1, 0x2, 0x4, 0x8, 0x10, 0x20,
0x40, 0x80, 0x100, 0x200, 0x400, 0x800,
0x1000, 0x2000, 0x4000, 0x8000, 0x10000, 0x20000,
@ -193,7 +193,7 @@ static unsigned long nbits[32]={
static int number_of_bits(unsigned long x, long* result)
{
const int count = sizeof(nbits)/sizeof(nbits[0]);
unsigned long *n=nbits;
const unsigned long *n=nbits;
*result=0;
while (x >= *n) {
n++;

22
src/grib_accessor_class_g1step_range.c
View File

@ -30,6 +30,7 @@
MEMBERS = const char *step_unit
MEMBERS = const char *stepType
MEMBERS = const char *patch_fp_precip
MEMBERS = int error_on_units
END_CLASS_DEF
*/
@ -71,6 +72,7 @@ typedef struct grib_accessor_g1step_range {
const char *step_unit;
const char *stepType;
const char *patch_fp_precip;
int error_on_units;
} grib_accessor_g1step_range;
extern grib_accessor_class* grib_accessor_class_abstract_long_vector;
@ -164,6 +166,7 @@ static void init(grib_accessor* a,const long l, grib_arguments* c)
self->step_unit = grib_arguments_get_name(h,c,n++);
self->stepType = grib_arguments_get_name(h,c,n++);
self->patch_fp_precip = grib_arguments_get_name(h,c,n++);
self->error_on_units = 1;
self->number_of_elements=2;
self->v=(long*)grib_context_malloc_clear(h->context,
@ -177,12 +180,9 @@ static void init(grib_accessor* a,const long l, grib_arguments* c)
static void dump(grib_accessor* a, grib_dumper* dumper)
{
grib_dump_string(dumper,a,NULL);
}
static int error_on_units=1;
static int u2s1[] = {
static const int u2s1[] = {
60, /* (0) minutes */
3600, /* (1) hour */
86400, /* (2) day */
@ -201,11 +201,11 @@ static int u2s1[] = {
1 /* (15) seconds */ /* See ECC-316 */
};
static int units_index[] = {
static const int units_index[] = {
1,0,10,11,12,2,0,13,14,15
};
static int u2s[] = {
static const int u2s[] = {
60, /* (0) minutes */
3600, /* (1) hour */
86400, /* (2) day */
@ -324,7 +324,7 @@ static int unpack_string(grib_accessor* a, char* val, size_t *len)
else
sprintf(step_unit_string,"h");
if (error_on_units) {
if (self->error_on_units) {
grib_get_long_internal(hand,self->unit,&unit);
if (unit==254) {
unit=15; /* See ECC-316 */
@ -640,9 +640,9 @@ static int pack_long(grib_accessor* a, const long* val, size_t *len)
return pack_string( a,buff,&bufflen);
case 0 :
self->pack_index=-1;
error_on_units=0;
self->error_on_units=0;
unpack_string(a,sval,&svallen);
error_on_units=1;
self->error_on_units=1;
while (*p != '-' && *p != '\0' ) p++;
if (*p=='-') {
sprintf(buff,"%ld-%s",*val,++p);
@ -656,9 +656,9 @@ static int pack_long(grib_accessor* a, const long* val, size_t *len)
return pack_string( a,buff,&bufflen);
case 1 :
self->pack_index=-1;
error_on_units=0;
self->error_on_units=0;
unpack_string(a,sval,&svallen);
error_on_units=1;
self->error_on_units=1;
while (*p != '-' && *p != '\0' ) p++;
if (*p=='-') {
*p='\0';

3
src/grib_accessor_class_g2_mars_labeling.c
View File

@ -369,7 +369,8 @@ static int extra_set(grib_accessor* a,long val)
break;
case 70: /* Ocean reanalysis (or) */
case 71: /* Flux forcing (fx) */
case 72: /* EFAS Fillup (fu) */
case 72: /* Fill-up (fu) */
case 73: /* Simulation forced with observations (sfo) */
case 80: /* Forecast mean (fcmean) */
case 81: /* Forecast maximum (fcmax) */
case 82: /* Forecast minimum (fcmin) */

4
src/grib_accessor_class_g2end_step.c
View File

@ -202,7 +202,7 @@ static void dump(grib_accessor* a, grib_dumper* dumper)
grib_dump_double(dumper,a,NULL);
}
static int u2s2[] = {
static const int u2s2[] = {
60, /* (0) minutes */
3600, /* (1) hour */
86400, /* (2) day */
@ -219,7 +219,7 @@ static int u2s2[] = {
1 /* (13) seconds */
};
static int u2s[] = {
static const int u2s[] = {
60, /* (0) minutes */
3600, /* (1) hour */
86400, /* (2) day */

2
src/grib_accessor_class_g2step.c
View File

@ -146,7 +146,7 @@ static void dump(grib_accessor* a, grib_dumper* dumper)
grib_dump_double(dumper,a,NULL);
}
static int units[] = {
static const int units[] = {
0, /* (0) minutes */
1, /* (1) hour */
24, /* (2) day */

6
src/grib_accessor_class_second_order_bits_per_value.c
View File

@ -141,7 +141,7 @@ static void init_class(grib_accessor_class* c)
/* END_CLASS_IMP */
/*
static unsigned long nbits[32]={
static const unsigned long nbits[32]={
0x1, 0x2, 0x4, 0x8, 0x10, 0x20,
0x40, 0x80, 0x100, 0x200, 0x400, 0x800,
0x1000, 0x2000, 0x4000, 0x8000, 0x10000, 0x20000,
@ -150,7 +150,7 @@ static unsigned long nbits[32]={
0x40000000, 0x80000000
};
*/
static unsigned long nbits[64]={
static const unsigned long nbits[64]={
0x1, 0x2, 0x4, 0x8,
0x10, 0x20, 0x40, 0x80,
0x100, 0x200, 0x400, 0x800,
@ -171,7 +171,7 @@ static unsigned long nbits[64]={
static int number_of_bits(unsigned long x, long* result)
{
unsigned long *n=nbits;
const unsigned long *n=nbits;
const int count = sizeof(nbits)/sizeof(nbits[0]);
*result=0;
while (x>=*n) {

4
src/grib_accessor_class_signed.c
View File

@ -148,7 +148,6 @@ static void init(grib_accessor* a, const long len , grib_arguments* arg )
a->length = len * count;
self->nbytes = len;
Assert(a->length>=0);
}
static void dump(grib_accessor* a, grib_dumper* dumper)
@ -161,7 +160,7 @@ static void dump(grib_accessor* a, grib_dumper* dumper)
grib_dump_values(dumper,a);
}
static long ones[] = {
static const long ones[] = {
0,
-0x7f,
-0x7fff,
@ -169,7 +168,6 @@ static long ones[] = {
-0x7fffffff,
};
static int unpack_long(grib_accessor* a, long* val, size_t *len)
{

4
src/grib_accessor_class_step_in_units.c
View File

@ -158,7 +158,7 @@ static void dump(grib_accessor* a, grib_dumper* dumper)
}
/* Conversion of to seconds - Grib edition 2 table 4.4 */
static int u2s2[] = {
static const int u2s2[] = {
60, /* (0) minutes */
3600, /* (1) hour */
86400, /* (2) day */
@ -176,7 +176,7 @@ static int u2s2[] = {
};
/* Note: 'stepUnits' has a different table with extra entries e.g. 15 and 30 mins */
static int u2s[] = {
static const int u2s[] = {
60, /* (0) minutes */
3600, /* (1) hour */
86400, /* (2) day */

2
src/grib_accessor_class_unsigned.c
View File

@ -170,7 +170,7 @@ static void dump(grib_accessor* a, grib_dumper* dumper)
grib_dump_values(dumper,a);
}
static unsigned long ones[] = {
static const unsigned long ones[] = {
0,
0xff,
0xffff,

2
src/grib_accessor_class_validity_date.c
View File

@ -144,7 +144,7 @@ static void init_class(grib_accessor_class* c)
/* END_CLASS_IMP */
/* Table of multipliers to convert step units to minutes */
static double u2m[] = {
static const double u2m[] = {
1, /* index 0: minutes */
60, /* index 1: hour */
24*60, /* index 2: day */

2
src/grib_accessor_class_validity_time.c
View File

@ -142,7 +142,7 @@ static void init_class(grib_accessor_class* c)
/* END_CLASS_IMP */
/* Table of multipliers to convert step units to minutes */
static double u2m[] = {
static const double u2m[] = {
1, /* index 0: minutes */
60, /* index 1: hour */
24*60, /* index 2: day */

4
src/grib_bits.c
View File

@ -30,9 +30,9 @@ long GRIB_MASK = -1; /* Mask of sword bits */
((b)==max_nbits ? GRIB_MASK : (~(GRIB_MASK<<(b))<<(max_nbits-((q)+(b)))))
static unsigned long dmasks[] = { 0xFF, 0xFE, 0xFC, 0xF8, 0xF0, 0xE0, 0xC0, 0x80, 0x00, };
static const unsigned long dmasks[] = { 0xFF, 0xFE, 0xFC, 0xF8, 0xF0, 0xE0, 0xC0, 0x80, 0x00, };
static int max_nbits = sizeof(unsigned long)*8;
static const int max_nbits = sizeof(unsigned long)*8;
unsigned long grib_decode_unsigned_byte_long(const unsigned char* p, long o, int l)
{

2
src/grib_box_class.c
View File

@ -21,7 +21,7 @@ struct table_entry
grib_box_class **cclass;
};
static struct table_entry table[] =
static const struct table_entry table[] =
{
/* This file is generated by ./make_class.pl */
#include "grib_box_factory.h"

2
src/grib_hash_keys.c
View File

@ -10673,7 +10673,7 @@ grib_keys_hash_get (const char *str, unsigned int len)
* Enrico Fucile
**************************************/
static int mapping[] = {
static const int mapping[] = {
0, /* 00 */
0, /* 01 */
0, /* 02 */

2
src/grib_iterator_class.c
View File

@ -27,7 +27,7 @@ struct table_entry
grib_iterator_class **cclass;
};
static struct table_entry table[] =
static const struct table_entry table[] =
{
/* This file is generated by ./make_class.pl */
#include "grib_iterator_factory.h"

2
src/grib_itrie.c
View File

@ -11,7 +11,7 @@
#include "grib_api_internal.h"
/* Note: all non-alpha are mapped to 0 */
static int mapping[] = {
static const int mapping[] = {
0, /* 00 */
0, /* 01 */
0, /* 02 */

2
src/grib_itrie_keys.c
View File

@ -12,7 +12,7 @@
* Enrico Fucile
**************************************/
static int mapping[] = {
static const int mapping[] = {
0, /* 00 */
0, /* 01 */
0, /* 02 */

2
src/grib_nearest_class.c
View File

@ -26,7 +26,7 @@ struct table_entry
grib_nearest_class **cclass;
};
static struct table_entry table[] =
static const struct table_entry table[] =
{
/* This file is generated by ./make_class.pl */
#include "grib_nearest_factory.h"

2
src/grib_trie.c
View File

@ -11,7 +11,7 @@
#include "grib_api_internal.h"
/* Note: all non-alpha are mapped to 0 */
static int mapping[] = {
static const int mapping[] = {
0, /* 00 */
0, /* 01 */
0, /* 02 */

2
src/grib_trie_with_rank.c
View File

@ -11,7 +11,7 @@
#include "grib_api_internal.h"
/* Note: all non-alpha are mapped to 0 */
static int mapping[] = {
static const int mapping[] = {
0, /* 00 */
0, /* 01 */
0, /* 02 */

4
src/md5.c
View File

@ -19,13 +19,13 @@
#pragma _CRI noopt
#endif
static unsigned long r[] = {
static const unsigned long r[] = {
7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22,
5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20,
4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23,
6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21};
static unsigned long k[] = {
static const unsigned long k[] = {
0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee, 0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501,
0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be, 0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821,
0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa, 0xd62f105d, 0x02441453, 0xd8a1e681, 0xe7d3fbc8,

7
tests/grib_efas.sh
View File

@ -65,12 +65,13 @@ ${tools_dir}/grib_set -s setLocalDefinition=1,localDefinitionNumber=41,yearOfFor
grib_check_key_equals $temp1 anoffset 25
grib_check_key_equals $temp1 anoffsetFirst,anoffsetLast,anoffsetFrequency "MISSING MISSING MISSING"
# ECC-663: MARS step
types="an fu go"
# MARS step
types="sfo fu go"
for t in $types; do
${tools_dir}/grib_set -s setLocalDefinition=1,localDefinitionNumber=41,type=$t,stepType=accum,stepRange=12-36,paramId=260268 \
$sample $temp1
grib_check_key_equals $temp1 mars.step 12 # start step
grib_check_key_equals $temp1 mars.step 36 # end step (ECC-701)
grib_check_key_exists $temp1 mars.anoffset
#${tools_dir}/grib_dump -Da $temp1 | grep mars.step
done

36
tests/grib_to_netcdf.sh
View File

@ -10,37 +10,19 @@
. ./include.sh
#set -x
# First decide if we have NetCDF enabled from configure.
# Temporarily turn off the '-e' flag for set so we dont exit on error
set +e
tmp_out=tmp_msg.out
skip_test=0
rm -f $tmp_out
# Invoke the grib_to_netcdf command with no options. If NetCDF was enabled
# we get a usage message otherwise we get a specific message. Note: In both cases
# the command fails.
${tools_dir}/grib_to_netcdf > $tmp_out
grep 'ecCodes was not compiled with NETCDF enabled' $tmp_out > /dev/null
if [ $? -eq 0 ]; then
# Message was found
skip_test=1
fi
# Restore the set -e so any error causes us to exit
set -e
rm -f $tmp_out
if [ $skip_test -eq 1 ]; then
echo "NetCDF was not enabled. Skipping this test."
# Disable if autotools being used
src_config=${src_dir}/config.h
if [ -f ${src_config} ]; then
exit 0
fi
echo "NetCDF was enabled."
grib_files="\
regular_latlon_surface.grib2 \
regular_latlon_surface.grib1 \
regular_gaussian_model_level.grib1 \
regular_gaussian_pressure_level.grib2 \
regular_gaussian_surface.grib1 \
missing.grib2"
regular_latlon_surface.grib2 \
regular_latlon_surface.grib1 \
regular_gaussian_model_level.grib1 \
regular_gaussian_pressure_level.grib2 \
regular_gaussian_surface.grib1 \
missing.grib2"
ncf_types="NC_SHORT NC_INT NC_FLOAT NC_DOUBLE"

20
tools/grib_tools.c
View File

@ -813,6 +813,17 @@ void grib_skip_check(grib_runtime_options* options,grib_handle* h)
}
}
/* See ECC-707 */
static int fix_for_lsdate_needed(grib_handle* h)
{
long lsdate_bug = 0;
int err = grib_get_long(h, "lsdate_bug", &lsdate_bug);
if (!err && lsdate_bug == 1) {
return 1;
}
return 0;
}
void grib_print_key_values(grib_runtime_options* options, grib_handle* h)
{
int i=0;
@ -864,15 +875,22 @@ void grib_print_key_values(grib_runtime_options* options, grib_handle* h)
}
} else {
/* Other products e.g. GRIB */
const int fix_lsdate = (fix_for_lsdate_needed(h) && options->name_space && strcmp(options->name_space,"ls")==0);
if (grib_is_missing(h,options->print_keys[i].name,&ret) && ret==GRIB_SUCCESS) {
sprintf(value,"MISSING");
}
else if ( ret == GRIB_SUCCESS ) {
const char* pName = NULL;
if (options->print_keys[i].type == GRIB_TYPE_UNDEFINED)
grib_get_native_type(h,options->print_keys[i].name,&(options->print_keys[i].type));
switch (options->print_keys[i].type) {
case GRIB_TYPE_STRING:
ret=grib_get_string( h,options->print_keys[i].name,value,&len);
pName = options->print_keys[i].name;
if (fix_lsdate && strcmp(pName, "date")==0) { /* ECC-707 */
pName = "ls.date";
}
ret=grib_get_string( h,pName,value,&len);
break;
case GRIB_TYPE_DOUBLE:
ret=grib_get_double( h,options->print_keys[i].name,&dvalue);