SUP-2101: grib_to_netcdf support of deflate option for netCDF-4 output formats

tools/grib_to_netcdf.c

@@ -1843,6 +1843,8 @@ typedef struct ncoptions {
     request *mars_description;
     boolean mmeans; /* Whether this dataset is Monthly Means */
     boolean climatology; /* Whether this dataset is climatology */
+    boolean shuffle;
+    long deflate;
 } ncoptions_t;
 
 ncoptions_t setup;
@@ -2044,11 +2046,15 @@ static void get_nc_options(const request *user_r)
     const char *checkvalidtime_env = NULL;
     const char *validtime = get_value(user_r, "usevalidtime", 0);
     const char *refdate = get_value(user_r, "referencedate", 0);
+    const char *shuffle = get_value(user_r, "shuffle", 0);
+    const char *deflate = get_value(user_r, "deflate", 0);
 
     const char *title = get_value(user_r, "title", 0);
     const char *history = get_value(user_r, "history", 0);
     const char *unlimited = get_value(user_r, "unlimited", 0);
 
+    setup.shuffle = shuffle ? (strcmp(shuffle, "true") == 0) : FALSE;
+    setup.deflate = deflate ? ((strcmp(deflate, "none") == 0) ? -1 : atol(deflate)) : -1;
     setup.usevalidtime = validtime ? (strcmp(validtime, "true") == 0) : FALSE;
     setup.refdate = refdate ? atol(refdate) : 19000101;
     setup.auto_refdate = refdate ? (strcmp(get_value(user_r, "referencedate", 0), "AUTOMATIC") == 0) : FALSE;
@@ -2856,10 +2862,40 @@ static int define_netcdf_dimensions(hypercube *h, fieldset *fs, int ncid, datase
     int var_id = 0; /* Variable ID */
     int dims[1024];
 
+    size_t count[NC_MAX_DIMS];
+    err e = 0;
+    
+    long ni;
+    long nj;
+    
+    field *f = get_field(fs, 0, expand_mem);
+
+    
+    /* Define longitude */
+    if((e = grib_get_long(f->handle, "Ni", &ni)) != GRIB_SUCCESS)
+    {
+        grib_context_log(ctx, GRIB_LOG_ERROR, "ecCodes: cannot get Ni %s", grib_get_error_message(e));
+        return e;
+    }
+    /* Define latitude */
+    if((e = grib_get_long(f->handle, "Nj", &nj)) != GRIB_SUCCESS)
+    {
+        grib_context_log(ctx, GRIB_LOG_ERROR, "ecCodes: cannot get Nj %s", grib_get_error_message(e));
+        return e;
+    }
+    release_field(f);
+
+    /* Count dimensions per axis */
+    for(i = 0; i < naxis; ++i)
+        count[naxis - i - 1] = 1;
+
+    count[naxis] = nj; /* latitude */
+    count[naxis + 1] = ni; /* longitude */
+
     /* START DEFINITIONS */
 
     /* Define latitude/longitude dimensions */
-    err e = def_latlon(ncid, fs);
+    e = def_latlon(ncid, fs);
     if (e != GRIB_SUCCESS)
         return e;
 
@@ -3010,6 +3046,14 @@ static int define_netcdf_dimensions(hypercube *h, fieldset *fs, int ncid, datase
         stat = nc_def_var(ncid, subsets[i].att.name, subsets[i].att.nctype, n, dims, &var_id);
         check_err(stat, __LINE__, __FILE__);
 
+        if (setup.deflate>-1)
+        {
+            stat = nc_def_var_chunking(ncid, var_id, NC_CHUNKED, count);
+            check_err(stat, __LINE__, __FILE__);
+
+            stat = nc_def_var_deflate(ncid, var_id, setup.shuffle, 1, setup.deflate);
+            check_err(stat, __LINE__, __FILE__);
+        }
         if(subsets[i].scale)
         {
             compute_scale(&subsets[i]);
@@ -3835,6 +3879,10 @@ grib_option grib_options[] = {
                 "\n\t\t3 -> netCDF-4 file format"
                 "\n\t\t4 -> netCDF-4 classic model file format\n"
                 , 0, 1, "2" },
+        { "d:", "level",          "\n\t\tDeflate data. Only for netCDF-4 kind output format."
+                "\n\t\tPossible values [0,9]. Default None." 
+                "\n\t\tChunking strategy based on GRIB message.\n", 0, 1, "6" },
+        { "s", 0, "Shuffle data before deflate.\n", 0, 1, 0 },
         { "u:", "dimension",  "\n\t\tSet dimension to be an unlimited dimension.\n", 0, 1, "time" }
 };
 
@@ -3843,6 +3891,7 @@ static fieldset *fs = NULL;
 static request* data_r = NULL;
 request *user_r = NULL;
 static int option_kind = 2; /* By default NetCDF3, 64-bit offset */
+static int deflate_option = 0;
 
 /* Table of formats for legal -k values. Inspired by nccopy */
 struct KindValue {
@@ -3978,6 +4027,44 @@ int grib_tool_init(grib_runtime_options* options)
         }
     }
 
+    if(grib_options_on("d:"))
+    {
+        if(option_kind == 3 || option_kind == 4)
+        {
+            char* theArg = grib_options_get_option("d:");
+            if (!is_number(theArg) || atol(theArg)<0 || atol(theArg)>9 ) {
+                fprintf(stderr, "Invalid deflate option: %s\n", theArg);
+                usage();
+                exit(1);
+            }
+            set_value(user_r, "deflate", theArg);
+            deflate_option=1;
+        }else{
+            fprintf(stderr, "Invalid deflate option for non netCDF-4 kind output formats\n");
+            usage();
+            exit(1);
+        }
+    }
+    else
+    {
+        set_value(user_r, "deflate", "none");
+    }
+
+    if(grib_options_on("s"))
+    {
+        if(deflate_option)
+            set_value(user_r, "shuffle", "true");
+        else
+        {
+            fprintf(stderr, "Invalid shuffle option. Deflate option needed.\n");
+            usage();
+            exit(1);
+        }
+
+    }
+    else
+        set_value(user_r, "shuffle", "false");
+
     if(grib_options_on("R:"))
     {
         char* theArg = grib_options_get_option("R:");