ECC-1344: Always call fflush() and fsync() before fclose() of a written file (Part 1)

examples/F90/grib_copy_message.f90

@@ -22,7 +22,7 @@ program copy
    character(len=32)                  :: product_kind
 
    call codes_open_file(infile, '../../data/constant_field.grib1', 'r')
-   call codes_open_file(outfile, 'out.copy.grib1', 'w')
+   call codes_open_file(outfile, 'out.copy.grib1', 'W')
 
    ! A new GRIB message is loaded from file
    ! igrib_in is the GRIB id to be used in subsequent calls

fortran/grib_fortran.c

@@ -82,6 +82,12 @@ static void init()
 }
 #endif
 
+typedef enum FileMode {
+    FILE_MODE_READ,
+    FILE_MODE_WRITE,
+    FILE_MODE_APPEND
+} FileMode;
+
 int GRIB_NULL=-1;
 int GRIB_NULL_NEAREST=-1;
 /*extern int errno;*/
@@ -92,6 +98,7 @@ struct l_grib_file {
     FILE* f;
     char* buffer;
     int id;
+    int mode; /* enum FileMode */
     l_grib_file* next;
 };
 
@@ -218,28 +225,35 @@ static void fort_char_clean(char* str,int len)
     *p=' ';
 }
 
-static int push_file(FILE* f,char* buffer)
+/* Note: the open_mode argument will be all lowercase. See grib_f_open_file_ */
+static int push_file(FILE* f, const char* open_mode, char* buffer)
 {
     l_grib_file* current  = file_set;
     l_grib_file* previous = file_set;
     l_grib_file* the_new      = NULL;
     int myindex = MIN_FILE_ID;
+    FileMode fmode = FILE_MODE_READ;
+
+    if (strcmp(open_mode, "w") == 0) fmode = FILE_MODE_WRITE;
+    else if (strcmp(open_mode, "a") == 0) fmode = FILE_MODE_APPEND;
 
     if(!file_set){
         file_set = (l_grib_file*)malloc(sizeof(l_grib_file));
         Assert(file_set);
-        file_set->id   = myindex;
+        file_set->id     = myindex;
-        file_set->f    = f;
+        file_set->f      = f;
-        file_set->buffer =buffer;
+        file_set->mode   = fmode;
-        file_set->next = NULL;
+        file_set->buffer = buffer;
+        file_set->next   = NULL;
         return myindex;
     }
 
     while(current){
         if(current->id < 0){
             current->id = -(current->id);
-            current->f    = f;
+            current->f      = f;
-            current->buffer    = buffer;
+            current->mode   = fmode;
+            current->buffer = buffer;
             return current->id ;
         } else{
             myindex++;
@@ -250,11 +264,12 @@ static int push_file(FILE* f,char* buffer)
 
     the_new = (l_grib_file*)malloc(sizeof(l_grib_file));
     Assert(the_new);
-    the_new->id   = myindex;
+    the_new->id     = myindex;
-    the_new->f    = f;
+    the_new->f      = f;
-    the_new->buffer    = buffer;
+    the_new->mode   = fmode;
-    the_new->next = current;
+    the_new->buffer = buffer;
-    previous->next = the_new;
+    the_new->next   = current;
+    previous->next  = the_new;
     return myindex;
 }
 
@@ -756,7 +771,10 @@ static int clear_file(int file_id)
         if(current->id == file_id){
             current->id = -(current->id);
             if (current->f) {
-                err = fclose(current->f);
+                if (current->mode == FILE_MODE_WRITE || current->mode == FILE_MODE_APPEND)
+                    err = codes_flush_sync_close_file(current->f);
+                else
+                    err = fclose(current->f);
                 if (err) {
                     int ioerr = errno;
                     grib_context* c = grib_context_get_default();
@@ -1049,7 +1067,7 @@ int grib_f_open_file_(int* fid, char* name , char* op, int lname, int lop) {
 #endif
             setvbuf(f,iobuf,_IOFBF,context->io_buffer_size);
         }
-        *fid = push_file(f,iobuf);
+        *fid = push_file(f, oper, iobuf);
         ret=GRIB_SUCCESS;
     }
     return ret;

src/codes_util.c

@@ -140,3 +140,54 @@ int codes_check_grib_ieee_packing_value(int value)
     }
     return GRIB_SUCCESS;
 }
+
+/* Note: To be called in cases where we are WRITING a file (Do not call when reading) */
+int codes_flush_sync_close_file(FILE* f)
+{
+    int err = 0;
+    int fd  = 0;
+    grib_context* c = grib_context_get_default();
+    Assert(f);
+
+    fd = fileno(f);
+    if (fd == -1) {
+        grib_context_log(c, GRIB_LOG_PERROR, "fileno() error: invalid stream");
+        return GRIB_IO_PROBLEM;
+    }
+
+#if 0
+#ifdef HAVE_FCNTL_H
+    /* Heavy handed way of getting the file access mode: only proceed if writing */
+    val = fcntl(fd, F_GETFL, 0);
+    if (val < 0) {
+        grib_context_log(c, GRIB_LOG_PERROR, "Call to fcntl failed");
+        return err;
+    }
+    if ((val & O_ACCMODE) != O_WRONLY) {
+        /* File is not being written */
+        return GRIB_SUCCESS;
+    }
+#endif
+#endif
+
+    err = fflush(f);
+    if (err) {
+        grib_context_log(c, GRIB_LOG_PERROR, "Call to fflush failed");
+        return err;
+    }
+
+    err = fsync(fd);
+    while (err < 0 && errno == EINTR) {
+        err = fsync(fd);
+    }
+    if (err < 0) {
+        grib_context_log(c, GRIB_LOG_PERROR, "Cannot fsync file");
+        return GRIB_IO_PROBLEM;
+    }
+    err = fclose(f);
+    if (err) {
+        grib_context_log(c, GRIB_LOG_PERROR, "Call to fclose failed");
+        return err;
+    }
+    return GRIB_SUCCESS;
+}

src/grib_api_prototypes.h

@@ -1447,6 +1447,7 @@ int path_is_regular_file(const char* path);
 int path_is_directory(const char* filename);
 char* codes_getenv(const char* name);
 int codes_check_grib_ieee_packing_value(int value);
+int codes_flush_sync_close_file(FILE* f);
 
 /* grib_util.c */
 grib_handle* grib_util_sections_copy(grib_handle* hfrom, grib_handle* hto, int what, int* err);

src/grib_handle.c

@@ -325,7 +325,7 @@ int grib_write_message(const grib_handle* h, const char* file, const char* mode)
         fclose(fh);
         return GRIB_IO_PROBLEM;
     }
-    if (fclose(fh) != 0) {
+    if (codes_flush_sync_close_file(fh) != GRIB_SUCCESS) {
         perror(file);
         return GRIB_IO_PROBLEM;
     }