eccodes/tools/codes_split_file.c

/*
 * Copyright 2005-2019 ECMWF.
 *
 * This software is licensed under the terms of the Apache Licence Version 2.0
 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
 *
 * In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
 * virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
 */

/*
 * Description:
 * Split an input file (GRIB, BUFR etc) into chunks of roughly the same size.
 * The output files are named input_01, input_02 etc. This is much faster than grib_copy/bufr_copy
 * 
 * 2019-07-26 W.Qu  Allow an input file to be split into each individual message (if nchunk=-1)
 *
 */

#include "grib_api_internal.h"
#include <assert.h>

static int verbose = 0;
static const char* OUTPUT_FILENAME_FORMAT = "%s_%03d"; /* x_001, x_002 etc */
static void usage(const char* prog)
{
    printf("Usage: %s [-v] nchunks infile\n",prog);
    printf("Setting nchunks=-1 splits infile into individual messages\n");
    exit(1);
}

static int split_file(FILE* in, const char* filename, const int nchunks, unsigned long *count)
{
    void* mesg=NULL;
    FILE* out;
    size_t size=0,read_size=0,insize=0,chunk_size, msg_size=0, num_msg=0;
    off_t offset=0;
    int err=GRIB_SUCCESS;
    int i;
    char* ofilename;
    grib_context* c=grib_context_get_default();

    if (!in) return 1;

    /* name of output file */
    ofilename=(char*)calloc(1,strlen(filename)+10);

    fseeko(in, 0, SEEK_END);
    insize = ftello(in);
    fseeko(in, 0, SEEK_SET);
    if(nchunks == -1){
      chunk_size = size;
    }else{
      assert(nchunks > 0);
      chunk_size=insize/nchunks;
    }

    i=1;
    sprintf(ofilename, OUTPUT_FILENAME_FORMAT, filename, i);
    out=fopen(ofilename,"w");
    if (!out) {
        perror(ofilename);
        free(ofilename);
        return GRIB_IO_PROBLEM;
    }

    while ( err!=GRIB_END_OF_FILE ) {
        mesg=wmo_read_any_from_file_malloc(in, 0, &size, &offset, &err);
        num_msg++;
        /*printf("=1=%d\t%d\t%d\n",*count,size,insize);*/
        if ( mesg!=NULL && err==0 ) {
            if (fwrite(mesg,1,size,out)!=size ) {
                perror(ofilename);
                free(ofilename);
                fclose(out);
                return GRIB_IO_PROBLEM;
            }
            grib_context_free(c,mesg);
            read_size+=size;
            msg_size+=size;
            if (read_size>chunk_size && msg_size < insize) {
                if (verbose) printf("Wrote output file %s (%lu msgs)\n", ofilename, (unsigned long)num_msg);
                fclose(out);
                i++;
                /* Start writing to the next file */
                /*printf("=2=%d\t%d\n",*count,msg_size);*/
                sprintf(ofilename, OUTPUT_FILENAME_FORMAT, filename, i);
                out=fopen(ofilename,"w");
                if (!out) {
                    perror(ofilename);
                    free(ofilename);
                    return GRIB_IO_PROBLEM;
                }
                read_size=0;
                num_msg=0;
            }
            (*count)++;
        }
    }
    if (verbose) printf("Wrote output file %s (%lu msgs)\n", ofilename, (unsigned long)num_msg-1);
    fclose(out);
    free(ofilename);

    if (err==GRIB_END_OF_FILE) err=GRIB_SUCCESS;

    return err;
}

int main(int argc,char* argv[])
{
    FILE* infh = NULL;
    char* filename;
    int i, status=0;
    struct stat s;
    int err=0,nchunks=0;
    unsigned long count=0;

    if (argc <3) usage(argv[0]);

    i=1;
    if (strcmp(argv[i], "-v")==0) {
        i++;
        verbose = 1;
        if (argc !=4) usage(argv[0]);
    }

    /* add some error checking */
    nchunks=atoi(argv[i]);
    if (nchunks<1 && nchunks!=-1) {
        fprintf(stderr,"ERROR: Invalid number %d. Please specify a positive integer or -1\n", nchunks);
        return 1;
    }

    i++;
    filename=argv[i];
    if (stat(filename, &s)==0) {
        if (S_ISDIR(s.st_mode)) {
            fprintf(stderr, "ERROR: %s: Is a directory\n", filename);
            return 1;
        }
    }
    infh=fopen(filename,"rb");
    if (!infh) {
        perror(filename);
        return 1;
    }

    count=0;
    err=split_file(infh, filename, nchunks, &count);
    if (err) {
        fprintf(stderr,"ERROR: Failed to split file %s", filename);
        fprintf(stderr,"\n");
        status = 1;
    } else {
        if (verbose) printf ("%7lu %s\n", count, filename);
    }

    fclose(infh);

    return status;
}