mirror of https://github.com/ecmwf/eccodes.git
566 lines
19 KiB
Perl
Executable File
566 lines
19 KiB
Perl
Executable File
#!/usr/bin/env perl
|
|
|
|
use strict;
|
|
use File::Path ;
|
|
use File::Basename ;
|
|
use File::Copy;
|
|
use Cwd;
|
|
use DBI;
|
|
|
|
my $basedir = dirname($0);
|
|
|
|
my $db="param";
|
|
my $host = $ENV{'PARAM_DB_HOST'} || 'unknown';
|
|
my $user = $ENV{'PARAM_DB_USER'} || 'unknown';
|
|
my $pass = $ENV{'PARAM_DB_PASS'} || 'unknown';
|
|
|
|
my $filename; my $filebase; my $out; my $conceptDir;
|
|
my $query; my $q; my $qh;
|
|
|
|
my $dbh = DBI->connect("dbi:mysql(RaiseError=>1):database=$db;host=$host",$user,$pass) or die $DBI::errstr;
|
|
|
|
# I have written to it already or not
|
|
#my $tarfilesflag = 0;
|
|
|
|
sub create_cfName {
|
|
my $p; my %seen;
|
|
my ($key) = "cfName";
|
|
my $field = "cf.name";
|
|
|
|
my $query= <<"EOF";
|
|
select $field,force128,edition,
|
|
centre.abbreviation,param_id,attribute.name,attribute_value,param.name,param.shortName from
|
|
param,grib_encoding,grib,attribute,centre,units,cf where
|
|
param.hide_def=0 and
|
|
param.retired=0 and
|
|
grib_encoding.published=1 and
|
|
grib_encoding.id=grib.encoding_id and
|
|
param.id=grib_encoding.param_id and
|
|
attribute.id=grib.attribute_id and
|
|
centre.id=grib_encoding.centre_id and
|
|
units.id=param.units_id and
|
|
param.id=cf.grib1_ecmwf and
|
|
grib_encoding.is_legacy=0 order by
|
|
edition,centre_id,param.o,param.id,grib_encoding.param_version,attribute.o;
|
|
EOF
|
|
|
|
my $qh=$dbh->prepare($query);
|
|
$qh->execute();
|
|
|
|
# file containing the list of grib api parameters files we want to tar and
|
|
# distribute to users for them to download and update their list of parameter
|
|
# to the latest
|
|
#open(TAR,$tarfilesflag ? ">>" : ">","tarfiles.txt") or die "Count not open file tarfiles.txt: $!";
|
|
#$tarfilesflag=1;
|
|
|
|
while (my ($keyval,$force128,$edition,$centre,$paramId,$attribute,$value,$name,$shortName)=$qh->fetchrow_array )
|
|
{
|
|
if ($centre eq "wmo" ) { $conceptDir=""; }
|
|
else { $conceptDir="/localConcepts/$centre"; }
|
|
|
|
if ($filebase ne "$basedir/grib$edition$conceptDir") {
|
|
if ($filebase) {
|
|
print $out "}\n";
|
|
close $out;
|
|
}
|
|
$filebase="$basedir/grib$edition$conceptDir";
|
|
mkpath($filebase);
|
|
|
|
print TAR "grib$edition$conceptDir/$key.def\n";
|
|
#system("cp -f $filebase/$key.def $filebase/$key.def.orig");
|
|
open($out,"> $filebase/$key.def")
|
|
or die "unable to open $filebase/$key.def";
|
|
print $out "# Automatically generated by $0, do not edit\n";
|
|
$p=();
|
|
}
|
|
if ($p ne $paramId || exists($seen{$attribute}) ) {
|
|
if ($p) { print $out "\t}\n"; }
|
|
print $out "#$name\n" ;
|
|
print $out "\'".$keyval."\' = {\n" ;
|
|
$p=$paramId;
|
|
%seen=();
|
|
}
|
|
$seen{$attribute}=1;
|
|
print "($key=$keyval) $edition,$centre,$shortName,$paramId,$name,$attribute,$value\n";
|
|
# we need to allow strings in the attribute_value field
|
|
# for the moment we apply a patch here
|
|
if ($attribute =~ /stepType/ ) {
|
|
$value="\"accum\"";
|
|
}
|
|
if ($value eq '') {
|
|
$value="missing()";
|
|
}
|
|
print $out "\t $attribute = $value ;\n" ;
|
|
}
|
|
if ($filebase) {
|
|
print $out "}\n";
|
|
close $out;
|
|
}
|
|
|
|
close(TAR);
|
|
}
|
|
|
|
sub create_cfName_legacy {
|
|
my $p; my %seen;
|
|
my ($key) = "cfName";
|
|
my $field = "cf.name";
|
|
|
|
my $query= <<"EOF";
|
|
select $field,force128,edition,
|
|
centre.abbreviation,param_id,attribute.name,attribute_value,param.name,param.shortName from
|
|
param,grib_encoding,grib,attribute,centre,units,cf where
|
|
param.hide_def=0 and
|
|
# param.retired=0 and
|
|
grib_encoding.id=grib.encoding_id and
|
|
param.id=grib_encoding.param_id and
|
|
attribute.id=grib.attribute_id and
|
|
centre.id=grib_encoding.centre_id and
|
|
units.id=param.units_id and
|
|
param.id=cf.grib1_ecmwf and
|
|
grib_encoding.is_legacy=1 order by
|
|
edition,centre_id,param.o,param.id,grib_encoding.param_version,attribute.o;
|
|
EOF
|
|
|
|
my $qh=$dbh->prepare($query);
|
|
$qh->execute();
|
|
|
|
# file containing the list of grib api parameters files we want to tar and
|
|
# distribute to users for them to download and update their list of parameter
|
|
# to the latest
|
|
#open(TAR,$tarfilesflag ? ">>" : ">","tarfiles.txt") or die "Count not open file tarfiles.txt: $!";
|
|
#$tarfilesflag=1;
|
|
|
|
while (my ($keyval,$force128,$edition,$centre,$paramId,$attribute,$value,$name,$shortName)=$qh->fetchrow_array )
|
|
{
|
|
if ($centre eq "wmo" ) { $conceptDir=""; }
|
|
else { $conceptDir="/localConcepts/$centre"; }
|
|
|
|
if ($filebase ne "$basedir/grib$edition$conceptDir") {
|
|
if ($filebase) {
|
|
print $out "}\n";
|
|
close $out;
|
|
}
|
|
$filebase="$basedir/grib$edition$conceptDir";
|
|
mkpath($filebase);
|
|
|
|
print TAR "grib$edition$conceptDir/$key.legacy.def\n";
|
|
#system("cp -f $filebase/$key.def $filebase/$key.def.orig");
|
|
open($out,"> $filebase/$key.legacy.def")
|
|
or die "unable to open $filebase/$key.legacy.def";
|
|
print $out "# Automatically generated by $0, do not edit\n";
|
|
$p=();
|
|
}
|
|
if ($p ne $paramId || exists($seen{$attribute}) ) {
|
|
if ($p) { print $out "\t}\n"; }
|
|
print $out "#$name\n" ;
|
|
print $out "\'".$keyval."\' = {\n" ;
|
|
$p=$paramId;
|
|
%seen=();
|
|
}
|
|
$seen{$attribute}=1;
|
|
print "($key=$keyval) $edition,$centre,$shortName,$paramId,$name,$attribute,$value\n";
|
|
# we need to allow strings in the attribute_value field
|
|
# for the moment we apply a patch here
|
|
if ($attribute =~ /stepType/ ) {
|
|
$value="\"accum\"";
|
|
}
|
|
if ($value eq '') {
|
|
$value="missing()";
|
|
}
|
|
print $out "\t $attribute = $value ;\n" ;
|
|
}
|
|
if ($filebase) {
|
|
print $out "}\n";
|
|
close $out;
|
|
}
|
|
|
|
close(TAR);
|
|
}
|
|
|
|
sub create_def {
|
|
my $p; my %seen;
|
|
my ($key) =@_;
|
|
my $field=$key;
|
|
|
|
if ($key =~ /paramId/) { $field="param.id"; }
|
|
if ($key =~ /name/) { $field="param.name"; }
|
|
if ($key =~ /units/) { $field="units.name"; }
|
|
|
|
my $query= <<"EOF";
|
|
select $field,force128,edition,
|
|
centre.abbreviation,param_id,attribute.name,attribute_value,param.name,param.shortName
|
|
from param,grib_encoding,grib,attribute,centre,units where
|
|
param.hide_def=0 and
|
|
param.retired=0 and
|
|
grib_encoding.published=1 and
|
|
grib_encoding.id=grib.encoding_id and
|
|
param.id=grib_encoding.param_id and
|
|
attribute.id=grib.attribute_id and
|
|
centre.id=grib_encoding.centre_id and
|
|
units.id=param.units_id and
|
|
grib_encoding.is_legacy=0
|
|
order by edition,centre_id,param.o,param.id,grib_encoding.param_version,attribute.o;
|
|
EOF
|
|
|
|
my $qh=$dbh->prepare($query);
|
|
$qh->execute();
|
|
|
|
# file containing the list of grib api parameters files we want to tar and
|
|
# distribute to users for them to download and update their list of parameter
|
|
# to the latest
|
|
#open(TAR,$tarfilesflag ? ">>" : ">","tarfiles.txt") or die "Count not open file tarfiles.txt: $!";
|
|
#$tarfilesflag=1;
|
|
|
|
while (my ($keyval,$force128,$edition,$centre,$paramId,$attribute,$value,$name,$shortName)=$qh->fetchrow_array )
|
|
{
|
|
if ($centre eq "wmo" ) { $conceptDir=""; }
|
|
else { $conceptDir="/localConcepts/$centre"; }
|
|
#if ($key =~ /paramId/ && $force128==1 && $keyval >1000) {
|
|
# $keyval= $keyval % 1000;
|
|
#}
|
|
|
|
if ($filebase ne "$basedir/grib$edition$conceptDir") {
|
|
if ($filebase) {
|
|
print $out "}\n";
|
|
close $out;
|
|
}
|
|
$filebase="$basedir/grib$edition$conceptDir";
|
|
mkpath($filebase);
|
|
|
|
#copy("$filebase/$key.def","$filebase/$key.def.bkp")
|
|
# or die ("unable to copy $filebase/$key.def");
|
|
|
|
print TAR "grib$edition$conceptDir/$key.def\n";
|
|
#system("cp -f $filebase/$key.def $filebase/$key.def.orig");
|
|
open($out,"> $filebase/$key.def")
|
|
or die "unable to open $filebase/$key.def";
|
|
print $out "# Automatically generated by $0, do not edit\n";
|
|
$p=();
|
|
}
|
|
if ($p ne $paramId || exists($seen{$attribute}) ) {
|
|
if ($p) { print $out "\t}\n"; }
|
|
print $out "#$name\n" ;
|
|
print $out "\'".$keyval."\' = {\n" ;
|
|
$p=$paramId;
|
|
%seen=();
|
|
}
|
|
$seen{$attribute}=1;
|
|
print "($key=$keyval) $edition,$centre,$shortName,$paramId,$name,$attribute,$value\n";
|
|
# we need to allow strings in the attribute_value field
|
|
# for the moment we apply a patch here
|
|
if ($attribute =~ /stepType/ ) {
|
|
$value="\"accum\"";
|
|
}
|
|
if ($value eq '') {
|
|
$value="missing()";
|
|
}
|
|
print $out "\t $attribute = $value ;\n" ;
|
|
}
|
|
if ($filebase) {
|
|
print $out "}\n";
|
|
close $out;
|
|
}
|
|
|
|
close(TAR);
|
|
}
|
|
|
|
sub create_def_legacy {
|
|
my $p; my %seen;
|
|
my ($key) =@_;
|
|
my $field=$key;
|
|
|
|
if ($key =~ /paramId/) { $field="param.id"; }
|
|
if ($key =~ /name/) { $field="param.name"; }
|
|
if ($key =~ /units/) { $field="units.name"; }
|
|
|
|
my $query= <<"EOF";
|
|
select $field,force128,edition,
|
|
centre.abbreviation,param_id,attribute.name,attribute_value,param.name,param.shortName
|
|
from param,grib_encoding,grib,attribute,centre,units where
|
|
param.hide_def=0 and
|
|
# param.retired=0 and
|
|
grib_encoding.id=grib.encoding_id and
|
|
param.id=grib_encoding.param_id and
|
|
attribute.id=grib.attribute_id and
|
|
centre.id=grib_encoding.centre_id and
|
|
units.id=param.units_id and
|
|
grib_encoding.is_legacy=1
|
|
order by edition,centre_id,param.o,param.id,grib_encoding.param_version,attribute.o;
|
|
EOF
|
|
|
|
my $qh=$dbh->prepare($query);
|
|
$qh->execute();
|
|
|
|
# file containing the list of grib api parameters files we want to tar and
|
|
# distribute to users for them to download and update their list of parameter
|
|
# to the latest
|
|
#open(TAR,$tarfilesflag ? ">>" : ">","tarfiles.txt") or die "Count not open file tarfiles.txt: $!";
|
|
#$tarfilesflag=1;
|
|
|
|
while (my ($keyval,$force128,$edition,$centre,$paramId,$attribute,$value,$name,$shortName)=$qh->fetchrow_array )
|
|
{
|
|
if ($centre eq "wmo" ) { $conceptDir=""; }
|
|
else { $conceptDir="/localConcepts/$centre"; }
|
|
#if ($key =~ /paramId/ && $force128==1 && $keyval >1000) {
|
|
# $keyval= $keyval % 1000;
|
|
#}
|
|
|
|
if ($filebase ne "$basedir/grib$edition$conceptDir") {
|
|
if ($filebase) {
|
|
print $out "}\n";
|
|
close $out;
|
|
}
|
|
$filebase="$basedir/grib$edition$conceptDir";
|
|
mkpath($filebase);
|
|
|
|
#copy("$filebase/$key.def","$filebase/$key.def.bkp")
|
|
# or die ("unable to copy $filebase/$key.def");
|
|
|
|
print TAR "grib$edition$conceptDir/$key.legacy.def\n";
|
|
#system("cp -f $filebase/$key.def $filebase/$key.def.orig");
|
|
open($out,"> $filebase/$key.legacy.def")
|
|
or die "unable to open $filebase/$key.legacy.def";
|
|
print $out "# Automatically generated by $0, do not edit\n";
|
|
$p=();
|
|
}
|
|
if ($p ne $paramId || exists($seen{$attribute}) ) {
|
|
if ($p) { print $out "\t}\n"; }
|
|
print $out "#$name\n" ;
|
|
print $out "\'".$keyval."\' = {\n" ;
|
|
$p=$paramId;
|
|
%seen=();
|
|
}
|
|
$seen{$attribute}=1;
|
|
print "($key=$keyval) $edition,$centre,$shortName,$paramId,$name,$attribute,$value\n";
|
|
# we need to allow strings in the attribute_value field
|
|
# for the moment we apply a patch here
|
|
if ($attribute =~ /stepType/ ) {
|
|
$value="\"accum\"";
|
|
}
|
|
if ($value eq '') {
|
|
$value="missing()";
|
|
}
|
|
print $out "\t $attribute = $value ;\n" ;
|
|
}
|
|
if ($filebase) {
|
|
print $out "}\n";
|
|
close $out;
|
|
}
|
|
|
|
close(TAR);
|
|
}
|
|
|
|
sub create_cfVarName {
|
|
my $p; my %seen;
|
|
my ($key) =@_;
|
|
my $field=$key;
|
|
|
|
#if ($key =~ /paramId/) { $field="param.id"; }
|
|
#if ($key =~ /name/) { $field="param.name"; }
|
|
#if ($key =~ /units/) { $field="units.name"; }
|
|
if ($key =~ /cfVarName/) { $field="cfVarName"; }
|
|
|
|
my $query= <<"EOF";
|
|
select $field,force128,edition,
|
|
centre.abbreviation,param_id,attribute.name,attribute_value,param.name,param.shortName
|
|
from param,grib_encoding,grib,attribute,centre,units where
|
|
param.hide_def=0 and
|
|
grib_encoding.id=grib.encoding_id and
|
|
param.id=grib_encoding.param_id and
|
|
attribute.id=grib.attribute_id and
|
|
centre.id=grib_encoding.centre_id and
|
|
units.id=param.units_id
|
|
and cfVarName IS NOT NULL
|
|
order by edition,centre_id,param.o,param.id,grib_encoding.param_version,attribute.o;
|
|
EOF
|
|
|
|
my $qh=$dbh->prepare($query);
|
|
$qh->execute();
|
|
|
|
# file containing the list of grib api parameters files we want to tar and
|
|
# distribute to users for them to download and update their list of parameter
|
|
# to the latest
|
|
#open(TAR,$tarfilesflag ? ">>" : ">","tarfiles.txt") or die "Count not open file tarfiles.txt: $!";
|
|
#$tarfilesflag=1;
|
|
|
|
while (my ($keyval,$force128,$edition,$centre,$paramId,$attribute,$value,$name,$shortName)=$qh->fetchrow_array )
|
|
{
|
|
if ($centre eq "wmo" ) { $conceptDir=""; }
|
|
else { $conceptDir="/localConcepts/$centre"; }
|
|
#if ($key =~ /paramId/ && $force128==1 && $keyval >1000) {
|
|
# $keyval= $keyval % 1000;
|
|
#}
|
|
|
|
if ($filebase ne "$basedir/grib$edition$conceptDir") {
|
|
if ($filebase) {
|
|
print $out "}\n";
|
|
close $out;
|
|
}
|
|
$filebase="$basedir/grib$edition$conceptDir";
|
|
mkpath($filebase);
|
|
|
|
#copy("$filebase/$key.def","$filebase/$key.def.bkp")
|
|
# or die ("unable to copy $filebase/$key.def");
|
|
|
|
print TAR "grib$edition$conceptDir/$key.def\n";
|
|
#system("cp -f $filebase/$key.def $filebase/$key.def.orig");
|
|
open($out,"> $filebase/$key.def")
|
|
or die "unable to open $filebase/$key.def";
|
|
print $out "# Automatically generated by $0, do not edit\n";
|
|
$p=();
|
|
}
|
|
if ($p ne $paramId || exists($seen{$attribute}) ) {
|
|
if ($p) { print $out "\t}\n"; }
|
|
print $out "#$name\n" ;
|
|
print $out "\'".$keyval."\' = {\n" ;
|
|
$p=$paramId;
|
|
%seen=();
|
|
}
|
|
$seen{$attribute}=1;
|
|
print "($key=$keyval) $edition,$centre,$shortName,$paramId,$name,$attribute,$value\n";
|
|
# we need to allow strings in the attribute_value field
|
|
# for the moment we apply a patch here
|
|
if ($attribute =~ /stepType/ ) {
|
|
$value="\"accum\"";
|
|
}
|
|
if ($value eq '') {
|
|
$value="missing()";
|
|
}
|
|
print $out "\t $attribute = $value ;\n" ;
|
|
}
|
|
if ($filebase) {
|
|
print $out "}\n";
|
|
close $out;
|
|
}
|
|
|
|
close(TAR);
|
|
}
|
|
|
|
sub create_paramId_def {
|
|
my $p; my %seen;
|
|
|
|
my $query="select edition,centre.abbreviation,param_id,attribute.name,attribute_value,param.name,param.shortName
|
|
from param,grib_encoding,grib,attribute,centre where
|
|
param.hide_def=0 and
|
|
param.retired=0 and
|
|
grib_encoding.published=1 and
|
|
grib_encoding.id=grib.encoding_id and
|
|
param.id=grib_encoding.param_id and
|
|
attribute.id=grib.attribute_id and
|
|
centre.id=grib_encoding.centre_id and
|
|
grib_encoding.is_legacy=0
|
|
order by edition,centre_id,param.o,param.id,attribute.o";
|
|
my $qh=$dbh->prepare($query);
|
|
$qh->execute();
|
|
|
|
while (my ($edition,$centre,$paramId,$attribute,$value,$name,$shortName)=$qh->fetchrow_array )
|
|
{
|
|
if ($centre eq "wmo" ) { $conceptDir=""; }
|
|
else { $conceptDir="/localConcepts/$centre"; }
|
|
|
|
if ($filebase ne "$basedir/grib$edition$conceptDir") {
|
|
if ($filebase) {
|
|
print $out "}\n";
|
|
close $out;
|
|
}
|
|
$filebase="$basedir/grib$edition$conceptDir";
|
|
mkpath($filebase);
|
|
|
|
copy("$filebase/paramId.def","$filebase/paramId.def.bkp")
|
|
or die ("unable to copy $filebase/paramId.def");
|
|
open($out,"> $filebase/paramId.def")
|
|
or die "unable to open $filebase/paramId.def";
|
|
print $out "# Automatically generated by $0, do not edit\n";
|
|
$p=();
|
|
}
|
|
if ($p ne $paramId || exists($seen{$attribute}) ) {
|
|
if ($p) { print $out "\t}\n"; }
|
|
print $out "#$name\n" ;
|
|
print $out "\'".$paramId."\' = {\n" ;
|
|
$p=$paramId;
|
|
%seen=();
|
|
}
|
|
$seen{$attribute}=1;
|
|
print "$edition,$centre,$shortName,$paramId,$name,$attribute,$value\n";
|
|
print $out "\t $attribute = $value ;\n" ;
|
|
}
|
|
if ($filebase) {
|
|
print $out "}\n";
|
|
close $out;
|
|
}
|
|
}
|
|
|
|
sub create_def_old {
|
|
my ($key,$query)=@_;
|
|
|
|
my $qh=$dbh->prepare($query);
|
|
$qh->execute();
|
|
|
|
while (my ($edition,$centre,$paramId,$value)=$qh->fetchrow_array )
|
|
{
|
|
if ($centre eq "wmo" ) { $conceptDir=""; }
|
|
else { $conceptDir="/localConcepts/$centre"; }
|
|
|
|
if ($filebase ne "$basedir/grib$edition$conceptDir") {
|
|
if ($filebase) {
|
|
close $out;
|
|
}
|
|
$filebase="$basedir/grib$edition$conceptDir";
|
|
mkpath($filebase);
|
|
|
|
copy("$filebase/$key.def","$filebase/$key.def.bkp")
|
|
or die ("unable to copy $filebase/$key.def");
|
|
open($out,"> $filebase/$key.def")
|
|
or die "unable to open $filebase/$key.def";
|
|
print $out "# Automatically generated by $0, do not edit\n";
|
|
}
|
|
print $out "\'$value\' \t= { paramId=$paramId; }\n";
|
|
}
|
|
if ($filebase) {
|
|
close $out;
|
|
}
|
|
}
|
|
|
|
create_def("paramId");
|
|
create_def_legacy("paramId");
|
|
create_def("shortName");
|
|
create_def_legacy("shortName");
|
|
create_def("name");
|
|
create_def_legacy("name");
|
|
create_def("units");
|
|
create_def_legacy("units");
|
|
create_cfVarName("cfVarName");
|
|
create_cfName("cfName");
|
|
create_cfName_legacy("cfName");
|
|
|
|
# #create_paramId_def();
|
|
|
|
# $query="select distinct edition,centre.abbreviation,param_id,param.shortName from param,grib_encoding,centre where
|
|
# param.hide_def=0 and
|
|
# param.id=grib_encoding.param_id and
|
|
# centre.id=grib_encoding.centre_id and
|
|
# shortName!='~' order by abbreviation,edition,param.o,param.id,shortName";
|
|
|
|
|
|
# #select distinct edition,centre.abbreviation,param_id,param.shortName
|
|
# #from param,grib_encoding,grib,centre where param.hide_def=0 and param.id=grib.param_id and
|
|
# #centre.id=grib_encoding.centre_id and shortName!='~'
|
|
# #order by centre,edition,param.o,param_id";
|
|
|
|
# #create_def("shortName",$query);
|
|
|
|
# $query="select distinct edition,centre.abbreviation,param_id,param.name
|
|
# from param,grib,centre where param.hide_def=0 and param.id=grib.param_id and
|
|
# centre.id=grib.centre and shortName!='~'
|
|
# order by centre,edition,param.o,param_id";
|
|
|
|
# #create_def("name",$query);
|
|
|
|
# $query="select distinct edition,centre.abbreviation,param_id,units.name
|
|
# from param,grib,centre,units where param.hide_def=0 and param.id=grib.param_id and units.id=param.units_id
|
|
# and centre.id=grib.centre and shortName!='~'
|
|
# order by centre,edition,param.o,param_id";
|
|
|
|
# #create_def("units",$query);
|
|
|