BUFR tools documentation for beta release ECC-14

This commit is contained in:
Sandor Kertesz 2015-03-16 11:37:24 +00:00
parent 28a42bcd3c
commit 48b7411648
9 changed files with 1043 additions and 0 deletions

60
confluence/confUtils.pm Normal file
View File

@ -0,0 +1,60 @@
package confUtils;
use strict;
my $CONF=$ENV{CONF};
#====================================================
#
# Generic functions related to confluence
#
#====================================================
#--------------------------------
# Upload a page to confluence
#--------------------------------
sub loadToConf {
my ($fHtml,$title,$parentPage) = @_;
my $cmd="$CONF -a addPage --noConvert --replace --space \"ECC\" --title \"$title\" --parent \"$parentPage\" --file $fHtml";
print "\t\tUpload to conflunce with command:\n\t$cmd\n";
system($cmd);
}
#---------------------------------------
# Create link to a page
#---------------------------------------
sub linkToPage {
my ($page,$label) = @_;
my $str="<ac:link>
<ri:page ri:content-title=\"".$page."\"/>
<ac:plain-text-link-body>
<![CDATA[".$label."]]>
</ac:plain-text-link-body>
</ac:link>";
return $str;
}
sub makeInfo {
my ($text) = @_;
return "<ac:structured-macro ac:name=\"info\"><ac:parameter ac:name=\"icon\">false</ac:parameter>".
"<ac:rich-text-body>".$text."</ac:rich-text-body></ac:structured-macro>";
}
sub makeCode {
my ($text) = @_;
return "<ac:structured-macro ac:name=\"code\"><ac:plain-text-body><![CDATA[".$text."]]></ac:plain-text-body></ac:structured-macro>";
}
1;

33
confluence/examples.par Normal file
View File

@ -0,0 +1,33 @@
#examples
GRIB (
clone
ensemble_index
get
index
iterator_bitmap
iterator
keys_iterator
multi_write
multi
nearest
precision
print_data
set_bitmap
set_data
set_pv
set
)
!BUFR (
bufr_attributes
bufr_clone
bufr_expanded
bufr_get_keys
bufr_keys_iterator
bufr_missing
bufr_print_header
bufr_print_data
bufr_set_keys
bufr_subset
)

389
confluence/load_examples.pl Executable file
View File

@ -0,0 +1,389 @@
#!/usr/local/apps/perl/current/bin/perl -I/usr/local/lib/metaps/perl
use Getopt::Long;
use strict;
#--------------------------------------------------
#
#--------------------------------------------------
#The root page of examples in confluence!!!
my $rootPage="API examples";
my @langKeys=("c","f","p");
my %langName = ("f" => "Fortran 90",
"c" => "C",
"p" => "Python");
my %langType = ("f" => "none",
"c" => "cpp",
"p" => "python");
my %langSuffix = ("f" => "f90",
"c" => "c",
"p" => "py");
my %langDir = ("f" => "F90",
"c" => "C",
"p" => "python");
my $CONF=$ENV{CONF};
my $inRootDir;
#---------------------------------
# Read arguments
#---------------------------------
GetOptions("inDir=s" => \$inRootDir)
or die("Error in command line arguments\n");
#----------------------------------
# Dirs
#----------------------------------
#The root dir of the examples
$inRootDir="../examples";
#The genareted html files are stored here
my $htmlDir=$ENV{TMPDIR}."/res_html";
my %allExamples=read_examples();
#----------------------------------------
# Preparations
#----------------------------------------
unless ( -d $htmlDir ) {
mkdir $htmlDir
}
#---------------------------------------------------
# Loop for the code types (e.g. grib, bufr, etc.)
#---------------------------------------------------
foreach my $cType (keys %allExamples) {
print "----------------------------------------\n";
print " Processing examples for: $cType\n";
print "----------------------------------------\n";
#Get the examples
my @examples=@{$allExamples{$cType}};
if($#examples+1 == 0) {
die "No examples are defined!\n";
}
#---------------------------------------------------
# Create a parent page for the examples of this type
#---------------------------------------------------
my $parentPage=parentPageTitle($cType);
makeParentPage($rootPage,$parentPage,$htmlDir,$cType,@examples);
#--------------------------------
# Loop for the examples
#--------------------------------
foreach my $name (@examples) {
my $fOut=$htmlDir."/".$cType."_".$name.".html";
open(OUT,">$fOut") or die "$fOut: $!";
#Description
print OUT "<h3>Description</h3>";
print OUT "This example shows: <i>".getDescription($name)."</i>\n";
#Create a tab container
print OUT "<h3>Source code</h3>";
print OUT "<ac:structured-macro ac:name=\"auitabs\"><ac:parameter ac:name=\"direction\">horizontal</ac:parameter><ac:rich-text-body>";
#Add a tab for each language
foreach my $i (@langKeys) {
my $tabName=$langName{$i};
my $fName=$name.".".$langSuffix{$i};
my $fPath=$inRootDir."/".$langDir{$i}."/".$fName;
my $lType=$langType{$i};
#Read sourcode into html format
my $c=getSourceCode($fName,$fPath,$lType);
#If there is a source creates a tab for it
#if(length($c) > 0) {
print OUT "<ac:structured-macro ac:name=\"auitabspage\"><ac:parameter ac:name=\"title\">".$tabName."</ac:parameter>
<ac:rich-text-body>".$c."</ac:rich-text-body></ac:structured-macro>";
}
#Close tab container
print OUT "</ac:rich-text-body></ac:structured-macro>";
close OUT;
#Upload the file to confluence
my $pageTitle=examplePageTitle($name);
loadToConf($fOut,$pageTitle,$parentPage);
}
}
#===========================================================
#===========================================================
#
# FUNCTIONS
#
#===========================================================
#===========================================================
#====================================================
#
# Functions related to the examples
#
#====================================================
#------------------------------------------
# Read in example list
#------------------------------------------
sub read_examples {
my %res=();
my $fIn="examples.par";
open(IN,"<$fIn") or die "$fIn: $!";
my $line;
my $actType;
while (defined ($line = <IN>)) {
chomp($line);
if ($line =~ /.*#/) {next;}
#print $line."\n";
$line =~ s/\"//g;
my ($a)=($line =~ /(\S+)\s*\(/);
if($a and $a !~ /\!/) {
$actType=$a;
#print "type: ".$a."\n";
#$res{$actType};
next;
}
if( $line =~ /\)/) {
$actType="";
next;
}
if(length($actType) > 0) {
my ($b) =($line =~ /(\S+)/);
#print "ex: ".$b."\n";
if($b) {
push(@{$res{$actType}},$b);
}
}
}
close IN;
return %res;
}
#-----------------------------------------------------------------------------
# Genarate the page title for the parent page for a given type of examples
#-----------------------------------------------------------------------------
sub parentPageTitle {
my ($codeType) = @_;
return $codeType." examples";
}
#--------------------------------------------------------
# Create parent page for a given type of examples
#--------------------------------------------------------
sub makeParentPage {
my ($rootPage,$pageTitle,$outDir,$cType,@xmp) = @_;
my $f=$outDir."/parent_".$cType."html";
open(OUT,">$f") or die "$f: $!";
my $str="<table><tbody>";
$str=$str."<tr><th>Example</th><th>Description</th></tr>";
foreach my $xm (@xmp) {
$str=$str."<tr><td>".linkToPage($xm,examplePageTitle($xm))."</td><td>".getDescription($xm)."</td></tr>";
}
$str=$str."</tbody></table>";
print OUT $str;
close OUT;
#Load it to confluence
loadToConf($f,$pageTitle,$rootPage);
}
#-----------------------------------------------------------------------------
# Genarate the page title for a given example
#-----------------------------------------------------------------------------
sub examplePageTitle {
my ($exName) = @_;
return $exName;
}
#-----------------------------------------------------------
# Get the file path for an example with the given language
#-----------------------------------------------------------
sub examplePath {
my ($name,$lang) = @_;
return $inRootDir."/".$langDir{$lang}."/".$name.".".$langSuffix{$lang};
}
#-----------------------------------------------------------
# Check if the file for a given example exists.
#-----------------------------------------------------------
sub hasExample {
my ($name,$lang) = @_;
my $f=examplePath($name,$lang);
if( -e $f ) {return 1};
print "$f no!!!\n";
return 0;
}
#------------------------------------------------------------
# Generate confluence formatted html string form a source file
#------------------------------------------------------------
sub getSourceCode {
my ($title,$fPath,$lang) = @_;
my $codeTxt;
print $fPath."\n";
my $str;
if($lang eq "") {
$codeTxt=fortran_file_to_string($fPath);
#return "<p /><code><ac:macro ac:name=\"html\"><ac:plain-text-body><![CDATA[".$codeTxt."]]></ac:plain-text-body></ac:macro></code>";
return "<p /><pre>".$codeTxt."</pre>";
} else {
open(IN,"<$fPath") or return ""; # die "$fPath: $!";
$codeTxt=join("", <IN>);
close IN;
}
$str="
<ac:structured-macro ac:name=\"code\">
<ac:parameter ac:name=\"title\">".$title."</ac:parameter>
<ac:parameter ac:name=\"linenumbers\">false</ac:parameter>
<ac:parameter ac:name=\"language\">".$lang."</ac:parameter>
<ac:plain-text-body><![CDATA[".$codeTxt."]]></ac:plain-text-body>
</ac:structured-macro>";
return $str;
}
#--------------------------------------------------------
# Extract the description of a given example
#--------------------------------------------------------
sub getDescription {
my ($name) = @_;
my $lang="c";
my $res;
if(hasExample($name,$lang)== 1) {
my $f=examplePath($name,$lang);
my $found=0;
open (IN,"<$f") or return "";
while (defined (my $line = <IN>)) {
#First row
if($found==0) {
if($line =~ /\*\s*Description:/) {
$found=1;
my ($r)=($line =~ /\*\s*Description:\s*(.+)/);
if(length($r) > 2) {
$res=$r;
}
}
} else {
my ($r)=($line =~ /\*\s+(.+)/);
if(length($r) > 2) {
$res=$res." ".$r;
} else {
last;
}
}
}
#print $f."\n";
#print "descr: ".$res."\n";
}
return $res;
}
#====================================================
#
# Generic functions related to confluence
#
#====================================================
#--------------------------------
# Upload a page to confluence
#--------------------------------
sub loadToConf {
my ($fHtml,$title,$parentPage) = @_;
system("$CONF -a addPage --noConvert --replace --space \"ECC\" --title \"$title\" --parent \"$parentPage\" --file $fHtml");
}
#---------------------------------------
# Create link to a page
#---------------------------------------
sub linkToPage {
my ($page,$label) = @_;
my $str="<ac:link>
<ri:page ri:content-title=\"".$page."\"/>
<ac:plain-text-link-body>
<![CDATA[".$label."]]>
</ac:plain-text-link-body>
</ac:link>";
return $str;
}
sub makeInfo {
my ($text) = @_;
return "<ac:structured-macro ac:name=\"info\"><ac:parameter ac:name=\"icon\">false</ac:parameter>".
"<ac:rich-text-body>".$text."</ac:rich-text-body></ac:structured-macro>";
}

482
confluence/load_tools.pl Executable file
View File

@ -0,0 +1,482 @@
#!/usr/local/apps/perl/current/bin/perl -I/usr/local/lib/metaps/perl -I.
use confUtils;
use Getopt::Long;
use strict;
#--------------------------------------------------
#
#--------------------------------------------------
#The root page of examples in confluence!!!
my $rootPage="Command line tools";
#---------------------------------
# Read arguments
#---------------------------------
#GetOptions("inDir=s" => \$inRootDir)
# or die("Error in command line arguments\n");
#----------------------------------
# Dirs
#----------------------------------
#The tools executales and shell scripts
my $toolsDir="../tools";
#The genareted html files are stored here
my $htmlDir=$ENV{TMPDIR}."/res_tools_html";
my %cmdSwitches=();
my %allTools=read_config();
#----------------------------------------
# Preparations
#----------------------------------------
unless ( -d $htmlDir ) {
mkdir $htmlDir
}
#---------------------------------------------------
# Loop for the code types (e.g. grib, bufr, etc.)
#---------------------------------------------------
foreach my $cType (keys %allTools) {
print "----------------------------------------\n";
print " Processing tools for: $cType\n";
print "----------------------------------------\n";
#Get the examples
my @tools=@{$allTools{$cType}};
if($#tools+1 == 0) {
die "No tools are defined!\n";
}
#---------------------------------------------------
# Create a parent page for the tools of this type
#---------------------------------------------------
print "--> Genarate parent page\n";
my $parentPage=parentPageTitle($cType);
makeParentPage($rootPage,$parentPage,$htmlDir,$cType,@tools);
#--------------------------------
# Loop for the tools
#--------------------------------
foreach my $name (@tools) {
print "-------------------------------\n--> tool: ".$name."\n";
my $fOut=$htmlDir."/".$cType."_".$name.".html";
open(OUT,">$fOut") or die "$fOut: $!";
my $str=getDoc($name);
$str=$str.getExample($name);
print OUT $str;
close OUT;
#Upload the file to confluence
my $pageTitle=toolsPageTitle($name);
confUtils::loadToConf($fOut,$pageTitle,$parentPage);
}
}
#===========================================================
#===========================================================
#
# FUNCTIONS
#
#===========================================================
#===========================================================
#====================================================
#
# Functions related to the examples
#
#====================================================
#------------------------------------------
# Read in example list
#------------------------------------------
sub read_config {
my %res=();
my $fIn="tools.par";
open(IN,"<$fIn") or die "$fIn: $!";
my $line;
my $actType;
while (defined ($line = <IN>)) {
chomp($line);
if ($line =~ /.*#/) {next;}
#print $line."\n";
$line =~ s/\"//g;
my ($a)=($line =~ /(\S+)\s*\(/);
if($a and $a !~ /\!/) {
$actType=$a;
#print "type: ".$a."\n";
#$res{$actType};
next;
}
if( $line =~ /\)/) {
$actType="";
next;
}
if(length($actType) > 0) {
#If ! presents we skip this tool
if($line =~ /^\s*!/) {
next;
#A switch is specified for the tool's shell script
} elsif( $line =~ /\S+ \S+/) {
my ($b,$c)=($line =~ /(\S+) (\S+)/);
if($b) {
push(@{$res{$actType}},$b);
$cmdSwitches{$b}="$c";
}
#The tools shell script has to be called without a switch
} else {
my ($b) =($line =~ /(\S+)/);
#print "ex: ".$b."\n";
if($b) {
push(@{$res{$actType}},$b);
}
}
}
}
close IN;
return %res;
}
#-----------------------------------------------------------------------------
# Genarate the page title for the parent page for a given type of examples
#-----------------------------------------------------------------------------
sub parentPageTitle {
my ($codeType) = @_;
return $codeType." tools";
}
#--------------------------------------------------------
# Create parent page for a given type of examples
#--------------------------------------------------------
sub makeParentPage {
my ($rootPage,$pageTitle,$outDir,$cType,@xmp) = @_;
my $f=$outDir."/parent_".$cType."html";
open(OUT,">$f") or die "$f: $!";
my $str="<table><tbody>";
$str=$str."<tr><th>Name</th><th>Description</th></tr>";
foreach my $xm (@xmp) {
$str=$str."<tr><td>".confUtils::linkToPage($xm,toolsPageTitle($xm))."</td><td>".getDescription($xm)."</td></tr>";
}
$str=$str."</tbody></table>";
print OUT $str;
close OUT;
#Load it to confluence
confUtils::loadToConf($f,$pageTitle,$rootPage);
}
#-----------------------------------------------------------------------------
# Genarate the page title for a given example
#-----------------------------------------------------------------------------
sub toolsPageTitle {
my ($exName) = @_;
return $exName;
}
#-----------------------------------------------------------
# Get the file path for an example with the given language
#-----------------------------------------------------------
sub toolsPath {
my ($name) = @_;
return $toolsDir."/".$name;
}
#-----------------------------------------------------------
# Check if the file for a given example exists.
#-----------------------------------------------------------
sub hasExample {
my ($name,$lang) = @_;
#my $f=examplePath($name,$lang);
#if( -e $f ) {return 1};
#print "$f no!!!\n";
return 0;
}
sub docPath {
my ($name) = @_;
my $fTmp=$htmlDir."/doc_".$name.".tmp";
my $exe=$name;
if( -e $toolsDir."/".$exe) {
my $swt;
if(defined($cmdSwitches{$name})) {
$swt=$cmdSwitches{$name};
}
my $cmd;
#If the switch is desc it does not run the shell script but
#use the .desc file"
if ($swt eq "desc") {
$cmd="cd ".$toolsDir."; cp ".$name.".desc ".$fTmp;
} else {
$cmd="cd ".$toolsDir.";./".$exe." > ".$fTmp;
}
print "\tGenerate documentation with command:\n\t\t".$cmd."\n";
system($cmd);
return $fTmp;
}
return "";
}
#------------------------------------------------------------
#
#------------------------------------------------------------
sub examplePath {
my ($name) = @_;
my $fTmp=$htmlDir."/ex_".$name.".tmp";
my $script=$name.".sh";
if( -e $toolsDir."/".$script) {
my $cmd="cd ".$toolsDir.";./".$script." > ".$fTmp;
print "\tGenerate examples text with command:\n\t\t".$cmd."\n";
system($cmd);
return $fTmp;
}
return "";
}
#------------------------------------------------------------
# Generate confluence formatted html string form a source file
#------------------------------------------------------------
sub getDoc {
my ($name) = @_;
my $fPath=docPath($name);
my $str;
my $tbody=0;
open (IN,"<$fPath") or return "";
while (defined (my $line = <IN>)) {
$line =~ s/^\s+//g;
$line=~ s/\n//g;
if($line =~ /^NAME\s*/) {
next;
}
if($line =~ /^DESCRIPTION|USAGE|OPTIONS\s*/) {
if($tbody == 1) {
$str=$str."</p>";
$tbody=0;
}
$str=$str."<h2>".$line."</h2>";
unless($line =~ /OPTIONS\s*/)
{
$str=$str."<p>";
$tbody=1;
}
} elsif( $line =~ /^\s*-\w /) {
if($tbody == 1) {
$str=$str."</p>";
$tbody=0;
}
my @v=split(/\t|\s{2,}/,$line);
if($#v+1 > 0) {
$str=$str."<br /><b>".@v[0]."</b><br /><p>";
if($#v+1 > 1) {
for(my $k=1; $k <= $#v; $k++) {
$str=$str.@v[$k]." ";
}
}
}
$tbody=1;
} else {
if($line =~ /\S/) {
$str=$str.$line." ";
} elsif ($tbody == 1) {
$str=$str."</p>";
$tbody=0;
}
}
}
close IN;
if($tbody == 1) {
$str=$str."</p>";
}
return $str;
}
#--------------------------------------------------------
# Extract the description of a given example
#--------------------------------------------------------
sub getDescription {
my ($name) = @_;
my $fPath=docPath($name);
print "\tExtract desription from: ".$fPath."\n";
my $str;
local $/=undef;
open (IN,"<$fPath") or return "";
$str=<IN>;
close IN;
($str) = ($str =~ /DESCRIPTION\s*([\S|\s]+)USAGE/);
$str =~ s/\n/ /g;
return $str;
}
#--------------------------------------------------------
#
#--------------------------------------------------------
sub getExample {
my ($name) = @_;
my $fPath=examplePath($name);
unless($fPath) {
return "";
}
my $str;
local $/=undef;
open (IN,"<$fPath") or return "";
$str=<IN>;
close IN;
my @v=split(/-#/,$str);
$str="<h2>".$name." examples</h2>";
$str=$str."<ol>";
foreach my $block (@v) {
print "block=".$block."\n";
#If there is text
if($block =~ /\S+/) {
$block =~ s/\\n//g;
$block =~ s/\n>/\n> /g;
$block =~ s/&&/&amp;&amp;/g;
my $blockMod;
my $prevLine;
my $inList=0;
foreach my $line (split(/\n/,$block)) {
if($line =~ /\\verbatim/) {
$line =~ s/\\verbatim/<ac:structured-macro ac:name="code"><ac:plain-text-body><!\[CDATA\[/g;
} elsif($line =~ /\\endverbatim/) {
$line =~ s/\\endverbatim/\]\]><\/ac:plain-text-body><\/ac:structured-macro>/g;
} elsif($line =~ /\\code/) {
$line =~ s/\\code/<ac:structured-macro ac:name="code"><ac:plain-text-body><!\[CDATA\[/g;
} elsif($line =~ /\\endcode/) {
$line =~ s/\\endcode/\]\]><\/ac:plain-text-body><\/ac:structured-macro>/g;
} elsif($line =~ /^\s*- /) {
$line =~ s/^\s*- /<li>/;
if($inList == 1) {
$line="</li>".$line;
} else {
$line="<ul>".$line;
}
$inList=1;
}
if($line =~ /^\s*\.\s*/ and $inList==1) {
$line="</li></ul>";
$inList=0;
}
$prevLine=$line;
$blockMod=$blockMod.$line."\n";
}
$str=$str."<li><p>".$blockMod."</p></li><br />";
}
}
$str=$str."</ol>";
return $str;
}

31
confluence/tools.par Normal file
View File

@ -0,0 +1,31 @@
#examples
GRIB (
grib_compare
grib_copy
grib_count desc
grib_dump
grib_filter
grib_get
grib_get_data
grib_index_build
grib_ls
grib_set
grib_to_netcdf
)
BUFR (
bufr_compare
bufr_copy
bufr_count desc
bufr_dump
bufr_filter
bufr_get
bufr_index_build
bufr_ls
bufr_set
)
Generic (
codes_count desc
codes_info desc
)

12
tools/bufr_count.desc Normal file
View File

@ -0,0 +1,12 @@
NAME bufr_count
DESCRIPTION
Print the total number of BUFR messages in the given files.
USAGE
bufr_count [options] file file ...
OPTIONS
-v Verbose mode. The number of messages is given for each file.

11
tools/codes_count.desc Normal file
View File

@ -0,0 +1,11 @@
NAME codes_count
DESCRIPTION
Print the total number of GRIB or BUFR messages in the given files.
USAGE
codes_count [options] file file ...
OPTIONS
-v Verbose mode. The number of messages is given for each file.

14
tools/codes_info.desc Normal file
View File

@ -0,0 +1,14 @@
NAME codes_ls
DESCRIPTION
Print information about the version of ecCodes, the default definitions and samples path. If the environment variables ECCODES_DEFINITION_PATH and/or ECCODES_SAMPLES_PATH are defined, then their values will be printed
USAGE
codes_info [options]
OPTIONS
-v Print only the version of ecCodes.
-d Print only the default definitions path.
-s Print only the default samples path.

11
tools/grib_count.desc Normal file
View File

@ -0,0 +1,11 @@
NAME grib_count
DESCRIPTION
Print the total number of GRIB messages in the given files.
USAGE
grib_count [options] file file ...
OPTIONS
-v Verbose mode. The number of messages is given for each file.