From 48b7411648165a103a91ace755236664c2aeda16 Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Mon, 16 Mar 2015 11:37:24 +0000 Subject: [PATCH] BUFR tools documentation for beta release ECC-14 --- confluence/confUtils.pm | 60 +++++ confluence/examples.par | 33 +++ confluence/load_examples.pl | 389 +++++++++++++++++++++++++++++ confluence/load_tools.pl | 482 ++++++++++++++++++++++++++++++++++++ confluence/tools.par | 31 +++ tools/bufr_count.desc | 12 + tools/codes_count.desc | 11 + tools/codes_info.desc | 14 ++ tools/grib_count.desc | 11 + 9 files changed, 1043 insertions(+) create mode 100644 confluence/confUtils.pm create mode 100644 confluence/examples.par create mode 100755 confluence/load_examples.pl create mode 100755 confluence/load_tools.pl create mode 100644 confluence/tools.par create mode 100644 tools/bufr_count.desc create mode 100644 tools/codes_count.desc create mode 100644 tools/codes_info.desc create mode 100644 tools/grib_count.desc diff --git a/confluence/confUtils.pm b/confluence/confUtils.pm new file mode 100644 index 000000000..22c9044d2 --- /dev/null +++ b/confluence/confUtils.pm @@ -0,0 +1,60 @@ +package confUtils; + +use strict; + +my $CONF=$ENV{CONF}; + +#==================================================== +# +# Generic functions related to confluence +# +#==================================================== + +#-------------------------------- +# Upload a page to confluence +#-------------------------------- + +sub loadToConf { + + my ($fHtml,$title,$parentPage) = @_; + + my $cmd="$CONF -a addPage --noConvert --replace --space \"ECC\" --title \"$title\" --parent \"$parentPage\" --file $fHtml"; + + print "\t\tUpload to conflunce with command:\n\t$cmd\n"; + system($cmd); +} + +#--------------------------------------- +# Create link to a page +#--------------------------------------- + +sub linkToPage { + + my ($page,$label) = @_; + + my $str=" + + + + + "; + + return $str; +} + +sub makeInfo { + + my ($text) = @_; + + return "false". + "".$text.""; +} + +sub makeCode { + + my ($text) = @_; + + return ""; +} + +1; diff --git a/confluence/examples.par b/confluence/examples.par new file mode 100644 index 000000000..6ed793e7d --- /dev/null +++ b/confluence/examples.par @@ -0,0 +1,33 @@ +#examples + +GRIB ( + clone + ensemble_index + get + index + iterator_bitmap + iterator + keys_iterator + multi_write + multi + nearest + precision + print_data + set_bitmap + set_data + set_pv + set + ) +!BUFR ( + bufr_attributes + bufr_clone + bufr_expanded + bufr_get_keys + bufr_keys_iterator + bufr_missing + bufr_print_header + bufr_print_data + bufr_set_keys + bufr_subset + ) + diff --git a/confluence/load_examples.pl b/confluence/load_examples.pl new file mode 100755 index 000000000..fc0ba80e9 --- /dev/null +++ b/confluence/load_examples.pl @@ -0,0 +1,389 @@ +#!/usr/local/apps/perl/current/bin/perl -I/usr/local/lib/metaps/perl + +use Getopt::Long; +use strict; + +#-------------------------------------------------- +# +#-------------------------------------------------- + +#The root page of examples in confluence!!! +my $rootPage="API examples"; + +my @langKeys=("c","f","p"); + +my %langName = ("f" => "Fortran 90", + "c" => "C", + "p" => "Python"); + +my %langType = ("f" => "none", + "c" => "cpp", + "p" => "python"); + +my %langSuffix = ("f" => "f90", + "c" => "c", + "p" => "py"); + +my %langDir = ("f" => "F90", + "c" => "C", + "p" => "python"); + + +my $CONF=$ENV{CONF}; +my $inRootDir; + +#--------------------------------- +# Read arguments +#--------------------------------- + +GetOptions("inDir=s" => \$inRootDir) + or die("Error in command line arguments\n"); + +#---------------------------------- +# Dirs +#---------------------------------- + +#The root dir of the examples +$inRootDir="../examples"; + +#The genareted html files are stored here +my $htmlDir=$ENV{TMPDIR}."/res_html"; + + +my %allExamples=read_examples(); + +#---------------------------------------- +# Preparations +#---------------------------------------- + +unless ( -d $htmlDir ) { + mkdir $htmlDir +} + +#--------------------------------------------------- +# Loop for the code types (e.g. grib, bufr, etc.) +#--------------------------------------------------- + +foreach my $cType (keys %allExamples) { + + print "----------------------------------------\n"; + print " Processing examples for: $cType\n"; + print "----------------------------------------\n"; + + #Get the examples + my @examples=@{$allExamples{$cType}}; + + if($#examples+1 == 0) { + die "No examples are defined!\n"; + } + + #--------------------------------------------------- + # Create a parent page for the examples of this type + #--------------------------------------------------- + + my $parentPage=parentPageTitle($cType); + makeParentPage($rootPage,$parentPage,$htmlDir,$cType,@examples); + + #-------------------------------- + # Loop for the examples + #-------------------------------- + + foreach my $name (@examples) { + + my $fOut=$htmlDir."/".$cType."_".$name.".html"; + open(OUT,">$fOut") or die "$fOut: $!"; + + #Description + print OUT "

Description

"; + print OUT "This example shows: ".getDescription($name)."\n"; + + #Create a tab container + print OUT "

Source code

"; + print OUT "horizontal"; + + #Add a tab for each language + foreach my $i (@langKeys) { + + my $tabName=$langName{$i}; + my $fName=$name.".".$langSuffix{$i}; + my $fPath=$inRootDir."/".$langDir{$i}."/".$fName; + my $lType=$langType{$i}; + + #Read sourcode into html format + my $c=getSourceCode($fName,$fPath,$lType); + + #If there is a source creates a tab for it + #if(length($c) > 0) { + print OUT "".$tabName." + ".$c.""; + + } + + #Close tab container + print OUT ""; + + close OUT; + + #Upload the file to confluence + my $pageTitle=examplePageTitle($name); + loadToConf($fOut,$pageTitle,$parentPage); + } +} + +#=========================================================== +#=========================================================== +# +# FUNCTIONS +# +#=========================================================== +#=========================================================== + +#==================================================== +# +# Functions related to the examples +# +#==================================================== + +#------------------------------------------ +# Read in example list +#------------------------------------------ + +sub read_examples { + + my %res=(); + my $fIn="examples.par"; + open(IN,"<$fIn") or die "$fIn: $!"; + + my $line; + my $actType; + while (defined ($line = )) { + chomp($line); + if ($line =~ /.*#/) {next;} + + #print $line."\n"; + + $line =~ s/\"//g; + + my ($a)=($line =~ /(\S+)\s*\(/); + if($a and $a !~ /\!/) { + $actType=$a; + #print "type: ".$a."\n"; + #$res{$actType}; + next; + } + + if( $line =~ /\)/) { + $actType=""; + next; + } + + if(length($actType) > 0) { + my ($b) =($line =~ /(\S+)/); + #print "ex: ".$b."\n"; + if($b) { + push(@{$res{$actType}},$b); + } + } + + } + + close IN; + + return %res; +} + +#----------------------------------------------------------------------------- +# Genarate the page title for the parent page for a given type of examples +#----------------------------------------------------------------------------- + +sub parentPageTitle { + + my ($codeType) = @_; + + return $codeType." examples"; +} + +#-------------------------------------------------------- +# Create parent page for a given type of examples +#-------------------------------------------------------- + +sub makeParentPage { + my ($rootPage,$pageTitle,$outDir,$cType,@xmp) = @_; + + my $f=$outDir."/parent_".$cType."html"; + + open(OUT,">$f") or die "$f: $!"; + + my $str=""; + $str=$str.""; + + foreach my $xm (@xmp) { + $str=$str.""; + } + $str=$str."
ExampleDescription
".linkToPage($xm,examplePageTitle($xm))."".getDescription($xm)."
"; + + print OUT $str; + + close OUT; + + #Load it to confluence + loadToConf($f,$pageTitle,$rootPage); +} + +#----------------------------------------------------------------------------- +# Genarate the page title for a given example +#----------------------------------------------------------------------------- + +sub examplePageTitle { + + my ($exName) = @_; + + return $exName; +} + +#----------------------------------------------------------- +# Get the file path for an example with the given language +#----------------------------------------------------------- + +sub examplePath { + my ($name,$lang) = @_; + return $inRootDir."/".$langDir{$lang}."/".$name.".".$langSuffix{$lang}; +} + +#----------------------------------------------------------- +# Check if the file for a given example exists. +#----------------------------------------------------------- + +sub hasExample { + + my ($name,$lang) = @_; + my $f=examplePath($name,$lang); + if( -e $f ) {return 1}; + + print "$f no!!!\n"; + + return 0; +} + +#------------------------------------------------------------ +# Generate confluence formatted html string form a source file +#------------------------------------------------------------ + +sub getSourceCode { + + my ($title,$fPath,$lang) = @_; + + my $codeTxt; + + print $fPath."\n"; + + my $str; + + if($lang eq "") { + $codeTxt=fortran_file_to_string($fPath); + #return "

"; + return "

".$codeTxt."
"; + } else { + open(IN,"<$fPath") or return ""; # die "$fPath: $!"; + $codeTxt=join("", ); + close IN; + } + + $str=" + + ".$title." + false + ".$lang." + + "; + + return $str; +} + +#-------------------------------------------------------- +# Extract the description of a given example +#-------------------------------------------------------- + +sub getDescription { + + my ($name) = @_; + + my $lang="c"; + + my $res; + + if(hasExample($name,$lang)== 1) { + my $f=examplePath($name,$lang); + + my $found=0; + open (IN,"<$f") or return ""; + while (defined (my $line = )) { + #First row + if($found==0) { + if($line =~ /\*\s*Description:/) { + $found=1; + my ($r)=($line =~ /\*\s*Description:\s*(.+)/); + if(length($r) > 2) { + $res=$r; + } + } + } else { + my ($r)=($line =~ /\*\s+(.+)/); + if(length($r) > 2) { + $res=$res." ".$r; + } else { + last; + } + } + } + + #print $f."\n"; + #print "descr: ".$res."\n"; + } + + return $res; +} + +#==================================================== +# +# Generic functions related to confluence +# +#==================================================== + +#-------------------------------- +# Upload a page to confluence +#-------------------------------- + +sub loadToConf { + + my ($fHtml,$title,$parentPage) = @_; + + system("$CONF -a addPage --noConvert --replace --space \"ECC\" --title \"$title\" --parent \"$parentPage\" --file $fHtml"); +} + +#--------------------------------------- +# Create link to a page +#--------------------------------------- + +sub linkToPage { + + my ($page,$label) = @_; + + my $str=" + + + + + "; + + return $str; +} + +sub makeInfo { + + my ($text) = @_; + + return "false". + "".$text.""; +} + diff --git a/confluence/load_tools.pl b/confluence/load_tools.pl new file mode 100755 index 000000000..19f237d56 --- /dev/null +++ b/confluence/load_tools.pl @@ -0,0 +1,482 @@ +#!/usr/local/apps/perl/current/bin/perl -I/usr/local/lib/metaps/perl -I. + +use confUtils; +use Getopt::Long; +use strict; + +#-------------------------------------------------- +# +#-------------------------------------------------- + +#The root page of examples in confluence!!! +my $rootPage="Command line tools"; + +#--------------------------------- +# Read arguments +#--------------------------------- + +#GetOptions("inDir=s" => \$inRootDir) +# or die("Error in command line arguments\n"); + +#---------------------------------- +# Dirs +#---------------------------------- + +#The tools executales and shell scripts +my $toolsDir="../tools"; + +#The genareted html files are stored here +my $htmlDir=$ENV{TMPDIR}."/res_tools_html"; + +my %cmdSwitches=(); +my %allTools=read_config(); + +#---------------------------------------- +# Preparations +#---------------------------------------- + +unless ( -d $htmlDir ) { + mkdir $htmlDir +} + +#--------------------------------------------------- +# Loop for the code types (e.g. grib, bufr, etc.) +#--------------------------------------------------- + +foreach my $cType (keys %allTools) { + + print "----------------------------------------\n"; + print " Processing tools for: $cType\n"; + print "----------------------------------------\n"; + + #Get the examples + my @tools=@{$allTools{$cType}}; + + if($#tools+1 == 0) { + die "No tools are defined!\n"; + } + + #--------------------------------------------------- + # Create a parent page for the tools of this type + #--------------------------------------------------- + + print "--> Genarate parent page\n"; + + my $parentPage=parentPageTitle($cType); + makeParentPage($rootPage,$parentPage,$htmlDir,$cType,@tools); + + #-------------------------------- + # Loop for the tools + #-------------------------------- + + foreach my $name (@tools) { + + print "-------------------------------\n--> tool: ".$name."\n"; + + my $fOut=$htmlDir."/".$cType."_".$name.".html"; + open(OUT,">$fOut") or die "$fOut: $!"; + + my $str=getDoc($name); + + $str=$str.getExample($name); + + print OUT $str; + + close OUT; + + #Upload the file to confluence + my $pageTitle=toolsPageTitle($name); + confUtils::loadToConf($fOut,$pageTitle,$parentPage); + } +} + +#=========================================================== +#=========================================================== +# +# FUNCTIONS +# +#=========================================================== +#=========================================================== + +#==================================================== +# +# Functions related to the examples +# +#==================================================== + +#------------------------------------------ +# Read in example list +#------------------------------------------ + +sub read_config { + + my %res=(); + my $fIn="tools.par"; + open(IN,"<$fIn") or die "$fIn: $!"; + + my $line; + my $actType; + while (defined ($line = )) { + chomp($line); + if ($line =~ /.*#/) {next;} + + #print $line."\n"; + + $line =~ s/\"//g; + + my ($a)=($line =~ /(\S+)\s*\(/); + if($a and $a !~ /\!/) { + $actType=$a; + #print "type: ".$a."\n"; + #$res{$actType}; + next; + } + + if( $line =~ /\)/) { + $actType=""; + next; + } + + if(length($actType) > 0) { + + #If ! presents we skip this tool + if($line =~ /^\s*!/) { + next; + + #A switch is specified for the tool's shell script + } elsif( $line =~ /\S+ \S+/) { + + my ($b,$c)=($line =~ /(\S+) (\S+)/); + if($b) { + push(@{$res{$actType}},$b); + $cmdSwitches{$b}="$c"; + } + + #The tools shell script has to be called without a switch + } else { + my ($b) =($line =~ /(\S+)/); + #print "ex: ".$b."\n"; + if($b) { + push(@{$res{$actType}},$b); + } + } + } + + } + + close IN; + + return %res; +} + +#----------------------------------------------------------------------------- +# Genarate the page title for the parent page for a given type of examples +#----------------------------------------------------------------------------- + +sub parentPageTitle { + + my ($codeType) = @_; + + return $codeType." tools"; +} + +#-------------------------------------------------------- +# Create parent page for a given type of examples +#-------------------------------------------------------- + +sub makeParentPage { + my ($rootPage,$pageTitle,$outDir,$cType,@xmp) = @_; + + my $f=$outDir."/parent_".$cType."html"; + + open(OUT,">$f") or die "$f: $!"; + + my $str=""; + $str=$str.""; + + foreach my $xm (@xmp) { + $str=$str.""; + } + $str=$str."
NameDescription
".confUtils::linkToPage($xm,toolsPageTitle($xm))."".getDescription($xm)."
"; + + print OUT $str; + + close OUT; + + #Load it to confluence + confUtils::loadToConf($f,$pageTitle,$rootPage); +} + +#----------------------------------------------------------------------------- +# Genarate the page title for a given example +#----------------------------------------------------------------------------- + +sub toolsPageTitle { + + my ($exName) = @_; + + return $exName; +} + +#----------------------------------------------------------- +# Get the file path for an example with the given language +#----------------------------------------------------------- + +sub toolsPath { + my ($name) = @_; + return $toolsDir."/".$name; +} + +#----------------------------------------------------------- +# Check if the file for a given example exists. +#----------------------------------------------------------- + +sub hasExample { + + my ($name,$lang) = @_; + #my $f=examplePath($name,$lang); + #if( -e $f ) {return 1}; + + #print "$f no!!!\n"; + + return 0; +} + +sub docPath { + + my ($name) = @_; + + my $fTmp=$htmlDir."/doc_".$name.".tmp"; + + my $exe=$name; + if( -e $toolsDir."/".$exe) { + + my $swt; + if(defined($cmdSwitches{$name})) { + $swt=$cmdSwitches{$name}; + } + + my $cmd; + + #If the switch is desc it does not run the shell script but + #use the .desc file" + if ($swt eq "desc") { + $cmd="cd ".$toolsDir."; cp ".$name.".desc ".$fTmp; + } else { + $cmd="cd ".$toolsDir.";./".$exe." > ".$fTmp; + } + + print "\tGenerate documentation with command:\n\t\t".$cmd."\n"; + + system($cmd); + return $fTmp; + } + + return ""; +} + +#------------------------------------------------------------ +# +#------------------------------------------------------------ + +sub examplePath { + + my ($name) = @_; + + my $fTmp=$htmlDir."/ex_".$name.".tmp"; + + my $script=$name.".sh"; + if( -e $toolsDir."/".$script) { + + my $cmd="cd ".$toolsDir.";./".$script." > ".$fTmp; + print "\tGenerate examples text with command:\n\t\t".$cmd."\n"; + system($cmd); + return $fTmp; + } + + return ""; +} + + +#------------------------------------------------------------ +# Generate confluence formatted html string form a source file +#------------------------------------------------------------ + +sub getDoc { + + my ($name) = @_; + + my $fPath=docPath($name); + + my $str; + my $tbody=0; + + open (IN,"<$fPath") or return ""; + while (defined (my $line = )) { + + $line =~ s/^\s+//g; + $line=~ s/\n//g; + + if($line =~ /^NAME\s*/) { + next; + } + + if($line =~ /^DESCRIPTION|USAGE|OPTIONS\s*/) { + + if($tbody == 1) { + $str=$str."

"; + $tbody=0; + } + + $str=$str."

".$line."

"; + + unless($line =~ /OPTIONS\s*/) + { + $str=$str."

"; + $tbody=1; + } + + } elsif( $line =~ /^\s*-\w /) { + + if($tbody == 1) { + $str=$str."

"; + $tbody=0; + } + + my @v=split(/\t|\s{2,}/,$line); + + if($#v+1 > 0) { + $str=$str."
".@v[0]."

"; + if($#v+1 > 1) { + for(my $k=1; $k <= $#v; $k++) { + $str=$str.@v[$k]." "; + } + } + } + + $tbody=1; + + } else { + + if($line =~ /\S/) { + $str=$str.$line." "; + } elsif ($tbody == 1) { + $str=$str."

"; + $tbody=0; + } + } + } + + close IN; + if($tbody == 1) { + $str=$str."

"; + } + + return $str; +} + +#-------------------------------------------------------- +# Extract the description of a given example +#-------------------------------------------------------- + +sub getDescription { + + my ($name) = @_; + + my $fPath=docPath($name); + + print "\tExtract desription from: ".$fPath."\n"; + + my $str; + + local $/=undef; + open (IN,"<$fPath") or return ""; + $str=; + close IN; + + ($str) = ($str =~ /DESCRIPTION\s*([\S|\s]+)USAGE/); + $str =~ s/\n/ /g; + + return $str; +} + +#-------------------------------------------------------- +# +#-------------------------------------------------------- + +sub getExample { + + my ($name) = @_; + + my $fPath=examplePath($name); + + unless($fPath) { + return ""; + } + + my $str; + + local $/=undef; + open (IN,"<$fPath") or return ""; + $str=; + close IN; + + my @v=split(/-#/,$str); + + $str="

".$name." examples

"; + $str=$str."
    "; + foreach my $block (@v) { + print "block=".$block."\n"; + + #If there is text + if($block =~ /\S+/) { + + $block =~ s/\\n//g; + $block =~ s/\n>/\n> /g; + $block =~ s/&&/&&/g; + + my $blockMod; + my $prevLine; + my $inList=0; + foreach my $line (split(/\n/,$block)) { + + if($line =~ /\\verbatim/) { + $line =~ s/\\verbatim/<\/ac:plain-text-body><\/ac:structured-macro>/g; + } elsif($line =~ /\\code/) { + $line =~ s/\\code/<\/ac:plain-text-body><\/ac:structured-macro>/g; + } elsif($line =~ /^\s*- /) { + $line =~ s/^\s*- /
  1. /; + + if($inList == 1) { + $line="
    2. ".$line; + } else { + $line="
      ".$line; + } + + $inList=1; + } + + if($line =~ /^\s*\.\s*/ and $inList==1) { + $line="
    "; + $inList=0; + } + + $prevLine=$line; + + $blockMod=$blockMod.$line."\n"; + } + + $str=$str."

  2. ".$blockMod."


    3. "; + } + + } + + $str=$str."
"; + + + return $str; +} diff --git a/confluence/tools.par b/confluence/tools.par new file mode 100644 index 000000000..d5d0b4e9b --- /dev/null +++ b/confluence/tools.par @@ -0,0 +1,31 @@ +#examples + +GRIB ( + grib_compare + grib_copy + grib_count desc + grib_dump + grib_filter + grib_get + grib_get_data + grib_index_build + grib_ls + grib_set + grib_to_netcdf + ) +BUFR ( + bufr_compare + bufr_copy + bufr_count desc + bufr_dump + bufr_filter + bufr_get + bufr_index_build + bufr_ls + bufr_set + ) +Generic ( + codes_count desc + codes_info desc + ) + diff --git a/tools/bufr_count.desc b/tools/bufr_count.desc new file mode 100644 index 000000000..701a54a66 --- /dev/null +++ b/tools/bufr_count.desc @@ -0,0 +1,12 @@ + +NAME bufr_count + +DESCRIPTION + Print the total number of BUFR messages in the given files. + +USAGE + bufr_count [options] file file ... + +OPTIONS + -v Verbose mode. The number of messages is given for each file. + diff --git a/tools/codes_count.desc b/tools/codes_count.desc new file mode 100644 index 000000000..2ca86efbe --- /dev/null +++ b/tools/codes_count.desc @@ -0,0 +1,11 @@ +NAME codes_count + +DESCRIPTION + Print the total number of GRIB or BUFR messages in the given files. + +USAGE + codes_count [options] file file ... + +OPTIONS + -v Verbose mode. The number of messages is given for each file. + diff --git a/tools/codes_info.desc b/tools/codes_info.desc new file mode 100644 index 000000000..2d6c7ea15 --- /dev/null +++ b/tools/codes_info.desc @@ -0,0 +1,14 @@ +NAME codes_ls + +DESCRIPTION + + Print information about the version of ecCodes, the default definitions and samples path. If the environment variables ECCODES_DEFINITION_PATH and/or ECCODES_SAMPLES_PATH are defined, then their values will be printed + +USAGE + codes_info [options] + +OPTIONS + -v Print only the version of ecCodes. + -d Print only the default definitions path. + -s Print only the default samples path. + diff --git a/tools/grib_count.desc b/tools/grib_count.desc new file mode 100644 index 000000000..dc1fc5aeb --- /dev/null +++ b/tools/grib_count.desc @@ -0,0 +1,11 @@ +NAME grib_count + +DESCRIPTION + Print the total number of GRIB messages in the given files. + +USAGE + grib_count [options] file file ... + +OPTIONS + -v Verbose mode. The number of messages is given for each file. +