mirror of https://github.com/ecmwf/eccodes.git
Removed ancient sms stuff
This commit is contained in:
Shahram Najm
parent
c06bd38aeb
commit
8198764240
|
|
@ -1,201 +0,0 @@
|
|||
|
||||
set suite grib_api
|
||||
|
||||
|
||||
suite $suite
|
||||
|
||||
clock hybrid
|
||||
|
||||
set USER max
|
||||
|
||||
set SMSHOME /home/ma/max/suites/$suite
|
||||
|
||||
set ARCHITECTURE 'linux itanium cluster hpia64 rs6000 ibm_power4 open_solaris sun4'
|
||||
# set MACHINES_HPIA64 'bilbo faramir boromir denethor belegorn'
|
||||
set MACHINES_HPIA64 'bilbo'
|
||||
set MACHINES_LINUX 'suse91 suse101'
|
||||
set MACHINES_ITANIUM 'itanium'
|
||||
set MACHINES_RS6000 'ecgate'
|
||||
set MACHINES_SGI ""
|
||||
set MACHINES_ALPHA ""
|
||||
set MACHINES_SUN4 'solar'
|
||||
set MACHINES_OPEN_SOLARIS 'borg2'
|
||||
set MACHINES_IBM_POWER4 'hpcd hpce'
|
||||
set MACHINES_CLUSTER 'bee01'
|
||||
|
||||
|
||||
edit RELEASE 0.8.5
|
||||
edit CURRENT 0.8.5
|
||||
|
||||
edit FLAGS32 ""
|
||||
edit FLAGS64 ""
|
||||
edit ENABLE32 ""
|
||||
edit ENABLE64 ""
|
||||
edit CC ""
|
||||
|
||||
edit BOTH ""
|
||||
|
||||
edit AMODE 32
|
||||
|
||||
edit SMSHOME $SMSHOME
|
||||
edit LOGDIR $SMSHOME
|
||||
|
||||
edit DESTINATION /usr/local/lib/metaps/lib
|
||||
edit DESTINATION64 /usr/local/lib64/metaps/lib
|
||||
edit BIN_DIR /usr/local/lib/metaps/bin
|
||||
edit BIN_DIR64 /usr/local/lib64/metaps/bin
|
||||
|
||||
edit SMSINCLUDE $SMSHOME/include
|
||||
edit SMSFILES $SMSHOME/smsfiles
|
||||
|
||||
edit DATA_DIR $SMSHOME/data
|
||||
edit SRC_DIR $SMSHOME/src
|
||||
|
||||
edit TOOLS 'grib_gen grib_ls grib_dump grib_gen'
|
||||
|
||||
# Perforce setting
|
||||
set P4_HOST suse91
|
||||
set P4_LOCAL `/usr/local/lib/metaps/bin/datasvc_perforce_dir $P4_HOST`
|
||||
|
||||
edit P4_HOST $P4_HOST
|
||||
edit P4_LOCAL $P4_LOCAL
|
||||
#
|
||||
edit USER $USER
|
||||
edit QSUB_TIME 50
|
||||
edit QSUB_MEMORY 200Mb
|
||||
edit QUEUE metapps
|
||||
|
||||
define hostmachine {
|
||||
edit HOST $1
|
||||
edit SMSCMD "/home/ma/emos/bin/smssubmit %USER% %HOST% %SMSJOB%"
|
||||
edit SMSKILL "/home/ma/emos/bin/smskill %USER% %HOST% %SMSJOB%"
|
||||
}
|
||||
|
||||
set LIBRARY $suite
|
||||
edit LIBRARY $LIBRARY
|
||||
|
||||
family admin
|
||||
task createTar
|
||||
label createTar -
|
||||
hostmachine $P4_HOST
|
||||
endfamily
|
||||
|
||||
|
||||
# Rest of the platforms
|
||||
loop PLATFORM ($ARCHITECTURE) do
|
||||
family $PLATFORM
|
||||
edit PLATFORM $PLATFORM
|
||||
case PLATFORM
|
||||
in (hpia64) do
|
||||
set MACHINES "$MACHINES_HPIA64"
|
||||
set ADDRESS '64'
|
||||
edit MAIN_ADDRESS 64
|
||||
# edit ENABLE64 "--without-perl"
|
||||
# edit ENABLE64 "--with-perl=/usr/bin/perl"
|
||||
edit CC "CC=cc"
|
||||
endin
|
||||
in (itanium) do
|
||||
set MACHINES "$MACHINES_ITANIUM"
|
||||
set ADDRESS 64
|
||||
edit MAIN_ADDRESS 64
|
||||
edit ENABLE64 "--without-perl"
|
||||
endin
|
||||
in (linux) do
|
||||
set MACHINES "$MACHINES_LINUX"
|
||||
set ADDRESS '32'
|
||||
edit MAIN_ADDRESS 32
|
||||
endin
|
||||
in (cluster) do
|
||||
set MACHINES "$MACHINES_CLUSTER"
|
||||
set ADDRESS '32 64'
|
||||
edit MAIN_ADDRESS 32
|
||||
edit ENABLE32 "--enable-32-bits-addressing-mode --without-perl"
|
||||
edit ENABLE64 "--without-perl"
|
||||
edit BOTH "yes"
|
||||
endin
|
||||
in (rs6000) do
|
||||
set MACHINES "$MACHINES_RS6000"
|
||||
set ADDRESS '32'
|
||||
edit MAIN_ADDRESS 32
|
||||
edit CC "CC=cc"
|
||||
endin
|
||||
in (sgimips) do
|
||||
set MACHINES "$MACHINES_SGI"
|
||||
set ADDRESS '32'
|
||||
edit MAIN_ADDRESS 32
|
||||
endin
|
||||
in (decalpha) do
|
||||
set MACHINES "$MACHINES_ALPHA"
|
||||
set ADDRESS '32'
|
||||
edit MAIN_ADDRESS 32
|
||||
endin
|
||||
in (sun4) do
|
||||
set MACHINES "$MACHINES_SUN4"
|
||||
set ADDRESS '32'
|
||||
edit MAIN_ADDRESS 32
|
||||
endin
|
||||
in (open_solaris) do
|
||||
set MACHINES "$MACHINES_OPEN_SOLARIS"
|
||||
set ADDRESS '64'
|
||||
edit MAIN_ADDRESS 64
|
||||
edit ENABLE64 "--without-perl"
|
||||
edit CC "CC=/opt/SUNWspro/bin/cc"
|
||||
# edit CC "CC=/opt/sfw/gcc-3/bin/gcc"
|
||||
endin
|
||||
in (ibm_power4) do
|
||||
set MACHINES "$MACHINES_IBM_POWER4"
|
||||
set ADDRESS '32 64'
|
||||
edit MAIN_ADDRESS 64
|
||||
edit ENABLE32 "--enable-32-bits-addressing-mode"
|
||||
edit ENABLE64 "--without-perl"
|
||||
edit QUEUE ns
|
||||
edit QUEUE_EPILOG ns
|
||||
edit USER_PRIORITY 100
|
||||
edit QSUB_MEMORY 160Mb
|
||||
edit QSUB_TIME 100
|
||||
edit LOGDIR /home/ma/$USER/logfiles
|
||||
endin
|
||||
endcase
|
||||
|
||||
|
||||
loop HOST ($MACHINES) do
|
||||
family $HOST
|
||||
set TMPBASE `/usr/local/lib/metaps/bin/datasvc_perforce_dir $HOST | sed 's/p4/build/g' `
|
||||
edit TMPBASE $TMPBASE
|
||||
edit HOST $HOST
|
||||
hostmachine $HOST
|
||||
task setup
|
||||
trigger /$LIBRARY/admin/createTar == complete
|
||||
label setup -
|
||||
loop AMODE ($ADDRESS) do
|
||||
family address$AMODE
|
||||
edit AMODE $AMODE
|
||||
task configure
|
||||
trigger ../setup == complete
|
||||
label configure -
|
||||
task buildLibrary
|
||||
trigger ./configure == complete
|
||||
label buildLibrary -
|
||||
# task checking
|
||||
# trigger ./buildLibrary == complete
|
||||
# label checking -
|
||||
task installLibrary
|
||||
defstatus suspended
|
||||
# trigger ./checking == complete
|
||||
trigger ./buildLibrary == complete
|
||||
label install -
|
||||
# end family address
|
||||
endfamily
|
||||
endloop
|
||||
task installDefinitions
|
||||
defstatus suspended
|
||||
trigger ./address$AMODE == complete
|
||||
# end family machines
|
||||
endfamily
|
||||
endloop
|
||||
# end family platform
|
||||
endfamily
|
||||
endloop
|
||||
|
||||
# end of suite
|
||||
endsuite
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
#==========================================================================
|
||||
# End task.
|
||||
# Clean up and exit
|
||||
#==========================================================================
|
||||
|
||||
wait
|
||||
/usr/local/bin/smscomplete
|
||||
trap 0
|
||||
|
||||
set +e
|
||||
|
||||
%include <eoj.h>
|
||||
|
||||
exit 0
|
||||
|
||||
# SMSCMD is %SMSCMD%
|
||||
# SMSNODE is %SMSNODE%
|
||||
#
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
|
||||
if [[ `hostname|cut -c1-3` = hpc ]]
|
||||
then
|
||||
JID=`/usr/local/bin/jid`
|
||||
rsh %HOST% -n /usr/local/bin/eoj -v $JID
|
||||
export NO_EPILOG='eoj'
|
||||
|
||||
rcp %LOGDIR%%SMSNAME%.%SMSTRYNO% \
|
||||
%SMSNODE%:%SMSHOME%/%SMSNAME%.%SMSTRYNO%
|
||||
|
||||
. /etc/epilog
|
||||
fi
|
||||
|
||||
if [[ "x${ARCH}x" = "xhpia64x" ]]
|
||||
then
|
||||
|
||||
rcp /var/spool/PBS/spool/${PBS_JOBID}.OU \
|
||||
%SMSNODE%:%SMSJOBOUT%
|
||||
|
||||
fi
|
||||
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
|
||||
SMSHOME=%SMSHOME%
|
||||
P4_LOCAL=%P4_LOCAL%
|
||||
P4_HOST=%P4_HOST%
|
||||
LIBRARY=%LIBRARY%
|
||||
RELEASE=%RELEASE%
|
||||
DESTINATION=%DESTINATION%
|
||||
ADDRESS=%ADDRESS%
|
||||
|
||||
set -a
|
||||
PACKAGE=${LIBRARY}-${RELEASE}
|
||||
PACKAGE_TAR_GZ=${LIBRARY}-${RELEASE}.tar.gz
|
||||
LIB_DIR=${P4_LOCAL}/grib_api/releases/${RELEASE}
|
||||
INSTALL_DIR=${DESTINATION}/${LIBRARY}/${RELEASE}
|
||||
set +a
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
#!/bin/ksh
|
||||
|
||||
set -e # stop the shell on first error
|
||||
set -u # fail when using an undefined variable
|
||||
set -x # echo script lines as they are executed
|
||||
|
||||
# Defines the three variables that are needed for any
|
||||
# communication with SMS
|
||||
|
||||
export SMS_PROG=%SMS_PROG% # SMS Remote Procedure Call number
|
||||
export SMSNODE=%SMSNODE% # The name sms that issued this task
|
||||
export SMSNAME=%SMSNAME% # The name of this current task
|
||||
export SMSPASS=%SMSPASS% # A unique password
|
||||
export SMSTRYNO=%SMSTRYNO% # Current try number of the task
|
||||
|
||||
# Tell SMS we have stated
|
||||
# The SMS variable SMSRID will be set to parameter of smsinit
|
||||
# Here we give the current PID.
|
||||
|
||||
smsinit $$
|
||||
|
||||
# Defined a error hanlder
|
||||
|
||||
ERROR() {
|
||||
set +e # Clear -e flag, so we don't fail
|
||||
smsabort # Notify SMS that something went wrong
|
||||
trap 0 # Remove the trap
|
||||
|
||||
%include <eoj.h>
|
||||
|
||||
exit 0 # End the script
|
||||
}
|
||||
|
||||
# Trap any calls to exit and errors caught by the -e flag
|
||||
|
||||
trap ERROR 0
|
||||
|
||||
# Trap any signal that may cause the script to fail
|
||||
|
||||
trap '{ echo "Killed by a signal"; ERROR ; }' 1 2 3 4 5 6 7 8 10 12 13 15
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
host=$(hostname)
|
||||
host=%HOST%
|
||||
package=%PACKAGE%
|
||||
root=%P4ROOT%/client/main
|
||||
[[ ! -d $root ]] && mkdir -p $root
|
||||
cd $root
|
||||
export P4CLIENT=${package}_${host}
|
||||
|
||||
export P4PORT=p4od:14001
|
||||
|
||||
p4login
|
||||
p4 client -t ${package}_template -o | p4 client -i
|
||||
|
||||
p4 sync %P4SYNCOPTION%
|
||||
make .init # creates make.dep
|
||||
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
:
|
||||
# QSUB -q %QUEUE%
|
||||
# QSUB -u %USER%
|
||||
# QSUB -eo
|
||||
# QSUB -ro
|
||||
# QSUB -ko
|
||||
# QSUB -nr
|
||||
# QSUB -s /bin/ksh
|
||||
# QSUB -r %TASK%_%FAMILY1%
|
||||
# QSUB -o %LOGDIR%%SMSNAME%.%SMSTRYNO%
|
||||
# QSUB -lT 500
|
||||
# QSUB -lt 450
|
||||
# QSUB -lM 200Mb
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
# QSUB -p 70
|
||||
#=======================================================================
|
||||
# step1.h
|
||||
# Define second step for copying the logfile when on the ibm-cluster
|
||||
#=======================================================================
|
||||
# @ step_name=step_1
|
||||
# @ queue
|
||||
# @ dependency=(step_1==0 || step_1!=0)
|
||||
# @ class=%QUEUE_EPILOG:emos-exp%
|
||||
# @ user_priority=100
|
||||
# @ output = /dev/null
|
||||
# @ error = /dev/null
|
||||
# @ job_name = $(jobid)
|
||||
# @ step_name = step_2
|
||||
|
||||
export LOADL_STEP_NAME=${LOADL_STEP_NAME:-step_1}
|
||||
case $LOADL_STEP_NAME in
|
||||
step_1)
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
#=======================================================================
|
||||
# step2.h
|
||||
# Define second step for copying the logfile when on the ibm-cluster
|
||||
#=======================================================================
|
||||
|
||||
#sleep for a few seconds to allow next task to get into queue
|
||||
|
||||
;;
|
||||
step_2)
|
||||
|
||||
if [[ "x${ARCH}x" = "xibm_power4x" ]]; then
|
||||
rcp %SMSJOBOUT% %SMSNODE%:%SMSJOBOUT%
|
||||
fi
|
||||
|
||||
exit
|
||||
;;
|
||||
esac
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
%manual
|
||||
|
||||
|
||||
The work is carried out
|
||||
|
||||
%end
|
||||
%include <qsub.h>
|
||||
%include <step1.h>
|
||||
%include <head.h>
|
||||
%include <grib_api.h>
|
||||
|
||||
TMPBASE=%TMPBASE%
|
||||
|
||||
cd ${TMPBASE}/${LIBRARY}/${ADDRESS}/${PACKAGE}
|
||||
|
||||
make
|
||||
|
||||
smslabel buildLibrary "`ls -l src/libgrib_api.a`"
|
||||
|
||||
%include <endt.h>
|
||||
|
||||
%include <step2.h>
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
%manual
|
||||
|
||||
Unpacks the tar file of gribex and pbio sources for the version
|
||||
defined by GRIBEX_CYCLE and "make"s the library.
|
||||
|
||||
The work is carried out in /scratch/ma/mas/CompareGribexVersions
|
||||
|
||||
%end
|
||||
%include <qsub.h>
|
||||
%include <step1.h>
|
||||
%include <head.h>
|
||||
%include <grib_api.h>
|
||||
|
||||
TMPBASE=%TMPBASE%
|
||||
|
||||
cd ${TMPBASE}/${LIBRARY}/${ADDRESS}
|
||||
|
||||
make check
|
||||
|
||||
%include <endt.h>
|
||||
|
||||
%include <step2.h>
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
%manual
|
||||
|
||||
The work is carried out in
|
||||
|
||||
%end
|
||||
%include <qsub.h>
|
||||
%include <step1.h>
|
||||
%include <head.h>
|
||||
%include <grib_api.h>
|
||||
|
||||
|
||||
cd ${P4_LOCAL}
|
||||
|
||||
[ -d $LIBRARY ] && cd $LIBRARY || { mkdir $LIBRARY ; cd $LIBRARY ; }
|
||||
[ -d releases ] && cd releases || { mkdir releases; cd releases ; }
|
||||
|
||||
|
||||
rm -rf ${RELEASE}
|
||||
mkdir ${RELEASE} ; cd ${RELEASE}
|
||||
|
||||
#[ -d ${RELEASE} ] && cd ${RELEASE} || { mkdir ${RELEASE} ; cd ${RELEASE} ; }
|
||||
|
||||
export P4CLIENT=${LIBRARY}_${P4_HOST}_${RELEASE}
|
||||
|
||||
export P4PORT=p4od:14001
|
||||
|
||||
p4login -f
|
||||
#p4login
|
||||
|
||||
p4 client -t ${LIBRARY}_${P4_HOST} -o | sed "s/main/releases\/${RELEASE}/g" | p4 client -i
|
||||
|
||||
#p4 sync -f
|
||||
p4 sync
|
||||
|
||||
p4 client -d ${LIBRARY}_${P4_HOST}_${RELEASE}
|
||||
|
||||
chmod 755 -R *
|
||||
|
||||
./autogen.sh
|
||||
|
||||
./configure
|
||||
|
||||
make dist
|
||||
|
||||
[ -f "${PACKAGE_TAR_GZ}" ] && smslabel createTar "Release number is ${RELEASE}" || { smslabel createTar "${PACKAGE_TAR_GZ} is not available" ; exit ; }
|
||||
|
||||
%include <endt.h>
|
||||
|
||||
%include <step2.h>
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
%manual
|
||||
|
||||
Unpacks the tar file of gribex and pbio sources for the version
|
||||
defined by GRIBEX_CYCLE and "make"s the library.
|
||||
|
||||
The work is carried out in /scratch/ma/mas/CompareGribexVersions
|
||||
|
||||
%end
|
||||
%include <qsub.h>
|
||||
%include <step1.h>
|
||||
%include <head.h>
|
||||
%include <grib_api.h>
|
||||
|
||||
PLATFORM=%PLATFORM%
|
||||
HOST=%HOST%
|
||||
TMPBASE=%TMPBASE%
|
||||
CURRENT=%CURRENT%
|
||||
|
||||
|
||||
[ -d ${INSTALL_DIR} ] || mkdir ${INSTALL_DIR}
|
||||
|
||||
cd ${TMPBASE}/${LIBRARY}/${ADDRESS}
|
||||
|
||||
make install
|
||||
|
||||
cd ${DESTINATION}/${LIBRARY}/
|
||||
|
||||
rm -f new current
|
||||
|
||||
ln -s ${RELEASE} new
|
||||
ln -s ${CURRENT} current
|
||||
|
||||
chmod -R 755 new current ${RELEASE} ${RELEASE}/*
|
||||
|
||||
smslabel install "`ls -l ${RELEASE}/lib`"
|
||||
|
||||
%include <endt.h>
|
||||
|
||||
%include <step2.h>
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
%manual
|
||||
|
||||
The work is carried out in
|
||||
|
||||
%end
|
||||
%include <qsub.h>
|
||||
%include <step1.h>
|
||||
%include <head.h>
|
||||
%include <grib_api.h>
|
||||
|
||||
TMPBASE=%TMPBASE%
|
||||
|
||||
cd ${TMPBASE}
|
||||
|
||||
[ -d $LIBRARY ] && cd $LIBRARY || { mkdir $LIBRARY ; cd $LIBRARY ; }
|
||||
|
||||
rcp $P4_HOST:$LIB_DIR/$PACKAGE_TAR_GZ $PACKAGE_TAR_GZ
|
||||
|
||||
[ -f "${PACKAGE_TAR_GZ}" ] && smslabel setup "${PACKAGE_TAR_GZ} is available" || { smslabel setup "${PACKAGE_TAR_GZ} is not available" ; exit ; }
|
||||
|
||||
[ -f ${PACKAGE}.tar ] && rm -f ${PACKAGE}.tar
|
||||
|
||||
gunzip $PACKAGE_TAR_GZ
|
||||
|
||||
%include <endt.h>
|
||||
|
||||
%include <step2.h>
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
#!/bin/ksh
|
||||
set -e
|
||||
job=$1
|
||||
out=$2
|
||||
$job > $out 2>&1 &
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
#!/bin/ksh
|
||||
job=$1
|
||||
out=$2
|
||||
q="-q mars"
|
||||
|
||||
. /usr/local/share/ecmwf/use/sh.sge
|
||||
|
||||
rm -f $out
|
||||
|
||||
exec 0</dev/null
|
||||
exec 1>>/marsdev/data/max/sms/submit.log 2>&1
|
||||
|
||||
n=0
|
||||
while [[ $n -lt 10 ]]
|
||||
do
|
||||
qsub $q -j yes -o $out $job && exit 0
|
||||
echo "$n: $(date) qsub failed: $?"
|
||||
sleep 1
|
||||
n=$((n+1))
|
||||
done
|
||||
echo "submit failed, too many retries"
|
||||
exit 1
|
||||
|
|
@ -1,283 +0,0 @@
|
|||
#!/usr/local/bin/perl56 -I/marsdev/data/max/webapps
|
||||
use strict;
|
||||
|
||||
|
||||
use LWP::UserAgent;
|
||||
my $ua = new LWP::UserAgent;
|
||||
|
||||
use webmars::parameter;
|
||||
use webmars::mars::request2;
|
||||
use Data::Dumper;
|
||||
|
||||
@ARGV=qw(marsod) unless(@ARGV);
|
||||
$ARGV[1]=9876 unless($ARGV[1]);
|
||||
|
||||
open(DEF,">$ARGV[0].def");
|
||||
|
||||
my $m = ($ARGV[0] eq "marsms") ? 1 : 0;
|
||||
|
||||
print DEF <<"EOF";
|
||||
suite mars_server
|
||||
limit test_archive 3
|
||||
limit test_retrieve 3
|
||||
limit compare 3
|
||||
limit grib_api 3
|
||||
limit sample 3
|
||||
limit qsub 50
|
||||
export MARSMS $m
|
||||
|
||||
family $ARGV[0]
|
||||
edit DATABASE $ARGV[0]
|
||||
limit $ARGV[0] 2
|
||||
inlimit qsub
|
||||
EOF
|
||||
|
||||
|
||||
my $host = "$ARGV[0]:$ARGV[1]";
|
||||
|
||||
my $tmp = tools::tmpfile->new;
|
||||
|
||||
my $cube = 0;
|
||||
my $empty = 0;
|
||||
|
||||
my @unset = qw(
|
||||
year month branch
|
||||
);
|
||||
|
||||
my %replace = (
|
||||
satellite => "ident",
|
||||
band => "obstype",
|
||||
);
|
||||
|
||||
my %seen;
|
||||
|
||||
my $url = "http://$host/cgi/lastcube";
|
||||
my $req = HTTP::Request->new(GET => $url);
|
||||
my $res = $ua->request($req);
|
||||
my $last = $res->content;
|
||||
|
||||
|
||||
my $suite={};
|
||||
my $smslabel = $ENV{SMSLABEL};
|
||||
|
||||
# $last=10;
|
||||
if($ARGV[2])
|
||||
{
|
||||
$cube=$ARGV[2];
|
||||
$last=$ARGV[2];
|
||||
}
|
||||
|
||||
while($cube <= $last)
|
||||
{
|
||||
system("smslabel $smslabel '$cube out of $last'")
|
||||
if($smslabel && (($cube % 100) == 0));
|
||||
|
||||
unless(-f $cube)
|
||||
{
|
||||
|
||||
for(;;)
|
||||
{
|
||||
|
||||
my $url = "http://$host/cgi/cube?id=$cube";
|
||||
my $req = HTTP::Request->new(GET => $url);
|
||||
my $res = $ua->request($req);
|
||||
|
||||
my $text = $res->content;
|
||||
|
||||
|
||||
open(OUT,">$cube") or die "$cube: $!";
|
||||
print OUT $text;
|
||||
close(OUT) or die "$cube: $!";
|
||||
|
||||
last unless($text =~ /exception/i);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
my $req = webmars::mars::request2->new_from_cube($cube);
|
||||
$cube++;
|
||||
|
||||
my ($v) = $req->get_values("expver");
|
||||
|
||||
next unless($v == 1 or $v eq "");
|
||||
|
||||
|
||||
my @r = $req->split_for_batch;
|
||||
|
||||
foreach my $s ( @r )
|
||||
{
|
||||
foreach my $r ( $s->individual_requests )
|
||||
{
|
||||
process($r->clone,1,2,2);
|
||||
process($r->clone,2,2,2);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
system("smslabel $smslabel '$cube out of $last'") if($smslabel);
|
||||
put($suite,qw(CLASS STREAM TYPE LEVTYPE RUN) );
|
||||
|
||||
|
||||
sub process {
|
||||
my ($r,$run,$max,$step) = @_;
|
||||
|
||||
my ($year) = $r->get_values("year");
|
||||
my ($month) = $r->get_values("month");
|
||||
|
||||
map { $r->unset_value($_) } @unset;
|
||||
|
||||
foreach my $k ( keys %replace )
|
||||
{
|
||||
my @x = $r->get_values($k);
|
||||
if(scalar(@x))
|
||||
{
|
||||
$r->unset_value($k);
|
||||
$r->set_values($replace{$k},@x);
|
||||
}
|
||||
}
|
||||
|
||||
# print Dumper($r);
|
||||
|
||||
|
||||
my $j = $run - 1;
|
||||
my @s;
|
||||
for(my $i = 0; $i < $max ; $i++) { push @s,$j; $j += $step;}
|
||||
|
||||
foreach my $x ( qw(date step levelist number direction frequency ) )
|
||||
{
|
||||
|
||||
my @x = $r->get_values($x);
|
||||
next unless(@x);
|
||||
|
||||
my @u;
|
||||
foreach my $s ( @s)
|
||||
{
|
||||
push @u,$x[$s] if(defined $x[$s]);
|
||||
}
|
||||
|
||||
unless(@u)
|
||||
{
|
||||
for(my $i = 0; $i < $max; $i++)
|
||||
{
|
||||
push @u,$x[$i] if(defined $x[$i]);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
$r->restrict_values($x,\@u);
|
||||
my @v = $r->get_values($x);
|
||||
die Dumper($r,\@x,\@u,\@s,$x) unless(@v);
|
||||
|
||||
}
|
||||
|
||||
my @x = $r->get_values("date");
|
||||
unless(@x)
|
||||
{
|
||||
#print "$cube year=$year month=$month @x\n";
|
||||
#print $r->as_batch,"\n";
|
||||
|
||||
if($month <= 12)
|
||||
{
|
||||
my @strmonth = qw(jan feb mar apr may jun jul aug sep oct nov dec);
|
||||
$month = @strmonth[$month-1];
|
||||
}
|
||||
else
|
||||
{
|
||||
$month =~ s/-//g;
|
||||
$month .= "01" unless(length($month) == 8);
|
||||
}
|
||||
|
||||
$r->set_values("date",$month);
|
||||
|
||||
# print $r->as_batch,"\n";
|
||||
|
||||
}
|
||||
|
||||
|
||||
my $year = int($year/10) . "0";
|
||||
|
||||
|
||||
|
||||
my $c = $r->clone;
|
||||
map { $c->unset_value($_) } qw(date time);
|
||||
|
||||
my $t = join("_",$c->target_name,$year,$run);
|
||||
$t =~ s/\W/_/g;
|
||||
|
||||
|
||||
next if($seen{$t});
|
||||
|
||||
|
||||
my $req = $r->as_batch;
|
||||
$req =~ s/\n//gs;
|
||||
$req =~ s/retrieve,//;
|
||||
|
||||
my ($class) = $r->get_values("class");
|
||||
my ($type) = $r->get_values("type");
|
||||
my ($stream) = $r->get_values("stream");
|
||||
my ($levtype) = $r->get_values("levtype");
|
||||
|
||||
$levtype = "sfc" unless($levtype);
|
||||
$type = "xx" unless($type);
|
||||
$stream = "xx" unless($stream);
|
||||
|
||||
|
||||
|
||||
if(0) {
|
||||
foreach my $e qw(country model icmodel bcmodel dbase rki rty)
|
||||
{
|
||||
my ($s) = $r->get_values($e);
|
||||
$stream .= "_$s" if(defined $s);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
push @{$suite->{$class}->{$stream}->{$type}->{$levtype}->{$run}}, [$t,$req,$cube-1];
|
||||
|
||||
# open(OUT,">reqs/$t.req") or die "reqs/$t.req: $!";
|
||||
# print OUT $r->as_batch,",target='$t.data'\n\n";
|
||||
# close(OUT) or die "reqs/$t.req: $!";
|
||||
|
||||
$seen{$t}++;
|
||||
}
|
||||
|
||||
|
||||
sub put {
|
||||
my ($suite,$name,@rest) = @_;
|
||||
|
||||
unless($name)
|
||||
{
|
||||
foreach my $s ( @{$suite} )
|
||||
{
|
||||
my ($t,$req,$cube) = @{$s};
|
||||
print DEF <<"EOF";
|
||||
|
||||
family $t
|
||||
edit REQ "$req"
|
||||
edit CUBE "$cube"
|
||||
task prod_retrieve ; inlimit $ARGV[0]
|
||||
task test_archive ; inlimit test_archive; trigger prod_retrieve == complete
|
||||
task test_retrieve ; inlimit test_retrieve; trigger test_archive == complete
|
||||
task compare ; inlimit compare; trigger test_retrieve == complete
|
||||
task sample ; label info "" ; inlimit sample; trigger prod_retrieve == complete
|
||||
#task sample ; label info "" ; inlimit sample; trigger prod_retrieve == complete
|
||||
#task request ; label info "" ; inlimit request; trigger prod_retrieve == complete
|
||||
#task gribex ; label info "" ; inlimit gribex; trigger prod_retrieve == complete
|
||||
#task cleanup ; inlimit compare; trigger compare == complete
|
||||
endfamily
|
||||
EOF
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
foreach my $x ( sort keys %{$suite} )
|
||||
{
|
||||
print DEF "family $x; edit $name '$x'\n";
|
||||
put($suite->{$x},@rest);
|
||||
print DEF "endfamily\n";
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1,93 +0,0 @@
|
|||
#!/usr/local/bin/perl56 -I/marsdev/data/max/webapps
|
||||
use strict;
|
||||
|
||||
|
||||
use LWP::UserAgent;
|
||||
my $ua = new LWP::UserAgent;
|
||||
|
||||
use webmars::parameter;
|
||||
use webmars::mars::request2;
|
||||
use Data::Dumper;
|
||||
|
||||
my @tests;
|
||||
opendir(DIR,"/marsdev/data/max/grib_api/tests.ecmwf");
|
||||
foreach my $t ( sort map { s/\.test$//; $_; } grep { /\.test$/} readdir(DIR) )
|
||||
{
|
||||
my $exit = 0;
|
||||
open(IN,"</marsdev/data/max/grib_api/tests.ecmwf/$t.test");
|
||||
while(<IN>)
|
||||
{
|
||||
chomp;
|
||||
$exit++ if(/^exit$/);
|
||||
}
|
||||
close(IN);
|
||||
push @tests,$t unless($exit);
|
||||
}
|
||||
closedir(DIR);
|
||||
|
||||
print "@tests\n";
|
||||
|
||||
open(DEF,">samples.def");
|
||||
|
||||
print DEF <<"EOF";
|
||||
suite mars_server
|
||||
limit grib_api 3
|
||||
limit qsub 50
|
||||
|
||||
family samples
|
||||
inlimit qsub
|
||||
EOF
|
||||
|
||||
foreach my $t ( @tests)
|
||||
{
|
||||
my $x = $t eq "memory" ? 1 : 5;
|
||||
print DEF "limit $t $x\n";
|
||||
}
|
||||
|
||||
opendir(S,"/vol/dataserv/grib_samples") or die "opendir /vol/dataserv/grib_samples: $!";
|
||||
|
||||
foreach my $d ( sort readdir(S) )
|
||||
{
|
||||
next if($d =~ /problem/ );
|
||||
# next unless($d =~ /brute/ );
|
||||
next if($d =~ /^\./);
|
||||
|
||||
opendir(DIR,"/vol/dataserv/grib_samples/$d");
|
||||
|
||||
print DEF "family $d\n";
|
||||
|
||||
foreach my $x ( sort readdir(DIR))
|
||||
{
|
||||
next unless($x =~ /\.\w\w\w\w/);
|
||||
|
||||
my $y = $x;
|
||||
$y =~ s/\W/_/g;
|
||||
print DEF <<"EOF";
|
||||
family $y #; autocancel +00:01
|
||||
edit SAMPLEDIR /vol/dataserv/grib_samples/$d
|
||||
edit FILE "$x"
|
||||
task copy
|
||||
EOF
|
||||
|
||||
|
||||
foreach my $t ( @tests)
|
||||
{
|
||||
print DEF <<"EOF";
|
||||
task $t
|
||||
trigger copy == complete
|
||||
inlimit /mars_server/samples:$t
|
||||
edit SMSSCRIPT $ENV{SMSFILES}/test.sms
|
||||
label info ""
|
||||
EOF
|
||||
}
|
||||
|
||||
print DEF <<"EOF";
|
||||
endfamily
|
||||
EOF
|
||||
}
|
||||
|
||||
print DEF "endfamily\n";
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
suite mars_server
|
||||
defstatus suspended
|
||||
limit test_archive 3
|
||||
limit test_retrieve 3
|
||||
limit compare 3
|
||||
limit grib_api 3
|
||||
limit gribex 3
|
||||
limit request 3
|
||||
limit sample 3
|
||||
limit qsub 50
|
||||
|
||||
edit MARSMS 0
|
||||
edit SMSTRIES 1
|
||||
|
||||
family build
|
||||
defstatus suspended
|
||||
inlimit qsub
|
||||
|
||||
task make
|
||||
edit SMSCMD "%SMSJOB% 1> %SMSJOBOUT% 2>&1 &"
|
||||
|
||||
family marser
|
||||
edit PORT 9876
|
||||
task cube
|
||||
label info ""
|
||||
endfamily
|
||||
|
||||
family marsod
|
||||
edit PORT 9876
|
||||
task cube
|
||||
label info ""
|
||||
endfamily
|
||||
|
||||
family marsms
|
||||
edit PORT 19876
|
||||
task cube
|
||||
label info ""
|
||||
endfamily
|
||||
|
||||
task samples
|
||||
|
||||
|
||||
|
||||
family extras
|
||||
task extras
|
||||
label info ""
|
||||
endfamily
|
||||
|
||||
endfamily
|
||||
|
||||
set HOME "/marsdev/data/max"
|
||||
set SMSHOME "$HOME/sms/test_gribs"
|
||||
|
||||
edit SMSHOME $SMSHOME
|
||||
edit SMSFILES $SMSHOME/smsfiles
|
||||
edit SMSINCLUDE $SMSHOME/smsinclude
|
||||
edit WORKDIR $HOME/work/test_gribs
|
||||
edit SMSCMD "$HOME/sms/submit %SMSJOB% %SMSJOBOUT%"
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
%include <head.h>
|
||||
|
||||
rm -f dump back
|
||||
#rm data
|
||||
|
||||
%include <tail.h>
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
%include <head.h>
|
||||
%include <tail.h>
|
||||
|
||||
ls -l data back
|
||||
|
||||
set +e
|
||||
cmp data back
|
||||
status=$?
|
||||
set -e
|
||||
|
||||
if [[ $status -ne 0 ]]
|
||||
then
|
||||
/home/ma/emos/bin/linux/compareGribFiles -random -fail -p0 data back
|
||||
fi
|
||||
|
||||
%include <tail.h>
|
||||
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
%include <head.h>
|
||||
|
||||
rm -f data
|
||||
ln -s %SAMPLEDIR%/%FILE% data
|
||||
|
||||
%include <tail.h>
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
%include <head.h>
|
||||
|
||||
what=%FAMILY1%
|
||||
export SMSLABEL=info
|
||||
$SMSHOME/bin/cube.pl $what %PORT%
|
||||
cdp<<@
|
||||
play -r /mars_server/$what $what.def
|
||||
exit 0
|
||||
@
|
||||
|
||||
%include <tail.h>
|
||||
|
|
@ -1,46 +0,0 @@
|
|||
%include <head.h>
|
||||
|
||||
name=%FAMILY1%
|
||||
rm -fr /scratch/ma/max/problems/$name
|
||||
|
||||
export ECCODES_DEFINITION_PATH=/home/ma/mab/src/q/definitions
|
||||
here=`pwd`
|
||||
cd /home/ma/mab/src/q
|
||||
test=$(ls *.test)
|
||||
cd $here
|
||||
cat > info
|
||||
fail=0
|
||||
for n in $test
|
||||
do
|
||||
t=$(basename $n .test)
|
||||
echo -n "$t " >> info
|
||||
banner $n
|
||||
cat info | perl -e '$/=undef;system("smslabel","info",<>);'
|
||||
set +e
|
||||
rm -f core
|
||||
/home/ma/mab/src/q/$n data > out 2>>info
|
||||
status=$?
|
||||
tail out
|
||||
[[ -f core ]] && ls -l core >> info
|
||||
rm -f core
|
||||
set -e
|
||||
|
||||
if [[ $status -eq 0 ]]
|
||||
then
|
||||
echo OK >> info
|
||||
else
|
||||
echo FAIL >> info
|
||||
fail=1
|
||||
fi
|
||||
cat info | perl -e '$/=undef;system("smslabel","info",<>);'
|
||||
|
||||
done
|
||||
|
||||
|
||||
if [[ $fail -ne 0 ]]
|
||||
then
|
||||
cp data /scratch/ma/max/problems/$name
|
||||
exit 1
|
||||
fi
|
||||
|
||||
%include <tail.h>
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
%include <head.h>
|
||||
|
||||
export ECCODES_DEFINITION_PATH=/home/ma/mab/src/q/definitions
|
||||
/home/ma/mab/src/q/gribex data || true
|
||||
|
||||
|
||||
%include <tail.h>
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
%include <head.h>
|
||||
|
||||
cd /marsdev/data/max/grib_api
|
||||
|
||||
use autoconf
|
||||
mkdir -p $TMPDIR
|
||||
|
||||
export P4CONFIG=p4config
|
||||
p4login
|
||||
p4 sync
|
||||
|
||||
|
||||
# export CFLAGS="-g -m32 -O3"
|
||||
# ./configure --prefix=`pwd`
|
||||
|
||||
pwd
|
||||
make
|
||||
|
||||
%include <tail.h>
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
%include <head.h>
|
||||
|
||||
cat %WORKDIR%/%DATABASE%/%CUBE% || true
|
||||
|
||||
if [[ ! -f data ]]
|
||||
then
|
||||
$MARSCMD <<@
|
||||
retrieve,
|
||||
database=%DATABASE%,
|
||||
%REQ%,
|
||||
target=tmp
|
||||
@
|
||||
|
||||
mv tmp data
|
||||
fi
|
||||
|
||||
%include <tail.h>
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
%include <head.h>
|
||||
|
||||
export ECCODES_DEFINITION_PATH=/home/ma/mab/src/q/definitions
|
||||
/home/ma/mab/src/q/sample data
|
||||
|
||||
|
||||
%include <tail.h>
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
%include <head.h>
|
||||
|
||||
export SMSFILES=%SMSFILES%
|
||||
export SMSLABEL=info
|
||||
$SMSHOME/bin/samples.pl
|
||||
cdp<<@
|
||||
play -r /mars_server/samples samples.def
|
||||
exit 0
|
||||
@
|
||||
|
||||
%include <tail.h>
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
%include <head.h>
|
||||
|
||||
home=/marsdev/data/max/grib_api/tests.ecmwf
|
||||
|
||||
name=%FAMILY1%
|
||||
rm -fr /scratch/ma/max/problems/$name
|
||||
|
||||
# export ECCODES_DEFINITION_PATH=/home/ma/mab/src/q/definitions
|
||||
here=`pwd`
|
||||
cd $home
|
||||
|
||||
export MARS_LANGUAGE_FILE=/home/ma/max/dwd/linux/etc/mars.def
|
||||
export MARS_CHECK_FILE=/home/ma/max/dwd/linux/chk/mars.chk
|
||||
|
||||
env | sort
|
||||
|
||||
export MARS_MM_FIRSTOFMONTH=1
|
||||
|
||||
t=%TASK%
|
||||
n=%TASK%.test
|
||||
|
||||
cd $here
|
||||
rm -f info.$t /scratch/ma/max/problems/$name.$t
|
||||
fail=0
|
||||
|
||||
set +e
|
||||
rm -f core
|
||||
$home/$n data 2>>info.$t
|
||||
status=$?
|
||||
[[ -f core ]] && ls -l core >> info.$t
|
||||
rm -f core
|
||||
set -e
|
||||
|
||||
if [[ $status -eq 0 ]]
|
||||
then
|
||||
echo OK >> info.$t
|
||||
else
|
||||
echo FAIL >> info.$t
|
||||
fail=1
|
||||
fi
|
||||
cat info.$t | perl -e '$/=undef;$x=substr(<>,-1024,1024);system("smslabel","info",$x);'
|
||||
|
||||
|
||||
if [[ $fail -ne 0 ]]
|
||||
then
|
||||
ln -s -f %SAMPLEDIR%/%FILE% /scratch/ma/max/problems/$name.$t
|
||||
# cp data /scratch/ma/max/problems/$name.$t
|
||||
echo "gribprofile /scratch/ma/max/problems/$name.$t"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
%include <tail.h>
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
%include <head.h>
|
||||
%include <tail.h>
|
||||
|
||||
|
||||
rm -f back
|
||||
export ECCODES_DEFINITION_PATH=/home/ma/mab/src/q/definitions
|
||||
/home/ma/mab/src/q/g2r data || true
|
||||
|
||||
$MARSCMD <<@
|
||||
archive,
|
||||
database=marsnext,
|
||||
%REQ%,
|
||||
source=data
|
||||
@
|
||||
|
||||
%include <tail.h>
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
%include <head.h>
|
||||
%include <tail.h>
|
||||
|
||||
#export MARS_MD5=1
|
||||
#MARSCMD="mars -n"
|
||||
|
||||
if [[ ! -f back ]]
|
||||
then
|
||||
$MARSCMD <<@
|
||||
retrieve,
|
||||
database=marsnext,
|
||||
%REQ%,
|
||||
# fi=x
|
||||
#write,fi=x,
|
||||
tar=tmp
|
||||
@
|
||||
|
||||
mv tmp back
|
||||
fi
|
||||
|
||||
%include <tail.h>
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
#!/bin/ksh
|
||||
|
||||
set -e # stop the shell on first error
|
||||
set -u # fail when using an undefined variable
|
||||
|
||||
# Defines the three variables that are needed for any
|
||||
# communication with SMS
|
||||
|
||||
export SMS_PROG=%SMS_PROG% # SMS Remote Procedure Call number
|
||||
export SMSNODE=%SMSNODE% # The name sms that issued this task
|
||||
export SMSNAME=%SMSNAME% # The name of this current task
|
||||
export SMSPASS=%SMSPASS% # A unique password
|
||||
export SMSTRYNO=%SMSTRYNO% # Current try number of the task
|
||||
|
||||
# Tell SMS we have stated
|
||||
# The SMS variable SMSRID will be set to parameter of smsinit
|
||||
# Here we give the current PID.
|
||||
|
||||
smsinit $$
|
||||
|
||||
# Defined a error hanlder
|
||||
|
||||
ERROR() {
|
||||
set +e # Clear -e flag, so we don't fail
|
||||
smsabort # Notify SMS that something went wrong
|
||||
trap 0 # Remove the trap
|
||||
. /usr/local/share/ecmwf/share/.epilog
|
||||
exit 0 # End the script
|
||||
}
|
||||
|
||||
# Trap any calls to exit and errors caught by the -e flag
|
||||
|
||||
trap ERROR 0
|
||||
|
||||
# Trap any signal that may cause the script to fail
|
||||
|
||||
trap '{ echo "Killed by a signal"; ERROR ; }' 1 2 3 4 5 6 7 8 10 12 13 15
|
||||
|
||||
ulimit -s
|
||||
ulimit -d
|
||||
ulimit -m
|
||||
ulimit -a
|
||||
|
||||
export SMSHOME=%SMSHOME%
|
||||
|
||||
workdir=%WORKDIR%/%FAMILY1%
|
||||
|
||||
[[ -d $workdir ]] || mkdir -p $workdir
|
||||
|
||||
export MARS_MM_FIRSTOFMONTH=1
|
||||
|
||||
[[ %MARSMS% -eq 1 ]] && export MARS_HOME=/home/ma/max/dwd/$ARCH
|
||||
|
||||
# hostname >> $SMSHOME/hosts
|
||||
|
||||
set -x # echo script lines as they are executed
|
||||
cd $workdir
|
||||
|
||||
export MARSCMD='mars -t'
|
||||
|
||||
#banner `hostname`
|
||||
|
||||
#export LOCAL_DEFINITION_TEMPLATES=/
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
smscomplete # Notify SMS of a normal end
|
||||
trap 0 # Remove all traps
|
||||
. /usr/local/share/ecmwf/share/.epilog
|
||||
exit 0 # End the shell
|
||||
Loading…
Reference in New Issue