first version

odb_manger.sh

+#!/bin/bash
+##SBATCH -N 1
+##SBATCH -J manage_odb_data
+##SBATCH -e err
+##SBATCH -o log
+
+module add odb_api/0.19.0
+
+function getyear(){
+    time_d=${1} 
+    year=`echo ${time_d}|cut -d- -f1`
+    echo ${year}
+}
+
+function getmon(){
+    time_d=${1} 
+    mon=`echo ${time_d}| cut -d- -f2`
+    echo ${mon}
+}
+
+function yearmonday(){
+    time_d=${1} 
+    year=`echo ${time_d}|cut -d- -f1`
+    mon=`echo ${time_d}| cut -d- -f2`
+    day=`echo ${time_d}| cut -d- -f3`
+    echo ${year}${mon}${day}
+}
+
+function addbackslash(){
+    str=${1}
+    echo ${str/,/\\\/}
+}
+
+function output_name(){
+    fin=${1}
+    fout=${fin/".odb"/".txt"}
+    echo ${fout}
+}
+
+function ismonthly(){
+
+    result=0
+    date1=$1
+    date2=$2
+    y1=`getyear ${date1}`
+    y2=`getyear ${date2}`
+    m1=`getmon ${date1}`
+    m2=`getmon ${date2}`
+    
+    if [ ${y1} -eq ${y2} -a ${m1} -eq ${m2} ];then
+	result=1
+    fi
+    echo ${result}
+}
+
+function do_filter(){
+    fin=${1}
+    fout=${fin/".odb"/".txt"}
+    if [ -f ${fin} ];then
+	odb sql 'SELECT                                       
+         date@hdr,                                                                                                  
+         time@hdr,                                                                                                  
+         lat@hdr,
+         lon@hdr,                                                                                                  
+         obsvalue@body,                                                                                               
+         varno@body,                                                                                                  
+         codetype@hdr,
+         count(*)                                                                                                
+         WHERE                                                                                               
+         datum_status.active@body == 1' -i ${fin} -o ${fout}
+    fi
+}
+
+do_era5=1
+do_precise=1
+PRECISE_ODB_DIR=""
+
+
+OUTPUT_DIR="OUTPUT"
+list_report=(16001 16005)
+startdate="2017-01-01"
+enddate="2017-01-10"
+time_cycle="00,12"
+
+checkmonthly=`ismonthly ${startdate} ${enddate}`
+
+if [ ${checkmonthly} -eq 0 ];then
+    echo "we only support the monthly data retrieval from mars"
+    echo "please adjust the startdate ${startdate} or enddate ${enddate}"
+    exit
+fi
+
+time_cycle_out=`addbackslash ${time_cycle}`
+startdate_out=`yearmonday ${startdate}`
+enddate_out=`yearmonday ${enddate}`
+HERE=`pwd`
+
+echo ${startdate_out}
+echo ${enddate_out}
+
+if [ ${do_era5} -eq 1 ];then
+
+    for j in ${!list_report[*]};do
+	vr=${list_report[j]}
+	fin="odb_era5_${vr}_${startdate_out}_${enddate_out}.odb"
+	fout=`output_name ${fin}`
+	if [ ! -f ${OUTPUT_DIR}/${fout} ];then
+	    echo "we are extracting odb for repportype ${vr}"
+	    sed -e "s/<reptype>/${vr}/g" -e "s/<time_cycle>/${time_cycle_out}/g" -e "s/<startdate_out>/${startdate_out}/g" -e "s/<enddate_out>/${enddate_out}/g" -e "s/<startdate>/${startdate}/g" -e "s/<enddate>/${enddate}/g" request_odb_temp_era5 > request_odb_temp
+	    mars request_odb_temp
+            do_filter ${fin}
+	    rm -f ${fin}
+	    mv ${fout} ${OUTPUT_DIR} 2>/dev/null
+	    rm -f request_odb_temp
+	else
+	    echo "${OUTPUT_DIR}/${fout} already exists"
+	fi
+    done
+fi
+
+
+if [ ${do_precise} -eq 1 ];then    
+
+    if [ ! -d ${PRECISE_ODB_DIR} ];then
+	echo "directory ${PRECISE_ODB_DIR} does not exist"
+	exit
+    fi
+
+    year="`getyear ${startdate}`"
+    mon="`getmon ${startdate}`"
+    if [ ! -d ${PRECISE_ODB_DIR}/${year}/${mon} ];then
+	echo "directory ${PRECISE_ODB_DIR}/${year}/${mon} does not exist"
+	exit
+    fi
+    
+    cd ${PRECISE_ODB_DIR}
+    cd ${year}
+    cd ${mon}
+    
+    HERE_P=`pwd`
+
+    list=`ls -d */`
+    nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
+    for va in ${list};do
+	for nf in `seq 1 ${nc}`;do
+	    vb=`echo ${time_cycle} | cut -d, -f${nf}`
+	    if [ -d ${va}/${vb}/odb_ccma/CCMA ];then
+		cd ${va}/${vb}/odb_ccma/CCMA
+		dcagen
+		odbsql -q "select andate,antime,degrees(lat),degrees(lon),varno,codetype, count(*) from hdr,desc,body where\
+ obsvalue/=\"NULL\" and reportype in (${report_type})" | grep -v "obsvalue" > odb_precise_${report_type}_${vb}.txt
+	    fi
+            cd ${HERE_P}
+	done
+	cd ${HERE_P}
+    done
+fi
+
+cd ${HERE}
+#!/bin/bash 
+
+HERE=`pwd`
+
+list=`ls -d */`
+
+report_type="16005"
+
+fout=${HERE}/"odb_precise_${report_type}_cycle12.txt"
+
+rm -f ${fout} 2>/dev/null
+
+for va in ${list};do
+    vb=12
+    cd ${va}/${vb}/odb_ccma/CCMA
+    if [ -f odb_precise_${report_type}_${vb}.txt ];then
+	sed '1d' odb_precise_${report_type}_${vb}.txt >> ${fout}
+    fi
+    cd ${HERE}
+done
+   

request_odb_temp_era5

+retrieve,
+class=ea,
+date=<startdate>/to/<enddate>,
+expver=1,
+reportype=<reptype>,
+stream=oper,
+time=<time_cycle>,
+type=ofb,
+area = 75.346607/-58.104964/20.292000/74.104614, #europe N/W/S/E area	
+target="odb_era5_<reptype>_<startdate_out>_<enddate_out>.odb"