odb_manger.sh 4.41 KB
Newer Older
saeed's avatar
saeed committed
1
2
3
4
5
6
#!/bin/bash
##SBATCH -N 1
##SBATCH -J manage_odb_data
##SBATCH -e err
##SBATCH -o log

saeed's avatar
saeed committed
7
8
module add odb/1.0.3
module add odb_api/0.19.1
saeed's avatar
saeed committed
9

saeed's avatar
m    
saeed committed
10
. ./functionsodb.sh
saeed's avatar
saeed committed
11

saeed's avatar
saeed committed
12
do_era5=1
saeed's avatar
saeed committed
13
do_precise=0
saeed's avatar
m    
saeed committed
14
do_only_monthy=1
saeed's avatar
saeed committed
15
use_report_file=0
saeed's avatar
m    
saeed committed
16
17
report_type_filename="report_type_list"
list_report=(16001 16005)
saeed's avatar
saeed committed
18

saeed's avatar
saeed committed
19
20
PRECISE_ODB_DIR="/scratch/ms/se/smdk/scratchdir/"
OUTPUT_DIR="/home/ms/se/smdk/odb_panel/OUTPUT"
saeed's avatar
saeed committed
21
path_cycle="/home/ms/se/smdk/odb_panel/bin/"
saeed's avatar
saeed committed
22

saeed's avatar
m    
saeed committed
23
24
25
26
27
startdate="2017-01-01"
enddate="2017-01-02"
time_cycle="00,12"


saeed's avatar
saeed committed
28
if [ ${use_report_file} -eq 1 ];then
saeed's avatar
m    
saeed committed
29
30
    if [ ! -f ${report_type_filename} ];then
	echo "please provide the file named ${report_type_filename}"
saeed's avatar
saeed committed
31
32
	exit
    fi
saeed's avatar
m    
saeed committed
33
    list_report=(`cat ${report_type_filename}`)
saeed's avatar
saeed committed
34
fi
saeed's avatar
saeed committed
35
36
37

checkmonthly=`ismonthly ${startdate} ${enddate}`

saeed's avatar
m    
saeed committed
38
39
if [ ${do_only_monthy} -eq 1 -a ${checkmonthly} -eq 0 ];then
    echo "we chose the monthly data retrieval from mars"
saeed's avatar
saeed committed
40
41
42
43
44
45
46
    echo "please adjust the startdate ${startdate} or enddate ${enddate}"
    exit
fi

time_cycle_out=`addbackslash ${time_cycle}`
startdate_out=`yearmonday ${startdate}`
enddate_out=`yearmonday ${enddate}`
saeed's avatar
saeed committed
47
48
49
50
51
52
53
54
55
56
57

day_1=`getday ${startdate}`
day_2=`getday ${enddate}`

drange=[]
ii=0
for j in `seq -w ${day_1} ${day_2}`;do
    drange[${ii}]=${j}
    ii=$((ii +1))
done

saeed's avatar
saeed committed
58
59
60
61
62
63
64
65
66
67
HERE=`pwd`

echo ${startdate_out}
echo ${enddate_out}

if [ ${do_era5} -eq 1 ];then
    for j in ${!list_report[*]};do
	vr=${list_report[j]}
	fin="odb_era5_${vr}_${startdate_out}_${enddate_out}.odb"
	fout=`output_name ${fin}`
saeed's avatar
saeed committed
68
69
70
71
72
73
74
75
76
77
78
	lexist=1
	nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
	for nf in `seq 1 ${nc}`;do
	    vb=`echo ${time_cycle} | cut -d, -f${nf}`
	    fout_cycle=`output_name_cycle ${fout} ${vb}`
	    if [ ! -f ${OUTPUT_DIR}/${fout_cycle} ];then
		lexist=0
	    fi
	done
	
	if [ ${lexist} -eq 0 ];then
saeed's avatar
saeed committed
79
80
81
82
	    echo "we are extracting odb for repportype ${vr}"
	    sed -e "s/<reptype>/${vr}/g" -e "s/<time_cycle>/${time_cycle_out}/g" -e "s/<startdate_out>/${startdate_out}/g" -e "s/<enddate_out>/${enddate_out}/g" -e "s/<startdate>/${startdate}/g" -e "s/<enddate>/${enddate}/g" request_odb_temp_era5 > request_odb_temp
	    mars request_odb_temp
            do_filter ${fin}
saeed's avatar
saeed committed
83
	    rm -f ${fin} 2>/dev/null
saeed's avatar
saeed committed
84
85
86
87
88
89
90
91
92
	    nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
	    for nf in `seq 1 ${nc}`;do
		vb=`echo ${time_cycle} | cut -d, -f${nf}`
		${path_cycle}/era5_cycle.exe ${fout} ${vb}
		fout_cycle=`output_name_cycle ${fout} ${vb}`
		echo "fout_cycle ${fout_cycle}"
		mv ${fout_cycle} ${OUTPUT_DIR} 2>/dev/null
	    done
	    rm -f ${fout}
saeed's avatar
saeed committed
93
	    rm -f request_odb_temp 2>/dev/null
saeed's avatar
saeed committed
94
95
96
97
98
99
100
101
	else
	    echo "${OUTPUT_DIR}/${fout} already exists"
	fi
    done
fi


if [ ${do_precise} -eq 1 ];then    
saeed's avatar
saeed committed
102
    
saeed's avatar
saeed committed
103
104
105
106
107
108
109
110
111
112
113
114
115
    if [ ! -d ${PRECISE_ODB_DIR} ];then
	echo "directory ${PRECISE_ODB_DIR} does not exist"
	exit
    fi

    year="`getyear ${startdate}`"
    mon="`getmon ${startdate}`"
    if [ ! -d ${PRECISE_ODB_DIR}/${year}/${mon} ];then
	echo "directory ${PRECISE_ODB_DIR}/${year}/${mon} does not exist"
	exit
    fi
    
    cd ${PRECISE_ODB_DIR}
saeed's avatar
saeed committed
116
        
saeed's avatar
saeed committed
117
118
119
    cd ${year}
    cd ${mon}
    HERE_P=`pwd`
saeed's avatar
saeed committed
120
    
saeed's avatar
saeed committed
121
122
123
124
125
126
127
128
129
130
131
132
133
    for j in ${!list_report[*]};do
	report_type=${list_report[j]}
	nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
	for nf in `seq 1 ${nc}`;do
	    vb=`echo ${time_cycle} | cut -d, -f${nf}`
	    fout="odb_precise_${report_type}_${startdate_out}_${enddate_out}_cycle${vb}.txt"
	    if [ -f ${OUTPUT_DIR}/${fout} ];then
		echo "file ${OUTPUT_DIR}/${fout} already exist"
	    fi
	done
    done

   
saeed's avatar
saeed committed
134
    for j in ${!list_report[*]};do
saeed's avatar
saeed committed
135
	cd ${HERE_P}
saeed's avatar
saeed committed
136
137
	report_type=${list_report[j]}
	nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
saeed's avatar
saeed committed
138
139
140
141
142
143
	for nf in `seq 1 ${nc}`;do
	    vb=`echo ${time_cycle} | cut -d, -f${nf}`
	    fout="odb_precise_${report_type}_${startdate_out}_${enddate_out}_cycle${vb}.txt"
	    if [ ! -f ${OUTPUT_DIR}/${fout} ];then
 		for va in ${drange[*]};do
		    echo "va " ${va}
saeed's avatar
saeed committed
144
145
146
147
148
149
150
151
152
153
154
155
156
		    if [ -d "${va}"/"${vb}"/odb_ccma/CCMA ];then
			cd "${va}"/"${vb}"/odb_ccma/CCMA
			dcagen
			odbsql -q "select andate,antime,degrees(lat),degrees(lon),varno,codetype, count(*) from hdr,desc,body where\
 obsvalue/=\"NULL\" and reportype in (${report_type})" | grep -v "obsvalue" > odb_precise_${report_type}_${vb}.txt
		    fi
		    cd ${HERE_P}
		done
		cd ${HERE_P}
		for va in ${drange[*]};do
		    if [ -d ${va}/${vb}/odb_ccma/CCMA ];then
			cd ${va}/${vb}/odb_ccma/CCMA
			if [ -f odb_precise_${report_type}_${vb}.txt ];then
saeed's avatar
saeed committed
157
			    sed '1d' odb_precise_${report_type}_${vb}.txt >> ${OUTPUT_DIR}/${fout}
saeed's avatar
saeed committed
158
159
			fi
		    fi
saeed's avatar
saeed committed
160
		    cd ${HERE_P}
saeed's avatar
saeed committed
161
		done
saeed's avatar
saeed committed
162
163
164
	    fi
	done
	cd ${HERE_P}
saeed's avatar
saeed committed
165
166
167
    done
fi

saeed's avatar
saeed committed
168
cd ${HERE}