odb_manger.sh 5.79 KB
Newer Older
saeed's avatar
saeed committed
1
2
3
4
5
6
#!/bin/bash
##SBATCH -N 1
##SBATCH -J manage_odb_data
##SBATCH -e err
##SBATCH -o log

saeed's avatar
saeed committed
7
8
module add odb/1.0.3
module add odb_api/0.19.1
saeed's avatar
m    
saeed committed
9
. ./functionsodb.sh
saeed's avatar
saeed committed
10

saeed's avatar
saeed committed
11
12
do_era5=1
do_precise=0
saeed's avatar
m    
saeed committed
13
14
do_only_monthy=0
use_report_file=0     ###if you want to have a list of report types in a file  
saeed's avatar
m    
saeed committed
15
report_type_filename="report_type_list"
saeed's avatar
m    
saeed committed
16
list_report=(16001)
saeed's avatar
saeed committed
17

saeed's avatar
m    
saeed committed
18
19
startdate="2017-01-05"
enddate="2017-01-06"
saeed's avatar
saeed committed
20
time_cycle="00,12"
saeed's avatar
saeed committed
21

saeed's avatar
saeed committed
22
23
PRECISE_ODB_DIR="/scratch/ms/se/smdk/scratchdir/"
OUTPUT_DIR="/home/ms/se/smdk/odb_panel/OUTPUT"
saeed's avatar
m    
saeed committed
24
path_bin="/home/ms/se/smdk/odb_panel/bin/"
saeed's avatar
saeed committed
25

saeed's avatar
saeed committed
26
if [ ${use_report_file} -eq 1 ];then
saeed's avatar
m    
saeed committed
27
28
    if [ ! -f ${report_type_filename} ];then
	echo "please provide the file named ${report_type_filename}"
saeed's avatar
saeed committed
29
30
	exit
    fi
saeed's avatar
m    
saeed committed
31
    list_report=(`cat ${report_type_filename}`)
saeed's avatar
saeed committed
32
fi
saeed's avatar
saeed committed
33
34
35

checkmonthly=`ismonthly ${startdate} ${enddate}`

saeed's avatar
m    
saeed committed
36
if [ ${do_only_monthy} -eq 1 -a ${checkmonthly} -eq 0 ];then
saeed's avatar
m    
saeed committed
37
    echo 
saeed's avatar
m    
saeed committed
38
39
    echo "you chose the monthly data retrieval from mars"
    echo "please adjust the startdate ${startdate} or enddate ${enddate} so that they have the same month and the same year"
saeed's avatar
m    
saeed committed
40
    echo
saeed's avatar
saeed committed
41
42
43
44
45
46
    exit
fi

time_cycle_out=`addbackslash ${time_cycle}`
startdate_out=`yearmonday ${startdate}`
enddate_out=`yearmonday ${enddate}`
saeed's avatar
saeed committed
47
48
49
50
51
52
53
54
55
56
57

day_1=`getday ${startdate}`
day_2=`getday ${enddate}`

drange=[]
ii=0
for j in `seq -w ${day_1} ${day_2}`;do
    drange[${ii}]=${j}
    ii=$((ii +1))
done

saeed's avatar
saeed committed
58
59
60
61
62
63
64
65
66
67
HERE=`pwd`

echo ${startdate_out}
echo ${enddate_out}

if [ ${do_era5} -eq 1 ];then
    for j in ${!list_report[*]};do
	vr=${list_report[j]}
	fin="odb_era5_${vr}_${startdate_out}_${enddate_out}.odb"
	fout=`output_name ${fin}`
saeed's avatar
saeed committed
68
69
70
71
72
73
	lexist=1
	nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
	for nf in `seq 1 ${nc}`;do
	    vb=`echo ${time_cycle} | cut -d, -f${nf}`
	    fout_cycle=`output_name_cycle ${fout} ${vb}`
	    if [ ! -f ${OUTPUT_DIR}/${fout_cycle} ];then
saeed's avatar
saeed committed
74
75
76
77
78
79
80
81
82
		echo "==========================================="
		echo "we are extracting odb for repportype ${vr} for a period ${startdate_out} ${enddate_out} cycle ${fout_cycle}"
		echo "==========================================="
		sed -e "s/<reptype>/${vr}/g" -e "s/<cycle>/${vb}/g" -e "s/<startdate_out>/${startdate_out}/g" -e "s/<enddate_out>/${enddate_out}/g" -e "s/<startdate>/${startdate}/g" -e "s/<enddate>/${enddate}/g" request_odb_temp_era5 > request_odb_temp
		mars request_odb_temp
		if [ ! -f ${fin} ];then
		    echo "something went wrong in creating odb file ${fin} so we quit check your mars request"
		fi
		do_filter ${fin} &>/dev/null
saeed's avatar
m    
saeed committed
83
	    
saeed's avatar
saeed committed
84
85
86
87
88
89
		if [ $? -ne 0 ];then
		    echo "some thing went wrong in odb filtering on file ${fin} so we quit"
		    echo "it is very likely that you have a problem with time_cycle ${time_cycle_out} so maybe you can change the cycle"
		    rm -f ${fin} 2>/dev/null
		    exit
		fi
saeed's avatar
m    
saeed committed
90
		rm -f ${fin} 2>/dev/null
saeed's avatar
m    
saeed committed
91
		${path_bin}/era5_cycle.exe ${fout} ${vb}
saeed's avatar
saeed committed
92
		mv ${fout_cycle} ${OUTPUT_DIR} 2>/dev/null
saeed's avatar
saeed committed
93
	    	rm -f ${fout}
saeed's avatar
saeed committed
94
95
96
97
98
		rm -f request_odb_temp 2>/dev/null
	    else
		echo "${OUTPUT_DIR}/${fout_cycle} already exists"
	    fi
	done
saeed's avatar
saeed committed
99
100
101
    done
fi

saeed's avatar
saeed committed
102
103
rm -f  start_end.dat 2>/dev/null

saeed's avatar
m    
saeed committed
104
${path_bin}/date_range.exe ${startdate_out} ${enddate_out}
saeed's avatar
saeed committed
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133

if [ ! -f start_end.dat ];then
    echo "start_end.dat file does not exist"
    exit
fi

list=(`cat start_end.dat`)
len=${#list[*]}

len=$((len / 2))
len_m1=$((len - 1))

year_array=[]
mon_array=[]
day_array1=[]
day_array2=[]

for j in `seq 0 ${len_m1}`;do
    j1=$((j * 2 + 0))
    j2=$((j * 2 + 1))
    year=`getyear_1 ${list[${j1}]}`
    month=`getmonth_1 ${list[${j1}]}`
    dy1=`getday_1 ${list[${j1}]}`
    dy2=`getday_1 ${list[${j2}]}`
    year_array[${j}]=${year}
    mon_array[${j}]=${month}
    day_array1[${j}]=${dy1}
    day_array2[${j}]=${dy2}
done
saeed's avatar
saeed committed
134
135

if [ ${do_precise} -eq 1 ];then    
saeed's avatar
saeed committed
136
    
saeed's avatar
saeed committed
137
138
139
140
    if [ ! -d ${PRECISE_ODB_DIR} ];then
	echo "directory ${PRECISE_ODB_DIR} does not exist"
	exit
    fi
saeed's avatar
saeed committed
141
          
saeed's avatar
saeed committed
142
    cd ${PRECISE_ODB_DIR}
saeed's avatar
saeed committed
143
        
saeed's avatar
saeed committed
144
    HERE_P=`pwd`
saeed's avatar
saeed committed
145
146
147
148
149
150
151
152
153
154
155
    for j in ${!list_report[*]};do
	report_type=${list_report[j]}
	nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
	for nf in `seq 1 ${nc}`;do
	    vb=`echo ${time_cycle} | cut -d, -f${nf}`
	    fout="odb_precise_${report_type}_${startdate_out}_${enddate_out}_cycle${vb}.txt"
	    if [ -f ${OUTPUT_DIR}/${fout} ];then
		echo "file ${OUTPUT_DIR}/${fout} already exist"
	    fi
	done
    done
saeed's avatar
saeed committed
156
    
saeed's avatar
saeed committed
157
    for j in ${!list_report[*]};do
saeed's avatar
saeed committed
158
	cd ${HERE_P}
saeed's avatar
saeed committed
159
160
	report_type=${list_report[j]}
	nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
saeed's avatar
saeed committed
161
162
163
164
	for nf in `seq 1 ${nc}`;do
	    vb=`echo ${time_cycle} | cut -d, -f${nf}`
	    fout="odb_precise_${report_type}_${startdate_out}_${enddate_out}_cycle${vb}.txt"
	    if [ ! -f ${OUTPUT_DIR}/${fout} ];then
saeed's avatar
saeed committed
165
166
167
168
169
170
171
172
173
174
175
176
177
178
		for ii in ${!year_array[*]};do
		    iy=${year_array[${ii}]}
		    im=${mon_array[${ii}]}
                    id1=${day_array1[${ii}]}
                    id2=${day_array2[${ii}]}
                    cd ${HERE_P}
		    for dy in `seq -w ${id1} ${id2}`;do
			va="${iy}/${im}/${dy}"
                	if [ -d "${va}"/"${vb}"/odb_ccma/CCMA ];then
			    echo "va " ${va}
			    cd "${va}"/"${vb}"/odb_ccma/CCMA
			    pwd
			    dcagen
			    odbsql -q "select andate,antime,degrees(lat),degrees(lon),varno,codetype, count(*) from hdr,desc,body where\
saeed's avatar
saeed committed
179
 obsvalue/=\"NULL\" and reportype in (${report_type})" | grep -v "obsvalue" > odb_precise_${report_type}_${vb}.txt
saeed's avatar
saeed committed
180
181
182
			fi
			cd ${HERE_P}
		    done
saeed's avatar
saeed committed
183
184
		    cd ${HERE_P}
		done
saeed's avatar
saeed committed
185
186
187
188
189
190
191
192
193
194
195
196
197
		for ii in ${!year_array[*]};do
		    iy=${year_array[${ii}]}
		    im=${mon_array[${ii}]}
                    id1=${day_array1[${ii}]}
                    id2=${day_array2[${ii}]}
                    cd ${HERE_P}
		    for dy in `seq -w ${id1} ${id2}`;do
			va="${iy}/${im}/${dy}"
                	if [ -d ${va}/${vb}/odb_ccma/CCMA ];then
			    cd ${va}/${vb}/odb_ccma/CCMA
			    if [ -f odb_precise_${report_type}_${vb}.txt ];then
				sed '1d' odb_precise_${report_type}_${vb}.txt >> ${OUTPUT_DIR}/${fout}
			    fi
saeed's avatar
saeed committed
198
			fi
saeed's avatar
saeed committed
199
200
			cd ${HERE_P}
		    done
saeed's avatar
saeed committed
201
		done
saeed's avatar
saeed committed
202
		cd ${HERE_P}
saeed's avatar
saeed committed
203
204
	    fi
	done
saeed's avatar
saeed committed
205
206
207
    done
fi

saeed's avatar
saeed committed
208
cd ${HERE}