odb_manger.sh 5.98 KB
Newer Older
saeed's avatar
saeed committed
1
2
3
4
5
6
#!/bin/bash
##SBATCH -N 1
##SBATCH -J manage_odb_data
##SBATCH -e err
##SBATCH -o log

saeed's avatar
saeed committed
7
8
module add odb/1.0.3
module add odb_api/0.19.1
saeed's avatar
saeed committed
9

saeed's avatar
m    
saeed committed
10
. ./functionsodb.sh
saeed's avatar
saeed committed
11

saeed's avatar
saeed committed
12
13
do_era5=0
do_precise=1
saeed's avatar
m    
saeed committed
14
do_only_monthy=1
saeed's avatar
saeed committed
15
use_report_file=0
saeed's avatar
m    
saeed committed
16
report_type_filename="report_type_list"
saeed's avatar
m    
saeed committed
17
list_report=(16001)
saeed's avatar
saeed committed
18

saeed's avatar
saeed committed
19
20
21
22
startdate="2017-01-01"
enddate="2017-01-06"
time_cycle="12"

saeed's avatar
saeed committed
23
24
PRECISE_ODB_DIR="/scratch/ms/se/smdk/scratchdir/"
OUTPUT_DIR="/home/ms/se/smdk/odb_panel/OUTPUT"
saeed's avatar
saeed committed
25
path_cycle="/home/ms/se/smdk/odb_panel/bin/"
saeed's avatar
saeed committed
26

saeed's avatar
saeed committed
27
if [ ${use_report_file} -eq 1 ];then
saeed's avatar
m    
saeed committed
28
29
    if [ ! -f ${report_type_filename} ];then
	echo "please provide the file named ${report_type_filename}"
saeed's avatar
saeed committed
30
31
	exit
    fi
saeed's avatar
m    
saeed committed
32
    list_report=(`cat ${report_type_filename}`)
saeed's avatar
saeed committed
33
fi
saeed's avatar
saeed committed
34
35
36

checkmonthly=`ismonthly ${startdate} ${enddate}`

saeed's avatar
m    
saeed committed
37
if [ ${do_only_monthy} -eq 1 -a ${checkmonthly} -eq 0 ];then
saeed's avatar
m    
saeed committed
38
39
    echo "you chose the monthly data retrieval from mars"
    echo "please adjust the startdate ${startdate} or enddate ${enddate} so that they have the same month and the same year"
saeed's avatar
saeed committed
40
41
42
43
44
45
    exit
fi

time_cycle_out=`addbackslash ${time_cycle}`
startdate_out=`yearmonday ${startdate}`
enddate_out=`yearmonday ${enddate}`
saeed's avatar
saeed committed
46
47
48
49
50
51
52
53
54
55
56

day_1=`getday ${startdate}`
day_2=`getday ${enddate}`

drange=[]
ii=0
for j in `seq -w ${day_1} ${day_2}`;do
    drange[${ii}]=${j}
    ii=$((ii +1))
done

saeed's avatar
saeed committed
57
58
59
60
61
62
63
64
65
66
HERE=`pwd`

echo ${startdate_out}
echo ${enddate_out}

if [ ${do_era5} -eq 1 ];then
    for j in ${!list_report[*]};do
	vr=${list_report[j]}
	fin="odb_era5_${vr}_${startdate_out}_${enddate_out}.odb"
	fout=`output_name ${fin}`
saeed's avatar
saeed committed
67
68
69
70
71
72
73
74
75
76
77
	lexist=1
	nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
	for nf in `seq 1 ${nc}`;do
	    vb=`echo ${time_cycle} | cut -d, -f${nf}`
	    fout_cycle=`output_name_cycle ${fout} ${vb}`
	    if [ ! -f ${OUTPUT_DIR}/${fout_cycle} ];then
		lexist=0
	    fi
	done
	
	if [ ${lexist} -eq 0 ];then
saeed's avatar
m    
saeed committed
78
79
80
	    echo "==========================================="
	    echo "we are extracting odb for repportype ${vr} for a period ${startdate_out} ${enddate_out}"
	    echo "==========================================="
saeed's avatar
saeed committed
81
82
	    sed -e "s/<reptype>/${vr}/g" -e "s/<time_cycle>/${time_cycle_out}/g" -e "s/<startdate_out>/${startdate_out}/g" -e "s/<enddate_out>/${enddate_out}/g" -e "s/<startdate>/${startdate}/g" -e "s/<enddate>/${enddate}/g" request_odb_temp_era5 > request_odb_temp
	    mars request_odb_temp
saeed's avatar
m    
saeed committed
83
84
85
86
87
88
89
90
91
92
93
	    if [ ! -f ${fin} ];then
		echo "something went wrong in creating odb file ${fin} so we quit check your mars request"
	    fi
            do_filter ${fin} &>/dev/null
	    
	    if [ $? -ne 0 ];then
		echo "some thing went wrong in odb filtering on file ${fin} so we quit"
		echo "it is very likely that you have a problem with time_cycle ${time_cycle_out} so maybe you can change the cycle"
		rm -f ${fin} 2>/dev/null
		exit
	    fi
saeed's avatar
saeed committed
94
	    rm -f ${fin} 2>/dev/null
saeed's avatar
saeed committed
95
96
97
98
99
	    nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
	    for nf in `seq 1 ${nc}`;do
		vb=`echo ${time_cycle} | cut -d, -f${nf}`
		${path_cycle}/era5_cycle.exe ${fout} ${vb}
		fout_cycle=`output_name_cycle ${fout} ${vb}`
saeed's avatar
m    
saeed committed
100
		#echo "fout_cycle ${fout_cycle}"
saeed's avatar
saeed committed
101
102
103
		mv ${fout_cycle} ${OUTPUT_DIR} 2>/dev/null
	    done
	    rm -f ${fout}
saeed's avatar
saeed committed
104
	    rm -f request_odb_temp 2>/dev/null
saeed's avatar
saeed committed
105
106
107
108
109
110
	else
	    echo "${OUTPUT_DIR}/${fout} already exists"
	fi
    done
fi

saeed's avatar
saeed committed
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
rm -f  start_end.dat 2>/dev/null

./bin/date_range.exe ${startdate_out} ${enddate_out}

if [ ! -f start_end.dat ];then
    echo "start_end.dat file does not exist"
    exit
fi

list=(`cat start_end.dat`)
len=${#list[*]}

len=$((len / 2))
len_m1=$((len - 1))

year_array=[]
mon_array=[]
day_array1=[]
day_array2=[]

for j in `seq 0 ${len_m1}`;do
    j1=$((j * 2 + 0))
    j2=$((j * 2 + 1))
    year=`getyear_1 ${list[${j1}]}`
    month=`getmonth_1 ${list[${j1}]}`
    dy1=`getday_1 ${list[${j1}]}`
    dy2=`getday_1 ${list[${j2}]}`
    year_array[${j}]=${year}
    mon_array[${j}]=${month}
    day_array1[${j}]=${dy1}
    day_array2[${j}]=${dy2}
done
saeed's avatar
saeed committed
143
144

if [ ${do_precise} -eq 1 ];then    
saeed's avatar
saeed committed
145
    
saeed's avatar
saeed committed
146
147
148
149
    if [ ! -d ${PRECISE_ODB_DIR} ];then
	echo "directory ${PRECISE_ODB_DIR} does not exist"
	exit
    fi
saeed's avatar
saeed committed
150
          
saeed's avatar
saeed committed
151
    cd ${PRECISE_ODB_DIR}
saeed's avatar
saeed committed
152
        
saeed's avatar
saeed committed
153
    HERE_P=`pwd`
saeed's avatar
saeed committed
154
155
156
157
158
159
160
161
162
163
164
    for j in ${!list_report[*]};do
	report_type=${list_report[j]}
	nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
	for nf in `seq 1 ${nc}`;do
	    vb=`echo ${time_cycle} | cut -d, -f${nf}`
	    fout="odb_precise_${report_type}_${startdate_out}_${enddate_out}_cycle${vb}.txt"
	    if [ -f ${OUTPUT_DIR}/${fout} ];then
		echo "file ${OUTPUT_DIR}/${fout} already exist"
	    fi
	done
    done
saeed's avatar
saeed committed
165
    
saeed's avatar
saeed committed
166
    for j in ${!list_report[*]};do
saeed's avatar
saeed committed
167
	cd ${HERE_P}
saeed's avatar
saeed committed
168
169
	report_type=${list_report[j]}
	nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
saeed's avatar
saeed committed
170
171
172
173
	for nf in `seq 1 ${nc}`;do
	    vb=`echo ${time_cycle} | cut -d, -f${nf}`
	    fout="odb_precise_${report_type}_${startdate_out}_${enddate_out}_cycle${vb}.txt"
	    if [ ! -f ${OUTPUT_DIR}/${fout} ];then
saeed's avatar
saeed committed
174
175
176
177
178
179
180
181
182
183
184
185
186
187
		for ii in ${!year_array[*]};do
		    iy=${year_array[${ii}]}
		    im=${mon_array[${ii}]}
                    id1=${day_array1[${ii}]}
                    id2=${day_array2[${ii}]}
                    cd ${HERE_P}
		    for dy in `seq -w ${id1} ${id2}`;do
			va="${iy}/${im}/${dy}"
                	if [ -d "${va}"/"${vb}"/odb_ccma/CCMA ];then
			    echo "va " ${va}
			    cd "${va}"/"${vb}"/odb_ccma/CCMA
			    pwd
			    dcagen
			    odbsql -q "select andate,antime,degrees(lat),degrees(lon),varno,codetype, count(*) from hdr,desc,body where\
saeed's avatar
saeed committed
188
 obsvalue/=\"NULL\" and reportype in (${report_type})" | grep -v "obsvalue" > odb_precise_${report_type}_${vb}.txt
saeed's avatar
saeed committed
189
190
191
			fi
			cd ${HERE_P}
		    done
saeed's avatar
saeed committed
192
193
		    cd ${HERE_P}
		done
saeed's avatar
saeed committed
194
195
196
197
198
199
200
201
202
203
204
205
206
		for ii in ${!year_array[*]};do
		    iy=${year_array[${ii}]}
		    im=${mon_array[${ii}]}
                    id1=${day_array1[${ii}]}
                    id2=${day_array2[${ii}]}
                    cd ${HERE_P}
		    for dy in `seq -w ${id1} ${id2}`;do
			va="${iy}/${im}/${dy}"
                	if [ -d ${va}/${vb}/odb_ccma/CCMA ];then
			    cd ${va}/${vb}/odb_ccma/CCMA
			    if [ -f odb_precise_${report_type}_${vb}.txt ];then
				sed '1d' odb_precise_${report_type}_${vb}.txt >> ${OUTPUT_DIR}/${fout}
			    fi
saeed's avatar
saeed committed
207
			fi
saeed's avatar
saeed committed
208
209
			cd ${HERE_P}
		    done
saeed's avatar
saeed committed
210
		done
saeed's avatar
saeed committed
211
		cd ${HERE_P}
saeed's avatar
saeed committed
212
213
	    fi
	done
saeed's avatar
saeed committed
214
215
216
    done
fi

saeed's avatar
saeed committed
217
cd ${HERE}