odb_manger.sh 5.01 KB
Newer Older
saeed's avatar
saeed committed
1
2
3
4
5
6
#!/bin/bash
##SBATCH -N 1
##SBATCH -J manage_odb_data
##SBATCH -e err
##SBATCH -o log

saeed's avatar
saeed committed
7
8
module add odb/1.0.3
module add odb_api/0.19.1
saeed's avatar
saeed committed
9

saeed's avatar
m    
saeed committed
10
. ./functionsodb.sh
saeed's avatar
saeed committed
11

saeed's avatar
saeed committed
12
do_era5=1
saeed's avatar
saeed committed
13
do_precise=0
saeed's avatar
m    
saeed committed
14
do_only_monthy=1
saeed's avatar
saeed committed
15
use_report_file=0
saeed's avatar
m    
saeed committed
16
report_type_filename="report_type_list"
saeed's avatar
m    
saeed committed
17
list_report=(16001)
saeed's avatar
saeed committed
18

saeed's avatar
saeed committed
19
20
PRECISE_ODB_DIR="/scratch/ms/se/smdk/scratchdir/"
OUTPUT_DIR="/home/ms/se/smdk/odb_panel/OUTPUT"
saeed's avatar
saeed committed
21
path_cycle="/home/ms/se/smdk/odb_panel/bin/"
saeed's avatar
saeed committed
22

saeed's avatar
m    
saeed committed
23
startdate="2017-01-01"
saeed's avatar
m    
saeed committed
24
25
enddate="2017-01-20"
time_cycle="21"
saeed's avatar
m    
saeed committed
26
27


saeed's avatar
saeed committed
28
if [ ${use_report_file} -eq 1 ];then
saeed's avatar
m    
saeed committed
29
30
    if [ ! -f ${report_type_filename} ];then
	echo "please provide the file named ${report_type_filename}"
saeed's avatar
saeed committed
31
32
	exit
    fi
saeed's avatar
m    
saeed committed
33
    list_report=(`cat ${report_type_filename}`)
saeed's avatar
saeed committed
34
fi
saeed's avatar
saeed committed
35
36
37

checkmonthly=`ismonthly ${startdate} ${enddate}`

saeed's avatar
m    
saeed committed
38
if [ ${do_only_monthy} -eq 1 -a ${checkmonthly} -eq 0 ];then
saeed's avatar
m    
saeed committed
39
40
    echo "you chose the monthly data retrieval from mars"
    echo "please adjust the startdate ${startdate} or enddate ${enddate} so that they have the same month and the same year"
saeed's avatar
saeed committed
41
42
43
44
45
46
    exit
fi

time_cycle_out=`addbackslash ${time_cycle}`
startdate_out=`yearmonday ${startdate}`
enddate_out=`yearmonday ${enddate}`
saeed's avatar
saeed committed
47
48
49
50
51
52
53
54
55
56
57

day_1=`getday ${startdate}`
day_2=`getday ${enddate}`

drange=[]
ii=0
for j in `seq -w ${day_1} ${day_2}`;do
    drange[${ii}]=${j}
    ii=$((ii +1))
done

saeed's avatar
saeed committed
58
59
60
61
62
63
64
65
66
67
HERE=`pwd`

echo ${startdate_out}
echo ${enddate_out}

if [ ${do_era5} -eq 1 ];then
    for j in ${!list_report[*]};do
	vr=${list_report[j]}
	fin="odb_era5_${vr}_${startdate_out}_${enddate_out}.odb"
	fout=`output_name ${fin}`
saeed's avatar
saeed committed
68
69
70
71
72
73
74
75
76
77
78
	lexist=1
	nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
	for nf in `seq 1 ${nc}`;do
	    vb=`echo ${time_cycle} | cut -d, -f${nf}`
	    fout_cycle=`output_name_cycle ${fout} ${vb}`
	    if [ ! -f ${OUTPUT_DIR}/${fout_cycle} ];then
		lexist=0
	    fi
	done
	
	if [ ${lexist} -eq 0 ];then
saeed's avatar
m    
saeed committed
79
80
81
	    echo "==========================================="
	    echo "we are extracting odb for repportype ${vr} for a period ${startdate_out} ${enddate_out}"
	    echo "==========================================="
saeed's avatar
saeed committed
82
83
	    sed -e "s/<reptype>/${vr}/g" -e "s/<time_cycle>/${time_cycle_out}/g" -e "s/<startdate_out>/${startdate_out}/g" -e "s/<enddate_out>/${enddate_out}/g" -e "s/<startdate>/${startdate}/g" -e "s/<enddate>/${enddate}/g" request_odb_temp_era5 > request_odb_temp
	    mars request_odb_temp
saeed's avatar
m    
saeed committed
84
85
86
87
88
89
90
91
92
93
94
	    if [ ! -f ${fin} ];then
		echo "something went wrong in creating odb file ${fin} so we quit check your mars request"
	    fi
            do_filter ${fin} &>/dev/null
	    
	    if [ $? -ne 0 ];then
		echo "some thing went wrong in odb filtering on file ${fin} so we quit"
		echo "it is very likely that you have a problem with time_cycle ${time_cycle_out} so maybe you can change the cycle"
		rm -f ${fin} 2>/dev/null
		exit
	    fi
saeed's avatar
saeed committed
95
	    rm -f ${fin} 2>/dev/null
saeed's avatar
saeed committed
96
97
98
99
100
	    nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
	    for nf in `seq 1 ${nc}`;do
		vb=`echo ${time_cycle} | cut -d, -f${nf}`
		${path_cycle}/era5_cycle.exe ${fout} ${vb}
		fout_cycle=`output_name_cycle ${fout} ${vb}`
saeed's avatar
m    
saeed committed
101
		#echo "fout_cycle ${fout_cycle}"
saeed's avatar
saeed committed
102
103
104
		mv ${fout_cycle} ${OUTPUT_DIR} 2>/dev/null
	    done
	    rm -f ${fout}
saeed's avatar
saeed committed
105
	    rm -f request_odb_temp 2>/dev/null
saeed's avatar
saeed committed
106
107
108
109
110
111
112
113
	else
	    echo "${OUTPUT_DIR}/${fout} already exists"
	fi
    done
fi


if [ ${do_precise} -eq 1 ];then    
saeed's avatar
saeed committed
114
    
saeed's avatar
saeed committed
115
116
117
118
119
120
121
122
123
124
125
126
127
    if [ ! -d ${PRECISE_ODB_DIR} ];then
	echo "directory ${PRECISE_ODB_DIR} does not exist"
	exit
    fi

    year="`getyear ${startdate}`"
    mon="`getmon ${startdate}`"
    if [ ! -d ${PRECISE_ODB_DIR}/${year}/${mon} ];then
	echo "directory ${PRECISE_ODB_DIR}/${year}/${mon} does not exist"
	exit
    fi
    
    cd ${PRECISE_ODB_DIR}
saeed's avatar
saeed committed
128
        
saeed's avatar
saeed committed
129
130
131
    cd ${year}
    cd ${mon}
    HERE_P=`pwd`
saeed's avatar
saeed committed
132
    
saeed's avatar
saeed committed
133
134
135
136
137
138
139
140
141
142
143
144
145
    for j in ${!list_report[*]};do
	report_type=${list_report[j]}
	nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
	for nf in `seq 1 ${nc}`;do
	    vb=`echo ${time_cycle} | cut -d, -f${nf}`
	    fout="odb_precise_${report_type}_${startdate_out}_${enddate_out}_cycle${vb}.txt"
	    if [ -f ${OUTPUT_DIR}/${fout} ];then
		echo "file ${OUTPUT_DIR}/${fout} already exist"
	    fi
	done
    done

   
saeed's avatar
saeed committed
146
    for j in ${!list_report[*]};do
saeed's avatar
saeed committed
147
	cd ${HERE_P}
saeed's avatar
saeed committed
148
149
	report_type=${list_report[j]}
	nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
saeed's avatar
saeed committed
150
151
152
153
154
155
	for nf in `seq 1 ${nc}`;do
	    vb=`echo ${time_cycle} | cut -d, -f${nf}`
	    fout="odb_precise_${report_type}_${startdate_out}_${enddate_out}_cycle${vb}.txt"
	    if [ ! -f ${OUTPUT_DIR}/${fout} ];then
 		for va in ${drange[*]};do
		    echo "va " ${va}
saeed's avatar
saeed committed
156
157
158
159
160
161
162
163
164
165
166
167
168
		    if [ -d "${va}"/"${vb}"/odb_ccma/CCMA ];then
			cd "${va}"/"${vb}"/odb_ccma/CCMA
			dcagen
			odbsql -q "select andate,antime,degrees(lat),degrees(lon),varno,codetype, count(*) from hdr,desc,body where\
 obsvalue/=\"NULL\" and reportype in (${report_type})" | grep -v "obsvalue" > odb_precise_${report_type}_${vb}.txt
		    fi
		    cd ${HERE_P}
		done
		cd ${HERE_P}
		for va in ${drange[*]};do
		    if [ -d ${va}/${vb}/odb_ccma/CCMA ];then
			cd ${va}/${vb}/odb_ccma/CCMA
			if [ -f odb_precise_${report_type}_${vb}.txt ];then
saeed's avatar
saeed committed
169
			    sed '1d' odb_precise_${report_type}_${vb}.txt >> ${OUTPUT_DIR}/${fout}
saeed's avatar
saeed committed
170
171
			fi
		    fi
saeed's avatar
saeed committed
172
		    cd ${HERE_P}
saeed's avatar
saeed committed
173
		done
saeed's avatar
saeed committed
174
175
176
	    fi
	done
	cd ${HERE_P}
saeed's avatar
saeed committed
177
178
179
    done
fi

saeed's avatar
saeed committed
180
cd ${HERE}