#!/bin/bash
##SBATCH -N 1
##SBATCH -J manage_odb_data
##SBATCH -e err
##SBATCH -o log

module add odb/1.0.3
module add odb_api/0.19.1
. ./functionsodb.sh

do_era5=1
do_precise=0
do_only_monthy=0
use_report_file=0     ###if you want to have a list of report types in a file  
report_type_filename="report_type_list"
list_report=(16001)

startdate="2017-01-05"
enddate="2017-01-06"
time_cycle="00,12"

PRECISE_ODB_DIR="/scratch/ms/se/smdk/scratchdir/"
OUTPUT_DIR="/home/ms/se/smdk/odb_panel/OUTPUT"
path_bin="/home/ms/se/smdk/odb_panel/bin/"

if [ ${use_report_file} -eq 1 ];then
    if [ ! -f ${report_type_filename} ];then
	echo "please provide the file named ${report_type_filename}"
	exit
    fi
    list_report=(`cat ${report_type_filename}`)
fi

checkmonthly=`ismonthly ${startdate} ${enddate}`

if [ ${do_only_monthy} -eq 1 -a ${checkmonthly} -eq 0 ];then
    echo 
    echo "you chose the monthly data retrieval from mars"
    echo "please adjust the startdate ${startdate} or enddate ${enddate} so that they have the same month and the same year"
    echo
    exit
fi

time_cycle_out=`addbackslash ${time_cycle}`
startdate_out=`yearmonday ${startdate}`
enddate_out=`yearmonday ${enddate}`

day_1=`getday ${startdate}`
day_2=`getday ${enddate}`

drange=[]
ii=0
for j in `seq -w ${day_1} ${day_2}`;do
    drange[${ii}]=${j}
    ii=$((ii +1))
done

HERE=`pwd`

echo ${startdate_out}
echo ${enddate_out}

if [ ${do_era5} -eq 1 ];then
    for j in ${!list_report[*]};do
	vr=${list_report[j]}
	fin="odb_era5_${vr}_${startdate_out}_${enddate_out}.odb"
	fout=`output_name ${fin}`
	lexist=1
	nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
	for nf in `seq 1 ${nc}`;do
	    vb=`echo ${time_cycle} | cut -d, -f${nf}`
	    fout_cycle=`output_name_cycle ${fout} ${vb}`
	    if [ ! -f ${OUTPUT_DIR}/${fout_cycle} ];then
		echo "==========================================="
		echo "we are extracting odb for repportype ${vr} for a period ${startdate_out} ${enddate_out} cycle ${fout_cycle}"
		echo "==========================================="
		sed -e "s/<reptype>/${vr}/g" -e "s/<cycle>/${vb}/g" -e "s/<startdate_out>/${startdate_out}/g" -e "s/<enddate_out>/${enddate_out}/g" -e "s/<startdate>/${startdate}/g" -e "s/<enddate>/${enddate}/g" request_odb_temp_era5 > request_odb_temp
		mars request_odb_temp
		if [ ! -f ${fin} ];then
		    echo "something went wrong in creating odb file ${fin} so we quit check your mars request"
		fi
		do_filter ${fin} &>/dev/null
	    
		if [ $? -ne 0 ];then
		    echo "some thing went wrong in odb filtering on file ${fin} so we quit"
		    echo "it is very likely that you have a problem with time_cycle ${time_cycle_out} so maybe you can change the cycle"
		    rm -f ${fin} 2>/dev/null
		    exit
		fi
		rm -f ${fin} 2>/dev/null
		${path_bin}/era5_cycle.exe ${fout} ${vb}
		mv ${fout_cycle} ${OUTPUT_DIR} 2>/dev/null
	    	rm -f ${fout}
		rm -f request_odb_temp 2>/dev/null
	    else
		echo "${OUTPUT_DIR}/${fout_cycle} already exists"
	    fi
	done
    done
fi

rm -f  start_end.dat 2>/dev/null

${path_bin}/date_range.exe ${startdate_out} ${enddate_out}

if [ ! -f start_end.dat ];then
    echo "start_end.dat file does not exist"
    exit
fi

list=(`cat start_end.dat`)
len=${#list[*]}

len=$((len / 2))
len_m1=$((len - 1))

year_array=[]
mon_array=[]
day_array1=[]
day_array2=[]

for j in `seq 0 ${len_m1}`;do
    j1=$((j * 2 + 0))
    j2=$((j * 2 + 1))
    year=`getyear_1 ${list[${j1}]}`
    month=`getmonth_1 ${list[${j1}]}`
    dy1=`getday_1 ${list[${j1}]}`
    dy2=`getday_1 ${list[${j2}]}`
    year_array[${j}]=${year}
    mon_array[${j}]=${month}
    day_array1[${j}]=${dy1}
    day_array2[${j}]=${dy2}
done

if [ ${do_precise} -eq 1 ];then    
    
    if [ ! -d ${PRECISE_ODB_DIR} ];then
	echo "directory ${PRECISE_ODB_DIR} does not exist"
	exit
    fi
          
    cd ${PRECISE_ODB_DIR}
        
    HERE_P=`pwd`
    for j in ${!list_report[*]};do
	report_type=${list_report[j]}
	nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
	for nf in `seq 1 ${nc}`;do
	    vb=`echo ${time_cycle} | cut -d, -f${nf}`
	    fout="odb_precise_${report_type}_${startdate_out}_${enddate_out}_cycle${vb}.txt"
	    if [ -f ${OUTPUT_DIR}/${fout} ];then
		echo "file ${OUTPUT_DIR}/${fout} already exist"
	    fi
	done
    done
    
    for j in ${!list_report[*]};do
	cd ${HERE_P}
	report_type=${list_report[j]}
	nc=`echo ${time_cycle} | awk -F',' '{print NF}'`
	for nf in `seq 1 ${nc}`;do
	    vb=`echo ${time_cycle} | cut -d, -f${nf}`
	    fout="odb_precise_${report_type}_${startdate_out}_${enddate_out}_cycle${vb}.txt"
	    if [ ! -f ${OUTPUT_DIR}/${fout} ];then
		for ii in ${!year_array[*]};do
		    iy=${year_array[${ii}]}
		    im=${mon_array[${ii}]}
                    id1=${day_array1[${ii}]}
                    id2=${day_array2[${ii}]}
                    cd ${HERE_P}
		    for dy in `seq -w ${id1} ${id2}`;do
			va="${iy}/${im}/${dy}"
                	if [ -d "${va}"/"${vb}"/odb_ccma/CCMA ];then
			    echo "va " ${va}
			    cd "${va}"/"${vb}"/odb_ccma/CCMA
			    pwd
			    dcagen
			    odbsql -q "select andate,antime,degrees(lat),degrees(lon),varno,codetype, count(*) from hdr,desc,body where\
 obsvalue/=\"NULL\" and reportype in (${report_type})" | grep -v "obsvalue" > odb_precise_${report_type}_${vb}.txt
			fi
			cd ${HERE_P}
		    done
		    cd ${HERE_P}
		done
		for ii in ${!year_array[*]};do
		    iy=${year_array[${ii}]}
		    im=${mon_array[${ii}]}
                    id1=${day_array1[${ii}]}
                    id2=${day_array2[${ii}]}
                    cd ${HERE_P}
		    for dy in `seq -w ${id1} ${id2}`;do
			va="${iy}/${im}/${dy}"
                	if [ -d ${va}/${vb}/odb_ccma/CCMA ];then
			    cd ${va}/${vb}/odb_ccma/CCMA
			    if [ -f odb_precise_${report_type}_${vb}.txt ];then
				sed '1d' odb_precise_${report_type}_${vb}.txt >> ${OUTPUT_DIR}/${fout}
			    fi
			fi
			cd ${HERE_P}
		    done
		done
		cd ${HERE_P}
	    fi
	done
    done
fi

cd ${HERE}