Skip to content

GitLab

Commit 45bbb39d authored by Richard Cornes's avatar Richard Cornes
Browse files

Added NOC dir and tidied file paths in sh scripts

parent e122b788
Hide whitespace changes
Inline Side-by-side
Showing with 355 additions and 0 deletions
rscripts/run_joins2.R → rscripts/run_joins.R
View file @ 45bbb39d
File moved
scr/run_full.bsh → scr/JASMIN/run_full.bsh
View file @ 45bbb39d
File moved
scr/NOC/run_disj.csh 0 → 100755
View file @ 45bbb39d
#!/bin/csh
# Immediately exit a script when it encounters an error
#set -e
# start year = $1
# end year = $2
source /noc/users/metman/SETUPS/setup_icproc
if ( ! -d ${basedir}OPFILES ) then
#echo Exists
#else
mkdir ${basedir}OPFILES
mkdir ${basedir}output_data
endif
#Adjust your log_output dir as needed
set log_output_dir="${basedir}OPFILES"
touch ${log_output_dir}/dop.$1.$2
/bin/rm ${log_output_dir}/dop.$1.$2
touch ${log_output_dir}/dop.$1.$2
echo "logfile" ${log_output_dir}/dop.$1.$2
echo 'running disjoint tracking' >> ${log_output_dir}/dop.$1.$2
echo 'start year' $1 >> ${log_output_dir}/dop.$1.$2
echo 'end year' $2 >> ${log_output_dir}/dop.$1.$2
echo 'start_at_disj' >> ${log_output_dir}/dop.$1.$2
echo "---------------" >> ${log_output_dir}/dop.$1.$2
set y = $1
while ( $y <= $2 )
echo "XXX getting disjoint" $y $y >> ${log_output_dir}/dop.$1.$2
if ( $y <= 1981 ) then
Rscript $codedir/run_disj_track2.R $y $y "post" "none" >> ${log_output_dir}/dop.$1.$2
else
Rscript $codedir/run_disj_track2.R $y $y "pre" "none" >> ${log_output_dir}/dop.$1.$2
endif
echo 'XXX running joining' >> ${log_output_dir}/dop.$1.$2
Rscript $codedir/run_joins.R $y >> ${log_output_dir}/dop.$1.$2
echo 'XXX getting height ensemble' >> ${log_output_dir}/dop.$1.$2
Rscript $codedir/get_height_ens.R $y $y >> ${log_output_dir}/dop.$1.$2
echo 'XXX generating gridded heights' >> ${log_output_dir}/dop.$1.$2
Rscript $codedir/get_gridded_vals.R $y $y >> ${log_output_dir}/dop.$1.$2
echo 'XXX getting ships with wrong diurnal cycle' >> ${log_output_dir}/dop.$1.$2
Rscript $codedir/add_diurnal_fails.R $y >> ${log_output_dir}/dop.$1.$2
@ y = $y + 1
end
echo "XXX finished disjoint, joins, heights and diurnal flags" $1 $2 >> ${log_output_dir}/dop.$1.$2
scr/run_disj.csh → scr/NOC/run_disj.csh~
View file @ 45bbb39d
File moved
scr/NOC/run_mid.csh 0 → 100755
View file @ 45bbb39d
#!/bin/csh
# Immediately exit a script when it encounters an error
#set -e
# start year = $1
# end year = $2
# Load required modules and set base/code directories
source setup_icproc
if ( ! -d ${basedir}OPFILES ) then
#echo Exists
#else
mkdir ${basedir}OPFILES
mkdir ${basedir}output_data
endif
#Adjust your log_output dir as needed
set log_output_dir="${basedir}OPFILES"
touch ${log_output_dir}/op.$1.$2
echo "logfile" ${log_output_dir}/op.$1.$2
echo 'running middle of processing' >> ${log_output_dir}/op.$1.$2
echo 'start year' $1 >> ${log_output_dir}/op.$1.$2
echo 'end year' $2 >> ${log_output_dir}/op.$1.$2
echo 'start_at_mid' >> ${log_output_dir}/op.$1.$2
echo "---------------" >> ${log_output_dir}/op.$1.$2
echo "XXX processing shipdata inc. ids" $1 $2 >> ${log_output_dir}/op.$1.$2
Rscript $codedir/process_ships.R $1 $2 >> ${log_output_dir}/op.$1.$2
set y = $1
while ( $y <= $2 )
if ( $y <= 2014 ) then
echo "XXX getting pairs" $y $y >> ${log_output_dir}/op.$1.$2
Rscript $codedir/get_pairs.R $y $y >> ${log_output_dir}/op.$1.$2
echo "XXX getting dups" $y $y >> ${log_output_dir}/op.$1.$2
Rscript $codedir/get_dups.R $y $y >> ${log_output_dir}/op.$1.$2
echo "XXX merging ids" $y $y >> ${log_output_dir}/op.$1.$2
Rscript $codedir/merge_ids_year.R $y $y >> ${log_output_dir}/op.$1.$2
else
set i = 1
while ( $i <= 12 )
echo "XXX getting NRT dups" $y $i >> ${log_output_dir}/op.$1.$2
Rscript $codedir/nrt_dup.R $y $i >> ${log_output_dir}/op.$1.$2
echo "XXX assigning station data" $y $y >> ${log_output_dir}/op.$1.$2
Rscript $codedir/proc_nrt.R $y $y >> ${log_output_dir}/op.$1.$2
@ i = $i + 1
end
endif
@ y = $y + 1
end
echo "XXX generating clean ship data" $1 $2 >> ${log_output_dir}/op.$1.$2
Rscript $codedir/clean_data.R $1 $2 >> ${log_output_dir}/op.$1.$2
echo "XXX generating track input files" $1 $2 >> ${log_output_dir}/op.$1.$2
Rscript $codedir/clean2track.R $1 $2 >> ${log_output_dir}/op.$1.$2
echo "XXX finished up to tracking" $1 $2 >> ${log_output_dir}/op.$1.$2
scr/NOC/run_mid.csh~ 0 → 100755
View file @ 45bbb39d
#!/bin/csh
# Immediately exit a script when it encounters an error
#set -e
# start year = $1
# end year = $2
#conda activate ~/miniconda3/envs/r_env
source /noc/users/metman/SETUPS/setup_icproc
#basedir="/gws/nopw/j04/glosat/development/data/interim/HOSTACE_PROC/TEST_CONFIG/"
set basedir="/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/"
set codedir="/noc/users/eck/nocgit/icoads-r-hostace/"
if ( ! -d ${basedir}OPFILES ) then
#echo Exists
#else
mkdir ${basedir}OPFILES
mkdir ${basedir}output_data
endif
#Adjust your log_output dir as needed
set log_output_dir="${basedir}OPFILES"
touch ${log_output_dir}/op.$1.$2
echo "logfile" ${log_output_dir}/op.$1.$2
echo 'running middle of processing' >> ${log_output_dir}/op.$1.$2
echo 'start year' $1 >> ${log_output_dir}/op.$1.$2
echo 'end year' $2 >> ${log_output_dir}/op.$1.$2
echo 'start_at_mid' >> ${log_output_dir}/op.$1.$2
echo "---------------" >> ${log_output_dir}/op.$1.$2
echo "XXX processing shipdata inc. ids" $1 $2 >> ${log_output_dir}/op.$1.$2
Rscript $codedir/rscripts/process_ships.R $1 $2 >> ${log_output_dir}/op.$1.$2
set y = $1
while ( $y <= $2 )
if ( $y <= 2014 ) then
echo "XXX getting pairs" $y $y >> ${log_output_dir}/op.$1.$2
Rscript $codedir/rscripts/get_pairs.R $y $y >> ${log_output_dir}/op.$1.$2
echo "XXX getting dups" $y $y >> ${log_output_dir}/op.$1.$2
Rscript $codedir/rscripts/get_dups.R $y $y >> ${log_output_dir}/op.$1.$2
echo "XXX merging ids" $y $y >> ${log_output_dir}/op.$1.$2
Rscript $codedir/rscripts/merge_ids_year.R $y $y >> ${log_output_dir}/op.$1.$2
else
set i = 1
while ( $i <= 12 )
echo "XXX getting NRT dups" $y $i >> ${log_output_dir}/op.$1.$2
Rscript $codedir/rscripts/nrt_dup.R $y $i >> ${log_output_dir}/op.$1.$2
echo "XXX assigning station data" $y $y >> ${log_output_dir}/op.$1.$2
Rscript $codedir/rscripts/proc_nrt.R $y $y >> ${log_output_dir}/op.$1.$2
@ i = $i + 1
end
endif
@ y = $y + 1
end
echo "XXX generating clean ship data" $1 $2 >> ${log_output_dir}/op.$1.$2
Rscript $codedir/rscripts/clean_data.R $1 $2 >> ${log_output_dir}/op.$1.$2
echo "XXX generating track input files" $1 $2 >> ${log_output_dir}/op.$1.$2
Rscript $codedir/rscripts/clean2track.R $1 $2 >> ${log_output_dir}/op.$1.$2
echo "XXX finished up to tracking" $1 $2 >> ${log_output_dir}/op.$1.$2
scr/NOC/run_start.csh 0 → 100755
View file @ 45bbb39d
#!/bin/csh
# Immediately exit a script when it encounters an error
#set -e
# start year = $1
# end year = $2
# remove = $3
# Load required modules and set base/code directories
source setup_icproc
if ( ! -d ${basedir}OPFILES ) then
#echo Exists
#else
mkdir ${basedir}OPFILES
mkdir ${basedir}output_data
endif
#Adjust your log_output dir as needed
set log_output_dir="${basedir}OPFILES"
echo "removing files in log dir to replace with new run" $1 $2
touch ${log_output_dir}/op.$1.$2
/bin/rm ${log_output_dir}/op.$1.$2
touch ${log_output_dir}/op.$1.$2
echo "logfile" ${log_output_dir}/op.$1.$2
#echo "removing existing files" >> ${log_output_dir}/op.$1
#Rscript $codedir/remove_output_files.R $1 >> ${log_output_dir}/op.$1
if ( $3 == "remove" ) then
echo "removing existing files" >> ${log_output_dir}/op.$1.$2
Rscript $codedir/remove_output_files.R $1 $2 >> ${log_output_dir}/op.$1.$2
else
echo "not removing existing files" >> ${log_output_dir}/op.$1.$2
endif
echo 'running start of processing' >> ${log_output_dir}/op.$1.$2
echo 'start year' $1 >> ${log_output_dir}/op.$1.$2
echo 'end year' $2 >> ${log_output_dir}/op.$1.$2
echo 'start_at_beg' >> ${log_output_dir}/op.$1.$2
echo "---------------" >> ${log_output_dir}/op.$1.$2
echo "XXX splitting data" $1 $2 >> ${log_output_dir}/op.$1.$2
Rscript $codedir/split_by_type.R $1 $2 >> ${log_output_dir}/op.$1.$2
#if [ "$1" -ge 1970 ]
if ( $1 >= 1970 ) then
#then
# find x-dups
echo "XXX ship dups with drifters" $1 $2 >> ${log_output_dir}/op.$1.$2
Rscript $codedir/simple_dup.R $1 $2 DRIFT >> ${log_output_dir}/op.$1.$2
echo "XXX ship dups with plat" $1 $2 >> ${log_output_dir}/op.$1.$2
Rscript $codedir/simple_dup.R $1 $2 PLAT >> ${log_output_dir}/op.$1.$2
echo "XXX ship dups with coast" $1 $2 >> ${log_output_dir}/op.$1.$2
Rscript $codedir/simple_dup.R $1 $2 COAST >> ${log_output_dir}/op.$1.$2
echo "XXX ship dups with moored" $1 $2 >> ${log_output_dir}/op.$1.$2
Rscript $codedir/simple_dup.R $1 $2 MOORED >> ${log_output_dir}/op.$1.$2
#fi # end of year >= 1970
endif
# split data in MFILES_TMP to _SHIP & _NOTSHIP
echo "XXX finding non-ship data" $1 $2 >> ${log_output_dir}/op.$1.$2
Rscript $codedir/ship2plat.R $1 $2 >> ${log_output_dir}/op.$1.$2
echo "XXX finished first steps" $1 $2 >> ${log_output_dir}/op.$1.$2
scr/NOC/run_start.csh~ 0 → 100755
View file @ 45bbb39d
#!/bin/csh
# Immediately exit a script when it encounters an error
#set -e
# start year = $1
# end year = $2
# remove = $3
#conda activate ~/miniconda3/envs/r_env
source /noc/users/metman/SETUPS/setup_icproc
#basedir="/gws/nopw/j04/glosat/development/data/interim/HOSTACE_PROC/TEST_CONFIG/"
set basedir="/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/"
set codedir="/noc/users/eck/nocgit/icoads-r-hostace/"
if ( ! -d ${basedir}OPFILES ) then
#echo Exists
#else
mkdir ${basedir}OPFILES
mkdir ${basedir}output_data
endif
#Adjust your log_output dir as needed
set log_output_dir="${basedir}OPFILES"
echo "removing files in log dir to replace with new run" $1 $2
touch ${log_output_dir}/op.$1.$2
/bin/rm ${log_output_dir}/op.$1.$2
touch ${log_output_dir}/op.$1.$2
echo "logfile" ${log_output_dir}/op.$1.$2
#echo "removing existing files" >> ${log_output_dir}/op.$1
#Rscript $codedir/rscripts/remove_output_files.R $1 >> ${log_output_dir}/op.$1
if ( $3 == "remove" ) then
echo "removing existing files" >> ${log_output_dir}/op.$1.$2
Rscript $codedir/rscripts/remove_output_files.R $1 $2 >> ${log_output_dir}/op.$1.$2
else
echo "not removing existing files" >> ${log_output_dir}/op.$1.$2
endif
echo 'running start of processing' >> ${log_output_dir}/op.$1.$2
echo 'start year' $1 >> ${log_output_dir}/op.$1.$2
echo 'end year' $2 >> ${log_output_dir}/op.$1.$2
echo 'start_at_beg' >> ${log_output_dir}/op.$1.$2
echo "---------------" >> ${log_output_dir}/op.$1.$2
echo "XXX splitting data" $1 $2 >> ${log_output_dir}/op.$1.$2
Rscript $codedir/rscripts/split_by_type.R $1 $2 >> ${log_output_dir}/op.$1.$2
#if [ "$1" -ge 1970 ]
if ( $1 >= 1970 ) then
#then
# find x-dups
echo "XXX ship dups with drifters" $1 $2 >> ${log_output_dir}/op.$1.$2
Rscript $codedir/rscripts/simple_dup.R $1 $2 DRIFT >> ${log_output_dir}/op.$1.$2
echo "XXX ship dups with plat" $1 $2 >> ${log_output_dir}/op.$1.$2
Rscript $codedir/rscripts/simple_dup.R $1 $2 PLAT >> ${log_output_dir}/op.$1.$2
echo "XXX ship dups with coast" $1 $2 >> ${log_output_dir}/op.$1.$2
Rscript $codedir/rscripts/simple_dup.R $1 $2 COAST >> ${log_output_dir}/op.$1.$2
echo "XXX ship dups with moored" $1 $2 >> ${log_output_dir}/op.$1.$2
Rscript $codedir/rscripts/simple_dup.R $1 $2 MOORED >> ${log_output_dir}/op.$1.$2
#fi # end of year >= 1970
endif
# split data in MFILES_TMP to _SHIP & _NOTSHIP
echo "XXX finding non-ship data" $1 $2 >> ${log_output_dir}/op.$1.$2
Rscript $codedir/rscripts/ship2plat.R $1 $2 >> ${log_output_dir}/op.$1.$2
echo "XXX finished first steps" $1 $2 >> ${log_output_dir}/op.$1.$2
scr/NOC/setup_icproc 0 → 100644
View file @ 45bbb39d
#!/bin/csh
#conda activate ~/miniconda3/envs/r_env
module load R
module load gdal
module load geos
module load nco
module load netcdf
module load intel_compiler
module load gcc/7.2
set basedir="/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/"
set codedir="/noc/users/eck/nocgit/icoads-r-hostace/rscripts/"
scr/NOC/setup_icproc~ 0 → 100644
View file @ 45bbb39d
module load R
module load gdal
module load geos
module load nco
module load netcdf
module load intel_compiler
module load gcc/7.2
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment