Fixed bug in reading ICOADS .Rda files

rscripts/process_ships.R

@@ -30,7 +30,7 @@ drop<-c("wmi","is","es","rs","ir","rrr","qci","mds","osz","oov","ooz","opv",
         "oav","oaz","wx","sx","eoh","hop","smf","sme","smv")
 
 #call.dcks <-c(128,233,254,255,555,700,708,709,732,735,749,781,792,849,874,875,888,889,892,926,927,992,993,995,999)
-if (!dir.exists(paste0(config$output_dir,"MFILES_SHIP_PROC"))) dir.create(paste0(config$output_dir,"MFILES_SHIP_PROC"))
+if (!dir.exists(paste0(config$output_dir,"MFILES_SHIP_PROC"))) dir.create(paste0(config$output_dir,"MFILES_SHIP_PROC"), recursive=TRUE)
 
 options(warn=1)
 if ( length(my_args) < 2 ) {my_args<-my_args2}

scr/NOC/config.yml

@@ -27,23 +27,24 @@ default:
 
   #output_dir: "../output_data/"
   # The next paths are need in icoads.utils::get_pub47() for log output
-  output_dir_pub47_txt: "/noc/mpoc/surface_data/ICOADS_R3_PROC/HOSTACE_PROC_ANC_INFO/"
-  output_dir: "/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/"
+  output_dir_pub47_txt: "/noc/mpoc/surface_data/ICOADS_RUN_TEST/HOSTACE_PROC_ANC_INFO/"
+  output_dir: "/noc/mpoc/surface_data/ICOADS_RUN_TEST/output_data/"
   profiler_output: ""
 
   # setup for GIULIA's tracking
   track_code_dir: "/noc/users/metman/GIULIA_TRACK_CODE/TRACK_CODE_FROM_JASMIN/"
   track_root_athena: "/pgdata/metman/TRACK"
-  track_input: "/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/TRACK_INPUT/"
-  track_output: "/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/GTRACK_OUT/"
+  track_input: "/noc/mpoc/surface_data/ICOADS_RUN_TEST/RUN_ZERO/output_data/TRACK_INPUT/"
+  track_output: "/noc/mpoc/surface_data/ICOADS_RUN_TEST/RUN_ZERO/output_data/GTRACK_OUT/"
 
   # Final Processing Dirs
-  data_dir: "/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/FINAL_PROC"
+  data_dir: "/noc/mpoc/surface_data/ICOADS_RUN_TEST/RUN_ZERO/output_data/FINAL_PROC"
 
   # Height Files/Dirs
-  ens_hts_dir: "/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/ENS_HTS/"
-  ens_grid_hts_dir: "/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/GRID_HTS/"
-  def_hts_file: "/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/default_height_ensemble.Rda"
+  ens_hts_dir: "/noc/mpoc/surface_data/ICOADS_RUN_TEST/RUN_ZERO/output_data/ENS_HTS/"
+  ens_grid_hts_dir: "/noc/mpoc/surface_data/ICOADS_RUN_TEST/RUN_ZERO/output_data/GRID_HTS/"
+  def_hts_file: "/noc/mpoc/surface_data/ICOADS_RUN_TEST/RUN_ZERO/output_data/default_height_ensemble.Rda"
 
   # Diurnal dirs
-  dir_flags_dir: "/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/DIURNAL_FAIL_IDS/"
\ No newline at end of file
+  dir_flags_dir: "/noc/mpoc/surface_data/ICOADS_RUN_TEST/RUN_ZERO/output_data/DIURNAL_FAIL_IDS/"
+  
\ No newline at end of file

scr/NOC/run_start.csh

-#!/bin/csh
+#!/bin/csh 
 
 # Immediately exit a script when it encounters an error
 #set -e
@@ -9,7 +9,7 @@
 
 # Load required modules and set base/code directories
 source setup_icproc
- 
+
 if ( ! -d ${basedir}OPFILES  ) then
     #echo Exists
 #else
@@ -29,10 +29,10 @@ echo "logfile" ${log_output_dir}/op.$1.$2
 #echo "removing existing files" >> ${log_output_dir}/op.$1
 #Rscript $codedir/remove_output_files.R $1 >> ${log_output_dir}/op.$1
 if ( $3 == "remove" ) then
-echo "removing existing files" >> ${log_output_dir}/op.$1.$2
-Rscript $codedir/remove_output_files.R $1 $2 >> ${log_output_dir}/op.$1.$2
-else
-echo "not removing existing files" >> ${log_output_dir}/op.$1.$2
+    echo "removing existing files" >> ${log_output_dir}/op.$1.$2
+    Rscript $codedir/remove_output_files.R $1 $2 >> ${log_output_dir}/op.$1.$2
+    else
+    echo "not removing existing files" >> ${log_output_dir}/op.$1.$2
 endif
 
 echo 'running start of processing' >> ${log_output_dir}/op.$1.$2

scr/NOC/setup_icproc

 #!/bin/csh
 
 #conda activate ~/miniconda3/envs/r_env
-module load R
+module load R/3.5.1
 module load gdal
 module load geos
 module load nco
@@ -9,6 +9,6 @@ module load netcdf
 module load intel_compiler
 module load gcc/7.2
 
-set basedir="/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/"
-set codedir="/noc/users/eck/nocgit/icoads-r-hostace/rscripts/"
+set basedir="/noc/mpoc/surface_data/ICOADS_RUN_TEST/"
+set codedir="/noc/users/ricorne/icoads-r-hostace/rscripts/"
 set configfile="config.yml"