Added ability to incorporate GDAC data

rscripts/split_by_type.R

@@ -103,6 +103,25 @@ for ( year in my_args[1]:my_args[2] ) {
             message("Using TOTAL data")
         }
 
+        ##
+        if(config$use_gdac){
+            fn.gdac <- file.path(config$gdac_path,sprintf("%s_%s.Rda",year,imo))
+            if(file.exists(fn.gdac)){
+                message("Using GDAC data")
+                mdata.gdac <- readRDS(fn.gdac)
+                mdata.gdac$sid <- 999
+                mdata.gdac$pt <- 5 # all GCC data should be from ships
+                mdata.gdac$dups <- NA
+
+                ## Fill missing columns with NA
+                miss.cols <- colnames(mdata)[which(!colnames(mdata)%in%colnames(mdata.gdac))]
+                mdata.gdac[,miss.cols] <- NA
+                mdata.gdac <- subset(mdata.gdac,select=colnames(mdata))
+
+                mdata <- rbind(mdata,mdata.gdac)
+            }
+        }    
+
         ## assign missing PT 
         ship.dck<-c(128, 150, 151, 152, 155, 156, 192, 201, 246, 255, 701, 721, 875, 897, 899)
         mdata$pt<-ifelse(is.na(mdata$pt) & mdata$dck %in% ship.dck, 0, mdata$pt)

scr/NOC/config.yml

@@ -28,26 +28,30 @@ default:
   #output_dir: "../output_data/"
   # The next paths are need in icoads.utils::get_pub47() for log output
   output_dir_pub47_txt: "/noc/mpoc/surface_data/ICOADS_FINAL_VERSION/HOSTACE_PROC_ANC_INFO/"
-  output_dir: "/noc/mpoc/surface_data/ICOADS_FINAL_VERSION/output_data/"
+  #output_dir: "/noc/mpoc/surface_data/ICOADS_FINAL_VERSION/output_data/"
+  output_dir: "/noc/mpoc/surface_data/ICOADS_v0.0.1/output_data/"
   profiler_output: ""
 
   # setup for GIULIA's tracking
   track_code_dir: "/noc/users/metman/GIULIA_TRACK_CODE/TRACK_CODE_FROM_JASMIN/"
   track_root_athena: "/pgdata/metman/TRACK"
-  track_input: "/noc/mpoc/surface_data/ICOADS_FINAL_VERSION/RUN_ZERO/output_data/TRACK_INPUT/"
-  track_output: "/noc/mpoc/surface_data/ICOADS_FINAL_VERSION/RUN_ZERO/output_data/GTRACK_OUT/"
+  track_input: "/noc/mpoc/surface_data/ICOADS_v0.0.1/RUN_ZERO/output_data/TRACK_INPUT/"
+  track_output: "/noc/mpoc/surface_data/ICOADS_v0.0.1/RUN_ZERO/output_data/GTRACK_OUT/"
 
   # Final Processing Dirs
-  data_dir: "/noc/mpoc/surface_data/ICOADS_FINAL_VERSION/RUN_ZERO/output_data/FINAL_PROC"
+  data_dir: "/noc/mpoc/surface_data/ICOADS_v0.0.1/RUN_ZERO/output_data/FINAL_PROC"
 
   # Height Files/Dirs
-  ens_hts_dir: "/noc/mpoc/surface_data/ICOADS_FINAL_VERSION/RUN_ZERO/output_data/ENS_HTS/"
-  ens_grid_hts_dir: "/noc/mpoc/surface_data/ICOADS_FINAL_VERSION/RUN_ZERO/output_data/GRID_HTS/"
-  def_hts_file: "/noc/mpoc/surface_data/ICOADS_FINAL_VERSION/RUN_ZERO/output_data/default_height_ensemble.Rda"
+  ens_hts_dir: "/noc/mpoc/surface_data/ICOADS_v0.0.1/RUN_ZERO/output_data/ENS_HTS/"
+  ens_grid_hts_dir: "/noc/mpoc/surface_data/ICOADS_v0.0.1/RUN_ZERO/output_data/GRID_HTS/"
+  def_hts_file: "/noc/mpoc/surface_data/ICOADS_v0.0.1/RUN_ZERO/output_data/default_height_ensemble.Rda"
 
   # Diurnal dirs
-  dir_flags_dir: "/noc/mpoc/surface_data/ICOADS_FINAL_VERSION/RUN_ZERO/output_data/DIURNAL_FAIL_IDS/"
+  dir_flags_dir: "/noc/mpoc/surface_data/ICOADS_v0.0.1/RUN_ZERO/output_data/DIURNAL_FAIL_IDS/"
 
   # Use TOTAL data or FINAL data
-  use_total: FALSE
-  
+  use_total: TRUE
+
\ No newline at end of file
+  # Merge in GDAC data?
+  use_gdac: TRUE
+  gdac_path: "/noc/mpoc/surface_data/VOS-GDAC/YRMO/"
\ No newline at end of file

scr/NOC/setup_icproc

@@ -9,6 +9,6 @@ module load netcdf
 module load intel_compiler
 module load gcc/7.2
 
-set basedir="/noc/mpoc/surface_data/ICOADS_FINAL_VERSION/"
+set basedir="/noc/mpoc/surface_data/ICOADS_v0.0.1/"
 set codedir="/noc/users/metman/icoads-r-hostace/rscripts/"
 set configfile="config.yml"