Added config file as an Rscript argument

rscripts/run_disj_track2.R

@@ -13,15 +13,19 @@
 # my_args[2] is end year
 # my_args[3] is "pre" or "post" tracking
 # my_args[4] dcks to select, either "none" or e.g. "123,124"
+# my_args[5] config.yml file
 
 ## Outputs the data to a home dir:
 # output_data/DISJTR
 # and duplicate text files to
 # output_data/PAIR_DUPFILES
 
+library(icoads.utils)
+
 my_args = commandArgs(trailingOnly=TRUE)
 
-source("~eck/keeping_going2.R")
+##source("~eck/keeping_going2.R")
+##source("/noc/users/eck/Rscripts/read_rdsfiles.R")
 
 if(length(my_args)<2)my_args<-my_args2
 
@@ -29,14 +33,14 @@ n.args<-length(my_args)
 if(n.args< 2) {my_args<-my_args2}
 n.args<-length(my_args)
 
-if(n.args < 2 | n.args> 4) {stop("wrong number of input my_args or my_args2 not set")}
+if(n.args < 2 | n.args> 5) {stop("wrong number of input my_args or my_args2 not set")}
 if(n.args == 2 & my_args[1] >= 1982 ) my_args[3] <- "pre"
 if(n.args == 2 ) my_args[3] <- "post"
 if(n.args < 4) my_args[4] <- "none"
+if(n.args < 4) my_args[5] <- "config.yml"
 
-config<-config::get(file = "config.yml")
+config<-config::get(file = my_args[5])
 exit <- function() { invokeRestart("abort") } 
-source("/noc/users/eck/Rscripts/read_rdsfiles.R")
 
 if (!dir.exists(paste0(config$output_dir,"DISJTR_NEW"))) dir.create(paste0(config$output_dir,"DISJTR_NEW"))
 

rscripts/run_joins.R

@@ -3,24 +3,29 @@
 # code written for the NERC GloSAT project 
 
 # Description:
-## IMPORTANT: all paths to all data are defined in config.yml
+## IMPORTANT: all paths to all data are defined in config.yml (as argument #2)
 ## input_data:
 # 1. arguments (e.g. 1990)
 # 2. output_data/HOOVER_RUN_ZERO
 # 3. output_data/DISJ/stat
 
+
 Sys.setenv(TZ='GMT')
 options(warnings=1)
 exit <- function() { invokeRestart("abort") }
 my_args = commandArgs(trailingOnly=TRUE)
 if(length(my_args)== 0) {my_args<-my_args2}
-if(length(my_args)!= 1) {stop("wrong number of input my_args or my_args2 not set")}
-require(icoads.utils)
+if(length(my_args)!= 2) {stop("wrong number of input my_args or my_args2 not set")}
+
+library(icoads.utils)
 
-config<-config::get(file = "config.yml")
+config<-config::get(file = my_args[2])
 print.comm <- TRUE
 dck.nodup<-c(701)
-p47vars<-c("ship_callsign","ship_name","vessel_length","vessel_breadth","vessel_freeboard","vessel_draught","recruiting_country","visual_observing_height","thermometer1_height_above_summer_load_line","anemometer1_height_above_summer_load_line","barometer1_height_above_summer_load_line","sea_thermometer1_height_above_summer_load_line","vessel_type2","p47.uid")
+p47vars<-c("ship_callsign","ship_name","vessel_length","vessel_breadth","vessel_freeboard","vessel_draught",
+           "recruiting_country","visual_observing_height","thermometer1_height_above_summer_load_line",
+           "anemometer1_height_above_summer_load_line","barometer1_height_above_summer_load_line",
+           "sea_thermometer1_height_above_summer_load_line","vessel_type2","p47.uid")
 
   #source("/noc/users/eck/nocgit/icoads.utils/R/check_track_func.R")
   #source("/noc/users/eck/nocgit/icoads.utils/R/read_rdsfiles.R")

scr/NOC/config.yml

+
+default:
+  ICOADS_data_path: "/noc/mpoc/surface_data/ICOADS_R3_PROC/INPUT_Rda/r3.0.0/ICOADS_R3.0.0T_"
+  ICOADS_RELEASE: "r3.0.0"
+  ICOADS_NRT_RELEASE: "r3.0.1"
+
+  # submarines in dck 245
+  # from https://github.com/oldweather/RN-WW2/blob/master/analyses/submarines/sub_names
+  subnames_data_path: "/noc/mpoc/surface_data/ICOADS_R3_PROC/HOSTACE_PROC_ANC_INFO/ww2_submarine_names.txt"
+
+  # Publication 47 data
+  fix_ids_pub47_path: "/noc/mpoc/surface_data/ICOADS_R3_PROC/HOSTACE_PROC_ANC_INFO/fixed_ids_pub47.txt"
+  not_fix_ids_pub47_path: "/noc/mpoc/surface_data/ICOADS_R3_PROC/HOSTACE_PROC_ANC_INFO/not_fix_ids_pub47.txt"
+  pub_47_by_month_path: "/noc/mpoc/surface_data/ICOADS_R3_PROC/HOSTACE_PROC_ANC_INFO/PUB47byMONTH"
+
+  # ids from external input
+  cliwoc_shiplogbookid_path: "/noc/mpoc/surface_data/ICOADS_R3_PROC/HOSTACE_PROC_ANC_INFO/cliwoc_shipLogbookid2_1.txt"
+  kobe762_metadata_path: "/noc/mpoc/surface_data/ICOADS_R3_PROC/HOSTACE_PROC_ANC_INFO/kobe2003_metadata.txt"
+  us_maury_names_path: "/noc/mpoc/surface_data/ICOADS_R3_PROC/HOSTACE_PROC_ANC_INFO/usmaury_names.txt"
+  seq_ids_path: "/noc/mpoc/surface_data/ICOADS_R3_PROC/HOSTACE_PROC_ANC_INFO/SEQ_IDS"
+  us_mmc_path: "/noc/mpoc/surface_data/ICOADS_R3_PROC/HOSTACE_PROC_ANC_INFO/US_MMC"
+
+  # processing info
+  generic_id_path: "/noc/mpoc/surface_data/ICOADS_R3_PROC/HOSTACE_PROC_ANC_INFO/gen_id.txt"
+  prec_input_path: "/noc/mpoc/surface_data/ICOADS_R3_PROC/HOSTACE_PROC_ANC_INFO/prec_input.txt"
+  json_files_path: "/noc/mpoc/surface_data/ICOADS_R3_PROC/HOSTACE_PROC_ANC_INFO/json_files"
+
+  #output_dir: "../output_data/"
+  # The next paths are need in icoads.utils::get_pub47() for log output
+  output_dir_pub47_txt: "/noc/mpoc/surface_data/ICOADS_R3_PROC/HOSTACE_PROC_ANC_INFO/"
+  output_dir: "/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/"
+  profiler_output: ""
+
+  # setup for GIULIA's tracking
+  track_code_dir: "/noc/users/metman/GIULIA_TRACK_CODE/TRACK_CODE_FROM_JASMIN/"
+  track_root_athena: "/pgdata/metman/TRACK"
+  track_input: "/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/TRACK_INPUT/"
+  track_output: "/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/GTRACK_OUT/"

scr/NOC/run_disj.csh

@@ -34,13 +34,13 @@ set y = $1
 
    echo "XXX getting disjoint" $y $y >> ${log_output_dir}/dop.$1.$2
    if ( $y <= 1981 ) then
-     Rscript $codedir/run_disj_track2.R $y $y "post" "none" >> ${log_output_dir}/dop.$1.$2
+     Rscript $codedir/run_disj_track2.R $y $y "post" "none" $configfile >> ${log_output_dir}/dop.$1.$2
    else
-     Rscript $codedir/run_disj_track2.R $y $y "pre" "none" >> ${log_output_dir}/dop.$1.$2
+     Rscript $codedir/run_disj_track2.R $y $y "pre" "none" $configfile >> ${log_output_dir}/dop.$1.$2
    endif
 
    echo 'XXX running joining' >> ${log_output_dir}/dop.$1.$2
-     Rscript $codedir/run_joins.R $y >> ${log_output_dir}/dop.$1.$2
+     Rscript $codedir/run_joins.R $y $configfile >> ${log_output_dir}/dop.$1.$2
 
    echo 'XXX getting height ensemble' >> ${log_output_dir}/dop.$1.$2
      Rscript $codedir/get_height_ens.R $y $y >> ${log_output_dir}/dop.$1.$2

scr/NOC/setup_icproc

@@ -11,4 +11,4 @@ module load gcc/7.2
 
 set basedir="/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/"
 set codedir="/noc/users/eck/nocgit/icoads-r-hostace/rscripts/"
-
+set configfile="config.yml"