Changes to reflect latest running code

rscripts/get_gridded_vals.R

@@ -3,8 +3,6 @@
 library(icoads.utils)
 data(basin_name)
 
-source("../../rscripts/get_basin.R") #Fix me. Include "basin" in data(basin) in icoads.utils
-
 ##-------------------------------------------------------------------
 ## my_args[1] = start year
 ## my_args[2] = end year
@@ -94,7 +92,7 @@ for ( year in my_args[1]:my_args[2] ) {
     ##hts.by.id  <-  read.table(paste0("ENS_HTS/htbyid_",year,".psv.gz"),header=F,sep="|")
     hts_file_id <- file.path(out_dir,sprintf("htbyid_%s.psv",year))
     if(file.exists(hts_file_id)){
-        hts.by.id  <-  read.table(hts_file_id,header=F,sep="|",comment.char="",quote="")
+        hts.by.id  <-  read.table(hts_file_id,header=F,sep="|",comment.char="",quote=FALSE)
         cat("got ID hts","\n")
     } else {
         cat('no height file, skipping',year,"\n")
@@ -163,7 +161,6 @@ for ( year in my_args[1]:my_args[2] ) {
         }
     }
 
-    if(!dir.exists(out_grid_dir)) dir.create(out_grid_dir)
     saveRDS(op.ht,paste0(out_grid_dir,"deg5_ht_",year,".Rda"))
     saveRDS(op.sd,paste0(out_grid_dir,"deg5_sd_",year,".Rda"))
     saveRDS(op.num,paste0(out_grid_dir,"deg5_sd_",year,".Rda"))

rscripts/get_height_ens.R

@@ -246,7 +246,7 @@ for ( year in my_args[1]:my_args[2] ) {
     }
 
     cat("about to assign country","\n")
-    source("../../rscripts/assign_country.R")
+    source(file.path(config$code_dir,"assign_country.R"))
     cat("assigned country","\n")
     ##stop()
 
@@ -271,7 +271,7 @@ for ( year in my_args[1]:my_args[2] ) {
                 def.sd.arr[,,i] <- def.sd.tmp[[i]]
             }
             rm(def.ht.tmp,def.sd.tmp)
-            mm.ht <- apply(def.ht.arr,c(1,2),mean,na.rm=T)
+            mm.ht <- apply(def.ht.arr,c(1,2),mean,na.rm=TRUE)
             mm.ht.mean <- mean(mm.ht,na.rm=T)
             mm.ht.sd <- sd(mm.ht,na.rm=T)
             mm.ht.diff <- (mm.ht-mm.ht.mean)
@@ -624,7 +624,6 @@ for ( year in my_args[1]:my_args[2] ) {
     } ## end test for pre-pub47
 
     if ( TRUE ) {
-        if(!dir.exists(out_dir)) dir.create(out_dir)
         ## output heights for reports with known ids
         fn.out <- paste0(out_dir,"htbyid_",year,".psv")
         write.table(ship.ht.ens, file=fn.out, append=F, sep="|", col.names = FALSE)

rscripts/hoover_data.R

@@ -11,7 +11,8 @@ source("~eck/Rscripts/read_rdsfiles.R")
 #require("imma")
 
 print.comm<-FALSE
-rootdir<-"/noc/mpoc/surface_data/ICOADS_v0.0.1/output_data/"
+                                        #rootdir<-"/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/"
+rootdir<-"/noc/mpoc/surface_data/ICOADS_FINAL_VERSION/output_data/"
 
 #ids.gen<-read.table("/noc/mpoc/surface/eck/HOSTACE_PROC/gen_id.txt")
 #ids.gen<-c(ids.gen$V1,"")

rscripts/split_by_type.R

@@ -102,8 +102,7 @@ for ( year in my_args[1]:my_args[2] ) {
         } else {
             message("Using TOTAL data")
         }
-
-        ##
+        
         if(config$use_gdac){
             fn.gdac <- file.path(config$gdac_path,sprintf("%s_%s.Rda",year,imo))
             if(file.exists(fn.gdac)){
@@ -112,8 +111,8 @@ for ( year in my_args[1]:my_args[2] ) {
                 mdata.gdac$sid <- 999
                 mdata.gdac$pt <- 5 # all GCC data should be from ships
                 mdata.gdac$dups <- NA
-                mdata.gdac$dck <- 0 
-                mdata.gdac$irf <- 0
+                mdata.gdac$data.type <- "ship"
+                mdata.gdac$irf <- 999
                 
                 ## Fill missing columns with NA
                 miss.cols <- colnames(mdata)[which(!colnames(mdata)%in%colnames(mdata.gdac))]
@@ -267,7 +266,7 @@ for ( year in my_args[1]:my_args[2] ) {
         mdata$data.type <- ifelse(mdata$dck == 151 & mdata$lon >= 260 & mdata$lon <= 270 & mdata$yr >= 1890 & mdata$yr <= 1919, "reject", mdata$data.type )
 
                                         #print(table(mdata$data.type))
-        
+
         if ( max(mdata$pt) == 99 ) {
             cat("PT problem")
             print(table(mdata$pt,mdata$dck))

scr/NOC/config.yml

 
 default:
-  ICOADS_data_path: "/noc/mpoc/surface_data/ICOADS_R3_TOTAL_RDA/ICOADS_R3.0.0T_"
   #ICOADS_data_path: "/noc/mpoc/surface_data/ICOADS_R3_PROC/INPUT_Rda/r3.0.0/ICOADS_R3.0.0T_"
+  ICOADS_data_path: "/noc/mpoc/surface_data/ICOADS_R3_PROC/INPUT_Rda/r3.0.2/ICOADS_R3.0.0T_"
   ICOADS_RELEASE: "r3.0.0"
-  ICOADS_NRT_RELEASE: "r3.0.1"
+  ICOADS_NRT_RELEASE: "r3.0.2"
+  code_dir: "/noc/users/metman/icoads-r-hostace/rscripts/"
 
   # submarines in dck 245
   # from https://github.com/oldweather/RN-WW2/blob/master/analyses/submarines/sub_names
@@ -28,27 +29,26 @@ default:
 
   #output_dir: "../output_data/"
   # The next paths are need in icoads.utils::get_pub47() for log output
-  output_dir_pub47_txt: "/noc/mpoc/surface_data/ICOADS_FINAL_VERSION/HOSTACE_PROC_ANC_INFO/"
-  #output_dir: "/noc/mpoc/surface_data/ICOADS_FINAL_VERSION/output_data/"
-  output_dir: "/noc/mpoc/surface_data/ICOADS_v0.0.1/output_data/"
+  output_dir_pub47_txt: "/noc/mpoc/surface_data/ICOADS_R3_PROC/HOSTACE_PROC_ANC_INFO/"
+  output_dir: "/noc/mpoc/surface_data/ICOADS_R3_PROC/output_data/"
   profiler_output: ""
 
   # setup for GIULIA's tracking
   track_code_dir: "/noc/users/metman/GIULIA_TRACK_CODE/TRACK_CODE_FROM_JASMIN/"
   track_root_athena: "/pgdata/metman/TRACK"
-  track_input: "/noc/mpoc/surface_data/ICOADS_v0.0.1/output_data/TRACK_INPUT/"
-  track_output: "/noc/mpoc/surface_data/ICOADS_v0.0.1/output_data/GTRACK_OUT/"
+  track_input: "/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/TRACK_INPUT/"
+  track_output: "/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/GTRACK_OUT/"
 
   # Final Processing Dirs
-  data_dir: "/noc/mpoc/surface_data/ICOADS_v0.0.1/output_data/FINAL_PROC"
+  data_dir: "/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/FINAL_PROC"
 
   # Height Files/Dirs
-  ens_hts_dir: "/noc/mpoc/surface_data/ICOADS_v0.0.1/output_data/ENS_HTS/"
-  ens_grid_hts_dir: "/noc/mpoc/surface_data/ICOADS_v0.0.1/output_data/GRID_HTS/"
-  def_hts_file: "/noc/mpoc/surface_data/ICOADS_v0.0.1/output_data/default_height_ensemble.Rda"
+  ens_hts_dir: "/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/ENS_HTS/"
+  ens_grid_hts_dir: "/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/GRID_HTS/"
+  def_hts_file: "/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/default_height_ensemble.Rda"
 
   # Diurnal dirs
-  dir_flags_dir: "/noc/mpoc/surface_data/ICOADS_v0.0.1/output_data/DIURNAL_FAIL_IDS/"
+  dir_flags_dir: "/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_ZERO/output_data/DIURNAL_FAIL_IDS/"
 
   # Use TOTAL data or FINAL data
   use_total: TRUE

scr/NOC/run_disj.csh

 #!/bin/csh
 
-# Immediately exit a script when it encounters an error
-#set -e
-
-# start year = $1
-# end year = $2
+    # Immediately exit a script when it encounters an error
+    #set -e
+    
+    # start year = $1
+    # end year = $2
 
+#source /noc/users/metman/SETUPS/setup_icproc
 source setup_icproc
 
 if ( ! -d ${basedir}OPFILES  ) then
@@ -23,9 +24,6 @@ touch ${log_output_dir}/dop.$1.$2
 touch ${log_output_dir}/dop.$1.$2
 echo "logfile" ${log_output_dir}/dop.$1.$2
 
-echo 'HOOVERING' >> ${log_output_dir}/dop.$1.$2
-Rscript $codedir/hoover_data.R $1 $2 >> ${log_output_dir}/dop.$1.$2
-
 echo 'running disjoint tracking' >> ${log_output_dir}/dop.$1.$2
 echo 'start year' $1 >> ${log_output_dir}/dop.$1.$2
 echo 'end year' $2 >> ${log_output_dir}/dop.$1.$2
@@ -35,15 +33,15 @@ echo "---------------" >> ${log_output_dir}/dop.$1.$2
 set y = $1
  while ( $y <= $2 )
 
-   echo "XXX getting disjoint" $y $y >> ${log_output_dir}/dop.$1.$2
-   if ( $y <= 1981 ) then
-     Rscript $codedir/run_disj_track2.R $y $y "post" "none" $configfile >> ${log_output_dir}/dop.$1.$2
-   else
-     Rscript $codedir/run_disj_track2.R $y $y "pre" "none" $configfile >> ${log_output_dir}/dop.$1.$2
-   endif
+   # echo "XXX getting disjoint" $y $y >> ${log_output_dir}/dop.$1.$2
+   # if ( $y <= 1981 ) then
+   #   Rscript $codedir/run_disj_track2.R $y $y "post" "none" $configfile >> ${log_output_dir}/dop.$1.$2
+   # else
+   #   Rscript $codedir/run_disj_track2.R $y $y "pre" "none" $configfile >> ${log_output_dir}/dop.$1.$2
+   # endif
 
-   echo 'XXX running joining' >> ${log_output_dir}/dop.$1.$2
-     Rscript $codedir/run_joins.R $y $configfile >> ${log_output_dir}/dop.$1.$2
+   # echo 'XXX running joining' >> ${log_output_dir}/dop.$1.$2
+   #   Rscript $codedir/run_joins.R  $y $configfile  >> ${log_output_dir}/dop.$1.$2
 
    echo 'XXX getting height ensemble' >> ${log_output_dir}/dop.$1.$2
      Rscript $codedir/get_height_ens.R $y $y $configfile >> ${log_output_dir}/dop.$1.$2

scr/NOC/setup_icproc

@@ -9,6 +9,6 @@ module load netcdf
 module load intel_compiler
 module load gcc/7.2
 
-set basedir="/noc/mpoc/surface_data/ICOADS_v0.0.1/"
-set codedir="/noc/users/ricorne/icoads-r-hostace/rscripts/"
+set basedir="/noc/mpoc/surface_data/ICOADS_R3_PROC/"
+set codedir="/noc/users/metman/icoads-r-hostace/rscripts/"
 set configfile="config.yml"