Add required file

rscripts/hoover_data.R

+#!/bin/R
+my_args = commandArgs(trailingOnly=TRUE)
+
+#--------------------------------------------------------------------------------
+Sys.setenv(TZ='GMT')
+#--------------------------------------------------------------------------------
+
+options(warn=1)
+source("~eck/Rscripts/read_rdsfiles.R")
+#source("/noc/mpoc/surface_data/TRACKING/FinalTrack/RSCRIPTS/get_speed.R")
+#require("imma")
+
+print.comm<-FALSE
+rootdir<-"/noc/mpoc/surface_data/ICOADS_v0.0.1/output_data/"
+
+#ids.gen<-read.table("/noc/mpoc/surface/eck/HOSTACE_PROC/gen_id.txt")
+#ids.gen<-c(ids.gen$V1,"")
+
+cmo<-c("01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12")
+
+if ( length(my_args) < 2 ) {my_args<-my_args2}
+
+dirs.old <- paste0("MFILES_",c("SHIP","PLAT","COAST","RESEARCH","REJECT","DRIFT","MOORED","SUBMARINE"))
+
+cat("running for years",my_args[1],my_args[2],"\n")
+
+  for ( year in my_args[1]:my_args[2] ) {
+  dl<-c()
+  #if ( year >= 2015 ) rootdir<-"/gws/nopw/j04/glosat/development/data/interim/HOSTACE_PROC/PROC_R301/output_data/"
+
+    cat("========================================================","\n")
+    cat(year,"\n")
+    cat("========================================================","\n")
+
+   data.orig<-c()
+   for (imo in 1:12) {
+   cat(imo," ")
+   #cat(imo,"\n")
+   #for (imo in 1:1) {
+     ff<-paste0(rootdir,dirs.old,"/",year,"_",imo,".Rda")
+     ff<-ff[file.exists(ff)]
+     if(length(ff) == 0 ) {
+       cat ("no data ")
+       next
+     }
+     data.orig<-rbind.files(ff)
+     sdir<-"MFILES_SHIP"
+     ff<-if(imo<12) paste0(rootdir,sdir,"/",year,"_",imo+1,".Rda") else paste0(rootdir,sdir,"/",year+1,"_1.Rda")
+     ff<-c(ff,if(imo>1) paste0(rootdir,sdir,"/",year,"_",imo-1,".Rda") else paste0(rootdir,sdir,"/",year-1,"_12.Rda"))
+     ff<-ff[file.exists(ff)]
+     if(length(ff) > 0) data.extra<-rbind.files(ff) else data.extra<-NULL
+   #data.orig$orig.id<-data.orig$id
+   #data.orig$id<-NULL
+
+   ff<-paste0(rootdir,"TRACK_INPUT/",year,"_",imo,".Rda")
+   data2track<-NULL
+   if(file.exists(ff)) data2track<-readRDS(ff)
+
+   #cat("rabbit","\n")
+
+   #tdirs<-c("AtlasE","AtlasL","Early_G","USNavy",paste0("Ex",seq(1,7)))
+   #tdirs<-c("AtlasE","AtlasL","Early_G","USNavy",paste0("Ex",seq(1,5)))
+   #tdirs<-c("Atlas1854","Atlas1880","Atlas1900","Atlas1910","Atlas1920")
+   #tdirs<-c(tdirs,"Atlas1930","Atlas1940","Atlas1950","Early_G","USNavy")
+   #tdirs<-c(tdirs,paste0("Ex",seq(1,6)))
+   tdirs<-c("AtlasE","AtlasL","Early_G","USNavy")
+   tdirs<-c(tdirs,paste0("Ex",seq(1,4)))
+   tdirs<-c(tdirs,paste0("dck",c("204","205","732","781","898","899")))
+   tdirs<-c(tdirs,paste0("ExCo0",seq(0,9)),paste0("ExCo",seq(10,35)))
+   #tdirs<-c(tdirs,"NRT301")
+   #tdirs<-c(tdirs,paste0("NRT201",seq(5,9)))
+   #tdirs<-c(tdirs,paste0("NRT2_20",seq(5,9)))
+   #if ( year >= 2015 ) cat("including tracking directories post 2015","\n")
+
+   datatracked<-NULL
+   flist<-list.files(paste0(rootdir,"GTRACK_OUT/",tdirs,"/TRACK_FILLED"),pattern=paste0(year,".",imo,".Rda"),full.name=T)
+   flist<-c(flist,list.files("/noc/mpoc/surface_data/ICOADS_R3_PROC/RUN_FOR_SHAWN/output_data/DISJTR_NEW",patt=as.character(year),full.names=T))
+   if ( length(flist) > 0 ) {
+     datatracked<-rbind.files.addname(flist)
+     datatracked$tgroup<-trimws(datatracked$tgroup)
+     xl<-unlist(gregexpr(pattern ='/TRACK_FILLED',datatracked$tgroup))
+     datatracked$tgroup<-substr(datatracked$tgroup,1,xl-1)
+     datatracked$tgroup<-gsub(paste0(rootdir,"GTRACK_OUT/"),"",datatracked$tgroup)
+     datatracked$tgroup[datatracked$tgroup=="Early_G"]<-"EaGer"
+     datatracked$track.id<-ifelse(!is.na(datatracked$track.id),paste(datatracked$track.id,datatracked$tgroup,sep="_"),NA)
+
+     uu<-datatracked$uid[duplicated(datatracked$uid)]
+     if ( length(uu) > 0 ) {
+       cat("duplicated data",year,imo,"\n")
+       print(table(datatracked$tgroup[datatracked$uid%in% uu],datatracked$dck[datatracked$uid%in% uu]))
+       #stop()
+     }
+   }
+
+   if (print.comm) cat("elephant",year,imo,"\n")
+   cleandata<-readRDS(paste0(rootdir,"SHIP_CLEAN/",year,"_",imo,".Rda"))
+   if ( !(is.null(data.extra))) {
+    data.extra<-subset(data.extra,uid%in%cleandata$uid)
+    if ( nrow(data.extra) > 0 ) {
+     data.extra[setdiff(names(data.orig), names(data.extra))] <- NA
+     data.orig[setdiff(names(data.extra), names(data.orig))] <- NA
+     data.orig<-rbind(data.orig,data.extra)
+    }
+   }
+
+   cleandata$track.id<-cleandata$new.id
+   cleandata$track.id[cleandata$id.type%in%c("generic","invalid")]<-NA
+
+   if (print.comm) cat("camel",year,imo,"\n")
+
+   if ( !is.null(datatracked) ) {
+    cleandata.t<-subset(cleandata,uid %in% datatracked$uid)
+    cleandata<-subset(cleandata,!(uid %in% datatracked$uid))
+    cleandata.t$track.id<-NULL 
+    cleandata.t$id.ok<-NULL 
+    cleandata.t<-merge(cleandata.t,datatracked[,c("uid","track.id","id.ok")],by="uid",all=T)
+    df.out<-rbind(cleandata,cleandata.t)
+   } else {
+    df.out<-cleandata
+   }
+   
+   #data$id.type<-ifelse(is.na(data$track.id),"missing",data$id.type)
+   #cat("old missing IDs =",round(sum(data$id.type!="valid",na.rm=T)/nrow(data)*100,2),"%","\n")
+
+   names.few<-names(table(df.out$track.id))[which(table(df.out$track.id)<4)]
+   df.out$id.type<-ifelse(df.out$track.id %in% names.few,"few",df.out$id.type)
+   df.out$track.id<-ifelse(df.out$track.id %in% names.few,NA,df.out$track.id)
+   #df.out$qcid[which(df.out$id.type=="few")]<-NA
+
+    # set output ID flags for final output
+    #df.out$final.id.flag <- 99
+    #df.out<-merge(df.out,data.orig,all.x=T)
+
+    # remove _g1 from JP data 705-707
+    #df.out$track.id <- ifelse(df.out$dck %in% c(705,706,707) & substr(df.out$track.id,1,2) == "JP", gsub("_g1","",df.out$track.id),df.out$track.id)
+
+   df.out$track.id<-trimws(df.out$track.id)
+   df.out<-subset(df.out,iso==0)
+   df.out<-subset(df.out,trk==0)
+
+   if (print.comm) cat("cabbage",year,imo,"\n")
+     base.outdir <- paste0(rootdir,"/HOOVER_RUNZERO/")
+     if ( !dir.exists(base.outdir) ) dir.create(base.outdir)
+   #cat("there",imo,"\n") 
+   if(nrow(df.out)>0){
+        YRMO <- split(df.out, data.frame(df.out$yr, df.out$mo), drop=TRUE)
+        filenames <- paste(base.outdir,names(YRMO), ".Rda",sep="")
+        jj<-mapply(saveRDS, YRMO, file = filenames )
+   } 
+    #cat("here",imo,"\n") 
+  }
+  cat("done","\n")
+}