Update process_shipdata.R

changed processing to output all data for single month by reading in adjacent months - avoids need to run long sequences of months

rscripts/process_shipdata.R

+  }
+  fn.out <- paste0(basedir,"MFILES_SHIP_PROC/",year,"_",imo,".Rda")
+
+  if ( file.exists(fn.out) ) file.remove(fn.out)
+  if ( year >= 1850 & year < 1996 ) {
+  files.want<-c(fn.in,fn.plus,fn.minus)
+  if ( !file.exists(fn.in) ) files.want<-files.want[files.want!=fn.in]
+  if ( !file.exists(fn.plus) ) files.want<-files.want[files.want!=fn.plus]
+  if ( !file.exists(fn.minus) ) files.want<-files.want[files.want!=fn.minus]
+  if ( length(files.want) == 0 ) next
+    mdata<-rbind.files(files.want)
+  } else {
+    if ( !file.exists(fn.in) & file.exists(fn.out) ) file.remove(fn.out)
+    if ( !file.exists(fn.in) ) next
+    mdata<-readRDS(fn.in)
+  }

rscripts/process_shipdata.R

 
 print.comm<-FALSE
 
-drop<-c("wmi","is","es","rs","ir","rrr","qci","mds","osz","oov","ooz","opv","opz","osiv","osiz","onv","onz","ophv","ophz","ocv","ocz","cce","wwe","ne","nhe","he","cle","cme","che","am","ah","uh","sbi","sa","ri","um","oav","oaz","wx","sx","eoh","hop","smf","sme","smv")
+drop<-c("wmi","is","es","rs","ir","rrr","qci","mds","osz","oov","ooz","opv","opz","osiv","osiz","onv","onz","ophv","ophz","ocv","ocz","cce","wwe","ne","nhe","he","cle","cme","che","am","ah","uh","sbi","sa","ri","um","oav","o
+az","wx","sx","eoh","hop","smf","sme","smv")
 
 if (!dir.exists("MFILES_SHIP_PROC")) dir.create("MFILES_SHIP_PROC")
 
 
 if ( length(args) == 0 ) {args<-args2}
 
-# remove files
-for ( year in args[1]:args[2] ) {
- for ( imo in 1:12 ) {
-  fn.out <- paste0("MFILES_SHIP_PROC/",year,"_",imo, ".Rda")
-  #print(fn)
-  if (file.exists(fn.out)) file.remove(fn.out)
- }
-}
-rm(fn.out)
-
 for ( year in args[1]:args[2] ) {
   cat("-----------------------------------------------","\n")
   cat(year," ")
   #fn.out<-paste0("MFILES_SHIP_PROC/",year,"-",imo,".Rda")
 
   fn.in <- paste0(basedir,"MFILES_SHIP_IDPROC/",year,"_",imo,".Rda")
-  if ( !file.exists(fn.in) ) next
-  mdata<-readRDS(fn.in)
+  if ( imo < 12 ) {
+    fn.plus <- paste0(basedir,"MFILES_SHIP_IDPROC/",year,"_",imo+1,".Rda")
+  } else {
+    fn.plus <- paste0(basedir,"MFILES_SHIP_IDPROC/",year+1,"_",1,".Rda")
+  }
+  if ( imo > 1 ) {
+    fn.minus <- paste0(basedir,"MFILES_SHIP_IDPROC/",year,"_",imo-1,".Rda")
+  } else {
+    fn.minus <- paste0(basedir,"MFILES_SHIP_IDPROC/",year-1,"_",12,".Rda")
+  }
+  fn.out <- paste0(basedir,"MFILES_SHIP_PROC/",year,"_",imo,".Rda")
+
+  if ( file.exists(fn.out) ) file.remove(fn.out)
+  if ( year >= 1850 & year < 1996 ) {
+  files.want<-c(fn.in,fn.plus,fn.minus)
+  if ( !file.exists(fn.in) ) files.want<-files.want[files.want!=fn.in]
+  if ( !file.exists(fn.plus) ) files.want<-files.want[files.want!=fn.plus]
+  if ( !file.exists(fn.minus) ) files.want<-files.want[files.want!=fn.minus]
+  if ( length(files.want) == 0 ) next
+    mdata<-rbind.files(files.want)
+  } else {
+    if ( !file.exists(fn.in) & file.exists(fn.out) ) file.remove(fn.out)
+    if ( !file.exists(fn.in) ) next
+    mdata<-readRDS(fn.in)
+  }
   mdata$date.ch.flag<-0
   # 0 is unchanged
   # 1 is hr missing, 12 local inserted
   var.want<-names(mdata)
   var.want<-var.want[!(var.want %in% drop)]
 
-  if ( print.comm ) cat("carrot","\n")
+  if ( print.comm ) cat("got input data","\n")
  
-  # put missing hour to midday local
-  #time_string <- sprintf("%4d-%02d-%02d 12:00", mdata$yr, mdata$mo, mdata$dy )
-  #my_date_time <- as.POSIXct(time_string) + (mdata$lon / 15)*3600
-  #mdata$date.flag<-ifelse(is.na(mdata$hr),1,mdata$date.flag)
-  #mdata$hr <- ifelse(is.na(mdata$hr),round(as.numeric(format.POSIXct( my_date_time ,"%H")) + as.numeric(format.POSIXct( my_date_time ,"%M"))/60,0),mdata$hr)
-  #rm(time_string)
-  #rm(my_date_time)
-
   # correct MDB times where needed
   # first calculate local time - appropriate for MO
   mdata<-add_date2(mdata)
-  #mdata<-subset(mdata,!is.na(date))  # shouldn't be needed after running again
-  #mdata$date<-as.POSIXct(mdata$date)
   mdata$old.date<-mdata$date
-  #if ( sum(is.na(mdata$date) ) > 0 ) {
-    tmplon <- ifelse (mdata$lon>=360,mdata$lon-360,mdata$lon)
-    mdata$offset <- round(tmplon/360*24*3600)
-    mdata$mlst <- mdata$date + mdata$offset
-    mdata$mlstdate <- strptime(mdata$mlst, format="%Y-%m-%d")
-    tmp1 <- strftime(mdata$mlst, format="%H")
-    tmp2 <- strftime(mdata$mlst, format="%M")
-    mdata$mlsttime <- as.numeric(tmp1) + as.numeric(tmp2)/60
-    rm(tmp1)
-    rm(tmp2)
-    mdata$mlsttime <- round(ifelse(mdata$mlsttime > 23.5,mdata$mlsttime-24,mdata$mlsttime))
-    mdata$mlsttime <- ifelse(mdata$mlsttime > 23.9,mdata$mlsttime-24,mdata$mlsttime)
-  #} else {
-  #  mdata$mlst<- NA
-  #  mdata$mlstdate <- NA
-  #  mdata$mlsttime <- NA
-  #}
-
-  if ( print.comm ) cat("cabbage","\n")
+  tmplon <- ifelse (mdata$lon>=360,mdata$lon-360,mdata$lon)
+  mdata$offset <- round(tmplon/360*24*3600)
+  mdata$mlst <- mdata$date + mdata$offset
+  mdata$mlstdate <- strptime(mdata$mlst, format="%Y-%m-%d")
+  tmp1 <- strftime(mdata$mlst, format="%H")
+  tmp2 <- strftime(mdata$mlst, format="%M")
+  mdata$mlsttime <- as.numeric(tmp1) + as.numeric(tmp2)/60
+  rm(tmp1)
+  rm(tmp2)
+  mdata$mlsttime <- round(ifelse(mdata$mlsttime > 23.5,mdata$mlsttime-24,mdata$mlsttime))
+  mdata$mlsttime <- ifelse(mdata$mlsttime > 23.9,mdata$mlsttime-24,mdata$mlsttime)
+
+  if ( print.comm ) cat("calculated local times","\n")
   # dck 781 adjust time assuming reported time is local
   if ( 781 %in% names(table(mdata$dck))) {
-  if(print.comm)cat('potato',"\n")
     mdata$yr<-ifelse(mdata$dck == 781, as.numeric(format(mdata$mlst,"%Y")), mdata$yr)
     mdata$mo<-ifelse(mdata$dck == 781, as.numeric(format(mdata$mlst,"%m")), mdata$mo)
     mdata$dy<-ifelse(mdata$dck == 781, as.numeric(format(mdata$mlst,"%d")), mdata$dy)
     if ( class(mdata$date)[1] == "numeric") mdata$date<-as.POSIXct(mdata$date,origin="1970-01-01")
   }
 
-  if(print.comm)cat('swede',"\n")
+  if(print.comm)cat('done dck 781',"\n")
 
-  # got local time, now lon2 is -179.99 -> 180; lon is 0 -> 360
-  #mdata$lon2<-ifelse(mdata$lon > 180, mdata$lon-360, mdata$lon)
-  # these are the mdb dcks
-  #d.mdb<-c(201, 202, 203, 204, 205, 206, 207, 209, 210, 211, 213, 214, 215, 216, 218, 221, 223, 224, 226, 227, 229, 230, 233, 234, 239, 245, 246, 247, 249, 254, 255)
-  # make MDB and other time corrections
-  #dtime<-as.difftime(1, unit="days")
-  #mdata$date2<-mdata$date
   # 201 0 GMT is a day early; up & inc 1899
   mdata$date<-ifelse(mdata$dck%in%c(201) & mdata$hr == 0 & mdata$yr<1900, mdata$date+lubridate::days(1),mdata$date)
   if ( class(mdata$date)[1] == "numeric") mdata$date<-as.POSIXct(mdata$date,origin="1970-01-01")
     rm(pig)
   }
 
-  if ( print.comm ) cat("turnip","\n")
+  if ( print.comm ) cat("done MDB date changes","\n")
   mdata$old.lat<-mdata$lat
   mdata$old.lon<-mdata$lon
   if ( class(mdata$date)[1] == "numeric") mdata$date<-as.POSIXct(mdata$date,origin="1970-01-01")
   mdata$hr<-as.numeric(format(mdata$date,"%H"))
   #print(table(mdata$yr,mdata$mo))
 
-  if ( print.comm ) cat("parsnip","\n")
+  mdata<-subset(mdata,yr==year & mo == imo)
+  if ( nrow(mdata)==0 & file.exists(fn.out) ) file.remove(fn.out)
+
+  if ( print.comm ) cat("about to output data","\n")
   YRMO <- split(mdata, data.frame(mdata$yr, mdata$mo), drop=TRUE)
   names(YRMO)<-gsub("\\.","_",names(YRMO))
   fn.out <- paste("MFILES_SHIP_PROC/",names(YRMO), ".Rda",sep="")
-  if ( length(fn.out) > 1 ) cat(paste(fn.out,collapse=";  "),"\n")
-  for ( ii in 1:length(fn.out) ) {
-   if ( file.exists(fn.out[ii]) ) {
-    toadd<-readRDS(fn.out[ii])
-    cat('adding to file',fn.out[ii],nrow(toadd),"reports","\n")
-    op<-rbind.all.columns(YRMO[[ii]],toadd)
-    rm(toadd)
-   } else {
-    op<-YRMO[[ii]]
-    #cat('not adding to file',fn[ii],"\n")
-   }
-   saveRDS(op,fn.out[ii])
-  }
-
-   #odf<-data.frame(sub$uid,sub$date2,sub$flag.date,round(sub$lat,2),sub$flag.lat,round(sub$lon2,2),sub$flag.lon)
+  ii<-1
+  
+  op<-YRMO[[ii]]
+  saveRDS(op,fn.out[ii])
 
   } # end loop over months
   cat("\n","-----------------------------------------------","\n")
  } # end loop over years
-cat('finishing processing ship data inc date changes',args[1],args[2],"\n")
-   
+cat('finishing processing ship data inc date changes',args[1],args[2],"\n")
\ No newline at end of file

[brivas]

Thanks! Your request has been merged.

[brivas]

Hello @eck (Lizz),

I am having problems with this change. I this gitlab interface you can see in red how the code was before and how it is now (in green) with the new changes that you commit last week.

When I run this for a single month it keeps assigning the wrong month to run. For example, if I am running the script for 1995_12.Rda keeps thinking that the month to analyse is 11 and when I get to write out the output of the file I keep getting an error in line 155 error: YRMO[[ii]] is out of bounds.

and the counter ii is always 1.

Are these changes necessary? 1. the adding of the if's at the beginning (lines 38-62) and 2. deleting the for loop at the end of the script lines 162 that we had in the previous version?

[brivas]

Hello @eck (Lizz),

I am having problems with this change. I this gitlab interface you can see in red how the code was before and how it is now (in green) with the new changes that you commit last week.

When I run this for a single month it keeps assigning the wrong month to run. For example, if I am running the script for 1995_12.Rda keeps thinking that the month to analyse is 11 and when I get to write out the output of the file I keep getting an error in line 155 error: YRMO[[ii]] is out of bounds.

and the counter ii is always 1.

Are these changes necessary? 1. the adding of the if's at the beginning (lines 38-62) and 2. deleting the for loop at the end of the script lines 162 that we had in the previous version?