Commit by BAK towards end of jc191 26 Feb 2020

ctd_linkscript

@@ -3,55 +3,54 @@
 #copy the raw and SBE-processed CTD files from the ship machine
 #then make links to the files to be ingested by mexec using the expected name format
 
-cd /local/users/pstar/cruise/data #mexecloc is replaced with $mexecloc by conf_script_cruise
-
-set cpre = JR18002
-
-rsync -av /mnt/data/jcr/current/ctd/jr18002/*.cnv ctd/ASCII_FILES/
-rsync -av /mnt/data/jcr/current/ctd/jr18002/*.bl ctd/ASCII_FILES/
-rsync -av /mnt/data/jcr/current/ctd/*.ros ctd/ASCII_FILES/
-rsync -av /mnt/data/jcr/current/ctd/jr18002/SBE35/ ctd/ASCII_FILES/SBE35/
+cd /local/users/pstar/cruise/data
+
+set cpre = 'CTD_JC191'
+set cruise = jc191
+#set ctdloc = /local/users/pstar/mounts/mnt_cruise_data/Specific_Equipment/CTD/
+set ctdloc = /local/users/pstar/mounts/mnt_cruise_data/Specific_Equipment/CTD/CTD # jc191 CTD appears twice in the directory name
+
+#bak jc191 we don't want all the cnvs with derived quantities, so explicitly pick up the raw and align_ctm
+rsync -av ${ctdloc}/Processed\ Data/${cpre}_???.cnv ctd/ASCII_FILES/
+#rsync -av ${ctdloc}/Processed\ Data/${cpre}_???_Align.cnv ctd/ASCII_FILES/
+rsync -av ${ctdloc}/Processed\ Data/${cpre}_???_Align_CTM.cnv ctd/ASCII_FILES/
+rsync -av ${ctdloc}/Raw\ Data/*.bl ctd/ASCII_FILES/
+rsync -av ${ctdloc}/Processed\ Data/${cpre}*.ros ctd/ASCII_FILES/
+rsync -av ${ctdloc}/Raw\ Data/*.hex ctd/RAW_CTD_FILES/
+rsync -av ${ctdloc}/Raw\ Data/*.XMLCON ctd/RAW_CTD_FILES/
+rsync -av ${ctdloc}/Raw\ Data/*.hdr ctd/RAW_CTD_FILES/
+rsync -av ${ctdloc}/Processed\ Data/${cpre}*.btl ctd/ASCII_FILES/
 
 cd ctd/ASCII_FILES
-foreach i (`ls ${cpre}_???_align_ctm.cnv`)
-   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,"_align")-3,3)}'`
-   set linkfile = ctd_jr18002_${num}_ctm.cnv
+
+foreach i (`ls ${cpre}_???_Align_CTM.cnv`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,"_Align")-3,3)}'`
+   set linkfile = ctd_${cruise}_${num}_ctm.cnv 
+   #echo $linkfile
    if (! -e $linkfile) then
       echo Making link $linkfile for raw file $i
-      ln -s $i $linkfile
+     ln -s $i $linkfile
    endif
 end
-#below could be commented out if T spikes aren't a problem
-foreach i (`ls ${cpre}_???_align.cnv`)
-   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,"_align")-3,3)}'`
-   set linkfile = ctd_jr18002_${num}_noctm.cnv
-   if (! -e $linkfile) then
-      echo Making link $linkfile for raw file
-      ln -s $i $linkfile
-   endif
-end
-#foreach i (`ls ${cpre}_???_align.cnv`)
-#   gzip -f $i
-#end
-#foreach i (`ls ${cpre}_???.cnv`)
-#   gzip -f $i
-#end
 
 foreach i (`ls ${cpre}_???.bl`)
    set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".bl")-3,3)}'`
-   set linkfile = ctd_jr18002_${num}.bl
+   set linkfile = ctd_${cruise}_${num}.bl
    if (! -e $linkfile) then
       echo Making link $linkfile for raw file $i
       ln -s $i $linkfile
    endif
 end
 
-cd ../RAW_CTD_FILES
-rsync -av /mnt/data/jcr/current/ctd/jr18002/*.hdr .
-rsync -av /mnt/data/jcr/current/ctd/jr18002/*.hex .
-rsync -av /mnt/data/jcr/current/ctd/jr18002/*.XMLCON .
-rsync -av /mnt/data/jcr/current/ctd/jr18002/*.btl .
-
-#gzip -f *.hex
+#below block could be commented out if your cruise doesn't have much of a problem with spikes
+foreach i (`ls ${cpre}_???.cnv`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,"cnv")-4,3)}'`
+   set linkfile = ctd_${cruise}_${num}_noctm.cnv 
+   #echo $linkfile
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+     ln -s $i $linkfile
+   endif
+end
 
 cd ../..

ctd_linkscript_atstart_jc191

+#!/bin/csh -f
+#
+#copy the raw and SBE-processed CTD files from the ship machine
+#then make links to the files to be ingested by mexec using the expected name format
+
+cd /local/users/pstar/cruise/data #mexecloc is replaced with $mexecloc by conf_script_cruise
+
+set cpre = JR18002
+
+rsync -av /mnt/data/jcr/current/ctd/jr18002/*.cnv ctd/ASCII_FILES/
+rsync -av /mnt/data/jcr/current/ctd/jr18002/*.bl ctd/ASCII_FILES/
+rsync -av /mnt/data/jcr/current/ctd/*.ros ctd/ASCII_FILES/
+rsync -av /mnt/data/jcr/current/ctd/jr18002/SBE35/ ctd/ASCII_FILES/SBE35/
+
+cd ctd/ASCII_FILES
+foreach i (`ls ${cpre}_???_align_ctm.cnv`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,"_align")-3,3)}'`
+   set linkfile = ctd_jr18002_${num}_ctm.cnv
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+      ln -s $i $linkfile
+   endif
+end
+#below could be commented out if T spikes aren't a problem
+foreach i (`ls ${cpre}_???_align.cnv`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,"_align")-3,3)}'`
+   set linkfile = ctd_jr18002_${num}_noctm.cnv
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file
+      ln -s $i $linkfile
+   endif
+end
+#foreach i (`ls ${cpre}_???_align.cnv`)
+#   gzip -f $i
+#end
+#foreach i (`ls ${cpre}_???.cnv`)
+#   gzip -f $i
+#end
+
+foreach i (`ls ${cpre}_???.bl`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".bl")-3,3)}'`
+   set linkfile = ctd_jr18002_${num}.bl
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+      ln -s $i $linkfile
+   endif
+end
+
+cd ../RAW_CTD_FILES
+rsync -av /mnt/data/jcr/current/ctd/jr18002/*.hdr .
+rsync -av /mnt/data/jcr/current/ctd/jr18002/*.hex .
+rsync -av /mnt/data/jcr/current/ctd/jr18002/*.XMLCON .
+rsync -av /mnt/data/jcr/current/ctd/jr18002/*.btl .
+
+#gzip -f *.hex
+
+cd ../..

ctd_linkscript_from_jc159

+#!/bin/csh -f
+#
+#copy the raw and SBE-processed CTD files from the ship machine
+#then make links to the files to be ingested by mexec using the expected name format
+
+cd /local/users/pstar/cruise/data
+
+set cpre = JC159
+set cruise = jc159
+set ctdloc = /local/users/pstar/mounts/mnt_cruise_data/Specific_Equipment/CTD/
+
+rsync -av ${ctdloc}/Processed\ Data/*.cnv ctd/ASCII_FILES/
+rsync -av ${ctdloc}/Raw\ Data/*.bl ctd/ASCII_FILES/
+rsync -av ${ctdloc}/Processed\ Data/*.ros ctd/ASCII_FILES/
+rsync -av ${ctdloc}/Raw\ Data/*.hex ctd/RAW_CTD_FILES/
+rsync -av ${ctdloc}/Raw\ Data/*.XMLCON ctd/RAW_CTD_FILES/
+rsync -av ${ctdloc}/Raw\ Data/*.hdr ctd/RAW_CTD_FILES/
+rsync -av ${ctdloc}/Processed\ Data/*.btl ctd/ASCII_FILES/
+
+cd ctd/ASCII_FILES
+
+foreach i (`ls ${cpre}_???_Align_CTM.cnv`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,"_Align")-3,3)}'`
+   set linkfile = ctd_${cruise}_${num}_ctm.cnv 
+   #echo $linkfile
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+     ln -s $i $linkfile
+   endif
+end
+
+foreach i (`ls ${cpre}_???.bl`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".bl")-3,3)}'`
+   set linkfile = ctd_${cruise}_${num}.bl
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+      ln -s $i $linkfile
+   endif
+end
+
+#below block could be commented out if your cruise doesn't have much of a problem with spikes
+foreach i (`ls ${cpre}_???.cnv`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,"cnv")-4,3)}'`
+   set linkfile = ctd_${cruise}_${num}_noctm.cnv 
+   #echo $linkfile
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+     ln -s $i $linkfile
+   endif
+end
+
+cd ../..

lad_linkscript_ix

 #!/bin/csh 
-# copy the raw .000 ladcp data into cruise/data/ladcp/raw/
+# copy the raw .000 ladcp data into cruise/data/ladcp/rawdata/
 # then make links to those files using the filename format expected by IX processing
 
+set cpre = JC191
+set cruise = jc191
+set ladloc = /local/users/pstar/mounts/mnt_cruise_data/Specific_Equipment/CTD/LADCP
+
 cd /local/users/pstar/cruise/data/ladcp/rawdata
 
-rsync -av /mnt/data/jcr/current/ladcp/* . 
+#sync files
+rsync -av ${ladloc}/ ./
+
+
+#make links for IX, for dual-instrument pairs of files
 
-#make links for IX
-foreach i (`ls JR18002_*.000`)
-   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".000")-3,3)}'`
+cd /local/users/pstar/cruise/data/ladcp/rawdata/Master/data
+foreach i (`ls ${cpre}_???M.000`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".000")-4,3)}'`
    set linkfile = ${num}DL000.000
-   if(! -e ../ix/raw/$num) then
+   cd /local/users/pstar/cruise/data/ladcp/ix/raw
+   if(! -e $num) then
       echo Making directory $num for raw file $i
-      mkdir -p ../ix/raw/$num
+      mkdir -p $num
    endif
-   cd ../ix/raw/$num
+   cd $num
    if (! -e $linkfile) then
       echo Making link $linkfile for raw file $i
-      ln -s ../../../rawdata/$i $linkfile
+      ln -s ../../../rawdata/Master/data/$i $linkfile
    endif
-   cd ../../../rawdata
+   cd /local/users/pstar/cruise/data/ladcp/rawdata/Master/data
 end
 
+cd /local/users/pstar/cruise/data/ladcp/rawdata/Slave/data
+foreach i (`ls ${cpre}_???S.000`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".000")-4,3)}'`
+   set linkfile = ${num}UL000.000
+   cd /local/users/pstar/cruise/data/ladcp/ix/raw
+   if(! -e $num) then
+      echo Making directory $num for raw file $i #(no master file???) bak jc191 bug fix, the 'no master file' was not commented out
+      mkdir -p $num
+   endif
+   cd $num
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+      ln -s ../../../rawdata/Slave/data/$i $linkfile
+   endif
+   cd /local/users/pstar/cruise/data/ladcp/rawdata/Slave/data
+end
+ 
 cd ../..
+

lad_linkscript_ix_atstart_jc191

+#!/bin/csh 
+# copy the raw .000 ladcp data into cruise/data/ladcp/raw/
+# then make links to those files using the filename format expected by IX processing
+
+cd /local/users/pstar/cruise/data/ladcp/rawdata
+
+rsync -av /mnt/data/jcr/current/ladcp/* . 
+
+#make links for IX
+foreach i (`ls JR18002_*.000`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".000")-3,3)}'`
+   set linkfile = ${num}DL000.000
+   if(! -e ../ix/raw/$num) then
+      echo Making directory $num for raw file $i
+      mkdir -p ../ix/raw/$num
+   endif
+   cd ../ix/raw/$num
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+      ln -s ../../../rawdata/$i $linkfile
+   endif
+   cd ../../../rawdata
+end
+
+cd ../..

lad_linkscript_ix_from_jc159

+#!/bin/csh 
+# copy the raw .000 ladcp data into cruise/data/ladcp/rawdata/
+# then make links to those files using the filename format expected by IX processing
+
+set cpre = JC159
+set cruise = jc159
+set ladloc = /local/users/pstar/mounts/mnt_cruise_data/Specific_Equipment/CTD/LADCP
+
+cd /local/users/pstar/cruise/data/ladcp/rawdata
+
+#sync files
+rsync -av ${ladloc}/ ./
+
+
+#make links for IX, for dual-instrument pairs of files
+
+cd /local/users/pstar/cruise/data/ladcp/rawdata/Master/data
+foreach i (`ls ${cpre}_???m.000`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".000")-4,3)}'`
+   set linkfile = ${num}DL000.000
+   cd /local/users/pstar/cruise/data/ladcp/ix/raw
+   if(! -e $num) then
+      echo Making directory $num for raw file $i
+      mkdir -p $num
+   endif
+   cd $num
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+      ln -s ../../../rawdata/Master/data/$i $linkfile
+   endif
+   cd /local/users/pstar/cruise/data/ladcp/rawdata/Master/data
+end
+
+cd /local/users/pstar/cruise/data/ladcp/rawdata/Slave/data
+foreach i (`ls ${cpre}_???s.000`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".000")-4,3)}'`
+   set linkfile = ${num}UL000.000
+   cd /local/users/pstar/cruise/data/ladcp/ix/raw
+   if(! -e $num) then
+      echo Making directory $num for raw file $i (no master file???)
+      mkdir -p $num
+   endif
+   cd $num
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+      ln -s ../../../rawdata/Slave/data/$i $linkfile
+   endif
+   cd /local/users/pstar/cruise/data/ladcp/rawdata/Slave/data
+end
+ 
+cd ../..
+

lad_linkscript_uh

@@ -9,11 +9,11 @@
 # j001_03.000
 #
 
-set cpre = JC159
-set cruise = jc159
+set cpre = JC191
+set cruise = jc191
 set ladloc = /local/users/pstar/mounts/mnt_cruise_data/Specific_Equipment/CTD/LADCP
 set cl = j
-set uhldir = jc1802
+set uhldir = jc2001
 
 #sync file
 cd /local/users/pstar/cruise/data/ladcp/rawdata
@@ -22,7 +22,7 @@ rsync -av ${ladloc}/ ./
 #make links
 
 cd /local/users/pstar/cruise/data/ladcp/rawdata/Master/data
-foreach i (`ls ${cpre}_???m.000`)
+foreach i (`ls ${cpre}_???M.000`)
    set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".000")-4,3)}'`
    set linkfile = {$cl}{$num}_02.000 #use cast 02 to indicate WHM
    cd /local/users/pstar/cruise/data/ladcp/uh/raw/$uhldir/ladcp
@@ -34,7 +34,7 @@ foreach i (`ls ${cpre}_???m.000`)
 end
 
 cd /local/users/pstar/cruise/data/ladcp/rawdata/Slave/data
-foreach i (`ls ${cpre}_???s.000`)
+foreach i (`ls ${cpre}_???S.000`)
    set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".000")-4,3)}'`
    set linkfile = {$cl}{$num}_03.000 #cast 03 to indicate WHS
    cd /local/users/pstar/cruise/data/ladcp/uh/raw/$uhldir/ladcp

lad_linkscript_uh_atstart_jc191

+#!/bin/csh
+#
+# copy the raw .000 ladcp data into cruise/data/ladcp/rawdata
+# then make links to those files using the filename formats expected by 
+# uh/woce (perl) processing
+#
+# j001_01.000
+# j001_02.000
+# j001_03.000
+#
+
+set cpre = JC191
+set cruise = jc191
+set ladloc = /local/users/pstar/mounts/mnt_cruise_data/Specific_Equipment/CTD/LADCP
+set cl = j
+set uhldir = jc1802
+
+#sync file
+cd /local/users/pstar/cruise/data/ladcp/rawdata
+rsync -av ${ladloc}/ ./
+
+#make links
+
+cd /local/users/pstar/cruise/data/ladcp/rawdata/Master/data
+foreach i (`ls ${cpre}_???M.000`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".000")-4,3)}'`
+   set linkfile = {$cl}{$num}_02.000 #use cast 02 to indicate WHM
+   cd /local/users/pstar/cruise/data/ladcp/uh/raw/$uhldir/ladcp
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+      ln -s ../../../../rawdata/Master/data/$i $linkfile
+   endif
+   cd /local/users/pstar/cruise/data/ladcp/rawdata/Master/data
+end
+
+cd /local/users/pstar/cruise/data/ladcp/rawdata/Slave/data
+foreach i (`ls ${cpre}_???S.000`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".000")-4,3)}'`
+   set linkfile = {$cl}{$num}_03.000 #cast 03 to indicate WHS
+   cd /local/users/pstar/cruise/data/ladcp/uh/raw/$uhldir/ladcp
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+      ln -s ../../../../rawdata/Slave/data/$i $linkfile
+   endif
+   cd /local/users/pstar/cruise/data/ladcp/rawdata/Slave/data
+end
+
+#and for ctd data
+cd /local/users/pstar/cruise/data/ladcp/ctd
+foreach i (`ls ctd.???.02.asc`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".02.asc")-3,3)}'`
+   set linkfile = ctd.{$num}.03.asc
+   if (! -e $linkfile) then
+      echo Making link $linkfile
+      ln -s $i $linkfile
+   endif
+end
+cd ..
+

mexec_cruise_backup_jc191

+# backup mexec cruise directory from workstation to external hard drive using rsync
+# usage: > mexec_cruise_backup drivename
+# e.g. > mexec_cruise_backup jr16002a
+# the external hard drives must be named/mounted as $cruise$letter
+# e.g. for cruise jr16002: jr16002a, jr16002b, etc.
+#
+
+# setup
+
+
+set target = $1
+set thiscruise = `awk -v nm="$target" 'BEGIN {print substr(nm,1,length(nm)-2)}'`
+#set thiscruise = `echo $target | sed s'/.$//'` 
+set target1 = /media/${target}/this_is_${target}
+set root =  /local/users/pstar/cruise
+set progroot = /local/users/pstar
+set now = `date +%Y%m%d%H%M%S`
+set backuplog = $root/data/other_backups/backup_log_${now}
+
+set thiscruise = jc191
+
+if (-e ${target1}) then
+  set backupdir = /media/${target}/${thiscruise}/backup_${now}
+  set location = /media/${target}
+else
+  echo "disk ${target} not found"
+exit
+endif
+
+if (-e $backuplog) then
+  echo " "
+exit
+else
+  touch $backuplog
+endif
+
+echo $backupdir
+if (-e $backupdir) then
+  echo " "
+  exit
+else
+  mkdir $backupdir
+endif
+
+pushd $root
+
+echo "disk ${target} found at ${location}" >> $backuplog
+
+# dot login and cshrc
+echo -------------        >> $backuplog
+echo Date `date`          >> $backuplog
+echo copying login and cshrc  >> $backuplog
+echo ------               >> $backuplog
+/bin/cp -p ~/.login  $root/data/other_backups/dot_login
+/bin/cp -p ~/.cshrc  $root/data/other_backups/dot_cshrc
+/bin/cp -p ~/.bashrc  $root/data/other_backups/dot_bashrc
+echo -------------        >> $backuplog
+echo Date `date`          >> $backuplog
+echo done login and cshrc  >> $backuplog
+echo ------               >> $backuplog
+
+# software
+echo -------------        >> $backuplog
+echo Date `date`          >> $backuplog
+echo start copying software >> $backuplog
+echo ------               >> $backuplog
+cd $root
+rsync -a sw ${backupdir}
+cd $progroot
+rsync -a programs ${backupdir}
+echo -------------        >> $backuplog
+echo Date `date`          >> $backuplog
+echo done copying software >> $backuplog
+echo ------               >> $backuplog
+
+# backup everything
+echo -------------        >> $backuplog
+echo Date `date`          >> $backuplog
+echo start copy to disk   >> $backuplog
+echo ------               >> $backuplog
+cd $root
+rsync -a --exclude jcrfs data ${backupdir}
+echo -------------        >> $backuplog
+echo Date `date`          >> $backuplog
+echo done copy to disk    >> $backuplog
+echo ------               >> $backuplog
+
+popd
+
+

modsal_unix

@@ -6,7 +6,7 @@
 #cd data
 #cd ctd
 
-set cruise = 'jc159'
+set cruise = 'jc191'
 
 cd ctd
 cd BOTTLE_SAL

techsas_linkscript

@@ -28,12 +28,17 @@ foreach techfile (`/bin/ls ../netcdf_files_rawdir/*/*`)
       set ss2 = `echo $sss | awk '{print $5}'`
       if ($ss2 > 10000) then
         echo "  making link $linkfile for raw file $techfile " >>! $link_log
+        echo "  making link $linkfile for raw file $techfile " 
         ln -s $techfile $linkfile
       else
         echo "  ignoring $linkfile length $ss2 is less than 10k bytes " >>! $link_log
+        echo "  ignoring $linkfile length $ss2 is less than 10k bytes " 
+        echo "  link command would have been ln -s $techfile $linkfile " >>! $link_log
+        echo "  link command would have been ln -s $techfile $linkfile " 
       endif
     else
       echo "  ignoring $linkfile has zero length " >>! $link_log
+      echo "  ignoring $linkfile has zero length " 
     endif
   endif
 end

uhdas_04_export_nc

@@ -45,8 +45,12 @@ echo working in ${thisdir}
 thisdir=`pwd`
 echo adcp_nc.py in ${thisdir}
 which adcp_nc.py
-adcp_nc.py adcpdb contour/${pingtype} jc184 ${pingtype}
+adcp_nc.py adcpdb contour/${pingtype} jc191 ${pingtype}
 
+thisdir=`pwd`
+echo quick_adcp.py in ${thisdir}
+which quick_adcp.py
+quick_adcp.py --steps2rerun matfiles --auto
 
 popd
 fi