dy105 version (original was jr18002), adds uhdas related scripts

ctd_linkscript~

+#!/bin/csh -f
+#
+#copy the raw and SBE-processed CTD files from the ship machine
+#then make links to the files to be ingested by mexec using the expected name format
+
+cd /local/users/pstar/cruise/data
+
+set cpre = JC159
+set cruise = jc159
+set ctdloc = /local/users/pstar/mounts/mnt_cruise_data/Specific_Equipment/CTD/
+
+rsync -av ${ctdloc}/Processed\ Data/*.cnv ctd/ASCII_FILES/
+rsync -av ${ctdloc}/Raw\ Data/*.bl ctd/ASCII_FILES/
+rsync -av ${ctdloc}/Processed\ Data/*.ros ctd/ASCII_FILES/
+rsync -av ${ctdloc}/Raw\ Data/*.hex ctd/RAW_CTD_FILES/
+rsync -av ${ctdloc}/Raw\ Data/*.XMLCON ctd/RAW_CTD_FILES/
+rsync -av ${ctdloc}/Raw\ Data/*.hdr ctd/RAW_CTD_FILES/
+rsync -av ${ctdloc}/Processed\ Data/*.btl ctd/ASCII_FILES/
+
+cd ctd/ASCII_FILES
+
+foreach i (`ls ${cpre}_???_Align_CTM.cnv`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,"_Align")-3,3)}'`
+   set linkfile = ctd_${cruise}_${num}_ctm.cnv 
+   echo $linkfile
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+     ln -s $i $linkfile
+   endif
+end
+
+foreach i (`ls ${cpre}_???.bl`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".bl")-3,3)}'`
+   set linkfile = ctd_${cruise}_${num}.bl
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+      ln -s $i $linkfile
+   endif
+end
+
+cd ../..

finddir.noread

+#
+find . -type d -exec ls -ld {} \; | egrep -ve "r.xr.xr.x"
+
+if($status == "0") then
+
+#the grep command found something because it finished with status zero
+
+echo " "
+echo "----------------------------------------------------"
+echo " The above directories do not have ugo rx permission"
+echo " This makes them unreadable to some users           "
+echo "----------------------------------------------------"
+
+endif
+

findfil.noread

+#
+find . -type f -exec ls -ld {} \; | egrep -ve "-r..r..r.."
+
+if($status == "0") then
+
+#the grep command found something because it finished with status zero
+ 
+echo " "
+echo "----------------------------------------------------"
+echo " The above files do not have ugo r permission"
+echo " This makes them unreadable to some users           "
+echo "----------------------------------------------------"
+
+endif

fixperms2

+#
+#BAK 18 Sep 2000
+#simple script to fix file permissions to make files more
+#readable. If the file has user read permission,
+#give read permission to group and other. If file
+#has x permission (ie executable or directory) then
+#add rx for group and other. The perm '-400'
+#means 'at least 400'.
+
+# I don't think the 'find .' construct works on SGIs
+
+find . -perm -400 -exec chmod go+r {} \;
+
+find . -perm -500 -exec chmod go+rx {} \;
+
+#check for any problems
+
+#~bak/commands/finddir.noread
+#~bak/commands/findfil.noread
+finddir.noread
+findfil.noread

jcr/conf_script_TPL

+# copy TPLs and edit oceanlogger
+
+set in = scs_raw
+set ot = scs_sed
+
+cd ~
+cd cruise
+cd data
+
+/bin/cp -p $in/*TPL $ot
+
+cd $ot
+
+cat << ! >! wk
+126,oceanlogger-sampletimeyyyy,YYYY
+126,oceanlogger-sampletimeddd,DDD
+126,oceanlogger-sampletimehhmmss,HH:MM:SS
+!
+
+cat oceanlogger.TPL | sed -n '2,$p' >> wk
+
+mv oceanlogger.TPL oceanlogger.TPL_original
+mv wk oceanlogger.TPL

jcr/vmadcp_linkscript_jc

+#!/bin/csh -f
+#
+# Go through loop once for each adcp
+
+set cruise = jc159
+set CRUISE = JC159
+set vmloc = /local/users/pstar/mounts/mnt_cruise_data/Ship_Systems/Acoustics/
+
+foreach adp ('75' '150')
+  echo $adp
+  set pre = ${CRUISE}_OS
+  cd /local/users/pstar/cruise/data/vmadcp
+  cd ${cruise}_os${adp}
+
+  echo "Synchronising raw data for the $adp"
+  rsync -av ${vmloc}/OS${adp}kHz/raw_data/${pre}* vmdas_data/
+
+end
+
+#now sync raw data to banba
+rsync -avz --exclude NMEA --exclude adcp_pyproc/ --exclude fake_uhdas_data/ --exclude mproc/ /local/users/pstar/cruise/data/vmadcp/ pstar@192.168.62.110:/local/users/pstar/codas_shared/$cruise/
+
+#and sync processed data back
+rsync -avz --exclude Temporary_bin/ --exclude 'vmdas_data*'/ --exclude fake_uhdas_data/ --exclude mproc/ pstar@192.168.62.110:/local/users/pstar/codas_shared/$cruise/ /local/users/pstar/cruise/data/vmadcp/
+

jcr/vmadcp_linkscript_jcr

+#!/bin/csh -f
+#
+#foreach i (`ls rawdata/*JC32*`)
+
+# overhaul on jr302 may 2014, to fit with new python codas
+# may need modification on cook/discovery to handle os75 and os150
+cd /local/users/pstar/cruise/data/vmadcp/
+
+echo "synchronising raw data"
+rsync -av ../jcrfs/current/adcp/ jr17001_os75/vmdas_data/
+# jcrfs is link to raw data on jrlb
+
+echo "making links to individual sequence directories"
+cd jr17001_os75/vmdas_data
+foreach i (`ls *.*`) # list all *.* to avoid collecting N1R directory
+set num = `echo $i | awk -F'.' '{print substr($1,length($1)-9,3)}'`
+
+set linkdir = rawdata{$num}
+
+if (! -e $linkdir) then
+  echo making $linkdir
+  mkdir $linkdir
+endif
+
+set rfile = $i
+set lfile = $i
+cd $linkdir
+if (! -e $lfile) then
+  echo linking ${lfile} to ${rfile}
+  ln -s ../${rfile} ${lfile}
+endif
+cd ..
+
+end

lad_linkscript_uh

+#!/bin/csh
+#
+# copy the raw .000 ladcp data into cruise/data/ladcp/rawdata
+# then make links to those files using the filename formats expected by 
+# uh/woce (perl) processing
+#
+# j001_01.000
+# j001_02.000
+# j001_03.000
+#
+
+set cpre = JC159
+set cruise = jc159
+set ladloc = /local/users/pstar/mounts/mnt_cruise_data/Specific_Equipment/CTD/LADCP
+set cl = j
+set uhldir = jc1802
+
+#sync file
+cd /local/users/pstar/cruise/data/ladcp/rawdata
+rsync -av ${ladloc}/ ./
+
+#make links
+
+cd /local/users/pstar/cruise/data/ladcp/rawdata/Master/data
+foreach i (`ls ${cpre}_???m.000`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".000")-4,3)}'`
+   set linkfile = {$cl}{$num}_02.000 #use cast 02 to indicate WHM
+   cd /local/users/pstar/cruise/data/ladcp/uh/raw/$uhldir/ladcp
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+      ln -s ../../../../rawdata/Master/data/$i $linkfile
+   endif
+   cd /local/users/pstar/cruise/data/ladcp/rawdata/Master/data
+end
+
+cd /local/users/pstar/cruise/data/ladcp/rawdata/Slave/data
+foreach i (`ls ${cpre}_???s.000`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".000")-4,3)}'`
+   set linkfile = {$cl}{$num}_03.000 #cast 03 to indicate WHS
+   cd /local/users/pstar/cruise/data/ladcp/uh/raw/$uhldir/ladcp
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+      ln -s ../../../../rawdata/Slave/data/$i $linkfile
+   endif
+   cd /local/users/pstar/cruise/data/ladcp/rawdata/Slave/data
+end
+
+#and for ctd data
+cd /local/users/pstar/cruise/data/ladcp/ctd
+foreach i (`ls ctd.???.02.asc`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".02.asc")-3,3)}'`
+   set linkfile = ctd.{$num}.03.asc
+   if (! -e $linkfile) then
+      echo Making link $linkfile
+      ln -s $i $linkfile
+   endif
+end
+cd ..
+

lad_linkscript_uhpy

+#!/bin/csh
+#
+# copy the raw .000 ladcp data into cruise/data/ladcp/rawdata
+# then make links to those files using the filename formats expected by 
+# uh/clivar (python) processing
+#
+
+set cpre = JC159
+set cruise = jc159
+set ladloc = /local/users/pstar/mounts/mnt_cruise_data/Specific_Equipment/CTD/LADCP
+
+#sync file
+cd /local/users/pstar/cruise/data/ladcp/rawdata
+rsync -av ${ladloc}/ ./
+
+#make links
+
+cd /local/users/pstar/cruise/data/ladcp/rawdata/Master/data
+foreach i (`ls ${cpre}_???m.000`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".000")-4,3)}'`
+   set linkfile = wh{$num}_02.dat #use cast 02 to indicate WHM
+   cd /local/users/pstar/cruise/data/ladcp/uhpy/raw
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+      ln -s ../../rawdata/Master/data/$i $linkfile
+   endif
+   cd /local/users/pstar/cruise/data/ladcp/rawdata/Master/data
+end
+
+cd /local/users/pstar/cruise/data/ladcp/rawdata/Slave/data
+foreach i (`ls ${cpre}_???s.000`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".000")-4,3)}'`
+   set linkfile = wh{$num}_03.dat #cast 03 to indicate WHS
+   cd /local/users/pstar/cruise/data/ladcp/uhpy/raw
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+      ln -s ../../rawdata/Slave/data/$i $linkfile
+   endif
+   cd /local/users/pstar/cruise/data/ladcp/rawdata/Slave/data
+end
+
+#and for ctd data
+cd /local/users/pstar/cruise/data/ladcp/ctd
+foreach i (`ls ctd.???.02.asc`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".02.asc")-3,3)}'`
+   set linkfile = ctd.{$num}.03.asc
+   if (! -e $linkfile) then
+      echo Making link $linkfile
+      ln -s $i $linkfile
+   endif
+end
+cd ..
+

mexec_cruise_backup_jc184

+# backup mexec cruise directory from workstation to external hard drive using rsync
+# usage: > mexec_cruise_backup drivename
+# e.g. > mexec_cruise_backup jr16002a
+# the external hard drives must be named/mounted as $cruise$letter
+# e.g. for cruise jr16002: jr16002a, jr16002b, etc.
+#
+
+# setup
+
+
+set target = $1
+set thiscruise = `awk -v nm="$target" 'BEGIN {print substr(nm,1,length(nm)-2)}'`
+#set thiscruise = `echo $target | sed s'/.$//'` 
+set target1 = /media/${target}/this_is_${target}
+set root =  /local/users/pstar/cruise
+set progroot = /local/users/pstar
+set now = `date +%Y%m%d%H%M%S`
+set backuplog = $root/data/other_backups/backup_log_${now}
+
+set thiscruise = jc184
+
+if (-e ${target1}) then
+  set backupdir = /media/${target}/${thiscruise}/backup_${now}
+  set location = /media/${target}
+else
+  echo "disk ${target} not found"
+exit
+endif
+
+if (-e $backuplog) then
+  echo " "
+exit
+else
+  touch $backuplog
+endif
+
+echo $backupdir
+if (-e $backupdir) then
+  echo " "
+  exit
+else
+  mkdir $backupdir
+endif
+
+pushd $root
+
+echo "disk ${target} found at ${location}" >> $backuplog
+
+# dot login and cshrc
+echo -------------        >> $backuplog
+echo Date `date`          >> $backuplog
+echo copying login and cshrc  >> $backuplog
+echo ------               >> $backuplog
+/bin/cp -p ~/.login  $root/data/other_backups/dot_login
+/bin/cp -p ~/.cshrc  $root/data/other_backups/dot_cshrc
+/bin/cp -p ~/.bashrc  $root/data/other_backups/dot_bashrc
+echo -------------        >> $backuplog
+echo Date `date`          >> $backuplog
+echo done login and cshrc  >> $backuplog
+echo ------               >> $backuplog
+
+# software
+echo -------------        >> $backuplog
+echo Date `date`          >> $backuplog
+echo start copying software >> $backuplog
+echo ------               >> $backuplog
+cd $root
+rsync -a sw ${backupdir}
+cd $progroot
+rsync -a programs ${backupdir}
+echo -------------        >> $backuplog
+echo Date `date`          >> $backuplog
+echo done copying software >> $backuplog
+echo ------               >> $backuplog
+
+# backup everything
+echo -------------        >> $backuplog
+echo Date `date`          >> $backuplog
+echo start copy to disk   >> $backuplog
+echo ------               >> $backuplog
+cd $root
+rsync -a --exclude jcrfs data ${backupdir}
+echo -------------        >> $backuplog
+echo Date `date`          >> $backuplog
+echo done copy to disk    >> $backuplog
+echo ------               >> $backuplog
+
+popd
+
+

mexec_cruise_backup_public

+rsync -av --exclude photos --exclude outreach /local/users/pstar/mounts/mnt_smb_public/JC159/ /local/users/pstar/jc159/from_public_jc159/
+

modsal_unix

-#!/bin/csh
+#!/bin/csh 
 #
 #Script by DADES on cruise jr302 to make mac files readable in unix.
+# tidied up by bak for dy040
 
 #cd data
 #cd ctd
+
+set cruise = 'jc159'
+
 cd ctd
 cd BOTTLE_SAL
-foreach i (`ls /local/users/pstar/cruise/data/ctd/BOTTLE_SAL/sal_jr302_???.csv`)
+foreach i (`ls /local/users/pstar/cruise/data/ctd/BOTTLE_SAL/sal_${cruise}_???.csv`)
+
 mac2unix -n $i ${i}_linux
 echo " " >> ${i}_linux
+echo " " >> ${i}_linux # extra newline added by bak for dy040; may not be needed
 end
 
-foreach i (`ls /local/users/pstar/cruise/data/ctd/BOTTLE_SAL/sal_jr302_????.csv`)
+foreach i (`ls /local/users/pstar/cruise/data/ctd/BOTTLE_SAL/tsg_${cruise}_???.csv`)
 mac2unix -n $i ${i}_linux
 echo " " >> ${i}_linux
+echo " " >> ${i}_linux
 end
 
-rm sal_jr302_01.csv
-cat *linux >>! sal_jr302_01.csv
+touch sal_${cruise}_01.csv
+/bin/rm sal_${cruise}_01.csv
+cat *linux >>! sal_${cruise}_01.csv
 
 
+chmod ugo+r *linux

sal_linkscript_incomplete

+#!/bin/csh -f
+#
+#run this after editing the .xls files to add a sampnum column and saving as .csv on 
+#the ship machine
+#
+#copies salinometer logs from the ship machine, makes links, and converts to unix format
+#then concatenates all the .csv_linux files into sal_jc159_01.csv
+#
+#if run with three input arguments: crate number, day, month (of analysis), it
+#will copy and convert only the files with name JC159*CTDnn*dd*mmm*.*
+#if run with no input arguments, it will copy and convert everything, clobbering
+#the existing .csv_linux files, which means that if you are going to make edits 
+#you should make them in the original .xls or .csv files i
+#(or introduce them in cruise_options/opt_jc159.m)
+
+cd /local/users/pstar/cruise/data/ctd/BOTTLE_SAL
+
+set cpre = JC159*CTD
+set cruise = jc159
+set rloc = /local/users/pstar/mounts/mnt_cruise_data/Specific_Equipment/CTD/Autosal
+
+if (-e $1) #do all files
+   rsync -av ${rloc}/*.csv ./
+   rsync -av ${rloc}/*.xls ./
+   foreach i (`ls ${cpre}*.csv`)
+      set n
+
+foreach i (`ls ${cpre}_???_Align_CTM.cnv`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,"_Align")-3,3)}'`
+   set linkfile = ctd_${cruise}_${num}_ctm.cnv 
+   echo $linkfile
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+     ln -s $i $linkfile
+   endif
+end
+
+foreach i (`ls ${cpre}_???.bl`)
+   set num = `awk -v nm="$i" 'BEGIN {print substr(nm,index(nm,".bl")-3,3)}'`
+   set linkfile = ctd_${cruise}_${num}.bl
+   if (! -e $linkfile) then
+      echo Making link $linkfile for raw file $i
+      ln -s $i $linkfile
+   endif
+end
+
+cd ../..

techsas_linkscript

@@ -7,9 +7,7 @@
 
 
 cd /local/users/pstar #~/h # cludge on eriu when pstar home directory is wrong
-cd cruise
-cd data
-cd techsas
+cd cruise/data/techsas
 cd netcdf_files_links
 
 set now = `date +%Y%m%d%H%M%S`

uhdas_00

+uhdas_00_linkmerge
\ No newline at end of file

uhdas_00_linkmerge

+#!/bin/csh -f
+#
+# bak in Amsterdam before start of jc184 uhdas trial
+# 3 July 2019
+#
+# 1) rsync uhdas data from uhdas server to local machine
+#
+# 2) ensure postprocessing directories exist
+#
+# If the script hasn't been run recently, there will 
+# be a lot of files to sync across. The processing on the uhdas
+# server updates plenty of things in routine processing every few minutes.
+# If the script is then immediately run again, there will be a few files
+# that are updated every ping.
+#
+# List of cruise segments to be processed is found in
+# ~/cruise/data/vmadcp/cruise_segments
+
+cd /local/users/pstar/cruise/data/vmadcp/spprocessing
+
+foreach i (`cat ~/cruise/data/vmadcp/cruise_segments`)
+echo linking ${i} to $1
+link_uhdaslegs.py ../atsea/${i} $1
+end
+
+if (! -e config) then
+mkdir config
+cp ../atsea/${i}/raw/config/*_proc.py config/$1_proc.py
+echo 'edit config/*proc.py cruiseid, uhdas_dir'
+endif
+

uhdas_01

+uhdas_01_syncraw
\ No newline at end of file

uhdas_01_syncraw

+#!/bin/csh -f
+#
+# bak in Amsterdam before start of jc184 uhdas trial
+# 3 July 2019
+#
+# 1) rsync uhdas data from uhdas server to local machine
+#
+# 2) ensure postprocessing directories exist
+#
+# If the script hasn't been run recently, there will 
+# be a lot of files to sync across. The processing on the uhdas
+# server updates plenty of things in routine processing every few minutes.
+# If the script is then immediately run again, there will be a few files
+# that are updated every ping.
+#
+# List of cruise segments to be processed is found in
+# ~/cruise/data/vmadcp/cruise_segments
+
+set rdir = ~/mounts/uhdas_data
+set ldir = ~/cruise/data/vmadcp
+
+mkdir -p ${ldir}/atsea
+mkdir -p ${ldir}/postprocessing
+
+foreach i (`cat ~/cruise/data/vmadcp/cruise_segments`)
+echo syncing ${i} to ${ldir}
+rsync -auv ${rdir}/${i} ${ldir}/atsea/
+
+mkdir -p ${ldir}/postprocessing/${i}/proc_editing
+mkdir -p ${ldir}/postprocessing/${i}/proc_archive
+end
+
+echo end of syncing ${i} to ${ldir}
+