#! /bin/csh -f
#
#	Utility for burning x-ray data sets onto DVD-R						-James Holton  7-30-07
#	using a rImage DVD producer robot
#
#
if($#argv == 0) then
    set BAD
    goto Help
endif

goto Setup
############################################################################
Help:
cat << EOF

usage: backup_rimage.com directory [otherdirectory] -not junk -only good -since time -copies 1 -splituser

where:
directory  - is the directory you want to back up onto a DVD-R
junk       - are files/subdirectories you want to leave out
good	   - is a string that the back-ed up filenames must contain
time       - either "hour", "day" or a filename no files as old or older than this file will be included
-priority  - urgency (default is 4)
-copies    - number of copies to make (default is 1)
-splituser - don't mix different users data on the same DVD
-finish    - change the "finished run signal" filename

example:

backup_rimage.com /data/mcfuser/whatever -not '_0_'
will back up everything but screening shots

backup_rimage.com /data/mcfuser/whatever -only img -only scan -not '_0_'
will back up every *.img or *.scan file but no screening shots

backup_rimage.com /data/mcfuser -since /data/mcfuser/blah/firstrun_1_100.img
will back up everything collected AFTER firstrun_1_100.img

EOF

if($?BAD) goto exit
############################################################################
Return_from_Setup:

echo "host:      `hostname`"
echo "cmdline:   $0 $*"
echo "smbfs:     ${archive_disk}"
echo "smbclient: $smbclient"
echo "remote:    ${remote_folder}"
echo "finish:    $finish_signal_file"
echo "temp:      ${tempfile}*"
echo "nice:     $?NICE"
echo "priority:  $priority"
echo "copies:    $copies"
echo "dirs:      $dirs"
echo "onlyfind:  $onlyfind"
echo "not:       "\"$not\"
echo "splituser: $splitusers"
echo "note:      $note"
echo "titles:    $titles"

startover:
echo -n "examining files...                                                  \r"
rm -f ${tempfile}* >& /dev/null
echo -n "" >! ${tempfile}filenames
if( "$dirs" != "") then
    find $dirs -type f -print >> ${tempfile}filenames
endif
if( "$lists" != "") then
    awk '{print $NF}' $lists >> ${tempfile}filenames
endif
if( -e "$user_diskimage") then
    echo "$user_diskimage" >! ${tempfile}filenames
    echo "x x x |$user_diskimage" >! ${tempfile}dated
    echo "x x x x |$user_diskimage" >! ${tempfile}grouped.txt
    echo "$user_diskimage" >! ${tempfile}dvd0001.list
    goto files_divided
endif

# filter and get dates
if($?weird_filenames) then
    find $dirs -type f -printf "%T@ %s %t |%p\n" |\
    egrep "$onlyfind" |\
    egrep -v "$not" |\
    sort -u |\
    sort -n |\
    cat >! ${tempfile}dated
else
    cat ${tempfile}filenames |\
    egrep "$onlyfind" |\
    egrep -v "$not" |\
    awk '/^\.\//{$0=substr($0,3)} {gsub("[()\\[\\]\\$]","\\\\&");print}' |\
    awk 'NF>=1{print "puts \"[file mtime \""$0"\"] [file size \""$0 "\"] [clock format [file mtime \""$0"\"]  -format \"%b %d %H:%M:%S %Y\"] |"$0"\""}' |\
    tclsh |\
    sort -u |\
    sort -n |\
    cat >! ${tempfile}dated
endif
# format: epoch date |original_filename

# find "groups" of files to keep together (if possible)
echo "end" |\
cat ${tempfile}dated - |\
awk -v lastarchive=$lastarchive '$1>lastarchive' |\
awk -v mediasize=$mediasize '{file=substr($0,index($0,"|")+1);pattern=file;\
  n="";$2=int($2/2048+2)*2048} \
  /[0-9][0-9][0-9]\.img$/{n=split(file,a,"."); ext=a[n];\
      pattern=substr(file,1,length(file)-length(ext)-4) "%03d." ext;}\
  /_E[0-9]_[0-9][0-9][0-9]\.img$/{n=split(file,a,"."); ext=a[n];\
      pattern=substr(file,1,length(file)-length(ext)-8) "_E%d_%03d." ext;}\
  pattern!=lastpattern || /^end$/ {if(gsize>mediasize){breakup=1;--group}\
    for(i=1;i<=files;++i){if(breakup){gsize=size[i];++group};\
	printf "%10s %d %10.0f %10.0f |%s\n",time[i],group,gsize,size[i],name[i]};\
    lastpattern=pattern;gsize=files=breakup=0;++group}\
  {++files;time[files]=$1;size[files]=$2;name[files]=file;gsize+=$2}' |\
sort -n >! ${tempfile}grouped.txt
# format: epoch group_number group_size file_size |filename

set groups = `cat ${tempfile}grouped.txt | wc -l`
if($groups == 0 && $?REALTIME) then
    echo -n "no files!  will check again in 5 min       \r"
    set timeout = 30
    if ($?DEBUG) set timeout = 1
    while ( $timeout > 0 )
	@ timeout = ( $timeout - 1 )
        sleep 10
	rm -f $finish_signal_file
    end
    goto startover
endif

if($?REALTIME && 0) then
    # don't bother trying to keep datasets together
    cat ${tempfile}dated |\
    awk -v lastarchive=$lastarchive '$1>lastarchive'{print $1,NR,$2,$2,substr($0,index($0,"|"))}' |\
    cat >! ${tempfile}grouped.txt
endif

# divide these images up into DVDs
rm -f ${tempfile}dvd????.list >& /dev/null
cat ${tempfile}grouped.txt |\
awk '{gsize=$3;file=substr($0,index($0,"|")+1)}\
   seen[$2]{gsize=0} {++seen[$2];print $1,$2,gsize,$4,"|" file}' |\
awk -v mediasize=$mediasize -v tempfile=$tempfile -v splitusers=$splitusers '\
  {file=substr($0,index($0,"|")+1)}\
  {n=split(file,w,"/")} n<5{delete w} w[2]~/data$/{user=w[3]} \
   w[2]~/data$/ && w[3]~/user$/{user=w[4]}\
   w[2]~/data$/ && w[3]=="sibyls"{user=w[4]}\
   n>5 && w[2]~/data$/ && w[3]~/user$/ && w[4]~/^uc[bs]/ && w[4] !~/[_-]/ {user=w[5]}\
  user != lastuser && splitusers && lastuser{print "new user:", lastuser"->"user}\
  NR==1 || dvdsize+$3>mediasize || (user != lastuser && splitusers) {\
    if(dvd) printf "%s %d%% %s ...\n", dvd, dvdsize/mediasize*100, lastname;\
    lastuser=user; dvd=sprintf("%04d",dvd+1);dvdsize=0}\
  {dvdsize+=$4;lastname=file;print file >> tempfile"dvd"dvd".list"} \
  END{if(dvdsize>0)printf "%s %d%% %s ...\n", dvd, dvdsize/mediasize*100, lastname;}' |\
cat
# > /dev/null

files_divided:

if(! -s ${tempfile}grouped.txt) then
    echo -n "no files!                                                                       \r"
    if ($?REALTIME) then
	# create an "empty" DVD and go forard...
	echo -n "" >! ${tempfile}dvd0001.list
    else
	set BAD = "empty DVD"
	goto exit
    endif
endif

if(! $?APPROVED) then
    echo ""
    # get a nice "starting" number for our archive
    set dvdnum = `ls -1 "${archive_disk}" | awk '{printf "%04d\n", substr($1,match($1,"[0-9]"))+1}' | sort -nr | head -1`
    if ("$dvdnum" == "") set dvdnum = 0001
    if ("$user_dvdnum" != "") then
	set dvdnum = $user_dvdnum
    endif

    # explain the filename patterns here
    foreach dvd ( ${tempfile}dvd????.list )
        echo ""
        echo "$media #$dvdnum"
        sumarize.com $sumopts $dvd
        set dvdnum = `echo $dvdnum | awk '{printf "%04d", $1+1}'`
    end
    
    set dvds = `ls -1 ${tempfile}dvd????.list | wc -l`
    set s = "s"
    if($dvds == 1) set s = ""
    set set = disk
    if ($copies > 1) set set = pair
    if ($copies > 2) set set = set
    set in = "Yes"
    echo "this will take $dvds ${media}-R ${set}${s}"
    echo "Okay? [${in}]"
    echo -n "-> "
    if (! $?AUTO) then
	set in = ( $< )
    endif
    if("$in" =~ [Nn]*) exit

    # only do this once
    set APPROVED
endif

# delete any files corresponding to already-done jobs (mainly for -realtime)
#if (-e "$last_done") then
#    # eliminate everything up to and including the last job we did
#    foreach dvd ( ${tempfile}dvd????.list )
#	rm -f $dvd
#	if("$dvd" == "$last_done") break
#    end
#endif

set dvdnum = 0
if("$user_dvdnum" != "") then
    set dvdnum = $user_dvdnum
endif

set lastdvd = `ls -1 ${tempfile}dvd????.list | tail -1`
set dvds = `ls -1 ${tempfile}dvd????.list | wc -l`
foreach dvd ( ${tempfile}dvd????.list )

    # sanity check: are there any files in this list?
    set files = `awk 'NF>0' "$dvd" | wc -l`
    if($files == 0) then
	echo "ERROR: disk #${dvdnum} is empty! "
	sleep 60
	continue
    endif

    if ($?REALTIME) then
	# wait until we have enough data
	if ($dvd == $lastdvd && ! -e $finish_signal_file) then
	    echo -n "waiting for more than $mediasize bytes to appear...                          \r"
	    set timeout = 30
	    if ($?DEBUG) set timeout = 1
	    while ( $timeout > 0 && ! -e $finish_signal_file )
		@ timeout = ( $timeout - 1 )
	        sleep 10
	    end
	    goto startover
	endif
	if (-e $finish_signal_file) then
	    # reset the "finished" flag
	    echo "user typed finished"
	    set lastarchive = `echo "puts [file mtime $finish_signal_file]]" | tclsh`
	    set finishdate = `echo "puts [clock format $lastarchive]" | tclsh`
	    echo "user typed finished at $finishdate"
	    set lastarchive = `echo "puts [clock seconds]" | tclsh`
	    set finishdate = `echo "puts [clock format $lastarchive]" | tclsh`
	    echo "found finish file at $finishdate"
	    rm -f $finish_signal_file > /dev/null
	endif
    endif

    # sanity check: are there any files in this list?
    # check if maybe someone deleted it?
    set files = `awk 'NF>0' "$dvd" | wc -l`
    if($files == 0) then
	echo "ERROR: disk #${dvdnum} is empty! "
	continue
    endif

increment:
    # make sure the disk is up
    while (! -w "${archive_disk}" )
	if (! $?waiting) echo waiting for "$archive_disk" to become ready
	set waiting
	sleep 10
    end
    unset waiting

    # make sure no two labels collide
    set dvdnum = `ls -1 "${archive_disk}" | awk '{printf "%04d\n", substr($1,match($1,"[0-9]"))+1}' | sort -nr | head -1`
    if ("$dvdnum" == "") set dvdnum = 0001
    if ("$user_dvdnum" != "") then
	set dvdnum = $user_dvdnum
    endif
   
retry:
    # sumarize each disk for the user's approval
    echo ""
    echo "$media #$dvdnum"
    sumarize.com $sumopts $dvd

    echo 'okay to write this '${media}'-R [Y/auto/skip]'
    echo -n "-> " 
    set answer = "Y"
    if(! $?AUTO) then
	set answer = ( $< )
	if ("$answer" == "") set answer = yes
    endif
    if("$answer" == "auto") then
	set AUTO
	set answer = yes
    endif
    if("$answer" !~ [Yy]*) then
	set lastdvd = "$dvd"
	continue
    endif

    # make sure the disk is up
    while (! -w "${archive_disk}" )
        if (! $?waiting) echo waiting for "$archive_disk" to become ready
        set waiting
        sleep 10
    end
    unset waiting

    # wait for the queue to clear a bit
#    while ( `ls -1 ${archive_disk} | egrep '\.nwp$|\.inp0$' | wc -l` > (3 - $copies) && $?NICE && ! ( "$dvd" == "$lastdvd") )
    while ( `ls -1 ${archive_disk} | egrep '\.nwp$|\.inp0$' | wc -l` > (5 - $copies) && $?NICE && ! ( "$dvd" == "$lastdvd") )
	if (! $?waiting) then
	    echo "waiting for robot to catch up...                      "
	    set waiting = `echo "puts [clock seconds]" | tclsh`
	else
	    set now = `echo "puts [clock seconds]" | tclsh`
	    if ($now > $waiting + 3600) then
		echo "it's been more than an hour! "
		set waiting = `echo "puts [clock seconds]" | tclsh`
		ls -lrt /archive | grep order | tail -1
		echo -n "it is now: "
		date
	    endif
	endif
	sleep 10
    end
    unset waiting

    # check if we have enough space for the iso image
    set spaceleft = 0
    while ( ! $spaceleft )
	# read megabytes available on disk
	set spaceleft = `df -k "${archive_disk}" | tail -1 | awk '{printf "%d", $(NF-2)/1024}'`

	# see if a DVD image will fit
	if ( `echo $mediasize $spaceleft | awk '{print ( 1.1*$1/1024/1024 > $2 && $2 > 1)}'` ) then
	    echo -n "waiting for space to be available on $archive_disk                               \r"
	    set spaceleft = 0
	    sleep 10
	endif
    end


    # hopefully, we have a pre-made label file
    if (! -e "${archive_disk}"/bmp_label.btw && -e ~jamesh/archiving/robot/bmp_label.btw ) then
	cp ~jamesh/archiving/robot/bmp_label.btw "${archive_disk}"/bmp_label.btw
    endif
    if (! -e "${archive_disk}"/bmp_label.btw) then
	# what do we do about this?
	set BAD = "no btw file found for rImage robot"
	goto exit
    endif

    # make sure the disk is up
    while (! -w "${archive_disk}" )
        if (! $?waiting) echo waiting for "$archive_disk" to become ready
        set waiting
        sleep 10
    end
    unset waiting

    # check again... make sure no two disks collide
    set test = `ls -1 "${archive_disk}" | awk '{printf "%04d\n", substr($1,match($1,"[0-9]"))+1}' | sort -nr | head -1`
    if("$test" != "$dvdnum" && "$user_dvdnum" == "") then
	echo "job #$dvdnum already taken, using job #$test"
	set dvdnum = $test
    endif

    if ("$user_dvdnum" != "") then
	# user specified a DVD serial number
	echo "re-using files for SN: $user_dvdnum"
	set dvdnum = $user_dvdnum
	if(! -e "${archive_disk}"/label_${dvdnum}.txt) then
	    echo "re-generating "${archive_disk}"/label_${dvdnum}.txt"
	    cat << EOF | awk '{printf "%s\r\n", $0}' >! "${archive_disk}"/label_${dvdnum}.txt
${remote_folder}label_${dvdnum}.bmp
EOF
	endif
    else
	# do this fast to "claim" the dvdnum slot number
	cp $dvd "${archive_disk}"/files_${dvdnum}.list
	echo "${remote_folder}label_${dvdnum}.bmp\r" >! "${archive_disk}"/label_${dvdnum}.txt
	cat << EOF | awk '{printf "%s\r\n", $0}' >! "${archive_disk}"/label_${dvdnum}.txt
${remote_folder}label_${dvdnum}.bmp
EOF

	if(-w /data/log/dvd_filesystem_sums.log) then
	    # make a record of the checsum of this disks's filesystem
	    set sum = `sort $dvd | md5sum`
	    # note the date when the order was submitted
	    set epoch = `echo 'puts [file mtime '"${archive_disk}"/files_${dvdnum}.list']' | tclsh`
	    echo "files_${dvdnum}.list $epoch $sum" |\
	    cat >> /data/log/dvd_filesystem_sums.log
	endif

	# worthwhile now to record the file checksums?
	cat $dvd |\
	 awk '{system("md5sum "$0)}' |\
	 awk '{file=substr($0,index($0,$2));\
	     print "set epoch [file mtime \""file"\"]";\
	     print "set year [clock format $epoch -format \"%Y\"]";\
	     print "puts \"$epoch $year "$1,file"\""}' |\
	tclsh >&! "${archive_disk}"/checksums_${dvdnum}.txt

	# wait a bit
	#sleep 5

	# check again... make sure nobody else took it
	set test = `diff $dvd "${archive_disk}"/files_${dvdnum}.list | wc -l`
	if("$test" != 0) then
	    set test = `ls -1 "${archive_disk}" | awk '{printf "%04d\n", substr($1,match($1,"[0-9]"))+1}' | sort -nr | head -1`
	    echo "WARNING: collided with another process on #$dvdnum"
	    goto startover
	endif
    endif

    # pass user-specified title through appropriately
    set title_option = ( "" "" )
    foreach title ( 1 2 )
	if("$titles[$title]" != "") then
	    set title_option[$title] = "-title"
	endif
    end
    set note_option = ""
    if("$note" != "") set note_option = "-note"

    # sanity check: are there any files in this list?
    set files = `awk 'NF>0' "${archive_disk}"/files_${dvdnum}.list | wc -l`
    if($files == 0) then
	echo "ERROR: disk #${dvdnum} is empty! "
	if("$user_dvdnum" != "") goto exit
	if(-e "${archive_disk}"/files_${dvdnum}.list ) goto startover
	continue
    endif

    if("$user_dvdnum" != "" && "$user_label" == "" && -e "${archive_disk}"/label_${dvdnum}.bmp) then
	echo "re-using label file for SN: $dvdnum"
	set user_label = "${archive_disk}"/label_${dvdnum}.bmp
	set titles = "repeat"
    endif

    if(! -e "$user_label") then
        # generate the label in BMP format
        echo -n "generating label for job #$dvdnum"
        rm -f label.bmp >& /dev/null
        rm -f ${tempfile}label.bmp >& /dev/null
        autolabel.com bmp ${tempfile}label.bmp $sumopts $title_option[1] "$titles[1]" $title_option[2] "$titles[2]" $note_option "$note" -serial $dvdnum  "${archive_disk}"/files_${dvdnum}.list >! ${tempfile}autolabel.log &
        echo ""
    else
	echo "using $user_label as the disk label"
    endif
    # we will get back to this job after the "wait" command below

    set orderid = ( $titles )
    if ("$titles" == " ") set orderid = `titles.com $dvd`
    set orderid = `echo "$dvdnum $orderid" | awk '$3!=""{$3="_"$3} {gsub("[^a-zA-Z0-9]","_");print substr($1"_"$2 $3,1,20)}'`
    set orderid = `echo "$orderid" | awk '{gsub("_$","");gsub("__","_");print}'`

diskimage:
    if(-e "$user_diskimage") then
	# use a pre-made disk image
	set diskimage_dir = `dirname $user_diskimage`
	set diskimage_dir = `cd $diskimage_dir ; pwd`
	set archive_disk_dir = `cd ${archive_disk} ; pwd`
	if("$diskimage_dir" == "$archive_disk_dir") then
	    # file is somewhere the robot can see it.
	    set user_diskimage_name = `basename $user_diskimage`
	else
	    echo "copying $user_diskimage to ${archive_disk}/image_${dvdnum}.iso"
	    cp "$user_diskimage" ${archive_disk}/image_${dvdnum}.iso
	endif
	goto wait_for_label
    endif
    # make the iso image
#    echo "making disk $dvdnum image..."
    set mkisofserror = 0
    # dry run...
    cat "${archive_disk}"/files_${dvdnum}.list |\
    awk '/^\.\//{$0=substr($0,3)} {gsub("[=]","\\="); print $0"="$0}' |\
    mkisofs -print-size -r $dashJ -D -graft-points -path-list - >&! ${tempfile}mkisofs.log
    if($status) then
	echo "ERROR calculating disk image size"
	set mkisofserror = 1
    endif
    set imagesize = `awk '/Total extents scheduled/{print $NF}' ${tempfile}mkisofs.log`

    if("$smbclient" == "") then
	# use filesystem to transfer the image
	cat "${archive_disk}"/files_${dvdnum}.list |\
	awk '/^\.\//{$0=substr($0,3)} {gsub("[=]","\\="); print $0"="$0}' |\
	sh -c "mkisofs -r $dashJ -D -graft-points -o "${archive_disk}"/image_${dvdnum}.iso -path-list - " |&\
        tee ${tempfile}mkisofs.log |\
	awk -v dvdnum=$dvdnum '/estimate finish/{\
 	    printf " preparing to write disk #%s  %s\r",dvdnum,$1}'
	if($status) then
	    echo "ERROR preparing disk image."
	    set mkisofserror = 1
	endif
    else
	# use smbclient to transfer the image	
	cat "$dvd" |\
	awk '/^\.\//{$0=substr($0,3)} {gsub("[=]","\\="); print $0"="$0}' |\
	( mkisofs -r $dashJ -D -graft-points -path-list - | $smbclient -c "put - image_${dvdnum}.iso" ) |&\
        tee ${tempfile}mkisofs.log |\
	awk -v dvdnum=$dvdnum '/estimate finish/{\
 	    printf " preparing to write disk #%s  %s\r",dvdnum,$1}'
	if($status) then
	    # something went wrong in the transfer...
	    set mkisofserror = 1
	endif
    endif
    grep -i error ${tempfile}mkisofs.log
    if(! $status) then
	echo "mkisofs gave an error? "
    endif
    set writtensize = `tail ${tempfile}mkisofs.log | awk '/extents written/{print $1}'`
    if("$writtensize" != "$imagesize") then
	echo "ERROR: disk image is not the right size"
	set mkisofserror = 1
    endif
    if ("$smbclient" == "") then
	# ${archive_disk} cannot be accessed via SMB, must be a filesystem
        set test = `ls -1 "${archive_disk}" | egrep "^image_${dvdnum}.iso" | wc -l`
        if("$test" == "1") then
#	    set test = `file "${archive_disk}"/image_${dvdnum}.iso`
#	    if("$test" !~ *ISO9660*) then
#	         echo "ERROR: disk image is corrupt"
#	         set mkisofserror = 1
#	     endif
	else
	    # the image file does not exist! 
	    echo "ERROR: image file does not exist"
	    set mkisofserror = 1
        endif
    else
        $smbclient -c "dir image_${dvdnum}.iso" | grep "File not found" > /dev/null
        if( ! $status ) set mkisofserror = 1
    endif
    if( $mkisofserror ) then
	# something went wrong
	cat ${tempfile}mkisofs.log
	echo "ERROR generating disk image. "
	if ("$smbclient" != "") then
	    $smbclient -c "rm image_${dvdnum}.iso" > /dev/null
	endif
	if (-e "${archive_disk}"/image_${dvdnum}.iso) then
            rm -f "${archive_disk}"/image_${dvdnum}.iso > /dev/null
	endif
	echo "Sorry! "
	set test = `awk '/Error/ && /Joliet/' ${tempfile}mkisofs.log | wc -l`
	if("$test") then
	    echo "abandoning Windows-compatible format"
	    set dashJ = ""
	    goto diskimage
	endif
	rm -f ${tempfile}mkisofs.log >& /dev/null
        # try again?
	wait
        goto startover
    endif
    set dashJ = "-J"
    rm -f ${tempfile}mkisofs.log >& /dev/null
    # finish off message
    echo " preparing to write disk #$dvdnum  100%     "

wait_for_label:
    # wait for the label to finish (it was generating in the background)
    wait
    if (-e "$user_label") then
	echo "using $user_label as the disk label"
	if ("$user_label" != "${archive_disk}"/label_${dvdnum}.bmp) then
	    cp -f "$user_label" "${archive_disk}"/label_${dvdnum}.bmp >& /dev/null
	endif
    else
        set bmpfile = `awk '/bmp is ready/{print $1;exit}' ${tempfile}autolabel.log`
	rm -f ${tempfile}autolabel.log >& /dev/null
	if(! -e "$bmpfile") then
	    cat ${tempfile}autolabel.log
	    echo "ERROR generating disk $dvdnum label. "
	    echo "Sorry! "
	    rm -f ${tempfile}autolabel.log
	    # try again?
	    goto startover
	endif
	rm -f ${tempfile}autolabel.log
	rm -f "${archive_disk}"/label_${dvdnum}.bmp > /dev/null
	mv ${bmpfile} "${archive_disk}"/label_${dvdnum}.bmp >& /dev/null
	rm -f ${bmpfile} > /dev/null
    endif

    # wait a tic for SMBFS to catch up?
    sleep 5

    # now write the order file
    echo "submitting to robot as:   $orderid"
    set diskimage_name = "${remote_folder}image_${dvdnum}.iso"
    set delete = "_DELETE"
    if( $?user_diskimage_name ) then
	set diskimage_name = "${remote_folder}$user_diskimage_name"
	set delete = ""
    endif
    cat << EOF | awk '{printf "%s\r\n", $0}' >! "${archive_disk}"/order_${dvdnum}.nwp
order_id=$orderid
priority=$priority
media=DVDR
filetype=IMAGE$delete
file=$diskimage_name
copies=$copies
label=${remote_folder}bmp_label.btw
merge=${remote_folder}label_${dvdnum}.txt
labtype=EDITOR_FMT
fixate=NOAPPEND
EOF

    # bump us up if we are in a hurry
    if($priority < 4) then
	foreach order ( ${archive_disk}"/order_* )
	    if("$order" != "${archive_disk}/order_${dvdnum}.nwp") then
		echo "bumping $order"
		touch $order
	    endif
	end
    endif

    set backlog = `ls -1 $archive_disk | awk -v dvdnum=$dvdnum '/^order_/ && ! /don0$/{++n} $1~"^order_"dvdnum{us=n} END{print us"/"n}'`
    if ("$backlog" == "1/1") set backlog = first
    echo "disk #$dvdnum is $backlog in the queue"

    # allow the Network Publisher to change/delete these files
    chmod a+rw "${archive_disk}"/order_${dvdnum}.nwp >& /dev/null
    chmod a+rw "${archive_disk}"/image_${dvdnum}.iso >& /dev/null
    chmod a+rw "${archive_disk}"/label_${dvdnum}.txt >& /dev/null
    chmod a+rw "${archive_disk}"/label_${dvdnum}.bmp >& /dev/null
    chmod a+rw "${archive_disk}"/files_${dvdnum}.list >& /dev/null

    # remember where we are (for -realtime mode)
    if(-e ${tempfile}dated) then
	# use this in case somebody touched the last file while we were busy archiving it
	awk '{print "DVD|"$0}' $dvd |\
	tac - ${tempfile}dated |\
	awk '{file=substr($0,index($0,"|")+1)} \
	    /^DVD/{dvd[file]=1;next} NF>6 && dvd[file]{print $1,"|"file;exit}' |\
	cat >! ${tempfile}lastfile
	set lastfile = `cat ${tempfile}lastfile`
	rm -f ${tempfile}lastfile >& /dev/null
	if($#lastfile < 2) then
	    # do an exhaustive search
	    awk '{print "DVD|"$0}' $dvd |\
	    cat - ${tempfile}dated |\
	    awk '{file=substr($0,index($0,"|")+1)} \
	        /^DVD/{dvd[file]=1;next} NF>8 && dvd[file]{print $1,"|"file}' |\
	    sort -n |\
	    tail -1 >! ${tempfile}lastfile
	    set lastfile = `cat ${tempfile}lastfile`
	    rm -f ${tempfile}lastfile >& /dev/null
	endif
    else
	# somebody deleted out tempfile?
	echo "WARNING: ${tempfile}dated deleted "
	set lastfile = `awk '{print "puts \"[file mtime "$0"] |"$0"\""}' $dvd | tclsh | sort -n | tail -1 | awk 'NF>1'`
    endif
    if("$lastfile" == "") then
	set BAD = "lost my place in time"
	goto exit
    endif
    set lastarchive = "$lastfile[1]"
    if("$note" != "" && -w /data/log) then
	# create a record of the last file we backed up in this "stream"
	set stream = `echo "$note $dirs" | awk -F "[ \/]" '$NF==""{$NF=$(NF-1)} {print tolower($1)"_"$NF}'`
	rm -f /data/log/last_${stream}_backup.txt >& /dev/null
	echo "$lastfile" >! /data/log/last_${stream}_backup.txt
	set last_file = `awk '$2~/^|/{print substr($0,index($0,"|")+1)}' /data/log/last_${stream}_backup.txt`
	touch --reference "$last_file" /data/log/last_${stream}_backup.txt
    endif
end

if ($?REALTIME) goto startover

goto exit


exit:
if($?BAD) then
    echo ""
    echo ""
    echo "ERROR: $BAD "
    exit 9
endif
exit

finishup:
onintr
unset REALTIME
goto startover


exit


Setup:
#############################################################
# platform-specific issues
set test = `echo "123asdf" | awk '{print $1+0}'`
if("$test" != "123") then
    alias awk nawk
    set test = `echo "123asdf" | awk '{print $1+0}'`
    if("$test" != "123") alias awk gawk
    set test = `echo "123asdf" | awk '{print $1+0}'`
    if("$test" != "123") then
	set BAD = "awk no good"
	goto exit
    endif
endif

set test = `echo -n "\r" |& od -c | wc -w`
if("$test" != 3) then
    alias echo /bin/echo
    set test = `echo -n "\r" |& od -c | wc -w`
    if("$test" != 3) alias echo '/bin/echo -e'
    set test = `echo -n "\r" |& od -c | wc -w`
    if("$test" != 3) alias echo /usr/bin/echo
    set test = `echo -n "\r" |& od -c | wc -w`
    if("$test" != 3) alias echo '/bin/echo \!* | awk '\''/^-n/{printf "%s", substr($0,4) ; exit} {print}'\'
    set test = `echo -n "\r" |& od -c | wc -w`
    if("$test" != 3) then
	set BAD = "echo command is broken"
	goto exit
    endif
endif

set test = `echo "string is digit 1" | /usr/bin/tclsh |& wc -l`
if($test != 0) then
    set test = `echo "string is digit 1" | tclsh |& wc -l`
    if($test != 0) alias tclsh tclsh8.2
    set test = `echo "string is digit 1" | tclsh |& wc -l`
    if($test != 0) alias tclsh tclsh8.3
    set test = `echo "string is digit 1" | tclsh |& wc -l`
    if($test != 0) then
        setenv LD_LIBRARY_PATH ~jamesh
        alias tclsh ~jamesh/tclsh8.2
	echo "" | tclsh >& /dev/null
	if("$status") then
	    set BAD = "tclsh not available"
	    goto exit
	endif
    endif
    alias sock_exchange.tcl "tclsh `which sock_exchange.tcl` \!*"
endif
######################################################################

set priority = 4
#set NICE
set media = DVD
set copies = 1
set splitusers = 0
set user_label = ""
set user_dvdnum = ""
#set noglob
set dirs
set lists
set onlyfind = "default   "
set not      = "default   "
set sumopts  = ""

set dummy
set dashJ = "-J"

set titles = ( "" "" )
set note = ""
set lastarchive = 0
set lastdvd = ""
set last_done = ""

set finish_signal_file = " default "

set user_diskimage = ""
set archive_disk = "/archive"
#set remote_folder = "C:\Orders\"
set remote_folder = "O:\Orders\"
#set remote_folder = ""
set tempfile = /tmp/tempfile_rimage
if($?USER) then
    if(-e /tmp/${USER}) set tempfile = /tmp/${USER}/tempfile_rimage
endif
# clear any old temp files
#rm -f ${tempfile}*
set tempfile = ${tempfile}$$
rm -f ${tempfile}* >& /dev/null


if ($?DEBUG) then
    echo "DEBUGGING"
    set tempfile = /tmp/archive/tmp/tempfile
    set archive_disk = "/tmp/archive"
    mkdir -p $archive_disk
endif

# this is how you should mount the windows share
# smbmount //bl831archive.dhcp.lbl.gov/archive /archive -o "dmask=777,fmask=777,username=Administrator%12345"
# smbmount //archive/orders /archive -o "lfs,dmask=777,fmask=777,username=Administrator%"
# or in fstab:
# //archive/orders /archive smbfs lfs,dmask=777,fmask=777,username=Administrator%12345 0 0


set path = ( `dirname $0` $path )

if($#argv == 0) then
    set BAD
    goto Help
endif

set i = 0
while($i < $#argv)

    @ i = ( $i + 1 )
    @ j = ( $i + 1 )
    set arg = "$argv[$i]"
    set ARG = `echo $arg | awk '{print toupper($1)}'`

    if("$arg" =~ *@*) then
	set alert_email = "$arg"
	echo "will alert $alert_email when each DVD is ready."
    endif

    if(-e "$arg") then
	if("$arg" =~ *.bmp) then
	    set user_label = $arg
	    echo "using $user_label as the disk label"
	    continue
	endif
	test -d $arg
	if((! $status) && ("$arg" !~ */)) then
	    set arg = "${arg}/"
	endif
	test -d $arg
	if($status) then
	    # not a directory
	    if("$arg" =~ *.iso) then
		set user_diskimage = "$arg"
		continue
	    endif
	    set lists = ( $lists $arg )
	    continue
	endif
	if(! $?NO) then
	    set dirs = ( $dirs $arg )
	else
	    set not  = ( $not $arg )
	endif
	continue
    endif

    # re-submit an old order under the same ID
    if("$arg" == "-repeat" && $j <= $#argv) then
	set test = `echo "$argv[$j]" | awk '{print $1}'`
	if(-e "${archive_disk}"/files_${test}.list) then
	    set lists = "${archive_disk}"/files_${test}.list
	    set user_dvdnum = $test
	    # make sure it all goes on one disk?
	    set mediasize = 5000000000
	    echo "re-submitting job SN: $user_dvdnum"
	else
	    echo "WARNING: $arg[$j] is not a valid job ID"
	endif
	unset noglob
	# skip two this time
        @ i = ( $i + 1 )
	continue
    endif

    # only back up certain patterns
    if("$arg" == "-only" && $j <= $#argv) then
	set noglob
	if("$onlyfind" == "default   ") set onlyfind = ""
	set onlyfind = ( $onlyfind $argv[$j] )
	unset noglob
	# skip two this time
        @ i = ( $i + 1 )
	continue
    endif
    if("$arg" == "-regardless" || "$arg" == "-allfiles") then
	set sumopts = "-nodatasets"
	set onlyfind = ""
	set not = "\n "
	continue
    endif
    if("$arg" == "-nice") then
	set NICE
	continue
    endif
    # only back up new files
    if("$arg" == "-since" && $j <= $#argv) then
	set file = "$argv[$j]"
	set epoch = `echo 'puts [clock scan "'$file'"]' | tclsh |& cat`
	if ("$epoch" =~ [1-9]*) then
	    set lastarchive = $epoch
	    set date = `echo "puts [clock format $epoch]" | tclsh`
	    echo "backing up files since $date"
	endif
	if ("$file" == "now") then
	    set lastarchive = `echo "puts [clock seconds]" | tclsh`
	    echo "backing up new files only"
	endif
	if ("$file" == "hour") then
	    set lastarchive = `echo "puts [expr [clock seconds] - 3600]" | tclsh`
	    echo "backing up everything from one hour ago"
	endif
	if("$file" == "day") then
	    set lastarchive = `echo "puts [expr [clock seconds] - 24*3600]" | tclsh`
	    echo "backing up everything since yesterday"
	endif
	if ("$file" == "last") then
	    set file = `ls -1rt ${archive_disk} | grep files | tail -1`
	    set file = ${archive_disk}/$file
	    set file = `awk '{print "puts \"[file mtime "$1"] "$1"\""}' $file | tclsh | sort -n | tail -1 | awk '{print $NF}'`
	endif
	if (-e "$file" ) then
	    # must be a file
	    echo "last backed-up file was $file"
	    set lastarchive = `echo "puts [file mtime $file]" | tclsh`
	endif
	# skip two this time
        @ i = ( $i + 1 )
	continue
    endif
    # set the robot priority
    if("$arg" == "-priority" && $j <= $#argv) then
	set num = `echo $argv[$j] | awk 'int($1+0)==$1{print $1}'`
	if ("$num" != "") then
	    set priority = $num
	endif
	# skip two this time
        @ i = ( $i + 1 )
	continue
    endif
    # select multiple copies
    if("$arg" == "-copies" && $j <= $#argv) then
	set num = `echo $argv[$j] | awk 'int($1+0)==$1{print $1}'`
	if ("$num" != "") then
	    set copies = $num
	endif
	# skip two this time
        @ i = ( $i + 1 )
	continue
    endif
    if("$arg" == "-realtime") then
	set REALTIME
	set AUTO
	if (-e $finish_signal_file) then
	    # reset the "finished" flag
	    rm -f $finish_signal_file > /dev/null
	endif
	onintr finishup
	continue
    endif
    if("$arg" =~ *splituse*) then
	# don't mix users on the same DVD
	set splitusers = 1
	if($?NO) set splitusers = 0
	continue
    endif
    # user specificed "finish" signal
    if("$arg" == "-finish" && $j <= $#argv) then
	set finish_signal_file = "$argv[$j]"
	# skip two this time
        @ i = ( $i + 1 )
	continue
    endif
    if("$arg" =~ *dpi) then
	set dpi = `echo $arg | awk '{print $1+0}'`
	continue
    endif
    if("$arg" == "CD") then
	set media = "CD"
	continue
    endif
    if("$arg" == "DVD") then
	set media = "DVD"
	continue
    endif
    if("$arg" == "-auto") then
	set AUTO
	continue
    endif
    # user-selectable title
    if("$arg" == "-title" && $j <= $#argv) then
	if("$titles[1]" == "") then
	    set titles[1] = "$argv[$j]"
	else
	    set titles[2] = "$argv[$j]"
	endif
	# skip two this time
        @ i = ( $i + 1 )
	continue
    endif
    # user-selectable note (to appear next to serial number)
    if("$arg" == "-note" && $j <= $#argv) then
        set note = "$argv[$j]"
        # skip two this time
        @ i = ( $i + 1 )
        continue
    endif

    # user-selectable media size
    if("$arg" == "-mediasize" && $j <= $#argv) then
	set size = `echo $argv[$j] | awk '/G/{$1*=1e9} /M/{$1*=1e6} /[kK]/{$1*=1e3} {printf "%.0f",$1}'`
	set test = `echo $size | awk '{print ($1>300e6)}'`
	if("$test") then
            set mediasize = "$size"
	else
	    echo "WARNING: media size of $size bytes is not realistic"
	    echo "WARNING: media size must be given in bytes"
	endif
        # skip two this time
        @ i = ( $i + 1 )
        continue
    endif


    if($?NO) then
	if("$not" == "default   ") set not = ""
	set not = ( $not $arg )
    endif

    if(("$ARG" =~ NO*) || ("$ARG" =~ "-NO"*)) then
	set NO
	continue
    endif
    unset NO
end

touch ${tempfile}filenames >& /dev/null
if($status && $?CCP4_SCR) then
    set tempfile = ${CCP4_SCR}/tempfile
endif
touch ${tempfile}filenames >& /dev/null
if($status && -e /tmp/${USER}) then
    set tempfile = /tmp/${USER}/tempfile
endif
touch ${tempfile}filenames >& /dev/null
if ($status ) set tempfile = /tmp/tempfile


if ("$finish_signal_file" == " default ") then
    set finish_signal_file = "/data/log/finished_run"
    if($#dirs == 1) then
	# there is only one monitor directory, let's watch it
	set finish_signal_file = "${dirs}/finished_run"
    endif
endif

# default to exclude common undesirables
if("$not" == "default   ") set not = "\.imx_ \.dkx_ mtz2various_TMP \.adxv_beam_center tempfile sent_home _temp align MiscScans scan\.inf scan\.efs on\.jpg beam\.jpg diff\.jpg in\.jpg diff.\.jpg sqr.\.jpg thresh\.jpg blob\.jpg over\.jpg marked\.jpg nih_\.jpg"

# create exclusion definitions
set not = `echo " $not " | awk 'BEGIN{RS=" "} NF==1{printf "%s|", $1}'`
set not = `echo $not | awk '{print substr($1,1,length($1)-1)}'`
if ("$not" == "") set not = "\n "

# default to just images 
if("$onlyfind" == "default   ") set onlyfind = "jpg jpeg img scan"

set onlyfind = `echo " $onlyfind " | awk 'BEGIN{RS=" "} NF==1{printf "%s|", $1}'`
set onlyfind = `echo "$onlyfind" | awk '{print substr($1,1,length($1)-1)}'`


if(! $?mediasize) then
    set mediasize = 4600000000
    set mediasize = 4000000000
    if ("$media" == "CD") set mediasize = 683483136
endif

# go into auto mode if user is not interactive
test -t 1
if($status) set AUTO

# smbclient command to use to access the DVD robot computer
set smbclient = "smbclient //archive/orders -U Administrator -N"

set computer = `df $archive_disk | tail -1 | awk '/^\/\//{print $1}' | awk -F "[/]" '{print $3}'`

# explicitly set up smbclient to mimic the smbmount mountpoint (if it works)
if("$computer" != "") then
    set sharename = `df $archive_disk | tail -1 | awk '{print $1}'`
    set username = `grep $archive_disk /etc/fstab | awk '{print $4}' | awk -F "[=]" 'BEGIN{RS=","} /username/{print $2}'`
    if ("$username" != "" && "$sharename" != "") then
	set smbclient = "smbclient $sharename -U $username -N"
    endif
else
    # $archive_disk must be a local filesystem
    set smbclient = ""
endif
# make sure we know the share name
set sharename = `echo "$smbclient" | awk '{print $2}'`

# this is how you should mount the windows share
# smbmount //archive/orders /archive -o "lfs,dmask=777,fmask=777,username=Administrator%password"
# or in fstab:
# //archive/orders /archive smbfs lfs,dmask=777,fmask=777,username=Administrator%12345 0 0


goto Return_from_Setup
end
end
end
end
end
end

# benchmarks:
mkisofs on local raid:  5:47.58
mkisofs onto grph2:    10:00.52
cp image onto grph2:    9:53.20
cp image onto grph1:    3:59.33
cp image onto grph3:    5:40.55
cp image on graphics1:  3:51.57
cp into null on grph1:  3:30.42
cp into null on servr:  3:17.52
cp into null on data4:  0:36.45
cp into null on grph3:  1:41.82