#! /bin/csh -f
#
#	Utility for burning x-ray data sets onto DVD-R
#
#
if($#argv == 0) then
    set BAD
    goto Help
endif

goto Setup
############################################################################
Help:
cat << EOF

usage: backup.com directory [otherdirectory] -not junk -only good -since timefile -nolabel -noimage

where:
directory  - is the directory you want to back up onto a DVD-R
junk       - are files/subdirectories you want to leave out
good	   - is a string that the back-ed up filenames must contain
timefile   - no files as old or older than timefile will be included
-nolabel   - don't generate labels
-noimage   - don't generate an intermediate disk image (fast and dangerous)
-noverify  - don't check the written files match the originals

example:

backup.com /data/mcfuser/whatever -not '_0_'
will back up everything but screening shots

backup.com /data/mcfuser/whatever -only img -only scan -not '_0_'
will back up every *.img or *.scan file but no screening shots

backup.com /data/mcfuser -since /data/mcfuser/blah/firstrun_1_100.img
will back up everything collected AFTER firstrun_1_100.img

EOF

if($?BAD) exit 9
############################################################################
Return_from_Setup:

echo "media:     ${media}-R ($mediasize bytes)"
echo "recorders: $recorders"
echo "drives:    $drives"
echo "program:   $dvdrecord"
echo "cache:     ${localdisk}"
echo "temp:      ${tempfile}*"
echo "image:     $?USE_IMAGE"
echo "verify:    $?VERIFY"
echo "dirs:      $dirs"
echo "onlyfind:  $onlyfind"
echo "not:       $not"

startover:
echo "examining files..."
rm -f ${tempfile}* >& /dev/null
echo -n "" >! ${tempfile}filenames
if( "$dirs" != "") then
    find $dirs -type f -print >> ${tempfile}filenames
endif
if( "$lists" != "") then
    awk '{print $NF}' $lists >> ${tempfile}filenames
endif

# filter and get dates
# find $dirs -type f -printf "%T@ %s %t %p\n" | sort -n
cat ${tempfile}filenames |\
egrep "$onlyfind" |\
egrep -v "$not" |\
awk '{print "puts \"[file mtime", $1 "] [file size", $1 "] [clock format [file mtime", $1 "]] "$1"\""}' |\
tclsh |\
sort -n |\
cat >! ${tempfile}dated

cat ${tempfile}dated |\
awk -v lastarchive=$lastarchive '$1>lastarchive' |\
awk -v mediasize=$mediasize '{pattern=$NF;n=""} \
  /[0-9][0-9][0-9]\.img$/{n=split($NF,a,"\."); ext=a[n];\
      pattern=substr($NF,1,length($NF)-length(ext)-4) "%03d." ext;\
      n=substr($NF,length($NF)-length(ext)-3)+0;}\
  {++files[pattern];file[pattern,files[pattern]]=$1" "$2" "$NF;\
      size[pattern]+=int($2/2048+2)*2048;date[pattern]=$1;num[pattern]=num[pattern]" "n;\
      if(low[pattern]==""){low[pattern]=n}}\
  n<low[pattern]{low[pattern]=n}  n>=high[pattern]{high[pattern]=n}\
    END{for(pattern in size) {\
   if(size[pattern]>mediasize){for(i=1;i<=files[pattern];++i)print file[pattern,i]; continue}\
   printf "%10s %10.0f %-70s %3s\n", date[pattern], size[pattern], pattern, num[pattern]}}' |\
sort -n >! ${tempfile}groups.txt

rm -f ${tempfile}dvd????.list >& /dev/null
cat ${tempfile}groups.txt |\
awk -v mediasize=$mediasize -v tempfile=$tempfile '{size[dvd]+=$2}\
  NR==1 || size[dvd]>=mediasize{size[dvd]-=$2;\
    printf "%s %d%% %s ...\n", dvd, size[dvd]/mediasize*100, $3;\
    dvd=sprintf("%04d",dvd+1);size[dvd]=$2}\
  NF==3{print $3 >> tempfile"dvd"dvd".list"}\
   NF>3{for(i=4;i<=NF;++i) printf $3"\n",$i >> tempfile"dvd"dvd".list"} \
END{printf "%s %d%% %s ...\n", dvd, size[dvd]/mediasize*100, $3}' |\
awk 'NR!=1'

if(! $?APPROVED) then
    # explain the filename patterns here
    foreach dvd ( ${tempfile}dvd????.list )
        set dvdnum = `basename $dvd | awk '{print substr($1,match($1,"[0-9]"))}' | awk '{print substr($1,1,match($1,"[^0-9\/]")-1)}'`
        echo ""
        echo "$media #$dvdnum"
        if (! $?FAST) sumarize.com $dvd
    end
    
    set dvds = `ls -1 ${tempfile}dvd????.list | wc -l`
    set s = "s"
    if($dvds == 1) set s = ""
    echo "this will take $dvds ${media}-R pair${s}"
    echo "Okay? [Yes]"
    echo -n "-> "
    if (! $?AUTO) then
	set in = ( $< )
    endif
    if("$in" =~ [Nn]*) exit
    set APPROVED
endif

# print out nice labels
#if(! $?NOLABEL) then
#    # first, kill any old label generation proceses
#    set others = `ps -fea | awk '$NF~/^autolabel/{print $3}'`
#    if("$others" != "") then
#	( kill -9 $others )
#    endif
#endif

set dvdnum = 0
set last_dvdnum = 0
set IMAGE = ""
foreach drive ( $drives )
    ( eject $drive & ) >& /dev/null
end
echo 'please insert a blank pair of '${media}'-Rs and press "Enter"'
set temp = ""
if (! $?AUTO) then
    set temp = ( $< )
else
    annoy.wish "please insert a blank pair of ${media}-Rs in `hostname -s`"
    if ($status) exit 9
endif
if("$temp" == "skip") then
    set SKIP
endif
set disk = blank
wait

foreach drive ( $drives )
    ( eject -t $drive & ) >& /dev/null
end

set second_to_last = `ls -1 ${tempfile}dvd????.list | tail -2 | head -1`
foreach dvd ( ${tempfile}dvd????.list )

if ($?REALTIME) then
    # fast-forward to the "current" dvd number
    if("$REALTIME" != "" && "$REALTIME" != $dvd) continue
endif

increment:
    set dvdnum = `basename $dvd | awk '{print substr($1,match($1,"[0-9]"))}' | awk '{print substr($1,1,match($1,"[^0-9\/]")-1)}'`
    
if ($?REALTIME) set REALTIME = "$dvd"

image:
    if ($?SKIP) goto try_again
    ls ${localdisk}/dvd.image >& /dev/null
    if ( ! $status && "$IMAGE" == "$dvdnum") goto burn
    if(! $?NOLABEL) then
	echo "printing label $dvdnum in the background..."
	( autolabel.com $media double $dvd >! ${tempfile}autolabel.log & )
    endif
    if ( ! $?USE_IMAGE ) then
	set IMAGE = $dvdnum
	goto burn
    endif
    echo "making disk $dvdnum image..."
    rm -f ${localdisk}/dvd.image >& /dev/null
    awk '{gsub("[=]","\\="); print $1"="$1}' $dvd |\
    sh -c "mkisofs -r -J -D -graft-points -o ${localdisk}/dvd.image -path-list - "
    set IMAGE = $dvdnum
    if("$disk" == "blank") goto burn

try_again:
    if(! $?SKIP) then
    	if("$last_dvdnum" != "") then
	    echo 'dvd# '$last_dvdnum' is ready'
	    if ($?alert_email) then
        	echo "alerting $alert_email"
        	echo "DVD $last_dvdnum is ready" | mail -s "DVD $last_dvdnum is ready" "$alert_email" 
	    endif
	endif
	foreach drive ( $drives )
	    ( eject $drive & ) >& /dev/null
	end
    endif
    unset SKIP
    echo 'please insert a blank pair of '${media}'-Rs for '$dvdnum' and press "Enter"'
    set temp = ""
    if(! $?AUTO) then
	set temp = ( $< )
    else
	annoy.wish "please insert a blank pair of ${media}-Rs in `hostname -s`"
	if($status) exit 9
    endif
    if("$temp" == "skip") then
	set SKIP
	continue
    endif
    if(! $?SKIP) then
	foreach drive ( $drives )
	    ( eject -t $drive & ) >& /dev/null
	end
	set disk = blank
    endif

burn:
    wait
    if("$disk" != "blank") goto try_again
    if(! $?USE_IMAGE ) then
	# burn directly to the drives!
	set extents = `awk '{gsub("[=]","\\="); print $1"="$1}' $dvd | sh -c "mkisofs -r -J -D -graft-points -path-list - -q -print-size"`
	
	set n = 0
	foreach recorder ( $recorders )
	    @ n = ( $n + 1)
	    awk '{gsub("[=]","\\="); print $1"="$1}' $dvd |\
	    mkisofs -r -J -D -graft-points -path-list - |\
	    $dvdrecord dev=$recorder -v -tsize ${extents}s -data - ; echo $status >! ${tempfile}$n &
	end
	wait
	goto checkstatus
    endif

    if("$IMAGE" != "$dvdnum") then
	echo "disk image is not up to date: IMAGE=$IMAGE dvdnum=$dvdnum "
	goto image
    endif
    ls ${localdisk}/dvd.image > /dev/null
    if( $status ) then
	echo "disk image does not exist! "
	goto image
    endif

    # burn it direct
    set n = 0
    foreach recorder ( $recorders )
	@ n = ( $n + 1)
	$dvdrecord dev=$recorder -v -data ${localdisk}/dvd.image |& tee ${tempfile}$n ; echo $status >! ${tempfile}$n &
	sleep 10
    end
    wait
    goto checkstatus

checkstatus:
    wait
    # now check and see if the burn worked
    set n = 0
    foreach drive ( $drives )
	@ n = ( $n + 1)
	set Status = `cat ${tempfile}$n`
        rm -f ${tempfile}$n >& /dev/null
	if("$Status" != "0") then
	    echo "disk write in recorder $recorder did not work! "
#	    ( eject $drive & ) >& /dev/null
	endif
    end


verify:
    if (! $?VERIFY) goto eject

    # get the filesystem mounted up
    foreach drive ( $drives )
	eject $drive
	sleep 1
	mount $drive
	if ($status) then
	    # try one more time
	    eject $drive
	    sleep 10
	    mount $drive
	endif
    end

    set n = 0
    foreach drive ( $drives )
	@ n = ( $n + 1)
	verify.com $drive ; echo $status >! ${tempfile}$n &
    end
    wait
    set n = 0
    foreach drive ( $drives )
	@ n = ( $n + 1)
	set Status = `cat ${tempfile}$n`
        rm -f ${tempfile}$n >& /dev/null
	if("$Status" != "0") then
	    echo "disk in $drive failed verification! "
	    annoy.wish "ERROR: DVD in $drive on `hostname -s` is BAD! "
	    ( eject $drive & ) >& /dev/null
	endif
    end

eject:
    # eject the disks wether they were good or not
    foreach drive ( $drives )
	( eject $drive & ) >& /dev/null
    end
    set disk = $dvdnum

    if($Status != 0) then
	echo "write failed!  Sorry."
	wait
	goto try_again
    endif
    
    # the write for "$dvdnum" succeeded
    set last_dvdnum = "$dvdnum"

    if($?REALTIME && "$dvd" == "$second_to_last") goto startover
end

goto exit


exit:
exit


Setup:
set USE_IMAGE
set VERIFY
set media = DVD
#set noglob
set dirs
set lists
set not
set onlyfind
set dummy
set lastarchive = 0
set tempfile = /dvd_temp/tempfile

if($#argv == 0) then
    set BAD
    goto Help
endif

set i = 0
while($i < $#argv)

    @ i = ( $i + 1 )
    @ j = ( $i + 1 )
    set arg = "$argv[$i]"
    set ARG = `echo "$arg" | awk '{print toupper($1)}'`

    if("$arg" =~ *@*) then
	set alert_email = "$arg"
	echo "will alert $alert_email when each DVD is ready."
    endif

    if(-e "$arg") then
	test -d $arg
	if((! $status) && ($arg !~ */)) then
	    set arg = "${arg}/"
	endif
	test -d $arg
	if($status) then
	    set lists = ( $lists $arg )
	    continue
	endif
	if(! $?NO) then
	    set dirs = ( $dirs $arg )
	else
	    set not  = ( $not $arg )
	endif
	continue
    endif

    # wether or not to use an intermediate disk image
    if("$arg" == "-image") then
	if(! $?NO) then
	    set USE_IMAGE
	else
	    unset USE_IMAGE
	endif
    endif
    if("$arg" == "-fast") then
	if(! $?NO) then
	    set FAST
	else
	    unset FAST
	endif
    endif
    # only back up certain patterns
    if("$arg" == "-only" && $j <= $#argv) then
	set noglob
	set onlyfind = ( $onlyfind $argv[$j] )
	unset noglob
	# skip two this time
        @ i = ( $i + 1 )
	continue
    endif
    # only back up new fies
    if("$arg" == "-since" && $j <= $#argv) then
	set lastarchive = "$argv[$j]"
	if (-e "$argv[$j]" ) then
	    # must be a file
	    set lastarchive = `echo "puts [file mtime $argv[$j]]" | tclsh`
	endif
	# skip two this time
        @ i = ( $i + 1 )
	continue
    endif
    if("$arg" == "-noimage") then
	unset USE_IMAGE
	continue
    endif
    if("$arg" == "-noverify") then
	unset VERIFY
	continue
    endif
    if("$arg" == "-fake") then
	set dummy = "-dummy"
	continue
    endif
    if("$arg" == "-dummy") then
	set dummy = "-dummy"
	continue
    endif
    if("$arg" == "-nolabel") then
	set NOLABEL
	continue
    endif
    if("$arg" == "-realtime") then
	set REALTIME
	continue
    endif
    if("$arg" =~ *dpi) then
	set dpi = `echo $arg | awk '{print $1+0}'`
	continue
    endif
    if("$arg" == "CD") then
	set media = "CD"
	continue
    endif
    if("$arg" == "DVD") then
	set media = "DVD"
	continue
    endif

    if($?NO) set not = ( $not $arg )

    if(("$ARG" =~ NO*) || ("$ARG" =~ "-NO"*)) then
	set NO
	continue
    endif
    unset NO
end

# figure out fastest storage
set fileserver = `df $dirs | awk '/^\/dev\//{print "localhost"} $1~/[:]/{print substr($1,1,index($1,":")-1)}' | head -1`
foreach localdisk ( /dvd_temp/ /tmp/$USER /tmp )
    df $localdisk |& egrep -q "^/dev"
    if(! $status) then
	rm -f ${localdisk}/this >& /dev/null
	touch ${localdisk}/this >& /dev/null
	if (! $status) then
	    # this is a local filesystem we can write to...
	    break
	endif
    endif
end
rm -f ${localdisk}/this >& /dev/null
set tempfile = ${localdisk}/tempfile
if($?CCP4_SCR) then
    set tempfile = ${CCP4_SCR}/tempfile
endif

# select an executable; count the DVD recorders
foreach dvdrecord ( dvdrecord cdrecord cdrecord-dvd )
    $dvdrecord -scanbus >& ${tempfile}
    if(! $status) break
end
set recorders = `awk '/D-R/{print $1}' ${tempfile}`
rm -f ${tempfile}
if ("$dvdrecord" == "dvdrecord") set dvdrecord = "dvdrecord -dao"
if ("$dvdrecord" == "cdrecord-dvd") set dvdrecord = "dvdrecord -packet"

# drives seem to be listed as /dev/scd* in the same order
set drives = `awk '/^\/dev/{printf "%s ",$2; system("ls -ld "$1)}' /etc/fstab | awk '$NF ~ /^\/dev\/scd/{print substr($NF,9)+0,$1}' | sort -n | awk '{print $NF}'`
#set drives = ( /dvdr-1 /dvdr-2 )


# create exclusion definitions
set not = `echo " $not " | awk 'BEGIN{RS=" "} NF==1{printf "%s|", $1}'`
set not = `echo $not | awk '{print substr($1,1,length($1)-1)}'`

set onlyfind = `echo " $onlyfind " | awk 'BEGIN{RS=" "} NF==1{printf "%s|", $1}'`
set onlyfind = `echo $onlyfind | awk '{print substr($1,1,length($1)-1)}'`

# default to excluding raw images
if("$not" == "") set not = "\.imx_|\.dkx_|mtz2various_TMP|\adxv_beam_center|tempfile|sent_home"
if("$onlyfind" == "") set onlyfind = "[.]"

set mediasize = 4700000000
if ("$media" == "CD") set mediasize = 683483136


# now look for problems
if("$recorders" == "") then
    echo "ERROR: cannot find a CD/DVD-R recorder! "
    goto Help
endif

goto Return_from_Setup