X-Git-Url: https://git.wh0rd.org/?p=dump.git;a=blobdiff_plain;f=dump%2Ftape.c;h=edcf5d691925a6532dd71e8b30f75514d8190109;hp=ce150568df6af077126891da2cfb5ca118be253b;hb=b8db960b14856a2ab86e044b2ffa53612d2aa489;hpb=1227625a12a66e0ded78a1997c2d23f23202a382 diff --git a/dump/tape.c b/dump/tape.c index ce15056..edcf5d6 100644 --- a/dump/tape.c +++ b/dump/tape.c @@ -1,8 +1,9 @@ /* * Ported to Linux's Second Extended File System as part of the * dump and restore backup suit - * Remy Card , 1994, 1995, 1996 - * + * Remy Card , 1994-1997 + * Stelian Pop , 1999-2000 + * Stelian Pop - Alcôve , 2000-2002 */ /*- @@ -17,11 +18,7 @@ * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the University of - * California, Berkeley and its contributors. - * 4. Neither the name of the University nor the names of its contributors + * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * @@ -39,18 +36,51 @@ */ #ifndef lint -static char sccsid[] = "@(#)tape.c 8.4 (Berkeley) 5/1/95"; +static const char rcsid[] = + "$Id: tape.c,v 1.91 2009/06/18 09:50:54 stelian Exp $"; #endif /* not lint */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __STDC__ +#include +#include +#include +#else +int write(), read(); +#endif + +#ifdef __linux__ +#include +#include +#include +#include /* for definition of BLKFLSBUF */ +#ifndef BLKFLSBUF /* last resort... */ +#define BLKFLSBUF _IO(0x12, 97) /* Flush buffer cache. */ +#endif +#include +#endif #include #include -#include #include +#include #ifdef __linux__ +#ifdef HAVE_EXT2FS_EXT2_FS_H +#include +#else #include +#endif +#include +#include #include -#else /* __linux__ */ -#ifdef sunos +#elif defined sunos #include #include @@ -58,46 +88,54 @@ static char sccsid[] = "@(#)tape.c 8.4 (Berkeley) 5/1/95"; #else #include #include -#endif #endif /* __linux__ */ #include -#include -#include -#include -#include -#include -#ifdef __STDC__ -#include -#include -#include -#else -int write(), read(); -#endif +#ifdef HAVE_ZLIB +#include +#endif /* HAVE_ZLIB */ + +#ifdef HAVE_BZLIB +#include +#endif /* HAVE_BZLIB */ + +#ifdef HAVE_LZO +#include +#endif /* HAVE_LZO */ -#ifdef __linux__ -#include -#endif #include "dump.h" -#include "pathnames.h" int writesize; /* size of malloc()ed buffer for tape */ long lastspclrec = -1; /* tape block number of last written header */ int trecno = 0; /* next record to write in current block */ -extern long blocksperfile; /* number of blocks per output file */ +extern long *blocksperfiles; /* number of blocks per output file(s) */ +long blocksperfiles_current; /* current position in blocksperfiles */ long blocksthisvol; /* number of blocks on current output file */ extern int ntrec; /* blocking factor on tape */ extern int cartridge; -extern char *host; char *nexttape; +extern pid_t rshpid; +int eot_code = 1; +long long tapea_bytes = 0; /* bytes_written at start of current volume */ +static int magtapeout; /* output is really a tape */ -static int atomic __P((int (*)(), int, char *, int)); +static ssize_t dump_atomic_read __P((int, char *, size_t)); +static ssize_t dump_atomic_write __P((int, const char *, size_t)); +#ifdef WRITEDEBUG +static void doslave __P((int, int, int)); +#else static void doslave __P((int, int)); +#endif static void enslave __P((void)); static void flushtape __P((void)); static void killall __P((void)); static void rollforward __P((void)); +#ifdef USE_QFA +static int GetTapePos __P((long long *)); +static int MkTapeString __P((struct s_spcl *, long long)); +#define FILESQFAPOS 20 +#endif /* * Concurrent dump mods (Caltech) - disk block reading and tape writing @@ -108,11 +146,16 @@ static void rollforward __P((void)); * The following structure defines the instruction packets sent to slaves. */ struct req { - daddr_t dblk; + ext2_loff_t dblk; int count; }; int reqsiz; +struct slave_results { + ssize_t unclen; /* uncompressed length */ + ssize_t clen; /* compressed length */ +}; + #define SLAVES 3 /* 1 slave writing, 1 reading, 1 for slack */ struct slave { int tapea; /* header number at start of this chunk */ @@ -130,16 +173,57 @@ struct slave *slp; char (*nextblock)[TP_BSIZE]; +static time_t tstart_volume; /* time of volume start */ +static int tapea_volume; /* value of spcl.c_tapea at volume start */ + int master; /* pid of master, for sending error signals */ -int tenths; /* length of tape used per block written */ +int tenths; /* length of tape overhead per block written */ static int caught; /* have we caught the signal to proceed? */ static int ready; /* have we reached the lock point without having */ /* received the SIGUSR2 signal from the prev slave? */ -static jmp_buf jmpbuf; /* where to jump to if we are ready when the */ +static sigjmp_buf jmpbuf; /* where to jump to if we are ready when the */ /* SIGUSR2 arrives from the previous slave */ +#ifdef USE_QFA +static int gtperr = 0; +#endif + +/* + * Determine if we can use Linux' clone system call. If so, call it + * with the CLONE_IO flag so that all processes will share the same I/O + * context, allowing the I/O schedulers to make better scheduling decisions. + */ +#ifdef __linux__ +/* first, pull in the header files that define sys_clone and CLONE_IO */ +#include +#define _GNU_SOURCE +#include +#include +#undef _GNU_SOURCE + +/* If either is not present, fall back on the fork behaviour */ +#if ! defined(SYS_clone) || ! defined (CLONE_IO) +#define fork_clone_io fork +#else /* SYS_clone */ +/* CLONE_IO is available, determine which version of sys_clone to use */ +#include +/* + * Kernel 2.5.49 introduced two extra parameters to the clone system call. + * Neither is useful in our case, so this is easy to handle. + */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,5,49) +/* clone_flags, child_stack, parent_tidptr, child_tidptr */ +#define CLONE_ARGS SIGCHLD|CLONE_IO, 0, NULL, NULL +#else +#define CLONE_ARGS SIGCHLD|CLONE_IO, 0 +#endif /* LINUX_VERSION_CODE */ +pid_t fork_clone_io(void); +#endif /* SYS_clone */ +#else /* __linux__ not defined */ +#define fork_clone_io fork +#endif /* __linux__ */ int -alloctape() +alloctape(void) { int pgoff = getpagesize() - 1; char *buf; @@ -153,9 +237,12 @@ alloctape() * repositioning after stopping, i.e, streaming mode, where the gap is * variable, 0.30" to 0.45". The gap is maximal when the tape stops. */ - if (blocksperfile == 0) - tenths = writesize / density + - (cartridge ? 16 : density == 625 ? 5 : 8); + if (!blocksperfiles && !unlimited) + tenths = (cartridge ? 16 : density == 625 ? 5 : 8); + else { + tenths = 0; + density = 1; + } /* * Allocate tape buffer contiguous with the array of instruction * packets, so flushtape() can write them together with one write(). @@ -183,14 +270,38 @@ alloctape() } void -writerec(dp, isspcl) - char *dp; - int isspcl; +writerec(const void *dp, int isspcl) { - slp->req[trecno].dblk = (daddr_t)0; + slp->req[trecno].dblk = (ext2_loff_t)0; slp->req[trecno].count = 1; - *(union u_spcl *)(*(nextblock)++) = *(union u_spcl *)dp; + /* XXX post increment triggers an egcs-1.1.2-12 bug on alpha/sparc */ + *(union u_spcl *)(*(nextblock)) = *(union u_spcl *)dp; + + /* Need to write it to the archive file */ + if (! AfileActive && isspcl && (spcl.c_type == TS_END)) + AfileActive = 1; + if (AfileActive && Afile >= 0 && !(spcl.c_flags & DR_EXTATTRIBUTES)) { + /* When we dump an inode which is not a directory, + * it means we ended the archive contents */ + if (isspcl && (spcl.c_type == TS_INODE) && + ((spcl.c_dinode.di_mode & S_IFMT) != IFDIR)) + AfileActive = 0; + else { + union u_spcl tmp; + tmp = *(union u_spcl *)dp; + /* Write the record, _uncompressed_ */ + if (isspcl) { + tmp.s_spcl.c_flags &= ~DR_COMPRESSED; + mkchecksum(&tmp); + } + if (write(Afile, &tmp, TP_BSIZE) != TP_BSIZE) + msg("error writing archive file: %s\n", + strerror(errno)); + } + } + + nextblock++; if (isspcl) lastspclrec = spcl.c_tapea; trecno++; @@ -200,11 +311,10 @@ writerec(dp, isspcl) } void -dumpblock(blkno, size) - daddr_t blkno; - int size; +dumpblock(blk_t blkno, int size) { - int avail, tpblks, dblkno; + int avail, tpblks; + ext2_loff_t dblkno; dblkno = fsbtodb(sblock, blkno); tpblks = size >> tp_bshift; @@ -222,47 +332,136 @@ dumpblock(blkno, size) int nogripe = 0; -void -tperror(signo) - int signo; +static void +tperror(int errnum) { if (pipeout) { - msg("write error on %s\n", tape); + msg("write error on %s: %s\n", tape, strerror(errnum)); quit("Cannot recover\n"); /* NOTREACHED */ } - msg("write error %d blocks into volume %d\n", blocksthisvol, tapeno); + msg("write error %d blocks into volume %d: %s\n", + blocksthisvol, tapeno, strerror(errnum)); broadcast("DUMP WRITE ERROR!\n"); - if (!query("Do you want to restart?")) - dumpabort(0); - msg("Closing this volume. Prepare to restart with new media;\n"); - msg("this dump volume will be rewritten.\n"); - killall(); - nogripe = 1; - close_rewind(); - Exit(X_REWRITE); + if (query("Do you want to rewrite this volume?")) { + msg("Closing this volume. Prepare to restart with new media;\n"); + msg("this dump volume will be rewritten.\n"); + killall(); + nogripe = 1; + close_rewind(); + Exit(X_REWRITE); + } + if (query("Do you want to start the next tape?")) + return; + dumpabort(0); } -void -sigpipe(signo) - int signo; +static void +sigpipe(UNUSED(int signo)) { quit("Broken pipe\n"); } +/* + * do_stats -- + * Update xferrate stats + */ +time_t +do_stats(void) +{ + time_t tnow, ttaken; + int blocks; + + tnow = time(NULL); + ttaken = tnow - tstart_volume; + blocks = spcl.c_tapea - tapea_volume; + msg("Volume %d completed at: %s", tapeno, ctime(&tnow)); + if (! compressed) + msg("Volume %d %ld blocks (%.2fMB)\n", tapeno, + blocks, ((double)blocks * TP_BSIZE / 1048576)); + if (ttaken > 0) { + long volkb = (bytes_written - tapea_bytes) / 1024; + long txfrate = volkb / ttaken; + msg("Volume %d took %d:%02d:%02d\n", tapeno, + ttaken / 3600, (ttaken % 3600) / 60, ttaken % 60); + msg("Volume %d transfer rate: %ld kB/s\n", tapeno, + txfrate); + xferrate += txfrate; + if (compressed) { + double rate = .0005 + (double) blocks / (double) volkb; + msg("Volume %d %ldkB uncompressed, %ldkB compressed," + " %1.3f:1\n", + tapeno, blocks, volkb, rate); + } + } + return(tnow); +} + +char * +mktimeest(time_t tnow) +{ + static char msgbuf[128]; + time_t deltat; + + msgbuf[0] = '\0'; + + if (blockswritten < 500) + return NULL; + if (blockswritten > tapesize) + tapesize = blockswritten; + deltat = tstart_writing - tnow + (1.0 * (tnow - tstart_writing)) + / blockswritten * tapesize; + if (tnow > tstart_volume) + (void)snprintf(msgbuf, sizeof(msgbuf), + "%3.2f%% done at %ld kB/s, finished in %d:%02d\n", + (blockswritten * 100.0) / tapesize, + (spcl.c_tapea - tapea_volume) / (tnow - tstart_volume), + (int)(deltat / 3600), (int)((deltat % 3600) / 60)); + else + (void)snprintf(msgbuf, sizeof(msgbuf), + "%3.2f%% done, finished in %d:%02d\n", + (blockswritten * 100.0) / tapesize, + (int)(deltat / 3600), (int)((deltat % 3600) / 60)); + + return msgbuf; +} + +#if defined(SIGINFO) +/* + * statussig -- + * information message upon receipt of SIGINFO + */ +void +statussig(int notused) +{ + int save_errno = errno; + char *buf; + + buf = mktimeest(time(NULL)); + if (buf) + write(STDERR_FILENO, buf, strlen(buf)); + errno = save_errno; +} +#endif + static void -flushtape() +flushtape(void) { int i, blks, got; long lastfirstrec; + struct slave_results returned; int siz = (char *)nextblock - (char *)slp->req; + /* make sure returned has sane values in case we don't read + * them from the slave in this pass */ + returned.unclen = returned.clen = writesize; + slp->req[trecno].count = 0; /* Sentinel */ - if (atomic(write, slp->fd, (char *)slp->req, siz) != siz) + if (dump_atomic_write( slp->fd, (char *)slp->req, siz) != siz) quit("error writing command pipe: %s\n", strerror(errno)); slp->sent = 1; /* we sent a request, read the response later */ @@ -273,16 +472,24 @@ flushtape() /* Read results back from next slave */ if (slp->sent) { - if (atomic(read, slp->fd, (char *)&got, sizeof got) - != sizeof got) { + if (dump_atomic_read( slp->fd, (char *)&returned, sizeof returned) + != sizeof returned) { perror(" DUMP: error reading command pipe in master"); dumpabort(0); } + got = returned.unclen; + bytes_written += returned.clen; + if (returned.unclen == returned.clen) + uncomprblks++; slp->sent = 0; - /* Check for end of tape */ - if (got < writesize) { - msg("End of tape detected\n"); + /* Check for errors or end of tape */ + if (got <= 0) { + /* Check for errors */ + if (got < 0) + tperror(-got); + else + msg("End of tape detected\n"); /* * Drain the results, don't care what the values were. @@ -290,9 +497,9 @@ flushtape() */ for (i = 0; i < SLAVES; i++) { if (slaves[i].sent) { - if (atomic(read, slaves[i].fd, - (char *)&got, sizeof got) - != sizeof got) { + if (dump_atomic_read( slaves[i].fd, + (char *)&returned, sizeof returned) + != sizeof returned) { perror(" DUMP: error reading command pipe in master"); dumpabort(0); } @@ -307,10 +514,14 @@ flushtape() } blks = 0; - if (spcl.c_type != TS_END) { - for (i = 0; i < spcl.c_count; i++) - if (spcl.c_addr[i] != 0) - blks++; + if (spcl.c_type == TS_CLRI || spcl.c_type == TS_BITS) + blks = spcl.c_count; + else { + if (spcl.c_type != TS_END) { + for (i = 0; i < spcl.c_count; i++) + if (spcl.c_addr[i] != 0) + blks++; + } } slp->count = lastspclrec + blks + 1 - spcl.c_tapea; slp->tapea = spcl.c_tapea; @@ -318,40 +529,57 @@ flushtape() slp->inode = curino; nextblock = slp->tblock; trecno = 0; - asize += tenths; + asize += tenths + returned.clen / density; blockswritten += ntrec; blocksthisvol += ntrec; - if (!pipeout && (blocksperfile ? - (blocksthisvol >= blocksperfile) : (asize > tsize))) { - close_rewind(); - startnewtape(0); + if (!pipeout && !unlimited) { + if (blocksperfiles && blocksperfiles[blocksperfiles_current]) { + if ( compressed ? (bytes_written - tapea_bytes + SLAVES * (writesize + sizeof(struct tapebuf))) >= (((long long)blocksperfiles[blocksperfiles_current]) * 1024) + : blocksthisvol >= blocksperfiles[blocksperfiles_current] ) { + close_rewind(); + startnewtape(0); + } + } + else if (asize > tsize) { + close_rewind(); + startnewtape(0); + } } timeest(); } -void -trewind() +time_t +trewind(void) { int f; int got; + struct slave_results returned; for (f = 0; f < SLAVES; f++) { /* - * Drain the results, but unlike EOT we DO (or should) care - * what the return values were, since if we detect EOT after - * we think we've written the last blocks to the tape anyway, + * Drain the results, but unlike EOT we DO (or should) care + * what the return values were, since if we detect EOT after + * we think we've written the last blocks to the tape anyway, * we have to replay those blocks with rollforward. * - * fixme: punt for now. + * fixme: punt for now. */ if (slaves[f].sent) { - if (atomic(read, slaves[f].fd, (char *)&got, sizeof got) - != sizeof got) { + if (dump_atomic_read( slaves[f].fd, (char *)&returned, sizeof returned) + != sizeof returned) { perror(" DUMP: error reading command pipe in master"); dumpabort(0); } + got = returned.unclen; + bytes_written += returned.clen; + if (returned.unclen == returned.clen) + uncomprblks++; slaves[f].sent = 0; - if (got != writesize) { + + if (got < 0) + tperror(-got); + + if (got == 0) { msg("EOT detected in last 2 tape records!\n"); msg("Use a longer tape, decrease the size estimate\n"); quit("or use no size estimate at all.\n"); @@ -362,31 +590,48 @@ trewind() while (wait((int *)NULL) >= 0) /* wait for any signals from slaves */ /* void */; - if (pipeout) - return; + if (!pipeout) { - msg("Closing %s\n", tape); + msg("Closing %s\n", tape); #ifdef RDUMP - if (host) { - rmtclose(); - while (rmtopen(tape, 0) < 0) - sleep(10); - rmtclose(); - return; - } + if (host) { + rmtclose(); + while (rmtopen(tape, O_RDONLY) < 0) + sleep(10); + rmtclose(); + } + else #endif - (void) close(tapefd); - while ((f = open(tape, 0)) < 0) - sleep (10); - (void) close(f); + { + (void) close(tapefd); + if (!fifoout) { + while ((f = OPEN(tape, O_RDONLY)) < 0) + sleep (10); + (void) close(f); + } + } + } + return do_stats(); } + void -close_rewind() +close_rewind(void) { - trewind(); - if (nexttape) + int eot_code = 1; + (void)trewind(); + if (eot_script) { + msg("Launching %s\n", eot_script); + eot_code = system_command(eot_script, tape, tapeno); + } + if (eot_code != 0 && eot_code != 1) { + msg("Dump aborted by the end of tape script\n"); + dumpabort(0); + } + if (eot_code == 0) + return; + if (nexttape || Mflag) return; if (!nogripe) { msg("Change Volumes: Mount volume #%d\n", tapeno+1); @@ -400,19 +645,28 @@ close_rewind() } void -rollforward() +rollforward(void) { - register struct req *p, *q, *prev; - register struct slave *tslp; + struct req *p, *q = NULL, *prev; + struct slave *tslp; int i, size, savedtapea, got; union u_spcl *ntb, *otb; + struct slave_results returned; +#ifdef __linux__ + int blks; + long lastfirstrec; +#endif tslp = &slaves[SLAVES]; ntb = (union u_spcl *)tslp->tblock[1]; + /* make sure returned has sane values in case we don't read + * them from the slave in this pass */ + returned.unclen = returned.clen = writesize; + /* - * Each of the N slaves should have requests that need to - * be replayed on the next tape. Use the extra slave buffers - * (slaves[SLAVES]) to construct request lists to be sent to + * Each of the N slaves should have requests that need to + * be replayed on the next tape. Use the extra slave buffers + * (slaves[SLAVES]) to construct request lists to be sent to * each slave in turn. */ for (i = 0; i < SLAVES; i++) { @@ -420,7 +674,7 @@ rollforward() otb = (union u_spcl *)slp->tblock; /* - * For each request in the current slave, copy it to tslp. + * For each request in the current slave, copy it to tslp. */ prev = NULL; @@ -452,11 +706,14 @@ rollforward() lastspclrec = savedtapea - 1; } size = (char *)ntb - (char *)q; - if (atomic(write, slp->fd, (char *)q, size) != size) { + if (dump_atomic_write( slp->fd, (char *)q, size) != size) { perror(" DUMP: error writing command pipe"); dumpabort(0); } slp->sent = 1; +#ifdef __linux__ + lastfirstrec = slp->firstrec; +#endif if (++slp >= &slaves[SLAVES]) slp = &slaves[0]; @@ -464,8 +721,8 @@ rollforward() if (prev->dblk != 0) { /* - * If the last one was a disk block, make the - * first of this one be the last bit of that disk + * If the last one was a disk block, make the + * first of this one be the last bit of that disk * block... */ q->dblk = prev->dblk + @@ -473,7 +730,7 @@ rollforward() ntb = (union u_spcl *)tslp->tblock; } else { /* - * It wasn't a disk block. Copy the data to its + * It wasn't a disk block. Copy the data to its * new location in the buffer. */ q->dblk = 0; @@ -483,8 +740,13 @@ rollforward() } slp->req[0] = *q; nextblock = slp->tblock; - if (q->dblk == 0) + if (q->dblk == 0) { +#ifdef __linux__ + /* XXX post increment triggers an egcs-1.1.2-12 bug on alpha/sparc */ + *(union u_spcl *)(*nextblock) = *(union u_spcl *)tslp->tblock; +#endif nextblock++; + } trecno = 1; /* @@ -492,18 +754,51 @@ rollforward() * worked ok, otherwise the tape is much too short! */ if (slp->sent) { - if (atomic(read, slp->fd, (char *)&got, sizeof got) - != sizeof got) { + if (dump_atomic_read( slp->fd, (char *)&returned, sizeof returned) + != sizeof returned) { perror(" DUMP: error reading command pipe in master"); dumpabort(0); } + got = returned.unclen; + bytes_written += returned.clen; + if (returned.clen == returned.unclen) + uncomprblks++; slp->sent = 0; - if (got != writesize) { + if (got < 0) + tperror(-got); + + if (got == 0) { quit("EOT detected at start of the tape!\n"); } } + +#ifdef __linux__ + blks = 0; + if (spcl.c_type != TS_END) { + for (i = 0; i < spcl.c_count; i++) + if (spcl.c_addr[i] != 0) + blks++; + } + + slp->firstrec = lastfirstrec + ntrec; + slp->count = lastspclrec + blks + 1 - spcl.c_tapea; + slp->inode = curino; + asize += tenths + returned.clen / density; + blockswritten += ntrec; + blocksthisvol += ntrec; +#endif +} + +#ifdef __linux__ +#if defined(SYS_clone) && defined(CLONE_IO) +pid_t +fork_clone_io(void) +{ + return syscall(SYS_clone, CLONE_ARGS); } +#endif +#endif /* * We implement taking and restoring checkpoints on the tape level. @@ -515,33 +810,43 @@ rollforward() * everything continues as if nothing had happened. */ void -startnewtape(top) - int top; +startnewtape(int top) { int parentpid; int childpid; int status; int waitpid; char *p; + #ifdef __linux__ - void (*interrupt_save)(); + sigset_t sigs; + sigemptyset(&sigs); + sigaddset(&sigs, SIGINT); + sigprocmask(SIG_BLOCK, &sigs, NULL); #else /* __linux__ */ #ifdef sunos void (*interrupt_save)(); #else sig_t interrupt_save; #endif + interrupt_save = signal(SIGINT, SIG_IGN); #endif /* __linux__ */ - interrupt_save = signal(SIGINT, SIG_IGN); parentpid = getpid(); + tapea_volume = spcl.c_tapea; + tapea_bytes = bytes_written; + tstart_volume = time(NULL); restore_check_point: +#ifdef __linux__ + sigprocmask(SIG_UNBLOCK, &sigs, NULL); +#else (void)signal(SIGINT, interrupt_save); +#endif /* * All signals are inherited... */ - childpid = fork(); + childpid = fork_clone_io(); if (childpid < 0) { msg("Context save fork fails in parent %d\n", parentpid); Exit(X_ABORT); @@ -553,13 +858,18 @@ restore_check_point: * until the child doing all of the work returns. * don't catch the interrupt */ +#ifdef __linux__ + sigprocmask(SIG_BLOCK, &sigs, NULL); +#else signal(SIGINT, SIG_IGN); +#endif #ifdef TDEBUG msg("Tape: %d; parent process: %d child process %d\n", tapeno+1, parentpid, childpid); #endif /* TDEBUG */ while ((waitpid = wait(&status)) != childpid) - msg("Parent %d waiting for child %d has another child %d return\n", + if (waitpid != rshpid) + msg("Parent %d waiting for child %d has another child %d return\n", parentpid, childpid, waitpid); if (status & 0xFF) { msg("Child %d returns LOB status %o\n", @@ -571,7 +881,7 @@ restore_check_point: case X_FINOK: msg("Child %d finishes X_FINOK\n", childpid); break; - case X_ABORT: + case X_ABORT: msg("Child %d finishes X_ABORT\n", childpid); break; case X_REWRITE: @@ -607,28 +917,50 @@ restore_check_point: * the remaining names for subsequent volumes. */ tapeno++; /* current tape sequence */ - if (nexttape || strchr(tape, ',')) { + if (Mflag) { + snprintf(tape, MAXPATHLEN, "%s%03d", tapeprefix, tapeno); + tape[MAXPATHLEN - 1] = '\0'; + msg("Dumping volume %d on %s\n", tapeno, tape); + } + else if (nexttape || strchr(tapeprefix, ',')) { if (nexttape && *nexttape) - tape = nexttape; - if ((p = strchr(tape, ',')) != NULL) { + tapeprefix = nexttape; + if ((p = strchr(tapeprefix, ',')) != NULL) { *p = '\0'; nexttape = p + 1; } else nexttape = NULL; + strncpy(tape, tapeprefix, MAXPATHLEN); + tape[MAXPATHLEN - 1] = '\0'; msg("Dumping volume %d on %s\n", tapeno, tape); } + if (blocksperfiles && blocksperfiles_current < *blocksperfiles) + blocksperfiles_current++; #ifdef RDUMP - while ((tapefd = (host ? rmtopen(tape, 2) : - pipeout ? 1 : open(tape, O_WRONLY|O_CREAT, 0666))) < 0) + while ((tapefd = (host ? rmtopen(tape, O_WRONLY|O_CREAT|O_TRUNC) : pipeout ? + fileno(stdout) : + OPEN(tape, O_WRONLY|O_CREAT|O_TRUNC, 0666))) < 0) #else - while ((tapefd = (pipeout ? 1 : - open(tape, O_WRONLY|O_CREAT, 0666))) < 0) + while ((tapefd = (pipeout ? fileno(stdout) : + OPEN(tape, O_WRONLY|O_CREAT|O_TRUNC, 0666))) < 0) #endif { - msg("Cannot open output \"%s\".\n", tape); + msg("Cannot open output \"%s\": %s\n", tape, + strerror(errno)); if (!query("Do you want to retry the open?")) dumpabort(0); } +#ifdef RDUMP + if (!host) +#endif + { + struct mtget mt_stat; + magtapeout = ioctl(tapefd, MTIOCGET, (char *)&mt_stat) == 0; + /* + msg("Output is to %s\n", + magtapeout ? "tape" : "file/pipe"); + */ + } enslave(); /* Share open tape file descriptor with slaves */ @@ -636,7 +968,7 @@ restore_check_point: blocksthisvol = 0; if (top) newtape++; /* new tape signal */ - spcl.c_count = slp->count; + spcl.c_count = slp->count; /* * measure firstrec in TP_BSIZE units since restore doesn't * know the correct ntrec value... @@ -645,17 +977,23 @@ restore_check_point: spcl.c_volume++; spcl.c_type = TS_TAPE; spcl.c_flags |= DR_NEWHEADER; - writeheader((ino_t)slp->inode); + spcl.c_ntrec = ntrec; + if (compressed) + spcl.c_flags |= DR_COMPRESSED; + writeheader((dump_ino_t)slp->inode); spcl.c_flags &=~ DR_NEWHEADER; + msg("Volume %d started with block %ld at: %s", tapeno, + spcl.c_tapea, ctime(&tstart_volume)); if (tapeno > 1) msg("Volume %d begins with blocks from inode %d\n", tapeno, slp->inode); + if (tapeno < (int)TP_NINOS) + volinfo[tapeno] = slp->inode; } } void -dumpabort(signo) - int signo; +dumpabort(UNUSED(int signo)) { if (master != 0 && master != getpid()) @@ -671,9 +1009,8 @@ dumpabort(signo) Exit(X_ABORT); } -__dead void -Exit(status) - int status; +void +Exit(int status) { #ifdef TDEBUG @@ -685,32 +1022,37 @@ Exit(status) /* * proceed - handler for SIGUSR2, used to synchronize IO between the slaves. */ -void -proceed(signo) - int signo; +static void +proceed(UNUSED(int signo)) { - if (ready) - longjmp(jmpbuf, 1); + siglongjmp(jmpbuf, 1); caught++; } void -enslave() +enslave(void) { int cmd[2]; #ifdef LINUX_FORK_BUG int i, j; #else - register int i, j; + int i, j; #endif master = getpid(); - signal(SIGTERM, dumpabort); /* Slave sends SIGTERM on dumpabort() */ - signal(SIGPIPE, sigpipe); - signal(SIGUSR1, tperror); /* Slave sends SIGUSR1 on tape errors */ - signal(SIGUSR2, proceed); /* Slave sends SIGUSR2 to next slave */ + { struct sigaction sa; + memset(&sa, 0, sizeof sa); + sigemptyset(&sa.sa_mask); + sa.sa_handler = dumpabort; + sigaction(SIGTERM, &sa, NULL); /* Slave sends SIGTERM on dumpabort() */ + sa.sa_handler = sigpipe; + sigaction(SIGPIPE, &sa, NULL); + sa.sa_handler = proceed; + sa.sa_flags = SA_RESTART; + sigaction(SIGUSR2, &sa, NULL); /* Slave sends SIGUSR2 to next slave */ + } for (i = 0; i < SLAVES; i++) { if (i == slp - &slaves[0]) { @@ -720,24 +1062,37 @@ enslave() } if (socketpair(AF_UNIX, SOCK_STREAM, 0, cmd) < 0 || - (slaves[i].pid = fork()) < 0) + (slaves[i].pid = fork_clone_io()) < 0) quit("too many slaves, %d (recompile smaller): %s\n", i, strerror(errno)); slaves[i].fd = cmd[1]; slaves[i].sent = 0; if (slaves[i].pid == 0) { /* Slave starts up here */ + sigset_t sigs; for (j = 0; j <= i; j++) (void) close(slaves[j].fd); - signal(SIGINT, SIG_IGN); /* Master handles this */ + sigemptyset(&sigs); + sigaddset(&sigs, SIGINT); /* Master handles this */ +#if defined(SIGINFO) + sigaddset(&sigs, SIGINFO); +#endif + sigprocmask(SIG_BLOCK, &sigs, NULL); + #ifdef LINUX_FORK_BUG - if (atomic(write, cmd[0], (char *) &i, sizeof i) + if (dump_atomic_write( cmd[0], (char *) &i, sizeof i) != sizeof i) quit("master/slave protocol botched 3\n"); #endif - doslave(cmd[0], i); + doslave(cmd[0], +#ifdef WRITEDEBUG + i, +#endif + (slaves[i].pid == slp->pid)); Exit(X_FINOK); } + else + close(cmd[0]); } #ifdef LINUX_FORK_BUG @@ -747,13 +1102,12 @@ enslave() * returned from fork() causes a SEGV in the child process */ for (i = 0; i < SLAVES; i++) - if (atomic(read, slaves[i].fd, (char *) &j, sizeof j) != sizeof j) + if (dump_atomic_read( slaves[i].fd, (char *) &j, sizeof j) != sizeof j) quit("master/slave protocol botched 4\n"); #endif - /* write pid of next slave to each slave */ for (i = 0; i < SLAVES; i++) - (void) atomic(write, slaves[i].fd, + (void) dump_atomic_write( slaves[i].fd, (char *) &slaves[(i + 1) % SLAVES].pid, sizeof slaves[0].pid); @@ -761,136 +1115,324 @@ enslave() } void -killall() +killall(void) { - register int i; + int i; for (i = 0; i < SLAVES; i++) - if (slaves[i].pid > 0) + if (slaves[i].pid > 0) { (void) kill(slaves[i].pid, SIGKILL); + slaves[i].sent = 0; + } } /* - * Synchronization - each process has a lockfile, and shares file - * descriptors to the following process's lockfile. When our write - * completes, we release our lock on the following process's lock- - * file, allowing the following process to lock it and proceed. We - * get the lock back for the next cycle by swapping descriptors. + * Synchronization - each process waits for a SIGUSR2 from the + * previous process before writing to the tape, and sends SIGUSR2 + * to the next process when the tape write completes. On tape errors + * a SIGUSR1 is sent to the master which then terminates all of the + * slaves. */ static void -doslave(cmd, slave_number) - register int cmd; - int slave_number; +doslave(int cmd, +#ifdef WRITEDEBUG + int slave_number, +#endif + int first) { - register int nread; - int nextslave, size, wrote, eot_count; + int nread; + int nextslave; + volatile int wrote = 0, size, eot_count, bufsize; + char * volatile buffer; +#if defined(HAVE_ZLIB) || defined(HAVE_BZLIB) || defined(HAVE_LZO) + struct tapebuf * volatile comp_buf = NULL; + int compresult; + volatile int do_compress = !first; + unsigned long worklen; +#ifdef HAVE_LZO + lzo_align_t __LZO_MMODEL *LZO_WorkMem; +#endif +#endif /* HAVE_ZLIB || HAVE_BZLIB || HAVE_LZO */ + struct slave_results returns; #ifdef __linux__ errcode_t retval; #endif +#ifdef USE_QFA + long long curtapepos; + union u_spcl *uspclptr; + struct s_spcl *spclptr; + /* long maxntrecs = 300000000 / (ntrec * 1024); last tested: 50 000 000 */ + long maxntrecs = 50000; /* every 50MB */ + long cntntrecs = maxntrecs; +#endif /* USE_QFA */ + sigset_t set; + + sigemptyset(&set); + sigaddset(&set, SIGUSR2); + sigprocmask(SIG_BLOCK, &set, NULL); + sigemptyset(&set); -#ifdef TDEBUG - msg("slave %d, pid %d\n", slave_number, getpid()); -#endif /* * Need our own seek pointer. */ (void) close(diskfd); - if ((diskfd = open(disk, O_RDONLY)) < 0) + if ((diskfd = OPEN(disk, O_RDONLY)) < 0) quit("slave couldn't reopen disk: %s\n", strerror(errno)); #ifdef __linux__ +#ifdef BLKFLSBUF + (void)ioctl(diskfd, BLKFLSBUF, 0); +#endif ext2fs_close(fs); - retval = ext2fs_open(disk, 0, 0, 0, unix_io_manager, &fs); + retval = dump_fs_open(disk, &fs); if (retval) - quit("slave couldn't reopen disk: %s\n", strerror(errno)); + quit("slave couldn't reopen disk: %s\n", error_message(retval)); #endif /* __linux__ */ /* * Need the pid of the next slave in the loop... */ - if ((nread = atomic(read, cmd, (char *)&nextslave, sizeof nextslave)) + if ((nread = dump_atomic_read( cmd, (char *)&nextslave, sizeof nextslave)) != sizeof nextslave) { quit("master/slave protocol botched - didn't get pid of next slave.\n"); } +#if defined(HAVE_ZLIB) || defined(HAVE_BZLIB) || defined(HAVE_LZO) + /* if we're doing a compressed dump, allocate the compress buffer */ + if (compressed) { + int bsiz = sizeof(struct tapebuf) + writesize; + /* Add extra space to deal with compression enlarging the buffer */ + if (TP_BSIZE > writesize/16 + 67) + bsiz += TP_BSIZE; + else + bsiz += writesize/16 + 67; + comp_buf = malloc(bsiz); + if (comp_buf == NULL) + quit("couldn't allocate a compress buffer.\n"); + if (zipflag == COMPRESS_ZLIB) + comp_buf->flags = COMPRESS_ZLIB; + else if (zipflag == COMPRESS_BZLIB) + comp_buf->flags = COMPRESS_BZLIB; + else if (zipflag == COMPRESS_LZO) { + comp_buf->flags = COMPRESS_LZO; + if (lzo_init() != LZO_E_OK) quit("lzo_init failed\n"); + } else + quit("internal error - unknown compression method: %d\n", zipflag); + } +#ifdef HAVE_LZO + LZO_WorkMem = malloc(LZO1X_1_MEM_COMPRESS); + if (!LZO_WorkMem) + quit("couldn't allocate a compress buffer.\n"); +#endif +#endif /* HAVE_ZLIB || HAVE_BZLIB || HAVE_LZO */ + /* * Get list of blocks to dump, read the blocks into tape buffer */ - while ((nread = atomic(read, cmd, (char *)slp->req, reqsiz)) == reqsiz) { - register struct req *p = slp->req; + while ((nread = dump_atomic_read( cmd, (char *)slp->req, reqsiz)) == reqsiz) { + struct req *p = slp->req; for (trecno = 0; trecno < ntrec; trecno += p->count, p += p->count) { - if (p->dblk) { + if (p->dblk) { /* read a disk block */ bread(p->dblk, slp->tblock[trecno], p->count * TP_BSIZE); - } else { - if (p->count != 1 || atomic(read, cmd, - (char *)slp->tblock[trecno], + } else { /* read record from pipe */ + if (p->count != 1 || dump_atomic_read( cmd, + (char *)slp->tblock[trecno], TP_BSIZE) != TP_BSIZE) quit("master/slave protocol botched.\n"); } } - if (setjmp(jmpbuf) == 0) { + + /* Try to write the data... */ + wrote = 0; + eot_count = 0; + size = 0; + buffer = (char *) slp->tblock[0]; /* set write pointer */ + bufsize = writesize; /* length to write */ + returns.clen = returns.unclen = bufsize; + +#if defined(HAVE_ZLIB) || defined(HAVE_BZLIB) || defined(HAVE_LZO) + /* + * When writing a compressed dump, each block except + * the first one on each tape is written + * from struct tapebuf with an 4 byte prefix + * followed by the data. This can be less than + * writesize. Restore, on a short read, can compare the + * length read to the compressed length in the header + * to verify that the read was good. Blocks which don't + * compress well are written uncompressed. + * The first block written by each slave is not compressed + * and does not have a prefix. + */ + + if (compressed && do_compress) { + comp_buf->length = bufsize; + worklen = TP_BSIZE + writesize; + compresult = 1; +#ifdef HAVE_ZLIB + if (zipflag == COMPRESS_ZLIB) { + compresult = compress2(comp_buf->buf, + &worklen, + (char *)slp->tblock[0], + writesize, + compressed); + if (compresult == Z_OK) + compresult = 1; + else + compresult = 0; + } +#endif /* HAVE_ZLIB */ +#ifdef HAVE_BZLIB + if (zipflag == COMPRESS_BZLIB) { + unsigned int worklen2 = worklen; + compresult = BZ2_bzBuffToBuffCompress( + comp_buf->buf, + &worklen2, + (char *)slp->tblock[0], + writesize, + compressed, + 0, 30); + worklen = worklen2; + if (compresult == BZ_OK) + compresult = 1; + else + compresult = 0; + } + +#endif /* HAVE_BZLIB */ +#ifdef HAVE_LZO + if (zipflag == COMPRESS_LZO) { + lzo_uint worklen2 = worklen; + compresult = lzo1x_1_compress((char *)slp->tblock[0],writesize, + comp_buf->buf, + &worklen2, + LZO_WorkMem); + worklen = worklen2; + if (compresult == LZO_E_OK) + compresult = 1; + else + compresult = 0; + } +#endif /* HAVE_LZO */ + if (compresult && worklen <= ((unsigned long)writesize - 16)) { + /* write the compressed buffer */ + comp_buf->length = worklen; + comp_buf->compressed = 1; + buffer = (char *) comp_buf; + returns.clen = bufsize = worklen + sizeof(struct tapebuf); + } + else { + /* write the data uncompressed */ + comp_buf->length = writesize; + comp_buf->compressed = 0; + buffer = (char *) comp_buf; + returns.clen = bufsize = writesize + sizeof(struct tapebuf); + returns.unclen = returns.clen; + memcpy(comp_buf->buf, (char *)slp->tblock[0], writesize); + } + } + /* compress the remaining blocks if we're compressing */ + do_compress = compressed; +#endif /* HAVE_ZLIB || HAVE_BZLIB || HAVE_LZO */ + + if (sigsetjmp(jmpbuf, 1) == 0) { ready = 1; if (!caught) - (void) pause(); + sigsuspend(&set); } ready = 0; caught = 0; - /* Try to write the data... */ - eot_count = 0; - size = 0; - - while (eot_count < 10 && size < writesize) { +#ifdef USE_QFA + if (gTapeposfd >= 0) { + int i; + int foundone = 0; + + for (i = 0; (i < ntrec) && !foundone; ++i) { + uspclptr = (union u_spcl *)&slp->tblock[i]; + spclptr = &uspclptr->s_spcl; + if ((spclptr->c_magic == NFS_MAGIC) && + (spclptr->c_type == TS_INODE) && + (spclptr->c_date == gThisDumpDate) && + !(spclptr->c_dinode.di_mode & S_IFDIR) && + !(spclptr->c_flags & DR_EXTATTRIBUTES) + ) { + foundone = 1; + /* if (cntntrecs >= maxntrecs) { only write every maxntrecs amount of data */ + cntntrecs = 0; + if (gtperr == 0) + gtperr = GetTapePos(&curtapepos); + /* if an error occured previously don't + * try again */ + if (gtperr == 0) { +#ifdef DEBUG_QFA + msg("inode %ld at tapepos %ld\n", spclptr->c_inumber, curtapepos); +#endif + gtperr = MkTapeString(spclptr, curtapepos); + } + /* } */ + } + } + } +#endif /* USE_QFA */ + + while (eot_count < 10 && size < bufsize) { #ifdef RDUMP if (host) - wrote = rmtwrite(slp->tblock[0]+size, - writesize-size); + wrote = rmtwrite(buffer + size, bufsize - size); else #endif - wrote = write(tapefd, slp->tblock[0]+size, - writesize-size); + wrote = write(tapefd, buffer + size, bufsize - size); #ifdef WRITEDEBUG - msg("slave %d wrote %d\n", slave_number, wrote); + printf("slave %d wrote %d\n", slave_number, wrote); #endif - if (wrote < 0) + if (wrote < 0 && errno != ENOSPC) break; - if (wrote == 0) + if (wrote == 0 || (wrote < 0 && errno == ENOSPC)) eot_count++; - size += wrote; + else + size += wrote; } #ifdef WRITEDEBUG - if (size != writesize) - msg("slave %d only wrote %d out of %d bytes and gave up.\n", - slave_number, size, writesize); + if (size != bufsize) + printf("slave %d only wrote %d out of %d bytes and gave up.\n", + slave_number, size, bufsize); #endif + /* + * Handle ENOSPC as an EOT condition. + */ + if (wrote < 0 && errno == ENOSPC) { + wrote = 0; + eot_count++; + } + if (eot_count > 0) - size = 0; + returns.clen = returns.unclen = 0; /* - * fixme: Pyramids running OSx return ENOSPC - * at EOT on 1/2 inch drives. + * pass errno back to master for special handling */ - if (size < 0) { - (void) kill(master, SIGUSR1); - for (;;) - (void) sigpause(0); - } else { - /* - * pass size of write back to master - * (for EOT handling) - */ - (void) atomic(write, cmd, (char *)&size, sizeof size); - } + if (wrote < 0) + returns.unclen = -errno; + + /* + * pass size of data and size of write back to master + * (for EOT handling) + */ + (void) dump_atomic_write( cmd, (char *)&returns, sizeof returns); /* - * If partial write, don't want next slave to go. - * Also jolts him awake. + * Signal the next slave to go. */ (void) kill(nextslave, SIGUSR2); +#ifdef USE_QFA + if (gTapeposfd >= 0) { + cntntrecs += ntrec; + } +#endif /* USE_QFA */ } if (nread != 0) quit("error reading command pipe: %s\n", strerror(errno)); @@ -901,15 +1443,111 @@ doslave(cmd, slave_number) * or a write may not write all we ask if we get a signal, * loop until the count is satisfied (or error). */ -static int -atomic(func, fd, buf, count) - int (*func)(), fd; - char *buf; - int count; +static ssize_t +dump_atomic_read(int fd, char *buf, size_t count) +{ + int got, need = count; + + do { + while ((got = read(fd, buf, need)) > 0 && (need -= got) > 0) + buf += got; + } while (got == -1 && errno == EINTR); + return (got < 0 ? got : (ssize_t)count - need); +} + +/* + * Since a read from a pipe may not return all we asked for, + * or a write may not write all we ask if we get a signal, + * loop until the count is satisfied (or error). + */ +static ssize_t +dump_atomic_write(int fd, const char *buf, size_t count) { int got, need = count; - while ((got = (*func)(fd, buf, need)) > 0 && (need -= got) > 0) - buf += got; - return (got < 0 ? got : count - need); + do { + while ((got = write(fd, buf, need)) > 0 && (need -= got) > 0) + buf += got; + } while (got == -1 && errno == EINTR); + return (got < 0 ? got : (ssize_t)count - need); +} + + +/* +int +SetLogicalPos(void) +{ + int err = 0; + struct mt_pos buf; + + buf.mt_op = MTSETDRVBUFFER; + buf.mt_count = MT_ST_BOOLEANS | MT_ST_SCSI2LOGICAL; + if (ioctl(tapefd, MTIOCTOP, &buf) == -1) { + err = errno; + msg("[%ld] error: %d (setting logical)\n", + (unsigned long)getpid(), err); + } + return err; +} +*/ + +#ifdef USE_QFA +#define LSEEK_GET_TAPEPOS 10 +#define LSEEK_GO2_TAPEPOS 11 +/* + * read the current tape position + */ +static int +GetTapePos(long long *pos) +{ + int err = 0; + +#ifdef RDUMP + if (host) { + *pos = (long long) rmtseek((OFF_T)0, (int)LSEEK_GET_TAPEPOS); + err = *pos < 0; + } + else +#endif + { + if (magtapeout) { + long mtpos; + *pos = 0; + err = (ioctl(tapefd, MTIOCPOS, &mtpos) < 0); + *pos = (long long)mtpos; + } + else { + *pos = LSEEK(tapefd, 0, SEEK_CUR); + err = (*pos < 0); + } + } + if (err) { + err = errno; + msg("[%ld] error: %d (getting tapepos: %lld)\n", getpid(), + err, *pos); + return err; + } + return err; +} + +static int +MkTapeString(struct s_spcl *spclptr, long long curtapepos) +{ + int err = 0; + +#ifdef DEBUG_QFA + msg("inode %ld at tapepos %lld\n", spclptr->c_inumber, curtapepos); +#endif + + snprintf(gTps, sizeof(gTps), "%ld\t%d\t%lld\n", + (unsigned long)spclptr->c_inumber, + tapeno, + curtapepos); + gTps[sizeof(gTps) - 1] = '\0'; + if (write(gTapeposfd, gTps, strlen(gTps)) != (ssize_t)strlen(gTps)) { + err = errno; + warn("error writing tapepos file. (error %d)\n", errno); + } + return err; } +#endif /* USE_QFA */