From: Stelian Pop Date: Fri, 10 Jun 2011 13:41:41 +0000 (+0000) Subject: SQLlite and QFA overhaul. X-Git-Url: https://git.wh0rd.org/?a=commitdiff_plain;h=70deb804d2211535f1f188a14e8fc867db9c6625;p=dump.git SQLlite and QFA overhaul. --- diff --git a/CHANGES b/CHANGES index a3f339c..32d742c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,4 @@ -$Id: CHANGES,v 1.329 2011/06/10 13:07:24 stelian Exp $ +$Id: CHANGES,v 1.330 2011/06/10 13:41:41 stelian Exp $ Changes between versions 0.4b44 and 0.4b45 (released ?????????????) =================================================================== @@ -11,6 +11,10 @@ Changes between versions 0.4b44 and 0.4b45 (released ?????????????) compression/encryption code as plugins. Thanks to Bear Giles for the patch. +3. Add sqlite index support and reorganize the QFA code + as plugins. Thanks to Bear Giles for + the patch. + Changes between versions 0.4b43 and 0.4b44 (released June 10, 2011) =================================================================== diff --git a/Makefile.in b/Makefile.in index f8f64f9..2dbd90a 100644 --- a/Makefile.in +++ b/Makefile.in @@ -5,7 +5,7 @@ top_builddir= . @MCONFIG@ RM= @RM@ -SUBDIRS= compat/lib compat/include common dump restore testing @RMTDIR@ +SUBDIRS= compat/lib compat/include common dump restore @RMTDIR@ all clean install dep depend realclean distclean:: for i in $(SUBDIRS); do \ diff --git a/common/Makefile.in b/common/Makefile.in index 2c03400..13f43bd 100644 --- a/common/Makefile.in +++ b/common/Makefile.in @@ -1,4 +1,4 @@ -# $Id: Makefile.in,v 1.6 2011/06/10 13:07:29 stelian Exp $ +# $Id: Makefile.in,v 1.7 2011/06/10 13:41:41 stelian Exp $ top_srcdir= @top_srcdir@ srcdir= @srcdir@ @@ -6,16 +6,29 @@ top_builddir= .. @MCONFIG@ -INC= -I$(top_srcdir)/dump -I$(top_srcdir)/common +INC= -I. -I$(top_srcdir)/dump -I$(top_srcdir)/compat/include ALL_CFLAGS= @CPPFLAGS@ @CFLAGS@ @CCOPTS@ -pipe $(OPT) $(GINC) $(INC) $(DEFS) @DUMPDEBUG@ ALL_LDFLAGS= @LDFLAGS@ @LDOPTS@ @STATIC@ -SRCS= dumprmt.c transformation_null.c transformation_lzo.c transformation_zlib.c transformation_bzlib.c transformation_ssl.c -OBJS= dumprmt.o transformation_null.o transformation_lzo.o transformation_zlib.o transformation_bzlib.o transformation_ssl.o +LIBS= $(GLIBS) @ZLIB@ @BZLIB@ @BLKID@ @SSLLIB@ @SQLITE3@ + +SRCS= dumprmt.c transformation_null.c transformation_lzo.c transformation_zlib.c transformation_bzlib.c \ + transformation_ssl.c legacy_indexer.c sqlite_indexer.c +OBJS= dumprmt.o transformation_null.o transformation_lzo.o transformation_zlib.o transformation_bzlib.o \ + transformation_ssl.o legacy_indexer.o sqlite_indexer.o .c.o: $(CC) -c $(ALL_CFLAGS) $< -o $@ -all:: $(OBJS) +all:: $(OBJS) transform_test indexer_test + +transform_test: transform_test.o $(OBJS) $(DEPLIBS) + $(LD) $(ALL_LDFLAGS) -o transform_test transform_test.o \ + transformation_null.o transformation_lzo.o transformation_zlib.o \ + transformation_bzlib.o transformation_ssl.o $(LIBS) + +indexer_test: indexer_test.o $(OBJS) $(DEPLIBS) + $(LD) $(ALL_LDFLAGS) -o indexer_test indexer_test.o legacy_indexer.o \ + sqlite_indexer.o dumprmt.o $(LIBS) install:: diff --git a/common/indexer.h b/common/indexer.h new file mode 100644 index 0000000..102251b --- /dev/null +++ b/common/indexer.h @@ -0,0 +1,54 @@ +#include +#include +#include + +#ifndef _ARCHIVE_H +#define _ARCHIVE_H 1 + +#ifdef USE_QFA +#define QFA_MAGIC "495115637697" +#define QFA_VERSION "1.0" +#endif /* USE_QFA */ + +#ifndef __P +#include +#endif + +/* state information when writing QFA file. */ +typedef struct qfa_state { + long long curtapepos; + /* long maxntrecs = 300000000 / (ntrec * 1024); last tested: 50 000 000 */ + long maxntrecs; + long cntntrecs; +} QFA_State; + +/* methods used to write 'Indexer' file. */ +typedef struct indexer { + struct indexer *next; + int (*open) __P((const char *filename, int mode)); + int (*close) __P((void)); + int (*foo) __P((void)); // what does this function do? + int (*writerec) __P((const void *dp, int isspcl)); + int (*addInode) __P((struct dinode *dp, dump_ino_t ino, int metaonly)); // dump entry + //int (*addDirEntry) __P((struct ext2_dir_entry *dirent, UNUSED(int offset), // dump inode + // UNUSED(int blocksize), UNUSED(char *buf), void *private)); + int (*addDirEntry) __P((struct direct *dp, dump_ino_t parent_ino)); + + int (*openQfa) __P(()); + int (*closeQfa) __P(()); + int (*openQfaState) __P((QFA_State *state)); + int (*updateQfa) __P((QFA_State *state)); + int (*updateQfaState) __P((QFA_State *state)); +} Indexer; + +extern Indexer *indexer; + +extern Indexer indexer_legacy; +extern Indexer indexer_sqlite; + +//extern Indexer *indexer_legacy_factory(); + +//#ifdef HAVE_SQLITE3 +//extern Indexer *indexer_sqlite_factory(); +//#endif /* HAVE_SQLITE3 */ +#endif diff --git a/common/indexer_test.c b/common/indexer_test.c new file mode 100644 index 0000000..003198b --- /dev/null +++ b/common/indexer_test.c @@ -0,0 +1,221 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#ifdef __linux__ +#include +#ifdef HAVE_EXT2FS_EXT2_FS_H +#include +#else +#include +#endif +#include +#elif defined sunos +#include +#include +#include +#else +#include +#include +#endif +#include + +#include "indexer.h" +#include "slave.h" + +int notify; +dump_ino_t volinfo[TP_NINOS]; + +int nddates; +struct dumpdates **ddatev; +char *host = NULL; +int tapefd; +int tapeno; + +struct slave slaves[SLAVES+1]; +struct slave *slp; + +union u_spcl u_spcl; + +#ifdef __linux__ +struct struct_ext2_filsys test_fs; +struct ext2_super_block test_super; +ext2_filsys fs = &test_fs; +#else +struct fs *sblock; /* the file system super block */ +char sblock_buf[MAXBSIZE]; +#endif + +/* for dumprmt.c */ +int dokerberos = 0; +int ntrec; +int abortifconnerr = 0; + +/* + * + */ +void +msg(const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + vprintf(fmt, ap); + va_end(ap); +} + +/* + * print error message and quit. + */ +int +quit(const char *msg) +{ + printf("%s\n", msg); + exit(1); +} + +/* + * Compute and fill in checksum information. (from dump/traverse.c) + */ +void +mkchecksum(union u_spcl *tmpspcl) +{ + int32_t sum, cnt, *lp; + + tmpspcl->s_spcl.c_checksum = 0; + lp = (int32_t *)&tmpspcl->s_spcl; + sum = 0; + cnt = sizeof(union u_spcl) / (4 * sizeof(int32_t)); + while (--cnt >= 0) { + sum += *lp++; + sum += *lp++; + sum += *lp++; + sum += *lp++; + } + tmpspcl->s_spcl.c_checksum = CHECKSUM - sum; +} + +/* + * + */ +int +dump_inode(Indexer *indexer, struct stat *buf) +{ + struct dinode dinode; + + memset(&dinode, 0, sizeof dinode); + dinode.di_mode = buf->st_mode; + dinode.di_nlink = buf->st_nlink; + dinode.di_uid = buf->st_uid; + dinode.di_gid = buf->st_gid; + dinode.di_rdev = buf->st_rdev; + dinode.di_size = buf->st_size; + dinode.di_atime = buf->st_atime; + dinode.di_mtime = buf->st_mtime; + dinode.di_ctime = buf->st_ctime; + dinode.di_blocks = buf->st_blocks; + + spcl.c_tapea++; + + indexer->addInode(&dinode, buf->st_ino, 0); +} + +/* + * + */ +int +dump_walk(Indexer *indexer, const char *root, int first) +{ + DIR *dirp; + struct dirent *dp; + char pathname[MAXPATHLEN]; + struct stat bufroot, bufentry; + struct direct direct; + + dirp = opendir(root); + if (dirp == NULL) { + printf("%s:%d: trying to open '%s'\n", __FILE__, __LINE__, root); + perror(root); + return -1; + } + + stat(root, &bufroot); + + //if (first) { + // spcl.c_volume = 1; + // spcl.c_tapea = 1; + // indexer->addDirEntry(&direct, direct.d_ino); + // dump_inode(indexer, &bufroot); + //} + + while ((dp = readdir(dirp)) != NULL) { + direct.d_ino = dp->d_ino; + direct.d_reclen = dp->d_reclen; + direct.d_type = dp->d_type; + direct.d_namlen = strlen(dp->d_name); + memcpy(direct.d_name, dp->d_name, MAXNAMLEN + 1); + direct.d_name[MAXNAMLEN] = '\0'; + + snprintf(pathname, sizeof pathname, "%s/%s", root, dp->d_name); + stat(pathname, &bufentry); + if (strcmp(dp->d_name, "..")) { + indexer->addDirEntry(&direct, bufentry.st_ino); + dump_inode(indexer, &bufentry); + if (S_ISDIR(bufentry.st_mode)) { + if (strcmp(dp->d_name, ".")) { + dump_walk(indexer, pathname, 0); + } + } + } + } + closedir(dirp); +} + +/* + * + */ +int +test_indexer(Indexer *indexer, const char *filename, const char *path) +{ + struct direct dp; + struct stat buf; + +#ifdef __linux__ + test_fs.super = &test_super; + uuid_generate_time((unsigned char *)&test_fs.super->s_uuid); +#endif + + indexer->open(filename, 1); + + dump_walk(indexer, path, 1); + indexer->close(); + + indexer->open(filename, 0); + indexer->close(); +} + +/* + * + */ +int +main(int argc, char **argv) +{ + if (argc == 1) { +// printf("usage: %s partition\n", argv[0]); +// exit(1); + argv[1] = "/usr/include"; + } + +// test_indexer(&indexer_legacy, "/tmp/archive.data", argv[1]); + +#ifdef HAVE_SQLITE3 + test_indexer(&indexer_sqlite, "/tmp/t.sqlite3", argv[1]); +#endif /* HAVE_SQLITE3 */ +} diff --git a/common/legacy_indexer.c b/common/legacy_indexer.c new file mode 100644 index 0000000..677895a --- /dev/null +++ b/common/legacy_indexer.c @@ -0,0 +1,359 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#ifdef __linux__ +#include +#include +#include +#include +#include /* for definition of BLKFLSBUF */ +#elif defined sunos +#include + +#include +#include +#else +#include +#include +#endif + +#include + +//#include "dump.h" +#include "indexer.h" +#include "slave.h" + +extern dump_ino_t volinfo[]; // TP_NINOS +extern int tapeno; + +static int Afile = -1; /* archive file descriptor */ +static int AfileActive = 1;/* Afile flag */ + +//extern char *Apath; +char *Apath; + +#ifdef USE_QFA +static int GetTapePos __P((long long *)); +static int MkTapeString __P((struct s_spcl *, long long)); +#define FILESQFAPOS 20 + +int tapepos; +int ntrec; /* blocking factor on tape */ +int magtapeout; /* is output a magnetic tape? */ +//extern int tapepos; +//extern int ntrec; /* blocking factor on tape */ +//extern int magtapeout; /* is output a magnetic tape? */ + +static int gtperr = 0; +static int gTapeposfd; +static char *gTapeposfile; +static char gTps[255]; +static int32_t gThisDumpDate; +#endif /* USE_QFA */ + +void msg __P((const char *fmt, ...)); +void quit __P((const char *fmt, ...)); + +extern char *host; +extern int tapefd; + +/* + * Open the indexer file. + */ +static int +legacy_open(const char *filename, int mode) +{ + if (filename == NULL) { + return 0; + } + + if (mode == 0) { + if ((Afile = OPEN(filename, O_RDONLY)) < 0) { + msg("Cannot open %s for reading: %s\n", + filename, strerror(errno)); + msg("The ENTIRE dump is aborted.\n"); + return -1; + } + } else if (mode == 1) { + if ((Afile = OPEN(filename, O_WRONLY|O_CREAT|O_TRUNC, + S_IRUSR | S_IWUSR | S_IRGRP | + S_IWGRP | S_IROTH | S_IWOTH)) < 0) { + msg("Cannot open %s for writing: %s\n", + filename, strerror(errno)); + msg("The ENTIRE dump is aborted.\n"); + return -1; + } + } + + return 0; +} + +/* + * Close the indexer file. + */ +static int +legacy_close() +{ + if (Afile >= 0) + msg("Archiving dump to %s\n", Apath); + return 0; +} + +/* + * Write a record to the indexer file. + */ +static int +legacy_writerec(const void *dp, int isspcl) +{ + if (! AfileActive && isspcl && (spcl.c_type == TS_END)) + AfileActive = 1; + if (AfileActive && Afile >= 0 && !(spcl.c_flags & DR_EXTATTRIBUTES)) { + /* When we dump an inode which is not a directory, + * it means we ended the archive contents */ + if (isspcl && (spcl.c_type == TS_INODE) && + ((spcl.c_dinode.di_mode & S_IFMT) != IFDIR)) + AfileActive = 0; + else { + union u_spcl tmp; + tmp = *(union u_spcl *)dp; + /* Write the record, _uncompressed_ */ + if (isspcl) { + tmp.s_spcl.c_flags &= ~DR_COMPRESSED; + mkchecksum(&tmp); + } + if (write(Afile, &tmp, TP_BSIZE) != TP_BSIZE) + msg("error writing archive file: %s\n", + strerror(errno)); + } + } + return 0; +} + +/* + * I'm not sure what this is used for... + */ +static int +legacy_foo() +{ + if (Afile >= 0) { + volinfo[1] = ROOTINO; + memcpy(spcl.c_inos, volinfo, TP_NINOS * sizeof(dump_ino_t)); + spcl.c_flags |= DR_INODEINFO; + } + return 0; +} + +/* + * Dump inode + */ +static int +legacy_addInode(struct dinode *dp, dump_ino_t ino, int metaonly) +{ + return 0; +} + +/* + * Dump directory (dirent) entry + */ +static int +legacy_addDirEntry(struct direct *dp, dump_ino_t parent_ino) +{ + return 0; +} + + +#define LSEEK_GET_TAPEPOS 10 +#define LSEEK_GO2_TAPEPOS 11 + +/* + * + */ +static int +legacy_openQfa() +{ +#ifdef USE_QFA + gThisDumpDate = spcl.c_date; + if (tapepos) { + msg("writing QFA positions to %s\n", gTapeposfile); + if ((gTapeposfd = open(gTapeposfile, + O_WRONLY|O_CREAT|O_TRUNC, + S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP + | S_IROTH | S_IWOTH)) < 0) + quit("can't open tapeposfile\n"); + /* print QFA-file header */ + snprintf(gTps, sizeof(gTps), "%s\n%s\n%ld\n\n", QFA_MAGIC, QFA_VERSION, (unsigned long)spcl.c_date); + gTps[sizeof(gTps) - 1] = '\0'; + if (write(gTapeposfd, gTps, strlen(gTps)) != (ssize_t)strlen(gTps)) + quit("can't write tapeposfile\n"); + sprintf(gTps, "ino\ttapeno\ttapepos\n"); + if (write(gTapeposfd, gTps, strlen(gTps)) != (ssize_t)strlen(gTps)) + quit("can't write tapeposfile\n"); + } +#endif /* USE_QFA */ + return 0; +} + +/* + * + */ +static int +legacy_closeQfa() +{ + return 0; +} + +/* + * + */ +static int +legacy_openQfaState(QFA_State *s) +{ + s->curtapepos = 0; + s->maxntrecs = 50000; /* every 50MB */ + s->cntntrecs = s->maxntrecs; + return 0; +} + +/* + * + */ +static int +legacy_updateQfaState(QFA_State *s) +{ + if (gTapeposfd >= 0) { + s->cntntrecs += ntrec; + } + + return 0; +} + + +/* + * + */ +static int +legacy_updateQfa(QFA_State *s) +{ +#ifdef USE_QFA + union u_spcl *uspclptr; + struct s_spcl *spclptr; + + if (gTapeposfd >= 0) { + int i; + int foundone = 0; + + for (i = 0; (i < ntrec) && !foundone; ++i) { + uspclptr = (union u_spcl *)&slp->tblock[i]; + spclptr = &uspclptr->s_spcl; + if ((spclptr->c_magic == NFS_MAGIC) && + (spclptr->c_type == TS_INODE) && + (spclptr->c_date == gThisDumpDate) && + !(spclptr->c_dinode.di_mode & S_IFDIR) && + !(spclptr->c_flags & DR_EXTATTRIBUTES) + ) { + foundone = 1; + /* if (s->cntntrecs >= s->maxntrecs) { only write every maxntrecs amount of data */ + s->cntntrecs = 0; + if (gtperr == 0) + gtperr = GetTapePos(&s->curtapepos); + /* if an error occured previously don't + * try again */ + if (gtperr == 0) { +#ifdef DEBUG_QFA + msg("inode %ld at tapepos %ld\n", spclptr->c_inumber, s->curtapepos); +#endif + gtperr = MkTapeString(spclptr, s->curtapepos); + } + /* } */ + } + } + } +#endif /* USE_QFA */ + return 0; +} + +#ifdef USE_QFA +/* + * read the current tape position + */ +static int +GetTapePos(long long *pos) +{ + int err = 0; + +#ifdef RDUMP + if (host) { + *pos = (long long) rmtseek((OFF_T)0, (int)LSEEK_GET_TAPEPOS); + err = *pos < 0; + } + else +#endif + { + if (magtapeout) { + long mtpos; + *pos = 0; + err = (ioctl(tapefd, MTIOCPOS, &mtpos) < 0); + *pos = (long long)mtpos; + } + else { + *pos = LSEEK(tapefd, 0, SEEK_CUR); + err = (*pos < 0); + } + } + if (err) { + err = errno; + msg("[%ld] error: %d (getting tapepos: %lld)\n", getpid(), + err, *pos); + return err; + } + return err; +} + +static int +MkTapeString(struct s_spcl *spclptr, long long curtapepos) +{ + int err = 0; + +#ifdef DEBUG_QFA + msg("inode %ld at tapepos %lld\n", spclptr->c_inumber, curtapepos); +#endif + + snprintf(gTps, sizeof(gTps), "%ld\t%d\t%lld\n", + (unsigned long)spclptr->c_inumber, + tapeno, + curtapepos); + gTps[sizeof(gTps) - 1] = '\0'; + if (write(gTapeposfd, gTps, strlen(gTps)) != (ssize_t)strlen(gTps)) { + err = errno; + warn("error writing tapepos file. (error %d)\n", errno); + } + return err; +} +#endif /* USE_QFA */ + +Indexer indexer_legacy = { + NULL, + &legacy_open, + &legacy_close, + &legacy_foo, + &legacy_writerec, + &legacy_addInode, + &legacy_addDirEntry, + &legacy_openQfa, + &legacy_closeQfa, + &legacy_updateQfa, + &legacy_updateQfaState +}; diff --git a/common/slave.h b/common/slave.h new file mode 100644 index 0000000..cd05d6d --- /dev/null +++ b/common/slave.h @@ -0,0 +1,39 @@ + +/* + * Structures pulled from dump/tape.c. We need it here so the 'archive' code can + * determine where we are in the tape. + */ +#ifndef _SLAVE_H +#define _SLAVE_H 1 + +/* + * Concurrent dump mods (Caltech) - disk block reading and tape writing + * are exported to several slave processes. While one slave writes the + * tape, the others read disk blocks; they pass control of the tape in + * a ring via signals. The parent process traverses the filesystem and + * sends writeheader()'s and lists of daddr's to the slaves via pipes. + * The following structure defines the instruction packets sent to slaves. + */ +struct req { + ext2_loff_t dblk; + int count; +}; + +#define SLAVES 3 /* 1 slave writing, 1 reading, 1 for slack */ + +struct slave { + int tapea; /* header number at start of this chunk */ + int count; /* count to next header (used for TS_TAPE */ + /* after EOT) */ + int inode; /* inode that we are currently dealing with */ + int fd; /* FD for this slave */ + int pid; /* PID for this slave */ + int sent; /* 1 == we've sent this slave requests */ + int firstrec; /* record number of this block */ + char (*tblock)[TP_BSIZE]; /* buffer for data blocks */ + struct req *req; /* buffer for requests */ +}; + +extern struct slave *slp; + +#endif diff --git a/common/sqlite_indexer.c b/common/sqlite_indexer.c new file mode 100644 index 0000000..4008341 --- /dev/null +++ b/common/sqlite_indexer.c @@ -0,0 +1,511 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#ifdef __linux__ +#include +#ifdef HAVE_EXT2FS_EXT2_FS_H +#include +#else +#include +#endif +#include +#include +#include +#include /* for definition of BLKFLSBUF */ +#elif defined sunos +#include + +#include +#include +#else +#include +#include +#endif + +#include + +//#include "dump.h" +#include "indexer.h" +#include "pathnames.h" +#include "bylabel.h" + +extern int tapeno; +extern dump_ino_t volinfo[]; // TP_NINOS + +extern void msg __P((const char *fmt,...)); + +#ifdef __linux__ +extern ext2_filsys fs; +#else /* __linux__ */ +extern struct fs *sblock; +#endif /* __linux__ */ + +//#ifdef HAVE_SQLITE3 +#include +#include + +static sqlite3 *db; + +// convenience macro that handles a basic sql statement. +#define EXEC(db, buffer) \ + rc = sqlite3_exec(db, buffer, NULL, 0, &errMsg); \ + if (rc != SQLITE_OK) { \ + msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db)); \ + msg(buffer); \ + sqlite3_free(errMsg); \ + sqlite3_close(db); \ + db = NULL; \ + return -1; \ + } + +/* prepared statement that allows us to use placeholder values. */ +/* it also acts as sentinel to indicate if this is read or write mode. */ +sqlite3_stmt *stmt = NULL; + +/* + * Ppopulate the 'backup' table. + */ +static int +sqlite_populate_backup_table() +{ + int rc; + char *errMsg = NULL; + char buffer[2000]; + char vol_uuid[40]; + char ts[40]; + char nts[40]; + char fs_uuid[40]; + time_t now; + const char *tail; + uuid_t uuid; + + time(&now); + + // populate 'backup table'. There will only be one entry. + uuid_generate_time(uuid); + uuid_unparse_lower(uuid, vol_uuid); + uuid_unparse_lower(fs->super->s_uuid, fs_uuid); + strftime(ts, sizeof ts, "%FT%T", gmtime((const time_t *) &spcl.c_date)); + strftime(nts, sizeof nts, "%FT%T", gmtime((const time_t *) &now)); + snprintf(buffer, sizeof buffer, + "insert into backup(backup_id, vol_uuid, dt, start_dt, fs_uuid, dev, label, host, level) values (1, '%s', '%s', '%s', '%s', ?, ?, ?, %d)", + vol_uuid, ts, nts, fs_uuid, spcl.c_level); + +// spcl.c_dev, spcl.c_label, spcl.c_host, + + // prepare statement. + rc = sqlite3_prepare_v2(db, buffer, strlen(buffer), &stmt, &tail); + if (rc != SQLITE_OK) { + msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db)); + msg(buffer); + sqlite3_free(errMsg); + sqlite3_close(db); + db = NULL; + return -1; + } + + sqlite3_bind_text(stmt, 1, spcl.c_dev, strlen(spcl.c_dev), SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, spcl.c_label, strlen(spcl.c_label), SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 3, spcl.c_host, strlen(spcl.c_host), SQLITE_TRANSIENT); + + while ((rc = sqlite3_step(stmt)) == SQLITE_BUSY) { + continue; + } + + if (rc != SQLITE_DONE) { + msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db)); + msg("(%s:%d) %s\n", __FILE__, __LINE__, sqlite3_sql(stmt)); + sqlite3_free(errMsg); + sqlite3_close(db); + db = NULL; + return -1; + } + + rc = sqlite3_finalize(stmt); + if (rc != SQLITE_OK) { + msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db)); + msg("(%s:%d) %s\n", __FILE__, __LINE__, sqlite3_sql(stmt)); + sqlite3_free(errMsg); + sqlite3_close(db); + db = NULL; + return -1; + } + + return 0; +} + +/* + * Create database schema + */ +static int +sqlite_create_schema() +{ + int rc; + char *errMsg = NULL; + char buffer[2000]; + const char *tail; + + // turn on foreign keys + snprintf(buffer, sizeof buffer, "pragma foreign_keys = ON"); + + EXEC(db, buffer); + + // turn off synchronous access. we have bigger problems + // if we lose power in the middle of a backup. + snprintf(buffer, sizeof buffer, "pragma synchronous = OFF\n"); + + EXEC(db, buffer); + + // create 'backup' table. + snprintf(buffer, sizeof buffer, + "create table backup(backup_id int primary key not null, vol_uuid char[36] not null, dt timestamp not null, start_dt timestamp not null, end_dt timestamp, fs_uuid char[36] not null, dev varchar[%1$d] not null, label varchar[%1$d] not null, host varchar[%1$d] not null, level int not null, blocks int, volumes int)\n", EXT2_NAME_LEN); + + EXEC(db, buffer); + + snprintf(buffer, sizeof buffer, + "create unique index i1 on backup(backup_id)\n"); + EXEC(db, buffer); + + // create 'entry' table. This contains information about + // each directory entry + snprintf(buffer, sizeof buffer, + "create table entry(backup_id int not null, ino int not null, parent_ino int not null, type int not null, depth int, name varchar[%d] not null, path varchar[%d], foreign key(backup_id) references backup(backup_id))\n", EXT2_NAME_LEN, 1024); + + // "create table entry(backup_id int not null, ino int not null, parent_ino int not null, type int not null, depth int, name varchar[%d] not null, path varchar[%d], foreign key(backup_id) references backup(backup_id), foreign key(parent_ino) references entry(ino))\n", EXT2_NAME_LEN, 1024); + + EXEC(db, buffer); + + // snprintf(buffer, sizeof buffer, + // "create unique index i2 on entry(ino)\n"); + + // EXEC(db, buffer); + + snprintf(buffer, sizeof buffer, + "create trigger t1 after insert on entry begin update entry set path = coalesce((select '.' where name = '.'), (select A.path || '/' || new.name from entry A where A.ino = new.parent_ino)) where rowid = new.rowid; end;\n"); + + EXEC(db, buffer); + + // create 'inode' table. This contains information about each inode. + snprintf(buffer, sizeof buffer, + "create table inode(backup_id int not null, ino int not null, is_deleted char[1] not null default 'N', mode int not null, nlink int not null, uid int not null, gid int not null, rdev int not null, size int not null, atime timestamp not null, mtime timestamp not null, ctime timestamp not null, has_xattr char[1] not null default 'N', has_acl char[1] not null default 'N', volume int not null, recno int not null, foreign key(backup_id) references backup(backup_id))\n"); + + // "create table inode(backup_id int not null, ino int not null, is_deleted char[1] not null default 'N', mode int not null, nlink int not null, uid int not null, gid int not null, rdev int not null, size int not null, atime timestamp not null, mtime timestamp not null, ctime timestamp not null, has_xattr char[1] not null default 'N', has_acl char[1] not null default 'N', volume int not null, recno int not null, foreign key(backup_id) references backup(backup_id), foreign key(ino) references entry(ino))\n"); + + EXEC(db, buffer); + + sqlite_populate_backup_table(); + + // prepare statement for entry inserts. + snprintf(buffer, sizeof buffer, + "insert into entry(backup_id, ino, parent_ino, type, name) values(1, ?, ?, ?, ?)\n"); + + rc = sqlite3_prepare_v2(db, buffer, strlen(buffer), &stmt, &tail); + if (rc != SQLITE_OK) { + msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db)); + msg(buffer); + sqlite3_free(errMsg); + sqlite3_close(db); + db = NULL; + return; + } +} + +/* + * Open database. + */ +static int +sqlite_open(const char *filename, int mode) +{ + if (mode == 0) { + if (sqlite3_open(filename, &db)) { + msg("Unable to open database: %s\n", sqlite3_errmsg(db)); + db = NULL; + return -1; + } + stmt = NULL; + } else if (mode == 1) { + unlink(filename); + + /* sqlite will always create DB if it doesn't already exist. */ + if (sqlite3_open(filename, &db)) { + msg("Unable to open database: %s\n", sqlite3_errmsg(db)); + db = NULL; + return -1; + } + + sqlite_create_schema(); + } + + return 0; +} + +/* + * Close database. + */ +static int +sqlite_close() +{ + int rc; + char *errMsg = NULL; + char buffer[2000]; + char uuid[40]; + char ts[40]; + time_t now; + + if (db == NULL) { + msg("database is already closed!"); + return -1; + } + + time(&now); + + /* if stmt is not null we're creating database */ + if (stmt != NULL) { + // close prepared statement. + rc = sqlite3_finalize(stmt); + if (rc != SQLITE_OK) { + msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db)); + msg("(%s:%d) %s\n", __FILE__, __LINE__, sqlite3_sql(stmt)); + sqlite3_free(errMsg); + sqlite3_close(db); + db = NULL; + return -1; + } + + // update 'tape header' with number of volumes and blocks. + uuid_unparse_lower(fs->super->s_uuid, uuid); + strftime(ts, sizeof ts, "%FT%T", gmtime((const time_t *) &now)); + snprintf(buffer, sizeof buffer, + "update backup set end_dt = '%s', blocks=%d, volumes=%d where fs_uuid='%s'", + ts, spcl.c_tapea, spcl.c_volume, uuid); + + EXEC(db, buffer); + } + + // close database. + rc = sqlite3_close(db); + if (rc != SQLITE_OK) { + msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db)); + msg(buffer); + sqlite3_free(errMsg); + db = NULL; + return -1; + } + db = NULL; + + return 0; +} + +/* + * Add directory entry. + */ +static int +sqlite_addDirEntry(struct direct *dp, dump_ino_t parent_ino) +{ + int rc; + char *errMsg = NULL; + char buffer[2000]; + const char *tail; + + if (db == NULL) { + return -1; + } + + // don't include backlink. + if (!strcmp(dp->d_name, "..")) { + return -1; + } + + sqlite3_bind_int(stmt, 1, dp->d_ino); + sqlite3_bind_int(stmt, 2, parent_ino); + sqlite3_bind_int(stmt, 3, dp->d_type); + sqlite3_bind_text(stmt, 4, dp->d_name, strlen(dp->d_name), SQLITE_TRANSIENT); + + while ((rc = sqlite3_step(stmt)) == SQLITE_BUSY) { + continue; + } + + if (rc != SQLITE_DONE) { + int i; + msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db)); + msg("(%s:%d) %s\n", __FILE__, __LINE__, sqlite3_sql(stmt)); + sqlite3_free(errMsg); + sqlite3_close(db); + db = NULL; + return -1; + } + + rc = sqlite3_reset(stmt); + if (rc != SQLITE_OK) { + msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db)); + msg(buffer); + sqlite3_free(errMsg); + sqlite3_close(db); + db = NULL; + return -1; + } + + rc = sqlite3_clear_bindings(stmt); + if (rc != SQLITE_OK) { + msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db)); + msg(buffer); + sqlite3_free(errMsg); + sqlite3_close(db); + db = NULL; + return -1; + } + + return 0; +} + +/* + * Add inode entry. + */ +static int +sqlite_addInode(struct dinode *dp, dump_ino_t ino, int metadata_only) +{ + int rc; + char *errMsg = NULL; + char buffer[2000]; + char ats[40]; + char mts[40]; + char cts[40]; + time_t t; + + if (db == NULL) { + return -1; + } + + ats[0] = 0; + mts[0] = 0; + cts[0] = 0; + + t = dp->di_atime; + strftime(ats, sizeof ats, "%FT%T", gmtime(&t)); + t = dp->di_mtime; + strftime(mts, sizeof ats, "%FT%T", gmtime(&t)); + t = dp->di_ctime; + strftime(cts, sizeof ats, "%FT%T", gmtime(&t)); + + // xattr: dp->di_extraisize != 0 + // acl: dp->di_file_acl (inode?) + // se linux?... + + snprintf(buffer, sizeof buffer, + "insert into inode(backup_id, ino, is_deleted, mode, nlink, uid, gid, rdev, size, atime, mtime, ctime, has_xattr, has_acl, volume, recno) values(1, %d, '%s', %d, %d, %d, %d, %d, %d, '%s', '%s', '%s', '%s', '%s', %d, %d)\n", + ino, "N", dp->di_mode, dp->di_nlink, dp->di_uid, dp->di_gid, dp->di_rdev, dp->di_size, ats, mts, cts, (dp->di_extraisize == 0) ? "N" : "Y", (dp->di_file_acl != 0) ? "Y" : "N", spcl.c_volume, spcl.c_tapea); + + EXEC(db, buffer); + + return 0; +} + +/* + * + */ +static int +foo() +{ + return 0; +} + +/* + * + */ +static int +sqlite_foo() +{ + return 0; +} + +/* + * + */ +static int +sqlite_writerec(const void *dp, int isspcl) +{ + return 0; +} + +/* + * + */ +static int +sqlite_openQfa() +{ + return 0; +} + +/* + * + */ +static int +sqlite_closeQfa() +{ + return 0; +} + +/* + * + */ +static int +sqlite_openQfaState(QFA_State *state) +{ + return 0; +} + +/* + * + */ +static int +sqlite_updateQfaState(QFA_State *state) +{ + return 0; +} + +/* + * + */ +static int +sqlite_updateQfa(QFA_State *state) +{ + return 0; +} + + +/* + * + */ +Indexer indexer_sqlite = +{ + NULL, + &sqlite_open, + &sqlite_close, + &sqlite_foo, + &sqlite_writerec, + &sqlite_addInode, + &sqlite_addDirEntry, + + &sqlite_openQfa, + &sqlite_closeQfa, + &sqlite_openQfaState, + &sqlite_updateQfa, + &sqlite_updateQfaState +}; + +//#endif diff --git a/config.h.in b/config.h.in index 71d6c7e..77b811f 100644 --- a/config.h.in +++ b/config.h.in @@ -63,6 +63,9 @@ /* Define to 1 if you have the `realpath' function. */ #undef HAVE_REALPATH +/* Define to 1 if you have the sqlite3 library. */ +#undef HAVE_SQLITE3 + /* Define to 1 if you have the header file. */ #undef HAVE_STDINT_H diff --git a/configure.in b/configure.in index 807df9b..70d7f68 100644 --- a/configure.in +++ b/configure.in @@ -562,6 +562,23 @@ else fi AC_SUBST(BZLIB) +dnl +dnl Check for sqlite3 headers and libraries +dnl +AC_CHECK_HEADER(sqlite3.h, [sqlite3_h=yes], [sqlite3_h=no], [-]) +AC_CHECK_LIB(sqlite3, sqlite3_initialize, [sqlite3_lib=yes], [sqlite3_lib=no]) +if test "$sqlite3_h" = yes -a "$sqlite3_lib" = yes; then + if test "$STATICZ" = yes; then + SQLITE3="-Wl,-Bstatic -lsqlite3 -Wl,-Bdynamic" + else + SQLITE3="-lsqlite3" + fi + AC_DEFINE([HAVE_SQLITE3],1,[Define this if you have sqlite3 library.]) +else + SQLITE3="" +fi +AC_SUBST(SQLITE3) + dnl dnl Check for library functions dnl diff --git a/dump/Makefile.in b/dump/Makefile.in index 9d312f9..1c79d07 100644 --- a/dump/Makefile.in +++ b/dump/Makefile.in @@ -1,4 +1,4 @@ -# $Id: Makefile.in,v 1.13 2011/06/10 13:07:29 stelian Exp $ +# $Id: Makefile.in,v 1.14 2011/06/10 13:41:41 stelian Exp $ top_srcdir= @top_srcdir@ srcdir= @srcdir@ @@ -9,7 +9,7 @@ top_builddir= .. INC= -I$(top_srcdir)/dump -I$(top_srcdir)/common ALL_CFLAGS= @CPPFLAGS@ @CFLAGS@ @CCOPTS@ -pipe $(OPT) $(GINC) $(INC) $(DEFS) @DUMPDEBUG@ ALL_LDFLAGS= @LDFLAGS@ @LDOPTS@ @STATIC@ -LIBS= $(GLIBS) @ZLIB@ @BZLIB@ @BLKID@ +LIBS= $(GLIBS) @ZLIB@ @BZLIB@ @BLKID@ @SQLITE3@ DEPLIBS= ../compat/lib/libcompat.a PROG= dump @@ -18,7 +18,8 @@ LINKS= ${SBINDIR}/dump ${SBINDIR}/rdump SRCS= itime.c main.c optr.c tape.c traverse.c unctime.c OBJS= itime.o main.o optr.o tape.o traverse.o unctime.o \ ../common/dumprmt.o ../common/transformation_null.o ../common/transformation_lzo.o \ - ../common/transformation_zlib.o ../common/transformation_bzlib.o + ../common/transformation_zlib.o ../common/transformation_bzlib.o \ + ../common/legacy_indexer.o ../common/sqlite.o MAN8= dump.8 RMAN8= rdump.8 diff --git a/dump/dump.h b/dump/dump.h index e525f3d..1e3866e 100644 --- a/dump/dump.h +++ b/dump/dump.h @@ -5,7 +5,7 @@ * Stelian Pop , 1999-2000 * Stelian Pop - Alcôve , 2000-2002 * - * $Id: dump.h,v 1.51 2011/06/10 13:07:29 stelian Exp $ + * $Id: dump.h,v 1.52 2011/06/10 13:41:41 stelian Exp $ */ /*- @@ -75,8 +75,6 @@ extern char *tapeprefix; /* prefix of the tape file */ extern char *dumpdates; /* name of the file containing dump date information*/ extern char lastlevel[NUM_STR_SIZE];/* dump level of previous dump */ extern char level[NUM_STR_SIZE];/* dump level of this dump */ -extern int Afile; /* archive file descriptor */ -extern int AfileActive; /* Afile flag */ extern int zipflag; /* which compression method */ extern int uflag; /* update flag */ extern int mflag; /* dump metadata only if possible flag */ @@ -119,15 +117,6 @@ extern int tp_bshift; /* log2(TP_BSIZE) */ extern dump_ino_t volinfo[]; /* which inode on which volume archive info */ extern Transformation *transformation; -#ifdef USE_QFA -#define QFA_MAGIC "495115637697" -#define QFA_VERSION "1.0" -extern int gTapeposfd; -extern char *gTapeposfile; -extern char gTps[255]; -extern int32_t gThisDumpDate; -#endif /* USE_QFA */ - #ifndef __P #include #endif @@ -287,5 +276,6 @@ extern void endgrent(); extern void exit(); extern off_t lseek(); extern const char *strerror(); + #endif diff --git a/dump/main.c b/dump/main.c index fa50044..20c2ad2 100644 --- a/dump/main.c +++ b/dump/main.c @@ -37,7 +37,7 @@ #ifndef lint static const char rcsid[] = - "$Id: main.c,v 1.98 2011/06/10 13:07:29 stelian Exp $"; + "$Id: main.c,v 1.99 2011/06/10 13:41:41 stelian Exp $"; #endif /* not lint */ #include @@ -85,6 +85,7 @@ static const char rcsid[] = #include "bylabel.h" #include "transformation.h" +#include "indexer.h" #ifndef SBOFF #define SBOFF (SBLOCK * DEV_BSIZE) @@ -151,13 +152,6 @@ int dev_bshift; /* log2(dev_bsize) */ int tp_bshift; /* log2(TP_BSIZE) */ dump_ino_t volinfo[TP_NINOS];/* which inode on which volume archive info */ -#ifdef USE_QFA -int gTapeposfd; -char *gTapeposfile; -char gTps[255]; -int32_t gThisDumpDate; -#endif /* USE_QFA */ - struct dumptime *dthead; /* head of the list version */ int nddates; /* number of records (might be zero) */ int ddates_in; /* we have read the increment file */ @@ -200,6 +194,8 @@ static void incompat_flags __P((int, char, char)); static char* iexclude_bitmap = NULL; /* the inode exclude bitmap */ static unsigned int iexclude_bitmap_bytes = 0; /* size of bitmap in bytes */ +Indexer *indexer = &indexer_legacy; + int main(int argc, char *argv[]) { @@ -251,10 +247,6 @@ main(int argc, char *argv[]) obsolete(&argc, &argv); -#ifdef USE_QFA - gTapeposfd = -1; -#endif /* USE_QFA */ - while ((ch = getopt(argc, argv, "0123456789A:aB:b:cd:D:e:E:f:F:h:I:" #ifdef HAVE_BZLIB @@ -415,7 +407,7 @@ main(int argc, char *argv[]) #ifdef USE_QFA case 'Q': /* create tapeposfile */ - gTapeposfile = optarg; + //gTapeposfile = optarg; // FIXME - communicate filename to indexer. tapepos = 1; break; #endif /* USE_QFA */ @@ -699,9 +691,6 @@ main(int argc, char *argv[]) msg("Date of this level %s dump: %s", level, ctime4(&spcl.c_date)); -#ifdef USE_QFA - gThisDumpDate = spcl.c_date; -#endif if (spcl.c_ddate) msg("Date of last level %s dump: %s", lastlevel, ctime4(&spcl.c_ddate)); @@ -933,22 +922,7 @@ main(int argc, char *argv[]) } #ifdef USE_QFA - if (tapepos) { - msg("writing QFA positions to %s\n", gTapeposfile); - if ((gTapeposfd = open(gTapeposfile, - O_WRONLY|O_CREAT|O_TRUNC, - S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP - | S_IROTH | S_IWOTH)) < 0) - quit("can't open tapeposfile\n"); - /* print QFA-file header */ - snprintf(gTps, sizeof(gTps), "%s\n%s\n%ld\n\n", QFA_MAGIC, QFA_VERSION, (unsigned long)spcl.c_date); - gTps[sizeof(gTps) - 1] = '\0'; - if (write(gTapeposfd, gTps, strlen(gTps)) != (ssize_t)strlen(gTps)) - quit("can't write tapeposfile\n"); - sprintf(gTps, "ino\ttapeno\ttapepos\n"); - if (write(gTapeposfd, gTps, strlen(gTps)) != (ssize_t)strlen(gTps)) - quit("can't write tapeposfile\n"); - } + indexer->openQfa(); #endif /* USE_QFA */ /* @@ -1023,6 +997,8 @@ main(int argc, char *argv[]) tend_writing = time(NULL); spcl.c_type = TS_END; + indexer->foo(); + if (Afile >= 0) { volinfo[1] = ROOTINO; memcpy(spcl.c_inos, volinfo, TP_NINOS * sizeof(dump_ino_t)); @@ -1069,8 +1045,8 @@ main(int argc, char *argv[]) spcl.c_tapea, tapekb, rate); } - if (Afile >= 0) - msg("Archiving dump to %s\n", Apath); + indexer->close(); + indexer->closeQfa(); broadcast("DUMP IS DONE!\7\7\n"); msg("DUMP IS DONE\n"); diff --git a/dump/tape.c b/dump/tape.c index e0b5411..bab3379 100644 --- a/dump/tape.c +++ b/dump/tape.c @@ -37,7 +37,7 @@ #ifndef lint static const char rcsid[] = - "$Id: tape.c,v 1.95 2011/06/10 13:07:29 stelian Exp $"; + "$Id: tape.c,v 1.96 2011/06/10 13:41:41 stelian Exp $"; #endif /* not lint */ #include @@ -93,6 +93,8 @@ int write(), read(); #include #include "dump.h" +#include "indexer.h" +#include "slave.h" int writesize; /* size of malloc()ed buffer for tape */ long lastspclrec = -1; /* tape block number of last written header */ @@ -119,24 +121,7 @@ static void enslave __P((void)); static void flushtape __P((void)); static void killall __P((void)); static void rollforward __P((void)); -#ifdef USE_QFA -static int GetTapePos __P((long long *)); -static int MkTapeString __P((struct s_spcl *, long long)); -#define FILESQFAPOS 20 -#endif -/* - * Concurrent dump mods (Caltech) - disk block reading and tape writing - * are exported to several slave processes. While one slave writes the - * tape, the others read disk blocks; they pass control of the tape in - * a ring via signals. The parent process traverses the filesystem and - * sends writeheader()'s and lists of daddr's to the slaves via pipes. - * The following structure defines the instruction packets sent to slaves. - */ -struct req { - ext2_loff_t dblk; - int count; -}; int reqsiz; struct slave_results { @@ -144,19 +129,7 @@ struct slave_results { ssize_t clen; /* compressed length */ }; -#define SLAVES 3 /* 1 slave writing, 1 reading, 1 for slack */ -struct slave { - int tapea; /* header number at start of this chunk */ - int count; /* count to next header (used for TS_TAPE */ - /* after EOT) */ - int inode; /* inode that we are currently dealing with */ - int fd; /* FD for this slave */ - int pid; /* PID for this slave */ - int sent; /* 1 == we've sent this slave requests */ - int firstrec; /* record number of this block */ - char (*tblock)[TP_BSIZE]; /* buffer for data blocks */ - struct req *req; /* buffer for requests */ -} slaves[SLAVES+1]; +struct slave slaves[SLAVES+1]; struct slave *slp; char (*nextblock)[TP_BSIZE]; @@ -176,9 +149,6 @@ static int ready2; /* have we reached the lock point without having */ /* received the SIGUSR2 signal from the prev slave? */ static sigjmp_buf jmpbuf2; /* where to jump to if we are ready when the */ /* SIGUSR2 arrives from the previous slave */ -#ifdef USE_QFA -static int gtperr = 0; -#endif /* * Determine if we can use Linux' clone system call. If so, call it @@ -280,27 +250,7 @@ writerec(const void *dp, int isspcl) *(union u_spcl *)(*(nextblock)) = *(union u_spcl *)dp; /* Need to write it to the archive file */ - if (! AfileActive && isspcl && (spcl.c_type == TS_END)) - AfileActive = 1; - if (AfileActive && Afile >= 0 && !(spcl.c_flags & DR_EXTATTRIBUTES)) { - /* When we dump an inode which is not a directory, - * it means we ended the archive contents */ - if (isspcl && (spcl.c_type == TS_INODE) && - ((spcl.c_dinode.di_mode & S_IFMT) != IFDIR)) - AfileActive = 0; - else { - union u_spcl tmp; - tmp = *(union u_spcl *)dp; - /* Write the record, _uncompressed_ */ - if (isspcl) { - tmp.s_spcl.c_flags &= ~DR_COMPRESSED; - mkchecksum(&tmp); - } - if (write(Afile, &tmp, TP_BSIZE) != TP_BSIZE) - msg("error writing archive file: %s\n", - strerror(errno)); - } - } + indexer->writerec(dp, isspcl); nextblock++; if (isspcl) @@ -1176,12 +1126,7 @@ doslave(int cmd, errcode_t retval; #endif #ifdef USE_QFA - long long curtapepos; - union u_spcl *uspclptr; - struct s_spcl *spclptr; - /* long maxntrecs = 300000000 / (ntrec * 1024); last tested: 50 000 000 */ - long maxntrecs = 50000; /* every 50MB */ - long cntntrecs = maxntrecs; + QFA_State qfa_state; #endif /* USE_QFA */ sigset_t set; @@ -1195,6 +1140,8 @@ doslave(int cmd, transformation->startDiskIOProcess(transformation); #endif /* HAVE_BLOCK_TRANSFORMATION */ + indexer->openQfaState(&qfa_state); + /* * Need our own seek pointer. */ @@ -1333,38 +1280,7 @@ doslave(int cmd, ready2 = 0; caught2 = 0; -#ifdef USE_QFA - if (gTapeposfd >= 0) { - int i; - int foundone = 0; - - for (i = 0; (i < ntrec) && !foundone; ++i) { - uspclptr = (union u_spcl *)&slp->tblock[i]; - spclptr = &uspclptr->s_spcl; - if ((spclptr->c_magic == NFS_MAGIC) && - (spclptr->c_type == TS_INODE) && - (spclptr->c_date == gThisDumpDate) && - !(spclptr->c_dinode.di_mode & S_IFDIR) && - !(spclptr->c_flags & DR_EXTATTRIBUTES) - ) { - foundone = 1; - /* if (cntntrecs >= maxntrecs) { only write every maxntrecs amount of data */ - cntntrecs = 0; - if (gtperr == 0) - gtperr = GetTapePos(&curtapepos); - /* if an error occured previously don't - * try again */ - if (gtperr == 0) { -#ifdef DEBUG_QFA - msg("inode %ld at tapepos %ld\n", spclptr->c_inumber, curtapepos); -#endif - gtperr = MkTapeString(spclptr, curtapepos); - } - /* } */ - } - } - } -#endif /* USE_QFA */ + indexer->updateQfa(&qfa_state); while (eot_count < 10 && size < bufsize) { #ifdef RDUMP @@ -1417,11 +1333,8 @@ doslave(int cmd, * Signal the next slave to go. */ (void) kill(nextslave, SIGUSR2); -#ifdef USE_QFA - if (gTapeposfd >= 0) { - cntntrecs += ntrec; - } -#endif /* USE_QFA */ + + indexer->updateQfaState(&qfa_state); } #ifdef HAVE_BLOCK_TRANSFORMATION @@ -1484,64 +1397,3 @@ SetLogicalPos(void) return err; } */ - -#ifdef USE_QFA -#define LSEEK_GET_TAPEPOS 10 -#define LSEEK_GO2_TAPEPOS 11 -/* - * read the current tape position - */ -int -GetTapePos(long long *pos) -{ - int err = 0; - -#ifdef RDUMP - if (host) { - *pos = (long long) rmtseek((OFF_T)0, (int)LSEEK_GET_TAPEPOS); - err = *pos < 0; - } - else -#endif - { - if (magtapeout) { - long mtpos; - *pos = 0; - err = (ioctl(tapefd, MTIOCPOS, &mtpos) < 0); - *pos = (long long)mtpos; - } - else { - *pos = LSEEK(tapefd, 0, SEEK_CUR); - err = (*pos < 0); - } - } - if (err) { - err = errno; - msg("[%ld] error: %d (getting tapepos: %lld)\n", getpid(), - err, *pos); - return err; - } - return err; -} - -static int -MkTapeString(struct s_spcl *spclptr, long long curtapepos) -{ - int err = 0; - -#ifdef DEBUG_QFA - msg("inode %ld at tapepos %lld\n", spclptr->c_inumber, curtapepos); -#endif - - snprintf(gTps, sizeof(gTps), "%ld\t%d\t%lld\n", - (unsigned long)spclptr->c_inumber, - tapeno, - curtapepos); - gTps[sizeof(gTps) - 1] = '\0'; - if (write(gTapeposfd, gTps, strlen(gTps)) != (ssize_t)strlen(gTps)) { - err = errno; - warn("error writing tapepos file. (error %d)\n", errno); - } - return err; -} -#endif /* USE_QFA */ diff --git a/dump/traverse.c b/dump/traverse.c index 7bdfef1..f762cf4 100644 --- a/dump/traverse.c +++ b/dump/traverse.c @@ -37,7 +37,7 @@ #ifndef lint static const char rcsid[] = - "$Id: traverse.c,v 1.73 2011/06/10 13:07:29 stelian Exp $"; + "$Id: traverse.c,v 1.74 2011/06/10 13:41:41 stelian Exp $"; #endif /* not lint */ #include @@ -77,6 +77,7 @@ static const char rcsid[] = #include #include "dump.h" +#include "indexer.h" #define HASDUMPEDFILE 0x1 #define HASSUBDIRS 0x2 @@ -914,6 +915,8 @@ dumpino(struct dinode *dp, dump_ino_t ino, int metaonly) spcl.c_type = TS_INODE; spcl.c_count = 0; + indexer->addInode(dp, ino, metaonly); + if (metaonly && (dp->di_mode & S_IFMT)) { spcl.c_flags |= DR_METAONLY; spcl.c_count = 0; @@ -1101,6 +1104,8 @@ convert_dir(struct ext2_dir_entry *dirent, UNUSED(int offset), p->prev_offset = p->offset; p->offset += reclen; + //indexer->addDirEntry(dirent, offset, blocksize, buf, private); + return 0; } @@ -1204,6 +1209,8 @@ dumpdirino(struct dinode *dp, dump_ino_t ino) spcl.c_type = TS_ADDR; } + indexer->addInode(dp, ino, 0); + (void)free(cdc.buf); dump_xattr(ino, dp); }