SQLlite and QFA overhaul.
authorStelian Pop <stelian@popies.net>
Fri, 10 Jun 2011 13:41:41 +0000 (13:41 +0000)
committerStelian Pop <stelian@popies.net>
Fri, 10 Jun 2011 13:41:41 +0000 (13:41 +0000)
15 files changed:
CHANGES
Makefile.in
common/Makefile.in
common/indexer.h [new file with mode: 0644]
common/indexer_test.c [new file with mode: 0644]
common/legacy_indexer.c [new file with mode: 0644]
common/slave.h [new file with mode: 0644]
common/sqlite_indexer.c [new file with mode: 0644]
config.h.in
configure.in
dump/Makefile.in
dump/dump.h
dump/main.c
dump/tape.c
dump/traverse.c

diff --git a/CHANGES b/CHANGES
index a3f339c2f3742560c27cf0eafba91fa7be9018b2..32d742cdcd2f70364ad0c62c8747090a06b92469 100644 (file)
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,4 @@
-$Id: CHANGES,v 1.329 2011/06/10 13:07:24 stelian Exp $
+$Id: CHANGES,v 1.330 2011/06/10 13:41:41 stelian Exp $
 
 Changes between versions 0.4b44 and 0.4b45 (released ?????????????)
 ===================================================================
@@ -11,6 +11,10 @@ Changes between versions 0.4b44 and 0.4b45 (released ?????????????)
        compression/encryption code as plugins. Thanks to
        Bear Giles <bgiles@coyotesong.com> for the patch.
 
+3.     Add sqlite index support and reorganize the QFA code
+       as plugins. Thanks to Bear Giles <bgiles@coyotesong.com> for
+       the patch.
+
 Changes between versions 0.4b43 and 0.4b44 (released June 10, 2011)
 ===================================================================
 
index f8f64f9519a4744ecf979238fd819152579c7271..2dbd90ae215cd08b81679fd72807b6eecd453a25 100644 (file)
@@ -5,7 +5,7 @@ top_builddir=   .
 @MCONFIG@
 
 RM=            @RM@
-SUBDIRS=       compat/lib compat/include common dump restore testing @RMTDIR@
+SUBDIRS=       compat/lib compat/include common dump restore @RMTDIR@
 
 all clean install dep depend realclean distclean::
        for i in $(SUBDIRS); do \
index 2c03400d9a9e1c4248f070c2731250a14c961339..13f43bd47bb5be16b0cef20a40a975c88c4c2ab0 100644 (file)
@@ -1,4 +1,4 @@
-# $Id: Makefile.in,v 1.6 2011/06/10 13:07:29 stelian Exp $
+# $Id: Makefile.in,v 1.7 2011/06/10 13:41:41 stelian Exp $
 
 top_srcdir=    @top_srcdir@
 srcdir=                @srcdir@
@@ -6,16 +6,29 @@ top_builddir= ..
 
 @MCONFIG@
 
-INC=           -I$(top_srcdir)/dump -I$(top_srcdir)/common
+INC=           -I. -I$(top_srcdir)/dump -I$(top_srcdir)/compat/include
 ALL_CFLAGS=    @CPPFLAGS@ @CFLAGS@ @CCOPTS@ -pipe $(OPT) $(GINC) $(INC) $(DEFS) @DUMPDEBUG@
 ALL_LDFLAGS=   @LDFLAGS@ @LDOPTS@ @STATIC@
-SRCS=          dumprmt.c transformation_null.c transformation_lzo.c transformation_zlib.c transformation_bzlib.c transformation_ssl.c
-OBJS=          dumprmt.o transformation_null.o transformation_lzo.o transformation_zlib.o transformation_bzlib.o transformation_ssl.o
+LIBS=          $(GLIBS) @ZLIB@ @BZLIB@ @BLKID@ @SSLLIB@ @SQLITE3@
+
+SRCS=          dumprmt.c transformation_null.c transformation_lzo.c transformation_zlib.c transformation_bzlib.c \
+    transformation_ssl.c legacy_indexer.c sqlite_indexer.c
+OBJS=          dumprmt.o transformation_null.o transformation_lzo.o transformation_zlib.o transformation_bzlib.o \
+    transformation_ssl.o legacy_indexer.o sqlite_indexer.o
 
 .c.o:
        $(CC) -c $(ALL_CFLAGS) $< -o $@
 
-all::          $(OBJS)
+all::          $(OBJS) transform_test indexer_test
+
+transform_test:        transform_test.o $(OBJS) $(DEPLIBS)
+       $(LD) $(ALL_LDFLAGS) -o transform_test transform_test.o \
+               transformation_null.o transformation_lzo.o transformation_zlib.o \
+               transformation_bzlib.o transformation_ssl.o  $(LIBS)
+
+indexer_test:  indexer_test.o $(OBJS) $(DEPLIBS)
+       $(LD) $(ALL_LDFLAGS) -o indexer_test indexer_test.o legacy_indexer.o \
+       sqlite_indexer.o dumprmt.o $(LIBS)
 
 install::
 
diff --git a/common/indexer.h b/common/indexer.h
new file mode 100644 (file)
index 0000000..102251b
--- /dev/null
@@ -0,0 +1,54 @@
+#include <config.h>
+#include <protocols/dumprestore.h>
+#include <compatlfs.h>
+
+#ifndef _ARCHIVE_H
+#define _ARCHIVE_H 1
+
+#ifdef USE_QFA
+#define        QFA_MAGIC       "495115637697"
+#define QFA_VERSION    "1.0"
+#endif /* USE_QFA */
+
+#ifndef __P
+#include <sys/cdefs.h>
+#endif
+
+/* state information when writing QFA file. */
+typedef struct qfa_state {
+       long long curtapepos;
+       /* long         maxntrecs = 300000000 / (ntrec * 1024);  last tested: 50 000 000 */
+       long            maxntrecs;
+       long            cntntrecs;
+} QFA_State;
+
+/* methods used to write 'Indexer' file. */
+typedef struct indexer {
+       struct indexer *next;
+       int (*open) __P((const char *filename, int mode));
+       int (*close) __P((void));
+       int (*foo) __P((void)); // what does this function do?
+       int (*writerec) __P((const void *dp, int isspcl));
+       int (*addInode) __P((struct dinode *dp, dump_ino_t ino, int metaonly));  // dump entry
+       //int (*addDirEntry) __P((struct ext2_dir_entry *dirent, UNUSED(int offset),  // dump inode
+       //          UNUSED(int blocksize), UNUSED(char *buf), void *private));
+       int (*addDirEntry) __P((struct direct *dp, dump_ino_t parent_ino));
+
+       int (*openQfa) __P(());
+       int (*closeQfa) __P(());
+       int (*openQfaState) __P((QFA_State *state));
+       int (*updateQfa) __P((QFA_State *state));
+       int (*updateQfaState) __P((QFA_State *state));
+} Indexer;
+
+extern Indexer *indexer;
+
+extern Indexer indexer_legacy;
+extern Indexer indexer_sqlite;
+
+//extern Indexer *indexer_legacy_factory();
+
+//#ifdef HAVE_SQLITE3
+//extern Indexer *indexer_sqlite_factory();
+//#endif /* HAVE_SQLITE3 */
+#endif
diff --git a/common/indexer_test.c b/common/indexer_test.c
new file mode 100644 (file)
index 0000000..003198b
--- /dev/null
@@ -0,0 +1,221 @@
+#include <config.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <dirent.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/param.h>
+#include <limits.h>
+
+#include <bsdcompat.h>
+#include <protocols/dumprestore.h>
+
+#ifdef __linux__
+#include <linux/types.h>
+#ifdef HAVE_EXT2FS_EXT2_FS_H
+#include <ext2fs/ext2_fs.h>
+#else
+#include <linux/ext2_fs.h>
+#endif
+#include <ext2fs/ext2fs.h>
+#elif defined sunos
+#include <sys/vnode.h>
+#include <ufs/inode.h>
+#include <ufs/fs.h>
+#else
+#include <ufs/ufs/dinode.h>
+#include <ufs/ffs/fs.h>
+#endif
+#include <uuid/uuid.h>
+
+#include "indexer.h"
+#include "slave.h"
+
+int notify;
+dump_ino_t volinfo[TP_NINOS];
+
+int nddates;
+struct dumpdates **ddatev;
+char *host = NULL;
+int tapefd;
+int tapeno;
+
+struct slave slaves[SLAVES+1];
+struct slave *slp;
+
+union u_spcl u_spcl;
+
+#ifdef __linux__
+struct struct_ext2_filsys test_fs;
+struct ext2_super_block test_super;
+ext2_filsys fs = &test_fs;
+#else
+struct fs *sblock; /* the file system super block */
+char sblock_buf[MAXBSIZE];
+#endif
+
+/* for dumprmt.c */
+int dokerberos = 0;
+int ntrec;
+int abortifconnerr = 0;
+
+/*
+ *
+ */
+void
+msg(const char *fmt, ...)
+{
+       va_list ap;
+       va_start(ap, fmt);
+       vprintf(fmt, ap);
+       va_end(ap);
+}
+
+/*
+ * print error message and quit.
+ */
+int
+quit(const char *msg)
+{
+       printf("%s\n", msg);
+       exit(1);
+}
+
+/*
+ * Compute and fill in checksum information. (from dump/traverse.c)
+ */
+void
+mkchecksum(union u_spcl *tmpspcl)
+{
+       int32_t sum, cnt, *lp;
+
+       tmpspcl->s_spcl.c_checksum = 0;
+       lp = (int32_t *)&tmpspcl->s_spcl;
+       sum = 0;
+       cnt = sizeof(union u_spcl) / (4 * sizeof(int32_t));
+       while (--cnt >= 0) {
+               sum += *lp++;
+               sum += *lp++;
+               sum += *lp++;
+               sum += *lp++;
+       }
+       tmpspcl->s_spcl.c_checksum = CHECKSUM - sum;
+}
+
+/*
+ *
+ */
+int
+dump_inode(Indexer *indexer, struct stat *buf)
+{
+       struct dinode dinode;
+
+       memset(&dinode, 0, sizeof dinode);
+       dinode.di_mode = buf->st_mode;
+       dinode.di_nlink = buf->st_nlink;
+       dinode.di_uid = buf->st_uid;
+       dinode.di_gid = buf->st_gid;
+       dinode.di_rdev = buf->st_rdev;
+       dinode.di_size = buf->st_size;
+       dinode.di_atime = buf->st_atime;
+       dinode.di_mtime = buf->st_mtime;
+       dinode.di_ctime = buf->st_ctime;
+       dinode.di_blocks = buf->st_blocks;
+
+       spcl.c_tapea++;
+
+       indexer->addInode(&dinode, buf->st_ino, 0);
+}
+
+/*
+ *
+ */
+int
+dump_walk(Indexer *indexer, const char *root, int first)
+{
+       DIR *dirp;
+       struct dirent *dp;
+       char pathname[MAXPATHLEN];
+       struct stat bufroot, bufentry;
+       struct direct direct;
+
+       dirp = opendir(root);
+       if (dirp == NULL) {
+               printf("%s:%d: trying to open '%s'\n", __FILE__, __LINE__, root);
+               perror(root);
+               return -1;
+       }
+
+       stat(root, &bufroot);
+
+       //if (first) {
+       //      spcl.c_volume = 1;
+       //      spcl.c_tapea = 1;
+       //      indexer->addDirEntry(&direct, direct.d_ino);
+       //      dump_inode(indexer, &bufroot);
+       //}
+       
+       while ((dp = readdir(dirp)) != NULL) {
+               direct.d_ino = dp->d_ino;
+               direct.d_reclen = dp->d_reclen;
+               direct.d_type = dp->d_type;
+               direct.d_namlen = strlen(dp->d_name);
+               memcpy(direct.d_name, dp->d_name, MAXNAMLEN + 1);
+               direct.d_name[MAXNAMLEN] = '\0';
+
+               snprintf(pathname, sizeof pathname, "%s/%s", root, dp->d_name);
+               stat(pathname, &bufentry);
+               if (strcmp(dp->d_name, "..")) {
+                       indexer->addDirEntry(&direct, bufentry.st_ino);
+                       dump_inode(indexer, &bufentry);
+                       if (S_ISDIR(bufentry.st_mode)) {
+                               if (strcmp(dp->d_name, ".")) {
+                                       dump_walk(indexer, pathname, 0);
+                               }
+                       }
+               }
+       }
+       closedir(dirp);
+}
+
+/*
+ *
+ */
+int
+test_indexer(Indexer *indexer, const char *filename, const char *path)
+{
+       struct direct dp;
+       struct stat buf;
+
+#ifdef __linux__
+       test_fs.super = &test_super;
+       uuid_generate_time((unsigned char *)&test_fs.super->s_uuid);
+#endif
+
+       indexer->open(filename, 1);
+
+       dump_walk(indexer, path, 1);
+       indexer->close();
+
+       indexer->open(filename, 0);
+       indexer->close();
+}
+
+/*
+ *
+ */
+int
+main(int argc, char **argv)
+{
+       if (argc == 1) {
+//             printf("usage: %s partition\n", argv[0]);
+//             exit(1);
+               argv[1] = "/usr/include";
+       }
+
+//     test_indexer(&indexer_legacy, "/tmp/archive.data", argv[1]);
+
+#ifdef HAVE_SQLITE3
+       test_indexer(&indexer_sqlite, "/tmp/t.sqlite3", argv[1]);
+#endif /* HAVE_SQLITE3 */
+}
diff --git a/common/legacy_indexer.c b/common/legacy_indexer.c
new file mode 100644 (file)
index 0000000..677895a
--- /dev/null
@@ -0,0 +1,359 @@
+#include <config.h>
+#include <compatlfs.h>
+#include <ctype.h>
+#include <compaterr.h>
+#include <fcntl.h>
+#include <fstab.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+
+#include <sys/param.h>
+#include <sys/time.h>
+#include <time.h>
+#ifdef __linux__
+#include <linux/types.h>
+#include <sys/stat.h>
+#include <sys/mtio.h>
+#include <bsdcompat.h>
+#include <linux/fs.h>  /* for definition of BLKFLSBUF */
+#elif defined sunos
+#include <sys/vnode.h>
+
+#include <ufs/inode.h>
+#include <ufs/fs.h>
+#else
+#include <ufs/ufs/dinode.h>
+#include <ufs/ffs/fs.h>
+#endif
+
+#include <protocols/dumprestore.h>
+
+//#include "dump.h"
+#include "indexer.h"
+#include "slave.h"
+
+extern dump_ino_t volinfo[];  // TP_NINOS
+extern int tapeno;
+
+static int Afile = -1; /* archive file descriptor */
+static int AfileActive = 1;/* Afile flag */
+
+//extern char *Apath;
+char *Apath;
+
+#ifdef USE_QFA
+static int GetTapePos __P((long long *));
+static int MkTapeString __P((struct s_spcl *, long long));
+#define FILESQFAPOS    20
+
+int tapepos;
+int ntrec;             /* blocking factor on tape */
+int magtapeout;  /* is output a magnetic tape? */
+//extern int tapepos;
+//extern int ntrec;            /* blocking factor on tape */
+//extern int magtapeout;  /* is output a magnetic tape? */
+
+static int gtperr = 0;
+static int gTapeposfd;
+static char *gTapeposfile;
+static char gTps[255];
+static int32_t gThisDumpDate;
+#endif /* USE_QFA */
+
+void   msg __P((const char *fmt, ...));
+void   quit __P((const char *fmt, ...));
+
+extern char *host;
+extern int tapefd;
+
+/*
+ * Open the indexer file.
+ */
+static int
+legacy_open(const char *filename, int mode)
+{
+       if (filename == NULL) {
+               return 0;
+       }
+
+       if (mode == 0) {
+               if ((Afile = OPEN(filename, O_RDONLY)) < 0) {
+                       msg("Cannot open %s for reading: %s\n",
+                                       filename, strerror(errno));
+                       msg("The ENTIRE dump is aborted.\n");
+                       return -1;
+               }
+       } else if (mode == 1) {
+               if ((Afile = OPEN(filename, O_WRONLY|O_CREAT|O_TRUNC,
+                                  S_IRUSR | S_IWUSR | S_IRGRP |
+                                  S_IWGRP | S_IROTH | S_IWOTH)) < 0) {
+                       msg("Cannot open %s for writing: %s\n",
+                                       filename, strerror(errno));
+                       msg("The ENTIRE dump is aborted.\n");
+                       return -1;
+               }
+       }
+
+       return 0;
+}
+
+/*
+ * Close the indexer file.
+ */
+static int
+legacy_close()
+{
+       if (Afile >= 0)
+               msg("Archiving dump to %s\n", Apath);
+       return 0;
+}
+
+/*
+ * Write a record to the indexer file.
+ */
+static int
+legacy_writerec(const void *dp, int isspcl)
+{
+       if (! AfileActive && isspcl && (spcl.c_type == TS_END))
+               AfileActive = 1;
+       if (AfileActive && Afile >= 0 && !(spcl.c_flags & DR_EXTATTRIBUTES)) {
+               /* When we dump an inode which is not a directory,
+                * it means we ended the archive contents */
+               if (isspcl && (spcl.c_type == TS_INODE) &&
+                   ((spcl.c_dinode.di_mode & S_IFMT) != IFDIR))
+                       AfileActive = 0;
+               else {
+                       union u_spcl tmp;
+                       tmp = *(union u_spcl *)dp;
+                       /* Write the record, _uncompressed_ */
+                       if (isspcl) {
+                               tmp.s_spcl.c_flags &= ~DR_COMPRESSED;
+                               mkchecksum(&tmp);
+                       }
+                       if (write(Afile, &tmp, TP_BSIZE) != TP_BSIZE)
+                               msg("error writing archive file: %s\n",
+                               strerror(errno));
+               }
+       }
+       return 0;
+}
+
+/*
+ * I'm not sure what this is used for...
+ */
+static int
+legacy_foo()
+{
+       if (Afile >= 0) {
+               volinfo[1] = ROOTINO;
+               memcpy(spcl.c_inos, volinfo, TP_NINOS * sizeof(dump_ino_t));
+               spcl.c_flags |= DR_INODEINFO;
+       }
+       return 0;
+}
+
+/*
+ * Dump inode
+ */
+static int
+legacy_addInode(struct dinode *dp, dump_ino_t ino, int metaonly)
+{
+       return 0;
+}
+
+/*
+ * Dump directory (dirent) entry
+ */
+static int
+legacy_addDirEntry(struct direct *dp, dump_ino_t parent_ino)
+{
+        return 0;
+}
+
+
+#define LSEEK_GET_TAPEPOS      10
+#define LSEEK_GO2_TAPEPOS      11
+
+/*
+ *
+ */
+static int
+legacy_openQfa()
+{
+#ifdef USE_QFA
+       gThisDumpDate = spcl.c_date;
+       if (tapepos) {
+               msg("writing QFA positions to %s\n", gTapeposfile);
+               if ((gTapeposfd = open(gTapeposfile,
+                                      O_WRONLY|O_CREAT|O_TRUNC,
+                                      S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP
+                                      | S_IROTH | S_IWOTH)) < 0)
+                       quit("can't open tapeposfile\n");
+               /* print QFA-file header */
+               snprintf(gTps, sizeof(gTps), "%s\n%s\n%ld\n\n", QFA_MAGIC, QFA_VERSION, (unsigned long)spcl.c_date);
+               gTps[sizeof(gTps) - 1] = '\0';
+               if (write(gTapeposfd, gTps, strlen(gTps)) != (ssize_t)strlen(gTps))
+                       quit("can't write tapeposfile\n");
+               sprintf(gTps, "ino\ttapeno\ttapepos\n");
+               if (write(gTapeposfd, gTps, strlen(gTps)) != (ssize_t)strlen(gTps))
+                       quit("can't write tapeposfile\n");
+       }
+#endif /* USE_QFA */
+       return 0;
+}
+
+/*
+ *
+ */
+static int
+legacy_closeQfa()
+{
+       return 0;
+}
+
+/*
+ *
+ */
+static int
+legacy_openQfaState(QFA_State *s)
+{
+       s->curtapepos = 0;
+       s->maxntrecs = 50000;   /* every 50MB */
+       s->cntntrecs = s->maxntrecs;
+       return 0;
+}
+
+/*
+ *
+ */
+static int
+legacy_updateQfaState(QFA_State *s)
+{
+       if (gTapeposfd >= 0) {
+               s->cntntrecs += ntrec;
+       }
+
+       return 0;
+}
+
+
+/*
+ *
+ */
+static int
+legacy_updateQfa(QFA_State *s)
+{
+#ifdef USE_QFA
+       union u_spcl *uspclptr;
+       struct s_spcl *spclptr;
+
+       if (gTapeposfd >= 0) {
+               int i;
+               int foundone = 0;
+
+               for (i = 0; (i < ntrec) && !foundone; ++i) {
+                       uspclptr = (union u_spcl *)&slp->tblock[i];
+                       spclptr = &uspclptr->s_spcl;
+                       if ((spclptr->c_magic == NFS_MAGIC) && 
+                                               (spclptr->c_type == TS_INODE) &&
+                                               (spclptr->c_date == gThisDumpDate) &&
+                                               !(spclptr->c_dinode.di_mode & S_IFDIR) &&
+                                               !(spclptr->c_flags & DR_EXTATTRIBUTES)
+                                       ) {
+                               foundone = 1;
+                               /* if (s->cntntrecs >= s->maxntrecs) {   only write every maxntrecs amount of data */
+                                       s->cntntrecs = 0;
+                                       if (gtperr == 0) 
+                                               gtperr = GetTapePos(&s->curtapepos);
+                                       /* if an error occured previously don't
+                                        * try again */
+                                       if (gtperr == 0) {
+#ifdef DEBUG_QFA
+                                               msg("inode %ld at tapepos %ld\n", spclptr->c_inumber, s->curtapepos);
+#endif
+                                               gtperr = MkTapeString(spclptr, s->curtapepos);
+                                       }
+                               /* } */
+                       }
+               }
+       }
+#endif /* USE_QFA */
+       return 0;
+}
+
+#ifdef USE_QFA
+/*
+ * read the current tape position
+ */
+static int
+GetTapePos(long long *pos)
+{
+       int err = 0;
+
+#ifdef RDUMP
+       if (host) {
+               *pos = (long long) rmtseek((OFF_T)0, (int)LSEEK_GET_TAPEPOS);
+               err = *pos < 0;
+       }
+       else 
+#endif
+       {
+       if (magtapeout) {
+               long mtpos;
+               *pos = 0;
+               err = (ioctl(tapefd, MTIOCPOS, &mtpos) < 0);
+               *pos = (long long)mtpos;
+       }
+       else {
+               *pos = LSEEK(tapefd, 0, SEEK_CUR);
+               err = (*pos < 0);
+       }
+       }
+       if (err) {
+               err = errno;
+               msg("[%ld] error: %d (getting tapepos: %lld)\n", getpid(), 
+                       err, *pos);
+               return err;
+       }
+       return err;
+}
+
+static int 
+MkTapeString(struct s_spcl *spclptr, long long curtapepos)
+{
+       int     err = 0;
+
+#ifdef DEBUG_QFA
+       msg("inode %ld at tapepos %lld\n", spclptr->c_inumber, curtapepos);
+#endif
+
+       snprintf(gTps, sizeof(gTps), "%ld\t%d\t%lld\n", 
+                (unsigned long)spclptr->c_inumber, 
+                tapeno, 
+                curtapepos);
+       gTps[sizeof(gTps) - 1] = '\0';
+       if (write(gTapeposfd, gTps, strlen(gTps)) != (ssize_t)strlen(gTps)) {
+               err = errno;
+       warn("error writing tapepos file. (error %d)\n", errno);
+       }
+       return err;
+}
+#endif /* USE_QFA */
+
+Indexer indexer_legacy = {
+       NULL,
+       &legacy_open,
+       &legacy_close,
+       &legacy_foo,
+       &legacy_writerec,
+       &legacy_addInode,
+       &legacy_addDirEntry,
+       &legacy_openQfa,
+       &legacy_closeQfa,
+       &legacy_updateQfa,
+       &legacy_updateQfaState
+};
diff --git a/common/slave.h b/common/slave.h
new file mode 100644 (file)
index 0000000..cd05d6d
--- /dev/null
@@ -0,0 +1,39 @@
+
+/*
+ * Structures pulled from dump/tape.c. We need it here so the 'archive' code can
+ * determine where we are in the tape.
+ */
+#ifndef _SLAVE_H
+#define _SLAVE_H 1
+
+/*
+ * Concurrent dump mods (Caltech) - disk block reading and tape writing
+ * are exported to several slave processes.  While one slave writes the
+ * tape, the others read disk blocks; they pass control of the tape in
+ * a ring via signals. The parent process traverses the filesystem and
+ * sends writeheader()'s and lists of daddr's to the slaves via pipes.
+ * The following structure defines the instruction packets sent to slaves.
+ */
+struct req {
+       ext2_loff_t dblk;
+       int count;
+};
+
+#define SLAVES 3               /* 1 slave writing, 1 reading, 1 for slack */
+
+struct slave {
+       int tapea;              /* header number at start of this chunk */
+       int count;              /* count to next header (used for TS_TAPE */
+                               /* after EOT) */
+       int inode;              /* inode that we are currently dealing with */
+       int fd;                 /* FD for this slave */
+       int pid;                /* PID for this slave */
+       int sent;               /* 1 == we've sent this slave requests */
+       int firstrec;           /* record number of this block */
+       char (*tblock)[TP_BSIZE]; /* buffer for data blocks */
+       struct req *req;        /* buffer for requests */
+};
+
+extern struct slave *slp;
+
+#endif
diff --git a/common/sqlite_indexer.c b/common/sqlite_indexer.c
new file mode 100644 (file)
index 0000000..4008341
--- /dev/null
@@ -0,0 +1,511 @@
+#include <config.h>
+#include <compatlfs.h>
+#include <ctype.h>
+#include <compaterr.h>
+#include <fcntl.h>
+#include <fstab.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <mntent.h>
+#include <stdarg.h>
+
+#include <sys/param.h>
+#include <sys/time.h>
+#include <time.h>
+#ifdef __linux__
+#include <linux/types.h>
+#ifdef HAVE_EXT2FS_EXT2_FS_H
+#include <ext2fs/ext2_fs.h>
+#else
+#include <linux/ext2_fs.h>
+#endif
+#include <ext2fs/ext2fs.h>
+#include <sys/stat.h>
+#include <bsdcompat.h>
+#include <linux/fs.h>  /* for definition of BLKFLSBUF */
+#elif defined sunos
+#include <sys/vnode.h>
+
+#include <ufs/inode.h>
+#include <ufs/fs.h>
+#else
+#include <ufs/ufs/dinode.h>
+#include <ufs/ffs/fs.h>
+#endif
+
+#include <protocols/dumprestore.h>
+
+//#include "dump.h"
+#include "indexer.h"
+#include "pathnames.h"
+#include "bylabel.h"
+
+extern int tapeno;
+extern dump_ino_t volinfo[]; // TP_NINOS
+
+extern void msg __P((const char *fmt,...));
+
+#ifdef __linux__
+extern ext2_filsys fs;
+#else /* __linux__ */
+extern struct fs *sblock;
+#endif /* __linux__ */
+
+//#ifdef HAVE_SQLITE3
+#include <sqlite3.h>
+#include <uuid/uuid.h>
+
+static sqlite3 *db;
+
+// convenience macro that handles a basic sql statement.
+#define EXEC(db, buffer) \
+       rc = sqlite3_exec(db, buffer, NULL, 0, &errMsg); \
+       if (rc != SQLITE_OK) { \
+               msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db)); \
+               msg(buffer); \
+               sqlite3_free(errMsg); \
+               sqlite3_close(db); \
+               db = NULL; \
+               return -1; \
+       }
+
+/* prepared statement that allows us to use placeholder values. */
+/* it also acts as sentinel to indicate if this is read or write mode. */
+sqlite3_stmt *stmt = NULL;
+
+/*
+ * Ppopulate the 'backup' table.
+ */
+static int
+sqlite_populate_backup_table()
+{
+       int rc;
+       char *errMsg = NULL;
+       char buffer[2000];
+       char vol_uuid[40];
+       char ts[40];
+       char nts[40];
+       char fs_uuid[40];
+       time_t now;
+       const char *tail;
+       uuid_t uuid;
+
+       time(&now);
+
+       // populate 'backup table'. There will only be one entry.
+       uuid_generate_time(uuid);
+       uuid_unparse_lower(uuid, vol_uuid);
+       uuid_unparse_lower(fs->super->s_uuid, fs_uuid);
+       strftime(ts, sizeof ts, "%FT%T", gmtime((const time_t *) &spcl.c_date));
+       strftime(nts, sizeof nts, "%FT%T", gmtime((const time_t *) &now));
+       snprintf(buffer, sizeof buffer,
+               "insert into backup(backup_id, vol_uuid, dt, start_dt, fs_uuid, dev, label, host, level) values (1, '%s', '%s', '%s', '%s', ?, ?, ?, %d)",
+               vol_uuid, ts, nts, fs_uuid, spcl.c_level);
+
+//             spcl.c_dev, spcl.c_label, spcl.c_host,
+
+       // prepare statement.
+       rc = sqlite3_prepare_v2(db, buffer, strlen(buffer), &stmt, &tail);
+       if (rc != SQLITE_OK) {
+               msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db));
+               msg(buffer);
+               sqlite3_free(errMsg);
+               sqlite3_close(db);
+               db = NULL;
+               return -1;
+       }
+
+       sqlite3_bind_text(stmt, 1, spcl.c_dev, strlen(spcl.c_dev), SQLITE_TRANSIENT);
+       sqlite3_bind_text(stmt, 2, spcl.c_label, strlen(spcl.c_label), SQLITE_TRANSIENT);
+       sqlite3_bind_text(stmt, 3, spcl.c_host, strlen(spcl.c_host), SQLITE_TRANSIENT);
+
+       while ((rc = sqlite3_step(stmt)) == SQLITE_BUSY) {
+               continue;
+       }
+
+       if (rc != SQLITE_DONE) {
+               msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db));
+               msg("(%s:%d) %s\n", __FILE__, __LINE__, sqlite3_sql(stmt));
+               sqlite3_free(errMsg);
+               sqlite3_close(db);
+               db = NULL;
+               return -1;
+       }
+
+       rc = sqlite3_finalize(stmt);
+       if (rc != SQLITE_OK) {
+               msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db));
+               msg("(%s:%d) %s\n", __FILE__, __LINE__, sqlite3_sql(stmt));
+               sqlite3_free(errMsg);
+               sqlite3_close(db);
+               db = NULL;
+               return -1;
+       }
+
+       return 0;
+}
+
+/*
+ * Create database schema
+ */
+static int
+sqlite_create_schema()
+{
+       int rc;
+       char *errMsg = NULL;
+       char buffer[2000];
+       const char *tail;
+
+       // turn on foreign keys
+       snprintf(buffer, sizeof buffer, "pragma foreign_keys = ON");
+
+       EXEC(db, buffer);
+
+       // turn off synchronous access. we have bigger problems
+       // if we lose power in the middle of a backup.
+       snprintf(buffer, sizeof buffer, "pragma synchronous = OFF\n");
+
+       EXEC(db, buffer);
+
+       // create 'backup' table.
+       snprintf(buffer, sizeof buffer,
+               "create table backup(backup_id int primary key not null, vol_uuid char[36] not null, dt timestamp not null, start_dt timestamp not null, end_dt timestamp, fs_uuid char[36] not null, dev varchar[%1$d] not null, label varchar[%1$d] not null, host varchar[%1$d] not null, level int not null, blocks int, volumes int)\n", EXT2_NAME_LEN);
+
+       EXEC(db, buffer);
+
+       snprintf(buffer, sizeof buffer,
+               "create unique index i1 on backup(backup_id)\n");
+       EXEC(db, buffer);
+
+       // create 'entry' table. This contains information about
+       // each directory entry
+       snprintf(buffer, sizeof buffer,
+               "create table entry(backup_id int not null, ino int not null, parent_ino int not null, type int not null, depth int, name varchar[%d] not null, path varchar[%d], foreign key(backup_id) references backup(backup_id))\n", EXT2_NAME_LEN, 1024);
+
+       //      "create table entry(backup_id int not null, ino int not null, parent_ino int not null, type int not null, depth int, name varchar[%d] not null, path varchar[%d], foreign key(backup_id) references backup(backup_id), foreign key(parent_ino) references entry(ino))\n", EXT2_NAME_LEN, 1024);
+
+       EXEC(db, buffer);
+
+       // snprintf(buffer, sizeof buffer,
+       //      "create unique index i2 on entry(ino)\n");
+
+       // EXEC(db, buffer);
+
+       snprintf(buffer, sizeof buffer,
+               "create trigger t1 after insert on entry begin update entry set path = coalesce((select '.' where name = '.'), (select A.path || '/' || new.name from entry A where A.ino = new.parent_ino)) where rowid = new.rowid; end;\n");
+
+       EXEC(db, buffer);
+
+       // create 'inode' table. This contains information about each inode.
+       snprintf(buffer, sizeof buffer,
+               "create table inode(backup_id int not null, ino int not null, is_deleted char[1] not null default 'N', mode int not null, nlink int not null, uid int not null, gid int not null, rdev int not null, size int not null, atime timestamp not null, mtime timestamp not null, ctime timestamp not null, has_xattr char[1] not null default 'N', has_acl char[1] not null default 'N', volume int not null, recno int not null, foreign key(backup_id) references backup(backup_id))\n");
+
+       //      "create table inode(backup_id int not null, ino int not null, is_deleted char[1] not null default 'N', mode int not null, nlink int not null, uid int not null, gid int not null, rdev int not null, size int not null, atime timestamp not null, mtime timestamp not null, ctime timestamp not null, has_xattr char[1] not null default 'N', has_acl char[1] not null default 'N', volume int not null, recno int not null, foreign key(backup_id) references backup(backup_id), foreign key(ino) references entry(ino))\n");
+
+       EXEC(db, buffer);
+
+       sqlite_populate_backup_table();
+
+       // prepare statement for entry inserts.
+       snprintf(buffer, sizeof buffer,
+               "insert into entry(backup_id, ino, parent_ino, type, name) values(1, ?, ?, ?, ?)\n");
+
+       rc = sqlite3_prepare_v2(db, buffer, strlen(buffer), &stmt, &tail);
+       if (rc != SQLITE_OK) {
+               msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db));
+               msg(buffer);
+               sqlite3_free(errMsg);
+               sqlite3_close(db);
+               db = NULL;
+               return;
+       }
+}
+
+/*
+ * Open database.
+ */
+static int
+sqlite_open(const char *filename, int mode)
+{
+       if (mode == 0) {
+               if (sqlite3_open(filename, &db)) {
+                       msg("Unable to open database: %s\n", sqlite3_errmsg(db));
+                       db = NULL;
+                       return -1;
+               }
+               stmt = NULL;
+       } else if (mode == 1) {
+               unlink(filename);
+
+               /* sqlite will always create DB if it doesn't already exist. */
+               if (sqlite3_open(filename, &db)) {
+                       msg("Unable to open database: %s\n", sqlite3_errmsg(db));
+                       db = NULL;
+                       return -1;
+               }
+
+               sqlite_create_schema();
+       }
+
+       return 0;
+}
+
+/*
+ * Close database.
+ */
+static int
+sqlite_close()
+{
+       int rc;
+       char *errMsg = NULL;
+       char buffer[2000];
+       char uuid[40];
+       char ts[40];
+       time_t now;
+
+       if (db == NULL) {
+               msg("database is already closed!");
+               return -1;
+       }
+
+       time(&now);
+
+       /* if stmt is not null we're creating database */
+       if (stmt != NULL) {
+               // close prepared statement.
+               rc = sqlite3_finalize(stmt);
+               if (rc != SQLITE_OK) {
+                       msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db));
+                       msg("(%s:%d) %s\n", __FILE__, __LINE__, sqlite3_sql(stmt));
+                       sqlite3_free(errMsg);
+                       sqlite3_close(db);
+                       db = NULL;
+                       return -1;
+               }
+
+               // update 'tape header' with number of volumes and blocks.
+               uuid_unparse_lower(fs->super->s_uuid, uuid);
+               strftime(ts, sizeof ts, "%FT%T", gmtime((const time_t *) &now));
+               snprintf(buffer, sizeof buffer,
+                               "update backup set end_dt = '%s', blocks=%d, volumes=%d where fs_uuid='%s'",
+                               ts, spcl.c_tapea, spcl.c_volume, uuid);
+
+               EXEC(db, buffer);
+       }
+
+       // close database.
+       rc = sqlite3_close(db);
+       if (rc != SQLITE_OK) {
+               msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db));
+               msg(buffer);
+               sqlite3_free(errMsg);
+               db = NULL;
+               return -1;
+       }
+       db = NULL;
+
+       return 0;
+}
+
+/*
+ * Add directory entry.
+ */
+static int
+sqlite_addDirEntry(struct direct *dp, dump_ino_t parent_ino)
+{
+       int rc;
+       char *errMsg = NULL;
+       char buffer[2000];
+       const char *tail;
+
+       if (db == NULL) {
+               return -1;
+       }
+
+       // don't include backlink.
+       if (!strcmp(dp->d_name, "..")) {
+               return -1;
+       }
+
+       sqlite3_bind_int(stmt, 1, dp->d_ino);
+       sqlite3_bind_int(stmt, 2, parent_ino);
+       sqlite3_bind_int(stmt, 3, dp->d_type);
+       sqlite3_bind_text(stmt, 4, dp->d_name, strlen(dp->d_name), SQLITE_TRANSIENT);
+
+       while ((rc = sqlite3_step(stmt)) == SQLITE_BUSY) {
+               continue;
+       }
+
+       if (rc != SQLITE_DONE) {
+               int i;
+               msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db));
+               msg("(%s:%d) %s\n", __FILE__, __LINE__, sqlite3_sql(stmt));
+               sqlite3_free(errMsg);
+               sqlite3_close(db);
+               db = NULL;
+               return -1;
+       }
+
+       rc = sqlite3_reset(stmt);
+       if (rc != SQLITE_OK) {
+               msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db));
+               msg(buffer);
+               sqlite3_free(errMsg);
+               sqlite3_close(db);
+               db = NULL;
+               return -1;
+       }
+
+       rc = sqlite3_clear_bindings(stmt);
+       if (rc != SQLITE_OK) {
+               msg("(%s:%d) SQL error: %s\n", __FILE__, __LINE__, sqlite3_errmsg(db));
+               msg(buffer);
+               sqlite3_free(errMsg);
+               sqlite3_close(db);
+               db = NULL;
+               return -1;
+       }
+
+       return 0;
+}
+
+/*
+ * Add inode entry.
+ */
+static int
+sqlite_addInode(struct dinode *dp, dump_ino_t ino, int metadata_only)
+{
+       int rc;
+       char *errMsg = NULL;
+       char buffer[2000];
+       char ats[40];
+       char mts[40];
+       char cts[40];
+       time_t t;
+
+       if (db == NULL) {
+               return -1;
+       }
+
+       ats[0] = 0;
+       mts[0] = 0;
+       cts[0] = 0;
+
+       t = dp->di_atime;
+       strftime(ats, sizeof ats, "%FT%T", gmtime(&t));
+       t = dp->di_mtime;
+       strftime(mts, sizeof ats, "%FT%T", gmtime(&t));
+       t = dp->di_ctime;
+       strftime(cts, sizeof ats, "%FT%T", gmtime(&t));
+
+       // xattr: dp->di_extraisize != 0
+       // acl: dp->di_file_acl (inode?)
+       // se linux?...
+
+       snprintf(buffer, sizeof buffer,
+               "insert into inode(backup_id, ino, is_deleted, mode, nlink, uid, gid, rdev, size, atime, mtime, ctime, has_xattr, has_acl, volume, recno) values(1, %d, '%s', %d, %d, %d, %d, %d, %d, '%s', '%s', '%s', '%s', '%s', %d, %d)\n",
+               ino, "N", dp->di_mode, dp->di_nlink, dp->di_uid, dp->di_gid, dp->di_rdev, dp->di_size, ats, mts, cts, (dp->di_extraisize == 0) ? "N" : "Y", (dp->di_file_acl != 0) ? "Y" : "N", spcl.c_volume, spcl.c_tapea);
+
+       EXEC(db, buffer);
+
+       return 0;
+}
+
+/*
+ *
+ */
+static int
+foo()
+{
+       return 0;
+}
+
+/*
+ *
+ */
+static int
+sqlite_foo()
+{
+       return 0;
+}
+
+/*
+ *
+ */
+static int
+sqlite_writerec(const void *dp, int isspcl)
+{
+       return 0;
+}
+
+/*
+ *
+ */
+static int
+sqlite_openQfa()
+{
+       return 0;
+}
+
+/*
+ *
+ */
+static int
+sqlite_closeQfa()
+{
+       return 0;
+}
+
+/*
+ *
+ */
+static int
+sqlite_openQfaState(QFA_State *state)
+{
+       return 0;
+}
+
+/*
+ *
+ */
+static int
+sqlite_updateQfaState(QFA_State *state)
+{
+       return 0;
+}
+
+/*
+ *
+ */
+static int
+sqlite_updateQfa(QFA_State *state)
+{
+       return 0;
+}
+
+
+/*
+ *
+ */
+Indexer indexer_sqlite =
+{
+               NULL,
+               &sqlite_open,
+               &sqlite_close,
+               &sqlite_foo,
+               &sqlite_writerec,
+               &sqlite_addInode,
+               &sqlite_addDirEntry,
+
+               &sqlite_openQfa,
+               &sqlite_closeQfa,
+               &sqlite_openQfaState,
+               &sqlite_updateQfa,
+               &sqlite_updateQfaState
+};
+
+//#endif
index 71d6c7ee826e774152b1123371f09faefaa81dec..77b811f4ab960797832ac012299623661e8f9edc 100644 (file)
@@ -63,6 +63,9 @@
 /* Define to 1 if you have the `realpath' function. */
 #undef HAVE_REALPATH
 
+/* Define to 1 if you have the sqlite3 library. */
+#undef HAVE_SQLITE3
+
 /* Define to 1 if you have the <stdint.h> header file. */
 #undef HAVE_STDINT_H
 
index 807df9bc34c640f4bed3e1e01a856b2467030305..70d7f68be93e73b6128d5d063b8219ba1b8abd32 100644 (file)
@@ -562,6 +562,23 @@ else
 fi
 AC_SUBST(BZLIB)
 
+dnl
+dnl Check for sqlite3 headers and libraries
+dnl
+AC_CHECK_HEADER(sqlite3.h, [sqlite3_h=yes], [sqlite3_h=no], [-])
+AC_CHECK_LIB(sqlite3, sqlite3_initialize, [sqlite3_lib=yes], [sqlite3_lib=no])
+if test "$sqlite3_h" = yes -a "$sqlite3_lib" = yes; then
+    if test "$STATICZ" = yes; then
+        SQLITE3="-Wl,-Bstatic -lsqlite3 -Wl,-Bdynamic"
+    else
+        SQLITE3="-lsqlite3"
+    fi
+    AC_DEFINE([HAVE_SQLITE3],1,[Define this if you have sqlite3 library.])
+else
+    SQLITE3=""
+fi
+AC_SUBST(SQLITE3)
+
 dnl
 dnl Check for library functions
 dnl
index 9d312f9c8a1a129ff33c264981821750a5b73bd3..1c79d0738d8e87ce64fc8030eac046ea83255e07 100644 (file)
@@ -1,4 +1,4 @@
-# $Id: Makefile.in,v 1.13 2011/06/10 13:07:29 stelian Exp $
+# $Id: Makefile.in,v 1.14 2011/06/10 13:41:41 stelian Exp $
 
 top_srcdir=    @top_srcdir@
 srcdir=                @srcdir@
@@ -9,7 +9,7 @@ top_builddir=   ..
 INC=           -I$(top_srcdir)/dump -I$(top_srcdir)/common
 ALL_CFLAGS=    @CPPFLAGS@ @CFLAGS@ @CCOPTS@ -pipe $(OPT) $(GINC) $(INC) $(DEFS) @DUMPDEBUG@ 
 ALL_LDFLAGS=   @LDFLAGS@ @LDOPTS@ @STATIC@
-LIBS=          $(GLIBS) @ZLIB@ @BZLIB@ @BLKID@
+LIBS=          $(GLIBS) @ZLIB@ @BZLIB@ @BLKID@ @SQLITE3@
 DEPLIBS=       ../compat/lib/libcompat.a
 
 PROG=          dump
@@ -18,7 +18,8 @@ LINKS=                ${SBINDIR}/dump ${SBINDIR}/rdump
 SRCS=          itime.c main.c optr.c tape.c traverse.c unctime.c
 OBJS=          itime.o main.o optr.o tape.o traverse.o unctime.o \
                ../common/dumprmt.o ../common/transformation_null.o ../common/transformation_lzo.o \
-               ../common/transformation_zlib.o ../common/transformation_bzlib.o
+               ../common/transformation_zlib.o ../common/transformation_bzlib.o \
+               ../common/legacy_indexer.o ../common/sqlite.o
 MAN8=          dump.8
 RMAN8=         rdump.8
 
index e525f3dfcc0fc648ecfe263a75bfd2634dc07461..1e3866e6bcc3ef0a8a7c2cb047abae9adbb28c32 100644 (file)
@@ -5,7 +5,7 @@
  *     Stelian Pop <stelian@popies.net>, 1999-2000
  *     Stelian Pop <stelian@popies.net> - AlcĂ´ve <www.alcove.com>, 2000-2002
  *
- *     $Id: dump.h,v 1.51 2011/06/10 13:07:29 stelian Exp $
+ *     $Id: dump.h,v 1.52 2011/06/10 13:41:41 stelian Exp $
  */
 
 /*-
@@ -75,8 +75,6 @@ extern char   *tapeprefix;    /* prefix of the tape file */
 extern char    *dumpdates;     /* name of the file containing dump date information*/
 extern char    lastlevel[NUM_STR_SIZE];/* dump level of previous dump */
 extern char    level[NUM_STR_SIZE];/* dump level of this dump */
-extern int     Afile;          /* archive file descriptor */
-extern int      AfileActive;    /* Afile flag */
 extern int     zipflag;        /* which compression method */
 extern int     uflag;          /* update flag */
 extern int     mflag;          /* dump metadata only if possible flag */
@@ -119,15 +117,6 @@ extern int tp_bshift;      /* log2(TP_BSIZE) */
 extern dump_ino_t volinfo[];   /* which inode on which volume archive info */
 extern Transformation *transformation;
 
-#ifdef USE_QFA
-#define        QFA_MAGIC       "495115637697"
-#define QFA_VERSION    "1.0"
-extern int     gTapeposfd;
-extern char    *gTapeposfile;
-extern char    gTps[255];
-extern int32_t gThisDumpDate;
-#endif /* USE_QFA */
-
 #ifndef __P
 #include <sys/cdefs.h>
 #endif
@@ -287,5 +276,6 @@ extern void endgrent();
 extern void exit();
 extern off_t lseek();
 extern const char *strerror();
+
 #endif
 
index fa50044de5861eddeaab61dec63d92c83c672182..20c2ad2be442db2d05413af784d9f88441503653 100644 (file)
@@ -37,7 +37,7 @@
 
 #ifndef lint
 static const char rcsid[] =
-       "$Id: main.c,v 1.98 2011/06/10 13:07:29 stelian Exp $";
+       "$Id: main.c,v 1.99 2011/06/10 13:41:41 stelian Exp $";
 #endif /* not lint */
 
 #include <config.h>
@@ -85,6 +85,7 @@ static const char rcsid[] =
 #include "bylabel.h"
 
 #include "transformation.h"
+#include "indexer.h"
 
 #ifndef SBOFF
 #define SBOFF (SBLOCK * DEV_BSIZE)
@@ -151,13 +152,6 @@ int        dev_bshift;     /* log2(dev_bsize) */
 int    tp_bshift;      /* log2(TP_BSIZE) */
 dump_ino_t volinfo[TP_NINOS];/* which inode on which volume archive info */
 
-#ifdef USE_QFA
-int    gTapeposfd;
-char   *gTapeposfile;
-char   gTps[255];
-int32_t        gThisDumpDate;
-#endif /* USE_QFA */
-
 struct dumptime *dthead;       /* head of the list version */
 int    nddates;                /* number of records (might be zero) */
 int    ddates_in;              /* we have read the increment file */
@@ -200,6 +194,8 @@ static void incompat_flags __P((int, char, char));
 static char* iexclude_bitmap = NULL;           /* the inode exclude bitmap */
 static unsigned int iexclude_bitmap_bytes = 0; /* size of bitmap in bytes */
 
+Indexer *indexer = &indexer_legacy;
+
 int
 main(int argc, char *argv[])
 {
@@ -251,10 +247,6 @@ main(int argc, char *argv[])
 
        obsolete(&argc, &argv);
 
-#ifdef USE_QFA
-       gTapeposfd = -1;
-#endif /* USE_QFA */
-
        while ((ch = getopt(argc, argv,
                            "0123456789A:aB:b:cd:D:e:E:f:F:h:I:"
 #ifdef HAVE_BZLIB
@@ -415,7 +407,7 @@ main(int argc, char *argv[])
 
 #ifdef USE_QFA
                case 'Q':               /* create tapeposfile */
-                       gTapeposfile = optarg;
+                       //gTapeposfile = optarg;  // FIXME - communicate filename to indexer.
                        tapepos = 1;
                        break;
 #endif /* USE_QFA */
@@ -699,9 +691,6 @@ main(int argc, char *argv[])
 
                msg("Date of this level %s dump: %s", level,
                    ctime4(&spcl.c_date));
-#ifdef USE_QFA
-               gThisDumpDate = spcl.c_date;
-#endif
                if (spcl.c_ddate)
                        msg("Date of last level %s dump: %s", lastlevel,
                            ctime4(&spcl.c_ddate));
@@ -933,22 +922,7 @@ main(int argc, char *argv[])
        }
 
 #ifdef USE_QFA
-       if (tapepos) {
-               msg("writing QFA positions to %s\n", gTapeposfile);
-               if ((gTapeposfd = open(gTapeposfile,
-                                      O_WRONLY|O_CREAT|O_TRUNC,
-                                      S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP
-                                      | S_IROTH | S_IWOTH)) < 0)
-                       quit("can't open tapeposfile\n");
-               /* print QFA-file header */
-               snprintf(gTps, sizeof(gTps), "%s\n%s\n%ld\n\n", QFA_MAGIC, QFA_VERSION, (unsigned long)spcl.c_date);
-               gTps[sizeof(gTps) - 1] = '\0';
-               if (write(gTapeposfd, gTps, strlen(gTps)) != (ssize_t)strlen(gTps))
-                       quit("can't write tapeposfile\n");
-               sprintf(gTps, "ino\ttapeno\ttapepos\n");
-               if (write(gTapeposfd, gTps, strlen(gTps)) != (ssize_t)strlen(gTps))
-                       quit("can't write tapeposfile\n");
-       }
+       indexer->openQfa();
 #endif /* USE_QFA */
 
        /*
@@ -1023,6 +997,8 @@ main(int argc, char *argv[])
        tend_writing = time(NULL);
        spcl.c_type = TS_END;
 
+       indexer->foo();
+
        if (Afile >= 0) {
                volinfo[1] = ROOTINO;
                memcpy(spcl.c_inos, volinfo, TP_NINOS * sizeof(dump_ino_t));
@@ -1069,8 +1045,8 @@ main(int argc, char *argv[])
                        spcl.c_tapea, tapekb, rate);
        }
 
-       if (Afile >= 0)
-               msg("Archiving dump to %s\n", Apath);
+       indexer->close();
+       indexer->closeQfa();
 
        broadcast("DUMP IS DONE!\7\7\n");
        msg("DUMP IS DONE\n");
index e0b54112bb602dbb69ecead4356c96b128471d25..bab3379782ca445b26caa219d4a244522c6ecc87 100644 (file)
@@ -37,7 +37,7 @@
 
 #ifndef lint
 static const char rcsid[] =
-       "$Id: tape.c,v 1.95 2011/06/10 13:07:29 stelian Exp $";
+       "$Id: tape.c,v 1.96 2011/06/10 13:41:41 stelian Exp $";
 #endif /* not lint */
 
 #include <config.h>
@@ -93,6 +93,8 @@ int    write(), read();
 #include <protocols/dumprestore.h>
 
 #include "dump.h"
+#include "indexer.h"
+#include "slave.h"
 
 int    writesize;              /* size of malloc()ed buffer for tape */
 long   lastspclrec = -1;       /* tape block number of last written header */
@@ -119,24 +121,7 @@ static     void enslave __P((void));
 static void flushtape __P((void));
 static void killall __P((void));
 static void rollforward __P((void));
-#ifdef USE_QFA
-static int GetTapePos __P((long long *));
-static int MkTapeString __P((struct s_spcl *, long long));
-#define FILESQFAPOS    20
-#endif
 
-/*
- * Concurrent dump mods (Caltech) - disk block reading and tape writing
- * are exported to several slave processes.  While one slave writes the
- * tape, the others read disk blocks; they pass control of the tape in
- * a ring via signals. The parent process traverses the filesystem and
- * sends writeheader()'s and lists of daddr's to the slaves via pipes.
- * The following structure defines the instruction packets sent to slaves.
- */
-struct req {
-       ext2_loff_t dblk;
-       int count;
-};
 int reqsiz;
 
 struct slave_results {
@@ -144,19 +129,7 @@ struct slave_results {
        ssize_t clen;           /* compressed length */
 };
 
-#define SLAVES 3               /* 1 slave writing, 1 reading, 1 for slack */
-struct slave {
-       int tapea;              /* header number at start of this chunk */
-       int count;              /* count to next header (used for TS_TAPE */
-                               /* after EOT) */
-       int inode;              /* inode that we are currently dealing with */
-       int fd;                 /* FD for this slave */
-       int pid;                /* PID for this slave */
-       int sent;               /* 1 == we've sent this slave requests */
-       int firstrec;           /* record number of this block */
-       char (*tblock)[TP_BSIZE]; /* buffer for data blocks */
-       struct req *req;        /* buffer for requests */
-} slaves[SLAVES+1];
+struct slave slaves[SLAVES+1];
 struct slave *slp;
 
 char   (*nextblock)[TP_BSIZE];
@@ -176,9 +149,6 @@ static int ready2;  /* have we reached the lock point without having */
                        /* received the SIGUSR2 signal from the prev slave? */
 static sigjmp_buf jmpbuf2;     /* where to jump to if we are ready when the */
                        /* SIGUSR2 arrives from the previous slave */
-#ifdef USE_QFA
-static int gtperr = 0;
-#endif
 
 /*
  * Determine if we can use Linux' clone system call.  If so, call it
@@ -280,27 +250,7 @@ writerec(const void *dp, int isspcl)
        *(union u_spcl *)(*(nextblock)) = *(union u_spcl *)dp;
 
        /* Need to write it to the archive file */
-       if (! AfileActive && isspcl && (spcl.c_type == TS_END))
-               AfileActive = 1;
-       if (AfileActive && Afile >= 0 && !(spcl.c_flags & DR_EXTATTRIBUTES)) {
-               /* When we dump an inode which is not a directory,
-                * it means we ended the archive contents */
-               if (isspcl && (spcl.c_type == TS_INODE) &&
-                   ((spcl.c_dinode.di_mode & S_IFMT) != IFDIR))
-                       AfileActive = 0;
-               else {
-                       union u_spcl tmp;
-                       tmp = *(union u_spcl *)dp;
-                       /* Write the record, _uncompressed_ */
-                       if (isspcl) {
-                               tmp.s_spcl.c_flags &= ~DR_COMPRESSED;
-                               mkchecksum(&tmp);
-                       }
-                       if (write(Afile, &tmp, TP_BSIZE) != TP_BSIZE)
-                               msg("error writing archive file: %s\n", 
-                               strerror(errno));
-               }
-       }
+       indexer->writerec(dp, isspcl);
 
        nextblock++;
        if (isspcl)
@@ -1176,12 +1126,7 @@ doslave(int cmd,
        errcode_t retval;
 #endif
 #ifdef USE_QFA
-       long long curtapepos;
-       union u_spcl *uspclptr;
-       struct s_spcl *spclptr;
-       /* long         maxntrecs = 300000000 / (ntrec * 1024);  last tested: 50 000 000 */
-       long            maxntrecs = 50000;      /* every 50MB */
-       long            cntntrecs = maxntrecs;
+    QFA_State qfa_state;
 #endif /* USE_QFA */
        sigset_t set;
 
@@ -1195,6 +1140,8 @@ doslave(int cmd,
        transformation->startDiskIOProcess(transformation);
 #endif /* HAVE_BLOCK_TRANSFORMATION */
 
+       indexer->openQfaState(&qfa_state);
+
        /*
         * Need our own seek pointer.
         */
@@ -1333,38 +1280,7 @@ doslave(int cmd,
                ready2 = 0;
                caught2 = 0;
 
-#ifdef USE_QFA
-               if (gTapeposfd >= 0) {
-                       int i;
-                       int foundone = 0;
-
-                       for (i = 0; (i < ntrec) && !foundone; ++i) {
-                               uspclptr = (union u_spcl *)&slp->tblock[i];
-                               spclptr = &uspclptr->s_spcl;
-                               if ((spclptr->c_magic == NFS_MAGIC) && 
-                                                       (spclptr->c_type == TS_INODE) &&
-                                                       (spclptr->c_date == gThisDumpDate) &&
-                                                       !(spclptr->c_dinode.di_mode & S_IFDIR) &&
-                                                       !(spclptr->c_flags & DR_EXTATTRIBUTES)
-                                               ) {
-                                       foundone = 1;
-                                       /* if (cntntrecs >= maxntrecs) {         only write every maxntrecs amount of data */
-                                               cntntrecs = 0;
-                                               if (gtperr == 0) 
-                                                       gtperr = GetTapePos(&curtapepos);
-                                               /* if an error occured previously don't
-                                                * try again */
-                                               if (gtperr == 0) {
-#ifdef DEBUG_QFA
-                                                       msg("inode %ld at tapepos %ld\n", spclptr->c_inumber, curtapepos);
-#endif
-                                                       gtperr = MkTapeString(spclptr, curtapepos);
-                                               }
-                                       /* } */
-                               }
-                       }
-               }
-#endif /* USE_QFA */
+               indexer->updateQfa(&qfa_state);
                                                
                while (eot_count < 10 && size < bufsize) {
 #ifdef RDUMP
@@ -1417,11 +1333,8 @@ doslave(int cmd,
                 * Signal the next slave to go.
                 */
                (void) kill(nextslave, SIGUSR2);
-#ifdef USE_QFA
-               if (gTapeposfd >= 0) {
-                       cntntrecs += ntrec;
-               }
-#endif /* USE_QFA */
+
+               indexer->updateQfaState(&qfa_state);
        }
 
 #ifdef HAVE_BLOCK_TRANSFORMATION
@@ -1484,64 +1397,3 @@ SetLogicalPos(void)
        return err;
 }
 */
-
-#ifdef USE_QFA
-#define LSEEK_GET_TAPEPOS      10
-#define LSEEK_GO2_TAPEPOS      11
-/*
- * read the current tape position
- */
-int
-GetTapePos(long long *pos)
-{
-       int err = 0;
-
-#ifdef RDUMP
-       if (host) {
-               *pos = (long long) rmtseek((OFF_T)0, (int)LSEEK_GET_TAPEPOS);
-               err = *pos < 0;
-       }
-       else 
-#endif
-       {
-       if (magtapeout) {
-               long mtpos;
-               *pos = 0;
-               err = (ioctl(tapefd, MTIOCPOS, &mtpos) < 0);
-               *pos = (long long)mtpos;
-       }
-       else {
-               *pos = LSEEK(tapefd, 0, SEEK_CUR);
-               err = (*pos < 0);
-       }
-       }
-       if (err) {
-               err = errno;
-               msg("[%ld] error: %d (getting tapepos: %lld)\n", getpid(), 
-                       err, *pos);
-               return err;
-       }
-       return err;
-}
-
-static int 
-MkTapeString(struct s_spcl *spclptr, long long curtapepos)
-{
-       int     err = 0;
-
-#ifdef DEBUG_QFA
-       msg("inode %ld at tapepos %lld\n", spclptr->c_inumber, curtapepos);
-#endif
-
-       snprintf(gTps, sizeof(gTps), "%ld\t%d\t%lld\n", 
-                (unsigned long)spclptr->c_inumber, 
-                tapeno, 
-                curtapepos);
-       gTps[sizeof(gTps) - 1] = '\0';
-       if (write(gTapeposfd, gTps, strlen(gTps)) != (ssize_t)strlen(gTps)) {
-               err = errno;
-       warn("error writing tapepos file. (error %d)\n", errno);
-       }
-       return err;
-}
-#endif /* USE_QFA */
index 7bdfef1c96c0e412a4cec3934b7ad42c212e6e7a..f762cf4b1c455c985ee2473b9536d1bd3578f7b0 100644 (file)
@@ -37,7 +37,7 @@
 
 #ifndef lint
 static const char rcsid[] =
-       "$Id: traverse.c,v 1.73 2011/06/10 13:07:29 stelian Exp $";
+       "$Id: traverse.c,v 1.74 2011/06/10 13:41:41 stelian Exp $";
 #endif /* not lint */
 
 #include <config.h>
@@ -77,6 +77,7 @@ static const char rcsid[] =
 #include <protocols/dumprestore.h>
 
 #include "dump.h"
+#include "indexer.h"
 
 #define        HASDUMPEDFILE   0x1
 #define        HASSUBDIRS      0x2
@@ -914,6 +915,8 @@ dumpino(struct dinode *dp, dump_ino_t ino, int metaonly)
        spcl.c_type = TS_INODE;
        spcl.c_count = 0;
 
+       indexer->addInode(dp, ino, metaonly);
+
        if (metaonly && (dp->di_mode & S_IFMT)) {
                spcl.c_flags |= DR_METAONLY;
                spcl.c_count = 0;
@@ -1101,6 +1104,8 @@ convert_dir(struct ext2_dir_entry *dirent, UNUSED(int offset),
        p->prev_offset = p->offset;
        p->offset += reclen;
 
+       //indexer->addDirEntry(dirent, offset, blocksize, buf, private);
+
        return 0;
 }
 
@@ -1204,6 +1209,8 @@ dumpdirino(struct dinode *dp, dump_ino_t ino)
                spcl.c_type = TS_ADDR;
        }
 
+       indexer->addInode(dp, ino, 0);
+
        (void)free(cdc.buf);
        dump_xattr(ino, dp);
 }