From e3956dfb7715a21919aa66dd4209a2dc1c3c82da Mon Sep 17 00:00:00 2001 From: Stelian Pop Date: Fri, 10 Jun 2011 13:07:24 +0000 Subject: [PATCH] Encryption (and compression as plugins) support. --- CHANGES | 6 +- Makefile.in | 2 +- common/Makefile.in | 8 +- common/transformation.h | 144 ++++++++++ common/transformation_bzlib.c | 140 +++++++++ common/transformation_lzo.c | 141 +++++++++ common/transformation_null.c | 94 ++++++ common/transformation_ssl.c | 519 ++++++++++++++++++++++++++++++++++ common/transformation_zlib.c | 136 +++++++++ config.h.in | 12 + configure | 33 +-- configure.in | 32 +-- dump/Makefile.in | 9 +- dump/dump.h | 4 +- dump/main.c | 11 +- dump/tape.c | 110 ++----- dump/traverse.c | 8 +- rmt/Makefile.in | 4 +- testing/Makefile.in | 38 +++ 19 files changed, 1318 insertions(+), 133 deletions(-) create mode 100644 common/transformation.h create mode 100644 common/transformation_bzlib.c create mode 100644 common/transformation_lzo.c create mode 100644 common/transformation_null.c create mode 100644 common/transformation_ssl.c create mode 100644 common/transformation_zlib.c create mode 100644 testing/Makefile.in diff --git a/CHANGES b/CHANGES index 8796462..a3f339c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,4 @@ -$Id: CHANGES,v 1.328 2011/06/10 12:41:53 stelian Exp $ +$Id: CHANGES,v 1.329 2011/06/10 13:07:24 stelian Exp $ Changes between versions 0.4b44 and 0.4b45 (released ?????????????) =================================================================== @@ -7,6 +7,10 @@ Changes between versions 0.4b44 and 0.4b45 (released ?????????????) allowing multiple slaves to compress in parallel. Thanks to Phillip Susi for the patch. +2. Add dump encryption support and reorganise the whole + compression/encryption code as plugins. Thanks to + Bear Giles for the patch. + Changes between versions 0.4b43 and 0.4b44 (released June 10, 2011) =================================================================== diff --git a/Makefile.in b/Makefile.in index 2dbd90a..f8f64f9 100644 --- a/Makefile.in +++ b/Makefile.in @@ -5,7 +5,7 @@ top_builddir= . @MCONFIG@ RM= @RM@ -SUBDIRS= compat/lib compat/include common dump restore @RMTDIR@ +SUBDIRS= compat/lib compat/include common dump restore testing @RMTDIR@ all clean install dep depend realclean distclean:: for i in $(SUBDIRS); do \ diff --git a/common/Makefile.in b/common/Makefile.in index 80ea04d..2c03400 100644 --- a/common/Makefile.in +++ b/common/Makefile.in @@ -1,4 +1,4 @@ -# $Id: Makefile.in,v 1.5 2003/05/08 21:11:39 stelian Exp $ +# $Id: Makefile.in,v 1.6 2011/06/10 13:07:29 stelian Exp $ top_srcdir= @top_srcdir@ srcdir= @srcdir@ @@ -6,11 +6,11 @@ top_builddir= .. @MCONFIG@ -INC= -I$(top_srcdir)/dump +INC= -I$(top_srcdir)/dump -I$(top_srcdir)/common ALL_CFLAGS= @CPPFLAGS@ @CFLAGS@ @CCOPTS@ -pipe $(OPT) $(GINC) $(INC) $(DEFS) @DUMPDEBUG@ ALL_LDFLAGS= @LDFLAGS@ @LDOPTS@ @STATIC@ -SRCS= dumprmt.c -OBJS= dumprmt.o +SRCS= dumprmt.c transformation_null.c transformation_lzo.c transformation_zlib.c transformation_bzlib.c transformation_ssl.c +OBJS= dumprmt.o transformation_null.o transformation_lzo.o transformation_zlib.o transformation_bzlib.o transformation_ssl.o .c.o: $(CC) -c $(ALL_CFLAGS) $< -o $@ diff --git a/common/transformation.h b/common/transformation.h new file mode 100644 index 0000000..5e667fa --- /dev/null +++ b/common/transformation.h @@ -0,0 +1,144 @@ +#include +#include +#include + +#ifndef __P +#include +#endif + +#ifdef HAVE_LZO +#include +#endif /* HAVE_LZO */ + +#ifdef HAVE_OPENSSL +#include +#include +#include +#endif + +#ifndef _transformation_H +#define _transformation_H + +/* + * Compression/encryption hooks. + * + * Open questions: + * 1. should it be a failure if compress/decompress is called and we DON'T have the code included? + */ + +typedef struct transformation { + int enc; + union { +#ifdef HAVE_LZO + struct { + lzo_align_t __LZO_MMODEL *LZO_WorkMem; + } lzo; +#endif /* HAVE_LZO */ +#ifdef HAVE_ZLIB + struct { + int complvl; + } zlib; +#endif /* HAVE_ZLIB */ +#ifdef HAVE_BZLIB + struct { + int complvl; + } bzlib; +#endif /* HAVE_ZLIB */ +#ifdef HAVE_OPENSSL + struct { + int complvl; + + // encryption/decryption key + unsigned char key[EVP_MAX_KEY_LENGTH]; + unsigned int keylen; + + // crypto + const EVP_CIPHER *cipher; + const EVP_MD *digest; + ENGINE *engine; + + // this assumes we're multi-process but not multi-threaded + EVP_CIPHER_CTX *dataCtx; + EVP_CIPHER_CTX *ivCtx; + } ssl; +#endif + } state; + + /* + * The name of the compression/encryption algorithm, for + * display purposes. + */ + const char *name; + + /* + * Is this mandatory even if the size of the buffer increases? + * As a general rule compression is optional * and encryption is + * mandatory. + */ + int mandatory; + + /* + * Initialize the system. + * (mode: 1 = compress/encrypt, 0 = decompress/decrypt) + */ + int (*initialize) __P((struct transformation *xform, int mode)); + + /* + * Shut down the system + */ + int (*shutdown) __P((struct transformation *xform)); + + /* + * Do anything necessary after forking the process. + */ + int (*startNewTape) __P((struct transformation *xform, + struct tapebuf *tpbin, unsigned long *destlen)); + + /* + * The dump process is master-slave with the actual + * disk and dump tape access handled by the slave processes. + * This method performs any initialization required by + * the latter process. (E.g., some encryption libraries + * must be reinitialized.) + */ + int (*startDiskIOProcess) __P((struct transformation *xform)); + + /* + * Clean up everything before the slave process ends. + */ + int (*endDiskIOProcess) __P((struct transformation *xform)); + + /* + * Compress/encrypt buffer. + */ + int (*compress) __P((struct transformation *xform, struct tapebuf *, unsigned long *destlen, + const char *src, int srclen)); + + /* + * Decompress/decrypt buffer. + */ + int (*decompress) __P((struct transformation *xform, struct tapebuf *, unsigned long *destlen, + const char *src, int srclen, char **reason)); + +} Transformation; + +extern Transformation transformation_null; + +#ifdef HAVE_LZO +extern Transformation *transformation_lzo_factory(int enc); +#endif /* HAVE_ZLIB */ + +#ifdef HAVE_ZLIB +extern Transformation *transformation_zlib_factory(int enc, int complvl); +#endif /* HAVE_ZLIB */ + +#ifdef HAVE_BZLIB +extern Transformation *transformation_bzlib_factory(int enc, int complvl); +#endif /* HAVE_BZLIB */ + +#ifdef HAVE_OPENSSL +extern Transformation *transformation_ssl_factory(int enc, int complvl, + const char *ciphername, const char *digestname); +#endif /* HAVE_OPENSSL */ + +#endif /* _transformation_H */ diff --git a/common/transformation_bzlib.c b/common/transformation_bzlib.c new file mode 100644 index 0000000..bfd0cca --- /dev/null +++ b/common/transformation_bzlib.c @@ -0,0 +1,140 @@ +#include +#include + +#ifdef HAVE_BZLIB +#include +#endif /* HAVE_BZLIB */ + +#include "transformation.h" + +/* + * Initialize + */ +static int +bzlib_initialize(Transformation *xform, int enc) +{ + return 0; +} + +/* + * Shut down. + */ +static int +bzlib_shutdown(Transformation *xform) +{ + return 0; +} + +/* + * Handle forks. + */ +static int +bzlib_startNewTape(Transformation *xform, struct tapebuf *tpbin, + unsigned long *destlen) +{ + return 0; +} + +/* + * Start slave process. + */ +static int +bzlib_startDiskIOProcess(Transformation *xform) +{ + return 0; +} + +/* + * End slave process. + */ +static int +bzlib_endDiskIOProcess(Transformation *xform) { + if (xform != NULL) { + free(xform); + } + + return 0; +} + +/* + * Compress a buffer. + */ +static int +bzlib_compress(Transformation *xform, struct tapebuf *tpbin, unsigned long *destlen, + const char *src, int srclen) { +#ifdef HAVE_BZLIB + int compresult; + unsigned int destlen2 = *destlen; + compresult = BZ2_bzBuffToBuffCompress( + tpbin->buf, + &destlen2, + (char *) src, + srclen, + xform->state.bzlib.complvl, + 0, 30); + *destlen = destlen2; + return compresult == BZ_OK ? 1 : 0; +#else + return 1; +#endif /* HAVE_BZLIB */ +} + +/* + * Decompress a buffer. + */ +static int +bzlib_decompress(Transformation *xform, struct tapebuf *tpbin, + unsigned long *destlen, const char *src, int srclen, char **reason) { +#ifdef HAVE_BZLIB + int cresult; + unsigned int destlen2 = *destlen; + cresult = BZ2_bzBuffToBuffDecompress(tpbin->buf, &destlen2, (char *) src, srclen, + 0, 0); + *destlen = destlen2; + switch (cresult) { + case BZ_OK: + *reason = ""; + break; + case BZ_MEM_ERROR: + *reason = "not enough memory"; + break; + case BZ_OUTBUFF_FULL: + *reason = "buffer too small"; + break; + case BZ_DATA_ERROR: + case BZ_DATA_ERROR_MAGIC: + case BZ_UNEXPECTED_EOF: + *reason = "data error"; + break; + default: + *reason = "unknown"; + } + return (cresult == BZ_OK) ? 1 : 0; +#else + return 1 +#endif +} + +/* + * Factory + */ +Transformation +*transformation_bzlib_factory(int enc, int complvl) +{ + Transformation *t = (Transformation *) malloc(sizeof (Transformation)); + + t->enc = enc; + t->state.bzlib.complvl = complvl; + + t->name = "bzlib"; + t->mandatory = 0; + t->initialize = &bzlib_initialize; + t->shutdown = &bzlib_shutdown; + t->startNewTape = &bzlib_startNewTape; + t->startDiskIOProcess = &bzlib_startDiskIOProcess; + t->endDiskIOProcess = &bzlib_endDiskIOProcess; + t->compress = &bzlib_compress; + t->decompress = &bzlib_decompress; + + return t; +} diff --git a/common/transformation_lzo.c b/common/transformation_lzo.c new file mode 100644 index 0000000..a18bed6 --- /dev/null +++ b/common/transformation_lzo.c @@ -0,0 +1,141 @@ +#include +#include + +#ifdef HAVE_LZO +#include +#endif /* HAVE_LZO */ + +#include "transformation.h" + +/* + * Initialize + */ +static int +lzo_initialize(Transformation *xform, int enc) +{ +} + +/* + * Shut down. + */ +static int +lzo_shutdown(Transformation *xform) +{ + return 0; +} + +/* + * Handle forks. + */ +static int +lzo_startNewTape(Transformation *xform, struct tapebuf *tpbin, + unsigned long *destlen) +{ + return 0; +} + +/* + * Start slave process. + */ +static int +lzo_startDiskIOProcess(Transformation *xform) +{ + return 0; +} + +/* + * End slave process. + */ +static int +lzo_endDiskIOProcess(Transformation *xform) +{ + if (xform != NULL) { +#ifdef HAVE_LZO + if (xform->state.lzo.LZO_WorkMem != NULL) { + free(xform->state.lzo.LZO_WorkMem); + } +#endif /* HAVE_LZO */ + free(xform); + } + + return 0; +} + +/* + * Compress a buffer. + */ +static int +lzo_compress(Transformation *xform, struct tapebuf *tpbin, + unsigned long *destlen, const char *src, int srclen, int compressed) +{ +#ifdef HAVE_LZO + lzo_uint *worklen2 = (lzo_uint *) destlen; + int compresult; + + compresult = lzo1x_1_compress(src, srclen, tpbin->buf, worklen2, + xform->state.lzo.LZO_WorkMem); + + return compresult == LZO_E_OK ? 1 : 0; +#else + return 1; +#endif /* HAVE_LZO */ +} + +/* + * Decompress a buffer. + */ +static int +lzo_decompress(Transformation *xform, struct tapebuf *tpbin, + unsigned long *destlen, const char *src, int srclen, char **reason) +{ +#ifdef HAVE_LZO + int cresult = 0; + lzo_uint destlen2 = *destlen; + cresult = lzo1x_decompress(src, srclen, tpbin->buf, &destlen2, NULL); + *destlen = destlen2; + switch (cresult) { + case LZO_E_OK: + *reason = ""; + break; + case LZO_E_ERROR: + case LZO_E_EOF_NOT_FOUND: + *reason = "data error"; + break; + default: + *reason = "unknown"; + } + + return (cresult == LZO_E_OK) ? 1 : 0; +#else + return 1; +#endif /* HAVE_LZO */ +} + +/* + * Factory + */ +Transformation +*transformation_lzo_factory(int enc) +{ + Transformation *t = (Transformation *) malloc(sizeof (Transformation)); + +#ifdef HAVE_LZO + t->state.lzo.LZO_WorkMem = malloc(LZO1X_1_MEM_COMPRESS); + if (!t->state.lzo.LZO_WorkMem) + quit("couldn't allocate a compress buffer.\n"); +#else /* HAVE_LZO */ + t->state = NULL; +#endif /* HAVE_LZO */ + + t->name = "lzo"; + t->mandatory = 0; + t->initialize = &lzo_initialize; + t->shutdown = &lzo_shutdown; + t->startNewTape = &lzo_startNewTape; + t->startDiskIOProcess = &lzo_startDiskIOProcess; + t->endDiskIOProcess = &lzo_endDiskIOProcess; + t->compress = &lzo_compress; + t->decompress = &lzo_decompress; + + return t; +} diff --git a/common/transformation_null.c b/common/transformation_null.c new file mode 100644 index 0000000..6f25752 --- /dev/null +++ b/common/transformation_null.c @@ -0,0 +1,94 @@ +#include +#include + +#include "transformation.h" + +/* + * Initialize + */ +static int +null_initialize(Transformation *xform, int enc) +{ + return 0; +} + +/* + * Shut down. + */ +static int +null_shutdown(Transformation *xform) +{ + return 0; +} + +/* + * Handle fork. + */ +static int +null_startNewTape(Transformation *xform, struct tapebuf *tpbin, + unsigned long *destlen) +{ + return 0; +} + +/* + * Start slave process. + */ +static int +null_startDiskIOProcess(Transformation *xform) +{ + return 0; +} + +/* + * End slave process. + */ +static int +null_endDiskIOProcess(Transformation *xform) +{ + return 0; +} + +/* + * Compress a buffer. + */ +static int +null_compress(Transformation *xform, struct tapebuf *tpbin, unsigned long *destlen, + const char *src, int srclen) +{ + memcpy(tpbin->buf, src, srclen); + *destlen = srclen; + + return 1; +} + +/* + * Decompress a buffer. + */ +static int +null_decompress(Transformation *xform, struct tapebuf *tpbin, unsigned long *destlen, + const char *src, int srclen, char **reason) +{ + memcpy(tpbin->buf, src, srclen); + *destlen = srclen; + + return 1; +} + +/* + * + */ +Transformation transformation_null = +{ + 0, + NULL, + "null", + 0, + &null_initialize, + &null_shutdown, + &null_startNewTape, + &null_startDiskIOProcess, + &null_endDiskIOProcess, + &null_compress, + &null_decompress +}; diff --git a/common/transformation_ssl.c b/common/transformation_ssl.c new file mode 100644 index 0000000..b97c150 --- /dev/null +++ b/common/transformation_ssl.c @@ -0,0 +1,519 @@ +#include +#include +#include + +#ifdef HAVE_OPENSSL +#include +#include +#include +#include +#endif + +#ifdef HAVE_ZLIB +#include +#endif /* HAVE_ZLIB */ + +#include "transformation.h" + +/* + * IMPORTANT: this is an important piece of the puzzle for native + * support of encrypted backups but Key management is much more + * important and much harder. The best encryption algorithms in the + * world won't help you if you have to reveal your one key to everyone. + * + * IMPORTANT: always comply with local laws! Do not modify the source + * to give you stronger encryption than permitted by local law. + * + * ----------------------- + * 30-second cryptanalysis + * ----------------------- + * + * This is a known-plaintext problem since many of the encrypted blocks + * will start with a BSD inode structure. (It will be compressed binary + * data but still (mostly) known.) This means that it is very important + * to choose a strong IV. + * + * The dump tape/file is written by multiple slave processes so we don't + * have ready access to an intrinsic value such as tape position. That + * leaves random salts that are included in the dump buffer. The salt + * doesn't need to be cryptographically strong since it will be published. + * + * The IV is created by calculating the hash of the salt, a function of + * the salt and session key, and the salt again. + * + * The greatest vulnerability is predictable keys (and weak ciphers + * required by local laws). The key should also be protected in memory + * while dumping and restoring files but it's not as critical since + * anyone able to read the key probably has the ability to read the + * filesystem directly. + */ + +/* + * Initialize. + * + * 1. load policy file, select ciphers based on it. + * 2. PKCS12 keystore contains certificate (public keys) and + * private keys. + * 3. private keys can be password-protected individually, and + * you can also password-protect the entire keystore. + * 4. PKCS12 keystores can be manipulated by openssl, java, + * windows apps, more. + * + * questions: + * 1. how to select which keystore? (command line option passed + * in as function argument? + * 2. how to select which key, if doing restore? + * 3. how to pass keystore password(s), if doing restore? + */ +static int +ssl_initialize(Transformation *xform, int enc) +{ +#ifdef HAVE_OPENSSL + int keylen; + + OpenSSL_add_all_algorithms(); // see below */ + OpenSSL_add_all_ciphers(); // see below */ + OpenSSL_add_all_digests(); // see below */ + ERR_load_crypto_strings(); + ERR_load_RAND_strings(); + ERR_load_ERR_strings(); + + // read policy file and call those two methods to load acceptable + // ciphers and digests. We always want CBC mode, not ECB mode. + +/* + If we are encrypting we need to + 1. create a random session key (good random numbers are critical!) + 2. read the X509 certificates for everyone who needs to be able to read the dump file - + extract the public keys. These certificates may be in a PKCS12 keystore which many + applications use. OpenSSL also provides a large number of programs for it. + 3. encrypt the session key with the public keys and put it into another PKCS12 keystore. + This keystore will be written to the tape in startnewtape(). + 4. set up the cipher context. + + If we are decrypting we need to + 1. open our keystore(PKCS12) containing our private keys. The keystore itself may be + password protected, the individual keys definitely should be. + 2. read the PKCS12 keystore from the tape header. + 3. look for matching key ids. If we can't find a match tell the user. + 4. we still need to be able to open the private key so get the password somewhere. + 5. set up the cipher context. +*/ + + // RAND_egd(path); + +#endif /* HAVE_OPENSSL */ + + return 0; +} + +/* + * Shut down. + * + * The OpenSSL library may be using a physical crypto device so + * we need to make sure we properly release it. + */ +static int +ssl_shutdown(Transformation *xform) +{ +#ifdef HAVE_OPENSSL + OPENSSL_cleanse(xform->state.ssl.key, sizeof xform->state.ssl.key); + + munlock(xform, sizeof(Transformation)); + free(xform); + + /* is this the right method? */ + EVP_cleanup(); + RAND_cleanup(); + free(xform); +#endif /* HAVE_OPENSSL */ + return 0; +} + +/* + * We need to write new crypto header containing the encrypted + * session key. + */ +static int +ssl_startNewTape(Transformation *xform, struct tapebuf *tpbin, + unsigned long *destlen) +{ +#ifdef HAVE_OPENSSL + /* write new TS containing PKCS12 containing encrypted session key. */ +#endif /* HAVE_OPENSSL */ + + return 0; +} + +/* + * Start slave process. We need to reinitialize the encryption + * engine. + */ +static int +ssl_startDiskIOProcess(Transformation *xform) +{ +#ifdef HAVE_OPENSSL + mlock(xform, sizeof(Transformation)); + + OpenSSL_add_all_algorithms(); // see below */ + OpenSSL_add_all_ciphers(); // see below */ + OpenSSL_add_all_digests(); // see below */ + ERR_load_crypto_strings(); + ERR_load_RAND_strings(); + ERR_load_ERR_strings(); + + // Initialize key information with values we obtained from parent + // thread's startNewTape(). + + xform->state.ssl.dataCtx = EVP_CIPHER_CTX_new(); + xform->state.ssl.ivCtx = EVP_CIPHER_CTX_new(); +#endif /* HAVE_OPENSSL */ + + return 0; +} + +/* + * End of slave process. Clear encryption keys, etc. + */ +static int +ssl_endDiskIOProcess(Transformation *xform) +{ +#ifdef HAVE_OPENSSL + EVP_CIPHER_CTX_cleanup(xform->state.ssl.dataCtx); + EVP_CIPHER_CTX_cleanup(xform->state.ssl.ivCtx); + EVP_CIPHER_CTX_free(xform->state.ssl.dataCtx); + EVP_CIPHER_CTX_free(xform->state.ssl.ivCtx); + + OPENSSL_cleanse(xform->state.ssl.key, sizeof xform->state.ssl.key); + + munlock(xform, sizeof(Transformation)); + free(xform); + + /* is this the right method? */ + EVP_cleanup(); + RAND_cleanup(); +#endif /* HAVE_OPENSSL */ + return 0; +} + +/* + * Method to generate 'random' salt and IV. + * + * The generated salt is 16 bytes long. It is duplicated to get a 256-bit value (to support + * 256-bit block ciphers.) + * + */ +#ifdef HAVE_OPENSSL +static int +generateIV(Transformation *xform, unsigned char *salt, unsigned int *saltlen, + unsigned char *iv, unsigned int *ivlen) +{ + unsigned char ivbuffer[64]; + unsigned int buflen, y; + + /* we can use pseudorandom bytes since they're going */ + /* to be exposed to any attacker anyway. */ + *saltlen = 16; + if (xform->enc == 1) { + RAND_pseudo_bytes(salt, *saltlen); + } + memcpy(ivbuffer, salt, 16); + + /* -decrypt- salt value */ + memset(ivbuffer, 0, sizeof(ivbuffer)); + EVP_CipherInit_ex(xform->state.ssl.ivCtx, xform->state.ssl.cipher, xform->state.ssl.engine, NULL, NULL, 0); + //EVP_CIPHER_CTX_set_key_length(&ctx, 8); + EVP_CIPHER_CTX_set_padding(xform->state.ssl.ivCtx, 0); // -nopad + EVP_CipherInit_ex(xform->state.ssl.ivCtx, NULL, NULL, xform->state.ssl.key, ivbuffer, 0); + buflen = 32; + if (!EVP_CipherUpdate(xform->state.ssl.ivCtx, ivbuffer + 16, &buflen, salt, 16)) { + y = 32 - buflen; + if (!EVP_CipherFinal(xform->state.ssl.ivCtx, ivbuffer + 16 + buflen, &y)) { + buflen += y; + } else { + memset(ivbuffer + 16, 0, 32); + } + } else { + memset(ivbuffer + 16, 0, 32); + } + memcpy(ivbuffer + 48, salt, 16); + + /* now digest it. */ + EVP_Digest(ivbuffer, 64, iv, ivlen, xform->state.ssl.digest, NULL); + + return 0; +} +#endif + + +/* + * Encrypt a single chunk of blocks. Each chunk is encrypted + * independently so it's critically important to choose a good + * initial vector (iv). + * + * The ciphertext format is: + * - 20 bytes of random salt + * - encrypted (plaintext . digest(plaintext)) + */ +static int +ssl_compress(Transformation *xform, struct tapebuf *tpbin, + unsigned long *destlen, const char *src, int srclen) +{ +#ifdef HAVE_OPENSSL + unsigned char salt[16], iv[EVP_MAX_MD_SIZE]; + unsigned int saltlen = sizeof(salt); + unsigned int ivlen = sizeof(iv); + unsigned char digest[EVP_MAX_MD_SIZE]; + unsigned int digestlen = 0; + char *buf; + unsigned int len, len2; + + len = srclen + 1000; + buf = malloc(len); + + digestlen = sizeof(digest); + + /* generate salt, put it in header */ + generateIV(xform, salt, &saltlen, iv, &ivlen); + memcpy(tpbin->buf, salt, saltlen); + + /* compress the buffer first - increase the entropy */ + int compresult; + compresult = compress2(buf, destlen, src, srclen, xform->state.ssl.complvl); + if (compresult != Z_OK) { + printf("unable to compress...\n"); + return 0; + } + + EVP_EncryptInit_ex(xform->state.ssl.dataCtx, xform->state.ssl.cipher, xform->state.ssl.engine, NULL, NULL); + //EVP_CIPHER_CTX_set_key_length(&ctx, 8); + EVP_EncryptInit_ex(xform->state.ssl.dataCtx, NULL, NULL, xform->state.ssl.key, iv); + + // encrypt content. + if (!EVP_EncryptUpdate(xform->state.ssl.dataCtx, tpbin->buf + saltlen, &len, buf, *destlen)) { + EVP_CIPHER_CTX_cleanup(xform->state.ssl.dataCtx); + ERR_print_errors_fp(stdout); + free(buf); + return 0; + } + + // calculate digest, add it. + EVP_Digest(src, srclen, digest, &digestlen, xform->state.ssl.digest, xform->state.ssl.engine); + + len2 = *destlen - len - saltlen; + if (!EVP_EncryptUpdate(xform->state.ssl.dataCtx, tpbin->buf + saltlen + len, &len2, digest, digestlen)) { + EVP_CIPHER_CTX_cleanup(xform->state.ssl.dataCtx); + ERR_print_errors_fp(stdout); + free(buf); + return 0; + } + + len += len2; + + // finish up + len2 = *destlen - len - saltlen; + if (!EVP_EncryptFinal(xform->state.ssl.dataCtx, tpbin->buf + saltlen + len, &len2)) { + EVP_CIPHER_CTX_cleanup(xform->state.ssl.dataCtx); + ERR_print_errors_fp(stdout); + free(buf); + return 0; + } + + *destlen = len + len2 + saltlen; + free(buf); + + OPENSSL_cleanse(iv, sizeof iv); +#endif /* HAVE_OPENSSL */ + + return 1; +} + +/* + * + */ +static int +ssl_decompress(Transformation *xform, struct tapebuf *tpbin, + unsigned long *destlen, const char *src, int srclen, char **reason) +{ +#ifdef HAVE_OPENSSL + unsigned char salt[16], iv[EVP_MAX_MD_SIZE]; + unsigned int saltlen = sizeof(salt); + unsigned int ivlen = sizeof(iv); + unsigned char digest[EVP_MAX_MD_SIZE]; + unsigned int digestlen; + char *buf; + unsigned int len, len2; + + digestlen = EVP_MD_size(xform->state.ssl.digest); + + len = *destlen + 1000; + buf = malloc(len); + + // how to know salt length? + memcpy(salt, src, saltlen); + generateIV(xform, salt, &saltlen, iv, &ivlen); + + EVP_DecryptInit_ex(xform->state.ssl.dataCtx, xform->state.ssl.cipher, xform->state.ssl.engine, NULL, NULL); + //EVP_CIPHER_CTX_set_key_length(&ctx, 8); + EVP_DecryptInit_ex(xform->state.ssl.dataCtx, NULL, NULL, xform->state.ssl.key, iv); + + if (!EVP_DecryptUpdate(xform->state.ssl.dataCtx, buf, &len, src+saltlen, srclen-saltlen)) { + EVP_CIPHER_CTX_cleanup(xform->state.ssl.dataCtx); + free(buf); + ERR_print_errors_fp(stdout); + printf("error 1\n"); + return 0; + } + + len2 = *destlen + 1000 - len; + if (!EVP_DecryptFinal(xform->state.ssl.dataCtx, buf + len, &len2)) { + EVP_CIPHER_CTX_cleanup(xform->state.ssl.dataCtx); + free(buf); + ERR_print_errors_fp(stdout); + printf("error 2\n"); + return 0; + } + len += len2; + len -= digestlen; + + OPENSSL_cleanse(iv, sizeof iv); + + int cresult; + cresult = uncompress(tpbin->buf, destlen, buf, len); + switch (cresult) { + case Z_OK: + *reason = ""; + break; + case Z_MEM_ERROR: + *reason = "not enough memory"; + break; + case Z_BUF_ERROR: + *reason = "buffer too small"; + break; + case Z_DATA_ERROR: + *reason = "data error"; + break; + default: + *reason = "unknown"; + } + if (cresult != Z_OK) { + printf("compression failed: %s\n", *reason); + free(buf); + return 0; + } + + /* verify digest */ + EVP_Digest(tpbin->buf, *destlen, digest, &digestlen, xform->state.ssl.digest, xform->state.ssl.engine); + + if (memcmp(buf + len, digest, digestlen)) { + *reason = "digests did not match"; + return 0; + } + + free(buf); + +#endif /* HAVE_OPENSSL */ + + return 1; +} + +/* + * + */ +#if 0 +static int +ssl_compress_ts_addr(char *state, struct tapebuf *comp_buf, + unsigned int *worklen, char *data, int writesize, int compressed) +{ +#ifdef HAVE_OPENSSL + SSLState *s = (SSLState *) state; + unsigned char iv[32]; + unsigned int len; + + EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *) malloc(sizeof(EVP_CIPHER_CTX)); + + //EVP_BytesToKey(cipher, EVP_md5(), NULL, buf, strlen(buf), 1, key, iv); + + //EVP_CIPHER_CTX_init(ctx); + //EVP_CipherInit_ex(ctx, cipher, NULL, key, iv, do_encrypt); + //blocksize = EVP_CIPHER_CTX_block_size(ctx); + + s->digest = EVP_md_null; + s->cipher = EVP_enc_null; + EVP_CIPHER_CTX_init(s->ctx); + + /* calculate 'random' IV. */ + EVP_MD_CTX_init(&md); + EVP_DigestInit_ex(s->md, s->md, NULL); + /* EVP_DigestUpdate(s->md, x, len); <-- use logical record number */ + EVP_DigestUpdate(s->md, s->salt, s->saltlen); + len = sizeof(iv); + EVP_DigestFinal(s->md, iv, &len); + + // mlock on state info... + + /* do the actual encryption */ + EVP_CipherInit(s->ctx, s->cipher, s->key, iv); + EVP_CipherUpdate(s->ctx, out, &n, buf, buflen); + EVP_CipherFinal(s->ctx, out + n, outlen - y); + return 1; +#endif /* HAVE_OPENSSL */ + + return 1; +} +#endif + + +/* + * Factory. The cipher and digestnames should be read from a localized + * policy file. + * + * TODO: indicate error if unknown cipher or digest. + */ +Transformation +*transformation_ssl_factory(int enc, int complvl, const char *ciphername, const char *digestname) +{ + int keylen; + Transformation *t; + + t = (Transformation *) malloc(sizeof (Transformation)); + mlock(t, sizeof(Transformation)); + + OpenSSL_add_all_ciphers(); + OpenSSL_add_all_digests(); + + t->enc = enc; + t->state.ssl.complvl = complvl; + t->state.ssl.cipher = EVP_get_cipherbyname(ciphername); + t->state.ssl.digest = EVP_get_digestbyname(digestname); + t->state.ssl.engine = NULL; + + t->name = "ssl"; + t->mandatory = 1; + t->initialize = &ssl_initialize; + t->shutdown = &ssl_shutdown; + t->startNewTape = &ssl_startNewTape; + t->startDiskIOProcess = &ssl_startDiskIOProcess; + t->endDiskIOProcess = &ssl_endDiskIOProcess; + t->compress = &ssl_compress; + t->decompress = &ssl_decompress; + + // we could use this to generate a key from a passphrase. + // using this as the actual encryption key has the problems + // discussed elsewhere. + //EVP_BytesToKey(cipher, EVP_md5(), NULL, buf, strlen(buf), 1, key, iv); + + if (enc) { + /* generate random session key */ + //EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + //EVP_CIPHER_CTX_init(ctx); + //EVP_CIPHER_CTX_rand_key(ctx, t->state.ssl.key); + //EVP_CIPHER_CTX_cleanup(ctx); + //EVP_CIPHER_CTX_free(ctx); + RAND_bytes(t->state.ssl.key, t->state.ssl.cipher->key_len); + } else { + // how do we get keys? + } + + return t; +} diff --git a/common/transformation_zlib.c b/common/transformation_zlib.c new file mode 100644 index 0000000..3bb9048 --- /dev/null +++ b/common/transformation_zlib.c @@ -0,0 +1,136 @@ +#include +#include + +#ifdef HAVE_ZLIB +#include +#endif /* HAVE_ZLIB */ + +#include "transformation.h" + +/* + * Initialize + */ +static int +zlib_initialize(Transformation *xform) +{ + return 0; +} + +/* + * Shut down. + */ +static int +zlib_shutdown(Transformation *xform) +{ + return 0; +} + +/* + * Handle forks. + */ +static int +zlib_startNewTape(Transformation *xform, struct tapebuf *bin, + unsigned long *destlen) +{ + return 0; +} + +/* + * Start slave process + */ +static int +zlib_startDiskIOProcess(Transformation *xform) +{ + return 0; +} + +/* + * End slave process + */ +static int +zlib_endDiskIOProcess(Transformation *xform) +{ + if (xform != NULL) { + free(xform); + } + + return 0; +} + +struct req { + ext2_loff_t dblk; + int count; +}; + +/* + * Compress a buffer. + */ +static int +zlib_compress(Transformation *xform, struct tapebuf *tpbin, + unsigned long *destlen, const char *src, int srclen) +{ +#ifdef HAVE_ZLIB + int compresult; + compresult = compress2(tpbin->buf, destlen, src, srclen, xform->state.zlib.complvl); + return compresult == Z_OK ? 1 : 0; +#else + return 1; +#endif /* HAVE_ZLIB */ +} + +/* + * Decompress a buffer. + */ +static int +zlib_decompress(Transformation *xform, struct tapebuf *tpbin, + unsigned long *destlen, const char *src, int srclen, char **reason) +{ +#ifdef HAVE_ZLIB + int cresult; + cresult = uncompress(tpbin->buf, destlen, src, srclen); + switch (cresult) { + case Z_OK: + *reason = ""; + break; + case Z_MEM_ERROR: + *reason = "not enough memory"; + break; + case Z_BUF_ERROR: + *reason = "buffer too small"; + break; + case Z_DATA_ERROR: + *reason = "data error"; + break; + default: + *reason = "unknown"; + } + return (cresult == Z_OK) ? 1 : 0; +#else + return 1; +#endif /* HAVE_ZLIB */ +} + + +/* + * Factory + */ +Transformation +*transformation_zlib_factory(int enc, int complvl) +{ + Transformation *t = (Transformation *) malloc(sizeof (Transformation)); + + t->enc = enc; + t->state.zlib.complvl = complvl; + + t->name = "zlib"; + t->mandatory = 0; + t->initialize = &zlib_initialize; + t->shutdown = &zlib_shutdown; + t->startNewTape = &zlib_startNewTape; + t->startDiskIOProcess = &zlib_startDiskIOProcess; + t->endDiskIOProcess = &zlib_endDiskIOProcess; + t->compress = &zlib_compress; + t->decompress = &zlib_decompress; + + return t; +} diff --git a/config.h.in b/config.h.in index 5cd2bcc..71d6c7e 100644 --- a/config.h.in +++ b/config.h.in @@ -9,6 +9,9 @@ /* Define this if you have the blkid library. */ #undef HAVE_BLKID +/* Define to 1 if have block transformation (compression or encryption) */ +#undef HAVE_BLOCK_TRANSFORMATION + /* Define this if you have bzlib compression library. */ #undef HAVE_BZLIB @@ -43,6 +46,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_MEMORY_H +/* Define to 1 if you have the openssl library */ +#undef HAVE_OPENSSL + /* Define if you want to include readline support. */ #undef HAVE_READLINE @@ -134,3 +140,9 @@ /* Define to `uint64_t' if does not define. */ #undef u_quad_t + +/* Define to 1 if we have any compression or encryption library. */ +#undef HAVE_BLOCK_TRANSFORMATION +#if defined(HAVE_LZO) || defined(HAVE_ZLIB) || defined(HAVE_BZLIB) +#define HAVE_BLOCK_TRANSFORMATION 1 +#endif diff --git a/configure b/configure index 1dbba1b..dd658df 100755 --- a/configure +++ b/configure @@ -594,7 +594,7 @@ ac_includes_default="\ ac_subst_vars='LTLIBOBJS LIBOBJS top_builddir -CRYPTO +SSLLIB BZLIB ZLIB READLINE @@ -5277,8 +5277,7 @@ rm -f conftest* fi -if test "$ERMT" != ""; then - ac_fn_c_check_header_mongrel "$LINENO" "openssl/evp.h" "ac_cv_header_openssl_evp_h" "$ac_includes_default" +ac_fn_c_check_header_mongrel "$LINENO" "openssl/evp.h" "ac_cv_header_openssl_evp_h" "$ac_includes_default" if test "x$ac_cv_header_openssl_evp_h" = x""yes; then : evp_h=yes else @@ -5286,7 +5285,7 @@ else fi - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for EVP_CIPHER_CTX_set_padding in -lcrypto" >&5 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for EVP_CIPHER_CTX_set_padding in -lcrypto" >&5 $as_echo_n "checking for EVP_CIPHER_CTX_set_padding in -lcrypto... " >&6; } if test "${ac_cv_lib_crypto_EVP_CIPHER_CTX_set_padding+set}" = set; then : $as_echo_n "(cached) " >&6 @@ -5328,23 +5327,18 @@ else crypto_lib=no fi - if test -n "$PKG_CONFIG" && \ - { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libcrypto\""; } >&5 - ($PKG_CONFIG --exists --print-errors "libcrypto") 2>&5 - ac_status=$? - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; }; then +if test "$evp_h" = yes -a "$crypto_lib" = yes; then + if test "$STATICZ" = yes; then + SSLLIB="-Wl,-Bstatic -lssl -Wl,-Bdynamic" + else + SSLLIB="-lssl" + fi - if test -n $STATIC ; then - CRYPTO=`$PKG_CONFIG --libs --static libcrypto` - else - CRYPTO=`$PKG_CONFIG --libs libcrypto` - fi +$as_echo "#define HAVE_OPENSSL 1" >>confdefs.h else - CRYPTO="" -fi - if test "$evp_h" = no -o "x$CRYPTO" = "x"; then + SSLLIB="" + if test "$ERMT" != ""; then as_fn_error or configure without --enable-ermt "You need to install the OpenSSL library (version 0.9.7a or later)" "$LINENO" 5 fi fi @@ -5379,7 +5373,7 @@ top_builddir=`cd .; pwd` test -d compat || mkdir compat test -d compat/lib || mkdir compat/lib -ac_config_files="$ac_config_files MCONFIG Makefile common/Makefile compat/include/Makefile compat/lib/Makefile dump/Makefile restore/Makefile $RMTMAKEFILE" +ac_config_files="$ac_config_files MCONFIG Makefile common/Makefile compat/include/Makefile compat/lib/Makefile dump/Makefile restore/Makefile testing/Makefile $RMTMAKEFILE" cat >confcache <<\_ACEOF # This file is a shell script that caches the results of configure @@ -6079,6 +6073,7 @@ do "compat/lib/Makefile") CONFIG_FILES="$CONFIG_FILES compat/lib/Makefile" ;; "dump/Makefile") CONFIG_FILES="$CONFIG_FILES dump/Makefile" ;; "restore/Makefile") CONFIG_FILES="$CONFIG_FILES restore/Makefile" ;; + "testing/Makefile") CONFIG_FILES="$CONFIG_FILES testing/Makefile" ;; "$RMTMAKEFILE") CONFIG_FILES="$CONFIG_FILES $RMTMAKEFILE" ;; *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5 ;; diff --git a/configure.in b/configure.in index 2d69157..807df9b 100644 --- a/configure.in +++ b/configure.in @@ -591,24 +591,24 @@ if test "$ac_cv_func_glob" = "yes"; then fi dnl -dnl Check for OpenSSL, for ermt -dnl -if test "$ERMT" != ""; then - AC_CHECK_HEADER(openssl/evp.h, [evp_h=yes], [evp_h=no]) - AC_CHECK_LIB(crypto, EVP_CIPHER_CTX_set_padding, [crypto_lib=yes], [crypto_lib=no]) - PKG_CHECK_EXISTS([libcrypto], - [ - if test -n $STATIC ; then - CRYPTO=`$PKG_CONFIG --libs --static libcrypto` - else - CRYPTO=`$PKG_CONFIG --libs libcrypto` - fi - ],[CRYPTO=""]) - if test "$evp_h" = no -o "x$CRYPTO" = "x"; then +dnl Check for OpenSSL, for ermt and encryption. +dnl +AC_CHECK_HEADER(openssl/evp.h, [evp_h=yes], [evp_h=no]) +AC_CHECK_LIB(crypto, EVP_CIPHER_CTX_set_padding, [crypto_lib=yes], [crypto_lib=no]) +if test "$evp_h" = yes -a "$crypto_lib" = yes; then + if test "$STATICZ" = yes; then + SSLLIB="-Wl,-Bstatic -lssl -Wl,-Bdynamic" + else + SSLLIB="-lssl" + fi + AC_DEFINE([HAVE_OPENSSL],1,[Define this if you have openssl library.]) +else + SSLLIB="" + if test "$ERMT" != ""; then AC_MSG_ERROR(You need to install the OpenSSL library (version 0.9.7a or later), or configure without --enable-ermt) fi fi -AC_SUBST(CRYPTO) +AC_SUBST(SSLLIB) dnl dnl Check for types @@ -631,4 +631,4 @@ test -d compat/lib || mkdir compat/lib dnl dnl Output files dnl -AC_OUTPUT(MCONFIG Makefile common/Makefile compat/include/Makefile compat/lib/Makefile dump/Makefile restore/Makefile $RMTMAKEFILE) +AC_OUTPUT(MCONFIG Makefile common/Makefile compat/include/Makefile compat/lib/Makefile dump/Makefile restore/Makefile testing/Makefile $RMTMAKEFILE) diff --git a/dump/Makefile.in b/dump/Makefile.in index f18c01e..9d312f9 100644 --- a/dump/Makefile.in +++ b/dump/Makefile.in @@ -1,4 +1,4 @@ -# $Id: Makefile.in,v 1.12 2004/07/05 15:02:36 stelian Exp $ +# $Id: Makefile.in,v 1.13 2011/06/10 13:07:29 stelian Exp $ top_srcdir= @top_srcdir@ srcdir= @srcdir@ @@ -6,8 +6,8 @@ top_builddir= .. @MCONFIG@ -INC= -I$(top_srcdir)/dump -ALL_CFLAGS= @CPPFLAGS@ @CFLAGS@ @CCOPTS@ -pipe $(OPT) $(GINC) $(INC) $(DEFS) @DUMPDEBUG@ +INC= -I$(top_srcdir)/dump -I$(top_srcdir)/common +ALL_CFLAGS= @CPPFLAGS@ @CFLAGS@ @CCOPTS@ -pipe $(OPT) $(GINC) $(INC) $(DEFS) @DUMPDEBUG@ ALL_LDFLAGS= @LDFLAGS@ @LDOPTS@ @STATIC@ LIBS= $(GLIBS) @ZLIB@ @BZLIB@ @BLKID@ DEPLIBS= ../compat/lib/libcompat.a @@ -17,7 +17,8 @@ RPROG= rdump LINKS= ${SBINDIR}/dump ${SBINDIR}/rdump SRCS= itime.c main.c optr.c tape.c traverse.c unctime.c OBJS= itime.o main.o optr.o tape.o traverse.o unctime.o \ - ../common/dumprmt.o + ../common/dumprmt.o ../common/transformation_null.o ../common/transformation_lzo.o \ + ../common/transformation_zlib.o ../common/transformation_bzlib.o MAN8= dump.8 RMAN8= rdump.8 diff --git a/dump/dump.h b/dump/dump.h index f26d12b..e525f3d 100644 --- a/dump/dump.h +++ b/dump/dump.h @@ -5,7 +5,7 @@ * Stelian Pop , 1999-2000 * Stelian Pop - Alcôve , 2000-2002 * - * $Id: dump.h,v 1.50 2010/04/28 09:29:50 stelian Exp $ + * $Id: dump.h,v 1.51 2011/06/10 13:07:29 stelian Exp $ */ /*- @@ -40,6 +40,7 @@ #include #include #include +#include "transformation.h" #define MAXINOPB (MAXBSIZE / sizeof(struct dinode)) #define MAXNINDIR (MAXBSIZE / sizeof(blk_t)) @@ -116,6 +117,7 @@ extern long dev_bsize; /* block size of underlying disk device */ extern int dev_bshift; /* log2(dev_bsize) */ extern int tp_bshift; /* log2(TP_BSIZE) */ extern dump_ino_t volinfo[]; /* which inode on which volume archive info */ +extern Transformation *transformation; #ifdef USE_QFA #define QFA_MAGIC "495115637697" diff --git a/dump/main.c b/dump/main.c index ae3e3ca..fa50044 100644 --- a/dump/main.c +++ b/dump/main.c @@ -37,7 +37,7 @@ #ifndef lint static const char rcsid[] = - "$Id: main.c,v 1.97 2010/04/28 09:29:50 stelian Exp $"; + "$Id: main.c,v 1.98 2011/06/10 13:07:29 stelian Exp $"; #endif /* not lint */ #include @@ -84,6 +84,8 @@ static const char rcsid[] = #include "pathnames.h" #include "bylabel.h" +#include "transformation.h" + #ifndef SBOFF #define SBOFF (SBLOCK * DEV_BSIZE) #endif @@ -178,6 +180,7 @@ int sizest = 0; /* return size estimate only */ int compressed = 0; /* use zlib to compress the output, compress level 1-9 */ long long bytes_written = 0; /* total bytes written */ long uncomprblks = 0;/* uncompressed blocks written */ +Transformation *transformation = &transformation_null; long smtc_errno; @@ -239,6 +242,7 @@ main(int argc, char *argv[]) quit("TP_BSIZE must be a multiple of DEV_BSIZE\n"); memset(&lastlevel, 0, NUM_STR_SIZE); memset(&level, 0, NUM_STR_SIZE); + transformation = &transformation_null; /* Default dump level is zero. */ level[0] = '0'; @@ -359,6 +363,7 @@ main(int argc, char *argv[]) case 'j': compressed = 2; zipflag = COMPRESS_BZLIB; + transformation = transformation_bzlib_factory(1, 2); if (optarg) compressed = numarg("compress level", 1L, 9L); break; @@ -450,6 +455,7 @@ main(int argc, char *argv[]) #ifdef HAVE_LZO case 'y': compressed = 2; + transformation = transformation_lzo_factory(1); zipflag = COMPRESS_LZO; break; #endif /* HAVE_LZO */ @@ -457,6 +463,7 @@ main(int argc, char *argv[]) #ifdef HAVE_ZLIB case 'z': compressed = 2; + transformation = transformation_zlib_factory(1, 2); zipflag = COMPRESS_ZLIB; if (optarg) compressed = numarg("compress level", 1L, 9L); @@ -793,7 +800,7 @@ main(int argc, char *argv[]) if (zipflag == COMPRESS_LZO) msg("Compressing output (lzo)\n"); else - msg("Compressing output at compression level %d (%s)\n", + msg("Compressing output at transformation level %d (%s)\n", compressed, zipflag == COMPRESS_ZLIB ? "zlib" : "bzlib"); } } diff --git a/dump/tape.c b/dump/tape.c index ec1b311..e0b5411 100644 --- a/dump/tape.c +++ b/dump/tape.c @@ -37,7 +37,7 @@ #ifndef lint static const char rcsid[] = - "$Id: tape.c,v 1.94 2011/06/10 12:41:54 stelian Exp $"; + "$Id: tape.c,v 1.95 2011/06/10 13:07:29 stelian Exp $"; #endif /* not lint */ #include @@ -92,18 +92,6 @@ int write(), read(); #include -#ifdef HAVE_ZLIB -#include -#endif /* HAVE_ZLIB */ - -#ifdef HAVE_BZLIB -#include -#endif /* HAVE_BZLIB */ - -#ifdef HAVE_LZO -#include -#endif /* HAVE_LZO */ - #include "dump.h" int writesize; /* size of malloc()ed buffer for tape */ @@ -286,7 +274,6 @@ alloctape(void) void writerec(const void *dp, int isspcl) { - slp->req[trecno].dblk = (ext2_loff_t)0; slp->req[trecno].count = 1; /* XXX post increment triggers an egcs-1.1.2-12 bug on alpha/sparc */ @@ -1003,6 +990,7 @@ restore_check_point: tapeno, slp->inode); if (tapeno < (int)TP_NINOS) volinfo[tapeno] = slp->inode; + transformation->startNewTape(transformation, NULL, 0); } } @@ -1176,15 +1164,13 @@ doslave(int cmd, int nextslave; volatile int wrote = 0, size, eot_count, bufsize; char * volatile buffer; -#if defined(HAVE_ZLIB) || defined(HAVE_BZLIB) || defined(HAVE_LZO) +#if defined(HAVE_BLOCK_TRANSFORMATION) struct tapebuf * volatile comp_buf = NULL; int compresult; volatile int do_compress = !first; unsigned long worklen; -#ifdef HAVE_LZO - lzo_align_t __LZO_MMODEL *LZO_WorkMem; -#endif -#endif /* HAVE_ZLIB || HAVE_BZLIB || HAVE_LZO */ +#endif /* HAVE_BLOCK_TRANSFORMATION */ + struct slave_results returns; #ifdef __linux__ errcode_t retval; @@ -1205,6 +1191,10 @@ doslave(int cmd, sigprocmask(SIG_BLOCK, &set, NULL); sigemptyset(&set); +#ifdef HAVE_BLOCK_TRANSFORMATION + transformation->startDiskIOProcess(transformation); +#endif /* HAVE_BLOCK_TRANSFORMATION */ + /* * Need our own seek pointer. */ @@ -1229,34 +1219,29 @@ doslave(int cmd, quit("master/slave protocol botched - didn't get pid of next slave.\n"); } -#if defined(HAVE_ZLIB) || defined(HAVE_BZLIB) || defined(HAVE_LZO) +#if defined(HAVE_BLOCK_TRANSFORMATION) /* if we're doing a compressed dump, allocate the compress buffer */ if (compressed) { int bsiz = sizeof(struct tapebuf) + writesize; - /* Add extra space to deal with compression enlarging the buffer */ - if (TP_BSIZE > writesize/16 + 67) + /* Add extra space to deal with compression or encryption enlarging the buffer */ + if (TP_BSIZE > writesize/16 + 200) bsiz += TP_BSIZE; else - bsiz += writesize/16 + 67; + bsiz += writesize/16 + 200; comp_buf = malloc(bsiz); if (comp_buf == NULL) quit("couldn't allocate a compress buffer.\n"); + transformation->initialize(transformation, 1); if (zipflag == COMPRESS_ZLIB) comp_buf->flags = COMPRESS_ZLIB; else if (zipflag == COMPRESS_BZLIB) comp_buf->flags = COMPRESS_BZLIB; - else if (zipflag == COMPRESS_LZO) { + else if (zipflag == COMPRESS_LZO) comp_buf->flags = COMPRESS_LZO; - if (lzo_init() != LZO_E_OK) quit("lzo_init failed\n"); - } else + else quit("internal error - unknown compression method: %d\n", zipflag); } -#ifdef HAVE_LZO - LZO_WorkMem = malloc(LZO1X_1_MEM_COMPRESS); - if (!LZO_WorkMem) - quit("couldn't allocate a compress buffer.\n"); -#endif -#endif /* HAVE_ZLIB || HAVE_BZLIB || HAVE_LZO */ +#endif /* HAVE_BLOCK_TRANSFORMATION */ /* * Get list of blocks to dump, read the blocks into tape buffer @@ -1275,6 +1260,7 @@ doslave(int cmd, for (trecno = 0; trecno < ntrec; trecno += p->count, p += p->count) { + if (p->dblk) { /* read a disk block */ bread(p->dblk, slp->tblock[trecno], p->count * TP_BSIZE); @@ -1296,7 +1282,7 @@ doslave(int cmd, bufsize = writesize; /* length to write */ returns.clen = returns.unclen = bufsize; -#if defined(HAVE_ZLIB) || defined(HAVE_BZLIB) || defined(HAVE_LZO) +#if defined(HAVE_BLOCK_TRANSFORMATION) /* * When writing a compressed dump, each block except * the first one on each tape is written @@ -1309,56 +1295,15 @@ doslave(int cmd, * The first block written by each slave is not compressed * and does not have a prefix. */ - if (compressed && do_compress) { comp_buf->length = bufsize; worklen = TP_BSIZE + writesize; compresult = 1; -#ifdef HAVE_ZLIB - if (zipflag == COMPRESS_ZLIB) { - compresult = compress2(comp_buf->buf, - &worklen, - (char *)slp->tblock[0], - writesize, - compressed); - if (compresult == Z_OK) - compresult = 1; - else - compresult = 0; - } -#endif /* HAVE_ZLIB */ -#ifdef HAVE_BZLIB - if (zipflag == COMPRESS_BZLIB) { - unsigned int worklen2 = worklen; - compresult = BZ2_bzBuffToBuffCompress( - comp_buf->buf, - &worklen2, - (char *)slp->tblock[0], - writesize, - compressed, - 0, 30); - worklen = worklen2; - if (compresult == BZ_OK) - compresult = 1; - else - compresult = 0; - } -#endif /* HAVE_BZLIB */ -#ifdef HAVE_LZO - if (zipflag == COMPRESS_LZO) { - lzo_uint worklen2 = worklen; - compresult = lzo1x_1_compress((char *)slp->tblock[0],writesize, - comp_buf->buf, - &worklen2, - LZO_WorkMem); - worklen = worklen2; - if (compresult == LZO_E_OK) - compresult = 1; - else - compresult = 0; - } -#endif /* HAVE_LZO */ + // tapebuf: compressed, flags, length + compresult = transformation->compress(transformation, comp_buf, + &worklen, slp->tblock[0], writesize); + if (compresult && worklen <= ((unsigned long)writesize - 16)) { /* write the compressed buffer */ comp_buf->length = worklen; @@ -1378,7 +1323,7 @@ doslave(int cmd, } /* compress the remaining blocks if we're compressing */ do_compress = compressed; -#endif /* HAVE_ZLIB || HAVE_BZLIB || HAVE_LZO */ +#endif /* HAVE_BLOCK_TRANSFORMATION */ if (sigsetjmp(jmpbuf2, 1) == 0) { ready2 = 1; @@ -1478,6 +1423,11 @@ doslave(int cmd, } #endif /* USE_QFA */ } + +#ifdef HAVE_BLOCK_TRANSFORMATION + transformation->endDiskIOProcess(transformation); +#endif /* HAVE_BLOCK_TRANSFORMATION */ + if (nread != 0) quit("error reading command pipe: %s\n", strerror(errno)); } @@ -1541,7 +1491,7 @@ SetLogicalPos(void) /* * read the current tape position */ -static int +int GetTapePos(long long *pos) { int err = 0; diff --git a/dump/traverse.c b/dump/traverse.c index 34be0ec..7bdfef1 100644 --- a/dump/traverse.c +++ b/dump/traverse.c @@ -37,7 +37,7 @@ #ifndef lint static const char rcsid[] = - "$Id: traverse.c,v 1.72 2011/02/21 10:36:47 stelian Exp $"; + "$Id: traverse.c,v 1.73 2011/06/10 13:07:29 stelian Exp $"; #endif /* not lint */ #include @@ -868,8 +868,9 @@ dump_xattr(dump_ino_t ino, struct dinode *dp) { void dumpino(struct dinode *dp, dump_ino_t ino, int metaonly) { - unsigned long cnt; - fsizeT size, remaining; + //unsigned long cnt; + //fsizeT size; + fsizeT remaining; char buf[TP_BSIZE]; struct new_bsd_inode nbi; int i; @@ -1348,6 +1349,7 @@ mkchecksum(union u_spcl *tmpspcl) void writeheader(dump_ino_t ino) { + char *state; /* need to have some place to put this! */ spcl.c_inumber = ino; spcl.c_magic = NFS_MAGIC; mkchecksum((union u_spcl *)&spcl); diff --git a/rmt/Makefile.in b/rmt/Makefile.in index f64f5fa..43413ff 100644 --- a/rmt/Makefile.in +++ b/rmt/Makefile.in @@ -1,4 +1,4 @@ -# $Id: Makefile.in,v 1.10 2003/05/08 21:11:39 stelian Exp $ +# $Id: Makefile.in,v 1.11 2011/06/10 13:07:29 stelian Exp $ top_srcdir= @top_srcdir@ srcdir= @srcdir@ @@ -25,7 +25,7 @@ $(PROG): $(OBJS) $(DEPLIBS) $(LD) $(ALL_LDFLAGS) -o $(PROG) $(OBJS) $(LIBS) ermt: ermt.o cipher.o $(DEPLIBS) - $(LD) $(ALL_LDFLAGS) -o ermt ermt.o cipher.o $(LIBS) @CRYPTO@ + $(LD) $(ALL_LDFLAGS) -o ermt ermt.o cipher.o $(LIBS) ermt.o: rmt.c $(CC) -c $(ALL_CFLAGS) -DERMT -o ermt.o rmt.c diff --git a/testing/Makefile.in b/testing/Makefile.in new file mode 100644 index 0000000..2d85d89 --- /dev/null +++ b/testing/Makefile.in @@ -0,0 +1,38 @@ +# $Id: Makefile.in,v 1.1 2011/06/10 13:07:29 stelian Exp $ + +top_srcdir= @top_srcdir@ +srcdir= @srcdir@ +top_builddir= .. + +@MCONFIG@ + +PROG= transform_test +INC= -I$(top_srcdir)/dump -I$(top_srcdir)/common +ALL_CFLAGS= @CPPFLAGS@ @CFLAGS@ @CCOPTS@ -pipe $(OPT) $(GINC) $(INC) $(DEFS) @DUMPDEBUG@ +ALL_LDFLAGS= @LDFLAGS@ @LDOPTS@ @STATIC@ +LIBS= $(GLIBS) @ZLIB@ @BZLIB@ @BLKID@ @SSLLIB@ +SRCS= transform_test.c ../common/transformation_null.c ../common/transformation_lzo.c ../common/transformation_zlib.c ../common/transformation_bzlib.c ../common/transformation_ssl.c +OBJS= transform_test.o ../common/transformation_null.o ../common/transformation_lzo.o ../common/transformation_zlib.o ../common/transformation_bzlib.o ../common/transformation_ssl.o + +.c.o: + $(CC) -c $(ALL_CFLAGS) $< -o $@ + +all:: $(PROG) + +$(PROG): $(OBJS) $(DEPLIBS) + $(LD) $(ALL_LDFLAGS) -o $(PROG) $(OBJS) $(LIBS) + +install:: + +clean:: + $(RM) -f \#* *.s *.o *.a *~ core + +distclean:: clean + $(RM) -f Makefile Makefile.old .depend + +# +++ Dependency line eater +++ +# +# Makefile dependencies follow. This must be the last section in +# the Makefile.in file +# + -- 2.39.2