Index: tags/1.05/makedocs.pl =================================================================== --- tags/1.05/makedocs.pl (revision 136) +++ tags/1.05/makedocs.pl (revision 136) @@ -0,0 +1,41 @@ +#!/usr/bin/perl + +use strict; + +my $base = "/home/lj/htdocs/dev/brackup/"; +my $pshb = Goats->new; +$pshb->batch_convert([qw(brackup brackup-restore lib)], $base); + +package Goats; + +use strict; +use base 'Pod::Simple::HTMLBatch'; + +sub modnames2paths { + my ($self, $dirs) = @_; + + my @files; + my @dirs; + + foreach my $path (@{$dirs || []}) { + if (-f $path) { + push @files, $path; + } else { + push @dirs, $path; + } + } + + my $m2p = $self->SUPER::modnames2paths(\@dirs); + + foreach my $file (@files) { + my ($tail) = $file =~ m!([^/]+)\z!; + $m2p->{$tail} = $file; + } + + # these are symlinks in brad's lib + foreach my $k (keys %$m2p) { + delete $m2p->{$k} if $k eq "Danga::blib::lib::Danga::Socket" || $k eq "Danga::Socket"; + } + + return $m2p; +} Index: tags/1.05/t/01-backup.t =================================================================== --- tags/1.05/t/01-backup.t (revision 157) +++ tags/1.05/t/01-backup.t (revision 157) @@ -0,0 +1,30 @@ +# -*-perl-*- + +use strict; +use Test::More tests => 12; + +use Brackup::Test; +use FindBin qw($Bin); +use Brackup::Util qw(tempfile); + +############### Backup + +my ($digdb_fh, $digdb_fn) = tempfile(); +close($digdb_fh); +my $root_dir = "$Bin/data"; +ok(-d $root_dir, "test data to backup exists"); +my $backup_file = do_backup( + with_confsec => sub { + my $csec = shift; + $csec->add("path", $root_dir); + $csec->add("chunk_size", "2k"); + $csec->add("digestdb_file", $digdb_fn); + }, + ); + +############### Restore + +my $restore_dir = do_restore($backup_file); + +ok_dirs_match($restore_dir, $root_dir); + Index: tags/1.05/t/03-combine-little-files.t =================================================================== --- tags/1.05/t/03-combine-little-files.t (revision 157) +++ tags/1.05/t/03-combine-little-files.t (revision 157) @@ -0,0 +1,48 @@ +# -*-perl-*- + +use strict; +use Test::More tests => 15; + +use Brackup::Test; +use FindBin qw($Bin); +use Brackup::Util qw(tempfile); + +############### Backup + +my ($digdb_fh, $digdb_fn) = tempfile(); +close($digdb_fh); +my $root_dir = "$Bin/data"; +ok(-d $root_dir, "test data to backup exists"); + +my ($backup_file, $backup) = + do_backup( + with_confsec => sub { + my $csec = shift; + $csec->add("path", $root_dir); + $csec->add("merge_files_under", "1k"); + $csec->add("max_composite_chunk_size", "500k"); + $csec->add("digestdb_file", $digdb_fn); + }, + ); + +# see if dup files were only stored once +my %seen; +$backup->foreach_saved_file(sub { + my ($file, $slist) = @_; + return unless $file->path =~ /000-dup[12]\.txt$/; + foreach my $sc (@$slist) { + $seen{$sc->to_meta}++; + } +}); +is(scalar keys %seen, 1, "stored just one uniq copy of 000-dup[12]"); +is((%seen)[-1], 2, "and stored it twice"); +like((%seen)[0], qr/-/, "and it was stored in a range"); + + + +############### Restore + +my $restore_dir = do_restore($backup_file); + +ok_dirs_match($restore_dir, $root_dir); + Index: tags/1.05/t/data/huge-file.txt =================================================================== --- tags/1.05/t/data/huge-file.txt (revision 31) +++ tags/1.05/t/data/huge-file.txt (revision 31) @@ -0,0 +1,64 @@ +This file is absolutely gigantic. It will be cut into two chunks. + +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +The end. Index: tags/1.05/t/data/my_dir/sub_dir/program.sh =================================================================== --- tags/1.05/t/data/my_dir/sub_dir/program.sh (revision 31) +++ tags/1.05/t/data/my_dir/sub_dir/program.sh (revision 31) @@ -0,0 +1,3 @@ +#!/bin/sh + +echo "Hello, world!" Index: tags/1.05/t/data/my_dir/sub_dir/another-file.txt =================================================================== --- tags/1.05/t/data/my_dir/sub_dir/another-file.txt (revision 31) +++ tags/1.05/t/data/my_dir/sub_dir/another-file.txt (revision 31) @@ -0,0 +1,1 @@ +This is another test file. Index: tags/1.05/t/data/000-dup1.txt =================================================================== --- tags/1.05/t/data/000-dup1.txt (revision 134) +++ tags/1.05/t/data/000-dup1.txt (revision 134) @@ -0,0 +1,1 @@ +Dup Index: tags/1.05/t/data/000-dup2.txt =================================================================== --- tags/1.05/t/data/000-dup2.txt (revision 134) +++ tags/1.05/t/data/000-dup2.txt (revision 134) @@ -0,0 +1,1 @@ +Dup Index: tags/1.05/t/data/test-file.txt =================================================================== --- tags/1.05/t/data/test-file.txt (revision 31) +++ tags/1.05/t/data/test-file.txt (revision 31) @@ -0,0 +1,2 @@ +Hello, world! + Index: tags/1.05/t/data/my-link.txt =================================================================== --- tags/1.05/t/data/my-link.txt (revision 31) +++ tags/1.05/t/data/my-link.txt (revision 31) @@ -0,0 +1,1 @@ +link test-file.txt Index: tags/1.05/t/00-use.t =================================================================== --- tags/1.05/t/00-use.t (revision 80) +++ tags/1.05/t/00-use.t (revision 80) @@ -0,0 +1,3 @@ +use Test::More tests => 1; +use Brackup; +ok(1); Index: tags/1.05/t/04-gc.t =================================================================== --- tags/1.05/t/04-gc.t (revision 162) +++ tags/1.05/t/04-gc.t (revision 162) @@ -0,0 +1,39 @@ +# -*-perl-*- + +use strict; +use Test::More tests => 9; + +use Brackup::Test; +use FindBin qw($Bin); +use Brackup::Util qw(tempfile); + +############### Backup + +my ($digdb_fh, $digdb_fn) = tempfile(); +close($digdb_fh); +my $root_dir = "$Bin/data"; +ok(-d $root_dir, "test data to backup exists"); +my ($backup_file, $backup, $target) = do_backup( + with_confsec => sub { + my $csec = shift; + $csec->add("path", $root_dir); + $csec->add("chunk_size", "2k"); + $csec->add("digestdb_file", $digdb_fn); + }, + ); + +############### Add an orphan chunk + +my $orphan_chunks_count = int(rand 10); +for (1..$orphan_chunks_count) { + my $chunk = Brackup::StoredChunk->new; + $chunk->{_chunkref} = \ "foobar $_"; + $target->store_chunk($chunk); +} + +############### Do garbage collection + +my $removed_count = $target->gc; +ok($removed_count == $orphan_chunks_count, "all orphan chunks removed"); + +__END__ Index: tags/1.05/t/02-gpg.t =================================================================== --- tags/1.05/t/02-gpg.t (revision 157) +++ tags/1.05/t/02-gpg.t (revision 157) @@ -0,0 +1,49 @@ +# -*-perl-*- + +use strict; +use Test::More; + +use Brackup::Test; +use FindBin qw($Bin); +use Brackup::Util qw(tempfile); + +############### Backup + +if (`gpg --version`) { + plan tests => 12; +} else { + plan skip_all => 'gpg binary not found, skipping encrypted tests'; +} + +my $gpg_args = ["--no-default-keyring", + "--keyring=$Bin/data/pubring-test.gpg", + "--secret-keyring=$Bin/data/secring-test.gpg"]; + +my ($digdb_fh, $digdb_fn) = tempfile(); +close($digdb_fh); +my $root_dir = "$Bin/data"; +ok(-d $root_dir, "test data to backup exists"); + +my $backup_file = do_backup( + with_confsec => sub { + my $csec = shift; + $csec->add("path", $root_dir); + $csec->add("chunk_size", "2k"); + $csec->add("digestdb_file", $digdb_fn); + $csec->add("gpg_recipient", "2149C469"); + }, + with_root => sub { + my $root = shift; + $root->{gpg_args} = $gpg_args; + }, + ); + +############### Restore + +my $restore_dir = do { + local @Brackup::GPG_ARGS = @$gpg_args; + do_restore($backup_file); +}; + +ok_dirs_match($restore_dir, $root_dir); + Index: tags/1.05/TODO =================================================================== --- tags/1.05/TODO (revision 153) +++ tags/1.05/TODO (revision 153) @@ -0,0 +1,54 @@ +-- in Restore.pm: + # TODO: inefficient! we don't want to download the chunk from the + # target multiple times. better to cache it locally, or at least + # only fetch a region from the target (but that's still kinda inefficient + # and pushes complexity into the Target interface) + +-- only use tempfiles in a 0700 directory under /tmp/brackup-$USER/ or + wherever. or option, when using encryption, to only doing + in-memory tempfiles, to avoid hitting disk? + +-- : if a file which existed when Brackup was + "discovering files" subsequently goes away while Brackup is working + it's magic (like a vim .swp file) Brackup tosses an error and bails: + +-- shouldn't backup the digest file (.brackup-digest.db) + +-- don't skip files ending in whitespace. figure out why GPG barfs. would + the metafile also barf, having a trailing \r or \n? + +-- "brackup-target gc": find/clean orphan chunks on a target + from metafiles + +-- figure out how files with intenal \r or \n in filename get written + to metafile. need to be escaped? + +-- FUSE script to mount a *.brackup file + +-- tool to clean digestcache, based on prefixes, looking for files that no longer + exist or have new mtimes, etc? or keep track of "last used" date + field in the digestcache and just delete things that are too old? + but then no longer just a dictionary. SQLite would work, but we'd ideally + like lots of dumber cache mechanisms. maybe a ->clean method is optional? + then a memcached backend/etc doesn't have to use it. + +-- Tools to rebuild your inventory database from the target's enumeration + of its chunks and the target's *.brackup metafiles isn't yet done, but + would be pretty easy. + +-- ionice stuff. network nice stuff. + +-- make tests pass without 'noatime' mount option + +-- --ignore-debian-files option (if managed by a package management + system (and not an unmodified conffile), don't back it up) + +-- reuse tempfiles in Chunk.pm as Restore.pm does + +-- restoring from existing config file. + +-- better test coverage. currently at 84%. should ignore + test coverage of test modules. move Brackup::Test to + t/lib/ probably + +-- should do TODOs in code. :) Index: tags/1.05/doc/exampleconfig.txt =================================================================== --- tags/1.05/doc/exampleconfig.txt (revision 4) +++ tags/1.05/doc/exampleconfig.txt (revision 4) @@ -0,0 +1,21 @@ +# a sample ~/.brackup.conf + +[TARGET:raidbackups] +type = Filesystem # this can be any Brackup::Target:: subclass +path = /raid/backup/brackup + +[SOURCE:proj] +path = /raid/bradfitz/proj/ +chunk_size = 5m +gpg_recipient = 5E1B3EC5 + +[SOURCE:bradhome] +chunk_size = 64MB +path = /raid/bradfitz/ +ignore = ^\.thumbnails/ +ignore = ^\.kde/share/thumbnails/ +ignore = ^\.ee/minis/ +ignore = ^build/ +ignore = ^(gqview|nautilus)/thumbnails/ + + Index: tags/1.05/doc/todo.txt =================================================================== --- tags/1.05/doc/todo.txt (revision 80) +++ tags/1.05/doc/todo.txt (revision 80) @@ -0,0 +1,28 @@ +Chunk description in .bracup file: + Chunks: offset;raw_length;stored_length;typed_digest +Proposal to change: + Chunks: offset;raw_length;stored_length;typed_digest(stored);typed_digest(raw);flags + +Where flags is comma separate list of \w+. e.g. "gz" for gzip compression + +------ + +Document: purpose of chunks named by their final digest is twofold: + 1) can verify integrity of storage medium. is it corrupt? + 2) hides proof of ownership of contents (when encrypted) +side-effect: + -- when we do compression, we'll be consistent and store it as its + compressed digest, even if not encrypted as well. + +--- + +Maybe we don't need per-chunk meta files: + -- can get it all from .brackup (meta)files on the server. + (TODO: abstract out parser for multiple users) + +--- + +smart chunk-sizing on certain files w/ metadata and data separate: +like mp3 files and their id3. have a smart chunker that's the data +part vs. the id3 part, so updating id3 later doesn't reupload the +entire data part. :-) Index: tags/1.05/doc/overview.txt =================================================================== --- tags/1.05/doc/overview.txt (revision 5) +++ tags/1.05/doc/overview.txt (revision 5) @@ -0,0 +1,37 @@ +Originally posted to: + + +There are lots of ways to store files on the net lately: + +-- Amazon S3 is the most interesting, +-- Google's rumored GDrive is surely soon coming +-- Apple has .Mac + +I want to back up to them. And more than one. So first off, abstract +out net-wide storage.... my backup tool (wsbackup) isn't targetting +one. They're all just providers. + +Also, don't trust sending my data in cleartext, and having it stored +in cleartext, so public key encryption is a must. Then I can run +automated backups from many hosts, without much fear of keys being +compromised. + +Don't want people being able to do size-analysis, and huge files are a pain anyway, so big files are cut into chunks. + +Files stored on Amazon/Google are of form: + +-- meta files: backup_rootname-yyyymmddnn.meta, encrypted (YAML?) file mapping relative paths from backup directory root to the stat() information, original SHA1, and array of chunk keys (SHA1s of encrypted chunks) that comprise the file. + +-- [sha1ofencryptedchunk].chunk -- content being <= ,say, 20MB chunk of encrypted data. + +Then every night different hosts/laptops recurse directory trees, +consult a stat() cache (on,say, inode number, mtime, size, whatever) +and do SHA1 calculations on changed files, lookup rest from cache, and +build the metafile, upload any new chunks, encrypt the metafile, +upload the metafile. + +Result: + +-- I can restore any host from any point in time, with Amazon/Google + storing all my data, and only paying $0.15 cents/GB-month. + Index: tags/1.05/doc/databases.txt =================================================================== --- tags/1.05/doc/databases.txt (revision 89) +++ tags/1.05/doc/databases.txt (revision 89) @@ -0,0 +1,27 @@ + +inventory DB: +------------- + default loc: + [TARGET:foo]'s "inventory_db" key, or + "$ENV{HOME}/.brackup-target-$name.invdb"; + + mapping: + + pchunk->inventory_key ---> join(" ", $schunk->backup_digest, $schunk->backup_length)) + ;to= + ;raw + + +Digest DB: +---------- + default loc: + [SOURCE:foo]'s 'digestdb_file' key, or + "$SOURCE_DIR/.brackup-digest.db" + + table: "digest_cache" + + => "sha1:xxxxxxxxx" + + cache_key ::= + []:,,, + Index: tags/1.05/doc/design-decisions.txt =================================================================== --- tags/1.05/doc/design-decisions.txt (revision 43) +++ tags/1.05/doc/design-decisions.txt (revision 43) @@ -0,0 +1,20 @@ +-- you should be able to restore without setting up a config file. + if you lost data, that'd be annoying. restoring from a config + file will be supported in the future, but it's not yet. + +-- backups must be automatable, never requiring user input. hence public + key encryption. + +-- restores may prompt for user input ("What's your Amazon S3 + password?" and "Enter your GPG passphrase."), because they won't be + automated or common. and I don't want a restore to require a fully + setup ~/.brackup.conf. You probably lost it anyway. So a *.brackup + metafile (the one you get after a backup) should contain all the + metadata necessary to restore (say, Amazon S3 username), but not + secret stuff. + +-- targets shouldn't include passwords (say, Amazon S3 password) + in the *.brackup (backup "index"/"meta" file). let the user + enter that on restore. you should, however, put in metadata + that'll ease restoring.... like Amazon username, or path, etc. + Index: tags/1.05/doc/notes.txt =================================================================== --- tags/1.05/doc/notes.txt (revision 79) +++ tags/1.05/doc/notes.txt (revision 79) @@ -0,0 +1,73 @@ +Chunk has: + + -- raw digest (always) + + -- file (many) + -- offset (many) + -- length (many) + + -- enc digest (many) + +Chunk description in .bracup file: + Chunks: offset;raw_length;stored_length;typed_digest + +Proposal to change: + Chunks: offset;raw_length;stored_length;typed_digest(stored);typed_digest(raw);flags + +Where flags is comma separate list of \w+. e.g. "gz" for gzip compression + +---- + +PositionedChunk (subclass of RawChunk) + - has a: + file + offset + RawChunk + - used by: + $file->foreach_chunk(sub { my $poschunk = shift; }); + restoring stuff? + - can: + write back to disk? + +RawChunk + - has a: + length + digest + contents + - used by: + positionedchunk. + +ChunkHandle + - has a: + digest of stored chunk + - used by: + return value from asking target if it has a raw chunk, + or after it stores a raw chunk. + +StoredChunk + + + + + + +------ + +Document: purpose of chunks named by their final digest is twofold: + 1) can verify integrity of storage medium. is it corrupt? + 2) hides proof of ownership of contents (when encrypted) +side-effect: + -- when we do compression, we'll be consistent and store it as its + compressed digest, even if not encrypted as well. + +Maybe we don't need per-chunk meta files: + -- can get it all from .brackup (meta)files on the server. + (TODO: abstract out parser for multiple users) + +--- + +smart chunk-sizing on certain files w/ metadata and data separate: +like mp3 files and their id3. have a smart chunker that's the data +part vs. the id3 part, so updating id3 later doesn't reupload the +entire data part. :-) + Index: tags/1.05/doc/data-structures.txt =================================================================== --- tags/1.05/doc/data-structures.txt (revision 58) +++ tags/1.05/doc/data-structures.txt (revision 58) @@ -0,0 +1,115 @@ +This file documents the main classes, data structures, file formats, +etc used in Brackup. + +---------------------------------------------------------------------------- +Class-wise, we have: +---------------------------------------------------------------------------- + + Root -- describes a path on the filesystem to be backed up. has as + properties how small large files are cut up into + ("chunk_size"), what files to ignore, and the encryption + settings. + + Target -- a destination for the backups. + + File -- a directory, symlink, or file in a Root. + + Chunk -- part of a file, defined as an offset and length. depending + on encryption settings, the serialized backup length can be + more or less than the unencrypted length. + + Backup -- a snapshot in time of all a Root's Files and Chunks. + during the backup, the Target is consulted to see if it + has chunks before they're re-stored. The backup upon + completion writes a structured file as described below. + + DigestDatabase -- the digest database, a property of the Root, acts + mostly as a cache, but is pretty important when + using encryption. If you lose the database, all your + files will need to be re-encrypted, as Brackup won't + know if the chunks already exist, as encryption makes + different files each time. Note that you don't need + the digest database to do a restore. + + +---------------------------------------------------------------------------- +DigestDatabase +---------------------------------------------------------------------------- + +The digest database is an SQLite file, but in theory can be anything +that implements a dictionary (get/set keys/values). + +The keys/values used are: + + --> + + --> + +Where: + + FileCacheKey ::= "[" "]" ":" join(",", , , , ) + + ChunkCacheKey ::= "-" "-" "-" + + ChunkDetails ::= " " + + TypedDigest ::= ":" + + DigestAlgo ::= { "sha1" } + + +---------------------------------------------------------------------------- +[backup-name].brackup format (RFC-822-like) +---------------------------------------------------------------------------- + +Keys: +----- + + Path: relative path + Size: unencrypted size + Digest: unencrypted digest (see TypedDigest format above) + Type: "l" for symlink, "d" for directory, else regular file if blank + Link: the symlink's target + Chunks: whitespace-separated "offset;length;enclength;encdigest" + +Example: +-------- + +Path: Some file.dat +Size: 4550656 +Digest: sha1:f822dd41714070a09df1cf19e80a12720ed20b43 +Chunks: 0;1048576;1032436;sha1:a303f69348cf6e4c40faf199e11d6705eb200eed + 1048576;1048576;1041619;sha1:95e81460845f27940d209b5482c672e3ad0e8646 + 2097152;1048576;1041937;sha1:a7b9d3eb26cb7b9969032d62576c0c1634ed8665 + 3145728;1048576;1042934;sha1:645689dfc08e35851ccfb4e9d2d3eb69a684ef92 + 4194304;356352;343473;sha1:14e65a999edd9f2a54fc218abbee07611c9743b9 + +Path: Another file.dat +Size: 3184274 +Digest: sha1:f7e3c4b75fe041f58464c36583fec1f4361a4676 +Chunks: 0;1048576;1030710;sha1:af185012fcf3d178c863b2aaef76f3f83863f579 + 1048576;1048576;1036044;sha1:1c08a500fba4751aea5d617a92f13373d0fd057e + 2097152;1048576;1035307;sha1:313f9ce3ba8a5e9c5361c587fed4e55d720e48c4 + 3145728;38546;38510;sha1:de1687f379f8b4ce505f0ee5652f1c85505fb5be + +Path: trunk/brackup +Size: 1510 +Digest: sha1:9242d98205094044a938e79b94a1fc505bdf50fe +Chunks: 0;1510;1819;sha1:34ddb242c4d88a4df82145de2b04dd6c0d26cd58 + +Path: trunk/brackup.dat +Size: 15151 +Digest: sha1:1e427622cadb31ea006c273b86457178f38a7c75 +Chunks: 0;15151;5096;sha1:5672f3d6ee89d0c7a039fc050ad4c315f3580533 + +Path: trunk/B_TO_THE_BIZZLE +Type: l +Link: F_TO_THE_FIZZLE + +Path: trunk/.svn +Type: d + +Path: trunk/.svn/entries +Size: 686 +Digest: sha1:9a9b269fa1c7ae74ca0a1a08f028c4e294bf9128 +Chunks: 0;686;1465;sha1:346d53cd2366efb4cb8b4ae918e860f0244dbd5d Index: tags/1.05/MANIFEST =================================================================== --- tags/1.05/MANIFEST (revision 162) +++ tags/1.05/MANIFEST (revision 162) @@ -0,0 +1,55 @@ +brackup +brackup-restore +brackup-target +Changes +doc/data-structures.txt +doc/databases.txt +doc/design-decisions.txt +doc/exampleconfig.txt +doc/notes.txt +doc/overview.txt +doc/todo.txt +lib/Brackup.pm +lib/Brackup/Backup.pm +lib/Brackup/BackupStats.pm +lib/Brackup/ChunkIterator.pm +lib/Brackup/CompositeChunk.pm +lib/Brackup/Config.pm +lib/Brackup/ConfigSection.pm +lib/Brackup/Dict/SQLite.pm +lib/Brackup/DigestCache.pm +lib/Brackup/File.pm +lib/Brackup/GPGProcess.pm +lib/Brackup/GPGProcManager.pm +lib/Brackup/InventoryDatabase.pm +lib/Brackup/Manual/Overview.pod +lib/Brackup/Metafile.pm +lib/Brackup/PositionedChunk.pm +lib/Brackup/Restore.pm +lib/Brackup/Root.pm +lib/Brackup/StoredChunk.pm +lib/Brackup/Target.pm +lib/Brackup/Target/Amazon.pm +lib/Brackup/Target/Filesystem.pm +lib/Brackup/TargetBackupStatInfo.pm +lib/Brackup/Test.pm +lib/Brackup/Util.pm +Makefile.PL +MANIFEST This list of files +MANIFEST.SKIP +META.yml Module meta-data (added by MakeMaker) +t/00-use.t +t/01-backup.t +t/02-gpg.t +t/03-combine-little-files.t +t/04-gc.t +t/data/000-dup1.txt +t/data/000-dup2.txt +t/data/huge-file.txt +t/data/my-link.txt +t/data/my_dir/sub_dir/another-file.txt +t/data/my_dir/sub_dir/program.sh +t/data/pubring-test.gpg +t/data/secring-test.gpg +t/data/test-file.txt +TODO Index: tags/1.05/lib/Brackup.pm =================================================================== --- tags/1.05/lib/Brackup.pm (revision 162) +++ tags/1.05/lib/Brackup.pm (revision 162) @@ -0,0 +1,33 @@ +package Brackup; +use strict; +use vars qw($VERSION); +$VERSION = '1.05'; + +use Brackup::Config; +use Brackup::ConfigSection; +use Brackup::File; +use Brackup::Metafile; +use Brackup::PositionedChunk; +use Brackup::StoredChunk; +use Brackup::Backup; +use Brackup::Root; # aka "source" +use Brackup::Restore; +use Brackup::Target; +use Brackup::BackupStats; + +1; + +__END__ + +=head1 NAME + +Brackup - Flexible backup tool. Slices, dices, encrypts, and sprays across the net. + +=head1 FURTHER READING + +L + +L + +L + Index: tags/1.05/lib/Brackup/Backup.pm =================================================================== --- tags/1.05/lib/Brackup/Backup.pm (revision 135) +++ tags/1.05/lib/Brackup/Backup.pm (revision 135) @@ -0,0 +1,340 @@ +package Brackup::Backup; +use strict; +use warnings; +use Carp qw(croak); +use Brackup::ChunkIterator; +use Brackup::CompositeChunk; +use Brackup::GPGProcManager; +use Brackup::GPGProcess; + +sub new { + my ($class, %opts) = @_; + my $self = bless {}, $class; + + $self->{root} = delete $opts{root}; # Brackup::Root + $self->{target} = delete $opts{target}; # Brackup::Target + $self->{dryrun} = delete $opts{dryrun}; # bool + $self->{verbose} = delete $opts{verbose}; # bool + + $self->{modecounts} = {}; # type -> mode(octal) -> count + + $self->{saved_files} = []; # list of Brackup::File objects backed up + + croak("Unknown options: " . join(', ', keys %opts)) if %opts; + + return $self; +} + +# returns true (a Brackup::BackupStats object) on success, or dies with error +sub backup { + my ($self, $backup_file) = @_; + + my $root = $self->{root}; + my $target = $self->{target}; + + my $stats = Brackup::BackupStats->new; + + my $gpg_rcpt = $self->{root}->gpg_rcpt; + + my $n_kb = 0.0; # num: kb of all files in root + my $n_files = 0; # int: # of files in root + my $n_kb_done = 0.0; # num: kb of files already done with (uploaded or skipped) + + # if we're pre-calculating the amount of data we'll + # actually need to upload, store it here. + my $n_kb_up = 0.0; + my $n_kb_up_need = 0.0; # by default, not calculated/used. + + my $n_files_done = 0; # int + my @files; # Brackup::File objs + + $self->debug("Discovering files in ", $root->path, "...\n"); + $root->foreach_file(sub { + my ($file) = @_; # a Brackup::File + push @files, $file; + $n_files++; + $n_kb += $file->size / 1024; + }); + + $self->debug("Number of files: $n_files\n"); + + # calc needed chunks + if ($ENV{CALC_NEEDED}) { + my $fn = 0; + foreach my $f (@files) { + $fn++; + if ($fn % 100 == 0) { warn "$fn / $n_files ...\n"; } + foreach my $pc ($f->chunks) { + if ($target->stored_chunk_from_inventory($pc)) { + $pc->forget_chunkref; + next; + } + $n_kb_up_need += $pc->length / 1024; + $pc->forget_chunkref; + } + } + warn "kb need to upload = $n_kb_up_need\n"; + } + + + my $chunk_iterator = Brackup::ChunkIterator->new(@files); + + my $gpg_iter; + my $gpg_pm; # gpg ProcessManager + if ($gpg_rcpt) { + ($chunk_iterator, $gpg_iter) = $chunk_iterator->mux_into(2); + $gpg_pm = Brackup::GPGProcManager->new($gpg_iter, $target); + } + + my $cur_file; # current (last seen) file + my @stored_chunks; + my $file_has_shown_status = 0; + + my $end_file = sub { + return unless $cur_file; + $self->add_file($cur_file, [ @stored_chunks ]); + $n_files_done++; + $n_kb_done += $cur_file->size / 1024; + $cur_file = undef; + }; + my $show_status = sub { + # use either size of files in normal case, or if we pre-calculated + # the size-to-upload (by looking in inventory, then we'll show the + # more accurate percentage) + my $percdone = 100 * ($n_kb_up_need ? + ($n_kb_up / $n_kb_up_need) : + ($n_kb_done / $n_kb)); + my $mb_remain = ($n_kb_up_need ? + ($n_kb_up_need - $n_kb_up) : + ($n_kb - $n_kb_done)) / 1024; + + $self->debug(sprintf("* %-60s %d/%d (%0.02f%%; remain: %0.01f MB)", + $cur_file->path, $n_files_done, $n_files, $percdone, + $mb_remain)); + }; + my $start_file = sub { + $end_file->(); + $cur_file = shift; + @stored_chunks = (); + $show_status->() if $cur_file->is_dir; + if ($gpg_iter) { + # catch our gpg iterator up. we want it to be ahead of us, + # nothing iteresting is behind us. + $gpg_iter->next while $gpg_iter->behind_by > 1; + } + $file_has_shown_status = 0; + }; + + my $merge_under = $root->merge_files_under; + my $comp_chunk = undef; + + # records are either Brackup::File (for symlinks, directories, etc), or + # PositionedChunks, in which case the file can asked of the chunk + while (my $rec = $chunk_iterator->next) { + if ($rec->isa("Brackup::File")) { + $start_file->($rec); + next; + } + my $pchunk = $rec; + if ($pchunk->file != $cur_file) { + $start_file->($pchunk->file); + } + + # have we already stored this chunk before? (iterative backup) + my $schunk; + if ($schunk = $target->stored_chunk_from_inventory($pchunk)) { + $pchunk->forget_chunkref; + push @stored_chunks, $schunk; + next; + } + + # weird case... have we stored this same pchunk digest in the + # current comp_chunk we're building? these aren't caught by + # the above inventory check, because chunks in a composite + # chunk aren't added to the inventory until after the the composite + # chunk has fully grown (because it's not until it's fully grown + # that we know the handle for it, its digest) + if ($comp_chunk && ($schunk = $comp_chunk->stored_chunk_from_dup_internal_raw($pchunk))) { + $pchunk->forget_chunkref; + push @stored_chunks, $schunk; + next; + } + + $show_status->() unless $file_has_shown_status++; + $self->debug(" * storing chunk: ", $pchunk->as_string, "\n"); + + unless ($self->{dryrun}) { + $schunk = Brackup::StoredChunk->new($pchunk); + + # encrypt it + if ($gpg_rcpt) { + $schunk->set_encrypted_chunkref($gpg_pm->enc_chunkref_of($pchunk)); + } + + # see if we should pack it into a bigger blob + my $chunk_size = $schunk->backup_length; + + # see if we should merge this chunk (in this case, file) together with + # other small files we encountered earlier, into a "composite chunk", + # to be stored on the target in one go. + + # Note: no technical reason for only merging small files (is_entire_file), + # and not the tails of larger files. just don't like the idea of files being + # both split up (for big head) and also merged together (for little end). + # would rather just have 1 type of magic per file. (split it or join it) + if ($merge_under && $chunk_size < $merge_under && $pchunk->is_entire_file) { + if ($comp_chunk && ! $comp_chunk->can_fit($chunk_size)) { + $self->debug("Finalizing composite chunk $comp_chunk..."); + $comp_chunk->finalize; + $comp_chunk = undef; + } + $comp_chunk ||= Brackup::CompositeChunk->new($root, $target); + $comp_chunk->append_little_chunk($schunk); + } else { + # store it regularly, as its own chunk on the target + $target->store_chunk($schunk) + or die "Chunk storage failed.\n"; + $target->add_to_inventory($pchunk => $schunk); + } + + # if only this worked... (LWP protocol handler seems to + # get confused by its syscalls getting interrupted?) + #local $SIG{CHLD} = sub { + # print "some child finished!\n"; + # $gpg_pm->start_some_processes; + #}; + + + $n_kb_up += $pchunk->length / 1024; + push @stored_chunks, $schunk; + } + + #$stats->note_stored_chunk($schunk); + + # DEBUG: verify it got written correctly + if ($ENV{BRACKUP_PARANOID}) { + die "FIX UP TO NEW API"; + #my $saved_ref = $target->load_chunk($handle); + #my $saved_len = length $$saved_ref; + #unless ($saved_len == $chunk->backup_length) { + # warn "Saved length of $saved_len doesn't match our length of " . $chunk->backup_length . "\n"; + # die; + #} + } + + $pchunk->forget_chunkref; + $schunk->forget_chunkref if $schunk; + } + $end_file->(); + $comp_chunk->finalize if $comp_chunk; + + unless ($self->{dryrun}) { + # write the metafile + $self->debug("Writing metafile ($backup_file)"); + open (my $metafh, ">$backup_file") or die "Failed to open $backup_file for writing: $!\n"; + print $metafh $self->backup_header; + $self->foreach_saved_file(sub { + my ($file, $schunk_list) = @_; + print $metafh $file->as_rfc822($schunk_list, $self); # arrayref of StoredChunks + }); + close $metafh or die; + + my $contents; + + # store the metafile, encrypted, on the target + if ($gpg_rcpt) { + my $encfile = $backup_file . ".enc"; + system($self->{root}->gpg_path, $self->{root}->gpg_args, + "--trust-model=always", + "--recipient", $gpg_rcpt, "--encrypt", "--output=$encfile", "--yes", $backup_file) + and die "Failed to run gpg while encryping metafile: $!\n"; + $contents = _contents_of($encfile); + unlink $encfile; + } else { + $contents = _contents_of($backup_file); + } + + # store it on the target + $self->debug("Storing metafile to " . ref($target)); + my $name = $self->{root}->publicname . "-" . $self->backup_time; + $target->store_backup_meta($name, $contents); + } + + return $stats; +} + +sub _contents_of { + my $file = shift; + open (my $fh, $file) or die "Failed to read contents of $file: $!\n"; + return do { local $/; <$fh>; }; +} + +sub default_file_mode { + my $self = shift; + return $self->{_def_file_mode} ||= $self->_default_mode('f'); +} + +sub default_directory_mode { + my $self = shift; + return $self->{_def_dir_mode} ||= $self->_default_mode('d'); +} + +sub _default_mode { + my ($self, $type) = @_; + my $map = $self->{modecounts}{$type} || {}; + return (sort { $map->{$b} <=> $map->{$a} } keys %$map)[0]; +} + +sub backup_time { + my $self = shift; + return $self->{backup_time} ||= time(); +} + +sub backup_header { + my $self = shift; + my $ret = ""; + my $now = $self->backup_time; + $ret .= "BackupTime: " . $now . " (" . localtime($now) . ")\n"; + $ret .= "BackupDriver: " . ref($self->{target}) . "\n"; + if (my $fields = $self->{target}->backup_header) { + foreach my $k (keys %$fields) { + die "Bogus header field from driver" unless $k =~ /^\w+$/; + my $val = $fields->{$k}; + die "Bogus header value from driver" if $val =~ /[\r\n]/; + $ret .= "Driver-$k: $val\n"; + } + } + $ret .= "RootName: " . $self->{root}->name . "\n"; + $ret .= "RootPath: " . $self->{root}->path . "\n"; + $ret .= "DefaultFileMode: " . $self->default_file_mode . "\n"; + $ret .= "DefaultDirMode: " . $self->default_directory_mode . "\n"; + if (my $rcpt = $self->{root}->gpg_rcpt) { + $ret .= "GPG-Recipient: $rcpt\n"; + } + $ret .= "\n"; + return $ret; +} + +sub add_file { + my ($self, $file, $handlelist) = @_; + $self->{modecounts}{$file->type}{$file->mode}++; + push @{ $self->{saved_files} }, [ $file, $handlelist ]; +} + +sub foreach_saved_file { + my ($self, $cb) = @_; + foreach my $rec (@{ $self->{saved_files} }) { + $cb->(@$rec); # Brackup::File, arrayref of Brackup::StoredChunk + } +} + +sub debug { + my ($self, @m) = @_; + return unless $self->{verbose}; + my $line = join("", @m); + chomp $line; + print $line, "\n"; +} + +1; + Index: tags/1.05/lib/Brackup/Config.pm =================================================================== --- tags/1.05/lib/Brackup/Config.pm (revision 161) +++ tags/1.05/lib/Brackup/Config.pm (revision 161) @@ -0,0 +1,180 @@ +package Brackup::Config; + +use strict; +use Brackup::ConfigSection; +use warnings; +use Carp qw(croak); + +sub new { + my ($class) = @_; + return bless {}, $class; +} + +sub add_section { + my ($self, $sec) = @_; + $self->{$sec->name} = $sec; +} + +sub load { + my ($class, $file) = @_; + $file ||= Brackup::Config->default_config_file_name; + + my $self = bless {}, $class; + + open (my $fh, $file) or do { + if (write_dummy_config($file)) { + die "Your config file needs tweaking. I put a commented-out template at: $file\n"; + } else { + die "No config file at: $file\n"; + } + }; + my $sec = undef; + while (my $line = <$fh>) { + $line =~ s/^\#.*//; # kill comments starting at beginning of line + $line =~ s/\s\#.*//; # kill comments with whitespace before the # (motivation: let # be in regexps) + $line =~ s/^\s+//; + $line =~ s/\s$//; + next unless $line ne ""; + + if ($line =~ /^\[(.+)\]$/) { + my $name = $1; + $sec = Brackup::ConfigSection->new($name); + die "Duplicate config section '$name'" if $self->{$name}; + $self->{$name} = $sec; + } elsif ($line =~ /^(\w+)\s*=\s*(.+)/) { + die "Declaration of '$1' outside of a section." unless $sec; + $sec->add($1, $2); + } else { + die "Bogus config line: $line"; + } + } + + unless ($sec) { + die "Your config file needs tweaking. There's a starting template at: $file\n"; + } + + return $self; +} + +sub default_config_file_name { + my ($class) = @_; + + if ($ENV{HOME}) { + # Default for UNIX folk + return "$ENV{HOME}/.brackup.conf"; + } + elsif ($ENV{APPDATA}) { + # For Windows users + return "$ENV{APPDATA}/brackup.conf"; + } + else { + # Fall back on the current directory + return "brackup.conf"; + } + +} + +sub write_dummy_config { + my $file = shift; + open (my $fh, ">$file") or return; + print $fh <{"SOURCE:$name"} or + die "Unknown source '$name'\n"; + + my $root = Brackup::Root->new($conf, $cache); + + # iterate over config's ignore, and add those + foreach my $pat ($conf->values("ignore")) { + $root->ignore($pat); + } + + # common things to ignore + $root->ignore(qr!~$!); + $root->ignore(qr!^\.thumbnails/!); + $root->ignore(qr!^\.kde/share/thumbnails/!); + $root->ignore(qr!^\.ee/minis/!); + $root->ignore(qr!^\.(gqview|nautilus)/thumbnails/!); + + # abort if the user had any configuration we didn't understand + if (my @keys = $conf->unused_config) { + die "Aborting, unknown configuration keys in SOURCE:$name: @keys\n"; + } + + return $root; +} + +sub load_target { + my ($self, $name) = @_; + my $confsec = $self->{"TARGET:$name"} or + die "Unknown target '$name'\n"; + + my $type = $confsec->value("type") or + die "Target '$name' has no 'type'"; + die "Invalid characters in $name's 'type'" + unless $type =~ /^\w+$/; + + my $class = "Brackup::Target::$type"; + eval "use $class; 1;" or die + "Failed to load ${name}'s driver: $@\n"; + my $target = $class->new($confsec); + + if (my @unk_config = $confsec->unused_config) { + die "Unknown config params in TARGET:$name: @unk_config\n"; + } + return $target; +} + +1; + +__END__ + +=head1 NAME + +Brackup::Config - configuration parsing/etc + +=head1 CONFIGURATION INFO + +For instructions on how to configure Brackup, see: + +L + +L + +L + + + + Index: tags/1.05/lib/Brackup/PositionedChunk.pm =================================================================== --- tags/1.05/lib/Brackup/PositionedChunk.pm (revision 135) +++ tags/1.05/lib/Brackup/PositionedChunk.pm (revision 135) @@ -0,0 +1,137 @@ +package Brackup::PositionedChunk; + +use strict; +use warnings; +use Carp qw(croak); +use Digest::SHA1 qw(sha1_hex); + +use fields ( + 'file', # the Brackup::File object + 'offset', # offset within said file + 'length', # length of data + '_raw_digest', + '_raw_chunkref', + ); + +sub new { + my ($class, %opts) = @_; + my $self = ref $class ? $class : fields::new($class); + + $self->{file} = delete $opts{'file'}; # Brackup::File object + $self->{offset} = delete $opts{'offset'}; + $self->{length} = delete $opts{'length'}; + + croak("Unknown options: " . join(', ', keys %opts)) if %opts; + croak("offset not numeric") unless $self->{offset} =~ /^\d+$/; + croak("length not numeric") unless $self->{length} =~ /^\d+$/; + return $self; +} + +sub as_string { + my $self = shift; + return $self->{file}->as_string . "{off=$self->{offset},len=$self->{length}}"; +} + +# the original length, pre-encryption +sub length { + my $self = shift; + return $self->{length}; +} + +sub offset { + my $self = shift; + return $self->{offset}; +} + +sub file { + my $self = shift; + return $self->{file}; +} + +sub root { + my $self = shift; + return $self->file->root; +} + +sub raw_digest { + my $self = shift; + return $self->{_raw_digest} ||= $self->_calc_raw_digest; +} + +sub _calc_raw_digest { + my $self = shift; + + my $n_chunks = $self->{file}->chunks + or die "zero chunks?"; + if ($n_chunks == 1) { + # don't calculate this chunk's digest.. it's the same as our + # file's digest, since this chunk spans the entire file. + die "ASSERT" unless $self->length == $self->{file}->size; + return $self->{file}->full_digest; + } + + my $cache = $self->root->digest_cache; + my $key = $self->cachekey; + my $dig; + + if ($dig = $cache->get($key)) { + return $self->{_raw_digest} = $dig; + } + + my $rchunk = $self->raw_chunkref; + $dig = "sha1:" . sha1_hex($$rchunk); + + $cache->set($key => $dig); + + return $self->{_raw_digest} = $dig; +} + +sub raw_chunkref { + my $self = shift; + return $self->{_raw_chunkref} if $self->{_raw_chunkref}; + + my $data; + my $fullpath = $self->{file}->fullpath; + open(my $fh, $fullpath) or die "Failed to open $fullpath: $!\n"; + binmode($fh); + seek($fh, $self->{offset}, 0) or die "Couldn't seek: $!\n"; + my $rv = read($fh, $data, $self->{length}) + or die "Failed to read: $!\n"; + unless ($rv == $self->{length}) { + Carp::confess("Read $rv bytes, not $self->{length}"); + } + + return $self->{_raw_chunkref} = \$data; +} + +# useful string for targets to key on. of one of the forms: +# ";to=" +# ";raw" +# ";gz" (future) +sub inventory_key { + my $self = shift; + my $key = $self->raw_digest; + if (my $rcpt = $self->root->gpg_rcpt) { + $key .= ";to=$rcpt"; + } else { + $key .= ";raw"; + } + return $key; +} + +sub forget_chunkref { + my $self = shift; + delete $self->{_raw_chunkref}; +} + +sub cachekey { + my $self = shift; + return $self->{file}->cachekey . ";o=$self->{offset};l=$self->{length}"; +} + +sub is_entire_file { + my $self = shift; + return $self->{file}->chunks == 1; +} + +1; Index: tags/1.05/lib/Brackup/Dict/SQLite.pm =================================================================== --- tags/1.05/lib/Brackup/Dict/SQLite.pm (revision 119) +++ tags/1.05/lib/Brackup/Dict/SQLite.pm (revision 119) @@ -0,0 +1,58 @@ +package Brackup::Dict::SQLite; +use strict; +use warnings; +use DBI; +use DBD::SQLite; + +sub new { + my ($class, $table, $file) = @_; + my $self = bless { + table => $table, + file => $file, + data => {}, + }, $class; + + my $dbh = $self->{dbh} = DBI->connect("dbi:SQLite:dbname=$file","","", { RaiseError => 1, PrintError => 0 }) or + die "Failed to connect to SQLite filesystem digest cache database at $file: " . DBI->errstr; + + eval { + $dbh->do("CREATE TABLE $table (key TEXT PRIMARY KEY, value TEXT)"); + }; + die "Error: $@" if $@ && $@ !~ /table \w+ already exists/; + return $self; +} + +sub get { + my ($self, $key) = @_; + unless ($self->{_loaded_all}++) { + # SQLite sucks at doing anything quickly (likes hundred thousand + # selects back-to-back), so we just suck the whole damn thing into + # a perl hash. cute, huh? then it doesn't have to + # open/read/seek/seek/seek/read/close for each select later. + my $sth = $self->{dbh}->prepare("SELECT key, value FROM $self->{table}"); + $sth->execute; + while (my ($k, $v) = $sth->fetchrow_array) { + $self->{data}{$k} = $v; + } + } + return $self->{data}{$key}; +} + +sub set { + my ($self, $key, $val) = @_; + $self->{dbh}->do("REPLACE INTO $self->{table} VALUES (?,?)", undef, $key, $val); + $self->{data}{$key} = $val; + return 1; +} + +sub backing_file { + my $self = shift; + return $self->{file}; +} + +sub wipe { + die "not implemented"; +} + +1; + Index: tags/1.05/lib/Brackup/Target/Filesystem.pm =================================================================== --- tags/1.05/lib/Brackup/Target/Filesystem.pm (revision 161) +++ tags/1.05/lib/Brackup/Target/Filesystem.pm (revision 161) @@ -0,0 +1,224 @@ +package Brackup::Target::Filesystem; +use strict; +use warnings; +use base 'Brackup::Target'; +use File::Basename; +use File::Find (); +use File::Path; +use File::stat (); + + +sub new { + my ($class, $confsec) = @_; + my $self = $class->SUPER::new($confsec); + $self->{path} = $confsec->path_value("path"); + $self->{nocolons} = $confsec->value("no_filename_colons"); + $self->{nocolons} = ($^O eq 'MSWin32') unless defined $self->{nocolons}; # LAME: Make it work on Windows + return $self; +} + +sub new_from_backup_header { + my ($class, $header) = @_; + my $self = bless {}, $class; + $self->{path} = $header->{"BackupPath"} or + die "No BackupPath specified in the backup metafile.\n"; + unless (-d $self->{path}) { + die "Restore path $self->{path} doesn't exist.\n"; + } + return $self; +} + +sub backup_header { + my $self = shift; + return { + "BackupPath" => $self->{path}, + }; +} + +sub _diskpath { + my ($self, $dig, $ext) = @_; + my @parts; + my $fulldig = $dig; + $dig =~ s/^\w+://; # remove the "hashtype:" from beginning + $fulldig =~ s/:/./g if $self->{nocolons}; # Convert colons to dots if we've been asked to + while (length $dig && @parts < 4) { + $dig =~ s/^(.{1,4})//; + push @parts, $1; + } + return $self->{path} . "/" . join("/", @parts) . "/$fulldig.$ext"; +} + +sub chunkpath { + my ($self, $dig) = @_; + return $self->_diskpath($dig, "chunk"); +} + +sub has_chunk_of_handle { + my ($self, $handle) = @_; + my $dig = $handle->digest; # "sha1:sdfsdf" format scalar + my $path = $self->chunkpath($dig); + return -e $path; +} + +sub load_chunk { + my ($self, $dig) = @_; + my $path = $self->chunkpath($dig); + open (my $fh, $path) or die "Error opening $path to load chunk: $!"; + my $chunk = do { local $/; <$fh>; }; + return \$chunk; +} + +sub store_chunk { + my ($self, $chunk) = @_; + my $dig = $chunk->backup_digest; + my $blen = $chunk->backup_length; + + my $path = $self->chunkpath($dig); + my $dir = $path; + $dir =~ s!/[^/]+$!!; + unless (-d $dir) { + File::Path::mkpath($dir) or die "Failed to mkdir: $dir: $!\n"; + } + open (my $fh, ">$path") or die "Failed to open $path for writing: $!\n"; + binmode($fh); + my $chunkref = $chunk->chunkref; + print $fh $$chunkref; + close($fh) or die "Failed to close $path\n"; + + my $actual_size = -s $path; + my $expected_size = length $$chunkref; + unless (defined($actual_size)) { + die "Chunk output file $path does not exist. Do you need to set no_filename_colons=1?"; + } + unless ($actual_size == $expected_size) { + die "Chunk $path was written to disk wrong: size is $actual_size, expecting $expected_size\n"; + } + + return 1; +} + +sub delete_chunk { + my ($self, $dig) = @_; + my $path = $self->chunkpath($dig); + unlink $path; +} + + +# returns a list of names of all chunks +sub chunks { + my $self = shift; + + my @chunks = (); + my $found_chunk = sub { + m/\.chunk$/ or return; + my $chunk_name = basename($_); + $chunk_name =~ s/\.chunk$//; + push @chunks, $chunk_name; + }; + File::Find::find({ wanted => $found_chunk, no_chdir => 1}, $self->{path}); + return @chunks; +} + +sub _metafile_dir { + return $_[0]->{path}."/backups/"; +} + +sub store_backup_meta { + my ($self, $name, $file) = @_; + my $dir = $self->_metafile_dir; + unless (-d $dir) { + mkdir $dir or die "Failed to mkdir $dir: $!\n"; + } + open (my $fh, ">$dir/$name.brackup") or die; + print $fh $file; + close $fh or die; + return 1; +} + +sub backups { + my ($self) = @_; + + my $dir = $self->_metafile_dir; + return () unless -d $dir; + + opendir(my $dh, $dir) or + die "Failed to open $dir: $!\n"; + + my @ret = (); + while (my $fn = readdir($dh)) { + next unless $fn =~ s/\.brackup$//; + my $stat = File::stat::stat("$dir/$fn.brackup"); + push @ret, Brackup::TargetBackupStatInfo->new($self, $fn, + time => $stat->mtime, + size => $stat->size); + } + closedir($dh); + return @ret; +} + +# downloads the given backup name to the current directory (with +# *.brackup extension) or to the specified location +sub get_backup { + my ($self, $name, $output_file) = @_; + my $dir = $self->_metafile_dir; + my $file = "$dir/$name.brackup"; + die "File doesn't exist: $file" unless -e $file; + open(my $in, $file) or die "Failed to open $file: $!\n"; + $output_file ||= "$name.brackup"; + open(my $out, ">$output_file") or die "Failed to open $output_file: $!\n"; + my $buf; + my $rv; + while ($rv = sysread($in, $buf, 128*1024)) { + my $outv = syswrite($out, $buf); + die "copy error" unless $outv == $rv; + } + die "copy error" unless defined $rv; + return 1; +} + +sub delete_backup { + my $self = shift; + my $name = shift; + + my $file = sprintf '%s/%s.brackup', $self->_metafile_dir, $name; + die "File doesn't exist: $file" unless -e $file; + unlink $file; + return 1; +} + +1; + + +=head1 NAME + +Brackup::Target::Filesystem - backup to a locally mounted filesystem + +=head1 DESCRIPTION + +Back up to an NFS or Samba server, another disk array (external storage), etc. + +=head1 EXAMPLE + +In your ~/.brackup.conf file: + + [TARGET:nfs_in_garage] + type = Filesystem + path = /mnt/nfs-garage/brackup/ + +=head1 CONFIG OPTIONS + +=over + +=item B + +Must be "B". + +=item B + +Path to backup to. + +=back + +=head1 SEE ALSO + +L Index: tags/1.05/lib/Brackup/Target/Amazon.pm =================================================================== --- tags/1.05/lib/Brackup/Target/Amazon.pm (revision 161) +++ tags/1.05/lib/Brackup/Target/Amazon.pm (revision 161) @@ -0,0 +1,234 @@ +package Brackup::Target::Amazon; +use strict; +use warnings; +use base 'Brackup::Target'; +use Net::Amazon::S3 0.37; + +# fields in object: +# s3 -- Net::Amazon::S3 +# access_key_id +# sec_access_key_id +# chunk_bucket : $self->{access_key_id} . "-chunks"; +# backup_bucket : $self->{access_key_id} . "-backups"; +# + +sub new { + my ($class, $confsec) = @_; + my $self = $class->SUPER::new($confsec); + + $self->{access_key_id} = $confsec->value("aws_access_key_id") + or die "No 'aws_access_key_id'"; + $self->{sec_access_key_id} = $confsec->value("aws_secret_access_key") + or die "No 'aws_secret_access_key'"; + + $self->_common_s3_init; + + my $s3 = $self->{s3}; + my $buckets = $s3->buckets or die "Failed to get bucket list"; + + unless (grep { $_->{bucket} eq $self->{chunk_bucket} } @{ $buckets->{buckets} }) { + $s3->add_bucket({ bucket => $self->{chunk_bucket} }) + or die "Chunk bucket creation failed\n"; + } + + unless (grep { $_->{bucket} eq $self->{backup_bucket} } @{ $buckets->{buckets} }) { + $s3->add_bucket({ bucket => $self->{backup_bucket} }) + or die "Backup bucket creation failed\n"; + } + + return $self; +} + +sub _common_s3_init { + my $self = shift; + $self->{chunk_bucket} = $self->{access_key_id} . "-chunks"; + $self->{backup_bucket} = $self->{access_key_id} . "-backups"; + $self->{s3} = Net::Amazon::S3->new({ + aws_access_key_id => $self->{access_key_id}, + aws_secret_access_key => $self->{sec_access_key_id}, + }); +} + +# ghetto +sub _prompt { + my ($q) = @_; + print "$q"; + my $ans = ; + $ans =~ s/^\s+//; + $ans =~ s/\s+$//; + return $ans; +} + +sub new_from_backup_header { + my ($class, $header) = @_; + + my $accesskey = ($ENV{'AWS_KEY'} || _prompt("Your Amazon AWS access key? ")) + or die "Need your Amazon access key.\n"; + my $sec_accesskey = ($ENV{'AWS_SEC_KEY'} || _prompt("Your Amazon AWS secret access key? ")) + or die "Need your Amazon secret access key.\n"; + + my $self = bless {}, $class; + $self->{access_key_id} = $accesskey; + $self->{sec_access_key_id} = $sec_accesskey; + $self->_common_s3_init; + return $self; +} + +sub has_chunk { + my ($self, $chunk) = @_; + my $dig = $chunk->backup_digest; # "sha1:sdfsdf" format scalar + + my $res = eval { $self->{s3}->head_key({ bucket => $self->{chunk_bucket}, key => $dig }); }; + return 0 unless $res; + return 0 if $@ && $@ =~ /key not found/; + return 0 unless $res->{content_type} eq "x-danga/brackup-chunk"; + return 1; +} + +sub load_chunk { + my ($self, $dig) = @_; + my $bucket = $self->{s3}->bucket($self->{chunk_bucket}); + + my $val = $bucket->get_key($dig) + or return 0; + return \ $val->{value}; +} + +sub store_chunk { + my ($self, $chunk) = @_; + my $dig = $chunk->backup_digest; + my $blen = $chunk->backup_length; + my $chunkref = $chunk->chunkref; + + my $try = sub { + eval { + $self->{s3}->add_key({ + bucket => $self->{chunk_bucket}, + key => $dig, + value => $$chunkref, + content_type => 'x-danga/brackup-chunk', + }); + }; + }; + + my $rv; + my $n_fails = 0; + while (!$rv && $n_fails < 5) { + $rv = $try->(); + last if $rv; + + # transient failure? + $n_fails++; + warn "Error uploading chunk $chunk [$@]... will do retry \#$n_fails in 5 seconds ...\n"; + sleep 5; + } + unless ($rv) { + warn "Error uploading chunk again: " . $self->{s3}->errstr . "\n"; + return 0; + } + return 1; +} + +sub delete_chunk { + my ($self, $dig) = @_; + my $bucket = $self->{s3}->bucket($self->{chunk_bucket}); + return $bucket->delete_key($dig); +} + +# returns a list of names of all chunks +sub chunks { + my $self = shift; + + my $chunks = $self->{s3}->list_bucket_all({ bucket => $self->{chunk_bucket} }); + return map { $_->{key} } @{ $chunks->{keys} }; +} + +sub store_backup_meta { + my ($self, $name, $file) = @_; + + my $rv = eval { $self->{s3}->add_key({ + bucket => $self->{backup_bucket}, + key => $name, + value => $file, + content_type => 'x-danga/brackup-meta', + })}; + + return $rv; +} + +sub backups { + my $self = shift; + + my @ret; + my $backups = $self->{s3}->list_bucket_all({ bucket => $self->{backup_bucket} }); + foreach my $backup (@{ $backups->{keys} }) { + push @ret, Brackup::TargetBackupStatInfo->new($self, $backup->{key}, + time => $backup->{last_modified}, + size => $backup->{size}); + } + return @ret; +} + +sub get_backup { + my $self = shift; + my ($name, $output_file) = @_; + + my $bucket = $self->{s3}->bucket($self->{backup_bucket}); + my $val = $bucket->get_key($name) + or return 0; + + $output_file ||= "$name.brackup"; + open(my $out, ">$output_file") or die "Failed to open $output_file: $!\n"; + my $outv = syswrite($out, $val->{value}); + die "download/write error" unless $outv == do { use bytes; length $val->{value} }; + close $out; + return 1; +} + +sub delete_backup { + my $self = shift; + my $name = shift; + + my $bucket = $self->{s3}->bucket($self->{backup_bucket}); + return $bucket->delete_key($name); +} + +1; + +=head1 NAME + +Brackup::Target::Amazon - backup to Amazon's S3 service + +=head1 EXAMPLE + +In your ~/.brackup.conf file: + + [TARGET:amazon] + type = Amazon + aws_access_key_id = ... + aws_secret_access_key = .... + +=head1 CONFIG OPTIONS + +=over + +=item B + +Must be "B". + +=item B + +Your Amazon Web Services access key id. + +=item B + +Your Amazon Web Services secret password for the above access key. (not your Amazon password) + +=back + +=head1 SEE ALSO + +L + +L -- required module to use Brackup::Target::Amazon + Index: tags/1.05/lib/Brackup/GPGProcess.pm =================================================================== --- tags/1.05/lib/Brackup/GPGProcess.pm (revision 151) +++ tags/1.05/lib/Brackup/GPGProcess.pm (revision 151) @@ -0,0 +1,78 @@ +package Brackup::GPGProcess; +use strict; +use warnings; +use Brackup::Util qw(tempfile); +use POSIX qw(_exit); + +sub new { + my ($class, $pchunk) = @_; + + my ($destfh, $destfn) = tempfile(); + + my $no_fork = 0; # if true (perhaps on Windows?), then don't fork... do all inline. + + my $pid = $no_fork ? 0 : fork; + if (!defined $pid) { + die "Failed to fork: $!"; + } + + # caller (parent) + if ($pid) { + return bless { + destfn => $destfn, + pid => $pid, + running => 1, + }, $class; + } + + # child: encrypt and exit(0)... + my $enc = $pchunk->root->encrypt($pchunk->raw_chunkref); + binmode($destfh); + print $destfh $enc + or die "failed to print: $!"; + close $destfh + or die "failed to close: $!"; + die "size not right" + unless -s $destfn == length $enc; + + if ($no_fork) { + return bless { + destfn => $destfn, + pid => 0, + }, $class; + } + + # Note: we have to do this, to avoid some END block, somewhere, + # from cleaning up something or doing something. probably tempfiles + # being destroyed in File::Temp. + POSIX::_exit(0); +} + +sub pid { $_[0]{pid} } + +sub running { $_[0]{running} } +sub note_stopped { $_[0]{running} = 0; } + +sub chunkref { + my ($self) = @_; + die "Still running!" if $self->{running}; + + open(my $fh, $self->{destfn}) + or die "Failed to open gpg temp file $self->{destfn}: $!"; + binmode($fh); + my $data = do { local $/; <$fh>; }; + die "No data in file" unless length $data; + close($fh); + unlink($self->{destfn}) or die "couldn't delete destfn"; + + # unlink + return \$data; +} + +sub size_on_disk { + my $self = shift; + return -s $self->{destfn}; +} + +1; + Index: tags/1.05/lib/Brackup/Util.pm =================================================================== --- tags/1.05/lib/Brackup/Util.pm (revision 153) +++ tags/1.05/lib/Brackup/Util.pm (revision 153) @@ -0,0 +1,49 @@ +package Brackup::Util; +use strict; +use warnings; +require Exporter; + +use vars qw(@ISA @EXPORT_OK); +@ISA = ('Exporter'); +@EXPORT_OK = qw(tempfile tempdir slurp valid_params); + +my $mainpid = $$; +my @TEMP_FILES = (); # ([filename, caller], ...) + +END { + # will happen after File::Temp's cleanup + if ($$ == $mainpid) { + foreach my $rec (@TEMP_FILES) { + next unless -e $rec->[0]; + unlink($rec->[0]); + } + } +} +use File::Temp (); + +sub tempfile { + my (@ret) = File::Temp::tempfile(); + my $from = join(" ", (caller())[0..2]); + push @TEMP_FILES, [$ret[1], $from]; + return wantarray ? @ret : $ret[0]; +} + +sub tempdir { + return File::Temp::tempdir(@_); +} + +sub slurp { + my $file = shift; + open(my $fh, $file) or die "Failed to open $file: $!\n"; + return do { local $/; <$fh>; } +} + +sub valid_params { + my ($vlist, %uarg) = @_; + my %ret; + $ret{$_} = delete $uarg{$_} foreach @$vlist; + croak("Bogus options: " . join(', ', sort keys %uarg)) if %uarg; + return %ret; +} + +1; Index: tags/1.05/lib/Brackup/Test.pm =================================================================== --- tags/1.05/lib/Brackup/Test.pm (revision 161) +++ tags/1.05/lib/Brackup/Test.pm (revision 161) @@ -0,0 +1,166 @@ +# support module for helping test brackup +package Brackup::Test; +require Exporter; +use strict; +use vars qw(@ISA @EXPORT); +@ISA = qw(Exporter); +@EXPORT = qw(do_backup do_restore ok_dirs_match); + +use Test::More; +use FindBin qw($Bin); +use Brackup::Util qw(tempdir tempfile); +use File::Find; +use File::stat (); +use Cwd; + +use Brackup; + +my $has_diff = eval "use Text::Diff; 1;"; + +my @to_unlink; +my $par_pid = $$; +END { + if ($$ == $par_pid) { + my $rv = unlink @to_unlink; + } +} + +sub do_backup { + my %opts = @_; + my $with_confsec = delete $opts{'with_confsec'} || sub {}; + my $with_root = delete $opts{'with_root'} || sub {}; + die if %opts; + + my $initer = shift; + + my $conf = Brackup::Config->new; + my $confsec; + + $confsec = Brackup::ConfigSection->new("SOURCE:test_root"); + $with_confsec->($confsec); + $conf->add_section($confsec); + + my $root = $conf->load_root("test_root"); + ok($root, "have a source root"); + $with_root->($root); + + my $backup_dir = tempdir( CLEANUP => 1 ); + ok_dir_empty($backup_dir); + + my ($inv_fh, $inv_filename) = tempfile(); + close($inv_fh); + push @to_unlink, $inv_filename; + + + $confsec = Brackup::ConfigSection->new("TARGET:test_restore"); + $confsec->add("type" => "Filesystem"); + $confsec->add("inventory_db" => $inv_filename); + $confsec->add("path" => $backup_dir); + $conf->add_section($confsec); + + my $target = $conf->load_target("test_restore"); + ok($target, "have a target"); + + my $backup = Brackup::Backup->new( + root => $root, + target => $target, + ); + ok($backup, "have a backup object"); + + my ($meta_fh, $meta_filename) = tempfile(); + ok(-e $meta_filename, "metafile exists"); + push @to_unlink, $meta_filename; + + ok(eval { $backup->backup($meta_filename) }, "backup succeeded"); + if ($@) { + warn "Died running backup: $@\n"; + } + ok(-s $meta_filename, "backup file has size"); + + return wantarray ? ($meta_filename, $backup, $target) : $meta_filename; +} + +sub do_restore { + my $backup_file = shift; + my $restore_dir = tempdir( CLEANUP => 1 ); + ok_dir_empty($restore_dir); + + my $restore = Brackup::Restore->new( + to => $restore_dir, + prefix => "", # backup everything + file => $backup_file, + ); + ok($restore, "have restore object"); + ok(eval { $restore->restore; }, "did the restore") + or die "restore failed: $@"; + return $restore_dir; +} + +sub ok_dirs_match { + my ($after, $before) = @_; + + my $pre_ls = dir_structure($before); + my $post_ls = dir_structure($after); + + if ($has_diff) { + use Data::Dumper; + my $pre_dump = Dumper($pre_ls); + my $post_dump = Dumper($post_ls); + my $diff = Text::Diff::diff(\$pre_dump, \$post_dump); + is($diff, "", "dirs match"); + } else { + is_deeply($post_ls, $pre_ls, "dirs match"); + } +} + +sub ok_dir_empty { + my $dir = shift; + unless (-d $dir) { ok(0, "not a dir"); return; } + opendir(my $dh, $dir) or die "failed to opendir: $!"; + is_deeply([ sort readdir($dh) ], ['.', '..'], "dir is empty: $dir"); +} + +# given a directory, returns a hashref of its contentn +sub dir_structure { + my $dir = shift; + my %files; # "filename" -> {metadata => ...} + my $cwd = getcwd; + chdir($dir) or die "Failed to chdir to $dir"; + + find({ + no_chdir => 1, + preprocess => sub { return sort @_ }, + wanted => sub { + my $path = $_; + my $st = File::stat::lstat($path); + + my $meta = {}; + $meta->{size} = $st->size unless -d $path; + $meta->{is_file} = 1 if -f $path; + $meta->{is_link} = 1 if -l $path; + if ($meta->{is_link}) { + $meta->{link} = readlink $path; + } else { + # we ignore these for links, since Linux doesn't let us restore anyway, + # as Linux as no lutimes(2) syscall, as of Linux 2.6.16 at least + $meta->{atime} = $st->atime if 0; # TODO: make tests work with atimes + $meta->{mtime} = $st->mtime; + $meta->{mode} = sprintf('%#o', $st->mode & 0777); + } + + # the gpg tests open/close the rings in the root, so + # mtimes get bumped around or something. the proper fix + # is too ugly for what it's worth, so let's just ignore + # the mtime of top-level + delete $meta->{mtime} if $path eq "."; + + $files{$path} = $meta; + }, + }, "."); + + chdir($cwd) or die "Failed to chdir back to $cwd"; + return \%files; +} + + +1; Index: tags/1.05/lib/Brackup/File.pm =================================================================== --- tags/1.05/lib/Brackup/File.pm (revision 124) +++ tags/1.05/lib/Brackup/File.pm (revision 124) @@ -0,0 +1,231 @@ +package Brackup::File; +# "everything is a file" +# ... this class includes symlinks and directories + +use strict; +use warnings; +use Carp qw(croak); +use File::stat (); +use Fcntl qw(S_ISREG S_ISDIR S_ISLNK); +use Digest::SHA1; +use Brackup::PositionedChunk; + +sub new { + my ($class, %opts) = @_; + my $self = bless {}, $class; + + $self->{root} = delete $opts{root}; + $self->{path} = delete $opts{path}; + $self->{stat} = delete $opts{stat}; # File::stat object + croak("Unknown options: " . join(', ', keys %opts)) if %opts; + + die "No root object provided." unless $self->{root} && $self->{root}->isa("Brackup::Root"); + die "No path provided." unless $self->{path}; + $self->{path} =~ s!^\./!!; + + return $self; +} + +sub root { + my $self = shift; + return $self->{root}; +} + +# returns File::stat object +sub stat { + my $self = shift; + return $self->{stat} if $self->{stat}; + my $path = $self->fullpath; + my $stat = File::stat::lstat($path); + return $self->{stat} = $stat; +} + +sub size { + my $self = shift; + return $self->stat->size; +} + +sub is_dir { + my $self = shift; + return S_ISDIR($self->stat->mode); +} + +sub is_link { + my $self = shift; + my $result = eval { S_ISLNK($self->stat->mode) }; + $result = -l $self->fullpath unless defined($result); + return $result; +} + +sub is_file { + my $self = shift; + return S_ISREG($self->stat->mode); +} + +sub supported_type { + my $self = shift; + return $self->type ne ""; +} + +# returns "f", "l", or "d" like find's -type +sub type { + my $self = shift; + return "f" if $self->is_file; + return "d" if $self->is_dir; + return "l" if $self->is_link; + return ""; +} + +sub fullpath { + my $self = shift; + return $self->{root}->path . "/" . $self->{path}; +} + +# a scalar that hopefully uniquely represents a single version of a file in time. +sub cachekey { + my $self = shift; + my $st = $self->stat; + return "[" . $self->{root}->name . "]" . $self->{path} . ":" . join(",", $st->ctime, $st->mtime, $st->size, $st->ino); +} + +# iterate over chunks sized by the root's configuration +sub foreach_chunk { + my ($self, $cb) = @_; + + foreach my $chunk ($self->chunks) { + $cb->($chunk); + } +} + +sub _min { + return (sort { $a <=> $b } @_)[0]; +} + +sub chunks { + my $self = shift; + return @{ $self->{chunks} } if $self->{chunks}; + + my $root = $self->{root}; + my $chunk_size = $root->chunk_size; + + # non-files don't have chunks + my @list; + if ($self->is_file) { + my $offset = 0; + my $size = $self->size; + while ($offset < $size) { + my $len = _min($chunk_size, $size - $offset); + my $chunk = Brackup::PositionedChunk->new( + file => $self, + offset => $offset, + length => $len, + ); + push @list, $chunk; + $offset += $len; + } + } + + $self->{chunks} = \@list; + return @list; +} + +sub full_digest { + my $self = shift; + return $self->{_full_digest} ||= $self->_calc_full_digest; +} + +sub _calc_full_digest { + my $self = shift; + return "" unless $self->is_file; + + my $cache = $self->{root}->digest_cache; + my $key = $self->cachekey; + + my $dig = $cache->get($key); + return $dig if $dig; + + # legacy migration thing... we used to more often store + # the chunk digests, not the file digests. so try that + # first... + if ($self->chunks == 1) { + my ($chunk) = $self->chunks; + $dig = $cache->get($chunk->cachekey); + } + + unless ($dig) { + my $sha1 = Digest::SHA1->new; + my $path = $self->fullpath; + open (my $fh, $path) or die "Couldn't open $path: $!\n"; + binmode($fh); + $sha1->addfile($fh); + close($fh); + + $dig = "sha1:" . $sha1->hexdigest; + } + + $cache->set($key => $dig); + return $dig; +} + +sub link_target { + my $self = shift; + return $self->{linktarget} if $self->{linktarget}; + return undef unless $self->is_link; + return $self->{linktarget} = readlink($self->fullpath); +} + +sub path { + my $self = shift; + return $self->{path}; +} + +sub as_string { + my $self = shift; + my $type = $self->type; + return "[" . $self->{root}->as_string . "] t=$type $self->{path}"; +} + +sub mode { + my $self = shift; + return sprintf('%#o', $self->stat->mode & 0777); +} + +sub as_rfc822 { + my ($self, $schunk_list, $backup) = @_; + my $ret = ""; + my $set = sub { + my ($key, $val) = @_; + return unless length $val; + $ret .= "$key: $val\n"; + }; + my $st = $self->stat; + + $set->("Path", $self->{path}); + my $type = $self->type; + if ($self->is_file) { + my $size = $self->size; + $set->("Size", $size); + $set->("Digest", $self->full_digest) if $size; + } else { + $set->("Type", $type); + if ($self->is_link) { + $set->("Link", $self->link_target); + } + } + $set->("Chunks", join("\n ", map { $_->to_meta } @$schunk_list)); + + unless ($self->is_link) { + $set->("Mtime", $st->mtime); + $set->("Atime", $st->atime) unless $self->root->noatime; + + my $mode = $self->mode; + unless (($type eq "d" && $mode eq $backup->default_directory_mode) || + ($type eq "f" && $mode eq $backup->default_file_mode)) { + $set->("Mode", $mode); + } + } + + return $ret . "\n"; +} + +1; Index: tags/1.05/lib/Brackup/ChunkIterator.pm =================================================================== --- tags/1.05/lib/Brackup/ChunkIterator.pm (revision 98) +++ tags/1.05/lib/Brackup/ChunkIterator.pm (revision 98) @@ -0,0 +1,78 @@ +package Brackup::ChunkIterator; +use strict; +use warnings; +use Carp qw(croak); + +sub new { + my ($class, @files) = @_; + return bless { + filelist => \@files, + chunkmag => [], + }, $class; +} + +# returns either PositionedChunks, or, in cases of files +# without contents (like directories/symlinks), returns +# File objects... returns undef on end of files/chunks. +sub next { + my $self = shift; + + # magazine already loaded? fire. + my $next = shift @{ $self->{chunkmag} }; + return $next if $next; + + # else reload... + my $file = shift @{ $self->{filelist} } or + return undef; + + ($next, @{$self->{chunkmag}}) = $file->chunks; + return $next if $next; + return $file; +} + +sub mux_into { + my ($self, $n_copies) = @_; + my @iters; + for (1..$n_copies) { + push @iters, Brackup::ChunkIterator::SlaveIterator->new; + } + my $on_empty = sub { + my $next = $self->next; + foreach my $peer (@iters) { + push @{$peer->{mag}}, $next; + } + }; + foreach (@iters) { + $_->{on_empty} = $on_empty; + } + return @iters; +} + +package Brackup::ChunkIterator::SlaveIterator; +use strict; +use warnings; +use Carp qw(croak); + +sub new { + my $class = shift; + return bless { + 'on_empty' => undef, # subref + 'mag' => [], + }, $class; +} + +sub next { + my $self = shift; + # the magazine itself could be true, but contain only undef: (undef), + # which signals the end. + return shift @{$self->{mag}} if @{$self->{mag}}; + $self->{on_empty}->(); + return shift @{$self->{mag}}; +} + +sub behind_by { + my $self = shift; + return scalar @{$self->{mag}}; +} + +1; Index: tags/1.05/lib/Brackup/InventoryDatabase.pm =================================================================== --- tags/1.05/lib/Brackup/InventoryDatabase.pm (revision 134) +++ tags/1.05/lib/Brackup/InventoryDatabase.pm (revision 134) @@ -0,0 +1,152 @@ +package Brackup::InventoryDatabase; +use strict; +use Brackup::Dict::SQLite; +use warnings; +use Carp qw(croak); + +sub new { + my ($class, $file) = @_; + my $self = bless {}, $class; + + # only SQLite is supported at present. future: gdbm, mysql, etc + $self->{dict} = Brackup::Dict::SQLite->new("target_inv", $file); +} + +# proxy through to underlying dictionary +sub get { shift->{dict}->get(@_) } +sub set { shift->{dict}->set(@_) } + + +1; +__END__ + +=head1 NAME + +Brackup::InventoryDatabase - track what chunks are already on a target + +=head1 DESCRIPTION + +The Brackup InventoryDatabase keeps track of which chunks (files) are +already on a given target, speeding up future iterative backups, +negating the need to ask the target for each chunk whether it exists +or not (which may be a lot of network roundtrips, slow even in the +best of network conditions). + +Unlike the L, the inventory +database is not a cache... its contents matter. Consider what happens +when the inventory database doesn't match reality: + +=over + +=item B<1) Exists in inventory database; not on target> + +If a chunk exists in the inventory database, but not on the target, brackup +won't store it on the target, and you'll think a backup succeeded, but +it's not actually there. + +=item B<2a) Exists on target; not in inventory database (without encryption)> + +You re-upload it to the target, so you waste time & bandwidth, but no +extra disk space is wasted, and no chunks are orphaned. Actually, +chunks are un-orphaned, as the inventory database is now updated and +contains the chunk you just uploaded. + +=item B<2b) Exists on target; not in inventory database (with encryption)> + +When using encryption, each time a chunk is encrypted with gpg, the +contents are different. So if the inventory database says a given +chunk isn't already stored on the server, it will be re-encrypted and +stored (uploaded) again. You may or may not have an orphaned chunk on +the server, depending on whether or not it's referenced by any other +*.brackup meta files. + +=back + +For those reasons, it's somewhat important that your inventory +database be kept around and not deleted. If you're running brackup to +the same target from different computers, you might want to sync up +your inventory databases with each other, so you don't do unnecessary +uploads to the target. + +Tools to rebuild your inventory database from the target's enumeration +of its chunks and the target's *.brackup metafiles isn't yet done, but +would be pretty easy. (this is a TODO item) + +In any case, it's not tragic if you lose your inventory database... it +just means you'll need to upload more stuff and maybe waste some disk +space. (A tool to clean orphaned chunks from a target is also easy, +and also a TODO item...) If you're feeling paranoid, it's safer to +delete your inventory database, tricking Brackup into thinking your +target is empty (even if it's not), rather than Brackup thinking your +target has something when it actually doesn't. + +=head1 DETAILS + +=head2 File format + +While designed to be abstract, the only supported digest cache format at +the moment is an SQLite database, stored in a single file. The schema +is created automatically as needed... no database maintenance is required. + +=head2 File location + +The SQLite file is stored in either the location specified in a +L's [TARGET] declaration in ~/.brackup.conf, as: + + [TARGET:amazon] + type = Amazon + aws_access_key_id = ... + aws_secret_access_key = ... + target_inv = /home/bradfitz/.amazon-already-has-these-chunks.db + +Or, more commonly (and recommended), is to not specify it and accept +the default location, which is ".brackup-target-TARGETNAME.invdb" in +your home directory. (where it might be shared by multiple backup +roots) + +=head2 SQLite Schema + +This is made automatically for you, but if you want to look around in +it, the schema is: + + CREATE TABLE digest_cache ( + key TEXT PRIMARY KEY, + value TEXT + ) + +=head2 Keys & Values stored in the cache + +B + +The key is the digest of the "raw" (pre-compression/encryption) +file/chunk (with GPG recipient, if using encryption), and the value is +the digest of the chunk stored on the target, which contains the raw +chunk. The chunk stored on the target may contain other chunks, may +be compressed, encrypted, etc. + + --> + ;to= --> + +For example: + + sha1:e23c4b5f685e046e7cc50e30e378ab11391e528e;to=6BAFF35F => + sha1:d7257184899c9e6c4e26506f1c46f8b6562d9ee7 71223 + +Means that the chunk with sha1 contents "e23c4...", intended to be +en/de-crypted for 6BAFF35F, can be got by asking the target for the +chunk with digest "d72571848...", with length 71,223 bytes. + +When using the Brackup feature which combines small files into larger +blobs, the inventory database instead stores values like: + + [;to=] --> + - + +Which is the same thing, but after fetching the composite chunk using +the stored digest provided, only the range provided from C to +C should be used. + +=head1 SEE ALSO + +L + Index: tags/1.05/lib/Brackup/Root.pm =================================================================== --- tags/1.05/lib/Brackup/Root.pm (revision 157) +++ tags/1.05/lib/Brackup/Root.pm (revision 157) @@ -0,0 +1,305 @@ +package Brackup::Root; +use strict; +use warnings; +use Carp qw(croak); +use File::Find; +use Brackup::DigestCache; +use Brackup::Util qw(tempfile); +use IPC::Open2; +use Symbol; + +sub new { + my ($class, $conf) = @_; + my $self = bless {}, $class; + + ($self->{name}) = $conf->name =~ m/^SOURCE:(.+)$/ + or die "No backup-root name provided."; + die "Backup-root name must be only a-z, A-Z, 0-9, and _." unless $self->{name} =~ /^\w+/; + + $self->{dir} = $conf->path_value('path'); + $self->{gpg_path} = $conf->value('gpg_path') || "gpg"; + $self->{gpg_rcpt} = $conf->value('gpg_recipient'); + $self->{chunk_size} = $conf->byte_value('chunk_size'); + $self->{ignore} = []; + + $self->{merge_files_under} = $conf->byte_value('merge_files_under'); + $self->{max_composite_size} = $conf->byte_value('max_composite_chunk_size') || 2**20; + + die "'max_composite_chunk_size' must be greater than 'merge_files_under'\n" unless + $self->{max_composite_size} > $self->{merge_files_under}; + + $self->{gpg_args} = []; # TODO: let user set this. for now, not possible + + $self->{digcache} = Brackup::DigestCache->new($self, $conf); + $self->{digcache_file} = $self->{digcache}->backing_file; # may be empty, if digest cache doesn't use a file + + $self->{noatime} = $conf->value('noatime'); + return $self; +} + +sub merge_files_under { $_[0]{merge_files_under} } +sub max_composite_size { $_[0]{max_composite_size} } + +sub gpg_path { + my $self = shift; + return $self->{gpg_path}; +} + +sub gpg_args { + my $self = shift; + return @{ $self->{gpg_args} }; +} + +sub gpg_rcpt { + my $self = shift; + return $self->{gpg_rcpt}; +} + +# returns Brackup::DigestCache object +sub digest_cache { + my $self = shift; + return $self->{digcache}; +} + +sub chunk_size { + my $self = shift; + return $self->{chunk_size} || (64 * 2**20); # default to 64MB +} + +sub publicname { + # FIXME: let users define the public (obscured) name of their roots. s/porn/media/, etc. + # because their metafile key names (which contain the root) aren't encrypted. + return $_[0]{name}; +} + +sub name { + return $_[0]{name}; +} + +sub ignore { + my ($self, $pattern) = @_; + push @{ $self->{ignore} }, $pattern; +} + +sub path { + return $_[0]{dir}; +} + +sub noatime { + return $_[0]{noatime}; +} + +sub foreach_file { + my ($self, $cb) = @_; + + chdir $self->{dir} or die "Failed to chdir to $self->{dir}"; + + my %statcache; # file -> statobj + + find({ + no_chdir => 1, + preprocess => sub { + my $dir = $File::Find::dir; + my @good_dentries; + DENTRY: + foreach my $dentry (@_) { + next if $dentry eq "." || $dentry eq ".."; + + my $path = "$dir/$dentry"; + $path =~ s!^\./!!; + + # skip the digest database file. not sure if this is smart or not. + # for now it'd be kinda nice to have, but it's re-creatable from + # the backup meta files later, so let's skip it. + next if $path eq $self->{digcache_file}; + + # gpg seems to barf on files ending in whitespace, blowing + # stuff up, so we just skip them instead... + if ($path =~ /\s+$/) { + warn "Skipping file ending in whitespace: <$path>\n"; + next; + } + + my $statobj = File::stat::lstat($path); + my $is_dir = -d _; + + foreach my $pattern (@{ $self->{ignore} }) { + next DENTRY if $path =~ /$pattern/; + next DENTRY if $is_dir && "$path/" =~ /$pattern/; + } + + $statcache{$path} = $statobj; + push @good_dentries, $dentry; + } + + # to let it recurse into the good directories we didn't + # already throw away: + return sort @good_dentries; + }, + + wanted => sub { + my $path = $_; + $path =~ s!^\./!!; + + my $stat_obj = delete $statcache{$path}; + my $file = Brackup::File->new(root => $self, + path => $path, + stat => $stat_obj, + ); + $cb->($file); + }, + }, "."); +} + +sub as_string { + my $self = shift; + return $self->{name} . "($self->{dir})"; +} + +sub du_stats { + my $self = shift; + + my $show_all = $ENV{BRACKUP_DU_ALL}; + my @dir_stack; + my %dir_size; + my $pop_dir = sub { + my $dir = pop @dir_stack; + printf("%-20d%s\n", $dir_size{$dir} || 0, $dir); + delete $dir_size{$dir}; + }; + my $start_dir = sub { + my $dir = shift; + unless ($dir eq ".") { + my @parts = (".", split(m!/!, $dir)); + while (@dir_stack >= @parts) { + $pop_dir->(); + } + } + push @dir_stack, $dir; + }; + $self->foreach_file(sub { + my $file = shift; + my $path = $file->path; + if ($file->is_dir) { + $start_dir->($path); + return; + } + if ($file->is_file) { + my $size = $file->size; + my $kB = int($size / 1024) + ($size % 1024 ? 1 : 0); + printf("%-20d%s\n", $kB, $path) if $show_all; + $dir_size{$_} += $kB foreach @dir_stack; + } + }); + + $pop_dir->() while @dir_stack; +} + +# given data (scalar or scalarref), returns encrypted data +sub encrypt { + my ($self, $data) = @_; + my $gpg_rcpt = $self->gpg_rcpt + or Carp::confess("Encryption not setup for this root"); + + $data = \$data unless ref $data; + + my ($tmpfh, $tmpfn) = tempfile(); + print $tmpfh $$data + or die "failed to print: $!"; + close $tmpfh + or die "failed to close: $!\n"; + Carp::confess("size not right") + unless -s $tmpfn == length $$data; + + my $cout = Symbol::gensym(); + my $cin = Symbol::gensym(); + + my $pid = IPC::Open2::open2($cout, $cin, + $self->gpg_path, $self->gpg_args, + "--recipient", $gpg_rcpt, + "--trust-model=always", + "--batch", + "--encrypt", + "--output=-", # Send output to stdout + "--yes", + $tmpfn + ); + + binmode $cout; + + my $ret = do { local $/; <$cout>; }; + + waitpid($pid, 0); + die "GPG failed: $!" if $? != 0; # If gpg return status is non-zero + + unlink($tmpfn); + + return $ret; +} + +1; + +__END__ + +=head1 NAME + +Brackup::Root - describes the source directory (and options) for a backup + +=head1 EXAMPLE + +In your ~/.brackup.conf file: + + [SOURCE:bradhome] + path = /home/bradfitz/ + gpg_recipient = 5E1B3EC5 + chunk_size = 64MB + ignore = ^\.thumbnails/ + ignore = ^\.kde/share/thumbnails/ + ignore = ^\.ee/(minis|icons|previews)/ + ignore = ^build/ + noatime = 1 + +=head1 CONFIG OPTIONS + +=over + +=item B + +The directory to backup (recursively) + +=item B + +The public key signature to encrypt data with. See L. + +=item B + +In units of bytes, kB, MB, etc. The max size of a chunk to be stored +on the target. Files over this size are cut up into chunks of this +size or smaller. The default is 64 MB if not specified. + +=item B + +Perl5 regular expression of files not to backup. You may have multiple ignore lines. + +=item B + +If true, don't backup access times. They're kinda useless anyway, and +just make the *.brackup metafiles larger. + +=item B + +In units of bytes, kB, MB, etc. If files are under this size. By +default this feature is off (value 0), purely because it's new, but 1 +kB is a recommended size, and will probably be the default in the +future. Set it to 0 to explicitly disable. + +=item B + +In units of bytes, kB, MB, etc. The maximum size of a composite +chunk, holding lots of little files. If this is too big, you'll waste +more space with future iterative backups updating files locked into +this chunk with unchanged chunks. + +Recommended, and default value, is 1 MB. + +=back Index: tags/1.05/lib/Brackup/Restore.pm =================================================================== --- tags/1.05/lib/Brackup/Restore.pm (revision 161) +++ tags/1.05/lib/Brackup/Restore.pm (revision 161) @@ -0,0 +1,284 @@ +package Brackup::Restore; +use strict; +use warnings; +use Carp qw(croak); +use Digest::SHA1; +use Brackup::Util qw(tempfile slurp valid_params); + +sub new { + my ($class, %opts) = @_; + my $self = bless {}, $class; + + $self->{to} = delete $opts{to}; # directory we're restoring to + $self->{prefix} = delete $opts{prefix}; # directory/file filename prefix, or "" for all + $self->{file} = delete $opts{file}; # filename we're restoring from + $self->{verbose} = delete $opts{verbose}; + + $self->{_stats_to_run} = []; # stack (push/pop) of subrefs to reset stat info on + + die "Destination directory doesn't exist" unless $self->{to} && -d $self->{to}; + die "Backup file doesn't exist" unless $self->{file} && -f $self->{file}; + croak("Unknown options: " . join(', ', keys %opts)) if %opts; + + $self->decrypt_file_if_needed; + + return $self; +} + +# returns a hashref of { "foo" => "bar" } from { ..., "Driver-foo" => "bar" } +sub _driver_meta { + my $src = shift; + my $ret = {}; + foreach my $k (keys %$src) { + next unless $k =~ /^Driver-(.+)/; + $ret->{$1} = $src->{$k}; + } + return $ret; +} + +sub decrypt_file_if_needed { + my $self = shift; + my $meta = slurp($self->{file}); + if ($meta =~ /[\x00-\x08]/) { # silly is-binary heuristic + my $new_file = $self->_decrypt_data(\$meta, + want_tempfile => 1, + no_batch => 1, + ); + warn "Decrypted metafile $self->{file} to $new_file; using that to restore from...\n"; + $self->{file} = $new_file; + scalar ; + } +} + +sub restore { + my ($self) = @_; + my $parser = $self->parser; + my $meta = $parser->readline; + my $driver_class = $meta->{BackupDriver}; + die "No driver specified" unless $driver_class; + + my $driver_meta = _driver_meta($meta); + + eval "use $driver_class; 1;" or die + "Failed to load driver ($driver_class) to restore from: $@\n"; + my $target = eval {"$driver_class"->new_from_backup_header($driver_meta); }; + if ($@) { + die "Failed to instantiate target ($driver_class) for restore, perhaps it doesn't support restoring yet?\n\nThe error was: $@"; + } + $self->{_target} = $target; + $self->{_meta} = $meta; + + while (my $it = $parser->readline) { + my $full = $self->{to} . "/" . $it->{Path}; + my $type = $it->{Type} || "f"; + die "Unknown filetype: type=$type, file: $it->{Path}" unless $type =~ /^[ldf]$/; + + # restore default modes from header + $it->{Mode} ||= $meta->{DefaultFileMode} if $type eq "f"; + $it->{Mode} ||= $meta->{DefaultDirMode} if $type eq "d"; + + warn " * restoring $it->{Path} to $full\n" if $self->{verbose}; + $self->_restore_link ($full, $it) if $type eq "l"; + $self->_restore_directory($full, $it) if $type eq "d"; + $self->_restore_file ($full, $it) if $type eq "f"; + } + + warn " * fixing stat info\n" if $self->{verbose}; + $self->_exec_statinfo_updates; + warn " * done\n" if $self->{verbose}; + return 1; +} + +sub _output_temp_filename { + my $self = shift; + return $self->{_output_temp_filename} ||= ( (tempfile())[1] || die ); +} + +sub _encrypted_temp_filename { + my $self = shift; + return $self->{_encrypted_temp_filename} ||= ( (tempfile())[1] || die ); +} + +sub _decrypt_data { + my $self = shift; + my $dataref = shift; + my %opts = valid_params(['no_batch', 'want_tempfile'], @_); + + # find which key we're decrypting it using, else return chunk + # unmodified in the not-encrypted case + if ($self->{_meta}) { + my $rcpt = $self->{_meta}{"GPG-Recipient"} or + return $dataref; + } + + unless ($ENV{'GPG_AGENT_INFO'} || + @Brackup::GPG_ARGS || + $self->{warned_about_gpg_agent}++) + { + my $err = q{# + # WARNING: trying to restore encrypted files, + # but $ENV{'GPG_AGENT_INFO'} not present. + # Are you running gpg-agent? + # + }; + $err =~ s/^\s+//gm; + warn $err; + } + + my $output_temp = $opts{want_tempfile} ? + (tempfile())[1] + : $self->_output_temp_filename; + my $enc_temp = $self->_encrypted_temp_filename; + + _write_to_file($enc_temp, $dataref); + + my @list = ("gpg", @Brackup::GPG_ARGS, + "--use-agent", + !$opts{no_batch} ? ("--batch") : (), + "--trust-model=always", + "--output", $output_temp, + "--yes", "--quiet", + "--decrypt", $enc_temp); + system(@list) + and die "Failed to decrypt with gpg: $!\n"; + + return $output_temp if $opts{want_tempfile}; + my $data = Brackup::Util::slurp($output_temp); + return \$data; +} + +sub _update_statinfo { + my ($self, $full, $it) = @_; + + push @{ $self->{_stats_to_run} }, sub { + if (defined $it->{Mode}) { + chmod(oct $it->{Mode}, $full) or + die "Failed to change mode of $full: $!"; + } + + if ($it->{Mtime} || $it->{Atime}) { + utime($it->{Atime}, $it->{Mtime}, $full) or + die "Failed to change utime of $full: $!"; + } + }; +} + +sub _exec_statinfo_updates { + my $self = shift; + + # change the modes/times in backwards order, going from deep + # files/directories to shallow ones. (so we can reliably change + # all the directory mtimes without kernel doing it for us when we + # modify files deeper) + while (my $sb = pop @{ $self->{_stats_to_run} }) { + $sb->(); + } +} + +sub _restore_directory { + my ($self, $full, $it) = @_; + + unless (-d $full) { + mkdir $full or # FIXME: permissions on directory + die "Failed to make directory: $full ($it->{Path})"; + } + + $self->_update_statinfo($full, $it); +} + +sub _restore_link { + my ($self, $full, $it) = @_; + + if (-e $full) { + # TODO: add --conflict={skip,overwrite} option, defaulting to nothing: which dies + die "Link $full ($it->{Path}) already exists. Aborting."; + } + symlink $it->{Link}, $full + or die "Failed to link"; +} + +sub _restore_file { + my ($self, $full, $it) = @_; + + if (-e $full && -s $full) { + # TODO: add --conflict={skip,overwrite} option, defaulting to nothing: which dies + die "File $full ($it->{Path}) already exists. Aborting."; + } + + open (my $fh, ">$full") or die "Failed to open $full for writing"; + my @chunks = grep { $_ } split(/\s+/, $it->{Chunks} || ""); + foreach my $ch (@chunks) { + my ($offset, $len, $enc_len, $dig) = split(/;/, $ch); + my $dataref = $self->{_target}->load_chunk($dig) or + die "Error loading chunk $dig from the restore target\n"; + + my $len_chunk = length $$dataref; + + # using just a range of the file + # TODO: inefficient! we don't want to download the chunk from the + # target multiple times. better to cache it locally, or at least + # only fetch a region from the target (but that's still kinda inefficient + # and pushes complexity into the Target interface) + if ($enc_len =~ /^(\d+)-(\d+)$/) { + my ($from, $to) = ($1, $2); + # file range. gotta be at least as big as bigger number + unless ($len_chunk >= $to) { + die "Backup chunk $dig isn't at least as big as range: got $len_chunk, needing $to\n"; + } + my $region = substr($$dataref, $from, $to-$from); + $dataref = \$region; + } else { + # using the whole chunk, so make sure fetched size matches + # expected size + unless ($len_chunk == $enc_len) { + die "Backup chunk $dig isn't of expected length: got $len_chunk, expecting $enc_len\n"; + } + } + + my $decrypted_ref = $self->_decrypt_data($dataref); + print $fh $$decrypted_ref; + } + close($fh) or die "Close failed"; + + if (my $good_dig = $it->{Digest}) { + die "not capable of verifying digests of from anything but sha1" + unless $good_dig =~ /^sha1:(.+)/; + $good_dig = $1; + + open (my $readfh, $full) or die "Couldn't reopen file for verification"; + my $sha1 = Digest::SHA1->new; + $sha1->addfile($readfh); + my $actual_dig = $sha1->hexdigest; + + # TODO: support --onerror={continue,prompt}, etc, but for now we just die + unless ($actual_dig eq $good_dig || $full =~ m!\.brackup-digest\.db\b!) { + die "Digest of restored file ($full) doesn't match"; + } + } + + $self->_update_statinfo($full, $it); +} + +# returns iterator subref which returns hashrefs or undef on EOF +sub parser { + my $self = shift; + return Brackup::Metafile->open($self->{file}); +} + +sub _write_to_file { + my ($file, $ref) = @_; + open (my $fh, ">$file") or die "Failed to open $file for writing: $!\n"; + print $fh $$ref; + close($fh) or die; + die unless -s $file == length $$ref; + return 1; +} + +sub DESTROY { + my $self = shift; + unlink(grep { $_ } ($self->{_output_temp_filename}, + $self->{_encrypted_temp_filename})); +} + +1; + Index: tags/1.05/lib/Brackup/TargetBackupStatInfo.pm =================================================================== --- tags/1.05/lib/Brackup/TargetBackupStatInfo.pm (revision 145) +++ tags/1.05/lib/Brackup/TargetBackupStatInfo.pm (revision 145) @@ -0,0 +1,38 @@ +package Brackup::TargetBackupStatInfo; + +use strict; +use warnings; +use Carp qw(croak); + +sub new { + my ($class, $target, $fn, %opts) = @_; + my $self = { + target => $target, + filename => $fn, + time => delete $opts{time}, + size => delete $opts{size}, + }; + croak "unknown options: " . join(", ", keys %opts) if %opts; + + return bless $self, $class; +} + +sub target { + return $_[0]->{target}; +} + +sub filename { + return $_[0]->{filename}; +} + +sub time { + return $_[0]->{time}; +} + +sub size { + return $_[0]->{size}; +} + + +1; + Index: tags/1.05/lib/Brackup/BackupStats.pm =================================================================== --- tags/1.05/lib/Brackup/BackupStats.pm (revision 59) +++ tags/1.05/lib/Brackup/BackupStats.pm (revision 59) @@ -0,0 +1,18 @@ +package Brackup::BackupStats; +use strict; + +sub new { + my $class = shift; + return bless {}, $class; +} + +sub print { + print "# BACKUPS STATS: [TODO]\n"; +} + +sub note_stored_chunk { + my ($self, $chunk) = @_; +} + + +1; Index: tags/1.05/lib/Brackup/Metafile.pm =================================================================== --- tags/1.05/lib/Brackup/Metafile.pm (revision 161) +++ tags/1.05/lib/Brackup/Metafile.pm (revision 161) @@ -0,0 +1,50 @@ +package Brackup::Metafile; +use strict; +use warnings; +use Carp qw(croak); + +sub new { + my ($class) = @_; + return bless {}, $class; +} + +sub open { + my ($class, $file) = @_; + unless (-e $file) { + die "Unable to open metafile $file\n"; + } + my $self = __PACKAGE__->new; + $self->{filename} = $file; + open $self->{fh}, "<", $file; + $self->{linenum} = 0; + return $self; +} + +sub readline { + my $self = shift; + my $ret = {}; + my $line; # + my $fh = $self->{fh}; + while (defined ($line = <$fh>)) { + $self->{linenum}++; + if ($line =~ /^([\w\-]+):\s*(.+)/) { + $ret->{$1} = $2; + $self->{last} = \$ret->{$1}; + next; + } + if ($line eq "\n") { + return $ret; + } + if ($line =~ /^\s+(.+)/) { + die "Can't continue line without start" unless $self->{last}; + ${ $self->{last} } .= " $1"; + next; + } + + $line =~ s/[:^print:]/?/g; + die "Unexpected line in metafile $self->{file}, line $self->{linenum}: $line"; + } + return undef; +} + +1; Index: tags/1.05/lib/Brackup/Target.pm =================================================================== --- tags/1.05/lib/Brackup/Target.pm (revision 161) +++ tags/1.05/lib/Brackup/Target.pm (revision 161) @@ -0,0 +1,190 @@ +package Brackup::Target; + +use strict; +use warnings; +use Brackup::InventoryDatabase; +use Brackup::TargetBackupStatInfo; +use Brackup::Util 'tempfile'; +use Carp qw(croak); + +sub new { + my ($class, $confsec) = @_; + my $self = bless {}, $class; + my $name = $confsec->name; + $name =~ s!^TARGET:!! or die; + + $self->{keep_backups} = $confsec->value("keep_backups"); + $self->{inv_db} = + Brackup::InventoryDatabase->new($confsec->value("inventory_db") || + "$ENV{HOME}/.brackup-target-$name.invdb"); + + return $self; +} + +# return hashref of key/value pairs you want returned to you during a restore +# you should include anything you need to restore. +# keys should only contain \w +sub backup_header { + return {} +} + +# returns bool +sub has_chunk { + my ($self, $chunk) = @_; + die "ERROR: has_chunk not implemented in sub-class $self"; +} + +# returns true on success, or returns false or dies otherwise. +sub store_chunk { + my ($self, $chunk) = @_; + die "ERROR: store_chunk not implemented in sub-class $self"; +} + +# returns true on success, or returns false or dies otherwise. +sub delete_chunk { + my ($self, $chunk) = @_; + die "ERROR: delete_chunk not implemented in sub-class $self"; +} + +# returns a list of names of all chunks +sub chunks { + my ($self) = @_; + die "ERROR: chunks not implemented in sub-class $self"; +} + +sub inventory_db { + my $self = shift; + return $self->{inv_db}; +} + +sub add_to_inventory { + my ($self, $pchunk, $schunk) = @_; + my $key = $pchunk->inventory_key; + my $db = $self->inventory_db; + $db->set($key => $schunk->inventory_value); +} + +# return stored chunk, given positioned chunk, or undef. no +# need to override this, unless you have a good reason. +sub stored_chunk_from_inventory { + my ($self, $pchunk) = @_; + my $key = $pchunk->inventory_key; + my $db = $self->inventory_db; + my $invval = $db->get($key) + or return undef; + return Brackup::StoredChunk->new_from_inventory_value($pchunk, $invval); +} + +# return a list of TargetBackupStatInfo objects representing the +# stored backup metafiles on this target. +sub backups { + my ($self) = @_; + die "ERROR: backups method not implemented in sub-class $self"; +} + +# downloads the given backup name to the current directory (with +# *.brackup extension) +sub get_backup { + my ($self, $name) = @_; + die "ERROR: get_backup method not implemented in sub-class $self"; +} + +# deletes the given backup from this target +sub delete_backup { + my ($self, $name) = @_; + die "ERROR: delete_backup method not implemented in sub-class $self"; +} + +# removes old metafiles from this target +sub prune { + my ($self, %opt) = @_; + + my $keep_backups = $self->{keep_backups} || $opt{keep_backups} + or die "ERROR: keep_backups option not set\n"; + die "ERROR: keep_backups option must be at least 1\n" + unless $keep_backups > 0; + + # select backups to delete + my (%backups, @backups_to_delete) = (); + foreach my $backup_name (map {$_->filename} $self->backups) { + $backup_name =~ /^(.+)-\d+$/; + $backups{$1} ||= []; + push @{ $backups{$1} }, $backup_name; + } + foreach my $source (keys %backups) { + my @b = reverse sort @{ $backups{$source} }; + push @backups_to_delete, splice(@b, ($keep_backups > $#b+1) ? $#b+1 : $keep_backups); + } + + unless ($opt{dryrun}) { + $self->delete_backup($_) for @backups_to_delete; + } + return scalar @backups_to_delete; +} + +# removes orphaned chunks in the target +sub gc { + my ($self, %opt) = @_; + + # get all chunks and then loop through metafiles to detect + # referenced ones + my %chunks = map {$_ => 1} $self->chunks; + my $tempfile = tempfile(); + BACKUP: foreach my $backup ($self->backups) { + $self->get_backup($backup->filename, $tempfile); + my $parser = Brackup::Metafile->open($tempfile); + $parser->readline; # skip header + ITEM: while (my $it = $parser->readline) { + next ITEM unless $it->{Chunks}; + my @item_chunks = map { (split /;/)[3] } grep { $_ } split(/\s+/, $it->{Chunks} || ""); + delete $chunks{$_} for (@item_chunks); + } + } + my @orphaned_chunks = keys %chunks; + + # remove orphaned chunks + unless ($opt{dryrun}) { + $self->delete_chunk($_) for @orphaned_chunks; + } + return scalar @orphaned_chunks; +} + + + +1; + +__END__ + +=head1 NAME + +Brackup::Target - describes the destination for a backup + +=head1 EXAMPLE + +In your ~/.brackup.conf file: + + [TARGET:amazon] + type = Amazon + aws_access_key_id = ... + aws_secret_access_key = .... + +=head1 GENERAL CONFIG OPTIONS + +=over + +=item B + +The driver for this target type. The type B corresponds to the Perl module Brackup::Target::B. + +As such, the only valid options for type, if you're just using the +Target modules that come with the Brackup core, are: + +B -- see L for configuration details + +B -- see L for configuration details + +=item B + +The number of recent backups to keep when running I. + +=back Index: tags/1.05/lib/Brackup/StoredChunk.pm =================================================================== --- tags/1.05/lib/Brackup/StoredChunk.pm (revision 134) +++ tags/1.05/lib/Brackup/StoredChunk.pm (revision 134) @@ -0,0 +1,230 @@ +package Brackup::StoredChunk; + +use strict; +use warnings; +use Carp qw(croak); +use Digest::SHA1 qw(sha1_hex); + +# fields: +# pchunk - always +# backlength - memoized +# backdigest - memoized +# _chunkref - memoized +# compchunk - composite chunk, if we were added to a composite chunk +# compfrom - offset in composite chunk where we start +# compto - offset in composite chunk where we end + +sub new { + my ($class, $pchunk) = @_; + my $self = bless {}, $class; + $self->{pchunk} = $pchunk; + return $self; +} + +sub pchunk { $_[0]{pchunk} } + +# create the 'lite' or 'handle' version of a storedchunk. can't get to +# the chunkref from this, but callers aren't won't. and we'll DIE if they +# try to access the chunkref. +sub new_from_inventory_value { + my ($class, $pchunk, $invval) = @_; + + my ($dig, $len, $range) = split /\s+/, $invval; + + my $sc = bless { + pchunk => $pchunk, + backdigest => $dig, + backlength => $len, + }, $class; + + # normal + return $sc unless $range; + + # in case of little file in a composite chunk, + # we gotta be a range. + my ($from, $to) = $range =~ /^(\d+)-(\d+)$/ + or die "bogus range: $range"; + $sc->{compfrom} = $from; + $sc->{compto} = $to; + return $sc; +} + +sub clone_but_for_pchunk { + my ($self, $pchunk) = @_; + my $copy = bless {}, ref $self; + foreach my $f (qw(backlength backdigest compchunk compfrom compto)) { + $copy->{$f} = $self->{$f}; + } + $copy->{pchunk} = $pchunk; + return $copy; +} + +sub set_composite_chunk { + my ($self, $cchunk, $from, $to) = @_; + $self->{compchunk} = $cchunk; + + # forget our backup length/digest. this handle information + # to the stored chunk should be asked of our composite + # chunk in the future, when it's done populating. + $self->{backdigest} = undef; + $self->{backlength} = undef; + $self->forget_chunkref; + + $self->{compfrom} = $from; + $self->{compto} = $to; +} + +sub range_in_composite { + my $self = shift; + return undef unless $self->{compfrom} || $self->{compto}; + return "$self->{compfrom}-$self->{compto}"; +} + +sub file { + my $self = shift; + return $self->{pchunk}->file; +} + +sub root { + my $self = shift; + return $self->file->root; +} + +# returns true if encrypted, false otherwise +sub encrypted { + my $self = shift; + return $self->root->gpg_rcpt ? 1 : 0; +} + +sub compressed { + my $self = shift; + # TODO/FUTURE: support compressed chunks (for non-encrypted + # content; gpg already compresses) + return 0; +} + +# the original length, pre-encryption +sub length { + my $self = shift; + return $self->{pchunk}->length; +} + +# the length, either encrypted or not +sub backup_length { + my $self = shift; + return $self->{backlength} if defined $self->{backlength}; + $self->_populate_lengthdigest; + return $self->{backlength}; +} + +# the digest, either encrypted or not +sub backup_digest { + my $self = shift; + return $self->{backdigest} if $self->{backdigest}; + $self->_populate_lengthdigest; + return $self->{backdigest}; +} + +sub _populate_lengthdigest { + my $self = shift; + if (my $cchunk = $self->{compchunk}) { + $self->{backlength} = $cchunk->backup_length; + $self->{backdigest} = $cchunk->digest; + return 1; + } + + my $dataref = $self->chunkref; + $self->{backlength} = CORE::length($$dataref); + $self->{backdigest} = "sha1:" . sha1_hex($$dataref); + return 1; +} + +sub chunkref { + my $self = shift; + return $self->{_chunkref} if $self->{_chunkref}; + + # caller/consistency check: + Carp::confess("Can't access chunkref on lite StoredChunk instance (handle only)") + if $self->{backlength} || $self->{backdigest}; + + # non-encrypting case + unless ($self->encrypted) { + return $self->{_chunkref} = $self->{pchunk}->raw_chunkref; + } + + # encrypting case. + my $enc = $self->root->encrypt($self->{pchunk}->raw_chunkref); + return $self->{_chunkref} = \$enc; +} + +# set scalar/scalarref of encryptd chunkref +sub set_encrypted_chunkref { + my ($self, $arg) = @_; + die "ASSERT: not enc" unless $self->encrypted; + die "ASSERT: already set?" if $self->{backlength} || $self->{backdigest}; + + return $self->{_chunkref} = ref $arg ? $arg : \$arg; +} + +# lose the chunkref data +sub forget_chunkref { + my $self = shift; + delete $self->{_chunkref}; +} + +# to the format used by the metafile +sub to_meta { + my $self = shift; + my @parts = ($self->{pchunk}->offset, + $self->{pchunk}->length); + + if (my $range = $self->range_in_composite) { + push @parts, ( + $range, + $self->backup_digest, + ); + } else { + push @parts, ( + $self->backup_length, + $self->backup_digest, + ); + } + + # if the inventory database is lost, it should be possible to + # recover the inventory database from the *.brackup files. + # if a file only has on chunk, the digest(raw) -> digest(enc) + # can be inferred from the file's digest, then the stored + # chunk's digest. but if we have multiple chunks, we need + # to store each chunk's raw digest as well in the chunk + # list. we could do this all the time, but considering + # most files are small, we want to save space in the *.brackup + # meta file and only do it when necessary. + if ($self->encrypted && $self->file->chunks > 1) { + push @parts, $self->{pchunk}->raw_digest; + } + + return join(";", @parts); +} + +# aka "instructions to attach to a pchunk, on how to recover the pchunk from a target" +sub inventory_value { + my $self = shift; + + # when this chunk was stored as part of a composite chunk, the instructions + # are of form: + # sha1:deadbeef 0-50 + # which means download "sha1:deadbeef", then the contents will be in from + # byte offset 0 to byte offset 50 (length of 50). + if (my $range = $self->range_in_composite) { + return join(" ", + $self->backup_digest, + $self->backup_length, + $range); + } + + # else, the historical format: + # sha1:deadbeef + return join(" ", $self->backup_digest, $self->backup_length); +} + +1; Index: tags/1.05/lib/Brackup/ConfigSection.pm =================================================================== --- tags/1.05/lib/Brackup/ConfigSection.pm (revision 80) +++ tags/1.05/lib/Brackup/ConfigSection.pm (revision 80) @@ -0,0 +1,64 @@ +package Brackup::ConfigSection; +use strict; +use warnings; + +sub new { + my ($class, $name) = @_; + return bless { + _name => $name, + _accessed => {}, # key => 1 + }, $class; +} + +sub name { + my $self = shift; + return $self->{_name}; +} + +sub add { + my ($self, $key, $val) = @_; + push @{ $self->{$key} ||= [] }, $val; +} + +sub unused_config { + my $self = shift; + return sort grep { $_ ne "_name" && $_ ne "_accessed" && ! $self->{_accessed}{$_} } keys %$self; +} + +sub path_value { + my ($self, $key) = @_; + my $val = $self->value($key) || ""; + die "Path '$key' of '$val' isn't a valid directory\n" + unless $val && -d $val; + return $val; +} + +sub byte_value { + my ($self, $key) = @_; + my $val = $self->value($key); + return 0 unless $val; + return $1 if $val =~ /^(\d+)b?$/i; + return $1 * 1024 if $val =~ /^(\d+)kb?$/i; + return $1 * 1024 * 1024 if $val =~ /^(\d+)mb?$/i; + die "Unrecognized size format: '$val'\n"; +} + +sub value { + my ($self, $key) = @_; + $self->{_accessed}{$key} = 1; + my $vals = $self->{$key}; + return undef unless $vals; + die "Configuration section '$self->{_name}' has multiple values of key '$key' where only one is expected.\n" + if @$vals > 1; + return $vals->[0]; +} + +sub values { + my ($self, $key) = @_; + $self->{_accessed}{$key} = 1; + my $vals = $self->{$key}; + return () unless $vals; + return @$vals; +} + +1; Index: tags/1.05/lib/Brackup/CompositeChunk.pm =================================================================== --- tags/1.05/lib/Brackup/CompositeChunk.pm (revision 134) +++ tags/1.05/lib/Brackup/CompositeChunk.pm (revision 134) @@ -0,0 +1,94 @@ +package Brackup::CompositeChunk; + +use strict; +use warnings; +use Carp qw(croak); +use Digest::SHA1 qw(sha1_hex); + +use fields ( + 'used_up', + 'max_size', + 'data', + 'target', + 'digest', # memoized + 'finalized', # if we've written ourselves to the target yet + 'subchunks', # the chunks this composite chunk is made of + ); + +sub new { + my ($class, $root, $target) = @_; + my $self = ref $class ? $class : fields::new($class); + $self->{used_up} = 0; # bytes + $self->{finalized} = 0; # false + $self->{max_size} = $root->max_composite_size; + $self->{data} = ''; + $self->{target} = $target; + $self->{subchunks} = []; + return $self; +} + +sub append_little_chunk { + my ($self, $schunk) = @_; + die "ASSERT" if $self->{digest}; # its digest was already requested? + + my $from = $self->{used_up}; + $self->{used_up} += $schunk->backup_length; + $self->{data} .= ${ $schunk->chunkref }; + my $to = $self->{used_up}; + die "ASSERT" unless CORE::length($self->{data}) == $self->{used_up}; + + $schunk->set_composite_chunk($self, $from, $to); + push @{$self->{subchunks}}, $schunk; +} + +sub digest { + my $self = shift; + return $self->{digest} ||= "sha1:" . Digest::SHA1::sha1_hex($self->{data}); +} + +sub can_fit { + my ($self, $len) = @_; + return $len <= ($self->{max_size} - $self->{used_up}); +} + +# return on success; die on any failure +sub finalize { + my $self = shift; + die "ASSERT" if $self->{finalized}++; + + $self->{target}->store_chunk($self) + or die "chunk storage of composite chunk failed.\n"; + + foreach my $schunk (@{$self->{subchunks}}) { + $self->{target}->add_to_inventory($schunk->pchunk => $schunk); + } + + return 1; +} + +sub stored_chunk_from_dup_internal_raw { + my ($self, $pchunk) = @_; + my $ikey = $pchunk->inventory_key; + foreach my $schunk (@{$self->{subchunks}}) { + next unless $schunk->pchunk->inventory_key eq $ikey; + # match! found a duplicate within ourselves + return $schunk->clone_but_for_pchunk($pchunk); + + } + return undef; +} + +# +# make this duck-typed like a StoredChunk, so targets can store it +*backup_digest = \&digest; +sub backup_length { + my $self = shift; + return CORE::length($self->{data}); +} +sub chunkref { return \ $_[0]->{data} } +sub inventory_value { + die "ASSERT: don't expect this to be called"; +} +# + +1; Index: tags/1.05/lib/Brackup/GPGProcManager.pm =================================================================== --- tags/1.05/lib/Brackup/GPGProcManager.pm (revision 107) +++ tags/1.05/lib/Brackup/GPGProcManager.pm (revision 107) @@ -0,0 +1,131 @@ +package Brackup::GPGProcManager; +use strict; +use warnings; +use Brackup::GPGProcess; +use POSIX ":sys_wait_h"; + +sub new { + my ($class, $iter, $target) = @_; + return bless { + chunkiter => $iter, + procs => {}, # "addr(pchunk)" => GPGProcess + target => $target, + procs_running => {}, # pid -> GPGProcess + uncollected_bytes => 0, + }, $class; +} + +sub enc_chunkref_of { + my ($self, $pchunk) = @_; + + my $proc = $self->{procs}{$pchunk}; + unless ($proc) { + # catch iterator up to the point that was + # requested, or blow up. + my $found = 0; + my $iters = 0; + while (my $ich = $self->{chunkiter}->next) { + if ($ich == $pchunk) { + $found = 1; + last; + } + $iters++; + warn "iters = $iters\n"; + } + die "Not found" unless $found; + $proc = $self->gen_process_for($pchunk); + } + + while ($proc->running) { + my $pid = $self->wait_for_a_process(1) or die + "No processes were reaped!"; + } + + $self->_proc_summary_dump; + my $cref = $self->get_proc_chunkref($proc); + $self->_proc_summary_dump; + $self->start_some_processes; + + return $cref; +} + +sub start_some_processes { + my $self = shift; + + # eat up any pending zombies + while ($self->wait_for_a_process(0)) {} + + my $pchunk; + # TODO: make this stuff configurable/auto-tuned + while ($self->num_running_procs < 5 && + $self->num_uncollected_bytes < 128 * 1024 * 1024 && + ($pchunk = $self->next_chunk_to_encrypt)) { + $self->_proc_summary_dump; + $self->gen_process_for($pchunk); + $self->_proc_summary_dump; + } +} + +sub _proc_summary_dump { + my $self = shift; + return unless $ENV{GPG_DEBUG}; + + printf STDERR "num_running=%d, num_outstanding_bytes=%d\n", + $self->num_running_procs, $self->num_uncollected_bytes; +} + +sub next_chunk_to_encrypt { + my $self = shift; + while (my $ev = $self->{chunkiter}->next) { + next if $ev->isa("Brackup::File"); + my $pchunk = $ev; + next if $self->{target}->stored_chunk_from_inventory($pchunk); + return $pchunk; + } + return undef; +} + +sub get_proc_chunkref { + my ($self, $proc) = @_; + my $cref = $proc->chunkref; + delete $self->{procs}{$proc}; + $self->{uncollected_bytes} -= length($$cref); + return $cref; +} + +# returns PID of a process that finished +sub wait_for_a_process { + my ($self, $block) = @_; + my $flags = $block ? 0 : WNOHANG; + my $kid = waitpid(-1, $flags); + return 0 if ! $block && $kid <= 0; + die "no child?" if $kid < 0; + return 0 unless $kid; + + my $proc = $self->{procs_running}{$kid} or die "Unknown child + process $kid finished!\n"; + + delete $self->{procs_running}{$proc->pid} or die; + $proc->note_stopped; + $self->{uncollected_bytes} += $proc->size_on_disk; + + return $kid; +} + +sub num_uncollected_bytes { $_[0]{uncollected_bytes} } + +sub gen_process_for { + my ($self, $pchunk) = @_; + my $proc = Brackup::GPGProcess->new($pchunk); + $self->{procs_running}{$proc->pid} = $proc; + $self->{procs}{$pchunk} = $proc; + return $proc; +} + +sub num_running_procs { + my $self = shift; + return scalar keys %{$self->{procs_running}}; +} + +1; + Index: tags/1.05/lib/Brackup/Manual/Overview.pod =================================================================== --- tags/1.05/lib/Brackup/Manual/Overview.pod (revision 161) +++ tags/1.05/lib/Brackup/Manual/Overview.pod (revision 161) @@ -0,0 +1,148 @@ +=head1 NAME + +Brackup::Manual::Overview - how Brackup works, and how to use it + +=head1 Quick Start Guide + +=head2 Setup your config file + +Run B to initialize your config file. You'll see: + + $ brackup + Error: + Your config file needs tweaking. + I put a commented-out template at: /home/bradfitz/.brackup.conf + + brackup --from=[source_name] --to=[target_name] [--output=] + brackup --help + +Now, go edit your config file: + + $ $EDITOR ~/.brackup.conf + +Tweak as appropriate. + +For details on what's tweakable, see L (a "source"), or +L (a destination). + +=head2 Do a backup + +Now that you've got a source and target named, run a backup. I like +watching it all happen with the --verbose (or -v) option: + + $ brackup --from=myhome --to=amazon -v + +=head2 What just happened? + +Let's look around at what just happened. + +First, you'll notice a file named, by default, +"myhome-yyyymmdd.brackup" in your current directory. Go look at it. +It describes the state of the tree (the "root", or "source") that you +just backed up. You might want to keep this file. Although, if you +don't, it's also stored on the target (in this case, Amazon), so it's +not critical. (You can always re-download lost .brackup files with +L) + +You might also notice two SQLite files at: + + $SOURCE_ROOT/.brackup-digest.db + $HOME/.brackup-target-amazon.invdb + +These are the L and +L files, both of which make future +incremental backups fast. + +=head1 Incremental backups + +Incremental backups are essentially free, only storing new chunks, +even if you rearrange your directory tree or rename all your files. +Brackup doesn't use the I of your files to decide what's new in +an incremental backup, only the contents. + +For two back-to-back backups, with no data changes in-between, the +only cost of an incremental backup is that another metafile (*.brackup) is +produced, which is proportional in size to the I +you're backing up (not the size of the files). + +Another good side-effect of storing backups based on their digests is +that multiple, duplicate files on your source are only stored on the +target once. (but yes, they're restored to all original locations) + +=head1 Using encryption + +Brackup supports backing up with public key encryption, using GNU +Privacy Guard (GnuPG). One of the great advantage of using public key +encryption is that your machines doing backups only need your public +key, so you can run automated backups from hosts which are on the +public Internet and might be get compromised, without worrying about +your private key getting stolen. (however, you'd still worry about +your machine getting compromised for lots of other reasons...) + +In any case, you encrypt files I, and this is a property +on a backup source (see L). For example, in my config +file, I have: + + [SOURCE:bradhome] + ... + path = /home/bradfitz/ + gpg_recipient = 5E1B3EC5 + ... + +Where 5E1B3EC5 corresponds to the key signature for myself as seen in: + + $ gpg --list-keys + ... + pub 1024D/5E1B3EC5 2006-03-20 + uid Brad Fitzpatrick + .... + +While you backup automatically without a human present, a restore from +encryption requires an interactive session for you to enter your +private key's passphrase into gpg-agent. + +To create a new key, run: + + $ gpg --gen-key + +But really, you should go read a gpg manual first. Notably, B. If you lose the disk +with your files which also contain your private key, your encrypted backups on +Amazon won't do you much good, since you'll have no way to decrypt them. +I recommend burning your private key to a CD, as well as printing it out +on paper. (Worst case you can type it back in, or use OCR.) Export with: + + $ gpg --export-secret-keys --armor + +=head1 Restores + +To do a restore, you'll need your *.brackup file handy. If you lost +it, you can re-download it from your backup target with +L. Then run: + + brackup-restore --from=foo.brackup --to= --all + +For more options, see: + + brackup-restore --help + +=head1 Number of backups to keep + +To free space on your target you can remove old backups. There are two steps +to do this: + + brackup-target prune + brackup-target gc + +The first command will look for backup metafiles in your target and remove the +oldest ones according to the I option you specified in the config +file. Thus, if you have, say, 15 backups stored and I is set to 10 +then I will remove the oldest 5 backups. + +The second command will remove from your target the orphaned chunks that are no +more referenced by any metafile. This will free some space while preserving chunks +that are still referenced by recent backups. + +=head1 SEE ALSO + +L Index: tags/1.05/lib/Brackup/DigestCache.pm =================================================================== --- tags/1.05/lib/Brackup/DigestCache.pm (revision 128) +++ tags/1.05/lib/Brackup/DigestCache.pm (revision 128) @@ -0,0 +1,99 @@ +package Brackup::DigestCache; +use strict; +use warnings; + +# for now. kinda ghetto. could be anything... +# should 'have-a' instead of 'is-a' dict, and proxy +# the methods through. but can do that later... +use base 'Brackup::Dict::SQLite'; + +sub new { + my ($class, $root, $rconf) = @_; + my $dir = $root->path; + my $file = $rconf->value('digestdb_file') || "$dir/" . default_filename(); + my $self = $class->SUPER::new("digest_cache", $file); + # ... + return $self; +} + +sub default_filename { ".brackup-digest.db" }; + + +1; + +__END__ + +=head1 NAME + +Brackup::DigestCache - cache digests of file and chunk contents + +=head1 DESCRIPTION + +The Brackup DigestCache caches the digests (currently SHA1) of files +and file chunks, to prevent untouched files from needing to be re-read +on subsequent, iterative backups. + +The digest cache is I a cache. It has no important data in it, +so don't worry about losing it. Worst case if you lose it: subsequent +backups take longer while the digest cache is re-built. + +=head1 DETAILS + +=head2 File format + +While designed to be abstract, the only supported digest cache format at +the moment is an SQLite database, stored in a single file. The schema +is created automatically as needed... no database maintenance is required. + +=head2 File location + +The SQLite file is stored in either the location specified in a +L's [SOURCE] declaration in ~/.brackup.conf, as: + + [SOURCE:home] + path = /home/bradfitz/ + # be explicit if you want: + digestdb_file = /var/cache/brackup-brad/digest-cache-bradhome.db + +Or, more commonly (and recommended), is to not specify it and accept +the default location, which is ".brackup-digest.db" in the root's +root directory. + + [SOURCE:home] + path = /home/bradfitz/ + # this is the default: + # digestdb_file = /home/bradfitz/.brackup-digest.db + +=head2 SQLite Schema + +This is made automatically for you, but if you want to look around in +it, the schema is: + + CREATE TABLE digest_cache ( + key TEXT PRIMARY KEY, + value TEXT + ) + +=head2 Keys & Values stored in the cache + +B (see L) + + [rootname]path/to/file.txt:,,, + +B (see L) + + [rootname]path/to/file.txt:,,,;o=;l= + +B + +In either case, the values are the digest of the chunk/file, in form: + + sha1:e23c4b5f685e046e7cc50e30e378ab11391e528e + +=head1 SEE ALSO + +L + + + + Index: tags/1.05/Makefile.PL =================================================================== --- tags/1.05/Makefile.PL (revision 146) +++ tags/1.05/Makefile.PL (revision 146) @@ -0,0 +1,17 @@ +#!/usr/bin/perl +use strict; +use ExtUtils::MakeMaker; + +WriteMakefile( NAME => 'Brackup', + VERSION_FROM => 'lib/Brackup.pm', + EXE_FILES => [ 'brackup', 'brackup-restore', 'brackup-target' ], + PREREQ_PM => { + 'DBD::SQLite' => 0, + 'Digest::SHA1' => 0, + 'DBI' => 0, + }, + ABSTRACT_FROM => 'lib/Brackup.pm', + AUTHOR => 'Brad Fitzpatrick ', + ); + + Index: tags/1.05/brackup =================================================================== --- tags/1.05/brackup (revision 146) +++ tags/1.05/brackup (revision 146) @@ -0,0 +1,144 @@ +#!/usr/bin/perl + +=head1 NAME + +brackup - do a backup using Brackup + +=head1 SYNOPSIS + + $ brackup --from= --to= --output=my_backup.brackup + +=head2 OPTIONS + +=over 4 + +=item --from=NAME + +Required. The source of your backup. Must match a [SOURCE:NAME] +config section in your ~/.brackup.conf (which is auto-created for you +on first run, so then you just have to go modify it) + +=item --to=NAME + +Required. The destination for your backup. Must match a [TARGET:NAME] +config section in your ~/.brackup.conf + +=item --output=FILE + +Option. Defaults to "source-YYYYMMDD.brackup". This is the "metafile" index +you'll need to restore. + +=item --verbose|-v + +Show status during backup. + +=back + +=head1 WARRANTY + +Brackup is distributed as-is and comes without warranty of any kind, +expressed or implied. We aren't responsible for your data loss. + +=head1 SEE ALSO + +brackup-restore + +=head1 AUTHOR + +Brad Fitzpatrick Ebrad@danga.comE + +Copyright (c) 2006-2007 Six Apart, Ltd. All rights reserved. + +This module is free software. You may use, modify, and/or redistribute this +software under the terms of same terms as perl itself. + +=cut + +use strict; +use warnings; +use Getopt::Long; + +use Cwd; +use FindBin qw($Bin); +use lib "$Bin/lib"; + +use Brackup; + +my ($src_name, $target_name, $backup_file, $opt_help); +my $opt_dryrun; +my $opt_verbose; +my $opt_du_stats; + +my $config_file = Brackup::Config->default_config_file_name; + +usage() unless + GetOptions( + 'from=s' => \$src_name, + 'to=s' => \$target_name, + 'verbose' => \$opt_verbose, + 'output=s' => \$backup_file, + 'help' => \$opt_help, + 'dry-run' => \$opt_dryrun, + 'du-stats' => \$opt_du_stats, + 'config=s' => \$config_file, + ); +usage() if @ARGV; + +if ($opt_help) { + eval "use Pod::Usage;"; + Pod::Usage::pod2usage( -verbose => 1, -exitval => 0 ); + exit 0; +} + +my $config = eval { Brackup::Config->load($config_file) } or + usage($@); + +if ($opt_du_stats && $src_name) { + my $root = eval { $config->load_root($src_name); } or + die "Bogus --from name"; + $root->du_stats; + exit 0; +} + +usage() unless $src_name && $target_name; + +my $cwd = getcwd(); + +sub usage { + my $why = shift || ""; + if ($why) { + $why =~ s/\s+$//; + $why = "Error: $why\n\n"; + } + die "${why}brackup --from=[source_name] --to=[target_name] [--output=]\nbrackup --help\n"; +} + +my $root = eval { $config->load_root($src_name); } or + usage($@); + +my $target = eval { $config->load_target($target_name); } or + usage($@); + + +my @now = localtime(); +$backup_file ||= $root->name . "-" . sprintf("%04d%02d%02d", $now[5]+1900, $now[4]+1, $now[3]) . ".brackup"; +$backup_file =~ s!^~/!$ENV{HOME}/! if $ENV{HOME}; +$backup_file = "$cwd/$backup_file" unless $backup_file =~ m!^/!; + +my $backup = Brackup::Backup->new( + root => $root, + target => $target, + dryrun => $opt_dryrun, + verbose => $opt_verbose, + ); + +if (my $stats = eval { $backup->backup($backup_file) }) { + warn "Backup complete." if $opt_verbose; + if ($opt_dryrun || $opt_verbose) { + $stats->print; + } + exit 0; +} else { + warn "Error running backup: $@\n"; + exit 1; +} Index: tags/1.05/Changes =================================================================== --- tags/1.05/Changes (revision 162) +++ tags/1.05/Changes (revision 162) @@ -0,0 +1,130 @@ +1.05 (august 2, 2007) + + - 'prune' and 'gc' commands commands for both Amazon + and Filesystem targets. from Alessandro Ranellucci . + +1.04 (july 30, 2007) + + - Amazon list_backups and delete backups (and delete for filesystem + target too), from Alessandro Ranellucci + + - make tests pass on OS X (Jesse Vincent) + +1.03 (may 23, 2007) + + - brackup-restore's verbose flag is more verbose now, showing files + as they're restored. + + - brackup-restore can restore from an encrypted *.brackup file now, + firing up gpg for user to decrypt to a tempfile + + - brackup-target tool, to list/get brackup files from a target, + and in the future do garbage collection on no-longer-referenced + chunks (once a command exists to delete a brackup file from a target) + + - stop leaking temp files + + - doc fixes/additions + +1.02 (may 22, 2007) + + - support for merging little files together into big chunks + on the backup target. aka "tail packing". requires no changes + to target drivers. this should speed backups, as less network + round-trips. will also be cheaper, once Amazon starts charging + per number of HTTP requests in June. + + - improved docs + +1.01 (may 21, 2007) + + - lot of new/updated docs + +1.00 (may 21, 2007) + + RELEASE NOTE: The author/maintainer of Brackup is finally happy now, + and has 40 GB of data stored on Amazon, encrypted. You can + trust this now. And the file formats aren't changing (or aren't + changing without being compatible with old *.brackup/Amazon + formats...) + + - track in meta header the default (most often occuring) modes for + files and directories, then don't list those for each file/dir + with those mode. saves on disk space on *.brackup files + + - support 'noatime = 1' option on a source root, because atimes are + often useless, so waste of space in metafile. + + - rename digestdb back to digestcache, now that it's purely a cache + again. + + - fix memory leak in case where chunk exists on target, but local + digest database was lost, and digest of chunk had to be recomputed. + in that case, the raw chunk was kept in memory until the end + (which it likely would never reach, accumulating GBs of RAM) + + - make PositionedChunk use the digest cache (which I guess was + re-fleshed out in the big refactor but never used...). so + iterative backups are fast again... no re-reading all files + in, blowing away all caches. + + - clean up old, dead code in Amazon target (the old inventory db which + is now an official part of the core, and in the Target base class) + + - retry PUTs to Amazon on failure, a few times, pausing in-between, + in case it was a transient error, as seems to happen occasionally + + - halve number of stats when walking backup root + + - cleanups, strictness + + - don't upload meta files when in dry-run mode + + - update amazon target support to work again, with the new inventory + database support (now separated from the old digest database) + + - merge in the refactoring branch, in which a lot of long-standing + pet peeves in the design were rethought/redone. + + - make decryption --use-agent and --batch, and help out if env not set + and gpg-agent probably not running + + - support putting .meta files besides .chunk files on the Target + to enable reconstructing the digest database in the future, should + it get lost. also start to flesh out per-chunk digests, which + would enable backing up large databases (say, InnoDB tablespaces) where + large chunks of the file never change. + + - new --du-stats to command to act like the du(1) command, but + based on a root in brackup.conf, and skipping ignored directories. + good to let you know how big a backup will be. + + - walk directories smarter: jump over directories early which ignore + patterns show as never matching. + + - deal w/ encryption better: tell chunks when the backup target + will need data, so it can forget cached digest/backlength + ahead of time w/o errors/warnings later. + + - start of stats code (to give stats after a backup). not done. + +0.91 (sep 29 2006) + - there's now a restore command (brackup-restore) + + - amazon restore support + + - use gpg --trust-model=always for new gpg that is more paranoid. + + - mostly usable. some more switches would be nice later. real + 1.00 release will come after few weeks/months of testing/tweaks. + +0.80 + - restore works + + - lot more tests + + - notable bug fix with encrypted backups. metafiles could have wrong sizes. + +0.71 + - first release to CPAN, didn't support restoring yet. + also didn't have a Changes file Index: tags/1.05/brackup-restore =================================================================== --- tags/1.05/brackup-restore (revision 153) +++ tags/1.05/brackup-restore (revision 153) @@ -0,0 +1,126 @@ +#!/usr/bin/perl + +# TODO: skip-if-exists (and ignore zero byte files) +# TODO: continue-on-errors +# Error doing restore: File restore/.brackup-digest.db (.brackup-digest.db) already exists. Aborting. at /raid/bradfitz/proj/brackup/trunk/lib/Brackup/Restore.pm line 157, <$fh> line 20. + +=head1 NAME + +brackup-restore - The brackup restore tool. + +=head1 SYNOPSIS + + $ brackup-restore --from=foo.brackup --to= --all + $ brackup-restore --from=foo.brackup --to= --just= + $ brackup-restore --from=foo.brackup --to= --just= + +=head2 OPTIONS + +=over 4 + +=item --from=NAME + +Required. The backup metafile, describing the tree you want to +restore. Probably named like "source-YYYYMMDD.brackup". If you lost +it, it's also stored on your backup target, and you can fetch it with +L. + +=item --to=NAME + +Required. The destination root directory for your restored files. + +=item --all + +Restore all files. + +=item --just="DIRECTORY" + +Restore just the directory named. (and all contents thereunder) + +=item --just="FILE" + +Restore just the file named. + +=back + +=head1 WARRANTY + +Brackup is distributed as-is and comes without warranty of any kind, +expressed or implied. We aren't responsible for your data loss. + +=head1 AUTHOR + +Brad Fitzpatrick Ebrad@danga.comE + +Copyright (c) 2006-2007 Six Apart, Ltd. All rights reserved. + +This module is free software. You may use, modify, and/or redistribute this +software under the terms of same terms as perl itself. + +=cut + +use strict; +use warnings; +use Getopt::Long; + +use FindBin qw($Bin); +use lib "$Bin/lib"; + +use Brackup; +use Brackup::Util qw(tempfile); + +my ($opt_verbose, $meta_file, $opt_help, $restore_dir, $opt_all, $prefix); + +my $config_file = Brackup::Config->default_config_file_name; + +usage() unless + GetOptions( + 'from=s' => \$meta_file, + 'to=s' => \$restore_dir, + 'verbose' => \$opt_verbose, + 'help' => \$opt_help, + 'all' => \$opt_all, + 'just=s' => \$prefix, + 'config=s' => \$config_file, + ); + +if ($opt_help) { + eval "use Pod::Usage;"; + Pod::Usage::pod2usage( -verbose => 1, -exitval => 0 ); + exit 0; +} + +my $config = eval { Brackup::Config->load($config_file) } or + usage($@); + +usage() unless $meta_file && $restore_dir && ($prefix || $opt_all); +usage("Given directory isn't a directory") unless -d $restore_dir; +usage("Given restore file doesn't exist") unless -e $meta_file; +usage("Given restore file isn't a file") unless -f $meta_file; +$prefix ||= ""; # with -all, "", which means everything + +my $restore = Brackup::Restore->new( + to => $restore_dir, + prefix => $prefix, + file => $meta_file, + verbose => $opt_verbose, + ); + +if (eval { $restore->restore }){ + warn "Restore complete." if $opt_verbose; + exit 0; +} else { + warn "Error doing restore: $@\n"; + exit 1; +} + + +sub usage { + my $why = shift || ""; + if ($why) { + $why =~ s/\s+$//; + $why = "Error: $why\n\n"; + } + die "${why}brackup-restore --from=[metafile.brackup] --to=[restore_dir] <--all|--just=[what]>\nbrackup-restore --help\n"; + +} Index: tags/1.05/MANIFEST.SKIP =================================================================== --- tags/1.05/MANIFEST.SKIP (revision 136) +++ tags/1.05/MANIFEST.SKIP (revision 136) @@ -0,0 +1,35 @@ +makedocs.pl +\.shipit +\.brackup$ + +# Avoid version control files. +\bRCS\b +\bCVS\b +\bSCCS\b +,v$ +\B\.svn\b +\b_darcs\b + +# Avoid Makemaker generated and utility files. +\bMANIFEST\.bak +\bMakefile$ +\bblib/ +\bMakeMaker-\d +\bpm_to_blib\.ts$ +\bpm_to_blib$ +\bblibdirs\.ts$ # 6.18 through 6.25 generated this + +# Avoid Module::Build generated and utility files. +\bBuild$ +\b_build/ + +# Avoid temp and backup files. +~$ +\.old$ +\#$ +\b\.# +\.bak$ + +# Avoid Devel::Cover files. +\bcover_db\b + Index: tags/1.05/brackup-target =================================================================== --- tags/1.05/brackup-target (revision 161) +++ tags/1.05/brackup-target (revision 161) @@ -0,0 +1,173 @@ +#!/usr/bin/perl + +=head1 NAME + +brackup-target - Manage your backup targets + +=head1 SYNOPSIS + + $ brackup-target [opts] list_backups + $ brackup-target [opts] get_backup + $ brackup-target [opts] get_backups + $ brackup-target [opts] delete_backup + $ brackup-target [opts] prune # remove old backups + $ brackup-target [opts] gc # run garbage collector + +=head2 OPTIONS + +=over 4 + +=item --dest=DIR + +Destination to write files to. Defaults to current working directory. + +=item --verbose|-v + +Be verbose with status. + +=item --dry-run + +Do not actually execute write operations. + +=item --keep-backups + +To be used in combination with the I command. This overrides the +I option specified in the configuration file. + +=back + +=head1 WARRANTY + +Brackup is distributed as-is and comes without warranty of any kind, +expressed or implied. We aren't responsible for your data loss. + +=head1 SEE ALSO + +brackup-restore + +=head1 AUTHOR + +Brad Fitzpatrick Ebrad@danga.comE + +Copyright (c) 2006-2007 Six Apart, Ltd. All rights reserved. + +This module is free software. You may use, modify, and/or redistribute this +software under the terms of same terms as perl itself. + +=cut + +use strict; +use warnings; +use Getopt::Long; + +use Cwd; +use FindBin qw($Bin); +use lib "$Bin/lib"; + +use Brackup; + +my $config_file; +my $destdir; +my $opt_help; +my $opt_verbose; +my $opt_keep_backups; +my $opt_dryrun; +usage() unless + GetOptions( + 'verbose' => \$opt_verbose, + 'dest=s' => \$destdir, + 'config=s' => \$config_file, + 'keep-backups=i' => \$opt_keep_backups, + 'dry-run' => \$opt_dryrun, + 'help' => \$opt_help, + ); + +if ($destdir) { + chdir $destdir or die "Failed to chdir to $destdir: $!\n"; +} + +if ($opt_help) { + eval "use Pod::Usage;"; + Pod::Usage::pod2usage( -verbose => 1, -exitval => 0 ); + exit 0; +} + +my $config = eval { Brackup::Config->load($config_file) } or + usage($@); + +my $target_name = shift or usage(); +my $cmd_name = shift or usage(); + +my $target = eval { $config->load_target($target_name); } or + usage($@); + +my $code = __PACKAGE__->can("CMD_$cmd_name") or + usage("Unknown/unimplemented command."); + +exit($code->() ? 0 : 1); + + +sub CMD_list_backups { + foreach my $si ($target->backups) { + printf("%-35s %-20s %10d\n", + $si->filename, + $si->time, + $si->size); + } + return 1; +} + +sub CMD_get_backup { + my $name = shift @ARGV or + die "get_backup requires a filename to download"; + $target->get_backup($name) + or die "Failed to retrieve backup $name\n"; +} + +sub CMD_get_backups { + foreach my $si ($target->backups) { + my $size = $si->size; + my $name = $si->filename; + no warnings 'uninitialized'; + if (-s "$name.brackup" == $size || -s "$name.brackup.orig" == $size) { + debug("Skipping $name; already have it"); + next; + } + debug("Fetching $name"); + $target->get_backup($si->filename); + } +} + +sub CMD_delete_backup { + my $name = shift @ARGV or + die "delete_backup requires a filename to download"; + $target->delete_backup($name) + or die "Failed to delete backup $name\n"; +} + +sub CMD_prune { + my $removed_count = $target->prune( keep_backups => $opt_keep_backups, + dryrun => $opt_dryrun); + debug("$removed_count backups removed from target"); +} + +sub CMD_gc { + my $removed_chunks = $target->gc(dryrun => $opt_dryrun); + debug("$removed_chunks chunks removed from target"); +} + +sub debug { + my $msg = shift; + return unless $opt_verbose; + warn "$msg\n"; +} + + +sub usage { + my $why = shift || ""; + if ($why) { + $why =~ s/\s+$//; + $why = "Error: $why\n\n"; + } + die "${why}brackup-target [...]\nbrackup-target --help\n"; +} Index: tags/1.05/.shipit =================================================================== --- tags/1.05/.shipit (revision 126) +++ tags/1.05/.shipit (revision 126) @@ -0,0 +1,5 @@ +steps = FindVersion, ChangeVersion, CheckChangeLog, DistTest, Commit, Tag, MakeDist, UploadCPAN, AddToSVNDir +svn.tagpattern = %v +AddToSVNDir.dir = /home/lj/cvs/web-danga/dist/Brackup/ + +