Changeset 65

Show
Ignore:
Timestamp:
10/08/06 01:04:31 (2 years ago)
Author:
bradfitz
Message:
  • support putting .meta files besides .chunk files on the Target
    to enable reconstructing the digest database in the future, should
    it get lost. also start to flesh out per-chunk digests, which
    would enable backing up large databases (say, InnoDB tablespaces) where
    large chunks of the file never change.
  • also move most encryption to Root class, to avoid duplication
Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved

  • trunk/Changes

    r63 r65  
     1  - support putting .meta files besides .chunk files on the Target 
     2    to enable reconstructing the digest database in the future, should 
     3    it get lost.  also start to flesh out per-chunk digests, which 
     4    would enable backing up large databases (say, InnoDB tablespaces) where  
     5    large chunks of the file never change. 
     6 
    17  - new --du-stats to command to act like the du(1) command, but 
    28    based on a root in brackup.conf, and skipping ignored directories. 

  • trunk/lib/Brackup/Backup.pm

    r62 r65  
    7777    # store the metafile, encrypted, on the target 
    7878    if (my $rcpt = $self->{root}->gpg_rcpt) { 
    79         my $encfile .= ".enc"; 
     79        my $encfile = $backup_file . ".enc"; 
    8080        system($self->{root}->gpg_path, $self->{root}->gpg_args, 
    8181               "--trust-model=always", 

  • trunk/lib/Brackup/Chunk.pm

    r62 r65  
    55use Carp qw(croak); 
    66use Digest::SHA1 qw(sha1_hex); 
    7 use File::Temp qw(tempfile); 
    87 
    98# fields 
    …… 
    1413# sometimes: 
    1514#   digest          if calculated "type:hex" 
     15#   _rawdigest      "type:hex" of not encrypted, not compressed. 
    1616#   _chunkref       if calculated, scalarref 
    1717#   backlength      if calculated 
    …… 
    6363} 
    6464 
    65 sub chunkref { 
    66     my $self = shift; 
    67     return $self->{_chunkref} if $self->{_chunkref}; 
     65# returns true if encrypted, false otherwise 
     66sub encrypted { 
     67    my $self = shift; 
     68    return $self->root->gpg_rcpt ? 1 : 0; 
     69} 
     70 
     71sub raw_digest { 
     72    my $self = shift; 
     73    return $self->{_rawdigest} if $self->{_rawdigest}; 
     74    my $rchunk = $self->raw_chunkref; 
     75    return $self->{_rawdigest} = "sha1:" . sha1_hex($$rchunk); 
     76} 
     77 
     78sub raw_chunkref { 
     79    my $self = shift; 
     80    return $self->{_raw_chunkref} if $self->{_raw_chunkref}; 
    6881 
    6982    my $data; 
    70     my $dataref_with_learning = sub { 
    71         # record the encrypted/ 
    72         $self->_learn_lengthdigest_from_data(\$data); 
    73         return $self->{_chunkref} = \$data; 
    74     }; 
    75  
    76     my $root = $self->root; 
    77     my $gpg_rcpt = $root->gpg_rcpt; 
    78  
    7983    my $fullpath = $self->{file}->fullpath; 
    8084    open(my $fh, $fullpath) or die "Failed to open $fullpath: $!\n"; 
    8185    seek($fh, $self->{offset}, 0) or die "Couldn't seek: $!\n"; 
    82     read($fh, $data, $self->{length}) or die "Failed to read: $!\n"; 
     86    my $rv = read($fh, $data, $self->{length}) 
     87        or die "Failed to read: $!\n"; 
     88    unless ($rv == $self->{length}) { 
     89        Carp::confess("Read $rv bytes, not $self->{length}"); 
     90    } 
     91 
     92    return $self->{_raw_chunkref} = \$data; 
     93} 
     94 
     95sub chunkref { 
     96    my $self = shift; 
     97    return $self->{_chunkref} if $self->{_chunkref}; 
     98 
     99    my $dataref_and_cache_it = sub { 
     100        my $ref = shift; 
     101        $self->_learn_lengthdigest_from_data($ref); 
     102        return $self->{_chunkref} = $ref; 
     103    }; 
    83104 
    84105    # non-encrypting case 
    85     return $dataref_with_learning->() unless $gpg_rcpt; 
    86  
    87     # FIXME: let users control where their temp files go? 
    88     my ($tmpfh, $tmpfn) = tempfile(); 
    89     print $tmpfh $data or die "failed to print: $!"; 
    90     close $tmpfh or die "failed to close: $!\n"; 
    91     die "size not right" unless -s $tmpfn == $self->{length}; 
    92  
    93     my ($etmpfh, $etmpfn) = tempfile(); 
    94  
    95     system($self->root->gpg_path, $self->root->gpg_args, 
    96            "--recipient", $gpg_rcpt, 
    97            "--trust-model=always", 
    98            "--batch", 
    99            "--encrypt", 
    100            "--output=$etmpfn", 
    101            "--yes", $tmpfn) 
    102         and die "Failed to run gpg: $!\n"; 
    103     open (my $enc_fh, $etmpfn) or die "Failed to open $etmpfn: $!\n"; 
    104     $data = do { local $/; <$enc_fh>; }; 
    105  
    106     return $dataref_with_learning->(); 
     106    unless ($self->encrypted) { 
     107        return $dataref_and_cache_it->($self->raw_chunkref); 
     108    } 
     109 
     110    # encrypting case. 
     111    my $enc = $self->root->encrypt($self->raw_chunkref); 
     112    return $dataref_and_cache_it->(\$enc); 
    107113} 
    108114 
    …… 
    111117    my $self = shift; 
    112118    delete $self->{_chunkref}; 
     119    delete $self->{_raw_chunkref}; 
    113120} 
    114121 
    …… 
    207214} 
    208215 
     216sub compressed { 
     217    my $self = shift; 
     218    return 0;  # FUTURE: support compressed chunks 
     219} 
     220 
     221sub meta_contents { 
     222    my $self = shift; 
     223    my $meta = ""; 
     224    my $addmeta = sub { 
     225        my ($k, $v) = @_; 
     226        $meta .= "$k: $v\n"; 
     227    }; 
     228 
     229    if ($self->encrypted) { 
     230        $addmeta->("chunkcachekey", $self->cachekey); 
     231    } 
     232 
     233    if ($self->compressed) { 
     234        #FUTURE: 
     235        #$addmeta->("compression", "gz"); 
     236    } 
     237 
     238    if ($self->encrypted || $self->compressed) { 
     239        $addmeta->("raw_digest", $self->raw_digest); 
     240    } 
     241 
     242    if ($self->encrypted) { 
     243        $meta = $self->root->encrypt($meta); 
     244    } 
     245 
     246    return $meta; 
     247 
     248} 
     249 
    2092501; 

  • trunk/lib/Brackup/Root.pm

    r64 r65  
    55use File::Find; 
    66use Brackup::DigestDatabase; 
     7use File::Temp qw(tempfile); 
    78 
    89sub new { 
    …… 
    165166 
    166167    $pop_dir->() while @dir_stack; 
    167  
    168  
     168} 
     169 
     170# given data (scalar or scalarref), returns encrypted data 
     171sub encrypt { 
     172    my ($self, $data) = @_; 
     173    my $gpg_rcpt = $self->gpg_rcpt 
     174        or Carp::confess("Encryption not setup for this root"); 
     175 
     176    $data = \$data unless ref $data; 
     177 
     178    # FIXME: let users control where their temp files go? 
     179    my ($tmpfh, $tmpfn) = tempfile(); 
     180    print $tmpfh $$data 
     181        or die "failed to print: $!"; 
     182    close $tmpfh 
     183        or die "failed to close: $!\n"; 
     184    Carp::confess("size not right") 
     185        unless -s $tmpfn == length $$data; 
     186 
     187    my ($etmpfh, $etmpfn) = tempfile(); 
     188 
     189    system($self->gpg_path, $self->gpg_args, 
     190           "--recipient", $gpg_rcpt, 
     191           "--trust-model=always", 
     192           "--batch", 
     193           "--encrypt", 
     194           "--output=$etmpfn", 
     195           "--yes", $tmpfn) 
     196        and die "Failed to run gpg: $!\n"; 
     197    open (my $enc_fh, $etmpfn) or die "Failed to open $etmpfn: $!\n"; 
     198    return do { local $/; <$enc_fh>; }; 
    169199} 
    170200 

  • trunk/lib/Brackup/Target/Filesystem.pm

    r41 r65  
    3030} 
    3131 
    32 sub chunkpath { 
    33     my ($self, $dig) = @_; 
     32sub _diskpath { 
     33    my ($self, $dig, $ext) = @_; 
    3434    my @parts; 
    3535    my $fulldig = $dig; 
    …… 
    3939        push @parts, $1; 
    4040    } 
    41     return $self->{path} . "/" . join("/", @parts) . "/$fulldig.chunk"; 
     41    return $self->{path} . "/" . join("/", @parts) . "/$fulldig.$ext"; 
     42} 
     43 
     44sub chunkpath { 
     45    my ($self, $dig) = @_; 
     46    return $self->_diskpath($dig, "chunk"); 
     47} 
     48 
     49sub metapath { 
     50    my ($self, $dig) = @_; 
     51    return $self->_diskpath($dig, "meta"); 
    4252} 
    4353 
    …… 
    8090    } 
    8191 
     92    if (my $mstr = $chunk->meta_contents) { 
     93        my $mpath = $self->metapath($dig); 
     94        open (my $fh, ">$mpath") or die "Failed to open $path for writing: $!\n"; 
     95        print $fh $mstr; 
     96        close($fh) or die "Failed to close $mpath\n"; 
     97    } 
     98 
    8299    return 1; 
    83100}