Changeset 161

Show
Ignore:
Timestamp:
08/02/07 20:54:41 (2 years ago)
Author:
bradfitz
Message:

'prune' and 'gc' commands commands for both Amazon
and Filesystem targets. from Alessandro Ranellucci <aar@…>.

Location:
trunk
Files:
2 added
10 modified

Legend:

Unmodified
Added
Removed

  • trunk/Changes

    r159 r161  
     1  - 'prune' and 'gc' commands commands for both Amazon 
     2     and Filesystem targets.  from Alessandro Ranellucci <aar@cpan.org>. 
     3 
    141.04 (july 30, 2007) 
    25 

  • trunk/brackup-target

    r158 r161  
    1111 $ brackup-target [opts] <target_name> get_backups 
    1212 $ brackup-target [opts] <target_name> delete_backup <backup_file> 
    13  $ brackup-target [opts] <target_name> gc   # run garbage collector 
     13 $ brackup-target [opts] <target_name> prune   # remove old backups 
     14 $ brackup-target [opts] <target_name> gc      # run garbage collector 
    1415 
    1516=head2 OPTIONS 
    … …  
    2425 
    2526Be verbose with status. 
     27 
     28=item --dry-run 
     29 
     30Do not actually execute write operations. 
     31 
     32=item --keep-backups 
     33 
     34To be used in combination with the I<prune> command. This overrides the  
     35I<keep_backups> option specified in the configuration file. 
    2636 
    2737=back 
    … …  
    6171my $opt_help; 
    6272my $opt_verbose; 
     73my $opt_keep_backups; 
     74my $opt_dryrun; 
    6375usage() unless 
    6476    GetOptions( 
    … …  
    6678               'dest=s'   => \$destdir, 
    6779               'config=s' => \$config_file, 
     80               'keep-backups=i' => \$opt_keep_backups, 
     81               'dry-run'   => \$opt_dryrun, 
    6882               'help'     => \$opt_help, 
    6983               ); 
    … …  
    132146} 
    133147 
     148sub CMD_prune { 
     149    my $removed_count = $target->prune( keep_backups => $opt_keep_backups, 
     150                                        dryrun => $opt_dryrun); 
     151    debug("$removed_count backups removed from target"); 
     152} 
     153 
     154sub CMD_gc { 
     155    my $removed_chunks = $target->gc(dryrun => $opt_dryrun); 
     156    debug("$removed_chunks chunks removed from target"); 
     157} 
     158 
    134159sub debug { 
    135160    my $msg = shift; 

  • trunk/lib/Brackup.pm

    r159 r161  
    77use Brackup::ConfigSection; 
    88use Brackup::File; 
     9use Brackup::Metafile; 
    910use Brackup::PositionedChunk; 
    1011use Brackup::StoredChunk; 

  • trunk/lib/Brackup/Config.pm

    r141 r161  
    8484#type = Filesystem 
    8585#path = /raid/backup/brackup 
     86#keep_backups = 10 
    8687 
    8788#[TARGET:amazon] 
    … …  
    8990#aws_access_key_id  = XXXXXXXXXX 
    9091#aws_secret_access_key =  XXXXXXXXXXXX 
     92#keep_backups = 10 
    9193 
    9294#[SOURCE:proj] 

  • trunk/lib/Brackup/Manual/Overview.pod

    r153 r161  
    127127   brackup-restore --help 
    128128 
     129=head1 Number of backups to keep 
     130 
     131To free space on your target you can remove old backups. There are two steps 
     132to do this: 
     133 
     134   brackup-target <target> prune 
     135   brackup-target <target> gc 
     136 
     137The first command will look for backup metafiles in your target and remove the 
     138oldest ones according to the I<keep_backups> option you specified in the config 
     139file. Thus, if you have, say, 15 backups stored and I<keep_backups> is set to 10 
     140then I<prune> will remove the oldest 5 backups. 
     141 
     142The second command will remove from your target the orphaned chunks that are no  
     143more referenced by any metafile. This will free some space while preserving chunks 
     144that are still referenced by recent backups. 
     145 
    129146=head1 SEE ALSO 
    130147 

  • trunk/lib/Brackup/Restore.pm

    r153 r161  
    5454    my ($self) = @_; 
    5555    my $parser = $self->parser; 
    56     my $meta = $parser->(); 
     56    my $meta = $parser->readline; 
    5757    my $driver_class = $meta->{BackupDriver}; 
    5858    die "No driver specified" unless $driver_class; 
    … …  
    6969    $self->{_meta}   = $meta; 
    7070 
    71     while (my $it = $parser->()) { 
     71    while (my $it = $parser->readline) { 
    7272        my $full = $self->{to} . "/" . $it->{Path}; 
    7373        my $type = $it->{Type} || "f"; 
    … …  
    263263sub parser { 
    264264    my $self = shift; 
    265     open (my $fh, $self->{file}) or die "Failed to open metafile: $!"; 
    266     my $linenum = 0; 
    267     return sub { 
    268         my $ret = {}; 
    269         my $last;  # scalarref to last item 
    270         my $line;  # 
    271         while (defined ($line = <$fh>)) { 
    272             $linenum++; 
    273             if ($line =~ /^([\w\-]+):\s*(.+)/) { 
    274                 $ret->{$1} = $2; 
    275                 $last = \$ret->{$1}; 
    276                 next; 
    277             } 
    278             if ($line eq "\n") { 
    279                 return $ret; 
    280             } 
    281             if ($line =~ /^\s+(.+)/) { 
    282                 die "Can't continue line without start" unless $last; 
    283                 $$last .= " $1"; 
    284                 next; 
    285             } 
    286  
    287             $line =~ s/[:^print:]/?/g; 
    288             die "Unexpected line in metafile $self->{file}, line $linenum: $line"; 
    289         } 
    290         return undef; 
    291     } 
     265    return Brackup::Metafile->open($self->{file}); 
    292266} 
    293267 

  • trunk/lib/Brackup/Target.pm

    r148 r161  
    55use Brackup::InventoryDatabase; 
    66use Brackup::TargetBackupStatInfo; 
     7use Brackup::Util 'tempfile'; 
    78use Carp qw(croak); 
    89 
    … …  
    1213    my $name = $confsec->name; 
    1314    $name =~ s!^TARGET:!! or die; 
    14  
     15     
     16    $self->{keep_backups} = $confsec->value("keep_backups"); 
    1517    $self->{inv_db} = 
    1618        Brackup::InventoryDatabase->new($confsec->value("inventory_db") || 
    1719                                        "$ENV{HOME}/.brackup-target-$name.invdb"); 
     20         
    1821    return $self; 
    1922} 
    … …  
    3639    my ($self, $chunk) = @_; 
    3740    die "ERROR: store_chunk not implemented in sub-class $self"; 
     41} 
     42 
     43# returns true on success, or returns false or dies otherwise. 
     44sub delete_chunk { 
     45    my ($self, $chunk) = @_; 
     46    die "ERROR: delete_chunk not implemented in sub-class $self"; 
     47} 
     48 
     49# returns a list of names of all chunks 
     50sub chunks { 
     51    my ($self) = @_; 
     52    die "ERROR: chunks not implemented in sub-class $self"; 
    3853} 
    3954 
    … …  
    7590} 
    7691 
     92# deletes the given backup from this target 
     93sub delete_backup { 
     94    my ($self, $name) = @_; 
     95    die "ERROR: delete_backup method not implemented in sub-class $self"; 
     96} 
     97 
     98# removes old metafiles from this target 
     99sub prune { 
     100    my ($self, %opt) = @_; 
     101     
     102    my $keep_backups = $self->{keep_backups} || $opt{keep_backups} 
     103        or die "ERROR: keep_backups option not set\n"; 
     104    die "ERROR: keep_backups option must be at least 1\n" 
     105        unless $keep_backups > 0; 
     106     
     107    # select backups to delete 
     108    my (%backups, @backups_to_delete) = (); 
     109    foreach my $backup_name (map {$_->filename} $self->backups) { 
     110        $backup_name =~ /^(.+)-\d+$/; 
     111        $backups{$1} ||= []; 
     112        push @{ $backups{$1} }, $backup_name; 
     113    } 
     114    foreach my $source (keys %backups) { 
     115        my @b = reverse sort @{ $backups{$source} }; 
     116        push @backups_to_delete, splice(@b, ($keep_backups > $#b+1) ? $#b+1 : $keep_backups); 
     117    } 
     118     
     119    unless ($opt{dryrun}) { 
     120         $self->delete_backup($_) for @backups_to_delete; 
     121    } 
     122    return scalar @backups_to_delete; 
     123} 
     124 
     125# removes orphaned chunks in the target 
     126sub gc { 
     127    my ($self, %opt) = @_; 
     128     
     129    # get all chunks and then loop through metafiles to detect 
     130    # referenced ones 
     131    my %chunks = map {$_ => 1} $self->chunks; 
     132    my $tempfile = tempfile(); 
     133    BACKUP: foreach my $backup ($self->backups) { 
     134        $self->get_backup($backup->filename, $tempfile); 
     135        my $parser = Brackup::Metafile->open($tempfile); 
     136        $parser->readline;  # skip header 
     137        ITEM: while (my $it = $parser->readline) { 
     138            next ITEM unless $it->{Chunks}; 
     139            my @item_chunks = map { (split /;/)[3] } grep { $_ } split(/\s+/, $it->{Chunks} || ""); 
     140            delete $chunks{$_} for (@item_chunks); 
     141        } 
     142    } 
     143    my @orphaned_chunks = keys %chunks; 
     144     
     145    # remove orphaned chunks 
     146    unless ($opt{dryrun}) { 
     147         $self->delete_chunk($_) for @orphaned_chunks; 
     148    } 
     149    return scalar @orphaned_chunks; 
     150} 
     151 
     152 
     153 
    771541; 
    78155 
    … …  
    107184B<Filesystem> -- see L<Brackup::Target::Filesystem> for configuration details 
    108185 
     186=item B<keep_backups> 
     187 
     188The number of recent backups to keep when running I<brackup-target prune>. 
     189 
    109190=back 

  • trunk/lib/Brackup/Target/Amazon.pm

    r158 r161  
    130130} 
    131131 
     132sub delete_chunk { 
     133    my ($self, $dig) = @_; 
     134    my $bucket = $self->{s3}->bucket($self->{chunk_bucket}); 
     135    return $bucket->delete_key($dig); 
     136} 
     137 
     138# returns a list of names of all chunks 
     139sub chunks { 
     140    my $self = shift; 
     141     
     142    my $chunks = $self->{s3}->list_bucket_all({ bucket => $self->{chunk_bucket} }); 
     143    return map { $_->{key} } @{ $chunks->{keys} }; 
     144} 
     145 
    132146sub store_backup_meta { 
    133147    my ($self, $name, $file) = @_; 
    … …  
    158172sub get_backup { 
    159173    my $self = shift; 
    160     my $name = shift; 
     174    my ($name, $output_file) = @_; 
    161175         
    162176    my $bucket = $self->{s3}->bucket($self->{backup_bucket}); 
    … …  
    164178        or return 0; 
    165179         
    166     open(my $out, ">$name.brackup") or die "Failed to open $name.brackup: $!\n"; 
     180        $output_file ||= "$name.brackup"; 
     181    open(my $out, ">$output_file") or die "Failed to open $output_file: $!\n"; 
    167182    my $outv = syswrite($out, $val->{value}); 
    168183    die "download/write error" unless $outv == do { use bytes; length $val->{value} }; 

  • trunk/lib/Brackup/Target/Filesystem.pm

    r158 r161  
    33use warnings; 
    44use base 'Brackup::Target'; 
     5use File::Basename; 
     6use File::Find (); 
    57use File::Path; 
    68use File::stat (); 
    … …  
    9698} 
    9799 
     100sub delete_chunk { 
     101    my ($self, $dig) = @_; 
     102    my $path = $self->chunkpath($dig); 
     103    unlink $path; 
     104} 
     105 
     106 
     107# returns a list of names of all chunks 
     108sub chunks { 
     109    my $self = shift; 
     110     
     111    my @chunks = (); 
     112    my $found_chunk = sub { 
     113        m/\.chunk$/ or return; 
     114        my $chunk_name = basename($_); 
     115        $chunk_name =~ s/\.chunk$//; 
     116        push @chunks, $chunk_name; 
     117    }; 
     118    File::Find::find({ wanted => $found_chunk, no_chdir => 1}, $self->{path}); 
     119    return @chunks; 
     120} 
     121 
    98122sub _metafile_dir { 
    99123    return $_[0]->{path}."/backups/"; 
    … …  
    134158 
    135159# downloads the given backup name to the current directory (with 
    136 # *.brackup extension) 
     160# *.brackup extension) or to the specified location 
    137161sub get_backup { 
    138     my ($self, $name) = @_; 
     162    my ($self, $name, $output_file) = @_; 
    139163    my $dir  = $self->_metafile_dir; 
    140164    my $file = "$dir/$name.brackup"; 
    141165    die "File doesn't exist: $file" unless -e $file; 
    142166    open(my $in,  $file)            or die "Failed to open $file: $!\n"; 
    143     open(my $out, ">$name.brackup") or die "Failed to open $name.brackup: $!\n"; 
     167        $output_file ||= "$name.brackup"; 
     168    open(my $out, ">$output_file") or die "Failed to open $output_file: $!\n"; 
    144169    my $buf; 
    145170    my $rv; 

  • trunk/lib/Brackup/Test.pm

    r157 r161  
    7878    ok(-s $meta_filename, "backup file has size"); 
    7979 
    80     return wantarray ? ($meta_filename, $backup) : $meta_filename; 
     80    return wantarray ? ($meta_filename, $backup, $target) : $meta_filename; 
    8181} 
    8282