Changeset 90

Show
Ignore:
Timestamp:
05/11/07 08:45:49 (2 years ago)
Author:
bradfitz
Message:
  • clean up old, dead code in Amazon target (the old inventory db which
    is now an official part of the core, and in the Target base class)
  • retry PUTs to Amazon on failure, once, in case it was a transient error,
    as seems to happen occasionally
Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved

  • trunk/Changes

    r88 r90  
     1  - clean up old, dead code in Amazon target (the old inventory db which 
     2    is now an official part of the core, and in the Target base class) 
     3 
     4  - retry PUTs to Amazon on failure, once, in case it was a transient error, 
     5    as seems to happen occasionally 
     6 
    17  - halve number of stats when walking backup root 
    28 

  • trunk/lib/Brackup/Target/Amazon.pm

    r84 r90  
    88# fields in object: 
    99#   s3  -- Net::Amazon::S3 
    10 #   dbh -- sqlite dbi exist cache 
    1110#   access_key_id 
    1211#   sec_access_key_id 
    …… 
    8180    my $dig = $chunk->backup_digest;   # "sha1:sdfsdf" format scalar 
    8281 
    83     if (my $dbh = $self->{dbh}) { 
    84         my $ans = $dbh->selectrow_array("SELECT COUNT(*) FROM amazon_key_exists WHERE key=?", undef, $dig); 
    85         return 1 if $ans; 
    86     } 
    87  
    8882    my $res = eval { $self->{s3}->head_key({ bucket => $self->{chunk_bucket}, key => $dig }); }; 
    8983    return 0 unless $res; 
    9084    return 0 if $@ && $@ =~ /key not found/; 
    9185    return 0 unless $res->{content_type} eq "x-danga/brackup-chunk"; 
    92     $self->_cache_existence_of($dig); 
    9386    return 1; 
    9487} 
    …… 
    108101    my $blen = $chunk->backup_length; 
    109102    my $len = $chunk->length; 
     103    my $chunkref = $chunk->chunkref; 
    110104 
    111     my $rv = eval { $self->{s3}->add_key({ 
    112         bucket        => $self->{chunk_bucket}, 
    113         key           => $dig, 
    114         value         => ${ $chunk->chunkref }, 
    115         content_type  => 'x-danga/brackup-chunk', 
    116     }) }; 
    117     return 0 unless $rv; 
    118     $self->_cache_existence_of($dig); 
     105    my $try = sub { 
     106        eval { 
     107            $self->{s3}->add_key({ 
     108                bucket        => $self->{chunk_bucket}, 
     109                key           => $dig, 
     110                value         => $$chunkref, 
     111                content_type  => 'x-danga/brackup-chunk', 
     112            }); 
     113        }; 
     114    }; 
     115 
     116    my $rv = $try->(); 
     117    unless ($rv) { 
     118        # transient failure? 
     119        warn "Error uploading chunk $chunk... retrying once...\n"; 
     120        $rv = $try->(); 
     121    } 
     122    unless ($rv) { 
     123        warn "Error uploading chunk again: " . $self->{s3}->errstr . "\n"; 
     124        return 0; 
     125    } 
    119126    return 1; 
    120 } 
    121  
    122 sub _cache_existence_of { 
    123     my ($self, $dig) = @_; 
    124     if (my $dbh = $self->{dbh}) { 
    125         $dbh->do("INSERT INTO amazon_key_exists VALUES (?,1)", undef, $dig); 
    126     } 
    127127} 
    128128