Changeset 1368

Timestamp:

11/25/09 07:43:08 (4 months ago)

Author:

dormando

Message:

Rebalance to repair overreplication errors.

If a fid is "too happy" (has too many copies), randomly rebalance
the devs until it's happy. This may momentarily make a fid unhappy
with its replication policy, but it will fix itself eventually.

Location:

trunk/server

Files:

• CHANGES (modified) (1 diff)
• lib/MogileFS/ReplicationPolicy/MultipleHosts.pm (modified) (2 diffs)
• lib/MogileFS/Worker/Replicate.pm (modified) (4 diffs)

trunk/server/CHANGES

@@ +1 @@
+
+        * Replication will now attempt to handle over replication situations.
+          FSCK will catch over replication as a policy violation, and
+          replicate will attempt to fix it. (dormando)
+          (rarely it might need multiple fscks)
 
         * Since FIDSizes is no longer used, remove it and remove gearman

trunk/server/lib/MogileFS/ReplicationPolicy/MultipleHosts.pm

@@ -50 +50 @@
     my $total_disks = scalar grep { $_->dstate->should_have_files } values %$all_devs;
 
-    # if we have two copies and that's all the disks there are
-    # anywhere, be happy enough, even if mindevcount is higher.  in
-    # that case, when they add more disks later, they'll need to fsck
-    # to make files replicate more.
-    return ALL_GOOD if $already_on >= 2 && $already_on == $total_disks;
-
     # see which and how many unique hosts we're already on.
     my %on_dev;
@@ -66 +60 @@
     my $total_uniq_hosts = unique_hosts($all_devs);
 
+    # if we are on two hosts but 10 devices, you want to weaken the number of
+    # devices you're on until you're on the right number of hosts with the
+    # right number of devices.
     return TOO_GOOD if $uniq_hosts_on >  $min;
-    return TOO_GOOD if $uniq_hosts_on == $min && $already_on > $min;
+    return TOO_GOOD if $already_on > $min;
     return ALL_GOOD if $uniq_hosts_on == $min;
     return ALL_GOOD if $uniq_hosts_on >= $total_uniq_hosts && $already_on >= $min;
+
+    # if we have two copies and that's all the disks there are
+    # anywhere, be happy enough, even if mindevcount is higher.  in
+    # that case, when they add more disks later, they'll need to fsck
+    # to make files replicate more.
+    # this is here instead of above in case an over replication error causes
+    # the file to be on all disks (where more than necessary)
+    return ALL_GOOD if $already_on >= 2 && $already_on == $total_disks;
 
     # if there are more hosts we're not on yet, we want to exclude devices we're already

trunk/server/lib/MogileFS/Worker/Replicate.pm

@@ -178 +178 @@
         # failed_getting_lock        => harmless.  skip.  somebody else probably doing.
         #
+        # -- ACTIONABLE --
+        # too_happy                  => too many copies, attempt to rebalance.
+        #
         # -- TEMPORARY; DO EXPONENTIAL BACKOFF --
         # source_down                => only source available is observed down.
@@ -220 +223 @@
         }
 
+        # try to shake off extra copies. fall through to the backoff logic
+        # so we don't flood if it's impossible to properly weaken the fid.
+        # there's a race where the fid could be checked again, but the
+        # exclusive locking prevents replication clobbering.
+        if ($errcode eq 'too_happy') {
+            $unlock->() if $unlock;
+            $unlock = undef;
+            my $f = MogileFS::FID->new($fid);
+            my @devs = List::Util::shuffle($f->devids);
+            my $devfid;
+            # First one we can delete from, we try to rebalance away from.
+            for (@devs) {
+                my $dev = MogileFS::Device->of_devid($_);
+                # Not positive 'can_read_from' needs to be here.
+                # We must be able to delete off of this dev so the fid can
+                # move.
+                if ($dev->can_delete_from && $dev->can_read_from) {
+                    $devfid = MogileFS::DevFID->new($dev, $f);
+                    last;
+                }
+            }
+            $self->rebalance_devfid($devfid) if $devfid;
+        }
+
         # at this point, the rest of the errors require exponential backoff.  define what this means
         # as far as failcount -> delay to next try.
@@ -418 +445 @@
     my $del_reason;
 
-    if ($ret eq "lost_race") {
+    if ($ret eq "too_happy" || $ret eq "lost_race") {
         # for some reason, we did no work. that could be because
         # either 1) we lost the race, as the error code implies,
@@ -693 +720 @@
     }
 
+    # We are over replicated. Let caller decide if it should rebalance.
+    if ($rr->too_happy) {
+        return $retunlock->(0, "too_happy", "fid $fidid is on too many devices");
+    }
+
     if ($rr->is_happy) {
         return $retunlock->(1) if $got_copy_request;