Changeset 360

Timestamp:

08/22/06 03:47:51 (2 years ago)

Author:

marksmith

Message:

* more work on the checker job

Files:

• branches/server-newrepl/lib/MogileFS/ProcManager.pm (modified) (3 diffs)
• branches/server-newrepl/lib/MogileFS/Worker/Checker.pm (modified) (7 diffs)

branches/server-newrepl/lib/MogileFS/ProcManager.pm

@@ -636 +636 @@
-    #warn "STATE CHANGE: $what<$whatid> = $state\n";
-    # TODO: can probably send this to all children now, not just certain types
-
+ checker
-        MogileFS::ProcManager->ImmediateSendToChildrenByJob($type, ":state_change $what $whatid $state", $child);
-    }
@@ -644 +644 @@
-    my ($what, $child) = @_;
-    # TODO: can probably send this to all children now, not just certain types
-
+ checker
-        MogileFS::ProcManager->ImmediateSendToChildrenByJob($type, ":invalidate_meta_once $what", $child);
-    }
@@ -651 +651 @@
-sub send_monitor_has_run {
-    my $child = shift;
-
+ checker
-        MogileFS::ProcManager->ImmediateSendToChildrenByJob($type, ":monitor_has_run", $child);
-    }

branches/server-newrepl/lib/MogileFS/Worker/Checker.pm

@@ -4 +4 @@
-use strict;
-use base 'MogileFS::Worker';
-every error
+ every error 
-use List::Util ();
+use LWP::UserAgent;
-use POSIX;
+use Time::HiRes qw( gettimeofday tv_interval );
+
+use constant SUCCESS => 0;
+use constant TEMPORARY => 1;
+use constant PERMANENT => 2;
+use constant REPLICATE => 3;
-
-sub new {
@@ -25 +32 @@
-    my $psock = $self->{psock};
-
+    my %retvals;
+    my ($run_count, $total_time, $total_done) = (0, 0, 0);
+
-    every(2, sub {
-        $self->parent_ping;
@@ -31 +41 @@
-        my $setting = Mgd::get_server_setting('fsck_enable');
-        return unless defined $setting && $setting ne 'off' && is_valid_level($setting);
+
+        # checking doesn't go well if the monitor job hasn't actively started
+        # marking things as being available
+        unless ($self->monitor_has_run) {
+            # only warn on runs after the first.  gives the monitor job some time to work
+            # before we throw a message.
+            if ($run_count++ > 0) {
+                error("waiting for monitor job to complete a cycle before beginning checking");
+            }
+            return;
+        }
-
-        # get dbh and create table if not exist
@@ -47 +68 @@
-            # select a group of fids to work on from the fsck table
-            my $fids = $dbh->selectcol_arrayref("SELECT fid FROM fsck WHERE nextcheck < UNIX_TIMESTAMP() " .
-
+0
-            if ($dbh->err) {
-                error("database error: " . $dbh->errstr);
@@ -53 +74 @@
-            }
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        }
-
@@ -65 +123 @@
-        # into the database to work on next
-        my $highest_fid = Mgd::get_server_setting('fsck_highest_fid_checked') || 0;
-
+ WHERE nextcheck < UNIX_TIMESTAMP()
-        my $limit = 10_000 - $total_already;  # but only up to $limit items
-
-        # now extract some files and re-insert them, but only if we need more
+        print "limit=$limit, highest=$highest_fid, total=$total_already\n";
-        if ($limit > 0) {
-            $dbh->do(qq{
@@ -88 +147 @@
-}
-
+# this sub actually does the checking of a fid.  we put it in its own sub so we can
+# return from it using the unlock coderef.  always returns a number:
+#
+#   0 - file is just fine, drop it from the list
+#   1 - temporary failure, check again later
+#   2 - permanent failure, this file shouldn't get tried again
+#   3 - needs replication, we found something not quite right
+#
+sub check_fid {
+    my ($dbh, $fid, $level) = @_;
+
+    # unlocker sub to be used 
+    my $lockname = "mgfs:fid:$fid:check";
+    my $retunlock = sub {
+        my $rv = shift()+0;
+
+        # 0 means success, else some sort of failure
+        if ($rv) {
+            my $msg = shift() || "no error text";
+            my $rvtype = {
+                1 => 'temporary failure',
+                2 => 'permanent failure',
+                3 => 'needs replication',
+            }->{$rv} || 'unknown error';
+            error("check_fid($fid, $level) = $rvtype: $msg");
+        }
+
+        $dbh->do("SELECT RELEASE_LOCK(?)", undef, $lockname);
+        return $rv;
+    };
+
+    # try to get the lock
+    my $lock = $dbh->selectrow_array("SELECT GET_LOCK(?, 1)", undef, $lockname);
+    return $retunlock->(TEMPORARY, "failed getting lock $lockname") unless $lock;
+
+    # all checks require us to get the file paths
+    my $devids = $dbh->selectcol_arrayref('SELECT devid FROM file_on WHERE fid = ?', undef, $fid);
+    return $retunlock->(PERMANENT, 'no sources found') unless $devids && @$devids;
+
+    # if it's a simple location check, we're done
+    return $retunlock->(SUCCESS) if $level eq 'locations';
+
+    # get the file size from the database, we're going to need it.  note that this could be
+    # a 0 size, so we have to watch for defined.
+    my $db_size = $dbh->selectrow_array('SELECT length FROM file WHERE fid = ?', undef, $fid);
+    return $retunlock->(TEMPORARY, "database does not contain file size") unless defined $db_size;
+
+    # iterate and do HEAD requests to determine some basic information about the file
+    my %devs;
+    foreach my $devid (@$devids) {
+        # setup and do the request.  these failures are total failures in that we expect
+        # them to work again later, as it's probably transient and will persist no matter
+        # how many paths we try.
+        my $path = Mgd::make_http_path($devid, $fid)
+            or return $retunlock->(TEMPORARY, 'failure to create HTTP path to file');
+        my $ua = LWP::UserAgent->new(timeout => 3)
+            or return $retunlock->(TEMPORARY, 'failed to create LWP::UserAgent object');
+        my $resp = $ua->head($path);
+
+        # at this point we're going to assume that any error is based on the device alone
+        # so we want to store the status and not return
+        if ($resp->is_success) {
+            # great, check the size against what's in the database
+            if ($resp->header('Content-Length') == $db_size) {
+                $devs{$devid} = SUCCESS;
+            } else {
+                $devs{$devid} = PERMANENT;
+            }
+
+        } else {
+            # easy one, the request failed for some reason, 500 would tend to imply that the
+            # mogstored is having issues so we should try again later, whereas a 404 is a
+            # total and permanent failure
+            error("check_fid($fid, $level): " . $resp->code . " on device $devid");
+            if ($resp->code == 404) {
+                $devs{$devid} = PERMANENT;
+            } else {
+                $devs{$devid} = TEMPORARY;
+            }
+        }
+    }
+
+    # at this point, we need to take actions.  if we discovered some PERMANENT failures in
+    # a device scan, then we need to take care of those now by removing them.  but DO NOT
+    # remove them if that would leave us with no mappings!  ONLY if there is at least one
+    # SUCCESS mapping.
+    # FIXME: implement
+
+    # if they wanted a quick scan, let's stop here and throw a result based on the contents
+    # of the %devs hash.  basically, if any of the devices had issues, then at this point we
+    # want to throw a flag saying "please replicate this".  if not, then we tell them that
+    # we're successful on this fid.
+    if ($level eq 'quick') {
+        foreach my $code (values %devs) {
+            return $retunlock->(REPLICATE, "permanent failure on one or more devices")
+                if $code != SUCCESS;
+        }
+        return $retunlock->(SUCCESS);
+    }
+
+    # full mode not here yet
+    return $retunlock->(TEMPORARY, "sorry, $level mode is not implemented yet");
+}
+
-1;
-