Changeset 856

Timestamp:

04/19/07 19:26:17 (3 years ago)

Author:

bradfitz

Message:

more fsck work. coming along well. went from 90 fid/second on my
local dev box, to 280 fids/s when using HTTPFile->at (which uses the
mogstored side-channel). much still to do, but basics work, so
polishing time. also more speed improvements possible later.

Location:

trunk/server/lib/MogileFS

Files:

• Store.pm (modified) (2 diffs)
• Worker/Fsck.pm (modified) (9 diffs)

trunk/server/lib/MogileFS/Store.pm

@@ -589 +589 @@
 utime  INT UNSIGNED NOT NULL,
 fid    INT UNSIGNED NULL,
-err    CHAR(4),
+evcode CHAR(4),
 devid  MEDIUMINT UNSIGNED,
 INDEX(utime)
@@ -1222 +1222 @@
 }
 
+sub fsck_log {
+    my ($self, %opts) = @_;
+    $self->dbh->do("INSERT INTO fsck_log (utime, fid, evcode, devid) ".
+                   "VALUES (" . $self->unix_timestamp . ",?,?,?)",
+                   undef,
+                   delete $opts{fid},
+                   delete $opts{code},
+                   delete $opts{devid});
+    croak("Unknown opts") if %opts;
+    return 1;
+}
+
+
 # run before daemonizing.  you can die from here if you see something's amiss.  or emit
 # warnings.

trunk/server/lib/MogileFS/Worker/Fsck.pm

@@ -22 +22 @@
     my $run_count = 0;
 
+    # <debug crap>
+    my $running = 0; # start time
+    my $n_check = 0; # items checked
+    my $start = sub {
+        return if $running;
+        $running = time();
+        print "START @" . time() . "\n";
+    };
+    my $stats = sub {
+        return unless $running;
+        my $now = time();
+        my $elap = $now - $running;
+        printf("In %d secs, %d fids, %0.02f fids/sec\n", $elap, $n_check, ($n_check / ($elap || 1)));
+    };
+    my $last_beat = 0;
+    my $beat = sub {
+        my $now = time();
+        return unless $now >= $last_beat + 5;
+        $stats->();
+        $last_beat = $now;
+    };
+    my $stop = sub {
+        return unless $running;
+        $stats->();
+        print "DONE.\n";
+        $running = 0;
+    };
+    # </debug crap>
+
     every(5.0, sub {
         my $sleep_set = shift;
@@ -45 +74 @@
 
         unless (@fids) {
-            warn "[fsck] no fids to check...\n";
+            $stop->();
             return;
         }
+        $start->();
 
         MogileFS::FID->mass_load_devids(@fids);
@@ -53 +83 @@
         my $new_max;
         foreach my $fid (@fids) {
+            $n_check++;
             $self->still_alive;
             last if $self->should_stop_running;
             last unless $self->check_fid($fid, no_stat => $opt_nostat);
             $new_max = $fid->id;
+            $beat->();
         }
 
@@ -87 +119 @@
 #   if no problems, no action.
 #   if problems, log & enqueue fixes
+use constant CANT_CHECK => 0;
 sub check_fid {
     my ($self, $fid, %opts) = @_;
@@ -93 +126 @@
 
     # first obvious fucked-up case:  no devids even presumed to exist.
-    unless (@{ $fid->devids }) {
+    unless ($fid->devids) {
         # first, log this weird condition. (N = NO PAths)
-        Mgd::get_store->fsck_log(code => "NOPA", fid => $fid->id);
+        Mgd::get_store()->fsck_log(code => "NOPA", fid => $fid->id);
         # weird, schedule a fix (which will do a search over all
         # devices as a last-ditch effort to locate it)
@@ -106 +139 @@
     unless ($fid->devids_meet_policy) {
         # log a policy violation
-        Mgd::get_store->fsck_log(code => "POVI", fid => $fid->id);
+        Mgd::get_store()->fsck_log(code => "POVI", fid => $fid->id);
         $self->fix_fid($fid);
         return 1;
@@ -115 +148 @@
     # check the replication policy, which is already done, so finish.
     return 1 if $opt_no_stat;
-
-    # **********************************************************************
-    # FIXME: temporary, until statting is done...
-    # **********************************************************************
-
-    # printf("fid %d is good.\n", $fid->id);
-    return 1;
 
     # iterate and do HEAD requests to determine some basic information about the file
@@ -132 +158 @@
         my $path = $dfid->get_url
             or die "FIXME";
-        # TODO: use side-channel?  eh, why?  LWP + ConnCache good enough for now.
-        my $ua = LWP::UserAgent->new(timeout => 3)
-            or die "FIXME";
-        my $resp = $ua->head($path);
-
-        # at this point we're going to assume that any error is based on the device alone
-        # so we want to store the status and not return
-        if ($resp->is_success) {
-            # great, check the size against what's in the database
-            if ($resp->header('Content-Length') == $fid->length) {
-                #yay
-            } else {
-                #shit.
-            }
-
+
+        my $disk_size = $dfid->size_on_disk;
+
+        if (! defined $disk_size) {
+            warn("Connectivity problem reaching XXXFIXME\n");
+            return CANT_CHECK;
+        }
+
+        # great, check the size against what's in the database
+        if ($disk_size == $fid->length) {
+            #yay
         } else {
-            # easy one, the request failed for some reason, 500 would tend to imply that the
-            # mogstored is having issues so we should try again later, whereas a 404 is a
-            # total and permanent failure
-            #error("check_fid($fid, $level): " . $resp->code . " on device $devid");
-
-            if ($resp->code == 404) {
-                #fucked!
-            } else {
-                # just unreachable
-                warn "TODO: foo is unreachable\n";
-                return 0;
-            }
+            printf("FID %d corruption on devid $devid: e=%d, a=%d\n",
+                   $fid->id,
+                   $fid->length,
+                   $disk_size,
+                   );
         }
     }
@@ -182 +197 @@
 
     # make devfid objects from the devids that this fid is on,
-    my @dfids = map { MogileFS::DevID->new($_, $fid) } @{ $fid->devids };
+    my @dfids = map { MogileFS::DevID->new($_, $fid) } $fid->devids;
 
     # keep track if we found the file (with the right size) at least somewhere.