Changeset 89

Timestamp:

08/15/06 18:22:35 (2 years ago)

Author:

bchoate

Message:

Added synchronization configuration option (support for synchronization can be enabled/disabled). Added support for syncing uploaded files. Prevent rebuild queue from queueing when app mode is an explicit rebuild* mode.

Files:

• trunk/RebuildQueue/README.txt (modified) (1 diff)
• trunk/RebuildQueue/plugins/RebuildQueue/RebuildQueue.pl (modified) (4 diffs)
• trunk/RebuildQueue/plugins/RebuildQueue/lib/RebuildQueue/Daemon.pm (modified) (9 diffs)
• trunk/RebuildQueue/plugins/RebuildQueue/lib/RebuildQueue/Plugin.pm (modified) (4 diffs)

trunk/RebuildQueue/README.txt

@@ -17 +17 @@
-Requirements
-============
+
+Movable Type version 3.3 or later (or Movable Type Enterprise).
-
-The 'rsync' command if using remote server synchronization.

trunk/RebuildQueue/plugins/RebuildQueue/RebuildQueue.pl

@@ -27 +27 @@
-    my $help      = 0;
-    my %throttle;
-
+
+
-    my $sync      = 0;
-    my $rsync_opt = '';
@@ -37 +38 @@
-        "help|?"      => \$help,
-        "throttle=i"  => \%throttle,
-i
+s
-        "sync"        => \$sync,
-        "to|target=s" => \@target,
@@ -63 +64 @@
-        }
-    }
+    if ($worker =~ m/,/) {
+        @worker = split(/,/, $worker);
+    } else {
+        @worker = ($worker) if $worker =~ m/^\d+$/;
+    }
-
-    require RebuildQueue::Daemon;
-    if ($sync) {
-        RebuildQueue::Daemon->new->sync(
-
+
+
+
-            target    => \@target,
-            rsync_opt => $rsync_opt,
@@ -76 +84 @@
-            'sleep'   => $sleep,
-            throttle  => \%throttle,
-$
+\@
-        );
-    }

trunk/RebuildQueue/plugins/RebuildQueue/lib/RebuildQueue/Daemon.pm

@@ -39 +39 @@
-    my (%opt) = @_;
-
-
+
+
+
-    my $targets   = $opt{target}    || [];
-    my $rsync_opt = $opt{rsync_opt} || '-a';
-
-
+
+
+
+
+
+
+
+
+
-        exit 1;
-    }
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-}
-
@@ -96 +131 @@
-    my (%opt) = @_;
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
-        # an existing daemon is running with this worker id. don't
-        # allow a second to run.
@@ -107 +144 @@
-    }
-
+    # We need the plugin for getting settings, but not for anything
+    # else...
+    require RebuildQueue::Plugin;
+    my $plugin = RebuildQueue::Plugin->instance;
+    $plugin->disable;
+
+    my $sync_support = $plugin->sync_support;
+
-    my $stop = 0;
-
+ 
-    local $SIG{QUIT} = sub { $stop = 1 };
-
@@ -115 +160 @@
-    $mt->cleanup_rebuild_queue;
-
-
+build 
-
-    my $pub = $mt->publisher;
@@ -121 +166 @@
-    while (!$stop) {
-        my $iter = RebuildQueue::File->load_iter({
-worker => $worker
+(@$worker ? ( worker => $worker ) : ())
-            build_time => [undef, time],
-        }, {
@@ -168 +213 @@
-                if ($mtime != $mtime2) {
-                    # file was updated; mark for syncing
-
-
-
+
+
+
+
+
+
+
+
-                } else {
-                    # touch file to help throttle mechanism
@@ -195 +245 @@
-}
-
+# Method to remove any RebuildQueue::File objects that
+# are no longer requiring rebuild or sync operations.
-sub cleanup_rebuild_queue {
-    my $mt = shift;
@@ -204 +256 @@
-}
-
+# Summarizes the current queue item for display in the log.
-sub _summary {
-    my $fi = shift;
@@ -214 +267 @@
-    $file =~ s/^\Q$root\E//;
-    $file =~ s/^\///;
+    # TBD: Handle this situation more gracefully.
-    die "failed to load template " . $fi->template_id unless $tmpl;
-# Output summary

trunk/RebuildQueue/plugins/RebuildQueue/lib/RebuildQueue/Plugin.pm

@@ -15 +15 @@
-use base 'MT::Plugin';
-
-
+1
-our $SCHEMA_VERSION = '1.01';
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-if (!MT->instance->isa('RebuildQueue::Daemon')) {
-    $plugin = new RebuildQueue::Plugin({
-        name           => "Rebuild Queue",
-        description    => "Queues rebuild operations for offline building.",
-        object_classes => ['RebuildQueue::File'],
-        version        => $VERSION,
-        schema_version => $SCHEMA_VERSION,
-        author_name    => "Six Apart, Ltd.",
-        author_link    => "http://www.sixapart.com/",
-        plugin_link    => "http://code.sixapart.com/",
-        icon           => "rebuildq.gif",
-        system_config_template => \&system_config_template,
-        blog_config_template   => \&blog_config_template,
-        settings       => new MT::PluginSettings([
-            ['workers',       { Default => 1,     Scope => 'system' }],
-            ['rebuildq_mode', { Default => 0,     Scope => 'blog'   }],
-            ['worker',        { Default => undef, Scope => 'blog'   }],
-        ]),
-        callbacks      => {
-            'BuildFileFilter' => {
-                priority => 10,
-                code     => \&build_file_filter,
-            },
-        }
-    });
-    MT->add_plugin($plugin);
-}
-
+sub instance {
+    $plugin;
+}
+
+sub enable {
+    $ENABLED = 1;
+}
+
+sub disable {
+    $ENABLED = 0;
+}
+
+sub enabled {
+    $ENABLED;
+}
+
+sub init_request {
+    my $plugin = shift;
+    my ($app) = @_;
+
+    $plugin->enable;
+    if ($app->isa('MT::App::CMS')) {
+        my $mode = $app->mode;
+        $plugin->disable if $mode =~ m/^rebuild/;
+    }
+
+    $plugin->SUPER::init_request(@_);
+}
+
+# The RebuildQueue can synchronize built pages, but in order to handle
+# files uploaded through the interface, we need to manage our own
+# FileInfo records. Upon synchronization of these, they can be removed.
+sub cms_upload_file {
+    my ($cb, %args) = @_;
+
+    my $url = $args{Url};
+    my $file = $args{File};
+    return unless -f $file;
+
+    return unless $plugin->sync_support;
+
+    my $blog = $args{Blog};
+    my $blog_id = $blog->id;
+    return unless $plugin->blog_enabled($blog_id);
+
+    require MT::FileInfo;
+    my $base_url = $url;
+    $base_url =~ s!^https?://[^/]+!!;
+    my $fi = MT::FileInfo->load({ blog_id => $blog_id, url => $base_url });
+    if (!$fi) {
+        $fi = new MT::FileInfo;
+        $fi->blog_id($blog_id);
+        $fi->url($base_url);
+        $fi->file_path($file);
+    } else {
+        $fi->file_path($file);
+    }
+    $fi->save;
+
+    require RebuildQueue::File;
+    my $rqf = RebuildQueue::File->load($fi->id);
+    if (!$rqf) {
+        $rqf = new RebuildQueue::File;
+        $rqf->id($fi->id);
+    }
+    $rqf->worker($plugin->blog_worker($blog_id));
+    $rqf->sync_me(1);
+    $rqf->save;
+}
+
+sub fileinfo_post_remove {
+    my ($cb, $fi) = @_;
+    require RebuildQueue::File;
+    if (my $rqf = RebuildQueue::File->load($fi->id)) {
+        $rqf->remove;
+    }
+}
+
+sub fileinfo_post_remove_all {
+    my ($cb, $fi) = @_;
+    require RebuildQueue::File;
+    RebuildQueue::File->remove_all;
+}
+
-sub system_config_template {
+    my $workers = $plugin->get_config_value('workers', 'system');
+
+    my $max = 10;
+    $max = $workers + 10 if $workers >= $max;
+
+    my $worker_html = '';
+    for (my $i = 1; $i <= $max; $i++) {
+        $worker_html .= qq{
+    <option value="$i" <TMPL_IF NAME=WORKERS_$i>selected="selected"</TMPL_IF>>$i</option>};
+    }
+
-    return <<HTML;
-<div class="setting">
-<div class="label"><MT_TRANS phrase="Number of Workers:"></div>
-<div class="field"><ul><li><select name="workers">
-
-
-
-
-
-
-
-
-
-
+
-</select></li></ul>
+</div>
+</div>
+
+<div class="setting">
+<div class="label"><MT_TRANS phrase="Synchronize:"></div>
+<div class="field"><ul><input name="rebuildq_sync" type="checkbox" <TMPL_IF NAME=REBUILDQ_SYNC_1>checked="checked"</TMPL_IF> value="1" />
+<MT_TRANS phrase="Check to support synchronization of queued items with other servers.">
+</li></ul>
-</div>
-</div>
@@ -121 +232 @@
-}
-
+sub sync_support {
+    $plugin->get_config_value('rebuildq_sync') ? 1 : 0;
+}
+
+sub blog_enabled {
+    my $plugin = shift;
+    my ($blog_id) = @_;
+    $plugin->get_config_value('rebuildq_mode', 'blog:'.$blog_id) ? 1 : 0;
+}
+
+sub blog_worker {
+    my $plugin = shift;
+    my ($blog_id) = @_;
+    my $workers = $plugin->get_config_value('workers') || 1;
+    my $worker = $plugin->get_config_value('workers', 'blog:'.$blog_id);
+    $worker = 1 if (defined $worker) && ($worker > $workers);
+    defined $worker ? $worker : int(rand($workers)) + 1;
+}
+
+# Adds an element to the rebuild queue when the plugin is enabled.
-sub build_file_filter {
-    my ($cb, %args) = @_;
+    return 1 unless $ENABLED;
-
-    my $fi = $args{FileInfo};
-    return 1 unless $fi;
-
-
-
+
-
-    require RebuildQueue::File;
@@ -138 +269 @@
-        $rqf->id($fi->id);
-    }
-    my $workers = $plugin->get_config_value('workers') || 1;
-    my $worker = $plugin->get_config_value('workers', 'blog:'.$fi->blog_id);
-    $worker = 1 if (defined $worker) && ($worker > $workers);
-
-    $rqf->rebuild_me(1);
-    $rqf->sync_me(0);
-
+
+
-
-    my $at = $fi->archive_type || '';
@@ -178 +307 @@
-}
-
+# Something was built while the queue itself was disabled; make sure
+# the item is marked for syncing.
+sub build_file {
+    my ($cb, %args) = @_;
+
+    return unless $plugin->sync_support;
+
+    my $fi = $args{FileInfo};
+    my $blog = $args{Blog};
+    if (!$fi) {
+        # FileInfo may still be there due to a bug in MT::WeblogPublisher
+        # passing an undef FileInfo element...
+        require MT::FileInfo;
+        $fi = MT::FileInfo->load({ blog_id => $blog->id, file_path => $args{File} });
+    }
+    return unless $fi;
+    return unless $plugin->blog_enabled($blog->id);
+
+    require RebuildQueue::File;
+    my $rqf = RebuildQueue::File->load($fi->id);
+    if (!$rqf) {
+        $rqf = new RebuildQueue::File;
+        $rqf->id($fi->id);
+        $rqf->build_time(time);
+        $rqf->worker($plugin->blog_worker($blog->id));
+        $rqf->sync_me(1);
+        $rqf->save;
+    } else {
+        if (!$rqf->rebuild_me) {
+            if (!$rqf->sync_me) {
+                $rqf->sync_me(1);
+                $rqf->build_time(time);
+                $rqf->worker($plugin->blog_worker($blog->id));
+                $rqf->save;
+            }
+        }
+    }
+}
+
-1;
+
+__END__
+
+=head1 NAME
+
+RebuildQueue::Plugin - Movable Type plugin for distributed rebuilding.
+
+=head1 SYNOPSIS
+
+=head1 TWEAKING
+
+You may wish to further tune the rebuild queue items. To do so, you
+should hook into the RebuildQueue::File::pre_save MT callback. This
+gives you a chance to modify the priority and build_time properties
+of a RebuildQueue::File record before it is saved. Here's an example:
+
+Within a separate plugin, register for the callback:
+
+    MT->add_callback('RebuildQueue::File', 5, undef, \&rqf_fix);
+
+Then, define your callback routine:
+
+    sub rqf_fix {
+        my ($cb, $obj, $orig) = @_;
+        my $fi = $obj->fileinfo;
+
+        # only tweak when being saved to the rebuild queue
+        return unless $obj->rebuild_me;
+
+        # Sidebar elements can be built last.
+        if ($fi->file_path =~ m!/sidebar/!) {
+            $obj->priority(1000); # really low priority
+        }
+
+        # Forces pages built for nagios to run on a particular
+        # worker that also has access to nagios local data.
+        if ($fi->file_path =~ m!/nagios/!) {
+            $obj->worker(3);
+        }
+    }
+
+As you can see, it's possible to really fine-tune your rebuild
+queue.
+
+=head1 AUTHORS
+
+Brad Choate and Jay Allen
+
+=head1 COPYRIGHT
+
+Copyright (c) 2006, Brad Choate and Jay Allen. This is free software.
+It may be distributed and modified under the same terms as Perl itself.
+
+=head1 AVAILABILITY
+
+http://code.sixapart.com/
+
+=cut