Changeset 617

Timestamp:

01/17/07 07:05:55 (2 years ago)

Author:

bradfitz

Message:

-- supported for "Transfer-Encoding: chunked" requests (HTTP/1.1 feature)

as well as the "Expect: 100-continue", which generally accompany
chunked requests. requires "buffered_uploads" be enabled. see
doc/http-versions.txt for details.

Files:

• trunk/CHANGES (modified) (1 diff)
• trunk/MANIFEST (modified) (3 diffs)
• trunk/META.yml (modified) (1 diff)
• trunk/doc/http-versions.txt (added)
• trunk/lib/Perlbal/ChunkedUploadState.pm (added)
• trunk/lib/Perlbal/ClientProxy.pm (modified) (11 diffs)
• trunk/t/52-chunked-upload.t (added)

trunk/CHANGES

@@ +1 @@
+    -- supported for "Transfer-Encoding: chunked" requests (HTTP/1.1 feature)
+       as well as the "Expect: 100-continue", which generally accompany
+       chunked requests.  requires "buffered_uploads" be enabled.  see
+       doc/http-versions.txt for details.
+
-1.53: 2006-12-05
-

trunk/MANIFEST

@@ -7 +7 @@
-lib/Perlbal/BackendHTTP.pm
-lib/Perlbal/Cache.pm
+lib/Perlbal/ChunkedUploadState.pm
-lib/Perlbal/ClientHTTP.pm
-lib/Perlbal/ClientHTTPBase.pm
@@ -33 +34 @@
-META.yml                                 Module meta-data (added by MakeMaker)
-devtools/gendocs.pl
+doc/http-versions.txt
-doc/config-guide.txt
-doc/service-parameters.txt
@@ -59 +61 @@
-t/45-buffereduploads.t
-t/50-plugins.t
+t/52-chunked-upload.t
-

trunk/META.yml

@@ -2 +2 @@
-#XXXXXXX This is a prototype!!!  It will change in the future!!! XXXXX#
-name:         Perlbal
-2
+3
-version_from: lib/Perlbal.pm
-installdirs:  site

trunk/lib/Perlbal/ClientProxy.pm

@@ -10 +10 @@
-use base "Perlbal::ClientHTTPBase";
-no  warnings qw(deprecated);
+
+use Perlbal::ChunkedUploadState;
-
-use fields (
@@ -36 +38 @@
-            'backend_stalled',   # boolean:  if backend has shut off its reads because we're too slow.
-            'unread_data_waiting',  # boolean:  if we shut off reads while we know data is yet to be read from client
+            'chunked_upload_state', # bool/obj:  if processing a chunked upload, Perlbal::ChunkedUploadState object, else undef
+            'request_body_length',  # integer:  request's body length, either as-declared,
+                                    #           or calculated after chunked upload is complete
-
-            # for perlbal sending out UDP packets related to upload status (for xmlhttprequest upload bar)
@@ -99 +104 @@
-    $self->{buoutpos} = 0;
-    $self->{bureason} = undef;
+    $self->{chunked_upload_state} = undef;
+    $self->{request_body_length} = undef;
-
-    $self->{reproxy_uris} = undef;
@@ -467 +474 @@
-    $self->{bureason} = undef;
-    $self->{upload_session} = undef;
+    $self->{chunked_upload_state} = undef;
+    $self->{request_body_length} = undef;
-    return 1;
-}
@@ -569 +578 @@
-        print "  disabling reads.\n" if Perlbal::DEBUG >= 3;
-        $self->watch_read(0);
+        return;
+    }
+
+    # deal with chunked uploads
+    if (my $cus = $self->{chunked_upload_state}) {
+        $cus->on_readable($self);
+
+        # if we got more than 1MB not flushed to disk,
+        # stop reading for a bit until disk catches up
+        if ($self->{read_ahead} > 1024*1024) {
+            $self->watch_read(0);
+        }
-        return;
-    }
@@ -713 +734 @@
-    return if $svc->run_hook('start_http_request',  $self);
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
-
-    # upload-tracking stuff.  both starting a new upload track session,
@@ -729 +756 @@
-    # either start buffering some of the request to memory, or
-    # immediately request a backend connection.
-
+
+
+
+
+
+
-        # the deeper path
-        $self->start_buffering_request;
@@ -741 +773 @@
-        $self->request_backend;
-    }
+}
+
+sub handle_chunked_upload {
+    my Perlbal::ClientProxy $self = shift;
+    my $req_hd = $self->{req_headers};
+    my $te = $req_hd->header("Transfer-Encoding");
+    return unless $te && $te eq "chunked";
+    return unless $self->{service}->{buffer_uploads};
+
+    $req_hd->header("Transfer-Encoding", undef); # remove it (won't go to backend)
+
+    # TODO: return false if we don't have buffered upload dir configured
+    my $eh = $req_hd->header("Expect");
+    if ($eh && $eh =~ /\b100-continue\b/) {
+        $self->write(\ "HTTP/1.1 100 Continue\r\n\r\n");
+        $req_hd->header("Expect", undef); # remove it (won't go to backend)
+    }
+
+    my $args = {
+        on_new_chunk => sub {
+            my $cref = shift;
+            my $len = length($$cref);
+            push @{$self->{read_buf}}, $cref;
+            $self->{read_ahead}          += $len;
+            $self->{request_body_length} += $len;
+            # TODO: if too large, disconnect?
+            $self->buffered_upload_update;
+        },
+        on_disconnect => sub {
+            $self->client_disconnected;
+        },
+        on_zero_chunk => sub {
+            $self->send_buffered_upload;
+        },
+    };
+
+    $self->{chunked_upload_state} = Perlbal::ChunkedUploadState->new(%$args);
+    return 1;
-}
-
@@ -915 +985 @@
-
-    # make sure our buoutpos is the same as the content length...
+    return if $self->{is_writing};
+
+    # set the content-length that goes to the backend...
+    if ($self->{chunked_upload_state}) {
+        $self->{req_headers}->header("Content-Length", $self->{request_body_length});
+    }
+
-    my $clen = $self->{req_headers}->content_length;
-    if ($clen != $self->{buoutpos}) {
@@ -935 +1012 @@
-
-    # now send the data
-
+
+
-    my $sent = Perlbal::Socket::sendfile($be->{fd}, fileno($self->{bufh}), $clen - $self->{buoutpos});
-    if ($sent < 0) {
@@ -1017 +1095 @@
-        }
-
+        # if we're processing a chunked upload, ...
+        if ($self->{chunked_upload_state}) {
+            # turn reads back on, if we haven't hit the end yet.
+            if ($self->{unread_data_waiting} && $self->{read_ahead} < 1024*1024) {
+                $self->watch_read(1);
+                $self->{unread_data_waiting} = 0;
+            }
+
+            if ($self->{read_ahead} == 0 && $self->{chunked_upload_state}->hit_zero_chunk) {
+                $self->watch_read(0);
+                $self->send_buffered_upload;
+                return;
+            }
+        }
+
-        # if we're done (no clr and no read ahead!) then send it
-
+els
-            $self->send_buffered_upload;
-            return;