Changeset 530

Show
Ignore:
Timestamp:
05/02/07 22:01:34 (2 years ago)
Author:
bradfitz
Message:

merging client-xs-20070328 branch into trunk

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved
  • trunk/api/xs/Cache-Memcached-GetParserXS

    • Property svn:ignore set to
      const-xs.inc
      const-c.inc
      GetParserXS.c
      *.diff
      notes.txt
      *blib
      *.bs
      Makefile
      Makefile.old


  • trunk/api/xs/Cache-Memcached-GetParserXS/GetParserXS.xs

    r305 r530  
    44 
    55#include "ppport.h" 
    6  
    7 #include "const-c.inc" 
    86 
    97#define DEST     0  /* destination hashref we're writing into */ 
     
    1513#define FLAGS    6 
    1614#define KEY      7  /* current key we're parsing (without the namespace prefix) */ 
     15#define FINISHED 8  /* hashref of keys and flags to be finalized at any time */ 
    1716 
    1817#define DEBUG    0 
     18 
     19#include "const-c.inc" 
    1920 
    2021int get_nslen (AV* self) { 
     
    2526} 
    2627 
    27 void set_key (AV* self, const char *key) { 
    28   av_store(self, KEY, newSVpv(key, strlen(key))); 
    29 } 
    30  
    31 SV *get_key_sv (AV* self) { 
     28inline void set_key (AV* self, const char *key, int len) { 
     29  av_store(self, KEY, newSVpv(key, len)); 
     30} 
     31 
     32inline SV *get_key_sv (AV* self) { 
    3233  SV** svp = av_fetch(self, KEY, 0); 
    3334  if (svp) 
     
    3637} 
    3738 
    38 SV *get_on_item (AV* self) { 
     39inline SV *get_on_item (AV* self) { 
    3940  SV** svp = av_fetch(self, ON_ITEM, 0); 
    4041  if (svp) 
     
    4344} 
    4445 
    45 void set_flags (AV* self, int flags) { 
     46inline SV *get_offset_sv (AV* self) { 
     47  SV** svp = av_fetch(self, OFFSET, 0); 
     48  if (svp) 
     49    return (SV*) *svp; 
     50 
     51  *svp = newSViv(0); 
     52  av_store(self, OFFSET, *svp); 
     53  return (SV*) *svp; 
     54
     55 
     56inline void clear_on_item (AV* self) { 
     57  SV** svp = av_store(self, ON_ITEM, newSV(0) ); 
     58
     59 
     60inline void set_flags (AV* self, int flags) { 
    4661  av_store(self, FLAGS, newSViv(flags)); 
    4762} 
    4863 
    49 void set_offset (AV* self, int offset) { 
     64inline void set_offset (AV* self, int offset) { 
    5065  av_store(self, OFFSET, newSViv(offset)); 
    5166} 
    5267 
    53 void set_state (AV* self, int state) { 
     68inline void set_state (AV* self, int state) { 
    5469  av_store(self, STATE, newSViv(state)); 
    5570} 
    5671 
    57 HV* get_dest (AV* self) { 
     72inline HV* get_dest (AV* self) { 
    5873  SV** svp = av_fetch(self, DEST, 0); 
    5974  if (svp) 
     
    6277} 
    6378 
    64 int get_state (AV* self) { 
     79inline HV* get_finished (AV* self) { 
     80  SV** svp = av_fetch(self, FINISHED, 0); 
     81  if (svp) 
     82    return (HV*) SvRV(*svp); 
     83  return 0; 
     84
     85 
     86inline IV get_state (AV* self) { 
    6587  SV** svp = av_fetch(self, STATE, 0); 
    6688  if (svp) 
     
    6991} 
    7092 
    71 SV* get_buffer (AV* self) { 
     93inline SV* get_buffer (AV* self) { 
    7294  SV** svp = av_fetch(self, BUF, 0); 
    7395  if (svp) 
     
    78100/* returns an answer, but also unsets ON_ITEM */ 
    79101int final_answer (AV* self, int ans) { 
    80   av_store(self, ON_ITEM, newSV(0)); 
     102//  av_store(self, ON_ITEM, newSV(0)); 
    81103  return ans; 
    82104} 
     
    88110  STRLEN len; 
    89111  char* buf; 
    90   char key[257]; 
    91112  unsigned int itemlen; 
    92113  unsigned int flags; 
     
    94115  int nslen = get_nslen(self); 
    95116  SV* on_item = get_on_item(self); 
     117  register signed char c; 
     118  char *key; 
     119  register char *p; 
     120  int key_len, barelen; 
     121  int state, copy, new_p; 
     122  char *barekey; 
     123 
     124  HV* finished = get_finished(self); 
    96125 
    97126  if (DEBUG) 
     
    101130    int rv; 
    102131    buf = SvPV(bufsv, len); 
    103  
    104     if (DEBUG) 
    105       printf(" buf (len=%d) = [%s]\n", len, buf); 
    106  
    107     scanned = 0; 
    108     rv = sscanf(buf, "VALUE %256s %u %u%n", key, &flags, &itemlen, &scanned); 
    109  
    110     if (DEBUG) 
    111       printf("rv=%d, scanned=%d, one=[%d], two=[%d]\n", 
    112              rv, scanned, buf[scanned], buf[scanned+1]); 
    113  
    114     if (rv >= 3 && scanned && buf[scanned+1] == '\n') { 
    115       int p     = scanned + 2;      /* 2 to skip \r\n */ 
    116       int state = itemlen + 2;      /* 2 to include reading final \r\n, a different \r\n */ 
    117       int copy  = len - p > state ? state : len - p; 
    118       char *barekey = key + nslen; 
    119  
    120       if (DEBUG) 
     132    p = buf; 
     133 
     134    if (DEBUG) { 
     135      char first_line[1000]; 
     136      int i; 
     137      char *end; 
     138      for (i = 0, end = buf; *end && *end != '\n' && i++ < 900; end++) 
     139              ; 
     140      end += 10; 
     141      strncpy (first_line, buf, end - buf + 1); 
     142      first_line[end - buf + 1] = '\0'; 
     143      printf("GOT buf (len=%d)\nFirst line: %s\n", len, first_line); 
     144    } 
     145 
     146    if ((c = *p++) == 'V') { 
     147      if (*p++ != 'A' || *p++ != 'L' || *p++ != 'U' || *p++ != 'E' || *p++ != ' ') { 
     148        if (DEBUG) 
     149          puts ("ERROR: Illegal command beginning with V"); 
     150        goto recover_from_partial_line; 
     151      } 
     152 
     153      // Parsing VALUE %s<key> %u<flags> %u<bytes> 
     154 
     155      for (key = p; *p++ > ' ';) 
     156        ; 
     157      key_len = p - key - 1; 
     158      if (*(p - 1) != ' ') { 
     159        if (DEBUG) 
     160          printf ("ERROR: key not space-terminated: key %s, char %c\n", key, *(p - 1)); 
     161        goto recover_from_partial_line; 
     162      } 
     163      // Note that key just points into the buffer and is not null-terminated 
     164      // yet.  Leave it that way in case we're dealing with a partial line. 
     165 
     166      // Get flags and itemlen as integers.  Note invalid characters  
     167      // are not caught and will result in strange numbers. 
     168 
     169      for (flags = 0; (c = *p++ - '0') >= 0; flags = flags * 10 + c) 
     170        ; 
     171      if (c != (signed char)' ' - '0') { 
     172        if (DEBUG) 
     173          puts ("ERROR: Flags not space terminated"); 
     174        goto recover_from_partial_line; 
     175      } 
     176 
     177 
     178      for (itemlen = 0; (c = *p++ - '0') >= 0; itemlen = itemlen * 10 + c) 
     179        ; 
     180      if (c != (signed char)'\r' - '0' || *p++ != '\n') { 
     181        if (DEBUG) 
     182          puts ("ERROR: byte count not CRLF-terminated"); 
     183        goto recover_from_partial_line; 
     184      } 
     185 
     186 
     187      // p is left at the start of the value data. 
     188 
     189      new_p = p - buf; 
     190      state = itemlen + 2;      /* 2 to include reading final \r\n, a different \r\n */ 
     191      copy  = len - new_p > state ? state : len - new_p; 
     192      barekey = key + nslen; 
     193      barelen = key_len - nslen; 
     194 
     195      if (DEBUG) { 
     196        char temp_key[256]; 
     197        strncpy (temp_key, key, key_len); 
     198        temp_key[key_len] = '\0'; 
    121199        printf("key=[%s], state=%d, copy=%d\n", key, state, copy); 
     200      } 
    122201 
    123202      if (copy) { 
    124         //SV*  newSVpv(const char*, STRLEN); 
    125         //SV**  hv_store(HV*, const char* key, U32 klen, SV* val, U32 hash); 
    126         /*  $ret->{$self->[KEY]} = substr($self->[BUF], $p, $copy) */ 
    127         hv_store(ret, barekey, strlen(barekey), newSVpv(buf + p, copy), 0); 
    128         buf[p + copy - 1] = '\0'; 
    129  
    130         if (DEBUG) 
    131           printf("doing store:  len=%d key=[%s] of data [%s]\n", 
    132                  strlen(barekey), barekey, 
    133                  buf + p); 
     203        *(key + key_len) = '\0';        // Null-terminate the key in-buffer 
     204        hv_store(ret, barekey, barelen, newSVpv(buf + new_p, copy), 0); 
     205        buf[new_p + copy - 1] = '\0'; 
     206 
     207        if (DEBUG) 
     208          printf("doing store:  len=%d key=[%s] of data [%c]\n", 
     209                 strlen(barekey), barekey, *(buf + new_p)); 
    134210      } 
    135211 
    136212      /* delete the stuff we used */ 
    137       sv_chop(bufsv, buf + p + copy); 
     213      sv_chop(bufsv, buf + new_p + copy); 
    138214 
    139215      if (copy == state) { 
    140         dSP ; 
    141  
    142          /* have it all? */ 
    143         ENTER ; 
    144         SAVETMPS ; 
    145         PUSHMARK(SP) ; 
    146         XPUSHs(sv_2mortal(newSVpv(barekey, strlen(barekey)))); 
    147         XPUSHs(sv_2mortal(newSViv(flags))); 
    148         PUTBACK ; 
    149         call_sv(on_item, G_VOID | G_DISCARD); 
    150         FREETMPS ; 
    151         LEAVE ; 
     216        hv_store(finished, barekey, barelen, newSViv(flags), 0); 
    152217 
    153218        set_offset(self, 0); 
     
    156221      } else { 
    157222        /* don't have it all... but buffer is now empty */ 
     223        hv_store(finished, barekey, barelen, newSViv(flags), 0); 
    158224        set_offset(self, copy); 
    159225        set_flags(self, flags); 
    160         set_key(self, barekey); 
     226        set_key(self, barekey, barelen); 
    161227        set_state(self, state); 
    162228 
     
    168234    } 
    169235 
    170     if (strncmp(buf, "END\r\n", 5) == 0) { 
    171       /* we're done successfully, return 1 to finish */ 
    172       return final_answer(self, 1); 
     236    else if (c == 'E') { 
     237 
     238      // Parsing END 
     239 
     240      if (*p++ == 'N' && *p++ == 'D' && *p++ == '\r' && *p == '\n') 
     241        return final_answer(self, 1); 
    173242    } 
     243    // Just fall through if after 'E' was not "ND\r\n" 
     244 
     245    else 
     246      ;         // Unknown command: not 'E' or 'V' at [0] 
    174247 
    175248 
     
    183256    */ 
    184257 
     258  recover_from_partial_line: 
    185259    set_offset(self, len); 
    186260    return 0; 
     
    188262} 
    189263 
    190 int parse_from_sock_xx (SV* selfref, SV* sock, int sockfd) { 
    191   int res; 
    192   AV* self = (AV*) SvRV(selfref); 
    193   HV* ret = get_dest(self); 
    194   int state = get_state(self); 
    195  
    196   if (state) { 
    197     //res = read(sockfd, *buf, state); 
    198   } 
    199  
    200   printf("fileno = %d\n", sockfd); 
    201  
    202   printf("got = %x, state = %d\n", ret, state); 
    203   return -1; 
    204  
    205   /* 
    206     # where are we reading into? 
    207     if ($self->[STATE]) { # reading value into $ret 
    208         $res = sysread($sock, $ret->{$self->[KEY]}, 
    209                        $self->[STATE] - $self->[OFFSET], 
    210                        $self->[OFFSET]); 
    211  
    212         return 0 
    213             if !defined($res) and $!==EWOULDBLOCK; 
    214  
    215         if ($res == 0) { # catches 0=conn closed or undef=error 
    216             $self->[ON_ITEM] = undef; 
    217             return -1; 
    218         } 
    219  
    220         $self->[OFFSET] += $res; 
    221         if ($self->[OFFSET] == $self->[STATE]) { # finished reading 
    222             $self->[ON_ITEM]->($self->[KEY], $self->[FLAGS]); 
    223             $self->[OFFSET] = 0; 
    224             $self->[STATE]  = 0; 
    225             # wait for another VALUE line or END... 
    226         } 
    227         return 0; # still working, haven't got to end yet 
    228     } 
    229  
    230     # we're reading a single line. 
    231     # first, read whatever's there, but be satisfied with 2048 bytes 
    232     $res = sysread($sock, $self->[BUF], 
    233                    2048, $self->[OFFSET]); 
    234     return 0 
    235         if !defined($res) and $!==EWOULDBLOCK; 
    236     if ($res == 0) { 
    237         $self->[ON_ITEM] = undef; 
    238         return -1; 
    239     } 
    240  
    241     $self->[OFFSET] += $res; 
    242  
    243   */ 
    244 
    245  
    246 MODULE = Cache::Memcached::GetParserXS          PACKAGE = Cache::Memcached::GetParserXS          
     264MODULE = Cache::Memcached::GetParserXS      PACKAGE = Cache::Memcached::GetParserXS 
    247265 
    248266INCLUDE: const-xs.inc 
    249  
    250 int 
    251 parse_from_sock_xx ( self, sock, sockfd ) 
    252     SV *self 
    253     SV *sock 
    254     int sockfd 
    255267 
    256268int 
  • trunk/api/xs/Cache-Memcached-GetParserXS/Makefile.PL

    r304 r530  
    55WriteMakefile( 
    66              OPTIMIZE => '-g', 
    7     NAME              => 'Cache::Memcached::GetParserXS', 
    8     VERSION_FROM      => 'lib/Cache/Memcached/GetParserXS.pm', # finds $VERSION 
    9     PREREQ_PM         => {}, # e.g., Module::Name => 1.1 
    10     ($] >= 5.005 ?     ## Add these new keywords supported since 5.005 
    11       (ABSTRACT_FROM  => 'lib/Cache/Memcached/GetParserXS.pm', # retrieve abstract from module 
    12        AUTHOR         => 'LiveJournal user <lj@>') : ()), 
    13     LIBS              => [''], # e.g., '-lm' 
    14     DEFINE            => '', # e.g., '-DHAVE_SOMETHING' 
    15     INC               => '-I.', # e.g., '-I. -I/usr/include/other' 
    16         # Un-comment this if you add C files to link with later: 
    17     # OBJECT            => '$(O_FILES)', # link all the C files too 
    18 ); 
     7              NAME              => 'Cache::Memcached::GetParserXS', 
     8              VERSION_FROM      => 'lib/Cache/Memcached/GetParserXS.pm', # finds $VERSION 
     9              PREREQ_PM         => { 
     10                  'Cache::Memcached' => 1.21, 
     11              }, # e.g., Module::Name => 1.1 
     12              ABSTRACT_FROM  => 'lib/Cache/Memcached/GetParserXS.pm', # retrieve abstract from module 
     13              AUTHOR         => 'LiveJournal user <lj@>', 
     14              LIBS              => [''], # e.g., '-lm' 
     15              DEFINE            => '', # e.g., '-DHAVE_SOMETHING' 
     16              INC               => '-I.', # e.g., '-I. -I/usr/include/other' 
     17              # Un-comment this if you add C files to link with later: 
     18              # OBJECT            => '$(O_FILES)', # link all the C files too 
     19              ); 
     20 
    1921if  (eval {require ExtUtils::Constant; 1}) { 
    2022  # If you edit these definitions to change the constants used by this module, 
     
    2224  # files to replace their "fallback" counterparts before distributing your 
    2325  # changes. 
    24   my @names = (qw()); 
     26  my @names = (qw(DEST NSLEN ON_ITEM BUF STATE OFFSET FLAGS KEY FINISHED)); 
    2527  ExtUtils::Constant::WriteConstants( 
    2628                                     NAME         => 'Cache::Memcached::GetParserXS', 
  • trunk/api/xs/Cache-Memcached-GetParserXS/lib/Cache/Memcached/GetParserXS.pm

    r304 r530  
    44use strict; 
    55use warnings; 
     6# We don't want to inherit from this, because our constants may be different. 
     7# use base 'Cache::Memcached::GetParser'; 
    68use Carp; 
    7  
    8 require Exporter; 
     9use Errno qw( EINPROGRESS EWOULDBLOCK EISCONN ); 
    910use AutoLoader; 
    10  
    11 our @ISA = qw(Exporter Cache::Memcached::GetParser); 
    12  
    13 # Items to export into callers namespace by default. Note: do not export 
    14 # names by default without a very good reason. Use EXPORT_OK instead. 
    15 # Do not simply export all your public functions/methods/constants. 
    16  
    17 # This allows declaration       use Cache::Memcached::GetParserXS ':all'; 
    18 # If you do not need this, moving things directly into @EXPORT or @EXPORT_OK 
    19 # will save memory. 
    20 our %EXPORT_TAGS = ( 'all' => [ qw( 
    21  
    22 ) ] ); 
    23  
    24 our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); 
    25  
    26 our @EXPORT = qw( 
    27  
    28 ); 
     11use Cache::Memcached 1.21; 
    2912 
    3013our $VERSION = '0.01'; 
     14 
     15require XSLoader; 
     16XSLoader::load('Cache::Memcached::GetParserXS', $VERSION); 
     17 
     18sub DEST; 
     19sub NSLEN; 
     20sub ON_ITEM; 
     21sub BUF; 
     22sub STATE; 
     23sub OFFSET; 
     24sub FLAGS; 
     25sub KEY; 
     26sub FINISHED; 
     27 
     28sub new { 
     29    my ($class, $dest, $nslen, $on_item) = @_; 
     30 
     31    my $self = bless [], (ref $class || $class); 
     32 
     33    $self->[DEST]     = $dest; 
     34    $self->[NSLEN]    = $nslen; 
     35    $self->[ON_ITEM]  = $on_item; 
     36    $self->[BUF]      = ''; 
     37    $self->[STATE]    = 0; 
     38    $self->[OFFSET]   = 0; 
     39    $self->[FLAGS]    = undef; 
     40    $self->[KEY]      = undef; 
     41    $self->[FINISHED] = {}; 
     42 
     43    return $self 
     44} 
     45 
     46sub current_key { 
     47    return $_[0][KEY]; 
     48} 
     49 
     50sub t_parse_buf { 
     51    my ($self, $buf) = @_; 
     52    # force buf into \r\n format 
     53    $buf =~ s/\n/\r\n/g; 
     54    $buf =~ s/\r\r/\r/g; 
     55 
     56    $self->[BUF] .= $buf; 
     57    $self->[OFFSET] += length $buf; 
     58    my $rv = $self->parse_buffer; 
     59    if ($rv > 0) { 
     60        $self->[ON_ITEM]->($self->[FINISHED]); 
     61        $self->[ON_ITEM] = undef; 
     62    } 
     63    return $rv; 
     64} 
     65 
     66# returns 1 on success, -1 on failure, and 0 if still working. 
     67sub parse_from_sock { 
     68    my ($self, $sock) = @_; 
     69    my $res; 
     70 
     71    # where are we reading into? 
     72    if ($self->[STATE]) { # reading value into $ret 
     73        my $ret = $self->[DEST]; 
     74        $res = sysread($sock, $ret->{$self->[KEY]}, 
     75                       $self->[STATE] - $self->[OFFSET], 
     76                       $self->[OFFSET]); 
     77 
     78        return 0 
     79            if !defined($res) and $!==EWOULDBLOCK; 
     80 
     81        if ($res == 0) { # catches 0=conn closed or undef=error 
     82            $self->[ON_ITEM] = undef; 
     83            return -1; 
     84        } 
     85 
     86        $self->[OFFSET] += $res; 
     87        if ($self->[OFFSET] == $self->[STATE]) { # finished reading 
     88            $self->[OFFSET] = 0; 
     89            $self->[STATE]  = 0; 
     90            # wait for another VALUE line or END... 
     91        } 
     92        return 0; # still working, haven't got to end yet 
     93    } 
     94 
     95    # we're reading a single line. 
     96    # first, read whatever's there, but be satisfied with 2048 bytes 
     97    $res = sysread($sock, $self->[BUF], 
     98                   128*1024, $self->[OFFSET]); 
     99    return 0 
     100        if !defined($res) and $!==EWOULDBLOCK; 
     101    if ($res == 0) { 
     102        $self->[ON_ITEM] = undef; 
     103        return -1; 
     104    } 
     105 
     106    $self->[OFFSET] += $res; 
     107 
     108    my $answer = $self->parse_buffer; 
     109 
     110    if ($answer > 0) { 
     111        $self->[ON_ITEM]->($self->[FINISHED]); 
     112        $self->[ON_ITEM] = undef; 
     113    } 
     114 
     115    return $answer; 
     116} 
     117 
     118sub DESTROY {} # Empty definition, so AUTOLOAD doesn't catch it 
     119 
     120# sub parse_buffer is defined in XS 
    31121 
    32122sub AUTOLOAD { 
     
    52142    goto &$AUTOLOAD; 
    53143} 
    54  
    55 require XSLoader; 
    56 XSLoader::load('Cache::Memcached::GetParserXS', $VERSION); 
    57  
    58 # Preloaded methods go here. 
    59  
    60 # Autoload methods go after =cut, and are processed by the autosplit program. 
    61144 
    621451; 
  • trunk/api/xs/Cache-Memcached-GetParserXS/t/Cache-Memcached-GetParserXS.t

    r304 r530  
    1 # Before `make install' is performed this script should be runnable with 
    2 # `make test'. After `make install' it should work as `perl Cache-Memcached-GetParserXS.t' 
     1#!/usr/bi/perl 
    32 
    4 ######################### 
     3use Test::More tests => 6; 
     4BEGIN { use_ok('Cache::Memcached::GetParserXS') }; 
     5use Data::Dumper; 
    56 
    6 # change 'tests => 1' to 'tests => last_test_to_print'; 
     7my $fin; 
     8my $p = new_parser(); 
     9ok($p, "Parser object was created"); 
    710 
    8 use Test::More tests => 1; 
    9 BEGIN { use_ok('Cache::Memcached::GetParserXS') }; 
     11# simple case 
     12$p->t_parse_buf("VALUE foo 0 3 
     13bar 
     14END 
     15"); 
     16is_deeply($fin, { foo => 0 }, "got foo"); 
    1017 
    11 ######################### 
     18# in chunks... 
     19$p = new_parser(); 
     20$p->t_parse_buf("VALUE foo 0 3 
     21bar 
     22VALUE bar 1 3 
     23baz 
     24"); 
     25is($fin, undef, "nothing yet"); 
     26$p->t_parse_buf("END"); 
     27is($fin, undef, "nothing yet"); 
     28$p->t_parse_buf("\n"); 
     29is_deeply($fin, { foo => 0, bar => 1 }, "got 'em"); 
    1230 
    13 # Insert your test code below, the Test::More module is use()ed here so read 
    14 # its man page ( perldoc Test::More ) for help writing this test script. 
    1531 
     32sub new_parser { 
     33    $fin = undef; 
     34    Cache::Memcached::GetParserXS->new({}, 0, sub { $fin = $_[0] }); 
     35}