Changeset 494

Show
Ignore:
Timestamp:
06/10/08 23:13:56 (5 months ago)
Author:
bchoate
Message:

Added 'window_size' for search method of BaseCache class.

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved

  • trunk/Changes

    r467 r494  
    1313    - Added a "distinct" method to D::OD::SQL that forces the DISTINCT keyword 
    1414      in the generated SQL statement. Thanks to John Berthels for the patch. 
     15    - Added a "window_size" argument for the search() method of the caching 
     16      layer to constrain the number of objects loaded from the database for 
     17      large or unbounded searches.  
    1518 
    16190.05  2008.02.24 

  • trunk/lib/Data/ObjectDriver.pm

    r482 r494  
    415415 
    416416A sql comment to watermark the SQL query. 
     417 
     418=item * window_size 
     419 
     420Used when requesting an iterator for the search method and selecting 
     421a large result set or a result set of unknown size. In such a case, 
     422no LIMIT clause is assigned, which can load all available objects into 
     423memory. Specifying C<window_size> will load objects in manageable chunks. 
     424This will also cause any caching driver to be bypassed for issuing 
     425the search itself. Objects are still placed into the cache upon load. 
     426 
     427This attribute is ignored when the search method is invoked in an array 
     428context, or if a C<limit> attribute is also specified that is smaller than 
     429the C<window_size>. 
    417430 
    418431=back 

  • trunk/lib/Data/ObjectDriver/BaseObject.pm

    r493 r494  
    490490    my($terms, $args) = @_; 
    491491    my $driver = $class->driver; 
    492     my @objs = $driver->search($class, $terms, $args); 
    493  
    494     ## Don't attempt to cache objects where the caller specified fetchonly, 
    495     ## because they won't be complete. 
    496     ## Also skip this step if we don't get any objects back from the search 
    497     if (!$args->{fetchonly} || !@objs) { 
    498         for my $obj (@objs) { 
    499             $driver->cache_object($obj) if $obj; 
     492    if (wantarray) { 
     493        my @objs = $driver->search($class, $terms, $args); 
     494 
     495        ## Don't attempt to cache objects where the caller specified fetchonly, 
     496        ## because they won't be complete. 
     497        ## Also skip this step if we don't get any objects back from the search 
     498        if (!$args->{fetchonly} || !@objs) { 
     499            for my $obj (@objs) { 
     500                $driver->cache_object($obj) if $obj; 
     501            } 
    500502        } 
    501     } 
    502     $driver->list_or_iterator(\@objs); 
     503        return @objs; 
     504    } else { 
     505        my $iter = $driver->search($class, $terms, $args); 
     506        return $iter if $args->{fetchonly}; 
     507 
     508        my $caching_iter = sub { 
     509            my $d = $driver; 
     510 
     511            my $o = $iter->(); 
     512            unless ($o) { 
     513                $iter->end; 
     514                return; 
     515            } 
     516            $driver->cache_object($o); 
     517            return $o; 
     518        }; 
     519        return Data::ObjectDriver::Iterator->new($caching_iter, sub { $iter->end }); 
     520    } 
    503521} 
    504522 

  • trunk/lib/Data/ObjectDriver/Driver/BaseCache.pm

    r482 r494  
    187187    ## Disable triggers for this load. We don't want the post_load trigger 
    188188    ## being called twice. 
    189     $args->{no_triggers} = 1; 
     189    local $args->{no_triggers} = 1; 
    190190    my @objs = $driver->fallback->search($class, $terms, $args); 
    191191 
    192     ## Load all of the objects using a lookup_multi, which is fast from 
    193     ## cache. 
    194     my $objs = $driver->lookup_multi($class, [ map { $_->primary_key } @objs ]); 
    195  
    196     $driver->list_or_iterator($objs); 
     192    my $windowed = (!wantarray) && $args->{window_size}; 
     193 
     194    if ( $windowed ) { 
     195        my @window; 
     196        my $window_size = $args->{window_size}; 
     197        my $iter = sub { 
     198            my $d = $driver; 
     199            while ( (!@window) && @objs ) { 
     200                my $objs = $driver->lookup_multi( 
     201                    $class, 
     202                    [ map { $_->primary_key } 
     203                          splice( @objs, 0, $window_size ) ] 
     204                ); 
     205                # A small possibility exists that we may fetch 
     206                # some IDs here that no longer exist; grep these out 
     207                @window = grep { defined $_ } @$objs if $objs; 
     208            } 
     209            return @window ? shift @window : undef; 
     210        }; 
     211        return Data::ObjectDriver::Iterator->new($iter, sub { @objs = (); @window = () }); 
     212    } else { 
     213        ## Load all of the objects using a lookup_multi, which is fast from 
     214        ## cache. 
     215        my $objs = $driver->lookup_multi($class, [ map { $_->primary_key } @objs ]); 
     216 
     217        return $driver->list_or_iterator($objs); 
     218    } 
    197219} 
    198220 

  • trunk/lib/Data/ObjectDriver/Driver/DBI.pm

    r493 r494  
    9999 
    100100    ## Use (shallow) duplicates so the pre_search trigger can modify them. 
    101     my $terms = defined $orig_terms ? ( ref $orig_terms eq 'ARRAY' ? [ @$orig_terms ] : { %$orig_terms } ) : undef; 
    102     my $args  = defined $orig_args  ? { %$orig_args  } : undef; 
     101    my $terms = defined $orig_terms ? ( ref $orig_terms eq 'ARRAY' ? [ @$orig_terms ] : { %$orig_terms } ) : {}; 
     102    my $args  = defined $orig_args  ? { %$orig_args  } : {}; 
    103103    $class->call_trigger('pre_search', $terms, $args); 
    104104 
    …… 
    188188    my @got; 
    189189    ## If it's a single-column PK, assume it's in one partition, and 
    190     ## use an OR search. 
     190    ## use an OR search. FIXME: can we instead check for partitioning? 
    191191    unless (ref($ids->[0])) { 
    192192        my $terms = $class->primary_key_to_terms([ $ids ]); 
    …… 
    239239 
    240240    my $class = ref $obj; 
     241    $terms ||= {}; 
    241242    $class->call_trigger('pre_search', $terms); 
    242243 

  • trunk/lib/Data/ObjectDriver/SQL.pm

    r492 r494  
    471471are as described for the C<joins> attribute member above. 
    472472 
     473=head2 C<$sql-E<gt>add_index_hint($table, $index)> 
     474 
     475Specifies a particular index to use for a particular table. 
     476 
    473477=head2 C<$sql-E<gt>add_where($column, $value)> 
    474478 

  • trunk/t/05-deflate.t

    r232 r494  
    4343{ 
    4444    no warnings 'once'; 
     45    no warnings 'redefine'; 
    4546    *Data::ObjectDriver::Driver::Cache::Cache::deflate = sub { 
    4647        $_[1]->deflate; 

  • trunk/t/32-partitioned.t

    r313 r494  
    4949is(ref $tmp, 'Recipe', 'Iterator gave us a recipe'); 
    5050is($tmp->title, 'My Banana Milkshake', 'Title is My Banana Milkshake'); 
     51$iter->end(); 
    5152 
    5253my $ingredient = Ingredient->new; 
    …… 
    7374is(ref $tmp, 'Ingredient', 'Iterator gave us an ingredient'); 
    7475is($tmp->name, 'Vanilla Ice Cream', 'Name is Vanilla Ice Cream'); 
     76$iter->end(); 
    7577 
    7678my $ingredient2 = Ingredient->new; 

  • trunk/t/34-both.t

    r480 r494  
    3333{ 
    3434    no warnings 'once'; 
     35    no warnings 'redefine'; 
    3536    *Data::ObjectDriver::Driver::Cache::Cache::deflate = sub { 
    3637        $_[1]->deflate; 

  • trunk/t/lib/both/Recipe.pm

    r232 r494  
    2121}); 
    2222 
     23my %drivers; 
    2324__PACKAGE__->has_partitions( 
    2425    number => 2, 
    2526    get_driver => sub { 
    26         return Data::ObjectDriver::Driver::DBI->new( 
    27             dsn => 'dbi:SQLite:dbname=cluster' . shift() . '.db', 
    28             @_, 
    29         ), 
     27        my $cluster = shift; 
     28        my $driver = $drivers{$cluster} ||=  
     29            Data::ObjectDriver::Driver::DBI->new( 
     30                dsn => 'dbi:SQLite:dbname=cluster' . $cluster . '.db', 
     31                @_, 
     32            ); 
     33        return $driver; 
    3034    }, 
    3135); 

  • trunk/t/lib/cached/Ingredient.pm

    r60 r494  
    77use Carp (); 
    88use Data::ObjectDriver::Driver::DBI; 
    9 use Data::ObjectDriver::Driver::Cache::Cache; 
    10 use Cache::Memory; 
     9use Data::ObjectDriver::Driver::Cache::RAM; 
    1110 
    1211our %IDs; 
    …… 
    1615    datasource => 'ingredients', 
    1716    primary_key => [ 'recipe_id', 'id' ], 
    18     driver      => Data::ObjectDriver::Driver::Cache::Cache->new( 
    19         cache => Cache::Memory->new, 
     17    driver      => Data::ObjectDriver::Driver::Cache::RAM->new( 
    2018        fallback => Data::ObjectDriver::Driver::DBI->new( 
    2119            dsn      => 'dbi:SQLite:dbname=global.db', 

  • trunk/t/lib/partitioned/Recipe.pm

    r232 r494  
    1616}); 
    1717 
     18my %drivers; 
    1819__PACKAGE__->has_partitions( 
    1920    number => 2, 
    2021    get_driver => sub { 
    21         return Data::ObjectDriver::Driver::DBI->new( 
    22             dsn => 'dbi:SQLite:dbname=cluster' . shift() . '.db', 
    23             @_, 
    24         ), 
     22        my $cluster = shift; 
     23        my $driver = $drivers{$cluster} ||=  
     24            Data::ObjectDriver::Driver::DBI->new( 
     25                dsn => 'dbi:SQLite:dbname=cluster' . $cluster . '.db', 
     26                @_, 
     27            ); 
     28        return $driver; 
    2529    }, 
    2630);