Merge pull request #43 from dcheckoway/feature-auto-grow

Add optional `auto_grow_threshold` for anticipatory growth

We have a use case where our pool needs to start relatively small (init_conn=1000) but grow quite large over time (max_conn=65535). And...we want to avoid at all costs getting error_no_members when trying to take a member from the pool. The caller should not ever have to wait for a new member to be started.

To that end, I've added a new auto_grow_threshold feature. It's fully backward compatible, in that it's undefined by default. There is no change whatsoever for existing pooler users.

For those who choose to set auto_grow_threshold to a non-negative integer, however, they will benefit from members being started prior to all free members having been exhausted.

The idea is that when free_count drops to auto_grow_threshold, a new batch of members will be started in the background. By the time free_count would have dropped to zero, there's already a new batch waiting & ready. In fact, free_count never drops to zero in the first place (as long as member start can keep up with the take demand).

This has already proven to work extremely well for us -- we avoid the "cliff" behavior and see the pool growing smartly in advance of the demand.

src/pooler.erl

@@ -564,9 +564,18 @@ take_member_from_pool(#pool{init_count = InitCount,
             {error_no_members, Pool2};
         [Pid|Rest] ->
             Pool2 = take_member_bookkeeping(Pid, From, Rest, Pool1),
-            send_metric(Pool, in_use_count, Pool2#pool.in_use_count, histogram),
-            send_metric(Pool, free_count, Pool2#pool.free_count, histogram),
-            {Pid, Pool2}
+            Pool3 = case Pool2#pool.auto_grow_threshold of
+                        N when is_integer(N) andalso
+                               Pool2#pool.free_count =< N andalso
+                               NumCanAdd > 0 ->
+                            NumToAdd = max(min(InitCount - NonStaleStartingMemberCount, NumCanAdd), 0),
+                            add_members_async(NumToAdd, Pool2);
+                        _ ->
+                            Pool2
+                    end,
+            send_metric(Pool, in_use_count, Pool3#pool.in_use_count, histogram),
+            send_metric(Pool, free_count, Pool3#pool.free_count, histogram),
+            {Pid, Pool3}
     end.
 
 -spec take_member_from_pool_queued(#pool{},

src/pooler.hrl

@@ -2,6 +2,7 @@
 -define(DEFAULT_CULL_INTERVAL, {1, min}).
 -define(DEFAULT_MAX_AGE, {30, sec}).
 -define(DEFAULT_MEMBER_START_TIMEOUT, {1, min}).
+-define(DEFAULT_AUTO_GROW_THRESHOLD, undefined).
 -define(POOLER_GROUP_TABLE, pooler_group_table).
 -define(DEFAULT_POOLER_QUEUE_MAX, 50).
 
@@ -73,6 +74,12 @@
           %% The maximum amount of time to allow for member start.
           member_start_timeout = ?DEFAULT_MEMBER_START_TIMEOUT :: time_spec(),
 
+          %% The optional threshold at which more members will be started if
+          %% free_count drops to this value.  Normally undefined, but may be
+          %% set to a non-negative integer in order to enable "anticipatory"
+          %% behavior (start members before they're actually needed).
+          auto_grow_threshold = ?DEFAULT_AUTO_GROW_THRESHOLD :: undefined | non_neg_integer(),
+
           %% The module to use for collecting metrics. If set to
           %% 'pooler_no_metrics', then metric sending calls do
           %% nothing. A typical value to actually capture metrics is

src/pooler_config.erl

@@ -20,6 +20,7 @@ list_to_pool(P) ->
        cull_interval     = ?gv(cull_interval, P, ?DEFAULT_CULL_INTERVAL),
        max_age           = ?gv(max_age, P, ?DEFAULT_MAX_AGE),
        member_start_timeout = ?gv(member_start_timeout, P, ?DEFAULT_MEMBER_START_TIMEOUT),
+       auto_grow_threshold = ?gv(auto_grow_threshold, P, ?DEFAULT_AUTO_GROW_THRESHOLD),
        metrics_mod       = ?gv(metrics_mod, P, pooler_no_metrics),
        metrics_api       = ?gv(metrics_api, P, folsom),
        queue_max         = ?gv(queue_max, P, ?DEFAULT_POOLER_QUEUE_MAX)}.

test/pooler_tests.erl

@@ -985,6 +985,81 @@ pooler_integration_test_() ->
      ]
     }.
 
+pooler_auto_grow_disabled_by_default_test_() ->
+    {setup,
+     fun() ->
+             application:set_env(pooler, metrics_module, fake_metrics),
+             fake_metrics:start_link()
+     end,
+     fun(_X) ->
+             fake_metrics:stop()
+     end,
+    {foreach,
+     % setup
+     fun() ->
+             Pool = [{name, test_pool_1},
+                     {max_count, 5},
+                     {init_count, 2},
+                     {start_mfa,
+                      {pooled_gs, start_link, [{"type-0"}]}}],
+             application:unset_env(pooler, pools),
+             error_logger:delete_report_handler(error_logger_tty_h),
+             application:start(pooler),
+             pooler:new_pool(Pool)
+     end,
+     fun(_X) ->
+             application:stop(pooler)
+     end,
+     [
+      {"take one, and it should not auto-grow",
+       fun() ->
+               ?assertEqual(2, (dump_pool(test_pool_1))#pool.free_count),
+               P = pooler:take_member(test_pool_1),
+               ?assertMatch({"type-0", _Id}, pooled_gs:get_id(P)),
+               timer:sleep(100),
+               ?assertEqual(1, (dump_pool(test_pool_1))#pool.free_count),
+               ok, pooler:return_member(test_pool_1, P)
+       end}
+     ]}}.
+
+pooler_auto_grow_enabled_test_() ->
+    {setup,
+     fun() ->
+             application:set_env(pooler, metrics_module, fake_metrics),
+             fake_metrics:start_link()
+     end,
+     fun(_X) ->
+             fake_metrics:stop()
+     end,
+    {foreach,
+     % setup
+     fun() ->
+             Pool = [{name, test_pool_1},
+                     {max_count, 5},
+                     {init_count, 2},
+                     {auto_grow_threshold, 1},
+                     {start_mfa,
+                      {pooled_gs, start_link, [{"type-0"}]}}],
+             application:unset_env(pooler, pools),
+             error_logger:delete_report_handler(error_logger_tty_h),
+             application:start(pooler),
+             pooler:new_pool(Pool)
+     end,
+     fun(_X) ->
+             application:stop(pooler)
+     end,
+     [
+      {"take one, and it should grow by 2",
+       fun() ->
+               ?assertEqual(2, (dump_pool(test_pool_1))#pool.free_count),
+               P = pooler:take_member(test_pool_1),
+               ?assertMatch({"type-0", _Id}, pooled_gs:get_id(P)),
+               timer:sleep(100),
+               ?assertEqual(3, (dump_pool(test_pool_1))#pool.free_count),
+               ok, pooler:return_member(test_pool_1, P)
+       end}
+     ]}}.
+
 time_as_millis_test_() ->
     Zeros = [ {{0, U}, 0} || U <- [min, sec, ms, mu] ],
     Ones = [{{1, min}, 60000},