Add pool utilization stats endpoint

This has two purposes.

1) The first is to provide some insight into the pool usage when
   producing diagnostic error messages, and debugging overload
   behavior.

2) The second is to allow experimentation with heuristics for load
   shedding in chef-server. Being able to probe the queue depth enables
   more intelligent choices of when and where to fail.

Signed-off-by: Mark Anderson <mark@chef.io>

src/pooler.erl

@@ -38,6 +38,7 @@
          return_member/2,
          return_member/3,
          pool_stats/1,
+         pool_utilization/1,
          manual_start/0,
          new_pool/1,
          pool_child_spec/1,
@@ -301,6 +302,14 @@ return_member(_, error_no_members) ->
 pool_stats(PoolName) ->
     gen_server:call(PoolName, pool_stats).
 
+%% @doc Obtain utilization info for a pool.
+%%
+%% Format of the return value is subject to change, but for now it
+%% will be a proplist to maintain backcompat with R16.
+-spec pool_utilization(atom() | pid()) -> [{atom(), integer()}].
+pool_utilization(PoolName) ->
+    gen_server:call(PoolName, pool_utilization).
+
 %% @doc Invokes `Fun' with arity 1 over all free members in pool with `PoolName'.
 %%
 -spec call_free_members(atom() | pid(), fun((pid()) -> term())) -> Res when
@@ -351,6 +360,8 @@ handle_call(stop, _From, Pool) ->
     {stop, normal, stop_ok, Pool};
 handle_call(pool_stats, _From, Pool) ->
     {reply, dict:to_list(Pool#pool.all_members), Pool};
+handle_call(pool_utilization, _From, Pool) ->
+    {reply, compute_utilization(Pool), Pool};
 handle_call(dump_pool, _From, Pool) ->
     {reply, Pool, Pool};
 handle_call({call_free_members, Fun}, _From, #pool{free_pids = Pids} = Pool) ->
@@ -475,7 +486,7 @@ reply_to_queued_requestor(TRef, Pid, From = {APid, _}, NewQueuedRequestors, Pool
     send_metric(Pool, events, error_no_members, history),
     gen_server:reply(From, Pid),
     Pool1.
-    
+
 
 -spec take_member_bookkeeping(pid(),
                               {pid(), _},
@@ -932,6 +943,17 @@ replace_placeholders(Name, Pid, Args) ->
        Arg
    end || Arg <- Args].
 
+compute_utilization(#pool{max_count = MaxCount,
+                          in_use_count = InUseCount,
+                          free_count = FreeCount,
+                          queued_requestors = Queue,
+                          queue_max = QueueMax}) ->
+    [{max_count, MaxCount},
+      {in_use_count, InUseCount},
+      {free_count, FreeCount},
+      {queued_count, queue:len(Queue)}, %% Note not O(n), so in pathological cases this might be expensive
+      {queue_max, QueueMax}].
+
 do_call_free_members(Fun, Pids) ->
     [do_call_free_member(Fun, P) || P <- Pids].
 

test/pooler_tests.erl

@@ -402,6 +402,17 @@ basic_tests() ->
                Bad = spawn(fun() -> ok end),
                FakeStarter = spawn(fun() -> starter end),
                ?assertEqual(ok, pooler:accept_member(test_pool_1, {FakeStarter, Bad}))
+       end},
+
+      {"utilization returns sane results",
+       fun() ->
+               #pool{max_count = MaxCount, queue_max = QueueMax} = Pool = sys:get_state(test_pool_1),
+
+               ?assertEqual(MaxCount, ?gv(max_count, pooler:pool_utilization(test_pool_1))),
+               ?assertEqual(0, ?gv(in_use_count, pooler:pool_utilization(test_pool_1))),
+               ?assertEqual(2, ?gv(free_count, pooler:pool_utilization(test_pool_1))),
+               ?assertEqual(0, ?gv(queued_count, pooler:pool_utilization(test_pool_1))),
+               ?assertEqual(QueueMax, ?gv(queue_max, pooler:pool_utilization(test_pool_1)))
        end}
       ].
 
@@ -621,7 +632,8 @@ pooler_scheduled_cull_test_() ->
                           [ pooler:return_member(test_pool_1, P) || P <- Pids ],
                           %% wait for longer than cull delay
                           timer:sleep(250),
-                          ?assertEqual(2, length(pooler:pool_stats(test_pool_1)))
+                          ?assertEqual(2, length(pooler:pool_stats(test_pool_1))),
+                          ?assertEqual(2, ?gv(free_count,pooler:pool_utilization(test_pool_1)))
                   end}
          end,
 
@@ -631,7 +643,8 @@ pooler_scheduled_cull_test_() ->
                           [ pooler:return_member(test_pool_1, P) || P <- Pids ],
                           %% wait for longer than cull delay
                           timer:sleep(250),
-                          ?assertEqual(2, length(pooler:pool_stats(test_pool_1)))
+                          ?assertEqual(2, length(pooler:pool_stats(test_pool_1))),
+                          ?assertEqual(2, ?gv(free_count,pooler:pool_utilization(test_pool_1)))
                   end}
          end,
 
@@ -668,6 +681,8 @@ in_use_members_not_culled(Pids, N) ->
              PidCount = length(Pids),
              ?assertEqual(PidCount,
                           length(pooler:pool_stats(test_pool_1))),
+             ?assertEqual(0, ?gv(free_count,pooler:pool_utilization(test_pool_1))),
+             ?assertEqual(PidCount, ?gv(in_use_count,pooler:pool_utilization(test_pool_1))),
              Returns = lists:sublist(Pids, N),
              [ pooler:return_member(test_pool_1, P)
                || P <- Returns ],
@@ -814,6 +829,7 @@ pooler_integration_queueing_test_() ->
                       ?assertEqual(0, (dump_pool(test_pool_1))#pool.free_count),
                       Val = pooler:take_member(test_pool_1, 0),
                       ?assertEqual(error_no_members, Val),
+                      ?assertEqual(0, ?gv(free_count,pooler:pool_utilization(test_pool_1))),
                       timer:sleep(50),
                       %Next request should be available
                       Pid = pooler:take_member(test_pool_1, 0),
@@ -847,8 +863,14 @@ pooler_integration_queueing_test_() ->
                                        end)
                         || _ <- lists:seq(1, (dump_pool(test_pool_1))#pool.max_count)
                       ],
+                      ?assertEqual(0, ?gv(free_count,pooler:pool_utilization(test_pool_1))),
+                      ?assert(?gv(queued_count,pooler:pool_utilization(test_pool_1)) >=  1),
+                      ?assertEqual(10, ?gv(queue_max,pooler:pool_utilization(test_pool_1))),
+
                       timer:sleep(50),
                       ?assertEqual(10, queue:len((dump_pool(test_pool_1))#pool.queued_requestors)),
+                      ?assertEqual(10, ?gv(queue_max,pooler:pool_utilization(test_pool_1))),
+
                       ?assertEqual(pooler:take_member(test_pool_1, 500), error_no_members),
                       ExpectKeys = lists:sort([<<"pooler.test_pool_1.error_no_members_count">>,
                                                <<"pooler.test_pool_1.events">>,