Browse Source

Merge pull request #85 from seriyps/code-formatter

Code formatter
Sergey Prokhorov 2 years ago
parent
commit
e5bc860aac

+ 17 - 0
.editorconfig

@@ -0,0 +1,17 @@
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 4
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+max_line_length = 120
+
+[*.{config,src}]
+indent_style = space
+
+[*.md]
+indent_style = space
+trim_trailing_whitespace = false

+ 2 - 0
.git-blame-ignore-revs

@@ -0,0 +1,2 @@
+# Code formatter introduced; code was re-formatted
+d8c64208e3f377cf5fd4236a9089c3f9822fde5a

+ 3 - 0
.github/workflows/ci.yml

@@ -43,6 +43,9 @@ jobs:
         run: make xref
         if: ${{ matrix.otp != '23.3' }}
 
+      - name: Code formatter validation
+        run: make format_check
+
       - name: Eunit test
         run: make test
 

+ 6 - 0
Makefile

@@ -18,6 +18,12 @@ test: $(REBAR)
 xref: $(REBAR)
 	$(REBAR) xref
 
+format: $(REBAR)
+	$(REBAR) fmt
+
+format_check: $(REBAR)
+	$(REBAR) fmt --check
+
 doc: $(REBAR)
 	$(REBAR) as dev edoc
 

+ 20 - 5
rebar.config

@@ -1,3 +1,4 @@
+%% -*- mode: erlang -*-
 {erl_opts, [
     debug_info,
 
@@ -24,7 +25,20 @@
     %, warn_missing_spec
 ]}.
 
-{deps, [
+{deps, []}.
+
+{project_plugins, [
+    erlfmt
+]}.
+
+{erlfmt, [
+    write,
+    {print_width, 120},
+    {files, [
+        "{src,include,test}/*.{hrl,erl}",
+        "src/*.app.src",
+        "rebar.config"
+    ]}
 ]}.
 
 {profiles, [
@@ -38,12 +52,13 @@
     ]},
     {test, [
         {erl_opts, [nowarn_export_all]}
-           ]
-    }
+    ]}
 ]}.
 
-{eunit_opts, [{report, {eunit_progress, [colored, profile]}},
-              {print_depth, 100}]}.
+{eunit_opts, [
+    {report, {eunit_progress, [colored, profile]}},
+    {print_depth, 100}
+]}.
 {eunit_compile_opts, [export_all]}.
 
 {ct_opts, []}.

+ 11 - 12
src/pooler.app.src

@@ -1,12 +1,11 @@
-{application, pooler,
- [
-  {description, "An OTP Process Pool Application"},
-  {vsn, git},
-  {registered, []},
-  {applications, [
-                  kernel,
-                  stdlib
-                 ]},
-  {mod, { pooler_app, []}},
-  {env, []}
- ]}.
+{application, pooler, [
+    {description, "An OTP Process Pool Application"},
+    {vsn, git},
+    {registered, []},
+    {applications, [
+        kernel,
+        stdlib
+    ]},
+    {mod, {pooler_app, []}},
+    {env, []}
+]}.

+ 461 - 291
src/pooler.erl

@@ -15,51 +15,57 @@
 -include_lib("kernel/include/logger.hrl").
 
 %% type specs for pool metrics
--type metric_value() :: 'unknown_pid' |
-                        non_neg_integer() |
-                        {'add_pids_failed', non_neg_integer(), non_neg_integer()} |
-                        {'inc',1} |
-                        'error_no_members'.
+-type metric_value() ::
+    'unknown_pid'
+    | non_neg_integer()
+    | {'add_pids_failed', non_neg_integer(), non_neg_integer()}
+    | {'inc', 1}
+    | 'error_no_members'.
 -type metric_type() :: 'counter' | 'histogram' | 'history' | 'meter'.
 
 %% ------------------------------------------------------------------
 %% API Function Exports
 %% ------------------------------------------------------------------
 
--export([start/0,
-         stop/0]).
-
--export([accept_member/2,
-         start_link/1,
-         take_member/1,
-         take_member/2,
-         take_group_member/1,
-         return_group_member/2,
-         return_group_member/3,
-         return_member/2,
-         return_member/3,
-         pool_stats/1,
-         pool_utilization/1,
-         manual_start/0,
-         new_pool/1,
-         pool_child_spec/1,
-         rm_pool/1,
-         rm_group/1,
-         call_free_members/2,
-         call_free_members/3
-        ]).
+-export([
+    start/0,
+    stop/0
+]).
+
+-export([
+    accept_member/2,
+    start_link/1,
+    take_member/1,
+    take_member/2,
+    take_group_member/1,
+    return_group_member/2,
+    return_group_member/3,
+    return_member/2,
+    return_member/3,
+    pool_stats/1,
+    pool_utilization/1,
+    manual_start/0,
+    new_pool/1,
+    pool_child_spec/1,
+    rm_pool/1,
+    rm_group/1,
+    call_free_members/2,
+    call_free_members/3
+]).
 
 %% ------------------------------------------------------------------
 %% gen_server Function Exports
 %% ------------------------------------------------------------------
 
--export([init/1,
-         handle_continue/2,
-         handle_call/3,
-         handle_cast/2,
-         handle_info/2,
-         terminate/2,
-         code_change/3]).
+-export([
+    init/1,
+    handle_continue/2,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2,
+    terminate/2,
+    code_change/3
+]).
 
 %% ------------------------------------------------------------------
 %% Application API
@@ -159,16 +165,17 @@ rm_group(GroupName) ->
 -spec rm_group_members([pid()]) -> [atom()].
 rm_group_members(MemberPids) ->
     lists:foldl(
-      fun(MemberPid, Acc) ->
-              Pool = gen_server:call(MemberPid, dump_pool),
-              PoolName = Pool#pool.name,
-              case pooler_sup:rm_pool(PoolName) of
-                  ok -> Acc;
-                  _  -> [PoolName | Acc]
-              end
-      end,
-      [],
-      MemberPids).
+        fun(MemberPid, Acc) ->
+            Pool = gen_server:call(MemberPid, dump_pool),
+            PoolName = Pool#pool.name,
+            case pooler_sup:rm_pool(PoolName) of
+                ok -> Acc;
+                _ -> [PoolName | Acc]
+            end
+        end,
+        [],
+        MemberPids
+    ).
 
 %% @doc Get child spec described by the proplist `PoolConfig'.
 %%
@@ -201,7 +208,6 @@ take_member(PoolName) when is_atom(PoolName) orelse is_pid(PoolName) ->
 take_member(PoolName, Timeout) when is_atom(PoolName) orelse is_pid(PoolName) ->
     gen_server:call(PoolName, {take_member, time_as_millis(Timeout)}, infinity).
 
-
 %% @doc Take a member from a randomly selected member of the group
 %% `GroupName'. Returns `MemberPid' or `error_no_members'.  If no
 %% members are available in the randomly chosen pool, all other pools
@@ -271,11 +277,13 @@ return_group_member(_GroupName, MemberPid, Status) when is_pid(MemberPid) ->
 %% `Status' is 'fail', the member is destroyed and a new member is
 %% added to the pool in its place.
 -spec return_member(atom() | pid(), pid() | error_no_members, ok | fail) -> ok.
-return_member(PoolName, Pid, Status) when is_pid(Pid) andalso
-                                          (is_atom(PoolName) orelse
-                                           is_pid(PoolName)) andalso
-                                          (Status =:= ok orelse
-                                           Status =:= fail) ->
+return_member(PoolName, Pid, Status) when
+    is_pid(Pid) andalso
+        (is_atom(PoolName) orelse
+            is_pid(PoolName)) andalso
+        (Status =:= ok orelse
+            Status =:= fail)
+->
     gen_server:call(PoolName, {return_member, Pid, Status}, infinity),
     ok;
 return_member(_, error_no_members, _) ->
@@ -284,8 +292,10 @@ return_member(_, error_no_members, _) ->
 %% @doc Return a member to the pool so it can be reused.
 %%
 -spec return_member(atom() | pid(), pid() | error_no_members) -> ok.
-return_member(PoolName, Pid) when is_pid(Pid) andalso
-                                  (is_atom(PoolName) orelse is_pid(PoolName)) ->
+return_member(PoolName, Pid) when
+    is_pid(Pid) andalso
+        (is_atom(PoolName) orelse is_pid(PoolName))
+->
     gen_server:call(PoolName, {return_member, Pid, ok}, infinity),
     ok;
 return_member(_, error_no_members) ->
@@ -309,25 +319,27 @@ pool_utilization(PoolName) ->
 %% @doc Invokes `Fun' with arity 1 over all free members in pool with `PoolName'.
 %%
 -spec call_free_members(atom() | pid(), fun((pid()) -> term())) -> Res when
-      Res :: [{ok, term()} | {error, term()}].
-call_free_members(PoolName, Fun)
-  when (is_atom(PoolName) orelse is_pid(PoolName)) andalso is_function(Fun, 1) ->
+    Res :: [{ok, term()} | {error, term()}].
+call_free_members(PoolName, Fun) when
+    (is_atom(PoolName) orelse is_pid(PoolName)) andalso is_function(Fun, 1)
+->
     call_free_members(PoolName, Fun, infinity).
 
 %% @doc Invokes `Fun' with arity 1 over all free members in pool with `PoolName'.
 %% `Timeout' sets the timeout of gen_server call.
 -spec call_free_members(atom() | pid(), Fun, timeout()) -> Res when
-      Fun :: fun((pid()) -> term()),
-      Res :: [{ok, term()} | {error, term()}].
-call_free_members(PoolName, Fun, Timeout)
-  when (is_atom(PoolName) orelse is_pid(PoolName)) andalso is_function(Fun, 1) ->
+    Fun :: fun((pid()) -> term()),
+    Res :: [{ok, term()} | {error, term()}].
+call_free_members(PoolName, Fun, Timeout) when
+    (is_atom(PoolName) orelse is_pid(PoolName)) andalso is_function(Fun, 1)
+->
     gen_server:call(PoolName, {call_free_members, Fun}, Timeout).
 
 %% ------------------------------------------------------------------
 %% gen_server Function Definitions
 %% ------------------------------------------------------------------
 
-init(#pool{}=Pool) ->
+init(#pool{} = Pool) ->
     #pool{init_count = N} = Pool,
     MemberSup = pooler_pool_sup:member_sup_name(Pool),
     Pool1 = set_member_sup(Pool, MemberSup),
@@ -350,7 +362,6 @@ set_member_sup(#pool{} = Pool, MemberSup) ->
 
 handle_call({take_member, Timeout}, From = {APid, _}, #pool{} = Pool) when is_pid(APid) ->
     maybe_reply(take_member_from_pool_queued(Pool, From, Timeout));
-
 handle_call({return_member, Pid, Status}, {_CPid, _Tag}, Pool) ->
     {reply, ok, do_return_member(Pid, Status, Pool)};
 handle_call({accept_member, Pid}, _From, Pool) ->
@@ -364,23 +375,27 @@ handle_call(pool_utilization, _From, Pool) ->
 handle_call(dump_pool, _From, Pool) ->
     {reply, Pool, Pool};
 handle_call({call_free_members, Fun}, _From, #pool{free_pids = Pids} = Pool) ->
-  {reply, do_call_free_members(Fun, Pids), Pool};
+    {reply, do_call_free_members(Fun, Pids), Pool};
 handle_call(_Request, _From, Pool) ->
     {noreply, Pool}.
 
--spec handle_cast(_,_) -> {'noreply', _}.
+-spec handle_cast(_, _) -> {'noreply', _}.
 handle_cast(_Msg, Pool) ->
     {noreply, Pool}.
 
 -spec handle_info(_, _) -> {'noreply', _}.
-handle_info({requestor_timeout, From}, Pool = #pool{ queued_requestors = RequestorQueue }) ->
-    NewQueue = queue:filter(fun({RequestorFrom, _TRef}) when RequestorFrom =:= From ->
-                                    gen_server:reply(RequestorFrom, error_no_members),
-                                    false;
-                               ({_, _}) ->
-                                    true
-                            end, RequestorQueue),
-    {noreply, Pool#pool{ queued_requestors = NewQueue} };
+handle_info({requestor_timeout, From}, Pool = #pool{queued_requestors = RequestorQueue}) ->
+    NewQueue = queue:filter(
+        fun
+            ({RequestorFrom, _TRef}) when RequestorFrom =:= From ->
+                gen_server:reply(RequestorFrom, error_no_members),
+                false;
+            ({_, _}) ->
+                true
+        end,
+        RequestorQueue
+    ),
+    {noreply, Pool#pool{queued_requestors = NewQueue}};
 handle_info({'DOWN', MRef, process, Pid, Reason}, State) ->
     State1 =
         case dict:find(Pid, State#pool.all_members) of
@@ -389,13 +404,16 @@ handle_info({'DOWN', MRef, process, Pid, Reason}, State) ->
             error ->
                 case dict:find(Pid, State#pool.consumer_to_pid) of
                     {ok, {MRef, Pids}} ->
-                        IsOk = case Reason of
-                                   normal -> ok;
-                                   _Crash -> fail
-                               end,
+                        IsOk =
+                            case Reason of
+                                normal -> ok;
+                                _Crash -> fail
+                            end,
                         lists:foldl(
-                          fun(P, S) -> do_return_member(P, IsOk, S) end,
-                          State, Pids);
+                            fun(P, S) -> do_return_member(P, IsOk, S) end,
+                            State,
+                            Pids
+                        );
                     error ->
                         State
                 end
@@ -418,14 +436,17 @@ code_change(_OldVsn, State, _Extra) ->
 %% Internal Function Definitions
 %% ------------------------------------------------------------------
 
-do_accept_member({StarterPid, Pid},
-                 #pool{
-                    all_members = AllMembers,
-                    starting_members = StartingMembers0,
-                    member_start_timeout = StartTimeout
-                   } = Pool) when is_pid(Pid) ->
+do_accept_member(
+    {StarterPid, Pid},
+    #pool{
+        all_members = AllMembers,
+        starting_members = StartingMembers0,
+        member_start_timeout = StartTimeout
+    } = Pool
+) when is_pid(Pid) ->
     %% make sure we don't accept a timedout member
-    Pool1 = #pool{starting_members = StartingMembers} =
+    Pool1 =
+        #pool{starting_members = StartingMembers} =
         remove_stale_starting_members(Pool, StartingMembers0, StartTimeout),
     case lists:keymember(StarterPid, 1, StartingMembers) of
         false ->
@@ -443,29 +464,44 @@ do_accept_member({StarterPid, Pid},
             Entry = {MRef, free, os:timestamp()},
             AllMembers1 = store_all_members(Pid, Entry, AllMembers),
             pooler_starter:stop(StarterPid),
-            maybe_reply_with_pid(Pid, Pool1#pool{all_members = AllMembers1,
-                                                 starting_members = StartingMembers1})
+            maybe_reply_with_pid(Pid, Pool1#pool{
+                all_members = AllMembers1,
+                starting_members = StartingMembers1
+            })
     end;
-do_accept_member({StarterPid, _Reason},
-                 #pool{starting_members = StartingMembers0,
-                       member_start_timeout = StartTimeout} = Pool) ->
+do_accept_member(
+    {StarterPid, _Reason},
+    #pool{
+        starting_members = StartingMembers0,
+        member_start_timeout = StartTimeout
+    } = Pool
+) ->
     %% member start failed, remove in-flight ref and carry on.
     pooler_starter:stop(StarterPid),
-    Pool1 = #pool{starting_members = StartingMembers} =
-        remove_stale_starting_members(Pool, StartingMembers0,
-                                      StartTimeout),
+    Pool1 =
+        #pool{starting_members = StartingMembers} =
+        remove_stale_starting_members(
+            Pool,
+            StartingMembers0,
+            StartTimeout
+        ),
     StartingMembers1 = lists:keydelete(StarterPid, 1, StartingMembers),
     Pool1#pool{starting_members = StartingMembers1}.
 
-
-maybe_reply_with_pid(Pid,
-                     Pool = #pool{queued_requestors = QueuedRequestors,
-                                  free_pids = Free,
-                                  free_count = NumFree}) when is_pid(Pid) ->
+maybe_reply_with_pid(
+    Pid,
+    Pool = #pool{
+        queued_requestors = QueuedRequestors,
+        free_pids = Free,
+        free_count = NumFree
+    }
+) when is_pid(Pid) ->
     case queue:out(QueuedRequestors) of
         {empty, _} ->
-            Pool#pool{free_pids = [Pid | Free],
-                      free_count = NumFree + 1};
+            Pool#pool{
+                free_pids = [Pid | Free],
+                free_count = NumFree + 1
+            };
         {{value, {From = {APid, _}, TRef}}, NewQueuedRequestors} when is_pid(APid) ->
             reply_to_queued_requestor(TRef, Pid, From, NewQueuedRequestors, Pool)
     end.
@@ -479,50 +515,66 @@ reply_to_queued_requestor(TRef, Pid, From = {APid, _}, NewQueuedRequestors, Pool
     gen_server:reply(From, Pid),
     Pool1.
 
-
--spec take_member_bookkeeping(pid(),
-                              {pid(), _},
-                              [pid()] | p_requestor_queue(),
-                              #pool{}) -> #pool{}.
-take_member_bookkeeping(MemberPid,
-                        {CPid, _},
-                        Rest,
-                        Pool = #pool{in_use_count = NumInUse,
-                                     free_count = NumFree,
-                                     consumer_to_pid = CPMap,
-                                     all_members = AllMembers})
-  when is_pid(MemberPid),
-       is_pid(CPid),
-       is_list(Rest) ->
-    Pool#pool{free_pids = Rest,
-              in_use_count = NumInUse + 1,
-              free_count = NumFree - 1,
-              consumer_to_pid = add_member_to_consumer(MemberPid, CPid, CPMap),
-              all_members = set_cpid_for_member(MemberPid, CPid, AllMembers)
-             };
-take_member_bookkeeping(MemberPid,
-                        {ReplyPid, _Tag},
-                        NewQueuedRequestors,
-                        Pool = #pool{
-                                  in_use_count = NumInUse,
-                                  all_members = AllMembers,
-                                  consumer_to_pid = CPMap
-                                 }) ->
+-spec take_member_bookkeeping(
+    pid(),
+    {pid(), _},
+    [pid()] | p_requestor_queue(),
+    #pool{}
+) -> #pool{}.
+take_member_bookkeeping(
+    MemberPid,
+    {CPid, _},
+    Rest,
+    Pool = #pool{
+        in_use_count = NumInUse,
+        free_count = NumFree,
+        consumer_to_pid = CPMap,
+        all_members = AllMembers
+    }
+) when
+    is_pid(MemberPid),
+    is_pid(CPid),
+    is_list(Rest)
+->
     Pool#pool{
-      in_use_count = NumInUse + 1,
-      all_members = set_cpid_for_member(MemberPid, ReplyPid, AllMembers),
-      consumer_to_pid = add_member_to_consumer(MemberPid, ReplyPid, CPMap),
-      queued_requestors = NewQueuedRequestors
-     }.
-
--spec remove_stale_starting_members(#pool{}, [{reference(), erlang:timestamp()}],
-                                    time_spec()) -> #pool{}.
+        free_pids = Rest,
+        in_use_count = NumInUse + 1,
+        free_count = NumFree - 1,
+        consumer_to_pid = add_member_to_consumer(MemberPid, CPid, CPMap),
+        all_members = set_cpid_for_member(MemberPid, CPid, AllMembers)
+    };
+take_member_bookkeeping(
+    MemberPid,
+    {ReplyPid, _Tag},
+    NewQueuedRequestors,
+    Pool = #pool{
+        in_use_count = NumInUse,
+        all_members = AllMembers,
+        consumer_to_pid = CPMap
+    }
+) ->
+    Pool#pool{
+        in_use_count = NumInUse + 1,
+        all_members = set_cpid_for_member(MemberPid, ReplyPid, AllMembers),
+        consumer_to_pid = add_member_to_consumer(MemberPid, ReplyPid, CPMap),
+        queued_requestors = NewQueuedRequestors
+    }.
+
+-spec remove_stale_starting_members(
+    #pool{},
+    [{reference(), erlang:timestamp()}],
+    time_spec()
+) -> #pool{}.
 remove_stale_starting_members(Pool, StartingMembers, MaxAge) ->
     Now = os:timestamp(),
     MaxAgeSecs = time_as_secs(MaxAge),
-    FilteredStartingMembers = lists:foldl(fun(SM, AccIn) ->
-                         accumulate_starting_member_not_stale(Pool, Now, SM, MaxAgeSecs, AccIn)
-                 end, [], StartingMembers),
+    FilteredStartingMembers = lists:foldl(
+        fun(SM, AccIn) ->
+            accumulate_starting_member_not_stale(Pool, Now, SM, MaxAgeSecs, AccIn)
+        end,
+        [],
+        StartingMembers
+    ),
     Pool#pool{starting_members = FilteredStartingMembers}.
 
 accumulate_starting_member_not_stale(Pool, Now, SM = {Pid, StartTime}, MaxAgeSecs, AccIn) ->
@@ -530,9 +582,13 @@ accumulate_starting_member_not_stale(Pool, Now, SM = {Pid, StartTime}, MaxAgeSec
         true ->
             [SM | AccIn];
         false ->
-            ?LOG_ERROR(#{label => "starting member timeout",
-                         pool => Pool#pool.name},
-                       #{domain => [pooler]}),
+            ?LOG_ERROR(
+                #{
+                    label => "starting member timeout",
+                    pool => Pool#pool.name
+                },
+                #{domain => [pooler]}
+            ),
             send_metric(Pool, starting_member_timeout, {inc, 1}, counter),
             pooler_starter:stop_member_async(Pid),
             AccIn
@@ -541,49 +597,59 @@ accumulate_starting_member_not_stale(Pool, Now, SM = {Pid, StartTime}, MaxAgeSec
 init_members_sync(N, #pool{name = PoolName} = Pool) ->
     Self = self(),
     StartTime = os:timestamp(),
-    StartRefs = [ {pooler_starter:start_member(Pool, Self), StartTime}
-                  || _I <- lists:seq(1, N) ],
+    StartRefs = [
+        {pooler_starter:start_member(Pool, Self), StartTime}
+     || _I <- lists:seq(1, N)
+    ],
     Pool1 = Pool#pool{starting_members = StartRefs},
     case collect_init_members(Pool1) of
         timeout ->
-            ?LOG_ERROR(#{label => "exceeded timeout waiting for members",
-                         pool => PoolName,
-                         init_count => Pool1#pool.init_count},
-                       #{domain => [pooler]}),
+            ?LOG_ERROR(
+                #{
+                    label => "exceeded timeout waiting for members",
+                    pool => PoolName,
+                    init_count => Pool1#pool.init_count
+                },
+                #{domain => [pooler]}
+            ),
             error({timeout, "unable to start members"});
         #pool{} = Pool2 ->
             {ok, Pool2}
     end.
 
-collect_init_members(#pool{starting_members = Empty} = Pool)
-  when Empty =:= [] ->
+collect_init_members(#pool{starting_members = Empty} = Pool) when
+    Empty =:= []
+->
     Pool;
 collect_init_members(#pool{member_start_timeout = StartTimeout} = Pool) ->
     Timeout = time_as_millis(StartTimeout),
     receive
         {accept_member, {Ref, Member}} ->
             collect_init_members(do_accept_member({Ref, Member}, Pool))
-    after
-        Timeout ->
-            timeout
+    after Timeout ->
+        timeout
     end.
 
 -spec take_member_from_pool(#pool{}, {pid(), term()}) ->
-                                   {error_no_members | pid(), #pool{}}.
-take_member_from_pool(#pool{init_count = InitCount,
-                            max_count = Max,
-                            free_pids = Free,
-                            in_use_count = NumInUse,
-                            free_count = NumFree,
-                            starting_members = StartingMembers,
-                            member_start_timeout = StartTimeout} = Pool,
-                      From) ->
+    {error_no_members | pid(), #pool{}}.
+take_member_from_pool(
+    #pool{
+        init_count = InitCount,
+        max_count = Max,
+        free_pids = Free,
+        in_use_count = NumInUse,
+        free_count = NumFree,
+        starting_members = StartingMembers,
+        member_start_timeout = StartTimeout
+    } = Pool,
+    From
+) ->
     send_metric(Pool, take_rate, 1, meter),
     Pool1 = remove_stale_starting_members(Pool, StartingMembers, StartTimeout),
     NonStaleStartingMemberCount = length(Pool1#pool.starting_members),
     NumCanAdd = Max - (NumInUse + NumFree + NonStaleStartingMemberCount),
     case Free of
-        [] when NumCanAdd =< 0  ->
+        [] when NumCanAdd =< 0 ->
             send_metric(Pool, error_no_members_count, {inc, 1}, counter),
             send_metric(Pool, events, error_no_members, history),
             {error_no_members, Pool1};
@@ -600,30 +666,39 @@ take_member_from_pool(#pool{init_count = InitCount,
             send_metric(Pool, error_no_members_count, {inc, 1}, counter),
             send_metric(Pool, events, error_no_members, history),
             {error_no_members, Pool2};
-        [Pid|Rest] ->
+        [Pid | Rest] ->
             Pool2 = take_member_bookkeeping(Pid, From, Rest, Pool1),
-            Pool3 = case Pool2#pool.auto_grow_threshold of
-                        N when is_integer(N) andalso
-                               Pool2#pool.free_count =< N andalso
-                               NumCanAdd > 0 ->
-                            NumToAdd = max(min(InitCount - NonStaleStartingMemberCount, NumCanAdd), 0),
-                            add_members_async(NumToAdd, Pool2);
-                        _ ->
-                            Pool2
-                    end,
+            Pool3 =
+                case Pool2#pool.auto_grow_threshold of
+                    N when
+                        is_integer(N) andalso
+                            Pool2#pool.free_count =< N andalso
+                            NumCanAdd > 0
+                    ->
+                        NumToAdd = max(min(InitCount - NonStaleStartingMemberCount, NumCanAdd), 0),
+                        add_members_async(NumToAdd, Pool2);
+                    _ ->
+                        Pool2
+                end,
             send_metric(Pool, in_use_count, Pool3#pool.in_use_count, histogram),
             send_metric(Pool, free_count, Pool3#pool.free_count, histogram),
             {Pid, Pool3}
     end.
 
--spec take_member_from_pool_queued(#pool{},
-                                   {pid(), _},
-                                   non_neg_integer()) ->
-                                   {error_no_members | queued | pid(), #pool{}}.
-take_member_from_pool_queued(Pool0 = #pool{queue_max = QMax,
-                                           queued_requestors = Requestors},
-                             From = {CPid, _},
-                             Timeout) when is_pid(CPid) ->
+-spec take_member_from_pool_queued(
+    #pool{},
+    {pid(), _},
+    non_neg_integer()
+) ->
+    {error_no_members | queued | pid(), #pool{}}.
+take_member_from_pool_queued(
+    Pool0 = #pool{
+        queue_max = QMax,
+        queued_requestors = Requestors
+    },
+    From = {CPid, _},
+    Timeout
+) when is_pid(CPid) ->
     case {take_member_from_pool(Pool0, From), queue:len(Requestors)} of
         {{error_no_members, Pool1}, QLen} when QLen >= QMax ->
             send_metric(Pool1, events, error_no_members, history),
@@ -644,32 +719,52 @@ take_member_from_pool_queued(Pool0 = #pool{queue_max = QMax,
 %% `starting_members'.
 add_members_async(Count, #pool{starting_members = StartingMembers} = Pool) ->
     StartTime = os:timestamp(),
-    StartRefs = [ {pooler_starter:start_member(Pool), StartTime}
-                  || _I <- lists:seq(1, Count) ],
+    StartRefs = [
+        {pooler_starter:start_member(Pool), StartTime}
+     || _I <- lists:seq(1, Count)
+    ],
     Pool#pool{starting_members = StartRefs ++ StartingMembers}.
 
 -spec do_return_member(pid(), ok | fail, #pool{}) -> #pool{}.
-do_return_member(Pid, ok, #pool{name = PoolName,
-                                all_members = AllMembers,
-                                queued_requestors = QueuedRequestors} = Pool) ->
+do_return_member(
+    Pid,
+    ok,
+    #pool{
+        name = PoolName,
+        all_members = AllMembers,
+        queued_requestors = QueuedRequestors
+    } = Pool
+) ->
     clean_group_table(Pid, Pool),
     case dict:find(Pid, AllMembers) of
         {ok, {_, free, _}} ->
-            ?LOG_WARNING(#{label => "ignored return of free member",
-                           pool => PoolName,
-                           pid => Pid},
-                         #{domain => [pooler]}),
+            ?LOG_WARNING(
+                #{
+                    label => "ignored return of free member",
+                    pool => PoolName,
+                    pid => Pid
+                },
+                #{domain => [pooler]}
+            ),
             Pool;
         {ok, {MRef, CPid, _}} ->
-            #pool{free_pids = Free, in_use_count = NumInUse,
-                  free_count = NumFree} = Pool,
+            #pool{
+                free_pids = Free,
+                in_use_count = NumInUse,
+                free_count = NumFree
+            } = Pool,
             Pool1 = Pool#pool{in_use_count = NumInUse - 1},
             Entry = {MRef, free, os:timestamp()},
-            Pool2 = Pool1#pool{all_members = store_all_members(Pid, Entry, AllMembers),
-                       consumer_to_pid = cpmap_remove(Pid, CPid,
-                                                      Pool1#pool.consumer_to_pid)},
+            Pool2 = Pool1#pool{
+                all_members = store_all_members(Pid, Entry, AllMembers),
+                consumer_to_pid = cpmap_remove(
+                    Pid,
+                    CPid,
+                    Pool1#pool.consumer_to_pid
+                )
+            },
             case queue:out(QueuedRequestors) of
-                {empty, _ } ->
+                {empty, _} ->
                     Pool2#pool{free_pids = [Pid | Free], free_count = NumFree + 1};
                 {{value, {From = {APid, _}, TRef}}, NewQueuedRequestors} when is_pid(APid) ->
                     reply_to_queued_requestor(TRef, Pid, From, NewQueuedRequestors, Pool2)
@@ -708,7 +803,7 @@ cpmap_remove(Pid, CPid, CPMap) ->
         {ok, {MRef, Pids0}} ->
             Pids1 = lists:delete(Pid, Pids0),
             case Pids1 of
-                [_H|_T] ->
+                [_H | _T] ->
                     dict:store(CPid, {MRef, Pids1}, CPMap);
                 [] ->
                     %% no more members for this consumer
@@ -727,10 +822,12 @@ cpmap_remove(Pid, CPid, CPMap) ->
 %
 -spec remove_pid(pid(), #pool{}) -> #pool{}.
 remove_pid(Pid, Pool) ->
-    #pool{name = PoolName,
-          all_members = AllMembers,
-          consumer_to_pid = CPMap,
-          stop_mfa = StopMFA} = Pool,
+    #pool{
+        name = PoolName,
+        all_members = AllMembers,
+        consumer_to_pid = CPMap,
+        stop_mfa = StopMFA
+    } = Pool,
     case dict:find(Pid, AllMembers) of
         {ok, {MRef, free, _Time}} ->
             % remove an unused member
@@ -748,28 +845,40 @@ remove_pid(Pid, Pool) ->
             Pool1 = Pool#pool{in_use_count = Pool#pool.in_use_count - 1},
             terminate_pid(PoolName, Pid, StopMFA),
             send_metric(Pool1, killed_in_use_count, {inc, 1}, counter),
-            Pool1#pool{consumer_to_pid = cpmap_remove(Pid, CPid, CPMap),
-                       all_members = dict:erase(Pid, AllMembers)};
+            Pool1#pool{
+                consumer_to_pid = cpmap_remove(Pid, CPid, CPMap),
+                all_members = dict:erase(Pid, AllMembers)
+            };
         error ->
-            ?LOG_ERROR(#{label => unknown_pid,
-                         pool => PoolName,
-                         pid => Pid},
-                       #{domain => [pooler]}),
+            ?LOG_ERROR(
+                #{
+                    label => unknown_pid,
+                    pool => PoolName,
+                    pid => Pid
+                },
+                #{domain => [pooler]}
+            ),
             send_metric(Pool, events, unknown_pid, history),
             Pool
     end.
 
--spec store_all_members(pid(),
-                        {reference(), free | pid(), {_, _, _}}, dict:dict()) -> dict:dict().
+-spec store_all_members(
+    pid(),
+    {reference(), free | pid(), {_, _, _}},
+    dict:dict()
+) -> dict:dict().
 store_all_members(Pid, Val = {_MRef, _CPid, _Time}, AllMembers) ->
     dict:store(Pid, Val, AllMembers).
 
 -spec set_cpid_for_member(pid(), pid(), dict:dict()) -> dict:dict().
 set_cpid_for_member(MemberPid, CPid, AllMembers) ->
-    dict:update(MemberPid,
-                fun({MRef, free, Time = {_, _, _}}) ->
-                        {MRef, CPid, Time}
-                end, AllMembers).
+    dict:update(
+        MemberPid,
+        fun({MRef, free, Time = {_, _, _}}) ->
+            {MRef, CPid, Time}
+        end,
+        AllMembers
+    ).
 
 -spec add_member_to_consumer(pid(), pid(), dict:dict()) -> dict:dict().
 add_member_to_consumer(MemberPid, CPid, CPMap) ->
@@ -792,30 +901,40 @@ cull_members_from_pool(#pool{init_count = C, max_count = C} = Pool) ->
     %% add capacity and should not schedule culling regardless of
     %% cull_interval config.
     Pool;
-cull_members_from_pool(#pool{name = PoolName,
-                             free_count = FreeCount,
-                             init_count = InitCount,
-                             in_use_count = InUseCount,
-                             cull_interval = Delay,
-                             max_age = MaxAge,
-                             all_members = AllMembers} = Pool) ->
+cull_members_from_pool(
+    #pool{
+        name = PoolName,
+        free_count = FreeCount,
+        init_count = InitCount,
+        in_use_count = InUseCount,
+        cull_interval = Delay,
+        max_age = MaxAge,
+        all_members = AllMembers
+    } = Pool
+) ->
     MaxCull = FreeCount - (InitCount - InUseCount),
-    Pool1 = case MaxCull > 0 of
-                true ->
-                    MemberInfo = member_info(Pool#pool.free_pids, AllMembers),
-                    ExpiredMembers =
-                        expired_free_members(MemberInfo, os:timestamp(), MaxAge),
-                    CullList = lists:sublist(ExpiredMembers, MaxCull),
-                    lists:foldl(fun({CullMe, _}, S) -> remove_pid(CullMe, S) end,
-                                Pool, CullList);
-                false ->
-                    Pool
-            end,
+    Pool1 =
+        case MaxCull > 0 of
+            true ->
+                MemberInfo = member_info(Pool#pool.free_pids, AllMembers),
+                ExpiredMembers =
+                    expired_free_members(MemberInfo, os:timestamp(), MaxAge),
+                CullList = lists:sublist(ExpiredMembers, MaxCull),
+                lists:foldl(
+                    fun({CullMe, _}, S) -> remove_pid(CullMe, S) end,
+                    Pool,
+                    CullList
+                );
+            false ->
+                Pool
+        end,
     schedule_cull(PoolName, Delay),
     Pool1.
 
--spec schedule_cull(PoolName :: atom() | pid(),
-                    Delay :: time_spec()) -> reference().
+-spec schedule_cull(
+    PoolName :: atom() | pid(),
+    Delay :: time_spec()
+) -> reference().
 %% @doc Schedule a pool cleaning or "cull" for `PoolName' in which
 %% members older than `max_age' will be removed until the pool has
 %% `init_count' members. Uses `erlang:send_after/3' for light-weight
@@ -828,56 +947,97 @@ schedule_cull(PoolName, Delay) ->
 
 -spec member_info([pid()], dict:dict()) -> [{pid(), member_info()}].
 member_info(Pids, AllMembers) ->
-    [ {P, dict:fetch(P, AllMembers)} || P <- Pids ].
+    [{P, dict:fetch(P, AllMembers)} || P <- Pids].
 
--spec expired_free_members(Members :: [{pid(), member_info()}],
-                           Now :: {_, _, _},
-                           MaxAge :: time_spec()) -> [{pid(), free_member_info()}].
+-spec expired_free_members(
+    Members :: [{pid(), member_info()}],
+    Now :: {_, _, _},
+    MaxAge :: time_spec()
+) -> [{pid(), free_member_info()}].
 expired_free_members(Members, Now, MaxAge) ->
     MaxMicros = time_as_micros(MaxAge),
-    [ MI || MI = {_, {_, free, LastReturn}} <- Members,
-            timer:now_diff(Now, LastReturn) >= MaxMicros ].
+    [
+        MI
+     || MI = {_, {_, free, LastReturn}} <- Members,
+        timer:now_diff(Now, LastReturn) >= MaxMicros
+    ].
 
 %% Send a metric using the metrics module from application config or
 %% do nothing.
--spec send_metric(Pool  :: #pool{},
-                  Label :: atom(),
-                  Value :: metric_value(),
-                  Type  :: metric_type()) -> ok.
+-spec send_metric(
+    Pool :: #pool{},
+    Label :: atom(),
+    Value :: metric_value(),
+    Type :: metric_type()
+) -> ok.
 send_metric(#pool{metrics_mod = pooler_no_metrics}, _Label, _Value, _Type) ->
     ok;
-send_metric(#pool{name = PoolName, metrics_mod = MetricsMod,
-                  metrics_api = exometer}, Label, {inc, Value}, counter) ->
+send_metric(
+    #pool{
+        name = PoolName,
+        metrics_mod = MetricsMod,
+        metrics_api = exometer
+    },
+    Label,
+    {inc, Value},
+    counter
+) ->
     MetricName = pool_metric_exometer(PoolName, Label),
     MetricsMod:update_or_create(MetricName, Value, counter, []),
     ok;
 % Exometer does not support 'history' type metrics right now.
-send_metric(#pool{name = _PoolName, metrics_mod = _MetricsMod,
-                  metrics_api = exometer}, _Label, _Value, history) ->
+send_metric(
+    #pool{
+        name = _PoolName,
+        metrics_mod = _MetricsMod,
+        metrics_api = exometer
+    },
+    _Label,
+    _Value,
+    history
+) ->
     ok;
-send_metric(#pool{name = PoolName, metrics_mod = MetricsMod,
-                   metrics_api = exometer}, Label, Value, Type) ->
+send_metric(
+    #pool{
+        name = PoolName,
+        metrics_mod = MetricsMod,
+        metrics_api = exometer
+    },
+    Label,
+    Value,
+    Type
+) ->
     MetricName = pool_metric_exometer(PoolName, Label),
     MetricsMod:update_or_create(MetricName, Value, Type, []),
     ok;
 %folsom API is the default one.
-send_metric(#pool{name = PoolName, metrics_mod = MetricsMod, metrics_api = folsom},
-                   Label, Value, Type) ->
+send_metric(
+    #pool{name = PoolName, metrics_mod = MetricsMod, metrics_api = folsom},
+    Label,
+    Value,
+    Type
+) ->
     MetricName = pool_metric(PoolName, Label),
     MetricsMod:notify(MetricName, Value, Type),
     ok.
 
 -spec pool_metric(atom(), atom()) -> binary().
 pool_metric(PoolName, Metric) ->
-    iolist_to_binary([<<"pooler.">>, atom_to_binary(PoolName, utf8),
-                      ".", atom_to_binary(Metric, utf8)]).
+    iolist_to_binary([
+        <<"pooler.">>,
+        atom_to_binary(PoolName, utf8),
+        ".",
+        atom_to_binary(Metric, utf8)
+    ]).
 
 %% Exometer metric names are lists, not binaries.
 -spec pool_metric_exometer(atom(), atom()) -> nonempty_list(binary()).
 pool_metric_exometer(PoolName, Metric) ->
-    [<<"pooler">>, atom_to_binary(PoolName, utf8),
-     atom_to_binary(Metric, utf8)].
-
+    [
+        <<"pooler">>,
+        atom_to_binary(PoolName, utf8),
+        atom_to_binary(Metric, utf8)
+    ].
 
 -spec time_as_secs(time_spec()) -> non_neg_integer().
 time_as_secs({Time, Unit}) ->
@@ -891,7 +1051,6 @@ time_as_millis({Time, Unit}) ->
 time_as_millis(Time) when is_integer(Time) ->
     Time.
 
-
 -spec time_as_micros(time_spec()) -> non_neg_integer().
 %% @doc Convert time unit into microseconds
 time_as_micros({Time, min}) ->
@@ -907,7 +1066,7 @@ secs_between({Mega1, Secs1, _}, {Mega2, Secs2, _}) ->
     (Mega2 - Mega1) * 1000000 + (Secs2 - Secs1).
 
 -spec maybe_reply({'queued' | 'error_no_members' | pid(), #pool{}}) ->
-                         {noreply, #pool{}} | {reply, 'error_no_members' | pid(), #pool{}}.
+    {noreply, #pool{}} | {reply, 'error_no_members' | pid(), #pool{}}.
 maybe_reply({Member, NewPool}) ->
     case Member of
         queued ->
@@ -934,75 +1093,86 @@ terminate_pid(PoolName, Pid, {Mod, Fun, Args}) when is_list(Args) ->
     end.
 
 replace_placeholders(Name, Pid, Args) ->
-  [case Arg of
-     ?POOLER_POOL_NAME ->
-       pooler_pool_sup:build_member_sup_name(Name);
-     ?POOLER_PID ->
-       Pid;
-     _ ->
-       Arg
-   end || Arg <- Args].
-
-compute_utilization(#pool{max_count = MaxCount,
-                          in_use_count = InUseCount,
-                          free_count = FreeCount,
-                          queued_requestors = Queue,
-                          queue_max = QueueMax}) ->
-    [{max_count, MaxCount},
-      {in_use_count, InUseCount},
-      {free_count, FreeCount},
-      {queued_count, queue:len(Queue)}, %% Note not O(n), so in pathological cases this might be expensive
-      {queue_max, QueueMax}].
+    [
+        case Arg of
+            ?POOLER_POOL_NAME ->
+                pooler_pool_sup:build_member_sup_name(Name);
+            ?POOLER_PID ->
+                Pid;
+            _ ->
+                Arg
+        end
+     || Arg <- Args
+    ].
+
+compute_utilization(#pool{
+    max_count = MaxCount,
+    in_use_count = InUseCount,
+    free_count = FreeCount,
+    queued_requestors = Queue,
+    queue_max = QueueMax
+}) ->
+    [
+        {max_count, MaxCount},
+        {in_use_count, InUseCount},
+        {free_count, FreeCount},
+        %% Note not O(n), so in pathological cases this might be expensive
+        {queued_count, queue:len(Queue)},
+        {queue_max, QueueMax}
+    ].
 
 do_call_free_members(Fun, Pids) ->
     [do_call_free_member(Fun, P) || P <- Pids].
 
 do_call_free_member(Fun, Pid) ->
-    try {ok, Fun(Pid)}
+    try
+        {ok, Fun(Pid)}
     catch
         _Class:Reason ->
             {error, Reason}
     end.
 
--ifdef(OTP_RELEASE). % >= OTP-21
+% >= OTP-21
+-ifdef(OTP_RELEASE).
 -if(?OTP_RELEASE >= 23).
 -define(USE_PG_NOT_PG2, true).
 -else.
 -undef(USE_PG_NOT_PG2).
 -endif.
--else. % < OTP-21
+% < OTP-21
+-else.
 -undef(USE_PG_NOT_PG2).
 -endif.
 
 -ifdef(USE_PG_NOT_PG2).
 
 pg_get_local_members(GroupName) ->
-  pg:get_local_members(GroupName).
+    pg:get_local_members(GroupName).
 
 pg_delete(_GroupName) ->
-  ok.
+    ok.
 
 pg_create(_Group) ->
-  ok.
+    ok.
 
 pg_join(Group, Pid) ->
-  pg:join(Group, Pid).
+    pg:join(Group, Pid).
 
 -else.
 
 pg_get_local_members(GroupName) ->
-  case pg2:get_local_members(GroupName) of
-    {error, {no_such_group, GroupName}} -> [];
-    Pids -> Pids
-  end.
+    case pg2:get_local_members(GroupName) of
+        {error, {no_such_group, GroupName}} -> [];
+        Pids -> Pids
+    end.
 
 pg_delete(GroupName) ->
-  pg2:delete(GroupName).
+    pg2:delete(GroupName).
 
 pg_create(Group) ->
-  pg2:create(Group).
+    pg2:create(Group).
 
 pg_join(Group, Pid) ->
-  pg2:join(Group, Pid).
+    pg2:join(Group, Pid).
 
 -endif.

+ 80 - 82
src/pooler.hrl

@@ -7,9 +7,7 @@
 -define(DEFAULT_POOLER_QUEUE_MAX, 50).
 -define(POOLER_POOL_NAME, '$pooler_pool_name').
 -define(POOLER_PID, '$pooler_pid').
--define(DEFAULT_STOP_MFA, {supervisor,
-                           terminate_child,
-                           [?POOLER_POOL_NAME, ?POOLER_PID]}).
+-define(DEFAULT_STOP_MFA, {supervisor, terminate_child, [?POOLER_POOL_NAME, ?POOLER_PID]}).
 
 -type member_info() :: {string(), free | pid(), {_, _, _}}.
 -type free_member_info() :: {string(), free, {_, _, _}}.
@@ -19,85 +17,85 @@
 -type p_requestor_queue() :: queue:queue({{pid(), _}, timer:tref()}).
 
 -record(pool, {
-          name             :: atom(),
-          group            :: atom(),
-          max_count = 100  :: non_neg_integer(),
-          init_count = 10  :: non_neg_integer(),
-          start_mfa        :: {atom(), atom(), [term()]},
-          free_pids = []   :: [pid()],
-          in_use_count = 0 :: non_neg_integer(),
-          free_count = 0   :: non_neg_integer(),
-          %% The number times to attempt adding a pool member if the
-          %% pool size is below max_count and there are no free
-          %% members. After this many tries, error_no_members will be
-          %% returned by a call to take_member. NOTE: this value
-          %% should be >= 2 or else the pool will not grow on demand
-          %% when max_count is larger than init_count.
-          add_member_retry = ?DEFAULT_ADD_RETRY :: non_neg_integer(),
-
-          %% The interval to schedule a cull message. Both
-          %% 'cull_interval' and 'max_age' are specified using a
-          %% `time_spec()' type.
-          cull_interval = ?DEFAULT_CULL_INTERVAL :: time_spec(),
-          %% The maximum age for members.
-          max_age = ?DEFAULT_MAX_AGE             :: time_spec(),
-
-          %% The supervisor used to start new members
-          member_sup :: atom() | pid(),
-
-          %% The supervisor used to start starter servers that start
-          %% new members. This is what enables async member starts.
-          starter_sup :: atom() | pid(),
-
-          %% Maps member pid to a tuple of the form:
-          %% {MonitorRef, Status, Time},
-          %% where MonitorRef is a monitor reference for the member,,
-          %% Status is either 'free' or the consumer pid, and Time is
-          %% an Erlang timestamp that records when the member became
-          %% free.
-
-          all_members = dict:new()     :: dict:dict(),
-
-          %% Maps consumer pid to a tuple of the form:
-          %% {MonitorRef, MemberList} where MonitorRef is a monitor
-          %% reference for the consumer and MemberList is a list of
-          %% members being consumed.
-          consumer_to_pid = dict:new() :: dict:dict(),
-
-          %% A list of `{References, Timestamp}' tuples representing
-          %% new member start requests that are in-flight. The
-          %% timestamp records when the start request was initiated
-          %% and is used to implement start timeout.
-          starting_members = [] :: [{reference(), erlang:timestamp()}],
-
-          %% The maximum amount of time to allow for member start.
-          member_start_timeout = ?DEFAULT_MEMBER_START_TIMEOUT :: time_spec(),
-
-          %% The optional threshold at which more members will be started if
-          %% free_count drops to this value.  Normally undefined, but may be
-          %% set to a non-negative integer in order to enable "anticipatory"
-          %% behavior (start members before they're actually needed).
-          auto_grow_threshold = ?DEFAULT_AUTO_GROW_THRESHOLD :: undefined | non_neg_integer(),
-
-          %% Stop callback to gracefully attempt to terminate pool members.
-          %% The list of arguments must contain the fixed atom '$pooler_pid'.
-          stop_mfa = ?DEFAULT_STOP_MFA :: {atom(), atom(), [term()]},
-
-          %% The module to use for collecting metrics. If set to
-          %% 'pooler_no_metrics', then metric sending calls do
-          %% nothing. A typical value to actually capture metrics is
-          %% folsom_metrics.
-          metrics_mod = pooler_no_metrics :: atom(),
-
-          %% The API used to call the metrics system. It supports both Folsom
-          %% and Exometer format.
-          metrics_api = folsom :: 'folsom' | 'exometer',
-
-          %% A queue of requestors for blocking take member requests
-          queued_requestors = queue:new() :: p_requestor_queue(),
-          %% The max depth of the queue
-          queue_max = 50 :: non_neg_integer()
-         }).
+    name :: atom(),
+    group :: atom(),
+    max_count = 100 :: non_neg_integer(),
+    init_count = 10 :: non_neg_integer(),
+    start_mfa :: {atom(), atom(), [term()]},
+    free_pids = [] :: [pid()],
+    in_use_count = 0 :: non_neg_integer(),
+    free_count = 0 :: non_neg_integer(),
+    %% The number times to attempt adding a pool member if the
+    %% pool size is below max_count and there are no free
+    %% members. After this many tries, error_no_members will be
+    %% returned by a call to take_member. NOTE: this value
+    %% should be >= 2 or else the pool will not grow on demand
+    %% when max_count is larger than init_count.
+    add_member_retry = ?DEFAULT_ADD_RETRY :: non_neg_integer(),
+
+    %% The interval to schedule a cull message. Both
+    %% 'cull_interval' and 'max_age' are specified using a
+    %% `time_spec()' type.
+    cull_interval = ?DEFAULT_CULL_INTERVAL :: time_spec(),
+    %% The maximum age for members.
+    max_age = ?DEFAULT_MAX_AGE :: time_spec(),
+
+    %% The supervisor used to start new members
+    member_sup :: atom() | pid(),
+
+    %% The supervisor used to start starter servers that start
+    %% new members. This is what enables async member starts.
+    starter_sup :: atom() | pid(),
+
+    %% Maps member pid to a tuple of the form:
+    %% {MonitorRef, Status, Time},
+    %% where MonitorRef is a monitor reference for the member,,
+    %% Status is either 'free' or the consumer pid, and Time is
+    %% an Erlang timestamp that records when the member became
+    %% free.
+
+    all_members = dict:new() :: dict:dict(),
+
+    %% Maps consumer pid to a tuple of the form:
+    %% {MonitorRef, MemberList} where MonitorRef is a monitor
+    %% reference for the consumer and MemberList is a list of
+    %% members being consumed.
+    consumer_to_pid = dict:new() :: dict:dict(),
+
+    %% A list of `{References, Timestamp}' tuples representing
+    %% new member start requests that are in-flight. The
+    %% timestamp records when the start request was initiated
+    %% and is used to implement start timeout.
+    starting_members = [] :: [{reference(), erlang:timestamp()}],
+
+    %% The maximum amount of time to allow for member start.
+    member_start_timeout = ?DEFAULT_MEMBER_START_TIMEOUT :: time_spec(),
+
+    %% The optional threshold at which more members will be started if
+    %% free_count drops to this value.  Normally undefined, but may be
+    %% set to a non-negative integer in order to enable "anticipatory"
+    %% behavior (start members before they're actually needed).
+    auto_grow_threshold = ?DEFAULT_AUTO_GROW_THRESHOLD :: undefined | non_neg_integer(),
+
+    %% Stop callback to gracefully attempt to terminate pool members.
+    %% The list of arguments must contain the fixed atom '$pooler_pid'.
+    stop_mfa = ?DEFAULT_STOP_MFA :: {atom(), atom(), [term()]},
+
+    %% The module to use for collecting metrics. If set to
+    %% 'pooler_no_metrics', then metric sending calls do
+    %% nothing. A typical value to actually capture metrics is
+    %% folsom_metrics.
+    metrics_mod = pooler_no_metrics :: atom(),
+
+    %% The API used to call the metrics system. It supports both Folsom
+    %% and Exometer format.
+    metrics_api = folsom :: 'folsom' | 'exometer',
+
+    %% A queue of requestors for blocking take member requests
+    queued_requestors = queue:new() :: p_requestor_queue(),
+    %% The max depth of the queue
+    queue_max = 50 :: non_neg_integer()
+}).
 
 -define(gv(X, Y), proplists:get_value(X, Y)).
 -define(gv(X, Y, D), proplists:get_value(X, Y, D)).

+ 15 - 14
src/pooler_config.erl

@@ -11,20 +11,21 @@
 -spec list_to_pool([{atom(), term()}]) -> #pool{}.
 list_to_pool(P) ->
     #pool{
-       name              = req(name, P),
-       group             = ?gv(group, P),
-       max_count         = req(max_count, P),
-       init_count        = req(init_count, P),
-       start_mfa         = req(start_mfa, P),
-       add_member_retry  = ?gv(add_member_retry, P, ?DEFAULT_ADD_RETRY),
-       cull_interval     = ?gv(cull_interval, P, ?DEFAULT_CULL_INTERVAL),
-       max_age           = ?gv(max_age, P, ?DEFAULT_MAX_AGE),
-       member_start_timeout = ?gv(member_start_timeout, P, ?DEFAULT_MEMBER_START_TIMEOUT),
-       auto_grow_threshold = ?gv(auto_grow_threshold, P, ?DEFAULT_AUTO_GROW_THRESHOLD),
-       stop_mfa          = ?gv(stop_mfa, P, ?DEFAULT_STOP_MFA),
-       metrics_mod       = ?gv(metrics_mod, P, pooler_no_metrics),
-       metrics_api       = ?gv(metrics_api, P, folsom),
-       queue_max         = ?gv(queue_max, P, ?DEFAULT_POOLER_QUEUE_MAX)}.
+        name = req(name, P),
+        group = ?gv(group, P),
+        max_count = req(max_count, P),
+        init_count = req(init_count, P),
+        start_mfa = req(start_mfa, P),
+        add_member_retry = ?gv(add_member_retry, P, ?DEFAULT_ADD_RETRY),
+        cull_interval = ?gv(cull_interval, P, ?DEFAULT_CULL_INTERVAL),
+        max_age = ?gv(max_age, P, ?DEFAULT_MAX_AGE),
+        member_start_timeout = ?gv(member_start_timeout, P, ?DEFAULT_MEMBER_START_TIMEOUT),
+        auto_grow_threshold = ?gv(auto_grow_threshold, P, ?DEFAULT_AUTO_GROW_THRESHOLD),
+        stop_mfa = ?gv(stop_mfa, P, ?DEFAULT_STOP_MFA),
+        metrics_mod = ?gv(metrics_mod, P, pooler_no_metrics),
+        metrics_api = ?gv(metrics_api, P, folsom),
+        queue_max = ?gv(queue_max, P, ?DEFAULT_POOLER_QUEUE_MAX)
+    }.
 
 %% Return `Value' for `Key' in proplist `P' or crashes with an
 %% informative message if no value is found.

+ 12 - 10
src/pooler_pool_sup.erl

@@ -2,10 +2,13 @@
 
 -behaviour(supervisor).
 
--export([start_link/1, init/1,
-         pool_sup_name/1,
-         member_sup_name/1,
-         build_member_sup_name/1]).
+-export([
+    start_link/1,
+    init/1,
+    pool_sup_name/1,
+    member_sup_name/1,
+    build_member_sup_name/1
+]).
 
 -include("pooler.hrl").
 
@@ -14,13 +17,12 @@ start_link(#pool{} = Pool) ->
     supervisor:start_link({local, SupName}, ?MODULE, Pool).
 
 init(#pool{} = Pool) ->
-    PoolerSpec = {pooler,
-                  {pooler, start_link, [Pool]},
-                  transient,  5000, worker, [pooler]},
+    PoolerSpec = {pooler, {pooler, start_link, [Pool]}, transient, 5000, worker, [pooler]},
     MemberSupName = member_sup_name(Pool),
-    MemberSupSpec = {MemberSupName,
-                     {pooler_pooled_worker_sup, start_link, [Pool]},
-                     transient, 5000, supervisor, [pooler_pooled_worker_sup]},
+    MemberSupSpec =
+        {MemberSupName, {pooler_pooled_worker_sup, start_link, [Pool]}, transient, 5000, supervisor, [
+            pooler_pooled_worker_sup
+        ]},
 
     %% five restarts in 60 seconds, then shutdown
     Restart = {one_for_all, 5, 60},

+ 32 - 24
src/pooler_starter.erl

@@ -12,22 +12,26 @@
 %% API Function Exports
 %% ------------------------------------------------------------------
 
--export([start_link/2,
-         start_member/1,
-         start_member/2,
-         stop_member_async/1,
-         stop/1]).
+-export([
+    start_link/2,
+    start_member/1,
+    start_member/2,
+    stop_member_async/1,
+    stop/1
+]).
 
 %% ------------------------------------------------------------------
 %% gen_server Function Exports
 %% ------------------------------------------------------------------
 
--export([init/1,
-         handle_call/3,
-         handle_cast/2,
-         handle_info/2,
-         terminate/2,
-         code_change/3]).
+-export([
+    init/1,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2,
+    terminate/2,
+    code_change/3
+]).
 
 %% ------------------------------------------------------------------
 %% API Function Definitions
@@ -79,9 +83,11 @@ stop_member_async(Pid) ->
 %% ------------------------------------------------------------------
 %% gen_server Function Definitions
 %% ------------------------------------------------------------------
--record(starter, {pool :: #pool{},
-                  parent :: pid() | atom(),
-                  msg :: term()}).
+-record(starter, {
+    pool :: #pool{},
+    parent :: pid() | atom(),
+    msg :: term()
+}).
 
 -spec init({#pool{}, pid() | atom()}) -> {'ok', #starter{}, 0}.
 init({Pool, Parent}) ->
@@ -97,24 +103,22 @@ handle_cast(stop_member, #starter{msg = {_Me, Pid}, pool = #pool{member_sup = Me
     %% Cleanup the process and stop normally.
     supervisor:terminate_child(MemberSup, Pid),
     {stop, normal, State};
-
 handle_cast(accept_member, #starter{msg = Msg, parent = Parent, pool = #pool{name = PoolName}} = State) ->
     %% Process creation has succeeded. Send the member to the pooler
     %% gen_server to be accepted. Pooler gen_server will notify
     %% us if the member was accepted or needs to cleaned up.
     send_accept_member(Parent, PoolName, Msg),
     {noreply, State};
-
 handle_cast(stop, State) ->
     {stop, normal, State};
-
-
 handle_cast(_Request, State) ->
     {noreply, State}.
 
 -spec handle_info(_, _) -> {'noreply', _}.
-handle_info(timeout,
-            #starter{pool = Pool} = State) ->
+handle_info(
+    timeout,
+    #starter{pool = Pool} = State
+) ->
     Msg = do_start_member(Pool),
     accept_member_async(self()),
     {noreply, State#starter{msg = Msg}};
@@ -134,10 +138,14 @@ do_start_member(#pool{member_sup = PoolSup, name = PoolName}) ->
         {ok, Pid} ->
             {self(), Pid};
         Error ->
-            ?LOG_ERROR(#{label => "failed to start member",
-                         pool => PoolName,
-                         error => Error},
-                       #{domain => [pooler]}),
+            ?LOG_ERROR(
+                #{
+                    label => "failed to start member",
+                    pool => PoolName,
+                    error => Error
+                },
+                #{domain => [pooler]}
+            ),
             {self(), Error}
     end.
 

+ 6 - 5
src/pooler_starter_sup.erl

@@ -6,9 +6,11 @@
 
 -behaviour(supervisor).
 
--export([new_starter/2,
-         start_link/0,
-         init/1]).
+-export([
+    new_starter/2,
+    start_link/0,
+    init/1
+]).
 
 -include("pooler.hrl").
 
@@ -19,8 +21,7 @@ start_link() ->
     supervisor:start_link({local, ?MODULE}, ?MODULE, []).
 
 init([]) ->
-    Worker = {pooler_starter, {pooler_starter, start_link, []},
-              temporary, brutal_kill, worker, [pooler_starter]},
+    Worker = {pooler_starter, {pooler_starter, start_link, []}, temporary, brutal_kill, worker, [pooler_starter]},
     Specs = [Worker],
     Restart = {simple_one_for_one, 1, 1},
     {ok, {Restart, Specs}}.

+ 28 - 21
src/pooler_sup.erl

@@ -2,11 +2,13 @@
 
 -behaviour(supervisor).
 
--export([init/1,
-         new_pool/1,
-         rm_pool/1,
-         pool_child_spec/1,
-         start_link/0]).
+-export([
+    init/1,
+    new_pool/1,
+    rm_pool/1,
+    pool_child_spec/1,
+    start_link/0
+]).
 
 -include("pooler.hrl").
 
@@ -15,17 +17,20 @@ start_link() ->
 
 init([]) ->
     %% a list of pool configs
-    Config = case application:get_env(pooler, pools) of
-                 {ok, C} ->
-                     C;
-                 undefined ->
-                     []
-             end,
+    Config =
+        case application:get_env(pooler, pools) of
+            {ok, C} ->
+                C;
+            undefined ->
+                []
+        end,
     {MetricsApi, MetricsMod} = metrics_module(),
-    MetricsConfig = [{metrics_mod, MetricsMod},
-                     {metrics_api, MetricsApi}],
-    Pools = [ pooler_config:list_to_pool(MetricsConfig ++ L) || L <- Config ],
-    PoolSupSpecs = [ pool_sup_spec(Pool) || Pool <- Pools ],
+    MetricsConfig = [
+        {metrics_mod, MetricsMod},
+        {metrics_api, MetricsApi}
+    ],
+    Pools = [pooler_config:list_to_pool(MetricsConfig ++ L) || L <- Config],
+    PoolSupSpecs = [pool_sup_spec(Pool) || Pool <- Pools],
     ets:new(?POOLER_GROUP_TABLE, [set, public, named_table, {write_concurrency, true}]),
     {ok, {{one_for_one, 5, 60}, [starter_sup_spec() | PoolSupSpecs]}}.
 
@@ -39,8 +44,12 @@ new_pool(PoolConfig) ->
 %% public API for this functionality is {@link pooler:pool_child_spec/1}.
 pool_child_spec(PoolConfig) ->
     {MetricsApi, MetricsMod} = metrics_module(),
-    NewPool = pooler_config:list_to_pool([{metrics_mod, MetricsMod},
-                                          {metrics_api, MetricsApi}] ++ PoolConfig),
+    NewPool = pooler_config:list_to_pool(
+        [
+            {metrics_mod, MetricsMod},
+            {metrics_api, MetricsApi}
+        ] ++ PoolConfig
+    ),
     pool_sup_spec(NewPool).
 
 %% @doc Shutdown the named pool.
@@ -56,13 +65,11 @@ rm_pool(Name) ->
     end.
 
 starter_sup_spec() ->
-    {pooler_starter_sup, {pooler_starter_sup, start_link, []},
-     transient, 5000, supervisor, [pooler_starter_sup]}.
+    {pooler_starter_sup, {pooler_starter_sup, start_link, []}, transient, 5000, supervisor, [pooler_starter_sup]}.
 
 pool_sup_spec(#pool{name = Name} = Pool) ->
     SupName = pool_sup_name(Name),
-    {SupName, {pooler_pool_sup, start_link, [Pool]},
-     transient, 5000, supervisor, [pooler_pool_sup]}.
+    {SupName, {pooler_pool_sup, start_link, [Pool]}, transient, 5000, supervisor, [pooler_pool_sup]}.
 
 pool_sup_name(Name) ->
     list_to_atom("pooler_" ++ atom_to_list(Name) ++ "_pool_sup").

+ 27 - 19
test/error_logger_mon.erl

@@ -12,24 +12,30 @@
 %% API Function Exports
 %% ------------------------------------------------------------------
 
--export([start_link/0,
-         report/1,
-         get_msg_count/0,
-         get_msgs/0,
-         reset/0,
-         stop/0,
-         install_handler/0,
-         install_handler/1,
-         uninstall_handler/0
-        ]).
+-export([
+    start_link/0,
+    report/1,
+    get_msg_count/0,
+    get_msgs/0,
+    reset/0,
+    stop/0,
+    install_handler/0,
+    install_handler/1,
+    uninstall_handler/0
+]).
 
 %% OTP logger
 -export([log/2]).
 
 %% gen_server
--export([init/1, handle_call/3, handle_cast/2, handle_info/2,
-         terminate/2, code_change/3]).
-
+-export([
+    init/1,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2,
+    terminate/2,
+    code_change/3
+]).
 
 %% Logger handler
 
@@ -64,18 +70,20 @@ install_handler() ->
 
 install_handler(FilterName) ->
     logger:add_handler(
-      ?MODULE,
-      ?MODULE,
-      #{level => all,
-        filter_default => stop,
-        filters => [{FilterName, filter(FilterName)}]}).
+        ?MODULE,
+        ?MODULE,
+        #{
+            level => all,
+            filter_default => stop,
+            filters => [{FilterName, filter(FilterName)}]
+        }
+    ).
 
 filter(error_logger) ->
     {fun error_logger_filter/2, []};
 filter(pooler) ->
     {fun logger_filters:domain/2, {log, sub, [pooler]}}.
 
-
 uninstall_handler() ->
     logger:remove_handler(?MODULE).
 

+ 1 - 1
test/fake_external_supervisor.erl

@@ -7,4 +7,4 @@
 -export([init/1]).
 
 init(Pool) ->
-	{ok, {{one_for_one, 1, 1}, [pooler:pool_child_spec(Pool)]}}.
+    {ok, {{one_for_one, 1, 1}, [pooler:pool_child_spec(Pool)]}}.

+ 20 - 14
test/fake_metrics.erl

@@ -11,19 +11,26 @@
 %% API Function Exports
 %% ------------------------------------------------------------------
 
--export([start_link/0,
-         notify/3,
-         get_metrics/0,
-         reset_metrics/0,
-         stop/0
-        ]).
+-export([
+    start_link/0,
+    notify/3,
+    get_metrics/0,
+    reset_metrics/0,
+    stop/0
+]).
 
 %% ------------------------------------------------------------------
 %% gen_server Function Exports
 %% ------------------------------------------------------------------
 
--export([init/1, handle_call/3, handle_cast/2, handle_info/2,
-         terminate/2, code_change/3]).
+-export([
+    init/1,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2,
+    terminate/2,
+    code_change/3
+]).
 
 %% ------------------------------------------------------------------
 %% API Function Definitions
@@ -48,15 +55,15 @@ get_metrics() ->
 %% gen_server Function Definitions
 %% ------------------------------------------------------------------
 -record(state, {
-          metrics = [] :: list()
-         }).
+    metrics = [] :: list()
+}).
 
 init(_) ->
     {ok, #state{}}.
 
 handle_call(reset, _From, State) ->
     {reply, ok, State#state{metrics = []}};
-handle_call(get_metrics, _From, #state{metrics = Metrics}=State) ->
+handle_call(get_metrics, _From, #state{metrics = Metrics} = State) ->
     {reply, Metrics, State};
 handle_call(stop, _From, State) ->
     {stop, normal, stop_ok, State};
@@ -64,8 +71,8 @@ handle_call(_Request, _From, State) ->
     erlang:error({what, _Request}),
     {noreply, ok, State}.
 
-handle_cast({_N, _V, _T}=M, #state{metrics = Metrics} = State) ->
-    {noreply, State#state{metrics = [M|Metrics]}};
+handle_cast({_N, _V, _T} = M, #state{metrics = Metrics} = State) ->
+    {noreply, State#state{metrics = [M | Metrics]}};
 handle_cast(_Msg, State) ->
     {noreply, State}.
 
@@ -77,4 +84,3 @@ terminate(_Reason, _State) ->
 
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
-

+ 26 - 23
test/pooled_gs.erl

@@ -16,32 +16,38 @@
 %% API Function Exports
 %% ------------------------------------------------------------------
 
--export([start_link/1,
-         get_id/1,
-         ping/1,
-         ping_count/1,
-         crash/1,
-         error_on_call/1,
-         do_work/2,
-         stop/1
-        ]).
+-export([
+    start_link/1,
+    get_id/1,
+    ping/1,
+    ping_count/1,
+    crash/1,
+    error_on_call/1,
+    do_work/2,
+    stop/1
+]).
 
 %% ------------------------------------------------------------------
 %% gen_server Function Exports
 %% ------------------------------------------------------------------
 
--export([init/1, handle_call/3, handle_cast/2, handle_info/2,
-         terminate/2, code_change/3]).
+-export([
+    init/1,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2,
+    terminate/2,
+    code_change/3
+]).
 
 %% ------------------------------------------------------------------
 %% API Function Definitions
 %% ------------------------------------------------------------------
 
-start_link(Args ={_Type}) ->
+start_link(Args = {_Type}) ->
     % not registered
     gen_server:start_link(?MODULE, Args, []);
-
-start_link(Args ={_Type, _InitFun}) ->
+start_link(Args = {_Type, _InitFun}) ->
     % not registered
     gen_server:start_link(?MODULE, Args, []).
 
@@ -77,10 +83,10 @@ stop(S) ->
 %% gen_server Function Definitions
 %% ------------------------------------------------------------------
 -record(state, {
-          type = "" :: string(),
-          id :: reference(),
-          ping_count = 0 :: non_neg_integer()
-         }).
+    type = "" :: string(),
+    id :: reference(),
+    ping_count = 0 :: non_neg_integer()
+}).
 
 init({Type}) ->
     {ok, #state{type = Type, id = make_ref()}};
@@ -88,18 +94,16 @@ init({Type, StartFun}) ->
     StartFun(),
     {ok, #state{type = Type, id = make_ref()}}.
 
-
-
 handle_call(get_id, _From, State) ->
     {reply, {State#state.type, State#state.id}, State};
 handle_call({do_work, T}, _From, State) ->
     Sleep = rand:uniform(T),
     timer:sleep(Sleep),
     {reply, {ok, Sleep}, State};
-handle_call(ping, _From, #state{ping_count = C } = State) ->
+handle_call(ping, _From, #state{ping_count = C} = State) ->
     State1 = State#state{ping_count = C + 1},
     {reply, pong, State1};
-handle_call(ping_count, _From, #state{ping_count = C } = State) ->
+handle_call(ping_count, _From, #state{ping_count = C} = State) ->
     {reply, C, State};
 handle_call(error_on_call, _From, _State) ->
     erlang:error({pooled_gs, requested_error});
@@ -121,4 +125,3 @@ terminate(_Reason, _State) ->
 
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
-

+ 82 - 67
test/pooler_perf_test.erl

@@ -10,16 +10,17 @@ setup() ->
 
 setup(InitCount, MaxCount, NumPools) ->
     MakePool = fun(I) ->
-                       N = integer_to_list(I),
-                       Name = "p" ++ N,
-                       Arg0 = "pool-" ++ Name,
-                       [{name, list_to_atom(Name)},
-                        {max_count, MaxCount},
-                        {init_count, InitCount},
-                        {start_mfa,
-                         {pooled_gs, start_link, [{Arg0}]}}]
-               end,
-    Pools = [ MakePool(I) || I <- lists:seq(1, NumPools) ],
+        N = integer_to_list(I),
+        Name = "p" ++ N,
+        Arg0 = "pool-" ++ Name,
+        [
+            {name, list_to_atom(Name)},
+            {max_count, MaxCount},
+            {init_count, InitCount},
+            {start_mfa, {pooled_gs, start_link, [{Arg0}]}}
+        ]
+    end,
+    Pools = [MakePool(I) || I <- lists:seq(1, NumPools)],
     application:set_env(pooler, pools, Pools),
     application:start(pooler).
 
@@ -41,15 +42,18 @@ consumer_cycle(0, NumOk, NumFail) ->
 
 test1() ->
     N = 10000,
-    {Time, _} = timer:tc(fun() ->
-                                  consumer_cycle(N)
-                          end, []),
+    {Time, _} = timer:tc(
+        fun() ->
+            consumer_cycle(N)
+        end,
+        []
+    ),
     Time / N.
 
 consumer_worker(N, Parent) ->
     spawn(fun() ->
-                  Parent ! {self(), consumer_cycle(N)}
-          end).
+        Parent ! {self(), consumer_cycle(N)}
+    end).
 
 test3(W, N) ->
     [consumer_worker(W, self()) || _I <- lists:seq(1, N)].
@@ -57,69 +61,80 @@ test3(W, N) ->
 test2(Iter, Workers) ->
     Self = self(),
     {Time, Res} =
-        timer:tc(fun() ->
-                         Pids = [ consumer_worker(Iter, Self)
-                                  || _I <- lists:seq(1, Workers) ],
-                         gather_pids(Pids)
-                 end, []),
-    {NumOk, NumFail} = lists:foldr(fun({_, L}, {O, F}) ->
-                                           {O + ?gv(ok, L),
-                                            F + ?gv(fail, L)}
-                                   end, {0, 0}, Res),
+        timer:tc(
+            fun() ->
+                Pids = [
+                    consumer_worker(Iter, Self)
+                 || _I <- lists:seq(1, Workers)
+                ],
+                gather_pids(Pids)
+            end,
+            []
+        ),
+    {NumOk, NumFail} = lists:foldr(
+        fun({_, L}, {O, F}) ->
+            {O + ?gv(ok, L), F + ?gv(fail, L)}
+        end,
+        {0, 0},
+        Res
+    ),
     {Time, [{ok, NumOk}, {fail, NumFail}]}.
 
 gather_pids(Pids) ->
     gather_pids(Pids, []).
 
-gather_pids([Pid|Rest], Acc) ->
+gather_pids([Pid | Rest], Acc) ->
     receive
         {Pid, Ans} ->
-            gather_pids(Rest, [{Pid, Ans}|Acc]);
-        stop -> stop
+            gather_pids(Rest, [{Pid, Ans} | Acc]);
+        stop ->
+            stop
     end;
 gather_pids([], Acc) ->
     Acc.
 
 pooler_take_return_test_() ->
     {foreach,
-     % setup
-     fun() ->
-             InitCount = 100,
-             MaxCount = 100,
-             NumPools = 5,
-             logger:set_handler_config(default, filters, []),
-             setup(InitCount, MaxCount, NumPools)
-     end,
-     fun(_X) ->
-             application:stop(pooler)
-     end,
-     [
-      {"take return cycle single worker",
-       fun() ->
-               NumCycles = 10000,
-               Ans = consumer_cycle(NumCycles),
-               ?assertEqual(NumCycles, ?gv(ok, Ans)),
-               ?assertEqual(0, ?gv(fail, Ans))
-       end},
-
-      {"take return cycle multiple workers",
-       fun() ->
-               Self = self(),
-               Iter = 100,
-               Workers = 100,
-               Pids = [ consumer_worker(Iter, Self)
-                        || _I <- lists:seq(1, Workers) ],
-               Res = gather_pids(Pids),
-               {NumOk, NumFail} =
-                   lists:foldr(fun({_, L}, {O, F}) ->
-                                       {O + ?gv(ok, L), F + ?gv(fail, L)}
-                               end, {0, 0}, Res),
-               %% not sure what to test here now. We expect some
-               %% failures if init count is less than max count
-               %% because of async start.
-               ?assertEqual(0, NumFail),
-               ?assertEqual(100*100, NumOk)
-       end}
-      ]
-    }.
+        % setup
+        fun() ->
+            InitCount = 100,
+            MaxCount = 100,
+            NumPools = 5,
+            logger:set_handler_config(default, filters, []),
+            setup(InitCount, MaxCount, NumPools)
+        end,
+        fun(_X) ->
+            application:stop(pooler)
+        end,
+        [
+            {"take return cycle single worker", fun() ->
+                NumCycles = 10000,
+                Ans = consumer_cycle(NumCycles),
+                ?assertEqual(NumCycles, ?gv(ok, Ans)),
+                ?assertEqual(0, ?gv(fail, Ans))
+            end},
 
+            {"take return cycle multiple workers", fun() ->
+                Self = self(),
+                Iter = 100,
+                Workers = 100,
+                Pids = [
+                    consumer_worker(Iter, Self)
+                 || _I <- lists:seq(1, Workers)
+                ],
+                Res = gather_pids(Pids),
+                {NumOk, NumFail} =
+                    lists:foldr(
+                        fun({_, L}, {O, F}) ->
+                            {O + ?gv(ok, L), F + ?gv(fail, L)}
+                        end,
+                        {0, 0},
+                        Res
+                    ),
+                %% not sure what to test here now. We expect some
+                %% failures if init count is less than max count
+                %% because of async start.
+                ?assertEqual(0, NumFail),
+                ?assertEqual(100 * 100, NumOk)
+            end}
+        ]}.

+ 1218 - 1145
test/pooler_tests.erl

@@ -104,1223 +104,1292 @@ user_loop(MyTC) ->
 
 pooler_basics_via_config_test_() ->
     {setup,
-     fun() ->
-             {ok, _} = error_logger_mon:start_link(),
-             error_logger_mon:install_handler(pooler),
-             logger:set_handler_config(default, filters, []),
-             application:set_env(pooler, metrics_module, fake_metrics),
-             fake_metrics:start_link()
-     end,
-     fun(_X) ->
-             error_logger_mon:uninstall_handler(),
-             ok = error_logger_mon:stop(),
-             fake_metrics:stop()
-     end,
-    {foreach,
-     % setup
-     fun() ->
-             error_logger_mon:reset(),
-             Pools = [[{name, test_pool_1},
-                       {max_count, 3},
-                       {init_count, 2},
-                       {cull_interval, {0, min}},
-                       {start_mfa,
-                        {pooled_gs, start_link, [{"type-0"}]}}]],
-             application:set_env(pooler, pools, Pools),
-             application:start(pooler)
-     end,
-     fun(_X) ->
-             application:stop(pooler)
-     end,
-     basic_tests()}}.
+        fun() ->
+            {ok, _} = error_logger_mon:start_link(),
+            error_logger_mon:install_handler(pooler),
+            logger:set_handler_config(default, filters, []),
+            application:set_env(pooler, metrics_module, fake_metrics),
+            fake_metrics:start_link()
+        end,
+        fun(_X) ->
+            error_logger_mon:uninstall_handler(),
+            ok = error_logger_mon:stop(),
+            fake_metrics:stop()
+        end,
+        {foreach,
+            % setup
+            fun() ->
+                error_logger_mon:reset(),
+                Pools = [
+                    [
+                        {name, test_pool_1},
+                        {max_count, 3},
+                        {init_count, 2},
+                        {cull_interval, {0, min}},
+                        {start_mfa, {pooled_gs, start_link, [{"type-0"}]}}
+                    ]
+                ],
+                application:set_env(pooler, pools, Pools),
+                application:start(pooler)
+            end,
+            fun(_X) ->
+                application:stop(pooler)
+            end,
+            basic_tests()}}.
 
 pooler_basics_dynamic_test_() ->
     {setup,
-     fun() ->
-             {ok, _} = error_logger_mon:start_link(),
-             error_logger_mon:install_handler(pooler),
-             logger:set_handler_config(default, filters, []),
-             application:set_env(pooler, metrics_module, fake_metrics),
-             fake_metrics:start_link()
-     end,
-     fun(_X) ->
-             error_logger_mon:uninstall_handler(),
-             ok = error_logger_mon:stop(),
-             fake_metrics:stop()
-     end,
-    {foreach,
-     % setup
-     fun() ->
-             error_logger_mon:reset(),
-             Pool = [{name, test_pool_1},
-                     {max_count, 3},
-                     {init_count, 2},
-                     {start_mfa,
-                      {pooled_gs, start_link, [{"type-0"}]}}],
-             application:unset_env(pooler, pools),
-             application:start(pooler),
-             pooler:new_pool(Pool)
-     end,
-     fun(_X) ->
-             application:stop(pooler)
-     end,
-     basic_tests()}}.
+        fun() ->
+            {ok, _} = error_logger_mon:start_link(),
+            error_logger_mon:install_handler(pooler),
+            logger:set_handler_config(default, filters, []),
+            application:set_env(pooler, metrics_module, fake_metrics),
+            fake_metrics:start_link()
+        end,
+        fun(_X) ->
+            error_logger_mon:uninstall_handler(),
+            ok = error_logger_mon:stop(),
+            fake_metrics:stop()
+        end,
+        {foreach,
+            % setup
+            fun() ->
+                error_logger_mon:reset(),
+                Pool = [
+                    {name, test_pool_1},
+                    {max_count, 3},
+                    {init_count, 2},
+                    {start_mfa, {pooled_gs, start_link, [{"type-0"}]}}
+                ],
+                application:unset_env(pooler, pools),
+                application:start(pooler),
+                pooler:new_pool(Pool)
+            end,
+            fun(_X) ->
+                application:stop(pooler)
+            end,
+            basic_tests()}}.
 
 pooler_basics_integration_to_other_supervisor_test_() ->
     {setup,
-     fun() ->
-             {ok, _} = error_logger_mon:start_link(),
-             error_logger_mon:install_handler(pooler),
-             logger:set_handler_config(default, filters, []),
-             application:set_env(pooler, metrics_module, fake_metrics),
-             fake_metrics:start_link()
-     end,
-     fun(_X) ->
-             error_logger_mon:uninstall_handler(),
-             ok = error_logger_mon:stop(),
-             fake_metrics:stop()
-     end,
-    {foreach,
-     % setup
-     fun() ->
-             error_logger_mon:reset(),
-             Pool = [{name, test_pool_1},
-                     {max_count, 3},
-                     {init_count, 2},
-                     {start_mfa,
-                      {pooled_gs, start_link, [{"type-0"}]}}],
-             application:unset_env(pooler, pools),
-             application:start(pooler),
-             supervisor:start_link(fake_external_supervisor, Pool)
-     end,
-     fun({ok, SupPid}) ->
-             exit(SupPid, normal),
-             application:stop(pooler)
-     end,
-     basic_tests()}}.
-
+        fun() ->
+            {ok, _} = error_logger_mon:start_link(),
+            error_logger_mon:install_handler(pooler),
+            logger:set_handler_config(default, filters, []),
+            application:set_env(pooler, metrics_module, fake_metrics),
+            fake_metrics:start_link()
+        end,
+        fun(_X) ->
+            error_logger_mon:uninstall_handler(),
+            ok = error_logger_mon:stop(),
+            fake_metrics:stop()
+        end,
+        {foreach,
+            % setup
+            fun() ->
+                error_logger_mon:reset(),
+                Pool = [
+                    {name, test_pool_1},
+                    {max_count, 3},
+                    {init_count, 2},
+                    {start_mfa, {pooled_gs, start_link, [{"type-0"}]}}
+                ],
+                application:unset_env(pooler, pools),
+                application:start(pooler),
+                supervisor:start_link(fake_external_supervisor, Pool)
+            end,
+            fun({ok, SupPid}) ->
+                exit(SupPid, normal),
+                application:stop(pooler)
+            end,
+            basic_tests()}}.
 
 basic_tests() ->
-     [
-      {"there are init_count members at start",
-       fun() ->
-               Stats = [ P || {P, {_, free, _}} <- pooler:pool_stats(test_pool_1) ],
-               ?assertEqual(2, length(Stats))
-       end},
-
-      {"take and return one",
-       fun() ->
-               P = pooler:take_member(test_pool_1),
-               ?assertMatch({"type-0", _Id}, pooled_gs:get_id(P)),
-               ok = pooler:return_member(test_pool_1, P, ok)
-       end},
-
-      {"take and return one, named pool",
-       fun() ->
-               P = pooler:take_member(test_pool_1),
-               ?assertMatch({"type-0", _Id}, pooled_gs:get_id(P)),
-               ok, pooler:return_member(test_pool_1, P)
-       end},
-
-      {"attempt to take form unknown pool",
-       fun() ->
-               %% since pools are now servers, an unknown pool will timeout
-               ?assertExit({noproc, _}, pooler:take_member(bad_pool_name))
-       end},
-
-      {"members creation is triggered after pool exhaustion until max",
-       fun() ->
-               %% init count is 2
-               Pids0 = [pooler:take_member(test_pool_1), pooler:take_member(test_pool_1)],
-               %% since new member creation is async, can only assert
-               %% that we will get a pid, but may not be first try.
-               Pids = get_n_pids(1, Pids0),
-               %% pool is at max now, requests should give error
-               ?assertEqual(error_no_members, pooler:take_member(test_pool_1)),
-               ?assertEqual(error_no_members, pooler:take_member(test_pool_1)),
-               PRefs = [ R || {_T, R} <- [ pooled_gs:get_id(P) || P <- Pids ] ],
-               % no duplicates
-               ?assertEqual(length(PRefs), length(lists:usort(PRefs)))
-       end
-      },
-
-      {"pids are reused most recent return first",
-       fun() ->
-               P1 = pooler:take_member(test_pool_1),
-               P2 = pooler:take_member(test_pool_1),
-               ?assertNot(P1 == P2),
-               ok = pooler:return_member(test_pool_1, P1, ok),
-               ok = pooler:return_member(test_pool_1, P2, ok),
-               % pids are reused most recent first
-               ?assertEqual(P2, pooler:take_member(test_pool_1)),
-               ?assertEqual(P1, pooler:take_member(test_pool_1))
-       end},
-
-      {"if an in-use pid crashes it is replaced",
-       fun() ->
-               Pids0 = get_n_pids(3, []),
-               Ids0 = [ pooled_gs:get_id(P) || P <- Pids0 ],
-               % crash them all
-               [ pooled_gs:crash(P) || P <- Pids0 ],
-               Pids1 = get_n_pids(3, []),
-               Ids1 = [ pooled_gs:get_id(P) || P <- Pids1 ],
-               [ ?assertNot(lists:member(I, Ids0)) || I <- Ids1 ]
-       end
-       },
-
-      {"if a free pid crashes it is replaced",
-       fun() ->
-               FreePids = [ P || {P, {_, free, _}} <- pooler:pool_stats(test_pool_1) ],
-               [ exit(P, kill) || P <- FreePids ],
-               Pids1 = get_n_pids(3, []),
-               ?assertEqual(3, length(Pids1))
-       end},
-
-      {"if a pid is returned with bad status it is replaced",
-       fun() ->
-               Pids0 = get_n_pids(3, []),
-               Ids0 = [ pooled_gs:get_id(P) || P <- Pids0 ],
-               % return them all marking as bad
-               [ pooler:return_member(test_pool_1, P, fail) || P <- Pids0 ],
-               Pids1 = get_n_pids(3, []),
-               Ids1 = [ pooled_gs:get_id(P) || P <- Pids1 ],
-               [ ?assertNot(lists:member(I, Ids0)) || I <- Ids1 ]
-       end
-      },
-
-      {"if a consumer crashes, pid is replaced",
-       fun() ->
-               Consumer = start_user(),
-               StartId = user_id(Consumer),
-               user_crash(Consumer),
-               NewPid = hd(get_n_pids(1, [])),
-               NewId = pooled_gs:get_id(NewPid),
-               ?assertNot(NewId == StartId)
-       end
-      },
-
-      {"it is ok to return an unknown pid",
-       fun() ->
-               Bogus1 = spawn(fun() -> ok end),
-               Bogus2 = spawn(fun() -> ok end),
-               ?assertEqual(ok, pooler:return_member(test_pool_1, Bogus1, ok)),
-               ?assertEqual(ok, pooler:return_member(test_pool_1, Bogus2, fail))
-       end
-      },
-
-      {"it is ok to return a pid more than once",
-       fun() ->
-               M = pooler:take_member(test_pool_1),
-               [ pooler:return_member(test_pool_1, M)
-                 || _I <- lists:seq(1, 37) ],
-               ?assertEqual(
-                  36,
-                  length(
+    [
+        {"there are init_count members at start", fun() ->
+            Stats = [P || {P, {_, free, _}} <- pooler:pool_stats(test_pool_1)],
+            ?assertEqual(2, length(Stats))
+        end},
+
+        {"take and return one", fun() ->
+            P = pooler:take_member(test_pool_1),
+            ?assertMatch({"type-0", _Id}, pooled_gs:get_id(P)),
+            ok = pooler:return_member(test_pool_1, P, ok)
+        end},
+
+        {"take and return one, named pool", fun() ->
+            P = pooler:take_member(test_pool_1),
+            ?assertMatch({"type-0", _Id}, pooled_gs:get_id(P)),
+            ok,
+            pooler:return_member(test_pool_1, P)
+        end},
+
+        {"attempt to take form unknown pool", fun() ->
+            %% since pools are now servers, an unknown pool will timeout
+            ?assertExit({noproc, _}, pooler:take_member(bad_pool_name))
+        end},
+
+        {"members creation is triggered after pool exhaustion until max", fun() ->
+            %% init count is 2
+            Pids0 = [pooler:take_member(test_pool_1), pooler:take_member(test_pool_1)],
+            %% since new member creation is async, can only assert
+            %% that we will get a pid, but may not be first try.
+            Pids = get_n_pids(1, Pids0),
+            %% pool is at max now, requests should give error
+            ?assertEqual(error_no_members, pooler:take_member(test_pool_1)),
+            ?assertEqual(error_no_members, pooler:take_member(test_pool_1)),
+            PRefs = [R || {_T, R} <- [pooled_gs:get_id(P) || P <- Pids]],
+            % no duplicates
+            ?assertEqual(length(PRefs), length(lists:usort(PRefs)))
+        end},
+
+        {"pids are reused most recent return first", fun() ->
+            P1 = pooler:take_member(test_pool_1),
+            P2 = pooler:take_member(test_pool_1),
+            ?assertNot(P1 == P2),
+            ok = pooler:return_member(test_pool_1, P1, ok),
+            ok = pooler:return_member(test_pool_1, P2, ok),
+            % pids are reused most recent first
+            ?assertEqual(P2, pooler:take_member(test_pool_1)),
+            ?assertEqual(P1, pooler:take_member(test_pool_1))
+        end},
+
+        {"if an in-use pid crashes it is replaced", fun() ->
+            Pids0 = get_n_pids(3, []),
+            Ids0 = [pooled_gs:get_id(P) || P <- Pids0],
+            % crash them all
+            [pooled_gs:crash(P) || P <- Pids0],
+            Pids1 = get_n_pids(3, []),
+            Ids1 = [pooled_gs:get_id(P) || P <- Pids1],
+            [?assertNot(lists:member(I, Ids0)) || I <- Ids1]
+        end},
+
+        {"if a free pid crashes it is replaced", fun() ->
+            FreePids = [P || {P, {_, free, _}} <- pooler:pool_stats(test_pool_1)],
+            [exit(P, kill) || P <- FreePids],
+            Pids1 = get_n_pids(3, []),
+            ?assertEqual(3, length(Pids1))
+        end},
+
+        {"if a pid is returned with bad status it is replaced", fun() ->
+            Pids0 = get_n_pids(3, []),
+            Ids0 = [pooled_gs:get_id(P) || P <- Pids0],
+            % return them all marking as bad
+            [pooler:return_member(test_pool_1, P, fail) || P <- Pids0],
+            Pids1 = get_n_pids(3, []),
+            Ids1 = [pooled_gs:get_id(P) || P <- Pids1],
+            [?assertNot(lists:member(I, Ids0)) || I <- Ids1]
+        end},
+
+        {"if a consumer crashes, pid is replaced", fun() ->
+            Consumer = start_user(),
+            StartId = user_id(Consumer),
+            user_crash(Consumer),
+            NewPid = hd(get_n_pids(1, [])),
+            NewId = pooled_gs:get_id(NewPid),
+            ?assertNot(NewId == StartId)
+        end},
+
+        {"it is ok to return an unknown pid", fun() ->
+            Bogus1 = spawn(fun() -> ok end),
+            Bogus2 = spawn(fun() -> ok end),
+            ?assertEqual(ok, pooler:return_member(test_pool_1, Bogus1, ok)),
+            ?assertEqual(ok, pooler:return_member(test_pool_1, Bogus2, fail))
+        end},
+
+        {"it is ok to return a pid more than once", fun() ->
+            M = pooler:take_member(test_pool_1),
+            [
+                pooler:return_member(test_pool_1, M)
+             || _I <- lists:seq(1, 37)
+            ],
+            ?assertEqual(
+                36,
+                length(
                     lists:filter(
-                      fun(#{msg := {report, #{label := "ignored return of free member",
-                                              pid := Pid}}}) ->
-                              Pid =:= M;
-                         (_) ->
-                              false
-                      end,
-                      error_logger_mon:get_msgs()))),
-               M1 = pooler:take_member(test_pool_1),
-               M2 = pooler:take_member(test_pool_1),
-               ?assert(M1 =/= M2),
-               Pool1 = gen_server:call(test_pool_1, dump_pool),
-               ?assertEqual(2, Pool1#pool.in_use_count),
-               ?assertEqual(0, Pool1#pool.free_count),
-               pooler:return_member(test_pool_1, M1),
-               pooler:return_member(test_pool_1, M2),
-               Pool2 = gen_server:call(test_pool_1, dump_pool),
-               ?assertEqual(0, Pool2#pool.in_use_count),
-               ?assertEqual(2, Pool2#pool.free_count),
-               ok
-       end},
-
-      {"calling return_member on error_no_members is ignored",
-       fun() ->
-               ?assertEqual(ok, pooler:return_member(test_pool_1, error_no_members)),
-               ?assertEqual(ok, pooler:return_member(test_pool_1, error_no_members, ok)),
-               ?assertEqual(ok, pooler:return_member(test_pool_1, error_no_members, fail))
-       end
-      },
-
-      {"dynamic pool creation",
-       fun() ->
-               PoolSpec = [{name, dyn_pool_1},
-                           {max_count, 3},
-                           {init_count, 2},
-                           {start_mfa,
-                            {pooled_gs, start_link, [{"dyn-0"}]}}],
-               {ok, SupPid1} = pooler:new_pool(PoolSpec),
-               ?assert(is_pid(SupPid1)),
-               M = pooler:take_member(dyn_pool_1),
-               ?assertMatch({"dyn-0", _Id}, pooled_gs:get_id(M)),
-               ?assertEqual(ok, pooler:rm_pool(dyn_pool_1)),
-               ?assertExit({noproc, _}, pooler:take_member(dyn_pool_1)),
-               %% verify pool of same name can be created after removal
-               {ok, SupPid2} = pooler:new_pool(PoolSpec),
-               ?assert(is_pid(SupPid2)),
-               %% remove non-existing pool
-               ?assertEqual(ok, pooler:rm_pool(dyn_pool_X)),
-               ?assertEqual(ok, pooler:rm_pool(dyn_pool_1))
-       end},
-
-      {"metrics have been called (no timeout/queue)",
-       fun() ->
-               %% exercise the API to ensure we have certain keys reported as metrics
-               fake_metrics:reset_metrics(),
-               Pids = [ pooler:take_member(test_pool_1) || _I <- lists:seq(1, 10) ],
-               [ pooler:return_member(test_pool_1, P) || P <- Pids ],
-               catch pooler:take_member(bad_pool_name),
-               %% kill and unused member
-               exit(hd(Pids), kill),
-               %% kill a used member
-               KillMe = pooler:take_member(test_pool_1),
-               exit(KillMe, kill),
-               %% FIXME: We need to wait for pooler to process the
-               %% exit message. This is ugly, will fix later.
-               timer:sleep(200),                % :(
-               ExpectKeys = lists:sort([<<"pooler.test_pool_1.error_no_members_count">>,
-                                        <<"pooler.test_pool_1.events">>,
-                                        <<"pooler.test_pool_1.free_count">>,
-                                        <<"pooler.test_pool_1.in_use_count">>,
-                                        <<"pooler.test_pool_1.killed_free_count">>,
-                                        <<"pooler.test_pool_1.killed_in_use_count">>,
-                                        <<"pooler.test_pool_1.take_rate">>]),
-               Metrics = fake_metrics:get_metrics(),
-               GotKeys = lists:usort([ Name || {Name, _, _} <- Metrics ]),
-               ?assertEqual(ExpectKeys, GotKeys)
-       end},
-
-      {"metrics have been called (with timeout/queue)",
-       fun() ->
-               %% exercise the API to ensure we have certain keys reported as metrics
-               fake_metrics:reset_metrics(),
-               %% pass a non-zero timeout here to exercise queueing
-               Pids = [ pooler:take_member(test_pool_1, 1) || _I <- lists:seq(1, 10) ],
-               [ pooler:return_member(test_pool_1, P) || P <- Pids ],
-               catch pooler:take_member(bad_pool_name),
-               %% kill and unused member
-               exit(hd(Pids), kill),
-               %% kill a used member
-               KillMe = pooler:take_member(test_pool_1),
-               exit(KillMe, kill),
-               %% FIXME: We need to wait for pooler to process the
-               %% exit message. This is ugly, will fix later.
-               timer:sleep(200),                % :(
-               ExpectKeys = lists:sort([<<"pooler.test_pool_1.error_no_members_count">>,
-                                        <<"pooler.test_pool_1.events">>,
-                                        <<"pooler.test_pool_1.free_count">>,
-                                        <<"pooler.test_pool_1.in_use_count">>,
-                                        <<"pooler.test_pool_1.killed_free_count">>,
-                                        <<"pooler.test_pool_1.killed_in_use_count">>,
-                                        <<"pooler.test_pool_1.take_rate">>,
-                                        <<"pooler.test_pool_1.queue_count">>]),
-               Metrics = fake_metrics:get_metrics(),
-               GotKeys = lists:usort([ Name || {Name, _, _} <- Metrics ]),
-               ?assertEqual(ExpectKeys, GotKeys)
-       end},
-
-      {"accept bad member is handled",
-       fun() ->
-               Bad = spawn(fun() -> ok end),
-               FakeStarter = spawn(fun() -> starter end),
-               ?assertEqual(ok, pooler:accept_member(test_pool_1, {FakeStarter, Bad}))
-       end},
-
-      {"utilization returns sane results",
-       fun() ->
-               #pool{max_count = MaxCount, queue_max = QueueMax} = sys:get_state(test_pool_1),
-
-               ?assertEqual(MaxCount, ?gv(max_count, pooler:pool_utilization(test_pool_1))),
-               ?assertEqual(0, ?gv(in_use_count, pooler:pool_utilization(test_pool_1))),
-               ?assertEqual(2, ?gv(free_count, pooler:pool_utilization(test_pool_1))),
-               ?assertEqual(0, ?gv(queued_count, pooler:pool_utilization(test_pool_1))),
-               ?assertEqual(QueueMax, ?gv(queue_max, pooler:pool_utilization(test_pool_1)))
-       end}
-      ].
+                        fun
+                            (
+                                #{
+                                    msg :=
+                                        {report, #{
+                                            label := "ignored return of free member",
+                                            pid := Pid
+                                        }}
+                                }
+                            ) ->
+                                Pid =:= M;
+                            (_) ->
+                                false
+                        end,
+                        error_logger_mon:get_msgs()
+                    )
+                )
+            ),
+            M1 = pooler:take_member(test_pool_1),
+            M2 = pooler:take_member(test_pool_1),
+            ?assert(M1 =/= M2),
+            Pool1 = gen_server:call(test_pool_1, dump_pool),
+            ?assertEqual(2, Pool1#pool.in_use_count),
+            ?assertEqual(0, Pool1#pool.free_count),
+            pooler:return_member(test_pool_1, M1),
+            pooler:return_member(test_pool_1, M2),
+            Pool2 = gen_server:call(test_pool_1, dump_pool),
+            ?assertEqual(0, Pool2#pool.in_use_count),
+            ?assertEqual(2, Pool2#pool.free_count),
+            ok
+        end},
+
+        {"calling return_member on error_no_members is ignored", fun() ->
+            ?assertEqual(ok, pooler:return_member(test_pool_1, error_no_members)),
+            ?assertEqual(ok, pooler:return_member(test_pool_1, error_no_members, ok)),
+            ?assertEqual(ok, pooler:return_member(test_pool_1, error_no_members, fail))
+        end},
+
+        {"dynamic pool creation", fun() ->
+            PoolSpec = [
+                {name, dyn_pool_1},
+                {max_count, 3},
+                {init_count, 2},
+                {start_mfa, {pooled_gs, start_link, [{"dyn-0"}]}}
+            ],
+            {ok, SupPid1} = pooler:new_pool(PoolSpec),
+            ?assert(is_pid(SupPid1)),
+            M = pooler:take_member(dyn_pool_1),
+            ?assertMatch({"dyn-0", _Id}, pooled_gs:get_id(M)),
+            ?assertEqual(ok, pooler:rm_pool(dyn_pool_1)),
+            ?assertExit({noproc, _}, pooler:take_member(dyn_pool_1)),
+            %% verify pool of same name can be created after removal
+            {ok, SupPid2} = pooler:new_pool(PoolSpec),
+            ?assert(is_pid(SupPid2)),
+            %% remove non-existing pool
+            ?assertEqual(ok, pooler:rm_pool(dyn_pool_X)),
+            ?assertEqual(ok, pooler:rm_pool(dyn_pool_1))
+        end},
+
+        {"metrics have been called (no timeout/queue)", fun() ->
+            %% exercise the API to ensure we have certain keys reported as metrics
+            fake_metrics:reset_metrics(),
+            Pids = [pooler:take_member(test_pool_1) || _I <- lists:seq(1, 10)],
+            [pooler:return_member(test_pool_1, P) || P <- Pids],
+            catch pooler:take_member(bad_pool_name),
+            %% kill and unused member
+            exit(hd(Pids), kill),
+            %% kill a used member
+            KillMe = pooler:take_member(test_pool_1),
+            exit(KillMe, kill),
+            %% FIXME: We need to wait for pooler to process the
+            %% exit message. This is ugly, will fix later.
+
+            % :(
+            timer:sleep(200),
+            ExpectKeys = lists:sort([
+                <<"pooler.test_pool_1.error_no_members_count">>,
+                <<"pooler.test_pool_1.events">>,
+                <<"pooler.test_pool_1.free_count">>,
+                <<"pooler.test_pool_1.in_use_count">>,
+                <<"pooler.test_pool_1.killed_free_count">>,
+                <<"pooler.test_pool_1.killed_in_use_count">>,
+                <<"pooler.test_pool_1.take_rate">>
+            ]),
+            Metrics = fake_metrics:get_metrics(),
+            GotKeys = lists:usort([Name || {Name, _, _} <- Metrics]),
+            ?assertEqual(ExpectKeys, GotKeys)
+        end},
+
+        {"metrics have been called (with timeout/queue)", fun() ->
+            %% exercise the API to ensure we have certain keys reported as metrics
+            fake_metrics:reset_metrics(),
+            %% pass a non-zero timeout here to exercise queueing
+            Pids = [pooler:take_member(test_pool_1, 1) || _I <- lists:seq(1, 10)],
+            [pooler:return_member(test_pool_1, P) || P <- Pids],
+            catch pooler:take_member(bad_pool_name),
+            %% kill and unused member
+            exit(hd(Pids), kill),
+            %% kill a used member
+            KillMe = pooler:take_member(test_pool_1),
+            exit(KillMe, kill),
+            %% FIXME: We need to wait for pooler to process the
+            %% exit message. This is ugly, will fix later.
+
+            % :(
+            timer:sleep(200),
+            ExpectKeys = lists:sort([
+                <<"pooler.test_pool_1.error_no_members_count">>,
+                <<"pooler.test_pool_1.events">>,
+                <<"pooler.test_pool_1.free_count">>,
+                <<"pooler.test_pool_1.in_use_count">>,
+                <<"pooler.test_pool_1.killed_free_count">>,
+                <<"pooler.test_pool_1.killed_in_use_count">>,
+                <<"pooler.test_pool_1.take_rate">>,
+                <<"pooler.test_pool_1.queue_count">>
+            ]),
+            Metrics = fake_metrics:get_metrics(),
+            GotKeys = lists:usort([Name || {Name, _, _} <- Metrics]),
+            ?assertEqual(ExpectKeys, GotKeys)
+        end},
+
+        {"accept bad member is handled", fun() ->
+            Bad = spawn(fun() -> ok end),
+            FakeStarter = spawn(fun() -> starter end),
+            ?assertEqual(ok, pooler:accept_member(test_pool_1, {FakeStarter, Bad}))
+        end},
+
+        {"utilization returns sane results", fun() ->
+            #pool{max_count = MaxCount, queue_max = QueueMax} = sys:get_state(test_pool_1),
+
+            ?assertEqual(MaxCount, ?gv(max_count, pooler:pool_utilization(test_pool_1))),
+            ?assertEqual(0, ?gv(in_use_count, pooler:pool_utilization(test_pool_1))),
+            ?assertEqual(2, ?gv(free_count, pooler:pool_utilization(test_pool_1))),
+            ?assertEqual(0, ?gv(queued_count, pooler:pool_utilization(test_pool_1))),
+            ?assertEqual(QueueMax, ?gv(queue_max, pooler:pool_utilization(test_pool_1)))
+        end}
+    ].
 
 pooler_groups_test_() ->
     {setup,
-     fun() ->
-             logger:set_handler_config(default, filters, []),
-             application:set_env(pooler, metrics_module, fake_metrics),
-             fake_metrics:start_link()
-     end,
-     fun(_X) ->
-             fake_metrics:stop()
-     end,
-    {foreach,
-     % setup
-     fun() ->
-             Pools = [[{name, test_pool_1},
-                       {group, group_1},
-                       {max_count, 3},
-                       {init_count, 2},
-                       {start_mfa,
-                        {pooled_gs, start_link, [{"type-1-1"}]}}],
-                      [{name, test_pool_2},
-                       {group, group_1},
-                       {max_count, 3},
-                       {init_count, 2},
-                       {start_mfa,
-                        {pooled_gs, start_link, [{"type-1-2"}]}}],
-                      %% test_pool_3 not part of the group
-                      [{name, test_pool_3},
-                       {group, undefined},
-                       {max_count, 3},
-                       {init_count, 2},
-                       {start_mfa,
-                        {pooled_gs, start_link, [{"type-3"}]}}]
-                     ],
-             application:set_env(pooler, pools, Pools),
-             pg_start(),
-             application:start(pooler)
-     end,
-     fun(_X) ->
-             application:stop(pooler),
-             pg_stop()
-     end,
-     [
-      {"take and return one group member (repeated)",
-       fun() ->
-               Types = [ begin
-                             Pid = pooler:take_group_member(group_1),
-                             ?assert(is_pid(Pid), [{result, Pid}, {i, I}]),
-                             {Type, _} = pooled_gs:get_id(Pid),
-                             ?assertMatch("type-1" ++ _, Type),
-                             ok = pooler:return_group_member(group_1, Pid, ok),
-                             timer:sleep(10),
-                             Type
-                         end
-                         || I <- lists:seq(1, 50) ],
-               Type_1_1 = [ X || "type-1-1" = X <- Types ],
-               Type_1_2 = [ X || "type-1-2" = X <- Types ],
-               ?assert(length(Type_1_1) > 0, [{types, Types}]),
-               ?assert(length(Type_1_2) > 0, [{types, Types}])
-       end},
-
-      {"take member from unknown group",
-       fun() ->
-               ?assertEqual(error_no_members,
-                            pooler:take_group_member(not_a_group))
-       end},
-
-      {"return member to unknown group",
-       fun() ->
-               Pid = pooler:take_group_member(group_1),
-               ?assertEqual(ok, pooler:return_group_member(no_such_group, Pid))
-       end},
-
-      {"return member to wrong group",
-       fun() ->
-               Pid = pooler:take_member(test_pool_3),
-               ?assertEqual(ok, pooler:return_group_member(group_1, Pid))
-       end},
-
-      {"return member with something which is not a pid",
-       fun() ->
-               ?assertException(error, _, pooler:return_group_member(group_1, not_pid))
-       end},
-
-      {"take member from empty group",
-       fun() ->
-               %% artificially empty group member list
-               [ pg_leave(group_1, M) || M <- pg_get_members(group_1) ],
-               ?assertEqual(error_no_members, pooler:take_group_member(group_1))
-       end},
-
-      {"return member to group, implied ok",
-       fun() ->
-               Pid = pooler:take_group_member(group_1),
-               ?assertEqual(ok, pooler:return_group_member(group_1, Pid))
-       end},
-
-      {"return error_no_member to group",
-       fun() ->
-               ?assertEqual(ok, pooler:return_group_member(group_1, error_no_members))
-       end},
-
-      {"exhaust pools in group",
-       fun() ->
-               Pids = get_n_pids_group(group_1, 6, []),
-               %% they should all be pids
-               [ begin
-                     {Type, _} = pooled_gs:get_id(P),
-                     ?assertMatch("type-1" ++ _, Type),
-                     ok
-                 end || P <- Pids ],
-               %% further attempts should be error
-               [error_no_members,
-                error_no_members,
-                error_no_members] = [ pooler:take_group_member(group_1)
-                                      || _I <- lists:seq(1, 3) ]
-       end},
-
-      {"rm_group with nonexisting group",
-       fun() ->
-               ?assertEqual(ok, pooler:rm_group(i_dont_exist))
-       end},
-
-      {"rm_group with existing empty group",
-       fun() ->
-               ?assertEqual(ok, pooler:rm_pool(test_pool_1)),
-               ?assertEqual(ok, pooler:rm_pool(test_pool_2)),
-               timer:sleep(100),  % process group de-registration is asynchronous
-               ?assertEqual(error_no_members, pooler:take_group_member(group_1)),
-               ?assertEqual(ok, pooler:rm_group(group_1)),
-
-               ?assertExit({noproc, _}, pooler:take_member(test_pool_1)),
-               ?assertExit({noproc, _}, pooler:take_member(test_pool_2)),
-               ?assertEqual(error_no_members,
-                            pooler:take_group_member(group_1))
-       end},
-
-      {"rm_group with existing non-empty group",
-       fun() ->
-               %% Verify that group members exist
-               MemberPid = pooler:take_group_member(group_1),
-               ?assert(is_pid(MemberPid)),
-               pooler:return_group_member(group_1, MemberPid),
-
-               Pool1Pid = pooler:take_member(test_pool_1),
-               ?assert(is_pid(Pool1Pid)),
-               pooler:return_member(test_pool_1, Pool1Pid),
-
-               Pool2Pid = pooler:take_member(test_pool_2),
-               ?assert(is_pid(Pool2Pid)),
-               pooler:return_member(test_pool_2, Pool2Pid),
-
-               %% Delete and verify that group and pools are destroyed
-               ?assertEqual(ok, pooler:rm_group(group_1)),
-
-               ?assertExit({noproc, _}, pooler:take_member(test_pool_1)),
-               ?assertExit({noproc, _}, pooler:take_member(test_pool_2)),
-               ?assertEqual(error_no_members,
-                            pooler:take_group_member(group_1))
-       end}
-     ]}}.
+        fun() ->
+            logger:set_handler_config(default, filters, []),
+            application:set_env(pooler, metrics_module, fake_metrics),
+            fake_metrics:start_link()
+        end,
+        fun(_X) ->
+            fake_metrics:stop()
+        end,
+        {foreach,
+            % setup
+            fun() ->
+                Pools = [
+                    [
+                        {name, test_pool_1},
+                        {group, group_1},
+                        {max_count, 3},
+                        {init_count, 2},
+                        {start_mfa, {pooled_gs, start_link, [{"type-1-1"}]}}
+                    ],
+                    [
+                        {name, test_pool_2},
+                        {group, group_1},
+                        {max_count, 3},
+                        {init_count, 2},
+                        {start_mfa, {pooled_gs, start_link, [{"type-1-2"}]}}
+                    ],
+                    %% test_pool_3 not part of the group
+                    [
+                        {name, test_pool_3},
+                        {group, undefined},
+                        {max_count, 3},
+                        {init_count, 2},
+                        {start_mfa, {pooled_gs, start_link, [{"type-3"}]}}
+                    ]
+                ],
+                application:set_env(pooler, pools, Pools),
+                pg_start(),
+                application:start(pooler)
+            end,
+            fun(_X) ->
+                application:stop(pooler),
+                pg_stop()
+            end,
+            [
+                {"take and return one group member (repeated)", fun() ->
+                    Types = [
+                        begin
+                            Pid = pooler:take_group_member(group_1),
+                            ?assert(is_pid(Pid), [{result, Pid}, {i, I}]),
+                            {Type, _} = pooled_gs:get_id(Pid),
+                            ?assertMatch("type-1" ++ _, Type),
+                            ok = pooler:return_group_member(group_1, Pid, ok),
+                            timer:sleep(10),
+                            Type
+                        end
+                     || I <- lists:seq(1, 50)
+                    ],
+                    Type_1_1 = [X || "type-1-1" = X <- Types],
+                    Type_1_2 = [X || "type-1-2" = X <- Types],
+                    ?assert(length(Type_1_1) > 0, [{types, Types}]),
+                    ?assert(length(Type_1_2) > 0, [{types, Types}])
+                end},
+
+                {"take member from unknown group", fun() ->
+                    ?assertEqual(
+                        error_no_members,
+                        pooler:take_group_member(not_a_group)
+                    )
+                end},
+
+                {"return member to unknown group", fun() ->
+                    Pid = pooler:take_group_member(group_1),
+                    ?assertEqual(ok, pooler:return_group_member(no_such_group, Pid))
+                end},
+
+                {"return member to wrong group", fun() ->
+                    Pid = pooler:take_member(test_pool_3),
+                    ?assertEqual(ok, pooler:return_group_member(group_1, Pid))
+                end},
+
+                {"return member with something which is not a pid", fun() ->
+                    ?assertException(error, _, pooler:return_group_member(group_1, not_pid))
+                end},
+
+                {"take member from empty group", fun() ->
+                    %% artificially empty group member list
+                    [pg_leave(group_1, M) || M <- pg_get_members(group_1)],
+                    ?assertEqual(error_no_members, pooler:take_group_member(group_1))
+                end},
+
+                {"return member to group, implied ok", fun() ->
+                    Pid = pooler:take_group_member(group_1),
+                    ?assertEqual(ok, pooler:return_group_member(group_1, Pid))
+                end},
+
+                {"return error_no_member to group", fun() ->
+                    ?assertEqual(ok, pooler:return_group_member(group_1, error_no_members))
+                end},
+
+                {"exhaust pools in group", fun() ->
+                    Pids = get_n_pids_group(group_1, 6, []),
+                    %% they should all be pids
+                    [
+                        begin
+                            {Type, _} = pooled_gs:get_id(P),
+                            ?assertMatch("type-1" ++ _, Type),
+                            ok
+                        end
+                     || P <- Pids
+                    ],
+                    %% further attempts should be error
+                    [
+                        error_no_members,
+                        error_no_members,
+                        error_no_members
+                    ] = [
+                        pooler:take_group_member(group_1)
+                     || _I <- lists:seq(1, 3)
+                    ]
+                end},
+
+                {"rm_group with nonexisting group", fun() ->
+                    ?assertEqual(ok, pooler:rm_group(i_dont_exist))
+                end},
+
+                {"rm_group with existing empty group", fun() ->
+                    ?assertEqual(ok, pooler:rm_pool(test_pool_1)),
+                    ?assertEqual(ok, pooler:rm_pool(test_pool_2)),
+                    % process group de-registration is asynchronous
+                    timer:sleep(100),
+                    ?assertEqual(error_no_members, pooler:take_group_member(group_1)),
+                    ?assertEqual(ok, pooler:rm_group(group_1)),
+
+                    ?assertExit({noproc, _}, pooler:take_member(test_pool_1)),
+                    ?assertExit({noproc, _}, pooler:take_member(test_pool_2)),
+                    ?assertEqual(
+                        error_no_members,
+                        pooler:take_group_member(group_1)
+                    )
+                end},
+
+                {"rm_group with existing non-empty group", fun() ->
+                    %% Verify that group members exist
+                    MemberPid = pooler:take_group_member(group_1),
+                    ?assert(is_pid(MemberPid)),
+                    pooler:return_group_member(group_1, MemberPid),
+
+                    Pool1Pid = pooler:take_member(test_pool_1),
+                    ?assert(is_pid(Pool1Pid)),
+                    pooler:return_member(test_pool_1, Pool1Pid),
+
+                    Pool2Pid = pooler:take_member(test_pool_2),
+                    ?assert(is_pid(Pool2Pid)),
+                    pooler:return_member(test_pool_2, Pool2Pid),
+
+                    %% Delete and verify that group and pools are destroyed
+                    ?assertEqual(ok, pooler:rm_group(group_1)),
+
+                    ?assertExit({noproc, _}, pooler:take_member(test_pool_1)),
+                    ?assertExit({noproc, _}, pooler:take_member(test_pool_2)),
+                    ?assertEqual(
+                        error_no_members,
+                        pooler:take_group_member(group_1)
+                    )
+                end}
+            ]}}.
 
 pooler_limit_failed_adds_test_() ->
     %% verify that pooler crashes completely if too many failures are
     %% encountered while trying to add pids.
     {setup,
-     fun() ->
-             logger:set_handler_config(default, filters, []),
-             Pools = [[{name, test_pool_1},
-                       {max_count, 10},
-                       {init_count, 10},
-                       {start_mfa,
-                        {pooled_gs, start_link, [crash]}}]],
-             application:set_env(pooler, pools, Pools)
-     end,
-     fun(_) ->
-             application:stop(pooler)
-     end,
-     fun() ->
-             application:start(pooler),
-             ?assertEqual(error_no_members, pooler:take_member(test_pool_1)),
-             ?assertEqual(error_no_members, pooler:take_member(test_pool_1))
-     end}.
+        fun() ->
+            logger:set_handler_config(default, filters, []),
+            Pools = [
+                [
+                    {name, test_pool_1},
+                    {max_count, 10},
+                    {init_count, 10},
+                    {start_mfa, {pooled_gs, start_link, [crash]}}
+                ]
+            ],
+            application:set_env(pooler, pools, Pools)
+        end,
+        fun(_) ->
+            application:stop(pooler)
+        end,
+        fun() ->
+            application:start(pooler),
+            ?assertEqual(error_no_members, pooler:take_member(test_pool_1)),
+            ?assertEqual(error_no_members, pooler:take_member(test_pool_1))
+        end}.
 
 pooler_scheduled_cull_test_() ->
     {setup,
-     fun() ->
-             logger:set_handler_config(default, filters, []),
-             application:set_env(pooler, metrics_module, fake_metrics),
-             fake_metrics:start_link(),
-             Pools = [[{name, test_pool_1},
-                       {max_count, 10},
-                       {init_count, 2},
-                       {start_mfa, {pooled_gs, start_link, [{"type-0"}]}},
-                       {cull_interval, {200, ms}},
-                       {max_age, {0, min}}]],
-             application:set_env(pooler, pools, Pools),
-             application:start(pooler)
-     end,
-     fun(_X) ->
-             fake_metrics:stop(),
-             application:stop(pooler)
-     end,
-     [
-       {foreach,
         fun() ->
-                Pids = get_n_pids(test_pool_1, 10, []),
-                ?assertEqual(10, length(pooler:pool_stats(test_pool_1))),
-                ?assertEqual(10, length(Pids)),
-                Pids
+            logger:set_handler_config(default, filters, []),
+            application:set_env(pooler, metrics_module, fake_metrics),
+            fake_metrics:start_link(),
+            Pools = [
+                [
+                    {name, test_pool_1},
+                    {max_count, 10},
+                    {init_count, 2},
+                    {start_mfa, {pooled_gs, start_link, [{"type-0"}]}},
+                    {cull_interval, {200, ms}},
+                    {max_age, {0, min}}
+                ]
+            ],
+            application:set_env(pooler, pools, Pools),
+            application:start(pooler)
         end,
-        fun(Pids) ->
-                [ pooler:return_member(test_pool_1, P) || P <- Pids ]
+        fun(_X) ->
+            fake_metrics:stop(),
+            application:stop(pooler)
         end,
         [
-         fun(Pids) ->
-                 {"excess members are culled run 1",
-                  fun() ->
-                          [ pooler:return_member(test_pool_1, P) || P <- Pids ],
-                          %% wait for longer than cull delay
-                          timer:sleep(250),
-                          ?assertEqual(2, length(pooler:pool_stats(test_pool_1))),
-                          ?assertEqual(2, ?gv(free_count,pooler:pool_utilization(test_pool_1)))
-                  end}
-         end,
-
-         fun(Pids) ->
-                 {"excess members are culled run 2",
-                  fun() ->
-                          [ pooler:return_member(test_pool_1, P) || P <- Pids ],
-                          %% wait for longer than cull delay
-                          timer:sleep(250),
-                          ?assertEqual(2, length(pooler:pool_stats(test_pool_1))),
-                          ?assertEqual(2, ?gv(free_count,pooler:pool_utilization(test_pool_1)))
-                  end}
-         end,
-
-         fun(Pids) -> in_use_members_not_culled(Pids, 1) end,
-         fun(Pids) -> in_use_members_not_culled(Pids, 2) end,
-         fun(Pids) -> in_use_members_not_culled(Pids, 3) end,
-         fun(Pids) -> in_use_members_not_culled(Pids, 4) end,
-         fun(Pids) -> in_use_members_not_culled(Pids, 5) end,
-         fun(Pids) -> in_use_members_not_culled(Pids, 6) end
-        ]},
-
-      {"no cull when init_count matches max_count",
-       %% not sure how to verify this. But this test at least
-       %% exercises the code path.
-       fun() ->
-               Config = [{name, test_static_pool_1},
-                         {max_count, 2},
-                         {init_count, 2},
-                         {start_mfa, {pooled_gs, start_link, [{"static-0"}]}},
-                         {cull_interval, {200, ms}}], % ignored
-               pooler:new_pool(Config),
-               P = pooler:take_member(test_static_pool_1),
-               ?assertMatch({"static-0", _}, pooled_gs:get_id(P)),
-               pooler:return_member(test_static_pool_1, P),
-               ok
-       end}
-     ]}.
+            {foreach,
+                fun() ->
+                    Pids = get_n_pids(test_pool_1, 10, []),
+                    ?assertEqual(10, length(pooler:pool_stats(test_pool_1))),
+                    ?assertEqual(10, length(Pids)),
+                    Pids
+                end,
+                fun(Pids) ->
+                    [pooler:return_member(test_pool_1, P) || P <- Pids]
+                end,
+                [
+                    fun(Pids) ->
+                        {"excess members are culled run 1", fun() ->
+                            [pooler:return_member(test_pool_1, P) || P <- Pids],
+                            %% wait for longer than cull delay
+                            timer:sleep(250),
+                            ?assertEqual(2, length(pooler:pool_stats(test_pool_1))),
+                            ?assertEqual(2, ?gv(free_count, pooler:pool_utilization(test_pool_1)))
+                        end}
+                    end,
+
+                    fun(Pids) ->
+                        {"excess members are culled run 2", fun() ->
+                            [pooler:return_member(test_pool_1, P) || P <- Pids],
+                            %% wait for longer than cull delay
+                            timer:sleep(250),
+                            ?assertEqual(2, length(pooler:pool_stats(test_pool_1))),
+                            ?assertEqual(2, ?gv(free_count, pooler:pool_utilization(test_pool_1)))
+                        end}
+                    end,
+
+                    fun(Pids) -> in_use_members_not_culled(Pids, 1) end,
+                    fun(Pids) -> in_use_members_not_culled(Pids, 2) end,
+                    fun(Pids) -> in_use_members_not_culled(Pids, 3) end,
+                    fun(Pids) -> in_use_members_not_culled(Pids, 4) end,
+                    fun(Pids) -> in_use_members_not_culled(Pids, 5) end,
+                    fun(Pids) -> in_use_members_not_culled(Pids, 6) end
+                ]},
+
+            {"no cull when init_count matches max_count",
+                %% not sure how to verify this. But this test at least
+                %% exercises the code path.
+                fun() ->
+                    Config = [
+                        {name, test_static_pool_1},
+                        {max_count, 2},
+                        {init_count, 2},
+                        {start_mfa, {pooled_gs, start_link, [{"static-0"}]}},
+                        % ignored
+                        {cull_interval, {200, ms}}
+                    ],
+                    pooler:new_pool(Config),
+                    P = pooler:take_member(test_static_pool_1),
+                    ?assertMatch({"static-0", _}, pooled_gs:get_id(P)),
+                    pooler:return_member(test_static_pool_1, P),
+                    ok
+                end}
+        ]}.
 
 in_use_members_not_culled(Pids, N) ->
-    {"in-use members are not culled " ++ erlang:integer_to_list(N),
-     fun() ->
-             %% wait for longer than cull delay
-             timer:sleep(250),
-             PidCount = length(Pids),
-             ?assertEqual(PidCount,
-                          length(pooler:pool_stats(test_pool_1))),
-             ?assertEqual(0, ?gv(free_count,pooler:pool_utilization(test_pool_1))),
-             ?assertEqual(PidCount, ?gv(in_use_count,pooler:pool_utilization(test_pool_1))),
-             Returns = lists:sublist(Pids, N),
-             [ pooler:return_member(test_pool_1, P)
-               || P <- Returns ],
-             timer:sleep(250),
-
-             ?assertEqual(PidCount - N,
-                          length(pooler:pool_stats(test_pool_1)))
-     end}.
+    {"in-use members are not culled " ++ erlang:integer_to_list(N), fun() ->
+        %% wait for longer than cull delay
+        timer:sleep(250),
+        PidCount = length(Pids),
+        ?assertEqual(
+            PidCount,
+            length(pooler:pool_stats(test_pool_1))
+        ),
+        ?assertEqual(0, ?gv(free_count, pooler:pool_utilization(test_pool_1))),
+        ?assertEqual(PidCount, ?gv(in_use_count, pooler:pool_utilization(test_pool_1))),
+        Returns = lists:sublist(Pids, N),
+        [
+            pooler:return_member(test_pool_1, P)
+         || P <- Returns
+        ],
+        timer:sleep(250),
 
+        ?assertEqual(
+            PidCount - N,
+            length(pooler:pool_stats(test_pool_1))
+        )
+    end}.
 
 random_message_test_() ->
     {setup,
-     fun() ->
-             logger:set_handler_config(default, filters, []),
-             Pools = [[{name, test_pool_1},
-                       {max_count, 2},
-                       {init_count, 1},
-                       {start_mfa,
-                        {pooled_gs, start_link, [{"type-0"}]}}]],
-             application:set_env(pooler, pools, Pools),
-             application:start(pooler),
-             %% now send some bogus messages
-             %% do the call in a throw-away process to avoid timeout error
-             spawn(fun() -> catch gen_server:call(test_pool_1, {unexpected_garbage_msg, 5}) end),
-             gen_server:cast(test_pool_1, {unexpected_garbage_msg, 6}),
-             whereis(test_pool_1) ! {unexpected_garbage_msg, 7},
-             ok
-     end,
-     fun(_) ->
-             application:stop(pooler)
-     end,
-    [
-     fun() ->
-             Pid = spawn(fun() -> ok end),
-             MonMsg = {'DOWN', erlang:make_ref(), process, Pid, because},
-             test_pool_1 ! MonMsg
-     end,
-
-     fun() ->
-             Pid = pooler:take_member(test_pool_1),
-             {Type, _} =  pooled_gs:get_id(Pid),
-             ?assertEqual("type-0", Type)
-     end,
-
-     fun() ->
-             RawPool = gen_server:call(test_pool_1, dump_pool),
-             ?assertEqual(pool, element(1, RawPool))
-     end
-    ]}.
+        fun() ->
+            logger:set_handler_config(default, filters, []),
+            Pools = [
+                [
+                    {name, test_pool_1},
+                    {max_count, 2},
+                    {init_count, 1},
+                    {start_mfa, {pooled_gs, start_link, [{"type-0"}]}}
+                ]
+            ],
+            application:set_env(pooler, pools, Pools),
+            application:start(pooler),
+            %% now send some bogus messages
+            %% do the call in a throw-away process to avoid timeout error
+            spawn(fun() -> catch gen_server:call(test_pool_1, {unexpected_garbage_msg, 5}) end),
+            gen_server:cast(test_pool_1, {unexpected_garbage_msg, 6}),
+            whereis(test_pool_1) ! {unexpected_garbage_msg, 7},
+            ok
+        end,
+        fun(_) ->
+            application:stop(pooler)
+        end,
+        [
+            fun() ->
+                Pid = spawn(fun() -> ok end),
+                MonMsg = {'DOWN', erlang:make_ref(), process, Pid, because},
+                test_pool_1 ! MonMsg
+            end,
+
+            fun() ->
+                Pid = pooler:take_member(test_pool_1),
+                {Type, _} = pooled_gs:get_id(Pid),
+                ?assertEqual("type-0", Type)
+            end,
+
+            fun() ->
+                RawPool = gen_server:call(test_pool_1, dump_pool),
+                ?assertEqual(pool, element(1, RawPool))
+            end
+        ]}.
 
 pooler_integration_long_init_test_() ->
     {foreach,
-     % setup
-     fun() ->
-             logger:set_handler_config(default, filters, []),
-             {ok, _} = error_logger_mon:start_link(),
-             error_logger_mon:install_handler(pooler),
-             Pool = [{name, test_pool_1},
-                       {max_count, 10},
-                       {init_count, 0},
-                       {member_start_timeout, {10, ms}},
-                       {start_mfa,
-                        {pooled_gs, start_link, [{"type-0", fun() -> timer:sleep(15) end}]}}],
-
-             application:set_env(pooler, pools, [Pool]),
-             application:start(pooler)
-     end,
-     % cleanup
-     fun(_) ->
-             error_logger_mon:uninstall_handler(),
-             ok = error_logger_mon:stop(),
-             application:stop(pooler)
-     end,
-     %
-     [
-      fun(_) ->
-              % Test what happens when pool members take too long to start.
-              % The pooler_starter should kill off stale members, there by
-              % reducing the number of children of the member_sup. This
-              % activity occurs both during take member and accept member.
-              % Accordingly, the count should go to zero once all starters
-              % check in.
-              fun() ->
-                      ?assertEqual(0, children_count(pooler_test_pool_1_member_sup)),
-                      [begin
-                           ?assertEqual(error_no_members, pooler:take_member(test_pool_1)),
-                           ?assertEqual(1, starting_members(test_pool_1))
-                       end
-                               || _ <- lists:seq(1,10)],
-                      ?assertEqual(10, children_count(pooler_test_pool_1_member_sup)),
-
-                      timer:sleep(150),
-                      ?assertEqual(0, children_count(pooler_test_pool_1_member_sup)),
-                      ?assertEqual(0, starting_members(test_pool_1)),
-                      %% there is a log when worker start times out
-                      ?assert(lists:any(
-                                fun(#{level := error,
-                                      msg := {report, #{label := "starting member timeout",
-                                                        pool := test_pool_1}}}) ->
-                                        true;
-                                   (_) ->
-                                        false
-                                end, error_logger_mon:get_msgs()))
-              end
-      end
-     ]
-     }.
+        % setup
+        fun() ->
+            logger:set_handler_config(default, filters, []),
+            {ok, _} = error_logger_mon:start_link(),
+            error_logger_mon:install_handler(pooler),
+            Pool = [
+                {name, test_pool_1},
+                {max_count, 10},
+                {init_count, 0},
+                {member_start_timeout, {10, ms}},
+                {start_mfa, {pooled_gs, start_link, [{"type-0", fun() -> timer:sleep(15) end}]}}
+            ],
+
+            application:set_env(pooler, pools, [Pool]),
+            application:start(pooler)
+        end,
+        % cleanup
+        fun(_) ->
+            error_logger_mon:uninstall_handler(),
+            ok = error_logger_mon:stop(),
+            application:stop(pooler)
+        end,
+        %
+        [
+            fun(_) ->
+                % Test what happens when pool members take too long to start.
+                % The pooler_starter should kill off stale members, there by
+                % reducing the number of children of the member_sup. This
+                % activity occurs both during take member and accept member.
+                % Accordingly, the count should go to zero once all starters
+                % check in.
+                fun() ->
+                    ?assertEqual(0, children_count(pooler_test_pool_1_member_sup)),
+                    [
+                        begin
+                            ?assertEqual(error_no_members, pooler:take_member(test_pool_1)),
+                            ?assertEqual(1, starting_members(test_pool_1))
+                        end
+                     || _ <- lists:seq(1, 10)
+                    ],
+                    ?assertEqual(10, children_count(pooler_test_pool_1_member_sup)),
+
+                    timer:sleep(150),
+                    ?assertEqual(0, children_count(pooler_test_pool_1_member_sup)),
+                    ?assertEqual(0, starting_members(test_pool_1)),
+                    %% there is a log when worker start times out
+                    ?assert(
+                        lists:any(
+                            fun
+                                (
+                                    #{
+                                        level := error,
+                                        msg :=
+                                            {report, #{
+                                                label := "starting member timeout",
+                                                pool := test_pool_1
+                                            }}
+                                    }
+                                ) ->
+                                    true;
+                                (_) ->
+                                    false
+                            end,
+                            error_logger_mon:get_msgs()
+                        )
+                    )
+                end
+            end
+        ]}.
 
 sleep_for_configured_timeout() ->
-    SleepTime = case application:get_env(pooler, sleep_time) of
-                    {ok, Val} ->
-                        Val;
-                    _  ->
-                        0
-                end,
+    SleepTime =
+        case application:get_env(pooler, sleep_time) of
+            {ok, Val} ->
+                Val;
+            _ ->
+                0
+        end,
     timer:sleep(SleepTime).
 
 pooler_integration_queueing_test_() ->
     {foreach,
-     % setup
-     fun() ->
-             logger:set_handler_config(default, filters, []),
-             Pool = [{name, test_pool_1},
-                     {max_count, 10},
-                     {queue_max, 10},
-                     {init_count, 0},
-                     {metrics, fake_metrics},
-                     {member_start_timeout, {5, sec}},
-                     {start_mfa,
-                      {pooled_gs, start_link, [
-                                               {"type-0",
-                                                fun pooler_tests:sleep_for_configured_timeout/0 }
-                                              ]
-                      }
-                     }
+        % setup
+        fun() ->
+            logger:set_handler_config(default, filters, []),
+            Pool = [
+                {name, test_pool_1},
+                {max_count, 10},
+                {queue_max, 10},
+                {init_count, 0},
+                {metrics, fake_metrics},
+                {member_start_timeout, {5, sec}},
+                {start_mfa,
+                    {pooled_gs, start_link, [
+                        {"type-0", fun pooler_tests:sleep_for_configured_timeout/0}
+                    ]}}
+            ],
+
+            application:set_env(pooler, pools, [Pool]),
+            fake_metrics:start_link(),
+            application:start(pooler)
+        end,
+        % cleanup
+        fun(_) ->
+            fake_metrics:stop(),
+            application:stop(pooler)
+        end,
+        [
+            fun(_) ->
+                fun() ->
+                    ?assertEqual(0, (dump_pool(test_pool_1))#pool.free_count),
+                    Val = pooler:take_member(test_pool_1, 10),
+                    ?assert(is_pid(Val)),
+                    pooler:return_member(test_pool_1, Val)
+                end
+            end,
+            fun(_) ->
+                fun() ->
+                    application:set_env(pooler, sleep_time, 1),
+                    ?assertEqual(0, (dump_pool(test_pool_1))#pool.free_count),
+                    Val = pooler:take_member(test_pool_1, 0),
+                    ?assertEqual(error_no_members, Val),
+                    ?assertEqual(0, ?gv(free_count, pooler:pool_utilization(test_pool_1))),
+                    timer:sleep(50),
+                    %Next request should be available
+                    Pid = pooler:take_member(test_pool_1, 0),
+                    ?assert(is_pid(Pid)),
+                    pooler:return_member(test_pool_1, Pid)
+                end
+            end,
+            fun(_) ->
+                fun() ->
+                    application:set_env(pooler, sleep_time, 10),
+                    ?assertEqual(0, (dump_pool(test_pool_1))#pool.free_count),
+                    [
+                        ?assertEqual(pooler:take_member(test_pool_1, 0), error_no_members)
+                     || _ <- lists:seq(1, (dump_pool(test_pool_1))#pool.max_count)
                     ],
-
-             application:set_env(pooler, pools, [Pool]),
-             fake_metrics:start_link(),
-             application:start(pooler)
-     end,
-     % cleanup
-     fun(_) ->
-             fake_metrics:stop(),
-             application:stop(pooler)
-     end,
-     [
-      fun(_) ->
-              fun() ->
-                      ?assertEqual(0, (dump_pool(test_pool_1))#pool.free_count),
-                      Val = pooler:take_member(test_pool_1, 10),
-                      ?assert(is_pid(Val)),
-                      pooler:return_member(test_pool_1, Val)
-              end
-      end,
-      fun(_) ->
-              fun() ->
-                      application:set_env(pooler, sleep_time, 1),
-                      ?assertEqual(0, (dump_pool(test_pool_1))#pool.free_count),
-                      Val = pooler:take_member(test_pool_1, 0),
-                      ?assertEqual(error_no_members, Val),
-                      ?assertEqual(0, ?gv(free_count,pooler:pool_utilization(test_pool_1))),
-                      timer:sleep(50),
-                      %Next request should be available
-                      Pid = pooler:take_member(test_pool_1, 0),
-                      ?assert(is_pid(Pid)),
-                      pooler:return_member(test_pool_1, Pid)
-              end
-      end,
-      fun(_) ->
-              fun() ->
-                      application:set_env(pooler, sleep_time, 10),
-                      ?assertEqual(0, (dump_pool(test_pool_1))#pool.free_count),
-                      [
-                       ?assertEqual(pooler:take_member(test_pool_1, 0), error_no_members) ||
-                          _ <- lists:seq(1, (dump_pool(test_pool_1))#pool.max_count)],
-                      timer:sleep(50),
-                      %Next request should be available
-                      Pid = pooler:take_member(test_pool_1, 0),
-                      ?assert(is_pid(Pid)),
-                      pooler:return_member(test_pool_1, Pid)
-              end
-      end,
-      fun(_) ->
-              fun() ->
-                      % fill to queue_max, next request should return immediately with no_members
-                      % Will return a if queue max is not enforced.
-                      application:set_env(pooler, sleep_time, 100),
-                      [ proc_lib:spawn(fun() ->
-                                               Val = pooler:take_member(test_pool_1, 200),
-                                               ?assert(is_pid(Val)),
-                                               pooler:return_member(Val)
-                                       end)
-                        || _ <- lists:seq(1, (dump_pool(test_pool_1))#pool.max_count)
-                      ],
-                      ?assertEqual(0, ?gv(free_count,pooler:pool_utilization(test_pool_1))),
-                      ?assert(?gv(queued_count,pooler:pool_utilization(test_pool_1)) >=  1),
-                      ?assertEqual(10, ?gv(queue_max,pooler:pool_utilization(test_pool_1))),
-
-                      timer:sleep(50),
-                      ?assertEqual(10, queue:len((dump_pool(test_pool_1))#pool.queued_requestors)),
-                      ?assertEqual(10, ?gv(queue_max,pooler:pool_utilization(test_pool_1))),
-
-                      ?assertEqual(pooler:take_member(test_pool_1, 500), error_no_members),
-                      ExpectKeys = lists:sort([<<"pooler.test_pool_1.error_no_members_count">>,
-                                               <<"pooler.test_pool_1.events">>,
-                                               <<"pooler.test_pool_1.take_rate">>,
-                                               <<"pooler.test_pool_1.queue_count">>,
-                                               <<"pooler.test_pool_1.queue_max_reached">>]),
-                      Metrics = fake_metrics:get_metrics(),
-                      GotKeys = lists:usort([ Name || {Name, _, _} <- Metrics ]),
-                      ?assertEqual(ExpectKeys, GotKeys),
-
-                      timer:sleep(100),
-                      Val = pooler:take_member(test_pool_1, 500),
-                      ?assert(is_pid(Val)),
-                      pooler:return_member(test_pool_1, Val)
-              end
-      end
-     ]
-    }.
+                    timer:sleep(50),
+                    %Next request should be available
+                    Pid = pooler:take_member(test_pool_1, 0),
+                    ?assert(is_pid(Pid)),
+                    pooler:return_member(test_pool_1, Pid)
+                end
+            end,
+            fun(_) ->
+                fun() ->
+                    % fill to queue_max, next request should return immediately with no_members
+                    % Will return a if queue max is not enforced.
+                    application:set_env(pooler, sleep_time, 100),
+                    [
+                        proc_lib:spawn(fun() ->
+                            Val = pooler:take_member(test_pool_1, 200),
+                            ?assert(is_pid(Val)),
+                            pooler:return_member(Val)
+                        end)
+                     || _ <- lists:seq(1, (dump_pool(test_pool_1))#pool.max_count)
+                    ],
+                    ?assertEqual(0, ?gv(free_count, pooler:pool_utilization(test_pool_1))),
+                    ?assert(?gv(queued_count, pooler:pool_utilization(test_pool_1)) >= 1),
+                    ?assertEqual(10, ?gv(queue_max, pooler:pool_utilization(test_pool_1))),
+
+                    timer:sleep(50),
+                    ?assertEqual(10, queue:len((dump_pool(test_pool_1))#pool.queued_requestors)),
+                    ?assertEqual(10, ?gv(queue_max, pooler:pool_utilization(test_pool_1))),
+
+                    ?assertEqual(pooler:take_member(test_pool_1, 500), error_no_members),
+                    ExpectKeys = lists:sort([
+                        <<"pooler.test_pool_1.error_no_members_count">>,
+                        <<"pooler.test_pool_1.events">>,
+                        <<"pooler.test_pool_1.take_rate">>,
+                        <<"pooler.test_pool_1.queue_count">>,
+                        <<"pooler.test_pool_1.queue_max_reached">>
+                    ]),
+                    Metrics = fake_metrics:get_metrics(),
+                    GotKeys = lists:usort([Name || {Name, _, _} <- Metrics]),
+                    ?assertEqual(ExpectKeys, GotKeys),
+
+                    timer:sleep(100),
+                    Val = pooler:take_member(test_pool_1, 500),
+                    ?assert(is_pid(Val)),
+                    pooler:return_member(test_pool_1, Val)
+                end
+            end
+        ]}.
 pooler_integration_queueing_return_member_test_() ->
     {foreach,
-     % setup
-     fun() ->
-             logger:set_handler_config(default, filters, []),
-             Pool = [{name, test_pool_1},
-                     {max_count, 10},
-                     {queue_max, 10},
-                     {init_count, 10},
-                     {metrics, fake_metrics},
-                     {member_start_timeout, {5, sec}},
-                     {start_mfa,
-                      {pooled_gs, start_link, [
-                                               {"type-0",
-                                                fun pooler_tests:sleep_for_configured_timeout/0 }
-                                              ]
-                      }
-                     }
+        % setup
+        fun() ->
+            logger:set_handler_config(default, filters, []),
+            Pool = [
+                {name, test_pool_1},
+                {max_count, 10},
+                {queue_max, 10},
+                {init_count, 10},
+                {metrics, fake_metrics},
+                {member_start_timeout, {5, sec}},
+                {start_mfa,
+                    {pooled_gs, start_link, [
+                        {"type-0", fun pooler_tests:sleep_for_configured_timeout/0}
+                    ]}}
+            ],
+
+            application:set_env(pooler, pools, [Pool]),
+            fake_metrics:start_link(),
+            application:start(pooler)
+        end,
+        % cleanup
+        fun(_) ->
+            fake_metrics:stop(),
+            application:stop(pooler)
+        end,
+        [
+            fun(_) ->
+                fun() ->
+                    application:set_env(pooler, sleep_time, 0),
+                    Parent = self(),
+                    Pids = [
+                        proc_lib:spawn_link(fun() ->
+                            Val = pooler:take_member(test_pool_1, 200),
+                            ?assert(is_pid(Val)),
+                            Parent ! {taken, self()},
+                            receive
+                                return ->
+                                    pooler:return_member(test_pool_1, Val)
+                            after 5000 ->
+                                pooler:return_member(test_pool_1, Val)
+                            end,
+                            Parent ! {returned, self()}
+                        end)
+                     || _ <- lists:seq(1, (dump_pool(test_pool_1))#pool.max_count)
                     ],
-
-             application:set_env(pooler, pools, [Pool]),
-             fake_metrics:start_link(),
-             application:start(pooler)
-     end,
-     % cleanup
-     fun(_) ->
-             fake_metrics:stop(),
-             application:stop(pooler)
-     end,
-     [
-      fun(_) ->
-              fun() ->
-                      application:set_env(pooler, sleep_time, 0),
-                      Parent = self(),
-                      Pids = [ proc_lib:spawn_link(fun() ->
-                                               Val = pooler:take_member(test_pool_1, 200),
-                                               ?assert(is_pid(Val)),
-                                               Parent ! {taken, self()},
-                                               receive
-                                                   return ->
-                                                       pooler:return_member(test_pool_1, Val)
-                                                   after
-                                                       5000 ->
-                                                           pooler:return_member(test_pool_1, Val)
-                                               end,
-                                               Parent ! {returned, self()}
-                                       end)
-                        || _ <- lists:seq(1, (dump_pool(test_pool_1))#pool.max_count)
-                      ],
-                      [receive {taken, Pid} -> ok end || Pid <- Pids],
-                      ?assertEqual(error_no_members, pooler:take_member(test_pool_1)),
-                      proc_lib:spawn_link(fun() ->
-                                             Val = pooler:take_member(test_pool_1, 200),
-                                             Parent ! {extra_result, Val}
-                                     end),
-                      [Pid ! return || Pid <- Pids],
-                      [receive {returned, Pid} -> ok end || Pid <- Pids],
-                      receive
-                          {extra_result, Result} ->
-                              ?assert(is_pid(Result)),
-                              pooler:return_member(test_pool_1, Result)
-                      end,
-                      ?assertEqual((dump_pool(test_pool_1))#pool.max_count, length((dump_pool(test_pool_1))#pool.free_pids)),
-                      ?assertEqual((dump_pool(test_pool_1))#pool.max_count, (dump_pool(test_pool_1))#pool.free_count)
-              end
-      end
-      ]
-     }.
-
+                    [
+                        receive
+                            {taken, Pid} -> ok
+                        end
+                     || Pid <- Pids
+                    ],
+                    ?assertEqual(error_no_members, pooler:take_member(test_pool_1)),
+                    proc_lib:spawn_link(fun() ->
+                        Val = pooler:take_member(test_pool_1, 200),
+                        Parent ! {extra_result, Val}
+                    end),
+                    [Pid ! return || Pid <- Pids],
+                    [
+                        receive
+                            {returned, Pid} -> ok
+                        end
+                     || Pid <- Pids
+                    ],
+                    receive
+                        {extra_result, Result} ->
+                            ?assert(is_pid(Result)),
+                            pooler:return_member(test_pool_1, Result)
+                    end,
+                    ?assertEqual(
+                        (dump_pool(test_pool_1))#pool.max_count, length((dump_pool(test_pool_1))#pool.free_pids)
+                    ),
+                    ?assertEqual((dump_pool(test_pool_1))#pool.max_count, (dump_pool(test_pool_1))#pool.free_count)
+                end
+            end
+        ]}.
 
 pooler_integration_test_() ->
     {foreach,
-     % setup
-     fun() ->
-             logger:set_handler_config(default, filters, []),
-             Pools = [[{name, test_pool_1},
-                       {max_count, 10},
-                       {init_count, 10},
-                       {start_mfa,
-                        {pooled_gs, start_link, [{"type-0"}]}}]],
-             application:set_env(pooler, pools, Pools),
-             application:start(pooler),
-             Users = [ start_user() || _X <- lists:seq(1, 10) ],
-             Users
-     end,
-     % cleanup
-     fun(Users) ->
-             [ user_stop(U) || U <- Users ],
-             application:stop(pooler)
-     end,
-     %
-     [
-      fun(Users) ->
-             fun() ->
-                     % each user has a different tc ID
-                     TcIds = lists:sort([ user_id(UPid) || UPid <- Users ]),
-                     ?assertEqual(lists:usort(TcIds), TcIds)
-             end
-      end
-      ,
-
-      fun(Users) ->
-              fun() ->
-                      % users still unique after a renew cycle
-                      [ user_new_tc(UPid) || UPid <- Users ],
-                      TcIds = lists:sort([ user_id(UPid) || UPid <- Users ]),
-                      ?assertEqual(lists:usort(TcIds), TcIds)
-              end
-      end
-      ,
-
-      fun(Users) ->
-              fun() ->
-                      % all users crash, pids are replaced
-                      TcIds1 = lists:sort([ user_id(UPid) || UPid <- Users ]),
-                      [ user_crash(UPid) || UPid <- Users ],
-                      Seq = lists:seq(1, 5),
-                      Users2 = [ start_user() || _X <- Seq ],
-                      TcIds2 = lists:sort([ user_id(UPid) || UPid <- Users2 ]),
-                      Both =
-                          sets:to_list(sets:intersection([sets:from_list(TcIds1),
-                                                          sets:from_list(TcIds2)])),
-                      ?assertEqual([], Both)
-              end
-      end
-     ]
-    }.
+        % setup
+        fun() ->
+            logger:set_handler_config(default, filters, []),
+            Pools = [
+                [
+                    {name, test_pool_1},
+                    {max_count, 10},
+                    {init_count, 10},
+                    {start_mfa, {pooled_gs, start_link, [{"type-0"}]}}
+                ]
+            ],
+            application:set_env(pooler, pools, Pools),
+            application:start(pooler),
+            Users = [start_user() || _X <- lists:seq(1, 10)],
+            Users
+        end,
+        % cleanup
+        fun(Users) ->
+            [user_stop(U) || U <- Users],
+            application:stop(pooler)
+        end,
+        %
+        [
+            fun(Users) ->
+                fun() ->
+                    % each user has a different tc ID
+                    TcIds = lists:sort([user_id(UPid) || UPid <- Users]),
+                    ?assertEqual(lists:usort(TcIds), TcIds)
+                end
+            end,
+
+            fun(Users) ->
+                fun() ->
+                    % users still unique after a renew cycle
+                    [user_new_tc(UPid) || UPid <- Users],
+                    TcIds = lists:sort([user_id(UPid) || UPid <- Users]),
+                    ?assertEqual(lists:usort(TcIds), TcIds)
+                end
+            end,
+
+            fun(Users) ->
+                fun() ->
+                    % all users crash, pids are replaced
+                    TcIds1 = lists:sort([user_id(UPid) || UPid <- Users]),
+                    [user_crash(UPid) || UPid <- Users],
+                    Seq = lists:seq(1, 5),
+                    Users2 = [start_user() || _X <- Seq],
+                    TcIds2 = lists:sort([user_id(UPid) || UPid <- Users2]),
+                    Both =
+                        sets:to_list(
+                            sets:intersection([
+                                sets:from_list(TcIds1),
+                                sets:from_list(TcIds2)
+                            ])
+                        ),
+                    ?assertEqual([], Both)
+                end
+            end
+        ]}.
 
 pooler_auto_grow_disabled_by_default_test_() ->
     {setup,
-     fun() ->
-             logger:set_handler_config(default, filters, []),
-             application:set_env(pooler, metrics_module, fake_metrics),
-             fake_metrics:start_link()
-     end,
-     fun(_X) ->
-             fake_metrics:stop()
-     end,
-    {foreach,
-     % setup
-     fun() ->
-             Pool = [{name, test_pool_1},
-                     {max_count, 5},
-                     {init_count, 2},
-                     {start_mfa,
-                      {pooled_gs, start_link, [{"type-0"}]}}],
-             application:unset_env(pooler, pools),
-             application:start(pooler),
-             pooler:new_pool(Pool)
-     end,
-     fun(_X) ->
-             application:stop(pooler)
-     end,
-     [
-      {"take one, and it should not auto-grow",
-       fun() ->
-               ?assertEqual(2, (dump_pool(test_pool_1))#pool.free_count),
-               P = pooler:take_member(test_pool_1),
-               ?assertMatch({"type-0", _Id}, pooled_gs:get_id(P)),
-               timer:sleep(100),
-               ?assertEqual(1, (dump_pool(test_pool_1))#pool.free_count),
-               ok, pooler:return_member(test_pool_1, P)
-       end}
-     ]}}.
+        fun() ->
+            logger:set_handler_config(default, filters, []),
+            application:set_env(pooler, metrics_module, fake_metrics),
+            fake_metrics:start_link()
+        end,
+        fun(_X) ->
+            fake_metrics:stop()
+        end,
+        {foreach,
+            % setup
+            fun() ->
+                Pool = [
+                    {name, test_pool_1},
+                    {max_count, 5},
+                    {init_count, 2},
+                    {start_mfa, {pooled_gs, start_link, [{"type-0"}]}}
+                ],
+                application:unset_env(pooler, pools),
+                application:start(pooler),
+                pooler:new_pool(Pool)
+            end,
+            fun(_X) ->
+                application:stop(pooler)
+            end,
+            [
+                {"take one, and it should not auto-grow", fun() ->
+                    ?assertEqual(2, (dump_pool(test_pool_1))#pool.free_count),
+                    P = pooler:take_member(test_pool_1),
+                    ?assertMatch({"type-0", _Id}, pooled_gs:get_id(P)),
+                    timer:sleep(100),
+                    ?assertEqual(1, (dump_pool(test_pool_1))#pool.free_count),
+                    ok,
+                    pooler:return_member(test_pool_1, P)
+                end}
+            ]}}.
 
 pooler_auto_grow_enabled_test_() ->
     {setup,
-     fun() ->
-             logger:set_handler_config(default, filters, []),
-             application:set_env(pooler, metrics_module, fake_metrics),
-             fake_metrics:start_link()
-     end,
-     fun(_X) ->
-             fake_metrics:stop()
-     end,
-    {foreach,
-     % setup
-     fun() ->
-             Pool = [{name, test_pool_1},
-                     {max_count, 5},
-                     {init_count, 2},
-                     {auto_grow_threshold, 1},
-                     {start_mfa,
-                      {pooled_gs, start_link, [{"type-0"}]}}],
-             application:unset_env(pooler, pools),
-             application:start(pooler),
-             pooler:new_pool(Pool)
-     end,
-     fun(_X) ->
-             application:stop(pooler)
-     end,
-     [
-      {"take one, and it should grow by 2",
-       fun() ->
-               ?assertEqual(2, (dump_pool(test_pool_1))#pool.free_count),
-               P = pooler:take_member(test_pool_1),
-               ?assertMatch({"type-0", _Id}, pooled_gs:get_id(P)),
-               timer:sleep(100),
-               ?assertEqual(3, (dump_pool(test_pool_1))#pool.free_count),
-               ok, pooler:return_member(test_pool_1, P)
-       end}
-     ]}}.
+        fun() ->
+            logger:set_handler_config(default, filters, []),
+            application:set_env(pooler, metrics_module, fake_metrics),
+            fake_metrics:start_link()
+        end,
+        fun(_X) ->
+            fake_metrics:stop()
+        end,
+        {foreach,
+            % setup
+            fun() ->
+                Pool = [
+                    {name, test_pool_1},
+                    {max_count, 5},
+                    {init_count, 2},
+                    {auto_grow_threshold, 1},
+                    {start_mfa, {pooled_gs, start_link, [{"type-0"}]}}
+                ],
+                application:unset_env(pooler, pools),
+                application:start(pooler),
+                pooler:new_pool(Pool)
+            end,
+            fun(_X) ->
+                application:stop(pooler)
+            end,
+            [
+                {"take one, and it should grow by 2", fun() ->
+                    ?assertEqual(2, (dump_pool(test_pool_1))#pool.free_count),
+                    P = pooler:take_member(test_pool_1),
+                    ?assertMatch({"type-0", _Id}, pooled_gs:get_id(P)),
+                    timer:sleep(100),
+                    ?assertEqual(3, (dump_pool(test_pool_1))#pool.free_count),
+                    ok,
+                    pooler:return_member(test_pool_1, P)
+                end}
+            ]}}.
 
 pooler_custom_stop_mfa_test_() ->
     {foreach,
-     fun() ->
-             logger:set_handler_config(default, filters, []),
-             Pool = [{name, test_pool_1},
-                     {max_count, 3},
-                     {init_count, 2},
-                     {cull_interval, {200, ms}},
-                     {max_age, {0, min}},
-                     {start_mfa, {pooled_gs, start_link, [{foo_type}]}}],
-             application:set_env(pooler, pools, [Pool])
-     end,
-     fun(_) ->
-             application:unset_env(pooler, pools)
-     end,
-     [
-      {"default behavior kills the pool member",
-       fun() ->
-               ok = application:start(pooler),
-               Reason = monitor_members_trigger_culling_and_return_reason(),
-               ok = application:stop(pooler),
-               ?assertEqual(killed, Reason)
-       end},
-      {"custom callback terminates the pool member normally",
-       fun() ->
-               {ok, [Pool]} = application:get_env(pooler, pools),
-               Stop = {stop_mfa, {pooled_gs, stop, ['$pooler_pid']}},
-               application:set_env(pooler, pools, [[Stop | Pool]]),
-               ok = application:start(pooler),
-               Reason = monitor_members_trigger_culling_and_return_reason(),
-               ok = application:stop(pooler),
-               ?assertEqual(normal, Reason)
-       end}]}.
+        fun() ->
+            logger:set_handler_config(default, filters, []),
+            Pool = [
+                {name, test_pool_1},
+                {max_count, 3},
+                {init_count, 2},
+                {cull_interval, {200, ms}},
+                {max_age, {0, min}},
+                {start_mfa, {pooled_gs, start_link, [{foo_type}]}}
+            ],
+            application:set_env(pooler, pools, [Pool])
+        end,
+        fun(_) ->
+            application:unset_env(pooler, pools)
+        end,
+        [
+            {"default behavior kills the pool member", fun() ->
+                ok = application:start(pooler),
+                Reason = monitor_members_trigger_culling_and_return_reason(),
+                ok = application:stop(pooler),
+                ?assertEqual(killed, Reason)
+            end},
+            {"custom callback terminates the pool member normally", fun() ->
+                {ok, [Pool]} = application:get_env(pooler, pools),
+                Stop = {stop_mfa, {pooled_gs, stop, ['$pooler_pid']}},
+                application:set_env(pooler, pools, [[Stop | Pool]]),
+                ok = application:start(pooler),
+                Reason = monitor_members_trigger_culling_and_return_reason(),
+                ok = application:stop(pooler),
+                ?assertEqual(normal, Reason)
+            end}
+        ]}.
 
 no_error_logger_reports_after_culling_test_() ->
     %% damn those wraiths! This is the cure
     {foreach,
-     fun() ->
-             logger:set_handler_config(default, filters, []),
-             {ok, _Pid} = error_logger_mon:start_link(),
-             Pool = [{name, test_pool_1},
-                     {max_count, 3},
-                     {init_count, 2},
-                     {cull_interval, {200, ms}},
-                     {max_age, {0, min}},
-                     {start_mfa, {pooled_gs, start_link, [{type}]}}],
-             application:set_env(pooler, pools, [Pool])
-     end,
-     fun(_) ->
-             ok = error_logger_mon:stop(),
-             error_logger_mon:uninstall_handler(),
-             application:unset_env(pooler, pools)
-     end,
-     [
-      {"Force supervisor to report by using exit/2 instead of terminate_child",
-       fun() ->
-               {ok, [Pool]} = application:get_env(pooler, pools),
-               Stop = {stop_mfa, {erlang, exit, ['$pooler_pid', kill]}},
-               application:set_env(pooler, pools, [[Stop | Pool]]),
-               ok = application:start(pooler),
-               error_logger_mon:install_handler(),
-               error_logger_mon:reset(),
-               Reason = monitor_members_trigger_culling_and_return_reason(),
-               %% we need to wait for the message to arrive before deleting handler
-               timer:sleep(250),
-               error_logger_mon:uninstall_handler(),
-               ok = application:stop(pooler),
-               ?assertEqual(killed, Reason),
-               ?assertEqual(1, error_logger_mon:get_msg_count(),
-                            [{msgs, error_logger_mon:get_msgs()},
-                             {m, [R || #{msg := {report, R}} <- error_logger_mon:get_msgs()]}])
-       end},
-      {"Default MFA shouldn't generate any reports during culling",
-       fun() ->
-               ok = application:start(pooler),
-               error_logger_mon:install_handler(),
-               Reason = monitor_members_trigger_culling_and_return_reason(),
-               error_logger_mon:uninstall_handler(),
-               ok = application:stop(pooler),
-               ?assertEqual(killed, Reason),
-               ?assertEqual(0, error_logger_mon:get_msg_count())
-       end}
-     ]}.
-
+        fun() ->
+            logger:set_handler_config(default, filters, []),
+            {ok, _Pid} = error_logger_mon:start_link(),
+            Pool = [
+                {name, test_pool_1},
+                {max_count, 3},
+                {init_count, 2},
+                {cull_interval, {200, ms}},
+                {max_age, {0, min}},
+                {start_mfa, {pooled_gs, start_link, [{type}]}}
+            ],
+            application:set_env(pooler, pools, [Pool])
+        end,
+        fun(_) ->
+            ok = error_logger_mon:stop(),
+            error_logger_mon:uninstall_handler(),
+            application:unset_env(pooler, pools)
+        end,
+        [
+            {"Force supervisor to report by using exit/2 instead of terminate_child", fun() ->
+                {ok, [Pool]} = application:get_env(pooler, pools),
+                Stop = {stop_mfa, {erlang, exit, ['$pooler_pid', kill]}},
+                application:set_env(pooler, pools, [[Stop | Pool]]),
+                ok = application:start(pooler),
+                error_logger_mon:install_handler(),
+                error_logger_mon:reset(),
+                Reason = monitor_members_trigger_culling_and_return_reason(),
+                %% we need to wait for the message to arrive before deleting handler
+                timer:sleep(250),
+                error_logger_mon:uninstall_handler(),
+                ok = application:stop(pooler),
+                ?assertEqual(killed, Reason),
+                ?assertEqual(
+                    1,
+                    error_logger_mon:get_msg_count(),
+                    [
+                        {msgs, error_logger_mon:get_msgs()},
+                        {m, [R || #{msg := {report, R}} <- error_logger_mon:get_msgs()]}
+                    ]
+                )
+            end},
+            {"Default MFA shouldn't generate any reports during culling", fun() ->
+                ok = application:start(pooler),
+                error_logger_mon:install_handler(),
+                Reason = monitor_members_trigger_culling_and_return_reason(),
+                error_logger_mon:uninstall_handler(),
+                ok = application:stop(pooler),
+                ?assertEqual(killed, Reason),
+                ?assertEqual(0, error_logger_mon:get_msg_count())
+            end}
+        ]}.
 
 monitor_members_trigger_culling_and_return_reason() ->
     Pids = get_n_pids(test_pool_1, 3, []),
-    [ erlang:monitor(process, P) || P <- Pids ],
-    [ pooler:return_member(test_pool_1, P) || P <- Pids ],
+    [erlang:monitor(process, P) || P <- Pids],
+    [pooler:return_member(test_pool_1, P) || P <- Pids],
     receive
         {'DOWN', _Ref, process, _Pid, Reason} ->
             Reason
-    after
-        250 -> timeout
+    after 250 -> timeout
     end.
 
 time_as_millis_test_() ->
-    Zeros = [ {{0, U}, 0} || U <- [min, sec, ms, mu] ],
-    Ones = [{{1, min}, 60000},
-            {{1, sec}, 1000},
-            {{1, ms}, 1},
-            {{1, mu}, 0}],
+    Zeros = [{{0, U}, 0} || U <- [min, sec, ms, mu]],
+    Ones = [
+        {{1, min}, 60000},
+        {{1, sec}, 1000},
+        {{1, ms}, 1},
+        {{1, mu}, 0}
+    ],
     Misc = [{{3000, mu}, 3}],
     Tests = Zeros ++ Ones ++ Misc,
-    [ ?_assertEqual(E, pooler:time_as_millis(I)) || {I, E} <- Tests ].
+    [?_assertEqual(E, pooler:time_as_millis(I)) || {I, E} <- Tests].
 
 time_as_micros_test_() ->
-    Zeros = [ {{0, U}, 0} || U <- [min, sec, ms, mu] ],
-    Ones = [{{1, min}, 60000000},
-            {{1, sec}, 1000000},
-            {{1, ms}, 1000},
-            {{1, mu}, 1}],
+    Zeros = [{{0, U}, 0} || U <- [min, sec, ms, mu]],
+    Ones = [
+        {{1, min}, 60000000},
+        {{1, sec}, 1000000},
+        {{1, ms}, 1000},
+        {{1, mu}, 1}
+    ],
     Misc = [{{3000, mu}, 3000}],
     Tests = Zeros ++ Ones ++ Misc,
-    [ ?_assertEqual(E, pooler:time_as_micros(I)) || {I, E} <- Tests ].
-
+    [?_assertEqual(E, pooler:time_as_micros(I)) || {I, E} <- Tests].
 
 call_free_members_test_() ->
     NumWorkers = 10,
     PoolName = test_pool_1,
 
     {setup,
-     fun() ->
-             application:set_env(pooler, metrics_module, fake_metrics),
-             fake_metrics:start_link()
-     end,
-     fun(_X) ->
-             fake_metrics:stop()
-     end,
-     {foreach,
-      fun() ->
-              Pool = [{name, PoolName},
-                      {max_count, NumWorkers},
-                      {init_count, NumWorkers},
-                      {start_mfa,
-                       {pooled_gs, start_link, [{"type-0"}]}}],
-              application:unset_env(pooler, pools),
-              application:start(pooler),
-              pooler:new_pool(Pool)
-      end,
-      fun(_X) ->
-              application:stop(pooler)
-      end,
-      [
-       {"perform a ping across the pool when all workers are free",
         fun() ->
-                ?assertEqual(NumWorkers, (dump_pool(PoolName))#pool.free_count),
-                Res = pooler:call_free_members(PoolName, fun pooled_gs:ping/1),
-
-                ?assertEqual(NumWorkers, count_pongs(Res))
-        end},
-       {"perform a ping across the pool when two workers are taken",
-        fun() ->
-                ?assertEqual(NumWorkers, (dump_pool(PoolName))#pool.free_count),
-                Pids = [pooler:take_member(PoolName) || _X <- lists:seq(0, 1)],
-                Res = pooler:call_free_members(PoolName, fun pooled_gs:ping/1),
-
-                ?assertEqual(NumWorkers -2, count_pongs(Res)),
-
-                [pooler:return_member(PoolName, P) || P <- Pids]
-        end},
-       {"perform an op where the op crashes all free members",
-        fun() ->
-                ?assertEqual(NumWorkers, (dump_pool(PoolName))#pool.free_count),
-                Res = pooler:call_free_members(PoolName,
-                                               fun pooled_gs:error_on_call/1),
-
-                ?assertEqual(NumWorkers, count_errors(Res))
-        end}
-      ]}}.
+            application:set_env(pooler, metrics_module, fake_metrics),
+            fake_metrics:start_link()
+        end,
+        fun(_X) ->
+            fake_metrics:stop()
+        end,
+        {foreach,
+            fun() ->
+                Pool = [
+                    {name, PoolName},
+                    {max_count, NumWorkers},
+                    {init_count, NumWorkers},
+                    {start_mfa, {pooled_gs, start_link, [{"type-0"}]}}
+                ],
+                application:unset_env(pooler, pools),
+                application:start(pooler),
+                pooler:new_pool(Pool)
+            end,
+            fun(_X) ->
+                application:stop(pooler)
+            end,
+            [
+                {"perform a ping across the pool when all workers are free", fun() ->
+                    ?assertEqual(NumWorkers, (dump_pool(PoolName))#pool.free_count),
+                    Res = pooler:call_free_members(PoolName, fun pooled_gs:ping/1),
+
+                    ?assertEqual(NumWorkers, count_pongs(Res))
+                end},
+                {"perform a ping across the pool when two workers are taken", fun() ->
+                    ?assertEqual(NumWorkers, (dump_pool(PoolName))#pool.free_count),
+                    Pids = [pooler:take_member(PoolName) || _X <- lists:seq(0, 1)],
+                    Res = pooler:call_free_members(PoolName, fun pooled_gs:ping/1),
+
+                    ?assertEqual(NumWorkers - 2, count_pongs(Res)),
+
+                    [pooler:return_member(PoolName, P) || P <- Pids]
+                end},
+                {"perform an op where the op crashes all free members", fun() ->
+                    ?assertEqual(NumWorkers, (dump_pool(PoolName))#pool.free_count),
+                    Res = pooler:call_free_members(
+                        PoolName,
+                        fun pooled_gs:error_on_call/1
+                    ),
+
+                    ?assertEqual(NumWorkers, count_errors(Res))
+                end}
+            ]}}.
 
 count_pongs(Result) ->
-    lists:foldl(fun({ok, pong}, Acc) -> Acc + 1;
-                   ({error, _}, Acc) -> Acc
-                end,
-                0,
-                Result).
+    lists:foldl(
+        fun
+            ({ok, pong}, Acc) -> Acc + 1;
+            ({error, _}, Acc) -> Acc
+        end,
+        0,
+        Result
+    ).
 
 count_errors(Result) ->
-    lists:foldl(fun({error, _}, Acc) -> Acc + 1;
-                   ({ok, _}, Acc) -> Acc
-                end,
-                0,
-                Result).
+    lists:foldl(
+        fun
+            ({error, _}, Acc) -> Acc + 1;
+            ({ok, _}, Acc) -> Acc
+        end,
+        0,
+        Result
+    ).
 
 % testing crash recovery means race conditions when either pids
 % haven't yet crashed or pooler hasn't recovered.  So this helper loops
@@ -1335,7 +1404,7 @@ get_n_pids(Pool, N, Acc) ->
         error_no_members ->
             get_n_pids(Pool, N, Acc);
         Pid ->
-            get_n_pids(Pool, N - 1, [Pid|Acc])
+            get_n_pids(Pool, N - 1, [Pid | Acc])
     end.
 
 get_n_pids_group(_Group, 0, Acc) ->
@@ -1345,7 +1414,7 @@ get_n_pids_group(Group, N, Acc) ->
         error_no_members ->
             get_n_pids_group(Group, N, Acc);
         Pid ->
-            get_n_pids_group(Group, N - 1, [Pid|Acc])
+            get_n_pids_group(Group, N - 1, [Pid | Acc])
     end.
 
 children_count(SupId) ->
@@ -1357,43 +1426,47 @@ starting_members(PoolName) ->
 dump_pool(PoolName) ->
     gen_server:call(PoolName, dump_pool).
 
--ifdef(OTP_RELEASE). % >= OTP-21
+% >= OTP-21
+-ifdef(OTP_RELEASE).
 -if(?OTP_RELEASE >= 23).
 -define(USE_PG_NOT_PG2, true).
 -else.
 -undef(USE_PG_NOT_PG2).
 -endif.
--else. % < OTP-21
+% < OTP-21
+-else.
 -undef(USE_PG_NOT_PG2).
 -endif.
 
 -ifdef(USE_PG_NOT_PG2).
 
 pg_start() ->
-  pg:start(_Scope = 'pg').
+    pg:start(_Scope = 'pg').
 
 pg_stop() ->
-  lists:foreach(fun(Group) -> pg:leave(Group, pg:get_members(Group)) end,
-                pg:which_groups()).
+    lists:foreach(
+        fun(Group) -> pg:leave(Group, pg:get_members(Group)) end,
+        pg:which_groups()
+    ).
 
 pg_leave(Group, Pid) ->
-  pg:leave(Group, Pid).
+    pg:leave(Group, Pid).
 
 pg_get_members(Group) ->
-  pg:get_members(Group).
+    pg:get_members(Group).
 
 -else.
 
 pg_start() ->
-  pg2:start().
+    pg2:start().
 
 pg_stop() ->
-  application:stop(pg2).
+    application:stop(pg2).
 
 pg_leave(Group, Pid) ->
-  pg2:leave(Group, Pid).
+    pg2:leave(Group, Pid).
 
 pg_get_members(Group) ->
-  pg2:get_members(Group).
+    pg2:get_members(Group).
 
 -endif.