Add empty groups.

src/syn_gen_scope.erl

@@ -204,8 +204,8 @@ handle_info({'3.0', discover, RemoteScopePid}, #state{
     nodes = Nodes
 } = State) ->
     RemoteScopeNode = node(RemoteScopePid),
-    error_logger:info_msg("SYN[~s] Received DISCOVER request from node '~s' and scope '~s'",
-        [node(), RemoteScopeNode, Scope]
+    error_logger:info_msg("SYN[~s] Received DISCOVER request from node '~s'for ~s and scope '~s'",
+        [node(), RemoteScopeNode,Handler, Scope]
     ),
     %% send local data to remote
     {ok, LocalData} = Handler:get_local_data(State),
@@ -228,8 +228,8 @@ handle_info({'3.0', ack_sync, RemoteScopePid, Data}, #state{
     scope = Scope
 } = State) ->
     RemoteScopeNode = node(RemoteScopePid),
-    error_logger:info_msg("SYN[~s] Received ACK SYNC from node '~s' and scope '~s'",
-        [node(), RemoteScopeNode, Scope]
+    error_logger:info_msg("SYN[~s] Received ACK SYNC from node '~s' for ~s and scope '~s'",
+        [node(), RemoteScopeNode, Handler, Scope]
     ),
     %% save remote data
     Handler:save_remote_data(Data, State),
@@ -258,8 +258,8 @@ handle_info({'DOWN', MRef, process, Pid, Reason}, #state{
     RemoteNode = node(Pid),
     case maps:take(RemoteNode, Nodes) of
         {Pid, Nodes1} ->
-            error_logger:info_msg("SYN[~s] Scope Process '~s' is DOWN on node '~s': ~p",
-                [node(), Scope, RemoteNode, Reason]
+            error_logger:info_msg("SYN[~s] Scope Process '~s' for ~s is DOWN on node '~s': ~p",
+                [node(), Scope, Handler, RemoteNode, Reason]
             ),
             Handler:purge_local_data_for_node(RemoteNode, State),
             {noreply, State#state{nodes = Nodes1}};
@@ -273,9 +273,12 @@ handle_info({nodedown, _Node}, State) ->
     %% ignore & wait for monitor DOWN message
     {noreply, State};
 
-handle_info({nodeup, RemoteNode}, #state{scope = Scope} = State) ->
-    error_logger:info_msg("SYN[~s] Node '~s' has joined the cluster, sending discover message for scope '~s'",
-        [node(), RemoteNode, Scope]
+handle_info({nodeup, RemoteNode}, #state{
+    handler = Handler,
+    scope = Scope
+} = State) ->
+    error_logger:info_msg("SYN[~s] Node '~s' has joined the cluster, sending discover message for ~s and scope '~s'",
+        [node(), RemoteNode, Handler, Scope]
     ),
     send_to_node(RemoteNode, {'3.0', discover, self()}, State),
     {noreply, State};
@@ -286,8 +289,11 @@ handle_info(Info, #state{handler = Handler} = State) ->
 %% ----------------------------------------------------------------------------------------------------------
 %% Continue messages
 %% ----------------------------------------------------------------------------------------------------------
-handle_continue(after_init, #state{scope = Scope} = State) ->
-    error_logger:info_msg("SYN[~s] Discovering the cluster with scope '~s'", [node(), Scope]),
+handle_continue(after_init, #state{
+    handler = Handler,
+    scope = Scope
+} = State) ->
+    error_logger:info_msg("SYN[~s] Discovering the cluster for ~s and scope '~s'", [node(), Handler, Scope]),
     broadcast_all_cluster({'3.0', discover, self()}, State),
     {noreply, State}.
 
@@ -295,8 +301,8 @@ handle_continue(after_init, #state{scope = Scope} = State) ->
 %% Terminate
 %% ----------------------------------------------------------------------------------------------------------
 -spec terminate(Reason :: any(), #state{}) -> terminated.
-terminate(Reason, _State) ->
-    error_logger:info_msg("SYN[~s] Terminating with reason: ~p", [node(), Reason]),
+terminate(Reason, #state{handler = Handler}) ->
+    error_logger:info_msg("SYN[~s] ~s terminating with reason: ~p", [node(), Handler, Reason]),
     terminated.
 
 %% ----------------------------------------------------------------------------------------------------------

src/syn_groups.erl

@@ -0,0 +1,121 @@
+%% ==========================================================================================================
+%% Syn - A global Process Registry and Process Group manager.
+%%
+%% The MIT License (MIT)
+%%
+%% Copyright (c) 2015-2021 Roberto Ostinelli <roberto@ostinelli.net> and Neato Robotics, Inc.
+%%
+%% Permission is hereby granted, free of charge, to any person obtaining a copy
+%% of this software and associated documentation files (the "Software"), to deal
+%% in the Software without restriction, including without limitation the rights
+%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+%% copies of the Software, and to permit persons to whom the Software is
+%% furnished to do so, subject to the following conditions:
+%%
+%% The above copyright notice and this permission notice shall be included in
+%% all copies or substantial portions of the Software.
+%%
+%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+%% IMPLIED, INCLUDING BUT NOT LIMITED TO THxE WARRANTIES OF MERCHANTABILITY,
+%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+%% THE SOFTWARE.
+%% ==========================================================================================================
+-module(syn_groups).
+-behaviour(syn_gen_scope).
+
+%% API
+-export([start_link/1]).
+-export([get_subcluster_nodes/1]).
+
+%% syn_gen_scope callbacks
+-export([
+    init/1,
+    handle_call/3,
+    handle_info/2,
+    save_remote_data/2,
+    get_local_data/1,
+    purge_local_data_for_node/2
+]).
+
+%% includes
+-include("syn.hrl").
+
+%% ===================================================================
+%% API
+%% ===================================================================
+-spec start_link(Scope :: atom()) ->
+    {ok, Pid :: pid()} | {error, {already_started, Pid :: pid()}} | {error, Reason :: any()}.
+start_link(Scope) when is_atom(Scope) ->
+    syn_gen_scope:start_link(?MODULE, Scope).
+
+-spec get_subcluster_nodes(#state{}) -> [node()].
+get_subcluster_nodes(State) ->
+    syn_gen_scope:get_subcluster_nodes(?MODULE, State).
+
+%% ===================================================================
+%% Callbacks
+%% ===================================================================
+
+%% ----------------------------------------------------------------------------------------------------------
+%% Init
+%% ----------------------------------------------------------------------------------------------------------
+-spec init(State :: term()) -> {ok, State :: term()}.
+init(State) ->
+    HandlerState = #{},
+    %% init
+    {ok, HandlerState}.
+
+%% ----------------------------------------------------------------------------------------------------------
+%% Call messages
+%% ----------------------------------------------------------------------------------------------------------
+-spec handle_call(Request :: term(), From :: {pid(), Tag :: term()},
+    State :: term()) ->
+    {reply, Reply :: term(), NewState :: term()} |
+    {reply, Reply :: term(), NewState :: term(), timeout() | hibernate | {continue, term()}} |
+    {noreply, NewState :: term()} |
+    {noreply, NewState :: term(), timeout() | hibernate | {continue, term()}} |
+    {stop, Reason :: term(), Reply :: term(), NewState :: term()} |
+    {stop, Reason :: term(), NewState :: term()}.
+
+handle_call(Request, From, State) ->
+    error_logger:warning_msg("SYN[~s] Received from ~p an unknown call message: ~p", [node(), From, Request]),
+    {reply, undefined, State}.
+
+%% ----------------------------------------------------------------------------------------------------------
+%% Info messages
+%% ----------------------------------------------------------------------------------------------------------
+-spec handle_info(Info :: timeout | term(), State :: term()) ->
+    {noreply, NewState :: term()} |
+    {noreply, NewState :: term(), timeout() | hibernate | {continue, term()}} |
+    {stop, Reason :: term(), NewState :: term()}.
+
+handle_info(Info, State) ->
+    error_logger:warning_msg("SYN[~s] Received an unknown info message: ~p", [node(), Info]),
+    {noreply, State}.
+
+%% ----------------------------------------------------------------------------------------------------------
+%% Data
+%% ----------------------------------------------------------------------------------------------------------
+-spec get_local_data(State :: term()) -> {ok, Data :: any()} | undefined.
+get_local_data(#state{table_by_name = TableByName}) ->
+    {ok, []}.
+
+-spec save_remote_data(RemoteData :: any(), State :: term()) -> any().
+save_remote_data(RegistryTuplesOfRemoteNode, #state{scope = Scope} = State) ->
+    %% insert tuples
+    ok.
+
+-spec purge_local_data_for_node(Node :: node(), State :: term()) -> any().
+purge_local_data_for_node(Node, #state{
+    scope = Scope,
+    table_by_name = TableByName,
+    table_by_pid = TableByPid
+}) ->
+    ok.
+
+%% ===================================================================
+%% Internal
+%% ===================================================================

src/syn_scope_sup.erl

@@ -49,7 +49,8 @@ init([Scope]) ->
     ok = syn_backbone:create_tables_for_scope(Scope),
     %% set children
     Children = [
-        scope_child_spec(syn_registry, Scope)
+        scope_child_spec(syn_registry, Scope),
+        scope_child_spec(syn_groups, Scope)
     ],
     {ok, {{one_for_one, 10, 10}, Children}}.
 

test/syn_groups_SUITE.erl

@@ -0,0 +1,339 @@
+%% ==========================================================================================================
+%% Syn - A global Process Registry and Process Group manager.
+%%
+%% The MIT License (MIT)
+%%
+%% Copyright (c) 2015-2021 Roberto Ostinelli <roberto@ostinelli.net> and Neato Robotics, Inc.
+%%
+%% Permission is hereby granted, free of charge, to any person obtaining a copy
+%% of this software and associated documentation files (the "Software"), to deal
+%% in the Software without restriction, including without limitation the rights
+%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+%% copies of the Software, and to permit persons to whom the Software is
+%% furnished to do so, subject to the following conditions:
+%%
+%% The above copyright notice and this permission notice shall be included in
+%% all copies or substantial portions of the Software.
+%%
+%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+%% THE SOFTWARE.
+%% ==========================================================================================================
+-module(syn_groups_SUITE).
+
+%% callbacks
+-export([all/0]).
+-export([init_per_suite/1, end_per_suite/1]).
+-export([groups/0, init_per_group/2, end_per_group/2]).
+-export([init_per_testcase/2, end_per_testcase/2]).
+
+%% tests
+-export([
+    three_nodes_discover_default_scope/1,
+    three_nodes_discover_custom_scope/1
+]).
+
+%% include
+-include_lib("common_test/include/ct.hrl").
+-include_lib("syn/src/syn.hrl").
+
+%% ===================================================================
+%% Callbacks
+%% ===================================================================
+
+%% -------------------------------------------------------------------
+%% Function: all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = any()
+%% -------------------------------------------------------------------
+all() ->
+    [
+        {group, three_nodes_groups}
+    ].
+
+%% -------------------------------------------------------------------
+%% Function: groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName =  atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%%			   repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% -------------------------------------------------------------------
+groups() ->
+    [
+        {three_nodes_groups, [shuffle], [
+            three_nodes_discover_default_scope,
+            three_nodes_discover_custom_scope
+        ]}
+    ].
+%% -------------------------------------------------------------------
+%% Function: init_per_suite(Config0) ->
+%%				Config1 | {skip,Reason} |
+%%              {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = any()
+%% -------------------------------------------------------------------
+init_per_suite(Config) ->
+    Config.
+
+%% -------------------------------------------------------------------
+%% Function: end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% -------------------------------------------------------------------
+end_per_suite(_Config) ->
+    ok.
+
+%% -------------------------------------------------------------------
+%% Function: init_per_group(GroupName, Config0) ->
+%%				Config1 | {skip,Reason} |
+%%              {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = any()
+%% -------------------------------------------------------------------
+init_per_group(three_nodes_groups, Config) ->
+    %% start slave
+    {ok, SlaveNode1} = syn_test_suite_helper:start_slave(syn_slave_1),
+    {ok, SlaveNode2} = syn_test_suite_helper:start_slave(syn_slave_2),
+    syn_test_suite_helper:connect_node(SlaveNode1),
+    syn_test_suite_helper:connect_node(SlaveNode2),
+    rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
+    %% wait full cluster
+    case syn_test_suite_helper:wait_cluster_mesh_connected([node(), SlaveNode1, SlaveNode2]) of
+        ok ->
+            %% config
+            [{slave_node_1, SlaveNode1}, {slave_node_2, SlaveNode2} | Config];
+
+        Other ->
+            ct:pal("*********** Could not get full cluster, skipping"),
+            end_per_group(three_nodes_groups, Config),
+            {skip, Other}
+    end;
+
+init_per_group(_GroupName, Config) ->
+    Config.
+
+%% -------------------------------------------------------------------
+%% Function: end_per_group(GroupName, Config0) ->
+%%				void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% -------------------------------------------------------------------
+end_per_group(three_nodes_groups, Config) ->
+    SlaveNode1 = proplists:get_value(slave_node_1, Config),
+    syn_test_suite_helper:connect_node(SlaveNode1),
+    SlaveNode2 = proplists:get_value(slave_node_2, Config),
+    syn_test_suite_helper:connect_node(SlaveNode2),
+    syn_test_suite_helper:clean_after_test(),
+    syn_test_suite_helper:stop_slave(syn_slave_1),
+    syn_test_suite_helper:stop_slave(syn_slave_2),
+    timer:sleep(1000);
+end_per_group(_GroupName, _Config) ->
+    syn_test_suite_helper:clean_after_test().
+
+%% -------------------------------------------------------------------
+%% Function: init_per_testcase(TestCase, Config0) ->
+%%				Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = any()
+%% -------------------------------------------------------------------
+init_per_testcase(TestCase, Config) ->
+    ct:pal("Starting test: ~p", [TestCase]),
+    Config.
+
+%% -------------------------------------------------------------------
+%% Function: end_per_testcase(TestCase, Config0) ->
+%%				void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = any()
+%% -------------------------------------------------------------------
+end_per_testcase(_, _Config) ->
+    syn_test_suite_helper:clean_after_test().
+
+%% ===================================================================
+%% Tests
+%% ===================================================================
+three_nodes_discover_default_scope(Config) ->
+    %% get slaves
+    SlaveNode1 = proplists:get_value(slave_node_1, Config),
+    SlaveNode2 = proplists:get_value(slave_node_2, Config),
+
+    %% start syn on nodes
+    ok = syn:start(),
+    ok = rpc:call(SlaveNode1, syn, start, []),
+    ok = rpc:call(SlaveNode2, syn, start, []),
+
+    %% check
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), default, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, default, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode2, default, [node(), SlaveNode1]),
+
+    %% simulate full netsplit
+    rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
+    syn_test_suite_helper:disconnect_node(SlaveNode1),
+    syn_test_suite_helper:disconnect_node(SlaveNode2),
+    syn_test_suite_helper:assert_cluster(node(), []),
+
+    %% check
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), default, []),
+
+    %% reconnect slave 1
+    syn_test_suite_helper:connect_node(SlaveNode1),
+    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1]),
+    syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
+
+    %% check
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), default, [SlaveNode1]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, default, [node()]),
+
+    %% reconnect all
+    syn_test_suite_helper:connect_node(SlaveNode2),
+    rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
+    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
+
+    %% check
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), default, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, default, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode2, default, [node(), SlaveNode1]),
+
+    %% simulate full netsplit, again
+    rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
+    syn_test_suite_helper:disconnect_node(SlaveNode1),
+    syn_test_suite_helper:disconnect_node(SlaveNode2),
+    syn_test_suite_helper:assert_cluster(node(), []),
+
+    %% check
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), default, []),
+    %% reconnect all, again
+    syn_test_suite_helper:connect_node(SlaveNode2),
+    rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
+    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
+
+    %% check
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), default, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, default, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode2, default, [node(), SlaveNode1]),
+
+    %% crash the scope process on local
+    syn_test_suite_helper:kill_process(syn_registry_default),
+    syn_test_suite_helper:wait_process_name_ready(syn_registry_default),
+
+    %% check, it should have rebuilt after supervisor restarts it
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), default, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, default, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode2, default, [node(), SlaveNode1]),
+
+    %% crash scopes supervisor on local
+    syn_test_suite_helper:kill_process(syn_scopes_sup),
+    syn_test_suite_helper:wait_process_name_ready(syn_registry_default),
+
+    %% check
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), default, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, default, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode2, default, [node(), SlaveNode1]).
+
+three_nodes_discover_custom_scope(Config) ->
+    %% get slaves
+    SlaveNode1 = proplists:get_value(slave_node_1, Config),
+    SlaveNode2 = proplists:get_value(slave_node_2, Config),
+
+    %% start syn on nodes
+    ok = syn:start(),
+    ok = rpc:call(SlaveNode1, syn, start, []),
+    ok = rpc:call(SlaveNode2, syn, start, []),
+
+    %% add custom scopes
+    ok = syn:add_node_to_scope(custom_scope_ab),
+    ok = syn:add_node_to_scope(custom_scope_all),
+    ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[custom_scope_ab, custom_scope_bc, custom_scope_all]]),
+    ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[custom_scope_bc, custom_scope_c, custom_scope_all]]),
+
+    %% get_subcluster_nodes should return invalid errors
+    {'EXIT', {{invalid_scope, custom_abcdef}, _}} = catch syn_registry:get_subcluster_nodes(custom_abcdef),
+
+    %% check
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), custom_scope_ab, [SlaveNode1]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), custom_scope_all, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, custom_scope_ab, [node()]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, custom_scope_bc, [SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, custom_scope_all, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode2, custom_scope_bc, [SlaveNode1]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode2, custom_scope_c, []),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode2, custom_scope_all, [node(), SlaveNode1]),
+
+    %% check default
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), default, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, default, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode2, default, [node(), SlaveNode1]),
+
+    %% disconnect node 2 (node 1 can still see node 2)
+    syn_test_suite_helper:disconnect_node(SlaveNode2),
+    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1]),
+    syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
+
+    %% check
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), custom_scope_ab, [SlaveNode1]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), custom_scope_all, [SlaveNode1]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, custom_scope_ab, [node()]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, custom_scope_bc, [SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, custom_scope_all, [node(), SlaveNode2]),
+
+    %% reconnect node 2
+    syn_test_suite_helper:connect_node(SlaveNode2),
+    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
+
+    %% check
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), custom_scope_ab, [SlaveNode1]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), custom_scope_all, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, custom_scope_ab, [node()]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, custom_scope_bc, [SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, custom_scope_all, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode2, custom_scope_bc, [SlaveNode1]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode2, custom_scope_c, []),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode2, custom_scope_all, [node(), SlaveNode1]),
+
+    %% crash a scope process on 2
+    rpc:call(SlaveNode2, syn_test_suite_helper, kill_process, [syn_registry_custom_scope_bc]),
+    rpc:call(SlaveNode2, syn_test_suite_helper, wait_process_name_ready, [syn_registry_default]),
+
+    %% check
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), custom_scope_ab, [SlaveNode1]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), custom_scope_all, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, custom_scope_ab, [node()]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, custom_scope_bc, [SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, custom_scope_all, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode2, custom_scope_bc, [SlaveNode1]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode2, custom_scope_c, []),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode2, custom_scope_all, [node(), SlaveNode1]),
+
+    %% crash scopes supervisor on local
+    syn_test_suite_helper:kill_process(syn_scopes_sup),
+    syn_test_suite_helper:wait_process_name_ready(syn_registry_default),
+
+    %% check
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), custom_scope_ab, [SlaveNode1]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(node(), custom_scope_all, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, custom_scope_ab, [node()]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, custom_scope_bc, [SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode1, custom_scope_all, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode2, custom_scope_bc, [SlaveNode1]),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode2, custom_scope_c, []),
+    syn_test_suite_helper:assert_groups_scope_subcluster(SlaveNode2, custom_scope_all, [node(), SlaveNode1]).

test/syn_registry_SUITE.erl

@@ -219,9 +219,9 @@ three_nodes_discover_default_scope(Config) ->
     ok = rpc:call(SlaveNode2, syn, start, []),
 
     %% check
-    syn_test_suite_helper:assert_scope_subcluster(node(), default, [SlaveNode1, SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, default, [node(), SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode2, default, [node(), SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), default, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, default, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, default, [node(), SlaveNode1]),
 
     %% simulate full netsplit
     rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
@@ -230,7 +230,7 @@ three_nodes_discover_default_scope(Config) ->
     syn_test_suite_helper:assert_cluster(node(), []),
 
     %% check
-    syn_test_suite_helper:assert_scope_subcluster(node(), default, []),
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), default, []),
 
     %% reconnect slave 1
     syn_test_suite_helper:connect_node(SlaveNode1),
@@ -238,8 +238,8 @@ three_nodes_discover_default_scope(Config) ->
     syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
 
     %% check
-    syn_test_suite_helper:assert_scope_subcluster(node(), default, [SlaveNode1]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, default, [node()]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), default, [SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, default, [node()]),
 
     %% reconnect all
     syn_test_suite_helper:connect_node(SlaveNode2),
@@ -249,9 +249,9 @@ three_nodes_discover_default_scope(Config) ->
     syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
 
     %% check
-    syn_test_suite_helper:assert_scope_subcluster(node(), default, [SlaveNode1, SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, default, [node(), SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode2, default, [node(), SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), default, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, default, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, default, [node(), SlaveNode1]),
 
     %% simulate full netsplit, again
     rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
@@ -260,7 +260,7 @@ three_nodes_discover_default_scope(Config) ->
     syn_test_suite_helper:assert_cluster(node(), []),
 
     %% check
-    syn_test_suite_helper:assert_scope_subcluster(node(), default, []),
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), default, []),
     %% reconnect all, again
     syn_test_suite_helper:connect_node(SlaveNode2),
     rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
@@ -269,27 +269,27 @@ three_nodes_discover_default_scope(Config) ->
     syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
 
     %% check
-    syn_test_suite_helper:assert_scope_subcluster(node(), default, [SlaveNode1, SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, default, [node(), SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode2, default, [node(), SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), default, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, default, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, default, [node(), SlaveNode1]),
 
     %% crash the scope process on local
     syn_test_suite_helper:kill_process(syn_registry_default),
     syn_test_suite_helper:wait_process_name_ready(syn_registry_default),
 
     %% check, it should have rebuilt after supervisor restarts it
-    syn_test_suite_helper:assert_scope_subcluster(node(), default, [SlaveNode1, SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, default, [node(), SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode2, default, [node(), SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), default, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, default, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, default, [node(), SlaveNode1]),
 
     %% crash scopes supervisor on local
     syn_test_suite_helper:kill_process(syn_scopes_sup),
     syn_test_suite_helper:wait_process_name_ready(syn_registry_default),
 
     %% check
-    syn_test_suite_helper:assert_scope_subcluster(node(), default, [SlaveNode1, SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, default, [node(), SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode2, default, [node(), SlaveNode1]).
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), default, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, default, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, default, [node(), SlaveNode1]).
 
 three_nodes_discover_custom_scope(Config) ->
     %% get slaves
@@ -311,19 +311,19 @@ three_nodes_discover_custom_scope(Config) ->
     {'EXIT', {{invalid_scope, custom_abcdef}, _}} = catch syn_registry:get_subcluster_nodes(custom_abcdef),
 
     %% check
-    syn_test_suite_helper:assert_scope_subcluster(node(), custom_scope_ab, [SlaveNode1]),
-    syn_test_suite_helper:assert_scope_subcluster(node(), custom_scope_all, [SlaveNode1, SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, custom_scope_ab, [node()]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, custom_scope_bc, [SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, custom_scope_all, [node(), SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode2, custom_scope_bc, [SlaveNode1]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode2, custom_scope_c, []),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode2, custom_scope_all, [node(), SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), custom_scope_ab, [SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), custom_scope_all, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, custom_scope_ab, [node()]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, custom_scope_bc, [SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, custom_scope_all, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, custom_scope_bc, [SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, custom_scope_c, []),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, custom_scope_all, [node(), SlaveNode1]),
 
     %% check default
-    syn_test_suite_helper:assert_scope_subcluster(node(), default, [SlaveNode1, SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, default, [node(), SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode2, default, [node(), SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), default, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, default, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, default, [node(), SlaveNode1]),
 
     %% disconnect node 2 (node 1 can still see node 2)
     syn_test_suite_helper:disconnect_node(SlaveNode2),
@@ -331,11 +331,11 @@ three_nodes_discover_custom_scope(Config) ->
     syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
 
     %% check
-    syn_test_suite_helper:assert_scope_subcluster(node(), custom_scope_ab, [SlaveNode1]),
-    syn_test_suite_helper:assert_scope_subcluster(node(), custom_scope_all, [SlaveNode1]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, custom_scope_ab, [node()]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, custom_scope_bc, [SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, custom_scope_all, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), custom_scope_ab, [SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), custom_scope_all, [SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, custom_scope_ab, [node()]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, custom_scope_bc, [SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, custom_scope_all, [node(), SlaveNode2]),
 
     %% reconnect node 2
     syn_test_suite_helper:connect_node(SlaveNode2),
@@ -344,42 +344,42 @@ three_nodes_discover_custom_scope(Config) ->
     syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
 
     %% check
-    syn_test_suite_helper:assert_scope_subcluster(node(), custom_scope_ab, [SlaveNode1]),
-    syn_test_suite_helper:assert_scope_subcluster(node(), custom_scope_all, [SlaveNode1, SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, custom_scope_ab, [node()]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, custom_scope_bc, [SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, custom_scope_all, [node(), SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode2, custom_scope_bc, [SlaveNode1]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode2, custom_scope_c, []),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode2, custom_scope_all, [node(), SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), custom_scope_ab, [SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), custom_scope_all, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, custom_scope_ab, [node()]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, custom_scope_bc, [SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, custom_scope_all, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, custom_scope_bc, [SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, custom_scope_c, []),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, custom_scope_all, [node(), SlaveNode1]),
 
     %% crash a scope process on 2
     rpc:call(SlaveNode2, syn_test_suite_helper, kill_process, [syn_registry_custom_scope_bc]),
     rpc:call(SlaveNode2, syn_test_suite_helper, wait_process_name_ready, [syn_registry_default]),
 
     %% check
-    syn_test_suite_helper:assert_scope_subcluster(node(), custom_scope_ab, [SlaveNode1]),
-    syn_test_suite_helper:assert_scope_subcluster(node(), custom_scope_all, [SlaveNode1, SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, custom_scope_ab, [node()]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, custom_scope_bc, [SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, custom_scope_all, [node(), SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode2, custom_scope_bc, [SlaveNode1]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode2, custom_scope_c, []),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode2, custom_scope_all, [node(), SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), custom_scope_ab, [SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), custom_scope_all, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, custom_scope_ab, [node()]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, custom_scope_bc, [SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, custom_scope_all, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, custom_scope_bc, [SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, custom_scope_c, []),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, custom_scope_all, [node(), SlaveNode1]),
 
     %% crash scopes supervisor on local
     syn_test_suite_helper:kill_process(syn_scopes_sup),
     syn_test_suite_helper:wait_process_name_ready(syn_registry_default),
 
     %% check
-    syn_test_suite_helper:assert_scope_subcluster(node(), custom_scope_ab, [SlaveNode1]),
-    syn_test_suite_helper:assert_scope_subcluster(node(), custom_scope_all, [SlaveNode1, SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, custom_scope_ab, [node()]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, custom_scope_bc, [SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode1, custom_scope_all, [node(), SlaveNode2]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode2, custom_scope_bc, [SlaveNode1]),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode2, custom_scope_c, []),
-    syn_test_suite_helper:assert_scope_subcluster(SlaveNode2, custom_scope_all, [node(), SlaveNode1]).
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), custom_scope_ab, [SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(node(), custom_scope_all, [SlaveNode1, SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, custom_scope_ab, [node()]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, custom_scope_bc, [SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, custom_scope_all, [node(), SlaveNode2]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, custom_scope_bc, [SlaveNode1]),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, custom_scope_c, []),
+    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, custom_scope_all, [node(), SlaveNode1]).
 
 three_nodes_register_unregister_and_monitor_default_scope(Config) ->
     %% get slaves

test/syn_test_suite_helper.erl

@@ -35,7 +35,7 @@
 -export([wait_cluster_mesh_connected/1]).
 -export([wait_process_name_ready/1, wait_process_name_ready/2]).
 -export([assert_cluster/2]).
--export([assert_scope_subcluster/3]).
+-export([assert_registry_scope_subcluster/3, assert_groups_scope_subcluster/3]).
 -export([assert_received_messages/1]).
 -export([assert_empty_queue/1]).
 -export([assert_wait/2]).
@@ -179,15 +179,11 @@ assert_cluster(Node, ExpectedNodes, StartAt) ->
         _ -> ok
     end.
 
-assert_scope_subcluster(Node, Scope, ExpectedNodes) ->
-    assert_scope_subcluster(Node, Scope, ExpectedNodes, os:system_time(millisecond)).
-assert_scope_subcluster(Node, Scope, ExpectedNodes, StartAt) ->
-    NodesMap = rpc:call(Node, syn_registry, get_subcluster_nodes, [Scope]),
-    Nodes = maps:keys(NodesMap),
-    case do_assert_cluster(Nodes, ExpectedNodes, StartAt) of
-        continue -> assert_scope_subcluster(Node, Scope, ExpectedNodes, StartAt);
-        _ -> ok
-    end.
+assert_registry_scope_subcluster(Node, Scope, ExpectedNodes) ->
+    do_assert_scope_subcluster(syn_registry, Node, Scope, ExpectedNodes).
+
+assert_groups_scope_subcluster(Node, Scope, ExpectedNodes) ->
+    do_assert_scope_subcluster(syn_groups, Node, Scope, ExpectedNodes).
 
 assert_received_messages(Messages) ->
     assert_received_messages(Messages, []).
@@ -247,6 +243,16 @@ process_main() ->
         _ -> process_main()
     end.
 
+do_assert_scope_subcluster(Module, Node, Scope, ExpectedNodes) ->
+    do_assert_scope_subcluster(Module, Node, Scope, ExpectedNodes, os:system_time(millisecond)).
+do_assert_scope_subcluster(Module, Node, Scope, ExpectedNodes, StartAt) ->
+    NodesMap = rpc:call(Node, Module, get_subcluster_nodes, [Scope]),
+    Nodes = maps:keys(NodesMap),
+    case do_assert_cluster(Nodes, ExpectedNodes, StartAt) of
+        continue -> do_assert_scope_subcluster(Module, Node, Scope, ExpectedNodes, StartAt);
+        _ -> ok
+    end.
+
 do_assert_cluster(Nodes, ExpectedNodes, StartAt) ->
     ExpectedCount = length(ExpectedNodes),
     %% count nodes