syn/test/syn_registry_SUITE.erl

%% ==========================================================================================================
%% Syn - A global Process Registry and Process Group manager.
%%
%% The MIT License (MIT)
%%
%% Copyright (c) 2015-2021 Roberto Ostinelli <roberto@ostinelli.net> and Neato Robotics, Inc.
%%
%% Permission is hereby granted, free of charge, to any person obtaining a copy
%% of this software and associated documentation files (the "Software"), to deal
%% in the Software without restriction, including without limitation the rights
%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
%% copies of the Software, and to permit persons to whom the Software is
%% furnished to do so, subject to the following conditions:
%%
%% The above copyright notice and this permission notice shall be included in
%% all copies or substantial portions of the Software.
%%
%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
%% THE SOFTWARE.
%% ==========================================================================================================
-module(syn_registry_SUITE).

%% callbacks
-export([all/0]).
-export([init_per_suite/1, end_per_suite/1]).
-export([groups/0, init_per_group/2, end_per_group/2]).
-export([init_per_testcase/2, end_per_testcase/2]).

%% tests
-export([
    one_node_via_register_unregister/1,
    one_node_strict_mode/1
]).
-export([
    three_nodes_discover/1,
    three_nodes_register_unregister_and_monitor/1,
    three_nodes_register_filter_unknown_node/1,
    three_nodes_cluster_changes/1,
    three_nodes_cluster_conflicts/1,
    three_nodes_custom_event_handler_reg_unreg/1,
    three_nodes_custom_event_handler_conflict_resolution/1
]).
-export([
    four_nodes_concurrency/1
]).

%% include
-include_lib("common_test/include/ct.hrl").
-include_lib("syn/src/syn.hrl").

%% ===================================================================
%% Callbacks
%% ===================================================================

%% -------------------------------------------------------------------
%% Function: all() -> GroupsAndTestCases | {skip,Reason}
%% GroupsAndTestCases = [{group,GroupName} | TestCase]
%% GroupName = atom()
%% TestCase = atom()
%% Reason = any()
%% -------------------------------------------------------------------
all() ->
    [
        {group, one_node_registry},
        {group, three_nodes_registry},
        {group, four_nodes_registry}
    ].

%% -------------------------------------------------------------------
%% Function: groups() -> [Group]
%% Group = {GroupName,Properties,GroupsAndTestCases}
%% GroupName =  atom()
%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
%% TestCase = atom()
%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
%%			   repeat_until_any_ok | repeat_until_any_fail
%% N = integer() | forever
%% -------------------------------------------------------------------
groups() ->
    [
        {one_node_registry, [shuffle], [
            one_node_via_register_unregister,
            one_node_strict_mode
        ]},
        {three_nodes_registry, [shuffle], [
            three_nodes_discover,
            three_nodes_register_unregister_and_monitor,
            three_nodes_register_filter_unknown_node,
            three_nodes_cluster_changes,
            three_nodes_cluster_conflicts,
            three_nodes_custom_event_handler_reg_unreg,
            three_nodes_custom_event_handler_conflict_resolution
        ]},
        {four_nodes_registry, [shuffle], [
            four_nodes_concurrency
        ]}
    ].
%% -------------------------------------------------------------------
%% Function: init_per_suite(Config0) ->
%%				Config1 | {skip,Reason} |
%%              {skip_and_save,Reason,Config1}
%% Config0 = Config1 = [tuple()]
%% Reason = any()
%% -------------------------------------------------------------------
init_per_suite(Config) ->
    Config.

%% -------------------------------------------------------------------
%% Function: end_per_suite(Config0) -> void() | {save_config,Config1}
%% Config0 = Config1 = [tuple()]
%% -------------------------------------------------------------------
end_per_suite(_Config) ->
    ok.

%% -------------------------------------------------------------------
%% Function: init_per_group(GroupName, Config0) ->
%%				Config1 | {skip,Reason} |
%%              {skip_and_save,Reason,Config1}
%% GroupName = atom()
%% Config0 = Config1 = [tuple()]
%% Reason = any()
%% -------------------------------------------------------------------
init_per_group(three_nodes_registry, Config) ->
    case syn_test_suite_helper:init_cluster(3) of
        {error_initializing_cluster, Other} ->
            end_per_group(three_nodes_registry, Config),
            {skip, Other};

        NodesConfig ->
            NodesConfig ++ Config
    end;

init_per_group(four_nodes_registry, Config) ->
    case syn_test_suite_helper:init_cluster(4) of
        {error_initializing_cluster, Other} ->
            end_per_group(four_nodes_registry, Config),
            {skip, Other};

        NodesConfig ->
            NodesConfig ++ Config
    end;

init_per_group(_GroupName, Config) ->
    Config.

%% -------------------------------------------------------------------
%% Function: end_per_group(GroupName, Config0) ->
%%				void() | {save_config,Config1}
%% GroupName = atom()
%% Config0 = Config1 = [tuple()]
%% -------------------------------------------------------------------
end_per_group(three_nodes_registry, Config) ->
    syn_test_suite_helper:end_cluster(3, Config);
end_per_group(four_nodes_registry, Config) ->
    syn_test_suite_helper:end_cluster(4, Config);
end_per_group(_GroupName, _Config) ->
    syn_test_suite_helper:clean_after_test().

%% -------------------------------------------------------------------
%% Function: init_per_testcase(TestCase, Config0) ->
%%				Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
%% TestCase = atom()
%% Config0 = Config1 = [tuple()]
%% Reason = any()
%% -------------------------------------------------------------------
init_per_testcase(TestCase, Config) ->
    ct:pal("Starting test: ~p", [TestCase]),
    Config.

%% -------------------------------------------------------------------
%% Function: end_per_testcase(TestCase, Config0) ->
%%				void() | {save_config,Config1} | {fail,Reason}
%% TestCase = atom()
%% Config0 = Config1 = [tuple()]
%% Reason = any()
%% -------------------------------------------------------------------
end_per_testcase(_, _Config) ->
    syn_test_suite_helper:clean_after_test().

%% ===================================================================
%% Tests
%% ===================================================================
one_node_via_register_unregister(_Config) ->
    %% start syn
    ok = syn:start(),

    %% --->  scope
    syn:add_node_to_scopes([scope]),

    %% start gen server via syn
    GenServerNameCustom = {scope, <<"my proc">>},
    TupleCustom = {via, syn, GenServerNameCustom},
    {ok, PidCustom} = syn_test_gen_server:start_link(TupleCustom),
    %% retrieve
    {PidCustom, undefined} = syn:lookup(scope, <<"my proc">>),
    %% call
    pong = syn_test_gen_server:ping(TupleCustom),
    %% send via syn
    syn:send(GenServerNameCustom, {self(), send_ping}),
    syn_test_suite_helper:assert_received_messages([
        reply_pong
    ]),
    %% stop server
    syn_test_gen_server:stop(TupleCustom),
    %% retrieve
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> syn:lookup(scope, <<"my proc">>) end
    ),
    %% send via syn
    {badarg, {GenServerNameCustom, anything}} = (catch syn:send(GenServerNameCustom, anything)).

one_node_strict_mode(_Config) ->
    %% start syn
    ok = syn:start(),
    syn:add_node_to_scopes([scope]),

    %% strict mode enabled
    application:set_env(syn, strict_mode, true),

    %% start process
    Pid = syn_test_suite_helper:start_process(),
    {error, not_self} = syn:register(scope, "strict-true", Pid, metadata),

    Self = self(),
    ok = syn:register(scope, "strict-true", Self, metadata),
    ok = syn:register(scope, "strict-true", Self, new_metadata),
    {Self, new_metadata} = syn:lookup(scope, "strict-true"),
    ok = syn:register(scope, "strict-true", Self),
    {Self, undefined} = syn:lookup(scope, "strict-true").

three_nodes_discover(Config) ->
    %% get slaves
    SlaveNode1 = proplists:get_value(syn_slave_1, Config),
    SlaveNode2 = proplists:get_value(syn_slave_2, Config),

    %% add scopes partially with ENV
    ok = rpc:call(SlaveNode2, application, set_env, [syn, scopes, [scope_all]]),

    %% start syn on nodes
    ok = syn:start(),
    ok = rpc:call(SlaveNode1, syn, start, []),
    ok = rpc:call(SlaveNode2, syn, start, []),

    %% add scopes
    ok = syn:add_node_to_scopes([scope_ab]),
    ok = syn:add_node_to_scopes([scope_all]),
    ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_ab, scope_bc, scope_all]]),
    ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_bc, scope_c]]),

    %% subcluster_nodes should return invalid errors
    {'EXIT', {{invalid_scope, custom_abcdef}, _}} = (catch syn_registry:subcluster_nodes(custom_abcdef)),

    %% check
    syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
    syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_bc, [SlaveNode1]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_c, []),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_all, [node(), SlaveNode1]),

    %% disconnect node 2 (node 1 can still see node 2)
    syn_test_suite_helper:disconnect_node(SlaveNode2),
    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1]),
    syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),

    %% check
    syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
    syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),

    %% reconnect node 2
    syn_test_suite_helper:connect_node(SlaveNode2),
    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),

    %% check
    syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
    syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_bc, [SlaveNode1]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_c, []),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_all, [node(), SlaveNode1]),

    %% crash a scope process on 2
    rpc:call(SlaveNode2, syn_test_suite_helper, kill_process, [syn_registry_scope_bc]),
    rpc:call(SlaveNode2, syn_test_suite_helper, wait_process_name_ready, [syn_registry_scope_bc]),

    %% check
    syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
    syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_bc, [SlaveNode1]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_c, []),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_all, [node(), SlaveNode1]),

    %% crash scopes supervisor on local
    syn_test_suite_helper:kill_process(syn_scopes_sup),
    syn_test_suite_helper:wait_process_name_ready(syn_registry_scope_ab),
    syn_test_suite_helper:wait_process_name_ready(syn_registry_scope_all),

    %% check
    syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
    syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_bc, [SlaveNode1]),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_c, []),
    syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_all, [node(), SlaveNode1]).

three_nodes_register_unregister_and_monitor(Config) ->
    %% get slaves
    SlaveNode1 = proplists:get_value(syn_slave_1, Config),
    SlaveNode2 = proplists:get_value(syn_slave_2, Config),

    %% start syn on nodes
    ok = syn:start(),
    ok = rpc:call(SlaveNode1, syn, start, []),
    ok = rpc:call(SlaveNode2, syn, start, []),

    %% add scopes
    ok = syn:add_node_to_scopes([scope_ab]),
    ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_ab, scope_bc]]),
    ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_bc]]),

    %% start processes
    Pid = syn_test_suite_helper:start_process(),
    PidWithMeta = syn_test_suite_helper:start_process(),
    PidRemoteWithMetaOn1 = syn_test_suite_helper:start_process(SlaveNode1),

    %% retrieve
    undefined = syn:lookup(scope_ab, "scope_a"),
    undefined = rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a"]),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a"])),
    undefined = syn:lookup(scope_ab, "scope_a_alias"),
    undefined = rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a_alias"]),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a_alias"])),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, {remote_scoped_bc})),
    undefined = rpc:call(SlaveNode1, syn, lookup, [scope_bc, {remote_scoped_bc}]),
    undefined = rpc:call(SlaveNode2, syn, lookup, [scope_bc, {remote_scoped_bc}]),
    0 = syn:registry_count(scope_ab),
    0 = syn:registry_count(scope_ab, node()),
    0 = syn:registry_count(scope_ab, SlaveNode1),
    0 = syn:registry_count(scope_ab, SlaveNode2),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc)),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, node())),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode1)),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode2)),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, node()]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode1]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode2]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab])),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, node()])),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode1])),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode2])),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),

    %% register
    ok = syn:register(scope_ab, "scope_a", Pid),
    ok = syn:register(scope_ab, "scope_a_alias", PidWithMeta, <<"with_meta">>),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:register(scope_bc, "scope_a", Pid)),
    {'EXIT', {{invalid_scope, non_existent_scope}, _}} = (catch syn:register(non_existent_scope, "scope_a", Pid)),
    ok = rpc:call(SlaveNode2, syn, register, [scope_bc, {remote_scoped_bc}, PidRemoteWithMetaOn1, <<"with_meta 1">>]),

    %% errors
    {error, taken} = syn:register(scope_ab, "scope_a", PidWithMeta),
    {error, not_alive} = syn:register(scope_ab, {"pid not alive"}, list_to_pid("<0.9999.0>")),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:register(scope_bc, "scope_a_noscope", Pid)),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:unregister(scope_bc, "scope_a_noscope")),
    LocalNode = node(),
    {badrpc, {'EXIT', {{invalid_remote_scope, scope_bc, LocalNode}, _}}} = (catch rpc:call(SlaveNode1, syn, register, [scope_bc, "pid-outside", Pid])),

    %% retrieve
    syn_test_suite_helper:assert_wait(
        {Pid, undefined},
        fun() -> syn:lookup(scope_ab, "scope_a") end
    ),
    syn_test_suite_helper:assert_wait(
        {Pid, undefined},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a"]) end
    ),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a"])),
    syn_test_suite_helper:assert_wait(
        {PidWithMeta, <<"with_meta">>},
        fun() -> syn:lookup(scope_ab, "scope_a_alias") end
    ),
    syn_test_suite_helper:assert_wait(
        {PidWithMeta, <<"with_meta">>},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a_alias"]) end
    ),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a_alias"])),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, {remote_scoped_bc})),
    syn_test_suite_helper:assert_wait(
        {PidRemoteWithMetaOn1, <<"with_meta 1">>},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, {remote_scoped_bc}]) end
    ),
    syn_test_suite_helper:assert_wait(
        {PidRemoteWithMetaOn1, <<"with_meta 1">>},
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, {remote_scoped_bc}]) end
    ),
    2 = syn:registry_count(scope_ab),
    2 = syn:registry_count(scope_ab, node()),
    0 = syn:registry_count(scope_ab, SlaveNode1),
    0 = syn:registry_count(scope_ab, SlaveNode2),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc)),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, node())),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode1)),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode2)),
    2 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab]),
    2 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, node()]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode1]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode2]),
    1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
    1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab])),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, node()])),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode1])),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode2])),
    1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
    1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),

    %% re-register to edit meta
    ok = syn:register(scope_ab, "scope_a_alias", PidWithMeta, <<"with_meta_updated">>),

    syn_test_suite_helper:assert_wait(
        {PidWithMeta, <<"with_meta_updated">>},
        fun() -> syn:lookup(scope_ab, "scope_a_alias") end
    ),
    syn_test_suite_helper:assert_wait(
        {PidWithMeta, <<"with_meta_updated">>},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a_alias"]) end
    ),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a_alias"])),

    %% register remote
    ok = syn:register(scope_ab, "ab_on_1", PidRemoteWithMetaOn1, <<"ab-on-1">>),
    syn_test_suite_helper:assert_wait(
        {PidRemoteWithMetaOn1, <<"ab-on-1">>},
        fun() -> syn:lookup(scope_ab, "ab_on_1") end
    ),

    %% crash scope process to ensure that monitors get recreated & data received from other nodes
    syn_test_suite_helper:kill_process(syn_registry_scope_ab),
    syn_test_suite_helper:wait_process_name_ready(syn_registry_scope_ab),

    %% check remote has been sync'ed back
    syn_test_suite_helper:assert_wait(
        {PidRemoteWithMetaOn1, <<"ab-on-1">>},
        fun() -> syn:lookup(scope_ab, "ab_on_1") end
    ),

    %% kill process
    syn_test_suite_helper:kill_process(Pid),
    syn_test_suite_helper:kill_process(PidWithMeta),
    %% unregister processes
    {error, undefined} = (catch syn:unregister(scope_ab, <<"my proc with meta">>)),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:unregister(scope_bc, <<"my proc with meta">>)),
    ok = rpc:call(SlaveNode1, syn, unregister, [scope_bc, {remote_scoped_bc}]),

    %% retrieve
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> syn:lookup(scope_ab, "scope_a") end
    ),
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a"]) end
    ),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a"])),
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> syn:lookup(scope_ab, "scope_a_alias") end
    ),
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a_alias"]) end
    ),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a_alias"])),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, {remote_scoped_bc})),
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, {remote_scoped_bc}]) end
    ),
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, {remote_scoped_bc}]) end
    ),
    1 = syn:registry_count(scope_ab),
    0 = syn:registry_count(scope_ab, node()),
    1 = syn:registry_count(scope_ab, SlaveNode1),
    0 = syn:registry_count(scope_ab, SlaveNode2),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc)),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, node())),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode1)),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode2)),
    1 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, node()]),
    1 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode1]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode2]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab])),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, node()])),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode1])),
    {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode2])),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),

    %% errors
    {error, undefined} = syn:unregister(scope_ab, {invalid_name}),

    %% (simulate race condition)
    Pid1 = syn_test_suite_helper:start_process(),
    Pid2 = syn_test_suite_helper:start_process(),
    ok = syn:register(scope_ab, <<"my proc">>, Pid1),
    syn_test_suite_helper:assert_wait(
        {Pid1, undefined},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, <<"my proc">>]) end
    ),
    remove_from_local_table(scope_ab, <<"my proc">>, Pid1),
    add_to_local_table(scope_ab, <<"my proc">>, Pid2, undefined, 0, undefined),
    {error, race_condition} = rpc:call(SlaveNode1, syn, unregister, [scope_ab, <<"my proc">>]).

three_nodes_register_filter_unknown_node(Config) ->
    %% get slaves
    SlaveNode1 = proplists:get_value(syn_slave_1, Config),
    SlaveNode2 = proplists:get_value(syn_slave_2, Config),

    %% start syn on 1 and 2
    ok = rpc:call(SlaveNode1, syn, start, []),
    ok = rpc:call(SlaveNode2, syn, start, []),

    %% add scopes
    ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_bc]]),
    ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_bc]]),

    %% send sync message from out of scope node
    InvalidPid = syn_test_suite_helper:start_process(),
    {syn_registry_scope_bc, SlaveNode1} ! {'3.0', sync_register, <<"proc-name">>, InvalidPid, undefined, os:system_time(millisecond), normal},

    %% check
    undefined = rpc:call(SlaveNode1, syn, lookup, [scope_bc, <<"proc-name">>]).

three_nodes_cluster_changes(Config) ->
    %% get slaves
    SlaveNode1 = proplists:get_value(syn_slave_1, Config),
    SlaveNode2 = proplists:get_value(syn_slave_2, Config),

    %% disconnect 1 from 2
    rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
    syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),

    %% start syn on 1 and 2, nodes don't know of each other
    ok = rpc:call(SlaveNode1, syn, start, []),
    ok = rpc:call(SlaveNode2, syn, start, []),

    %% add scopes
    ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
    ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_all, scope_bc]]),

    %% start processes
    PidRemoteOn1 = syn_test_suite_helper:start_process(SlaveNode1),
    PidRemoteOn2 = syn_test_suite_helper:start_process(SlaveNode2),

    %% register
    ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-1", PidRemoteOn1, "meta-1"]),
    ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-2", PidRemoteOn2, "meta-2"]),
    ok = rpc:call(SlaveNode1, syn, register, [scope_bc, "BC-proc-1", PidRemoteOn1, "meta-1"]),
    ok = rpc:call(SlaveNode1, syn, register, [scope_bc, "BC-proc-1 alias", PidRemoteOn1, "meta-1 alias"]),

    %% form full cluster
    ok = syn:start(),
    ok = syn:add_node_to_scopes([scope_all]),

    rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),

    %% retrieve
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn1, "meta-1"},
        fun() -> syn:lookup(scope_all, "proc-1") end
    ),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn1, "meta-1"},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-1"]) end
    ),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn1, "meta-1"},
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-1"]) end
    ),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn2, "meta-2"},
        fun() -> syn:lookup(scope_all, "proc-2") end
    ),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn2, "meta-2"},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-2"]) end
    ),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn2, "meta-2"},
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-2"]) end
    ),
    2 = syn:registry_count(scope_all),
    0 = syn:registry_count(scope_all, node()),
    1 = syn:registry_count(scope_all, SlaveNode1),
    1 = syn:registry_count(scope_all, SlaveNode2),
    2 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
    1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
    1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
    2 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
    1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
    1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),

    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, "BC-proc-1")),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn1, "meta-1"},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1"]) end
    ),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn1, "meta-1"},
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1"]) end
    ),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, "BC-proc-1 alias")),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn1, "meta-1 alias"},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
    ),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn1, "meta-1 alias"},
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
    ),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc)),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, node())),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode1)),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode2)),
    2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
    2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
    2 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
    2 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),

    %% partial netsplit (1 cannot see 2)
    rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
    syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),

    %% retrieve
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn1, "meta-1"},
        fun() -> syn:lookup(scope_all, "proc-1") end
    ),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn1, "meta-1"},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-1"]) end
    ),
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-1"]) end
    ),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn2, "meta-2"},
        fun() -> syn:lookup(scope_all, "proc-2") end
    ),
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-2"]) end
    ),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn2, "meta-2"},
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-2"]) end
    ),
    2 = syn:registry_count(scope_all),
    0 = syn:registry_count(scope_all, node()),
    1 = syn:registry_count(scope_all, SlaveNode1),
    1 = syn:registry_count(scope_all, SlaveNode2),
    1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
    1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
    1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
    1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),

    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, "BC-proc-1")),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn1, "meta-1"},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1"]) end
    ),
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1"]) end
    ),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, "BC-proc-1 alias")),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn1, "meta-1 alias"},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
    ),
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
    ),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc)),
    2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
    2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),

    %% re-join
    rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),

    %% retrieve
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn1, "meta-1"},
        fun() -> syn:lookup(scope_all, "proc-1") end
    ),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn1, "meta-1"},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-1"]) end
    ),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn1, "meta-1"},
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-1"]) end
    ),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn2, "meta-2"},
        fun() -> syn:lookup(scope_all, "proc-2") end
    ),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn2, "meta-2"},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-2"]) end
    ),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn2, "meta-2"},
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-2"]) end
    ),
    2 = syn:registry_count(scope_all),
    0 = syn:registry_count(scope_all, node()),
    1 = syn:registry_count(scope_all, SlaveNode1),
    1 = syn:registry_count(scope_all, SlaveNode2),
    2 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
    1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
    1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
    2 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
    1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
    1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),

    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, "BC-proc-1")),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn1, "meta-1"},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1"]) end
    ),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn1, "meta-1"},
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1"]) end
    ),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, "BC-proc-1 alias")),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn1, "meta-1 alias"},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
    ),
    syn_test_suite_helper:assert_wait(
        {PidRemoteOn1, "meta-1 alias"},
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
    ),
    {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc)),
    2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
    2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
    2 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
    2 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]).

three_nodes_cluster_conflicts(Config) ->
    %% get slaves
    SlaveNode1 = proplists:get_value(syn_slave_1, Config),
    SlaveNode2 = proplists:get_value(syn_slave_2, Config),

    %% start syn on nodes
    ok = syn:start(),
    ok = rpc:call(SlaveNode1, syn, start, []),
    ok = rpc:call(SlaveNode2, syn, start, []),

    %% add scopes
    ok = syn:add_node_to_scopes([scope_all]),
    ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
    ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_all, scope_bc]]),

    %% partial netsplit (1 cannot see 2)
    rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
    syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),

    %% start conflict processes
    Pid2RemoteOn1 = syn_test_suite_helper:start_process(SlaveNode1),
    Pid2RemoteOn2 = syn_test_suite_helper:start_process(SlaveNode2),

    %% --> conflict by netsplit
    ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit", Pid2RemoteOn1, "meta-1"]),
    ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit", Pid2RemoteOn2, "meta-2"]),
    ok = rpc:call(SlaveNode1, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped", Pid2RemoteOn1, "meta-1"]),
    ok = rpc:call(SlaveNode2, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped", Pid2RemoteOn2, "meta-2"]),

    %% re-join
    rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),

    %% retrieve
    syn_test_suite_helper:assert_wait(
        {Pid2RemoteOn2, "meta-2"},
        fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit") end
    ),
    syn_test_suite_helper:assert_wait(
        {Pid2RemoteOn2, "meta-2"},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit"]) end
    ),
    syn_test_suite_helper:assert_wait(
        {Pid2RemoteOn2, "meta-2"},
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit"]) end
    ),
    1 = syn:registry_count(scope_all),
    0 = syn:registry_count(scope_all, node()),
    0 = syn:registry_count(scope_all, SlaveNode1),
    1 = syn:registry_count(scope_all, SlaveNode2),
    1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
    1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
    1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
    1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
    syn_test_suite_helper:assert_wait(
        {Pid2RemoteOn2, "meta-2"},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped"]) end
    ),
    syn_test_suite_helper:assert_wait(
        {Pid2RemoteOn2, "meta-2"},
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped"]) end
    ),
    1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
    1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
    1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
    1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),

    %% process alive
    syn_test_suite_helper:assert_wait(
        false,
        fun() -> rpc:call(SlaveNode1, erlang, is_process_alive, [Pid2RemoteOn1]) end
    ),
    syn_test_suite_helper:assert_wait(
        true,
        fun() -> rpc:call(SlaveNode2, erlang, is_process_alive, [Pid2RemoteOn2]) end
    ),

    %% --> conflict by race condition
    PidOnMaster = syn_test_suite_helper:start_process(),
    PidOn1 = syn_test_suite_helper:start_process(SlaveNode1),
    rpc:call(SlaveNode1, syn_registry, add_to_local_table,
        [default, <<"my proc">>, PidOn1, "meta-2", erlang:system_time(), undefined]
    ),
    ok = syn:register(scope_all, <<"my proc">>, PidOnMaster, "meta-1"),

    %% retrieve
    syn_test_suite_helper:assert_wait(
        {PidOnMaster, "meta-1"},
        fun() -> syn:lookup(scope_all, <<"my proc">>) end
    ),
    syn_test_suite_helper:assert_wait(
        {PidOnMaster, "meta-1"},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, <<"my proc">>]) end
    ),
    syn_test_suite_helper:assert_wait(
        {PidOnMaster, "meta-1"},
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, <<"my proc">>]) end
    ),
    %% NB: we can't check for process alive here because we injected the conflicting process in the DB
    %% -> it's not actually monitored
    ok.

three_nodes_custom_event_handler_reg_unreg(Config) ->
    %% get slaves
    SlaveNode1 = proplists:get_value(syn_slave_1, Config),
    SlaveNode2 = proplists:get_value(syn_slave_2, Config),

    %% add custom handler for callbacks (using ENV)
    rpc:call(SlaveNode2, application, set_env, [syn, event_handler, syn_test_event_handler_callbacks]),

    %% start syn on nodes
    ok = syn:start(),
    ok = rpc:call(SlaveNode1, syn, start, []),
    ok = rpc:call(SlaveNode2, syn, start, []),

    %% add custom handler for callbacks (using method call)
    syn:set_event_handler(syn_test_event_handler_callbacks),
    rpc:call(SlaveNode1, syn, set_event_handler, [syn_test_event_handler_callbacks]),

    %% add scopes
    ok = syn:add_node_to_scopes([scope_all]),
    ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all]]),
    ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_all]]),

    %% init
    LocalNode = node(),

    %% start process
    Pid = syn_test_suite_helper:start_process(),
    Pid2 = syn_test_suite_helper:start_process(),

    %% ---> on registration
    ok = syn:register(scope_all, "proc-handler", Pid, {recipient, self(), <<"meta">>}),

    %% check callbacks called
    syn_test_suite_helper:assert_received_messages([
        {on_process_registered, LocalNode, scope_all, "proc-handler", Pid, <<"meta">>, normal},
        {on_process_registered, SlaveNode1, scope_all, "proc-handler", Pid, <<"meta">>, normal},
        {on_process_registered, SlaveNode2, scope_all, "proc-handler", Pid, <<"meta">>, normal}
    ]),

    %% registration from another node
    ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-handler-2", Pid2, {recipient, self(), <<"meta-for-2">>}]),

    %% check callbacks called
    syn_test_suite_helper:assert_received_messages([
        {on_process_registered, LocalNode, scope_all, "proc-handler-2", Pid2, <<"meta-for-2">>, normal},
        {on_process_registered, SlaveNode1, scope_all, "proc-handler-2", Pid2, <<"meta-for-2">>, normal},
        {on_process_registered, SlaveNode2, scope_all, "proc-handler-2", Pid2, <<"meta-for-2">>, normal}
    ]),

    %% ---> on meta update
    ok = syn:register(scope_all, "proc-handler", Pid, {recipient, self(), <<"new-meta">>}),

    %% check callbacks called
    syn_test_suite_helper:assert_received_messages([
        {on_registry_process_updated, LocalNode, scope_all, "proc-handler", Pid, <<"new-meta">>, normal},
        {on_registry_process_updated, SlaveNode1, scope_all, "proc-handler", Pid, <<"new-meta">>, normal},
        {on_registry_process_updated, SlaveNode2, scope_all, "proc-handler", Pid, <<"new-meta">>, normal}
    ]),

    %% meta update from another node
    ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-handler-2", Pid2, {recipient, self(), <<"meta-for-2-update">>}]),

    %% check callbacks called
    syn_test_suite_helper:assert_received_messages([
        {on_registry_process_updated, LocalNode, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal},
        {on_registry_process_updated, SlaveNode1, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal},
        {on_registry_process_updated, SlaveNode2, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal}
    ]),

    %% ---> on unregister
    ok = syn:unregister(scope_all, "proc-handler"),

    %% check callbacks called
    syn_test_suite_helper:assert_received_messages([
        {on_process_unregistered, LocalNode, scope_all, "proc-handler", Pid, <<"new-meta">>, normal},
        {on_process_unregistered, SlaveNode1, scope_all, "proc-handler", Pid, <<"new-meta">>, normal},
        {on_process_unregistered, SlaveNode2, scope_all, "proc-handler", Pid, <<"new-meta">>, normal}
    ]),

    %% unregister from another node
    ok = rpc:call(SlaveNode1, syn, unregister, [scope_all, "proc-handler-2"]),

    %% check callbacks called
    syn_test_suite_helper:assert_received_messages([
        {on_process_unregistered, LocalNode, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal},
        {on_process_unregistered, SlaveNode1, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal},
        {on_process_unregistered, SlaveNode2, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal}
    ]),

    %% clean & check
    syn_test_suite_helper:kill_process(Pid),
    %% no messages
    syn_test_suite_helper:assert_empty_queue(),

    %% ---> after a netsplit
    PidRemoteOn1 = syn_test_suite_helper:start_process(SlaveNode1),
    ok = syn:register(scope_all, remote_on_1, PidRemoteOn1, {recipient, self(), <<"netsplit">>}),

    %% check callbacks called
    syn_test_suite_helper:assert_received_messages([
        {on_process_registered, LocalNode, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, normal},
        {on_process_registered, SlaveNode1, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, normal},
        {on_process_registered, SlaveNode2, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, normal}
    ]),

    %% partial netsplit (1 cannot see 2)
    rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
    syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),

    %% check callbacks called
    syn_test_suite_helper:assert_received_messages([
        {on_process_unregistered, SlaveNode2, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, {syn_remote_scope_node_down, scope_all, SlaveNode1}}
    ]),

    %% ---> after a re-join
    %% re-join
    rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),

    %% check callbacks called
    syn_test_suite_helper:assert_received_messages([
        {on_process_registered, SlaveNode2, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, {syn_remote_scope_node_up, scope_all, SlaveNode1}}
    ]),

    %% clean
    syn_test_suite_helper:kill_process(PidRemoteOn1),

    %% check callbacks called
    syn_test_suite_helper:assert_received_messages([
        {on_process_unregistered, LocalNode, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, killed},
        {on_process_unregistered, SlaveNode1, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, killed},
        {on_process_unregistered, SlaveNode2, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, killed}
    ]),

    %% ---> after a conflict resolution
    %% partial netsplit (1 cannot see 2)
    rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
    syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),

    %% start conflict processes
    Pid2RemoteOn1 = syn_test_suite_helper:start_process(SlaveNode1),
    Pid2RemoteOn2 = syn_test_suite_helper:start_process(SlaveNode2),

    ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict", Pid2RemoteOn1, {recipient, self(), <<"meta-1">>}]),
    ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict", Pid2RemoteOn2, {recipient, self(), <<"meta-2">>}]),

    %% check callbacks called
    syn_test_suite_helper:assert_received_messages([
        {on_process_registered, LocalNode, scope_all, "proc-confict", Pid2RemoteOn1, <<"meta-1">>, normal},
        {on_process_unregistered, LocalNode, scope_all, "proc-confict", Pid2RemoteOn1, <<"meta-1">>, normal},
        {on_process_registered, LocalNode, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, normal},
        {on_process_registered, SlaveNode1, scope_all, "proc-confict", Pid2RemoteOn1, <<"meta-1">>, normal},
        {on_process_registered, SlaveNode2, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, normal}
    ]),

    %% re-join
    rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),

    %% check callbacks called
    syn_test_suite_helper:assert_received_messages([
        {on_process_unregistered, SlaveNode1, scope_all, "proc-confict", Pid2RemoteOn1, <<"meta-1">>, syn_conflict_resolution},
        {on_process_registered, SlaveNode1, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, syn_conflict_resolution}
    ]),

    %% kill
    syn_test_suite_helper:kill_process(Pid2RemoteOn1),
    syn_test_suite_helper:kill_process(Pid2RemoteOn2),

    %% check callbacks called
    syn_test_suite_helper:assert_received_messages([
        {on_process_unregistered, LocalNode, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, killed},
        {on_process_unregistered, SlaveNode1, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, killed},
        {on_process_unregistered, SlaveNode2, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, killed}
    ]),

    %% ---> don't call on monitor rebuild
    %% crash the scope process on local
    syn_test_suite_helper:kill_process(syn_registry_scope_all),
    syn_test_suite_helper:wait_process_name_ready(syn_registry_scope_all),

    %% no messages
    syn_test_suite_helper:assert_empty_queue(),

    %% ---> call if process died during the scope process crash
    TransientPid = syn_test_suite_helper:start_process(),
    ok = syn:register(scope_all, "transient-pid", TransientPid, {recipient, self(), "transient-meta"}),

    %% check callbacks called
    syn_test_suite_helper:assert_received_messages([
        {on_process_registered, LocalNode, scope_all, "transient-pid", TransientPid, "transient-meta", normal},
        {on_process_registered, SlaveNode1, scope_all, "transient-pid", TransientPid, "transient-meta", normal},
        {on_process_registered, SlaveNode2, scope_all, "transient-pid", TransientPid, "transient-meta", normal}
    ]),

    %% crash the scope process & fake a died process on local
    InvalidPid = list_to_pid("<0.9999.0>"),
    add_to_local_table(scope_all, "invalid-pid", InvalidPid, {recipient, self(), "invalid-meta"}, 0, undefined),
    syn_test_suite_helper:kill_process(syn_registry_scope_all),

    %% check callbacks called
    syn_test_suite_helper:assert_received_messages([
        {on_process_unregistered, LocalNode, scope_all, "invalid-pid", InvalidPid, "invalid-meta", undefined},
        {on_process_unregistered, SlaveNode1, scope_all, "transient-pid", TransientPid, "transient-meta", {syn_remote_scope_node_down, scope_all, LocalNode}},
        {on_process_unregistered, SlaveNode2, scope_all, "transient-pid", TransientPid, "transient-meta", {syn_remote_scope_node_down, scope_all, LocalNode}},
        {on_process_registered, SlaveNode1, scope_all, "transient-pid", TransientPid, "transient-meta", {syn_remote_scope_node_up, scope_all, LocalNode}},
        {on_process_registered, SlaveNode2, scope_all, "transient-pid", TransientPid, "transient-meta", {syn_remote_scope_node_up, scope_all, LocalNode}}
    ]).

three_nodes_custom_event_handler_conflict_resolution(Config) ->
    %% get slaves
    SlaveNode1 = proplists:get_value(syn_slave_1, Config),
    SlaveNode2 = proplists:get_value(syn_slave_2, Config),

    %% add custom handler for resolution & scopes (using ENV)
    rpc:call(SlaveNode2, application, set_env, [syn, event_handler, syn_test_event_handler_resolution]),
    rpc:call(SlaveNode2, application, set_env, [syn, scopes, [scope_all, scope_bc]]),

    %% start syn on nodes
    ok = syn:start(),
    ok = rpc:call(SlaveNode1, syn, start, []),
    ok = rpc:call(SlaveNode2, syn, start, []),

    %% add custom handler for resolution (using method call)
    syn:set_event_handler(syn_test_event_handler_resolution),
    rpc:call(SlaveNode1, syn, set_event_handler, [syn_test_event_handler_resolution]),

    %% add scopes
    ok = syn:add_node_to_scopes([scope_all]),
    ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all, scope_bc]]),

    %% current node
    TestPid = self(),
    LocalNode = node(),

    %% partial netsplit (1 cannot see 2)
    rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
    syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),

    %% start conflict processes
    PidOn1 = syn_test_suite_helper:start_process(SlaveNode1),
    PidOn2 = syn_test_suite_helper:start_process(SlaveNode2),

    %% --> conflict by netsplit
    ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit-custom", PidOn1, {recipient, TestPid, keepthis}]),
    ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit-custom", PidOn2, {recipient, TestPid, "meta-2"}]),
    ok = rpc:call(SlaveNode1, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, {recipient, TestPid, keepthis}]),
    ok = rpc:call(SlaveNode2, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn2, {recipient, TestPid, "meta-2"}]),

    %% check callbacks
    syn_test_suite_helper:assert_received_messages([
        {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
        {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
        {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn2, "meta-2", normal},

        {on_process_registered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
        {on_process_registered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom", PidOn2, "meta-2", normal},
        {on_process_registered, SlaveNode1, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, keepthis, normal},
        {on_process_registered, SlaveNode2, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn2, "meta-2", normal}
    ]),

    %% re-join
    rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),

    %% retrieve
    syn_test_suite_helper:assert_wait(
        {PidOn1, {recipient, TestPid, keepthis}},
        fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit-custom") end
    ),
    syn_test_suite_helper:assert_wait(
        {PidOn1, {recipient, TestPid, keepthis}},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom"]) end
    ),
    syn_test_suite_helper:assert_wait(
        {PidOn1, {recipient, TestPid, keepthis}},
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom"]) end
    ),
    1 = syn:registry_count(scope_all),
    0 = syn:registry_count(scope_all, node()),
    1 = syn:registry_count(scope_all, SlaveNode1),
    0 = syn:registry_count(scope_all, SlaveNode2),
    1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
    1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
    1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
    1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
    syn_test_suite_helper:assert_wait(
        {PidOn1, {recipient, TestPid, keepthis}},
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]) end
    ),
    syn_test_suite_helper:assert_wait(
        {PidOn1, {recipient, TestPid, keepthis}},
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]) end
    ),
    1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
    1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
    1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
    1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),

    syn_test_suite_helper:assert_received_messages([
        {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn2, "meta-2", syn_conflict_resolution},
        {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, syn_conflict_resolution},

        {on_process_unregistered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom", PidOn2, "meta-2", syn_conflict_resolution},
        {on_process_registered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, syn_conflict_resolution},
        {on_process_unregistered, SlaveNode2, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn2, "meta-2", syn_conflict_resolution},
        {on_process_registered, SlaveNode2, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, keepthis, syn_conflict_resolution}
    ]),

    %% process alive (discarded process does not get killed with a custom handler)
    syn_test_suite_helper:assert_wait(
        true,
        fun() -> rpc:call(SlaveNode1, erlang, is_process_alive, [PidOn1]) end
    ),
    syn_test_suite_helper:assert_wait(
        true,
        fun() -> rpc:call(SlaveNode2, erlang, is_process_alive, [PidOn2]) end
    ),

    %% clean up default scope
    syn:unregister(scope_all, "proc-confict-by-netsplit-custom"),
    ok = rpc:call(SlaveNode1, syn, unregister, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]),

    %% retrieve
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit-custom") end
    ),
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom"]) end
    ),
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom"]) end
    ),
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]) end
    ),
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]) end
    ),

    %% check callbacks
    syn_test_suite_helper:assert_received_messages([
        {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
        {on_process_unregistered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
        {on_process_unregistered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
        {on_process_unregistered, SlaveNode1, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, keepthis, normal},
        {on_process_unregistered, SlaveNode2, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, keepthis, normal}
    ]),

    %% --> conflict by netsplit, which returns invalid pid
    %% partial netsplit (1 cannot see 2)
    rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
    syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),

    %% register with meta with no 'keepthis' element
    ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, {recipient, TestPid, "meta-1"}]),
    ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, {recipient, TestPid, "meta-2"}]),

    %% check callbacks
    syn_test_suite_helper:assert_received_messages([
        {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, "meta-1", normal},
        {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, "meta-1", normal},
        {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, "meta-2", normal},

        {on_process_registered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, "meta-1", normal},
        {on_process_registered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, "meta-2", normal}
    ]),

    %% re-join
    rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),

    %% retrieve (names get freed)
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit-custom-other-pid") end
    ),
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom-other-pid"]) end
    ),
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom-other-pid"]) end
    ),
    0 = syn:registry_count(scope_all),
    0 = syn:registry_count(scope_all, node()),
    0 = syn:registry_count(scope_all, SlaveNode1),
    0 = syn:registry_count(scope_all, SlaveNode2),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),

    %% check callbacks
    syn_test_suite_helper:assert_received_messages([
        {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, "meta-2", syn_conflict_resolution},
        {on_process_unregistered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, "meta-1", syn_conflict_resolution},
        {on_process_unregistered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, "meta-2", syn_conflict_resolution}
    ]),

    %% process alive (discarded process does not get killed with a custom handler)
    syn_test_suite_helper:assert_wait(
        true,
        fun() -> rpc:call(SlaveNode1, erlang, is_process_alive, [PidOn1]) end
    ),
    syn_test_suite_helper:assert_wait(
        true,
        fun() -> rpc:call(SlaveNode2, erlang, is_process_alive, [PidOn2]) end
    ),

    %% --> conflict by netsplit, which crashes

    %% partial netsplit (1 cannot see 2)
    rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
    syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),

    %% register with meta with no 'crash' element
    ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, {recipient, TestPid, crash}]),
    ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, {recipient, TestPid, crash}]),

    %% check callbacks
    syn_test_suite_helper:assert_received_messages([
        {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, crash, normal},
        {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, crash, normal},
        {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, crash, normal},

        {on_process_registered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, crash, normal},
        {on_process_registered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, crash, normal}
    ]),

    %% re-join
    rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
    syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
    syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),

    %% retrieve (names get freed)
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit-custom-crash") end
    ),
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom-crash"]) end
    ),
    syn_test_suite_helper:assert_wait(
        undefined,
        fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom-crash"]) end
    ),
    0 = syn:registry_count(scope_all),
    0 = syn:registry_count(scope_all, node()),
    0 = syn:registry_count(scope_all, SlaveNode1),
    0 = syn:registry_count(scope_all, SlaveNode2),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
    0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
    0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),

    %% check callbacks
    syn_test_suite_helper:assert_received_messages([
        {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, crash, syn_conflict_resolution},
        {on_process_unregistered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, crash, syn_conflict_resolution},
        {on_process_unregistered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, crash, syn_conflict_resolution}
    ]),

    %% process alive (discarded process does not get killed with a custom handler)
    syn_test_suite_helper:assert_wait(
        true,
        fun() -> rpc:call(SlaveNode1, erlang, is_process_alive, [PidOn1]) end
    ),
    syn_test_suite_helper:assert_wait(
        true,
        fun() -> rpc:call(SlaveNode2, erlang, is_process_alive, [PidOn2]) end
    ).

four_nodes_concurrency(Config) ->
    %% get slaves
    SlaveNode1 = proplists:get_value(syn_slave_1, Config),
    SlaveNode2 = proplists:get_value(syn_slave_2, Config),
    SlaveNode3 = proplists:get_value(syn_slave_3, Config),

    %% start syn on nodes
    ok = syn:start(),
    ok = rpc:call(SlaveNode1, syn, start, []),
    ok = rpc:call(SlaveNode2, syn, start, []),
    ok = rpc:call(SlaveNode3, syn, start, []),

    %% add scopes
    ok = syn:add_node_to_scopes([scope_all]),
    ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all]]),
    ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_all]]),
    ok = rpc:call(SlaveNode3, syn, add_node_to_scopes, [[scope_all]]),

    %% ref
    TestPid = self(),
    Iterations = 250,

    %% concurrent test
    WorkerFun = fun() ->
        lists:foreach(fun(_) ->
            %% start pid
            Pid = syn_test_suite_helper:start_process(),
            RandomMeta = rand:uniform(99999),
            %% loop
            case syn:register(scope_all, <<"concurrent">>, Pid, RandomMeta) of
                ok ->
                    ok;

                {error, taken} ->
                    case syn:unregister(scope_all, <<"concurrent">>) of
                        {error, undefined} ->
                            ok;
                        {error, race_condition} ->
                            ok;
                        ok ->
                            syn:register(scope_all, <<"concurrent">>, Pid, RandomMeta)
                    end
            end,
            %% random kill
            case rand:uniform(10) of
                1 -> exit(Pid, kill);
                _ -> ok
            end,
            %% random sleep
            RndTime = rand:uniform(30),
            timer:sleep(RndTime)
        end, lists:seq(1, Iterations)),
        TestPid ! {done, node()}
    end,

    %% spawn concurrent
    LocalNode = node(),
    spawn(LocalNode, WorkerFun),
    spawn(SlaveNode1, WorkerFun),
    spawn(SlaveNode2, WorkerFun),
    spawn(SlaveNode3, WorkerFun),

    %% wait for workers done
    syn_test_suite_helper:assert_received_messages([
        {done, LocalNode},
        {done, SlaveNode1},
        {done, SlaveNode2},
        {done, SlaveNode3}
    ]),

    %% check results are same across network
    syn_test_suite_helper:assert_wait(
        1,
        fun() ->
            ResultPidLocal = syn:lookup(scope_all, <<"concurrent">>),
            ResultPidOn1 = rpc:call(SlaveNode1, syn, lookup, [scope_all, <<"concurrent">>]),
            ResultPidOn2 = rpc:call(SlaveNode2, syn, lookup, [scope_all, <<"concurrent">>]),
            ResultPidOn3 = rpc:call(SlaveNode3, syn, lookup, [scope_all, <<"concurrent">>]),

            %% if unique set is of 1 element then they all contain the same result
            Ordset = ordsets:from_list([ResultPidLocal, ResultPidOn1, ResultPidOn2, ResultPidOn3]),
            ordsets:size(Ordset)
        end
    ).

%% ===================================================================
%% Internal
%% ===================================================================
add_to_local_table(Scope, Name, Pid, Meta, Time, MRef) ->
    TableByName = syn_backbone:get_table_name(syn_registry_by_name, Scope),
    TableByPid = syn_backbone:get_table_name(syn_registry_by_pid, Scope),
    syn_registry:add_to_local_table(Name, Pid, Meta, Time, MRef, TableByName, TableByPid).

remove_from_local_table(Scope, Name, Pid) ->
    TableByName = syn_backbone:get_table_name(syn_registry_by_name, Scope),
    TableByPid = syn_backbone:get_table_name(syn_registry_by_pid, Scope),
    syn_registry:remove_from_local_table(Name, Pid, TableByName, TableByPid).