syn_registry_SUITE.erl 73 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456
  1. %% ==========================================================================================================
  2. %% Syn - A global Process Registry and Process Group manager.
  3. %%
  4. %% The MIT License (MIT)
  5. %%
  6. %% Copyright (c) 2015-2021 Roberto Ostinelli <roberto@ostinelli.net> and Neato Robotics, Inc.
  7. %%
  8. %% Permission is hereby granted, free of charge, to any person obtaining a copy
  9. %% of this software and associated documentation files (the "Software"), to deal
  10. %% in the Software without restriction, including without limitation the rights
  11. %% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  12. %% copies of the Software, and to permit persons to whom the Software is
  13. %% furnished to do so, subject to the following conditions:
  14. %%
  15. %% The above copyright notice and this permission notice shall be included in
  16. %% all copies or substantial portions of the Software.
  17. %%
  18. %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19. %% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20. %% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21. %% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22. %% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  23. %% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  24. %% THE SOFTWARE.
  25. %% ==========================================================================================================
  26. -module(syn_registry_SUITE).
  27. %% callbacks
  28. -export([all/0]).
  29. -export([init_per_suite/1, end_per_suite/1]).
  30. -export([groups/0, init_per_group/2, end_per_group/2]).
  31. -export([init_per_testcase/2, end_per_testcase/2]).
  32. %% tests
  33. -export([
  34. one_node_via_register_unregister/1
  35. ]).
  36. -export([
  37. three_nodes_discover/1,
  38. three_nodes_register_unregister_and_monitor/1,
  39. three_nodes_register_filter_unknown_node/1,
  40. three_nodes_cluster_changes/1,
  41. three_nodes_cluster_conflicts/1,
  42. three_nodes_custom_event_handler_reg_unreg/1,
  43. three_nodes_custom_event_handler_conflict_resolution/1
  44. ]).
  45. %% include
  46. -include_lib("common_test/include/ct.hrl").
  47. -include_lib("syn/src/syn.hrl").
  48. %% ===================================================================
  49. %% Callbacks
  50. %% ===================================================================
  51. %% -------------------------------------------------------------------
  52. %% Function: all() -> GroupsAndTestCases | {skip,Reason}
  53. %% GroupsAndTestCases = [{group,GroupName} | TestCase]
  54. %% GroupName = atom()
  55. %% TestCase = atom()
  56. %% Reason = any()
  57. %% -------------------------------------------------------------------
  58. all() ->
  59. [
  60. {group, one_node_process_registration},
  61. {group, three_nodes_process_registration}
  62. ].
  63. %% -------------------------------------------------------------------
  64. %% Function: groups() -> [Group]
  65. %% Group = {GroupName,Properties,GroupsAndTestCases}
  66. %% GroupName = atom()
  67. %% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
  68. %% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
  69. %% TestCase = atom()
  70. %% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
  71. %% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
  72. %% repeat_until_any_ok | repeat_until_any_fail
  73. %% N = integer() | forever
  74. %% -------------------------------------------------------------------
  75. groups() ->
  76. [
  77. {one_node_process_registration, [shuffle], [
  78. one_node_via_register_unregister
  79. ]},
  80. {three_nodes_process_registration, [shuffle], [
  81. three_nodes_discover,
  82. three_nodes_register_unregister_and_monitor,
  83. three_nodes_register_filter_unknown_node,
  84. three_nodes_cluster_changes,
  85. three_nodes_cluster_conflicts,
  86. three_nodes_custom_event_handler_reg_unreg,
  87. three_nodes_custom_event_handler_conflict_resolution
  88. ]}
  89. ].
  90. %% -------------------------------------------------------------------
  91. %% Function: init_per_suite(Config0) ->
  92. %% Config1 | {skip,Reason} |
  93. %% {skip_and_save,Reason,Config1}
  94. %% Config0 = Config1 = [tuple()]
  95. %% Reason = any()
  96. %% -------------------------------------------------------------------
  97. init_per_suite(Config) ->
  98. Config.
  99. %% -------------------------------------------------------------------
  100. %% Function: end_per_suite(Config0) -> void() | {save_config,Config1}
  101. %% Config0 = Config1 = [tuple()]
  102. %% -------------------------------------------------------------------
  103. end_per_suite(_Config) ->
  104. ok.
  105. %% -------------------------------------------------------------------
  106. %% Function: init_per_group(GroupName, Config0) ->
  107. %% Config1 | {skip,Reason} |
  108. %% {skip_and_save,Reason,Config1}
  109. %% GroupName = atom()
  110. %% Config0 = Config1 = [tuple()]
  111. %% Reason = any()
  112. %% -------------------------------------------------------------------
  113. init_per_group(three_nodes_process_registration, Config) ->
  114. %% start slave
  115. {ok, SlaveNode1} = syn_test_suite_helper:start_slave(syn_slave_1),
  116. {ok, SlaveNode2} = syn_test_suite_helper:start_slave(syn_slave_2),
  117. syn_test_suite_helper:connect_node(SlaveNode1),
  118. syn_test_suite_helper:connect_node(SlaveNode2),
  119. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  120. %% wait full cluster
  121. case syn_test_suite_helper:wait_cluster_mesh_connected([node(), SlaveNode1, SlaveNode2]) of
  122. ok ->
  123. %% config
  124. [{slave_node_1, SlaveNode1}, {slave_node_2, SlaveNode2} | Config];
  125. Other ->
  126. ct:pal("*********** Could not get full cluster, skipping"),
  127. end_per_group(three_nodes_process_registration, Config),
  128. {skip, Other}
  129. end;
  130. init_per_group(_GroupName, Config) ->
  131. Config.
  132. %% -------------------------------------------------------------------
  133. %% Function: end_per_group(GroupName, Config0) ->
  134. %% void() | {save_config,Config1}
  135. %% GroupName = atom()
  136. %% Config0 = Config1 = [tuple()]
  137. %% -------------------------------------------------------------------
  138. end_per_group(three_nodes_process_registration, Config) ->
  139. SlaveNode1 = proplists:get_value(slave_node_1, Config),
  140. syn_test_suite_helper:connect_node(SlaveNode1),
  141. SlaveNode2 = proplists:get_value(slave_node_2, Config),
  142. syn_test_suite_helper:connect_node(SlaveNode2),
  143. syn_test_suite_helper:clean_after_test(),
  144. syn_test_suite_helper:stop_slave(syn_slave_1),
  145. syn_test_suite_helper:stop_slave(syn_slave_2),
  146. timer:sleep(1000);
  147. end_per_group(_GroupName, _Config) ->
  148. syn_test_suite_helper:clean_after_test().
  149. %% -------------------------------------------------------------------
  150. %% Function: init_per_testcase(TestCase, Config0) ->
  151. %% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
  152. %% TestCase = atom()
  153. %% Config0 = Config1 = [tuple()]
  154. %% Reason = any()
  155. %% -------------------------------------------------------------------
  156. init_per_testcase(TestCase, Config) ->
  157. ct:pal("Starting test: ~p", [TestCase]),
  158. Config.
  159. %% -------------------------------------------------------------------
  160. %% Function: end_per_testcase(TestCase, Config0) ->
  161. %% void() | {save_config,Config1} | {fail,Reason}
  162. %% TestCase = atom()
  163. %% Config0 = Config1 = [tuple()]
  164. %% Reason = any()
  165. %% -------------------------------------------------------------------
  166. end_per_testcase(_, _Config) ->
  167. syn_test_suite_helper:clean_after_test().
  168. %% ===================================================================
  169. %% Tests
  170. %% ===================================================================
  171. one_node_via_register_unregister(_Config) ->
  172. %% start syn
  173. ok = syn:start(),
  174. %% ---> scope
  175. syn:add_node_to_scopes([scope]),
  176. %% start gen server via syn
  177. GenServerNameCustom = {scope, <<"my proc">>},
  178. TupleCustom = {via, syn, GenServerNameCustom},
  179. {ok, PidCustom} = syn_test_gen_server:start_link(TupleCustom),
  180. %% retrieve
  181. {PidCustom, undefined} = syn:lookup(scope, <<"my proc">>),
  182. %% call
  183. pong = syn_test_gen_server:ping(TupleCustom),
  184. %% send via syn
  185. syn:send(GenServerNameCustom, {self(), send_ping}),
  186. syn_test_suite_helper:assert_received_messages([
  187. reply_pong
  188. ]),
  189. %% stop server
  190. syn_test_gen_server:stop(TupleCustom),
  191. %% retrieve
  192. syn_test_suite_helper:assert_wait(
  193. undefined,
  194. fun() -> syn:lookup(scope, <<"my proc">>) end
  195. ),
  196. %% send via syn
  197. {badarg, {GenServerNameCustom, anything}} = catch syn:send(GenServerNameCustom, anything).
  198. three_nodes_discover(Config) ->
  199. %% get slaves
  200. SlaveNode1 = proplists:get_value(slave_node_1, Config),
  201. SlaveNode2 = proplists:get_value(slave_node_2, Config),
  202. %% start syn on nodes
  203. ok = syn:start(),
  204. ok = rpc:call(SlaveNode1, syn, start, []),
  205. ok = rpc:call(SlaveNode2, syn, start, []),
  206. %% add scopes
  207. ok = syn:add_node_to_scopes([scope_ab]),
  208. ok = syn:add_node_to_scopes([scope_all]),
  209. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_ab, scope_bc, scope_all]]),
  210. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_bc, scope_c, scope_all]]),
  211. %% subcluster_nodes should return invalid errors
  212. {'EXIT', {{invalid_scope, custom_abcdef}, _}} = catch syn_registry:subcluster_nodes(custom_abcdef),
  213. %% check
  214. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
  215. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1, SlaveNode2]),
  216. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
  217. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
  218. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
  219. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_bc, [SlaveNode1]),
  220. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_c, []),
  221. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_all, [node(), SlaveNode1]),
  222. %% disconnect node 2 (node 1 can still see node 2)
  223. syn_test_suite_helper:disconnect_node(SlaveNode2),
  224. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1]),
  225. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  226. %% check
  227. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
  228. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1]),
  229. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
  230. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
  231. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
  232. %% reconnect node 2
  233. syn_test_suite_helper:connect_node(SlaveNode2),
  234. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  235. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  236. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  237. %% check
  238. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
  239. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1, SlaveNode2]),
  240. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
  241. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
  242. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
  243. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_bc, [SlaveNode1]),
  244. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_c, []),
  245. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_all, [node(), SlaveNode1]),
  246. %% crash a scope process on 2
  247. rpc:call(SlaveNode2, syn_test_suite_helper, kill_process, [syn_registry_scope_bc]),
  248. rpc:call(SlaveNode2, syn_test_suite_helper, wait_process_name_ready, [syn_registry_default]),
  249. %% check
  250. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
  251. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1, SlaveNode2]),
  252. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
  253. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
  254. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
  255. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_bc, [SlaveNode1]),
  256. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_c, []),
  257. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_all, [node(), SlaveNode1]),
  258. %% crash scopes supervisor on local
  259. syn_test_suite_helper:kill_process(syn_scopes_sup),
  260. syn_test_suite_helper:wait_process_name_ready(syn_registry_scope_ab),
  261. syn_test_suite_helper:wait_process_name_ready(syn_registry_scope_all),
  262. %% check
  263. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
  264. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1, SlaveNode2]),
  265. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
  266. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
  267. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
  268. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_bc, [SlaveNode1]),
  269. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_c, []),
  270. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_all, [node(), SlaveNode1]).
  271. three_nodes_register_unregister_and_monitor(Config) ->
  272. %% get slaves
  273. SlaveNode1 = proplists:get_value(slave_node_1, Config),
  274. SlaveNode2 = proplists:get_value(slave_node_2, Config),
  275. %% start syn on nodes
  276. ok = syn:start(),
  277. ok = rpc:call(SlaveNode1, syn, start, []),
  278. ok = rpc:call(SlaveNode2, syn, start, []),
  279. %% add scopes
  280. ok = syn:add_node_to_scopes([scope_ab]),
  281. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_ab, scope_bc]]),
  282. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_bc]]),
  283. %% start processes
  284. Pid = syn_test_suite_helper:start_process(),
  285. PidWithMeta = syn_test_suite_helper:start_process(),
  286. PidRemoteWithMetaOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  287. %% retrieve
  288. undefined = syn:lookup(scope_ab, "scope_a"),
  289. undefined = rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a"]),
  290. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a"]),
  291. undefined = syn:lookup(scope_ab, "scope_a_alias"),
  292. undefined = rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a_alias"]),
  293. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a_alias"]),
  294. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:lookup(scope_bc, {remote_scoped_bc}),
  295. undefined = rpc:call(SlaveNode1, syn, lookup, [scope_bc, {remote_scoped_bc}]),
  296. undefined = rpc:call(SlaveNode2, syn, lookup, [scope_bc, {remote_scoped_bc}]),
  297. 0 = syn:registry_count(scope_ab),
  298. 0 = syn:registry_count(scope_ab, node()),
  299. 0 = syn:registry_count(scope_ab, SlaveNode1),
  300. 0 = syn:registry_count(scope_ab, SlaveNode2),
  301. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:registry_count(scope_bc),
  302. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:registry_count(scope_bc, node()),
  303. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:registry_count(scope_bc, SlaveNode1),
  304. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:registry_count(scope_bc, SlaveNode2),
  305. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab]),
  306. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, node()]),
  307. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode1]),
  308. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode2]),
  309. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  310. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  311. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  312. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  313. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab]),
  314. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, node()]),
  315. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode1]),
  316. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode2]),
  317. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  318. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  319. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  320. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  321. %% register
  322. ok = syn:register(scope_ab, "scope_a", Pid),
  323. ok = syn:register(scope_ab, "scope_a_alias", PidWithMeta, <<"with_meta">>),
  324. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:register(scope_bc, "scope_a", Pid),
  325. {'EXIT', {{invalid_scope, non_existent_scope}, _}} = catch syn:register(non_existent_scope, "scope_a", Pid),
  326. ok = rpc:call(SlaveNode2, syn, register, [scope_bc, {remote_scoped_bc}, PidRemoteWithMetaOn1, <<"with_meta 1">>]),
  327. %% errors
  328. {error, taken} = syn:register(scope_ab, "scope_a", PidWithMeta),
  329. {error, not_alive} = syn:register(scope_ab, {"pid not alive"}, list_to_pid("<0.9999.0>")),
  330. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:register(scope_bc, "scope_a_noscope", Pid),
  331. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:unregister(scope_bc, "scope_a_noscope"),
  332. LocalNode = node(),
  333. {badrpc, {'EXIT', {{invalid_remote_scope, scope_bc, LocalNode}, _}}} = catch rpc:call(SlaveNode1, syn, register, [scope_bc, "pid-outside", Pid]),
  334. %% retrieve
  335. syn_test_suite_helper:assert_wait(
  336. {Pid, undefined},
  337. fun() -> syn:lookup(scope_ab, "scope_a") end
  338. ),
  339. syn_test_suite_helper:assert_wait(
  340. {Pid, undefined},
  341. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a"]) end
  342. ),
  343. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a"]),
  344. syn_test_suite_helper:assert_wait(
  345. {PidWithMeta, <<"with_meta">>},
  346. fun() -> syn:lookup(scope_ab, "scope_a_alias") end
  347. ),
  348. syn_test_suite_helper:assert_wait(
  349. {PidWithMeta, <<"with_meta">>},
  350. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a_alias"]) end
  351. ),
  352. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a_alias"]),
  353. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:lookup(scope_bc, {remote_scoped_bc}),
  354. syn_test_suite_helper:assert_wait(
  355. {PidRemoteWithMetaOn1, <<"with_meta 1">>},
  356. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, {remote_scoped_bc}]) end
  357. ),
  358. syn_test_suite_helper:assert_wait(
  359. {PidRemoteWithMetaOn1, <<"with_meta 1">>},
  360. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, {remote_scoped_bc}]) end
  361. ),
  362. 2 = syn:registry_count(scope_ab),
  363. 2 = syn:registry_count(scope_ab, node()),
  364. 0 = syn:registry_count(scope_ab, SlaveNode1),
  365. 0 = syn:registry_count(scope_ab, SlaveNode2),
  366. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:registry_count(scope_bc),
  367. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:registry_count(scope_bc, node()),
  368. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:registry_count(scope_bc, SlaveNode1),
  369. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:registry_count(scope_bc, SlaveNode2),
  370. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab]),
  371. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, node()]),
  372. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode1]),
  373. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode2]),
  374. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  375. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  376. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  377. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  378. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab]),
  379. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, node()]),
  380. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode1]),
  381. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode2]),
  382. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  383. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  384. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  385. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  386. %% re-register to edit meta
  387. ok = syn:register(scope_ab, "scope_a_alias", PidWithMeta, <<"with_meta_updated">>),
  388. syn_test_suite_helper:assert_wait(
  389. {PidWithMeta, <<"with_meta_updated">>},
  390. fun() -> syn:lookup(scope_ab, "scope_a_alias") end
  391. ),
  392. syn_test_suite_helper:assert_wait(
  393. {PidWithMeta, <<"with_meta_updated">>},
  394. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a_alias"]) end
  395. ),
  396. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a_alias"]),
  397. %% register remote
  398. syn:register(scope_ab, "ab_on_1", PidRemoteWithMetaOn1, <<"ab-on-1">>),
  399. syn_test_suite_helper:assert_wait(
  400. {PidRemoteWithMetaOn1, <<"ab-on-1">>},
  401. fun() -> syn:lookup(scope_ab, "ab_on_1") end
  402. ),
  403. %% crash scope process to ensure that monitors get recreated & data received from other nodes
  404. syn_test_suite_helper:kill_process(syn_registry_scope_ab),
  405. syn_test_suite_helper:wait_process_name_ready(syn_registry_scope_ab),
  406. %% check remote has been sync'ed back
  407. syn_test_suite_helper:assert_wait(
  408. {PidRemoteWithMetaOn1, <<"ab-on-1">>},
  409. fun() -> syn:lookup(scope_ab, "ab_on_1") end
  410. ),
  411. %% kill process
  412. syn_test_suite_helper:kill_process(Pid),
  413. syn_test_suite_helper:kill_process(PidWithMeta),
  414. %% unregister processes
  415. {error, undefined} = catch syn:unregister(scope_ab, <<"my proc with meta">>),
  416. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:unregister(scope_bc, <<"my proc with meta">>),
  417. ok = rpc:call(SlaveNode1, syn, unregister, [scope_bc, {remote_scoped_bc}]),
  418. %% retrieve
  419. syn_test_suite_helper:assert_wait(
  420. undefined,
  421. fun() -> syn:lookup(scope_ab, "scope_a") end
  422. ),
  423. syn_test_suite_helper:assert_wait(
  424. undefined,
  425. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a"]) end
  426. ),
  427. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a"]),
  428. syn_test_suite_helper:assert_wait(
  429. undefined,
  430. fun() -> syn:lookup(scope_ab, "scope_a_alias") end
  431. ),
  432. syn_test_suite_helper:assert_wait(
  433. undefined,
  434. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a_alias"]) end
  435. ),
  436. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a_alias"]),
  437. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:lookup(scope_bc, {remote_scoped_bc}),
  438. syn_test_suite_helper:assert_wait(
  439. undefined,
  440. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, {remote_scoped_bc}]) end
  441. ),
  442. syn_test_suite_helper:assert_wait(
  443. undefined,
  444. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, {remote_scoped_bc}]) end
  445. ),
  446. 1 = syn:registry_count(scope_ab),
  447. 0 = syn:registry_count(scope_ab, node()),
  448. 1 = syn:registry_count(scope_ab, SlaveNode1),
  449. 0 = syn:registry_count(scope_ab, SlaveNode2),
  450. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:registry_count(scope_bc),
  451. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:registry_count(scope_bc, node()),
  452. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:registry_count(scope_bc, SlaveNode1),
  453. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:registry_count(scope_bc, SlaveNode2),
  454. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab]),
  455. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, node()]),
  456. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode1]),
  457. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode2]),
  458. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  459. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  460. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  461. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  462. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab]),
  463. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, node()]),
  464. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode1]),
  465. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode2]),
  466. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  467. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  468. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  469. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  470. %% errors
  471. {error, undefined} = syn:unregister(scope_ab, {invalid_name}),
  472. %% (simulate race condition)
  473. Pid1 = syn_test_suite_helper:start_process(),
  474. Pid2 = syn_test_suite_helper:start_process(),
  475. ok = syn:register(scope_ab, <<"my proc">>, Pid1),
  476. syn_test_suite_helper:assert_wait(
  477. {Pid1, undefined},
  478. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, <<"my proc">>]) end
  479. ),
  480. remove_from_local_table(scope_ab, <<"my proc">>, Pid1),
  481. add_to_local_table(scope_ab, <<"my proc">>, Pid2, undefined, 0, undefined),
  482. {error, race_condition} = rpc:call(SlaveNode1, syn, unregister, [scope_ab, <<"my proc">>]).
  483. three_nodes_register_filter_unknown_node(Config) ->
  484. %% get slaves
  485. SlaveNode1 = proplists:get_value(slave_node_1, Config),
  486. SlaveNode2 = proplists:get_value(slave_node_2, Config),
  487. %% start syn on 1 and 2
  488. ok = rpc:call(SlaveNode1, syn, start, []),
  489. ok = rpc:call(SlaveNode2, syn, start, []),
  490. %% add scopes
  491. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_bc]]),
  492. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_bc]]),
  493. %% send sync message from out of scope node
  494. InvalidPid = syn_test_suite_helper:start_process(),
  495. {syn_registry_scope_bc, SlaveNode1} ! {'3.0', sync_register, <<"proc-name">>, InvalidPid, undefined, os:system_time(millisecond), normal},
  496. %% check
  497. undefined = rpc:call(SlaveNode1, syn, lookup, [scope_bc, <<"proc-name">>]).
  498. three_nodes_cluster_changes(Config) ->
  499. %% get slaves
  500. SlaveNode1 = proplists:get_value(slave_node_1, Config),
  501. SlaveNode2 = proplists:get_value(slave_node_2, Config),
  502. %% disconnect 1 from 2
  503. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  504. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  505. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  506. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  507. %% start syn on 1 and 2, nodes don't know of each other
  508. ok = rpc:call(SlaveNode1, syn, start, []),
  509. ok = rpc:call(SlaveNode2, syn, start, []),
  510. %% add scopes
  511. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  512. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  513. %% start processes
  514. PidRemoteOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  515. PidRemoteOn2 = syn_test_suite_helper:start_process(SlaveNode2),
  516. %% register
  517. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-1", PidRemoteOn1, "meta-1"]),
  518. ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-2", PidRemoteOn2, "meta-2"]),
  519. ok = rpc:call(SlaveNode1, syn, register, [scope_bc, "BC-proc-1", PidRemoteOn1, "meta-1"]),
  520. ok = rpc:call(SlaveNode1, syn, register, [scope_bc, "BC-proc-1 alias", PidRemoteOn1, "meta-1 alias"]),
  521. %% form full cluster
  522. ok = syn:start(),
  523. ok = syn:add_node_to_scopes([scope_all]),
  524. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  525. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  526. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  527. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  528. %% retrieve
  529. syn_test_suite_helper:assert_wait(
  530. {PidRemoteOn1, "meta-1"},
  531. fun() -> syn:lookup(scope_all, "proc-1") end
  532. ),
  533. syn_test_suite_helper:assert_wait(
  534. {PidRemoteOn1, "meta-1"},
  535. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-1"]) end
  536. ),
  537. syn_test_suite_helper:assert_wait(
  538. {PidRemoteOn1, "meta-1"},
  539. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-1"]) end
  540. ),
  541. syn_test_suite_helper:assert_wait(
  542. {PidRemoteOn2, "meta-2"},
  543. fun() -> syn:lookup(scope_all, "proc-2") end
  544. ),
  545. syn_test_suite_helper:assert_wait(
  546. {PidRemoteOn2, "meta-2"},
  547. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-2"]) end
  548. ),
  549. syn_test_suite_helper:assert_wait(
  550. {PidRemoteOn2, "meta-2"},
  551. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-2"]) end
  552. ),
  553. 2 = syn:registry_count(scope_all),
  554. 0 = syn:registry_count(scope_all, node()),
  555. 1 = syn:registry_count(scope_all, SlaveNode1),
  556. 1 = syn:registry_count(scope_all, SlaveNode2),
  557. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  558. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  559. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  560. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  561. 2 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  562. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  563. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  564. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  565. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:lookup(scope_bc, "BC-proc-1"),
  566. syn_test_suite_helper:assert_wait(
  567. {PidRemoteOn1, "meta-1"},
  568. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1"]) end
  569. ),
  570. syn_test_suite_helper:assert_wait(
  571. {PidRemoteOn1, "meta-1"},
  572. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1"]) end
  573. ),
  574. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:lookup(scope_bc, "BC-proc-1 alias"),
  575. syn_test_suite_helper:assert_wait(
  576. {PidRemoteOn1, "meta-1 alias"},
  577. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
  578. ),
  579. syn_test_suite_helper:assert_wait(
  580. {PidRemoteOn1, "meta-1 alias"},
  581. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
  582. ),
  583. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:registry_count(scope_bc),
  584. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:registry_count(scope_bc, node()),
  585. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:registry_count(scope_bc, SlaveNode1),
  586. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:registry_count(scope_bc, SlaveNode2),
  587. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  588. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  589. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  590. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  591. 2 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  592. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  593. 2 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  594. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  595. %% partial netsplit (1 cannot see 2)
  596. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  597. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  598. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  599. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  600. %% retrieve
  601. syn_test_suite_helper:assert_wait(
  602. {PidRemoteOn1, "meta-1"},
  603. fun() -> syn:lookup(scope_all, "proc-1") end
  604. ),
  605. syn_test_suite_helper:assert_wait(
  606. {PidRemoteOn1, "meta-1"},
  607. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-1"]) end
  608. ),
  609. syn_test_suite_helper:assert_wait(
  610. undefined,
  611. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-1"]) end
  612. ),
  613. syn_test_suite_helper:assert_wait(
  614. {PidRemoteOn2, "meta-2"},
  615. fun() -> syn:lookup(scope_all, "proc-2") end
  616. ),
  617. syn_test_suite_helper:assert_wait(
  618. undefined,
  619. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-2"]) end
  620. ),
  621. syn_test_suite_helper:assert_wait(
  622. {PidRemoteOn2, "meta-2"},
  623. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-2"]) end
  624. ),
  625. 2 = syn:registry_count(scope_all),
  626. 0 = syn:registry_count(scope_all, node()),
  627. 1 = syn:registry_count(scope_all, SlaveNode1),
  628. 1 = syn:registry_count(scope_all, SlaveNode2),
  629. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  630. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  631. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  632. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  633. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  634. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  635. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  636. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  637. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:lookup(scope_bc, "BC-proc-1"),
  638. syn_test_suite_helper:assert_wait(
  639. {PidRemoteOn1, "meta-1"},
  640. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1"]) end
  641. ),
  642. syn_test_suite_helper:assert_wait(
  643. undefined,
  644. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1"]) end
  645. ),
  646. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:lookup(scope_bc, "BC-proc-1 alias"),
  647. syn_test_suite_helper:assert_wait(
  648. {PidRemoteOn1, "meta-1 alias"},
  649. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
  650. ),
  651. syn_test_suite_helper:assert_wait(
  652. undefined,
  653. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
  654. ),
  655. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:registry_count(scope_bc),
  656. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  657. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  658. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  659. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  660. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  661. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  662. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  663. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  664. %% re-join
  665. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  666. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  667. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  668. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  669. %% retrieve
  670. syn_test_suite_helper:assert_wait(
  671. {PidRemoteOn1, "meta-1"},
  672. fun() -> syn:lookup(scope_all, "proc-1") end
  673. ),
  674. syn_test_suite_helper:assert_wait(
  675. {PidRemoteOn1, "meta-1"},
  676. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-1"]) end
  677. ),
  678. syn_test_suite_helper:assert_wait(
  679. {PidRemoteOn1, "meta-1"},
  680. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-1"]) end
  681. ),
  682. syn_test_suite_helper:assert_wait(
  683. {PidRemoteOn2, "meta-2"},
  684. fun() -> syn:lookup(scope_all, "proc-2") end
  685. ),
  686. syn_test_suite_helper:assert_wait(
  687. {PidRemoteOn2, "meta-2"},
  688. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-2"]) end
  689. ),
  690. syn_test_suite_helper:assert_wait(
  691. {PidRemoteOn2, "meta-2"},
  692. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-2"]) end
  693. ),
  694. 2 = syn:registry_count(scope_all),
  695. 0 = syn:registry_count(scope_all, node()),
  696. 1 = syn:registry_count(scope_all, SlaveNode1),
  697. 1 = syn:registry_count(scope_all, SlaveNode2),
  698. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  699. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  700. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  701. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  702. 2 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  703. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  704. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  705. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  706. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:lookup(scope_bc, "BC-proc-1"),
  707. syn_test_suite_helper:assert_wait(
  708. {PidRemoteOn1, "meta-1"},
  709. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1"]) end
  710. ),
  711. syn_test_suite_helper:assert_wait(
  712. {PidRemoteOn1, "meta-1"},
  713. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1"]) end
  714. ),
  715. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:lookup(scope_bc, "BC-proc-1 alias"),
  716. syn_test_suite_helper:assert_wait(
  717. {PidRemoteOn1, "meta-1 alias"},
  718. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
  719. ),
  720. syn_test_suite_helper:assert_wait(
  721. {PidRemoteOn1, "meta-1 alias"},
  722. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
  723. ),
  724. {'EXIT', {{invalid_scope, scope_bc}, _}} = catch syn:registry_count(scope_bc),
  725. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  726. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  727. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  728. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  729. 2 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  730. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  731. 2 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  732. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]).
  733. three_nodes_cluster_conflicts(Config) ->
  734. %% get slaves
  735. SlaveNode1 = proplists:get_value(slave_node_1, Config),
  736. SlaveNode2 = proplists:get_value(slave_node_2, Config),
  737. %% start syn on nodes
  738. ok = syn:start(),
  739. ok = rpc:call(SlaveNode1, syn, start, []),
  740. ok = rpc:call(SlaveNode2, syn, start, []),
  741. %% add scopes
  742. ok = syn:add_node_to_scopes([scope_all]),
  743. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  744. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  745. %% partial netsplit (1 cannot see 2)
  746. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  747. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  748. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  749. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  750. %% start conflict processes
  751. Pid2RemoteOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  752. Pid2RemoteOn2 = syn_test_suite_helper:start_process(SlaveNode2),
  753. %% --> conflict by netsplit
  754. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit", Pid2RemoteOn1, "meta-1"]),
  755. ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit", Pid2RemoteOn2, "meta-2"]),
  756. ok = rpc:call(SlaveNode1, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped", Pid2RemoteOn1, "meta-1"]),
  757. ok = rpc:call(SlaveNode2, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped", Pid2RemoteOn2, "meta-2"]),
  758. %% re-join
  759. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  760. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  761. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  762. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  763. %% retrieve
  764. syn_test_suite_helper:assert_wait(
  765. {Pid2RemoteOn2, "meta-2"},
  766. fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit") end
  767. ),
  768. syn_test_suite_helper:assert_wait(
  769. {Pid2RemoteOn2, "meta-2"},
  770. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit"]) end
  771. ),
  772. syn_test_suite_helper:assert_wait(
  773. {Pid2RemoteOn2, "meta-2"},
  774. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit"]) end
  775. ),
  776. 1 = syn:registry_count(scope_all),
  777. 0 = syn:registry_count(scope_all, node()),
  778. 0 = syn:registry_count(scope_all, SlaveNode1),
  779. 1 = syn:registry_count(scope_all, SlaveNode2),
  780. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  781. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  782. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  783. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  784. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  785. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  786. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  787. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  788. syn_test_suite_helper:assert_wait(
  789. {Pid2RemoteOn2, "meta-2"},
  790. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped"]) end
  791. ),
  792. syn_test_suite_helper:assert_wait(
  793. {Pid2RemoteOn2, "meta-2"},
  794. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped"]) end
  795. ),
  796. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  797. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  798. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  799. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  800. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  801. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  802. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  803. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  804. %% process alive
  805. syn_test_suite_helper:assert_wait(
  806. false,
  807. fun() -> rpc:call(SlaveNode1, erlang, is_process_alive, [Pid2RemoteOn1]) end
  808. ),
  809. syn_test_suite_helper:assert_wait(
  810. true,
  811. fun() -> rpc:call(SlaveNode2, erlang, is_process_alive, [Pid2RemoteOn2]) end
  812. ),
  813. %% --> conflict by race condition
  814. PidOnMaster = syn_test_suite_helper:start_process(),
  815. PidOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  816. rpc:call(SlaveNode1, syn_registry, add_to_local_table,
  817. [default, <<"my proc">>, PidOn1, "meta-2", erlang:system_time(), undefined]
  818. ),
  819. ok = syn:register(scope_all, <<"my proc">>, PidOnMaster, "meta-1"),
  820. %% retrieve
  821. syn_test_suite_helper:assert_wait(
  822. {PidOnMaster, "meta-1"},
  823. fun() -> syn:lookup(scope_all, <<"my proc">>) end
  824. ),
  825. syn_test_suite_helper:assert_wait(
  826. {PidOnMaster, "meta-1"},
  827. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, <<"my proc">>]) end
  828. ),
  829. syn_test_suite_helper:assert_wait(
  830. {PidOnMaster, "meta-1"},
  831. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, <<"my proc">>]) end
  832. ).
  833. %% NB: we can't check for process alive here because we injected the conflicting process in the DB
  834. %% -> it's not actually monitored
  835. three_nodes_custom_event_handler_reg_unreg(Config) ->
  836. %% get slaves
  837. SlaveNode1 = proplists:get_value(slave_node_1, Config),
  838. SlaveNode2 = proplists:get_value(slave_node_2, Config),
  839. %% add custom handler for callbacks
  840. syn:set_event_handler(syn_test_event_handler_callbacks),
  841. rpc:call(SlaveNode1, syn, set_event_handler, [syn_test_event_handler_callbacks]),
  842. rpc:call(SlaveNode2, syn, set_event_handler, [syn_test_event_handler_callbacks]),
  843. %% start syn on nodes
  844. ok = syn:start(),
  845. ok = rpc:call(SlaveNode1, syn, start, []),
  846. ok = rpc:call(SlaveNode2, syn, start, []),
  847. %% add scopes
  848. ok = syn:add_node_to_scopes([scope_all]),
  849. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all]]),
  850. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_all]]),
  851. %% init
  852. LocalNode = node(),
  853. %% start process
  854. Pid = syn_test_suite_helper:start_process(),
  855. Pid2 = syn_test_suite_helper:start_process(),
  856. %% ---> on registration
  857. ok = syn:register(scope_all, "proc-handler", Pid, {recipient, self(), <<"meta">>}),
  858. %% check callbacks called
  859. syn_test_suite_helper:assert_received_messages([
  860. {on_process_registered, LocalNode, scope_all, "proc-handler", Pid, <<"meta">>, normal},
  861. {on_process_registered, SlaveNode1, scope_all, "proc-handler", Pid, <<"meta">>, normal},
  862. {on_process_registered, SlaveNode2, scope_all, "proc-handler", Pid, <<"meta">>, normal}
  863. ]),
  864. syn_test_suite_helper:assert_empty_queue(self()),
  865. %% registration from another node
  866. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-handler-2", Pid2, {recipient, self(), <<"meta-for-2">>}]),
  867. %% check callbacks called
  868. syn_test_suite_helper:assert_received_messages([
  869. {on_process_registered, LocalNode, scope_all, "proc-handler-2", Pid2, <<"meta-for-2">>, normal},
  870. {on_process_registered, SlaveNode1, scope_all, "proc-handler-2", Pid2, <<"meta-for-2">>, normal},
  871. {on_process_registered, SlaveNode2, scope_all, "proc-handler-2", Pid2, <<"meta-for-2">>, normal}
  872. ]),
  873. syn_test_suite_helper:assert_empty_queue(self()),
  874. %% ---> on meta update
  875. ok = syn:register(scope_all, "proc-handler", Pid, {recipient, self(), <<"new-meta">>}),
  876. %% check callbacks called
  877. syn_test_suite_helper:assert_received_messages([
  878. {on_registry_process_updated, LocalNode, scope_all, "proc-handler", Pid, <<"new-meta">>, normal},
  879. {on_registry_process_updated, SlaveNode1, scope_all, "proc-handler", Pid, <<"new-meta">>, normal},
  880. {on_registry_process_updated, SlaveNode2, scope_all, "proc-handler", Pid, <<"new-meta">>, normal}
  881. ]),
  882. syn_test_suite_helper:assert_empty_queue(self()),
  883. %% meta update from another node
  884. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-handler-2", Pid2, {recipient, self(), <<"meta-for-2-update">>}]),
  885. %% check callbacks called
  886. syn_test_suite_helper:assert_received_messages([
  887. {on_registry_process_updated, LocalNode, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal},
  888. {on_registry_process_updated, SlaveNode1, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal},
  889. {on_registry_process_updated, SlaveNode2, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal}
  890. ]),
  891. syn_test_suite_helper:assert_empty_queue(self()),
  892. %% ---> on unregister
  893. ok = syn:unregister(scope_all, "proc-handler"),
  894. %% check callbacks called
  895. syn_test_suite_helper:assert_received_messages([
  896. {on_process_unregistered, LocalNode, scope_all, "proc-handler", Pid, <<"new-meta">>, normal},
  897. {on_process_unregistered, SlaveNode1, scope_all, "proc-handler", Pid, <<"new-meta">>, normal},
  898. {on_process_unregistered, SlaveNode2, scope_all, "proc-handler", Pid, <<"new-meta">>, normal}
  899. ]),
  900. syn_test_suite_helper:assert_empty_queue(self()),
  901. %% unregister from another node
  902. ok = rpc:call(SlaveNode1, syn, unregister, [scope_all, "proc-handler-2"]),
  903. %% check callbacks called
  904. syn_test_suite_helper:assert_received_messages([
  905. {on_process_unregistered, LocalNode, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal},
  906. {on_process_unregistered, SlaveNode1, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal},
  907. {on_process_unregistered, SlaveNode2, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal}
  908. ]),
  909. syn_test_suite_helper:assert_empty_queue(self()),
  910. %% clean & check
  911. syn_test_suite_helper:kill_process(Pid),
  912. syn_test_suite_helper:assert_empty_queue(self()),
  913. %% ---> after a netsplit
  914. PidRemoteOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  915. syn:register(scope_all, remote_on_1, PidRemoteOn1, {recipient, self(), <<"netsplit">>}),
  916. %% check callbacks called
  917. syn_test_suite_helper:assert_received_messages([
  918. {on_process_registered, LocalNode, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, normal},
  919. {on_process_registered, SlaveNode1, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, normal},
  920. {on_process_registered, SlaveNode2, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, normal}
  921. ]),
  922. syn_test_suite_helper:assert_empty_queue(self()),
  923. %% partial netsplit (1 cannot see 2)
  924. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  925. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  926. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  927. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  928. %% check callbacks called
  929. syn_test_suite_helper:assert_received_messages([
  930. {on_process_unregistered, SlaveNode2, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, {syn_remote_scope_node_down, scope_all, SlaveNode1}}
  931. ]),
  932. syn_test_suite_helper:assert_empty_queue(self()),
  933. %% ---> after a re-join
  934. %% re-join
  935. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  936. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  937. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  938. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  939. %% check callbacks called
  940. syn_test_suite_helper:assert_received_messages([
  941. {on_process_registered, SlaveNode2, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, {syn_remote_scope_node_up, scope_all, SlaveNode1}}
  942. ]),
  943. syn_test_suite_helper:assert_empty_queue(self()),
  944. %% clean
  945. syn_test_suite_helper:kill_process(PidRemoteOn1),
  946. %% check callbacks called
  947. syn_test_suite_helper:assert_received_messages([
  948. {on_process_unregistered, LocalNode, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, killed},
  949. {on_process_unregistered, SlaveNode1, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, killed},
  950. {on_process_unregistered, SlaveNode2, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, killed}
  951. ]),
  952. syn_test_suite_helper:assert_empty_queue(self()),
  953. %% ---> after a conflict resolution
  954. %% partial netsplit (1 cannot see 2)
  955. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  956. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  957. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  958. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  959. %% start conflict processes
  960. Pid2RemoteOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  961. Pid2RemoteOn2 = syn_test_suite_helper:start_process(SlaveNode2),
  962. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict", Pid2RemoteOn1, {recipient, self(), <<"meta-1">>}]),
  963. ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict", Pid2RemoteOn2, {recipient, self(), <<"meta-2">>}]),
  964. %% check callbacks called
  965. syn_test_suite_helper:assert_received_messages([
  966. {on_process_registered, LocalNode, scope_all, "proc-confict", Pid2RemoteOn1, <<"meta-1">>, normal},
  967. {on_process_unregistered, LocalNode, scope_all, "proc-confict", Pid2RemoteOn1, <<"meta-1">>, normal},
  968. {on_process_registered, LocalNode, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, normal},
  969. {on_process_registered, SlaveNode1, scope_all, "proc-confict", Pid2RemoteOn1, <<"meta-1">>, normal},
  970. {on_process_registered, SlaveNode2, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, normal}
  971. ]),
  972. syn_test_suite_helper:assert_empty_queue(self()),
  973. %% re-join
  974. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  975. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  976. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  977. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  978. %% check callbacks called
  979. syn_test_suite_helper:assert_received_messages([
  980. {on_process_unregistered, SlaveNode1, scope_all, "proc-confict", Pid2RemoteOn1, <<"meta-1">>, syn_conflict_resolution},
  981. {on_process_registered, SlaveNode1, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, syn_conflict_resolution}
  982. ]),
  983. syn_test_suite_helper:assert_empty_queue(self()),
  984. %% kill
  985. syn_test_suite_helper:kill_process(Pid2RemoteOn1),
  986. syn_test_suite_helper:kill_process(Pid2RemoteOn2),
  987. %% check callbacks called
  988. syn_test_suite_helper:assert_received_messages([
  989. {on_process_unregistered, LocalNode, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, killed},
  990. {on_process_unregistered, SlaveNode1, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, killed},
  991. {on_process_unregistered, SlaveNode2, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, killed}
  992. ]),
  993. syn_test_suite_helper:assert_empty_queue(self()),
  994. %% ---> don't call on monitor rebuild
  995. %% crash the scope process on local
  996. syn_test_suite_helper:kill_process(syn_registry_scope_all),
  997. syn_test_suite_helper:wait_process_name_ready(syn_registry_scope_all),
  998. %% no messages
  999. syn_test_suite_helper:assert_wait(
  1000. ok,
  1001. fun() -> syn_test_suite_helper:assert_empty_queue(self()) end
  1002. ),
  1003. %% ---> call if process died during the scope process crash
  1004. TransientPid = syn_test_suite_helper:start_process(),
  1005. syn:register(scope_all, "transient-pid", TransientPid, {recipient, self(), "transient-meta"}),
  1006. %% check callbacks called
  1007. syn_test_suite_helper:assert_received_messages([
  1008. {on_process_registered, LocalNode, scope_all, "transient-pid", TransientPid, "transient-meta", normal},
  1009. {on_process_registered, SlaveNode1, scope_all, "transient-pid", TransientPid, "transient-meta", normal},
  1010. {on_process_registered, SlaveNode2, scope_all, "transient-pid", TransientPid, "transient-meta", normal}
  1011. ]),
  1012. syn_test_suite_helper:assert_empty_queue(self()),
  1013. %% crash the scope process & transient process on local
  1014. syn_test_suite_helper:kill_process(syn_registry_scope_all),
  1015. syn_test_suite_helper:kill_process(TransientPid),
  1016. %% check callbacks called
  1017. syn_test_suite_helper:assert_received_messages([
  1018. {on_process_unregistered, LocalNode, scope_all, "transient-pid", TransientPid, "transient-meta", undefined},
  1019. {on_process_unregistered, SlaveNode1, scope_all, "transient-pid", TransientPid, "transient-meta", {syn_remote_scope_node_down, scope_all, LocalNode}},
  1020. {on_process_unregistered, SlaveNode2, scope_all, "transient-pid", TransientPid, "transient-meta", {syn_remote_scope_node_down, scope_all, LocalNode}}
  1021. ]),
  1022. syn_test_suite_helper:assert_empty_queue(self()).
  1023. three_nodes_custom_event_handler_conflict_resolution(Config) ->
  1024. %% get slaves
  1025. SlaveNode1 = proplists:get_value(slave_node_1, Config),
  1026. SlaveNode2 = proplists:get_value(slave_node_2, Config),
  1027. %% add custom handler for resolution
  1028. syn:set_event_handler(syn_test_event_handler_resolution),
  1029. rpc:call(SlaveNode1, syn, set_event_handler, [syn_test_event_handler_resolution]),
  1030. rpc:call(SlaveNode2, syn, set_event_handler, [syn_test_event_handler_resolution]),
  1031. %% start syn on nodes
  1032. ok = syn:start(),
  1033. ok = rpc:call(SlaveNode1, syn, start, []),
  1034. ok = rpc:call(SlaveNode2, syn, start, []),
  1035. %% add scopes
  1036. ok = syn:add_node_to_scopes([scope_all]),
  1037. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  1038. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  1039. %% current node
  1040. TestPid = self(),
  1041. LocalNode = node(),
  1042. %% partial netsplit (1 cannot see 2)
  1043. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  1044. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1045. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  1046. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  1047. %% start conflict processes
  1048. PidOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  1049. PidOn2 = syn_test_suite_helper:start_process(SlaveNode2),
  1050. %% --> conflict by netsplit
  1051. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit-custom", PidOn1, {recipient, TestPid, keepthis}]),
  1052. ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit-custom", PidOn2, {recipient, TestPid, "meta-2"}]),
  1053. ok = rpc:call(SlaveNode1, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, {recipient, TestPid, keepthis}]),
  1054. ok = rpc:call(SlaveNode2, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn2, {recipient, TestPid, "meta-2"}]),
  1055. %% check callbacks
  1056. syn_test_suite_helper:assert_received_messages([
  1057. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
  1058. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
  1059. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn2, "meta-2", normal},
  1060. {on_process_registered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
  1061. {on_process_registered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom", PidOn2, "meta-2", normal},
  1062. {on_process_registered, SlaveNode1, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, keepthis, normal},
  1063. {on_process_registered, SlaveNode2, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn2, "meta-2", normal}
  1064. ]),
  1065. syn_test_suite_helper:assert_empty_queue(self()),
  1066. %% re-join
  1067. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  1068. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1069. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  1070. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  1071. %% retrieve
  1072. syn_test_suite_helper:assert_wait(
  1073. {PidOn1, {recipient, TestPid, keepthis}},
  1074. fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit-custom") end
  1075. ),
  1076. syn_test_suite_helper:assert_wait(
  1077. {PidOn1, {recipient, TestPid, keepthis}},
  1078. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom"]) end
  1079. ),
  1080. syn_test_suite_helper:assert_wait(
  1081. {PidOn1, {recipient, TestPid, keepthis}},
  1082. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom"]) end
  1083. ),
  1084. 1 = syn:registry_count(scope_all),
  1085. 0 = syn:registry_count(scope_all, node()),
  1086. 1 = syn:registry_count(scope_all, SlaveNode1),
  1087. 0 = syn:registry_count(scope_all, SlaveNode2),
  1088. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  1089. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  1090. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  1091. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  1092. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  1093. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  1094. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  1095. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  1096. syn_test_suite_helper:assert_wait(
  1097. {PidOn1, {recipient, TestPid, keepthis}},
  1098. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]) end
  1099. ),
  1100. syn_test_suite_helper:assert_wait(
  1101. {PidOn1, {recipient, TestPid, keepthis}},
  1102. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]) end
  1103. ),
  1104. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  1105. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  1106. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  1107. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  1108. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  1109. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  1110. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  1111. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  1112. syn_test_suite_helper:assert_received_messages([
  1113. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn2, "meta-2", syn_conflict_resolution},
  1114. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, syn_conflict_resolution},
  1115. {on_process_unregistered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom", PidOn2, "meta-2", syn_conflict_resolution},
  1116. {on_process_registered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, syn_conflict_resolution},
  1117. {on_process_unregistered, SlaveNode2, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn2, "meta-2", syn_conflict_resolution},
  1118. {on_process_registered, SlaveNode2, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, keepthis, syn_conflict_resolution}
  1119. ]),
  1120. syn_test_suite_helper:assert_empty_queue(self()),
  1121. %% process alive (discarded process does not get killed with a custom handler)
  1122. syn_test_suite_helper:assert_wait(
  1123. true,
  1124. fun() -> rpc:call(SlaveNode1, erlang, is_process_alive, [PidOn1]) end
  1125. ),
  1126. syn_test_suite_helper:assert_wait(
  1127. true,
  1128. fun() -> rpc:call(SlaveNode2, erlang, is_process_alive, [PidOn2]) end
  1129. ),
  1130. %% clean up default scope
  1131. syn:unregister(scope_all, "proc-confict-by-netsplit-custom"),
  1132. ok = rpc:call(SlaveNode1, syn, unregister, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]),
  1133. %% retrieve
  1134. syn_test_suite_helper:assert_wait(
  1135. undefined,
  1136. fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit-custom") end
  1137. ),
  1138. syn_test_suite_helper:assert_wait(
  1139. undefined,
  1140. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom"]) end
  1141. ),
  1142. syn_test_suite_helper:assert_wait(
  1143. undefined,
  1144. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom"]) end
  1145. ),
  1146. syn_test_suite_helper:assert_wait(
  1147. undefined,
  1148. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]) end
  1149. ),
  1150. syn_test_suite_helper:assert_wait(
  1151. undefined,
  1152. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]) end
  1153. ),
  1154. %% check callbacks
  1155. syn_test_suite_helper:assert_received_messages([
  1156. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
  1157. {on_process_unregistered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
  1158. {on_process_unregistered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
  1159. {on_process_unregistered, SlaveNode1, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, keepthis, normal},
  1160. {on_process_unregistered, SlaveNode2, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, keepthis, normal}
  1161. ]),
  1162. syn_test_suite_helper:assert_empty_queue(self()),
  1163. %% --> conflict by netsplit, which returns invalid pid
  1164. %% partial netsplit (1 cannot see 2)
  1165. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  1166. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1167. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  1168. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  1169. %% register with meta with no 'keepthis' element
  1170. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, {recipient, TestPid, "meta-1"}]),
  1171. ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, {recipient, TestPid, "meta-2"}]),
  1172. %% check callbacks
  1173. syn_test_suite_helper:assert_received_messages([
  1174. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, "meta-1", normal},
  1175. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, "meta-1", normal},
  1176. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, "meta-2", normal},
  1177. {on_process_registered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, "meta-1", normal},
  1178. {on_process_registered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, "meta-2", normal}
  1179. ]),
  1180. syn_test_suite_helper:assert_empty_queue(self()),
  1181. %% re-join
  1182. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  1183. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1184. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  1185. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  1186. %% retrieve (names get freed)
  1187. syn_test_suite_helper:assert_wait(
  1188. undefined,
  1189. fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit-custom-other-pid") end
  1190. ),
  1191. syn_test_suite_helper:assert_wait(
  1192. undefined,
  1193. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom-other-pid"]) end
  1194. ),
  1195. syn_test_suite_helper:assert_wait(
  1196. undefined,
  1197. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom-other-pid"]) end
  1198. ),
  1199. 0 = syn:registry_count(scope_all),
  1200. 0 = syn:registry_count(scope_all, node()),
  1201. 0 = syn:registry_count(scope_all, SlaveNode1),
  1202. 0 = syn:registry_count(scope_all, SlaveNode2),
  1203. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  1204. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  1205. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  1206. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  1207. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  1208. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  1209. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  1210. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  1211. %% check callbacks
  1212. syn_test_suite_helper:assert_received_messages([
  1213. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, "meta-2", syn_conflict_resolution},
  1214. {on_process_unregistered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, "meta-1", syn_conflict_resolution},
  1215. {on_process_unregistered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, "meta-2", syn_conflict_resolution}
  1216. ]),
  1217. syn_test_suite_helper:assert_empty_queue(self()),
  1218. %% process alive (discarded process does not get killed with a custom handler)
  1219. syn_test_suite_helper:assert_wait(
  1220. true,
  1221. fun() -> rpc:call(SlaveNode1, erlang, is_process_alive, [PidOn1]) end
  1222. ),
  1223. syn_test_suite_helper:assert_wait(
  1224. true,
  1225. fun() -> rpc:call(SlaveNode2, erlang, is_process_alive, [PidOn2]) end
  1226. ),
  1227. %% --> conflict by netsplit, which crashes
  1228. %% partial netsplit (1 cannot see 2)
  1229. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  1230. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1231. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  1232. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  1233. %% register with meta with no 'crash' element
  1234. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, {recipient, TestPid, crash}]),
  1235. ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, {recipient, TestPid, crash}]),
  1236. %% check callbacks
  1237. syn_test_suite_helper:assert_received_messages([
  1238. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, crash, normal},
  1239. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, crash, normal},
  1240. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, crash, normal},
  1241. {on_process_registered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, crash, normal},
  1242. {on_process_registered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, crash, normal}
  1243. ]),
  1244. syn_test_suite_helper:assert_empty_queue(self()),
  1245. %% re-join
  1246. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  1247. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1248. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  1249. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  1250. %% retrieve (names get freed)
  1251. syn_test_suite_helper:assert_wait(
  1252. undefined,
  1253. fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit-custom-crash") end
  1254. ),
  1255. syn_test_suite_helper:assert_wait(
  1256. undefined,
  1257. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom-crash"]) end
  1258. ),
  1259. syn_test_suite_helper:assert_wait(
  1260. undefined,
  1261. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom-crash"]) end
  1262. ),
  1263. 0 = syn:registry_count(scope_all),
  1264. 0 = syn:registry_count(scope_all, node()),
  1265. 0 = syn:registry_count(scope_all, SlaveNode1),
  1266. 0 = syn:registry_count(scope_all, SlaveNode2),
  1267. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  1268. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  1269. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  1270. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  1271. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  1272. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  1273. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  1274. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  1275. %% check callbacks
  1276. syn_test_suite_helper:assert_received_messages([
  1277. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, crash, syn_conflict_resolution},
  1278. {on_process_unregistered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, crash, syn_conflict_resolution},
  1279. {on_process_unregistered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, crash, syn_conflict_resolution}
  1280. ]),
  1281. syn_test_suite_helper:assert_empty_queue(self()),
  1282. %% process alive (discarded process does not get killed with a custom handler)
  1283. syn_test_suite_helper:assert_wait(
  1284. true,
  1285. fun() -> rpc:call(SlaveNode1, erlang, is_process_alive, [PidOn1]) end
  1286. ),
  1287. syn_test_suite_helper:assert_wait(
  1288. true,
  1289. fun() -> rpc:call(SlaveNode2, erlang, is_process_alive, [PidOn2]) end
  1290. ).
  1291. %% ===================================================================
  1292. %% Internal
  1293. %% ===================================================================
  1294. add_to_local_table(Scope, Name, Pid, Meta, Time, MRef) ->
  1295. TableByName = syn_backbone:get_table_name(syn_registry_by_name, Scope),
  1296. TableByPid = syn_backbone:get_table_name(syn_registry_by_pid, Scope),
  1297. syn_registry:add_to_local_table(Name, Pid, Meta, Time, MRef, TableByName, TableByPid).
  1298. remove_from_local_table(Scope, Name, Pid) ->
  1299. TableByName = syn_backbone:get_table_name(syn_registry_by_name, Scope),
  1300. TableByPid = syn_backbone:get_table_name(syn_registry_by_pid, Scope),
  1301. syn_registry:remove_from_local_table(Name, Pid, TableByName, TableByPid).