syn_registry_SUITE.erl 82 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711
  1. %% ==========================================================================================================
  2. %% Syn - A global Process Registry and Process Group manager.
  3. %%
  4. %% The MIT License (MIT)
  5. %%
  6. %% Copyright (c) 2015-2021 Roberto Ostinelli <roberto@ostinelli.net> and Neato Robotics, Inc.
  7. %%
  8. %% Permission is hereby granted, free of charge, to any person obtaining a copy
  9. %% of this software and associated documentation files (the "Software"), to deal
  10. %% in the Software without restriction, including without limitation the rights
  11. %% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  12. %% copies of the Software, and to permit persons to whom the Software is
  13. %% furnished to do so, subject to the following conditions:
  14. %%
  15. %% The above copyright notice and this permission notice shall be included in
  16. %% all copies or substantial portions of the Software.
  17. %%
  18. %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19. %% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20. %% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21. %% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22. %% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  23. %% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  24. %% THE SOFTWARE.
  25. %% ==========================================================================================================
  26. -module(syn_registry_SUITE).
  27. %% callbacks
  28. -export([all/0]).
  29. -export([init_per_suite/1, end_per_suite/1]).
  30. -export([groups/0, init_per_group/2, end_per_group/2]).
  31. -export([init_per_testcase/2, end_per_testcase/2]).
  32. %% tests
  33. -export([
  34. one_node_via_register_unregister/1,
  35. one_node_via_register_unregister_with_metadata/1,
  36. one_node_strict_mode/1
  37. ]).
  38. -export([
  39. three_nodes_discover/1,
  40. three_nodes_register_unregister_and_monitor/1,
  41. three_nodes_register_filter_unknown_node/1,
  42. three_nodes_cluster_changes/1,
  43. three_nodes_cluster_conflicts/1,
  44. three_nodes_custom_event_handler_reg_unreg/1,
  45. three_nodes_custom_event_handler_conflict_resolution/1,
  46. three_nodes_update/1
  47. ]).
  48. -export([
  49. four_nodes_concurrency/1
  50. ]).
  51. %% include
  52. -include_lib("common_test/include/ct.hrl").
  53. -include_lib("syn/src/syn.hrl").
  54. %% ===================================================================
  55. %% Callbacks
  56. %% ===================================================================
  57. %% -------------------------------------------------------------------
  58. %% Function: all() -> GroupsAndTestCases | {skip,Reason}
  59. %% GroupsAndTestCases = [{group,GroupName} | TestCase]
  60. %% GroupName = atom()
  61. %% TestCase = atom()
  62. %% Reason = any()
  63. %% -------------------------------------------------------------------
  64. all() ->
  65. [
  66. {group, one_node_registry},
  67. {group, three_nodes_registry},
  68. {group, four_nodes_registry}
  69. ].
  70. %% -------------------------------------------------------------------
  71. %% Function: groups() -> [Group]
  72. %% Group = {GroupName,Properties,GroupsAndTestCases}
  73. %% GroupName = atom()
  74. %% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
  75. %% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
  76. %% TestCase = atom()
  77. %% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
  78. %% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
  79. %% repeat_until_any_ok | repeat_until_any_fail
  80. %% N = integer() | forever
  81. %% -------------------------------------------------------------------
  82. groups() ->
  83. [
  84. {one_node_registry, [shuffle], [
  85. one_node_via_register_unregister,
  86. one_node_via_register_unregister_with_metadata,
  87. one_node_strict_mode
  88. ]},
  89. {three_nodes_registry, [shuffle], [
  90. three_nodes_discover,
  91. three_nodes_register_unregister_and_monitor,
  92. three_nodes_register_filter_unknown_node,
  93. three_nodes_cluster_changes,
  94. three_nodes_cluster_conflicts,
  95. three_nodes_custom_event_handler_reg_unreg,
  96. three_nodes_custom_event_handler_conflict_resolution,
  97. three_nodes_update
  98. ]},
  99. {four_nodes_registry, [shuffle], [
  100. four_nodes_concurrency
  101. ]}
  102. ].
  103. %% -------------------------------------------------------------------
  104. %% Function: init_per_suite(Config0) ->
  105. %% Config1 | {skip,Reason} |
  106. %% {skip_and_save,Reason,Config1}
  107. %% Config0 = Config1 = [tuple()]
  108. %% Reason = any()
  109. %% -------------------------------------------------------------------
  110. init_per_suite(Config) ->
  111. Config.
  112. %% -------------------------------------------------------------------
  113. %% Function: end_per_suite(Config0) -> void() | {save_config,Config1}
  114. %% Config0 = Config1 = [tuple()]
  115. %% -------------------------------------------------------------------
  116. end_per_suite(_Config) ->
  117. ok.
  118. %% -------------------------------------------------------------------
  119. %% Function: init_per_group(GroupName, Config0) ->
  120. %% Config1 | {skip,Reason} |
  121. %% {skip_and_save,Reason,Config1}
  122. %% GroupName = atom()
  123. %% Config0 = Config1 = [tuple()]
  124. %% Reason = any()
  125. %% -------------------------------------------------------------------
  126. init_per_group(three_nodes_registry, Config) ->
  127. case syn_test_suite_helper:init_cluster(3) of
  128. {error_initializing_cluster, Other} ->
  129. end_per_group(three_nodes_registry, Config),
  130. {skip, Other};
  131. NodesConfig ->
  132. NodesConfig ++ Config
  133. end;
  134. init_per_group(four_nodes_registry, Config) ->
  135. case syn_test_suite_helper:init_cluster(4) of
  136. {error_initializing_cluster, Other} ->
  137. end_per_group(four_nodes_registry, Config),
  138. {skip, Other};
  139. NodesConfig ->
  140. NodesConfig ++ Config
  141. end;
  142. init_per_group(_GroupName, Config) ->
  143. Config.
  144. %% -------------------------------------------------------------------
  145. %% Function: end_per_group(GroupName, Config0) ->
  146. %% void() | {save_config,Config1}
  147. %% GroupName = atom()
  148. %% Config0 = Config1 = [tuple()]
  149. %% -------------------------------------------------------------------
  150. end_per_group(three_nodes_registry, Config) ->
  151. syn_test_suite_helper:end_cluster(3, Config);
  152. end_per_group(four_nodes_registry, Config) ->
  153. syn_test_suite_helper:end_cluster(4, Config);
  154. end_per_group(_GroupName, _Config) ->
  155. syn_test_suite_helper:clean_after_test().
  156. %% -------------------------------------------------------------------
  157. %% Function: init_per_testcase(TestCase, Config0) ->
  158. %% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
  159. %% TestCase = atom()
  160. %% Config0 = Config1 = [tuple()]
  161. %% Reason = any()
  162. %% -------------------------------------------------------------------
  163. init_per_testcase(TestCase, Config) ->
  164. ct:pal("Starting test: ~p", [TestCase]),
  165. Config.
  166. %% -------------------------------------------------------------------
  167. %% Function: end_per_testcase(TestCase, Config0) ->
  168. %% void() | {save_config,Config1} | {fail,Reason}
  169. %% TestCase = atom()
  170. %% Config0 = Config1 = [tuple()]
  171. %% Reason = any()
  172. %% -------------------------------------------------------------------
  173. end_per_testcase(_, _Config) ->
  174. syn_test_suite_helper:clean_after_test().
  175. %% ===================================================================
  176. %% Tests
  177. %% ===================================================================
  178. one_node_via_register_unregister(_Config) ->
  179. %% start syn
  180. ok = syn:start(),
  181. %% ---> scope
  182. syn:add_node_to_scopes([scope]),
  183. %% start gen server via syn
  184. GenServerNameCustom = {scope, <<"my proc">>},
  185. TupleCustom = {via, syn, GenServerNameCustom},
  186. {ok, PidCustom} = syn_test_gen_server:start_link(TupleCustom),
  187. %% retrieve
  188. {PidCustom, undefined} = syn:lookup(scope, <<"my proc">>),
  189. %% call
  190. pong = syn_test_gen_server:ping(TupleCustom),
  191. %% send via syn
  192. syn:send(GenServerNameCustom, {self(), send_ping}),
  193. syn_test_suite_helper:assert_received_messages([
  194. reply_pong
  195. ]),
  196. %% stop server
  197. syn_test_gen_server:stop(TupleCustom),
  198. %% retrieve
  199. syn_test_suite_helper:assert_wait(
  200. undefined,
  201. fun() -> syn:lookup(scope, <<"my proc">>) end
  202. ),
  203. %% send via syn
  204. {badarg, {GenServerNameCustom, anything}} = (catch syn:send(GenServerNameCustom, anything)).
  205. one_node_via_register_unregister_with_metadata(_Config) ->
  206. %% start syn
  207. ok = syn:start(),
  208. syn:add_node_to_scopes([scope]),
  209. %% start gen server via syn
  210. GenServerNameCustom = {scope, <<"my proc">>},
  211. GenServerNameCustomMeta = {scope, <<"my proc">>, my_metadata},
  212. TupleCustom = {via, syn, GenServerNameCustom},
  213. TupleCustomMeta = {via, syn, GenServerNameCustomMeta},
  214. {ok, PidCustom} = syn_test_gen_server:start_link(TupleCustomMeta),
  215. %% retrieve
  216. {PidCustom, my_metadata} = syn:lookup(scope, <<"my proc">>),
  217. %% call
  218. pong = syn_test_gen_server:ping(TupleCustomMeta),
  219. pong = syn_test_gen_server:ping(TupleCustom),
  220. %% send via syn
  221. syn:send(GenServerNameCustomMeta, {self(), send_ping}),
  222. syn_test_suite_helper:assert_received_messages([
  223. reply_pong
  224. ]),
  225. syn:send(GenServerNameCustom, {self(), send_ping}),
  226. syn_test_suite_helper:assert_received_messages([
  227. reply_pong
  228. ]),
  229. %% stop server
  230. syn_test_gen_server:stop(TupleCustomMeta),
  231. %% retrieve
  232. syn_test_suite_helper:assert_wait(
  233. undefined,
  234. fun() -> syn:lookup(scope, <<"my proc">>) end
  235. ),
  236. %% send via syn
  237. {badarg, {GenServerNameCustomMeta, anything}} = (catch syn:send(GenServerNameCustomMeta, anything)),
  238. {badarg, {GenServerNameCustom, anything}} = (catch syn:send(GenServerNameCustom, anything)).
  239. one_node_strict_mode(_Config) ->
  240. %% start syn
  241. ok = syn:start(),
  242. syn:add_node_to_scopes([scope]),
  243. %% strict mode enabled
  244. application:set_env(syn, strict_mode, true),
  245. %% start process
  246. Pid = syn_test_suite_helper:start_process(),
  247. {error, not_self} = syn:register(scope, "strict-true", Pid, metadata),
  248. Self = self(),
  249. ok = syn:register(scope, "strict-true", Self, metadata),
  250. ok = syn:register(scope, "strict-true", Self, new_metadata),
  251. {Self, new_metadata} = syn:lookup(scope, "strict-true"),
  252. ok = syn:register(scope, "strict-true", Self),
  253. {Self, undefined} = syn:lookup(scope, "strict-true").
  254. three_nodes_discover(Config) ->
  255. %% get slaves
  256. SlaveNode1 = proplists:get_value(syn_slave_1, Config),
  257. SlaveNode2 = proplists:get_value(syn_slave_2, Config),
  258. %% add scopes partially with ENV
  259. ok = rpc:call(SlaveNode2, application, set_env, [syn, scopes, [scope_all]]),
  260. %% start syn on nodes
  261. ok = syn:start(),
  262. ok = rpc:call(SlaveNode1, syn, start, []),
  263. ok = rpc:call(SlaveNode2, syn, start, []),
  264. %% add scopes
  265. ok = syn:add_node_to_scopes([scope_ab]),
  266. ok = syn:add_node_to_scopes([scope_all]),
  267. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_ab, scope_bc, scope_all]]),
  268. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_bc, scope_c]]),
  269. %% subcluster_nodes should return invalid errors
  270. {'EXIT', {{invalid_scope, custom_abcdef}, _}} = (catch syn_registry:subcluster_nodes(custom_abcdef)),
  271. %% check
  272. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
  273. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1, SlaveNode2]),
  274. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
  275. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
  276. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
  277. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_bc, [SlaveNode1]),
  278. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_c, []),
  279. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_all, [node(), SlaveNode1]),
  280. %% disconnect node 2 (node 1 can still see node 2)
  281. syn_test_suite_helper:disconnect_node(SlaveNode2),
  282. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1]),
  283. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  284. %% check
  285. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
  286. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1]),
  287. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
  288. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
  289. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
  290. %% reconnect node 2
  291. syn_test_suite_helper:connect_node(SlaveNode2),
  292. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  293. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  294. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  295. %% check
  296. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
  297. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1, SlaveNode2]),
  298. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
  299. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
  300. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
  301. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_bc, [SlaveNode1]),
  302. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_c, []),
  303. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_all, [node(), SlaveNode1]),
  304. %% crash a scope process on 2
  305. rpc:call(SlaveNode2, syn_test_suite_helper, kill_process, [syn_registry_scope_bc]),
  306. rpc:call(SlaveNode2, syn_test_suite_helper, wait_process_name_ready, [syn_registry_scope_bc]),
  307. %% check
  308. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
  309. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1, SlaveNode2]),
  310. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
  311. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
  312. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
  313. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_bc, [SlaveNode1]),
  314. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_c, []),
  315. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_all, [node(), SlaveNode1]),
  316. %% crash scopes supervisor on local
  317. syn_test_suite_helper:kill_process(syn_scopes_sup),
  318. syn_test_suite_helper:wait_process_name_ready(syn_registry_scope_ab),
  319. syn_test_suite_helper:wait_process_name_ready(syn_registry_scope_all),
  320. %% check
  321. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
  322. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1, SlaveNode2]),
  323. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
  324. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
  325. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
  326. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_bc, [SlaveNode1]),
  327. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_c, []),
  328. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_all, [node(), SlaveNode1]).
  329. three_nodes_register_unregister_and_monitor(Config) ->
  330. %% get slaves
  331. SlaveNode1 = proplists:get_value(syn_slave_1, Config),
  332. SlaveNode2 = proplists:get_value(syn_slave_2, Config),
  333. %% start syn on nodes
  334. ok = syn:start(),
  335. ok = rpc:call(SlaveNode1, syn, start, []),
  336. ok = rpc:call(SlaveNode2, syn, start, []),
  337. %% add scopes
  338. ok = syn:add_node_to_scopes([scope_ab]),
  339. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_ab, scope_bc]]),
  340. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_bc]]),
  341. %% start processes
  342. Pid = syn_test_suite_helper:start_process(),
  343. PidWithMeta = syn_test_suite_helper:start_process(),
  344. PidRemoteWithMetaOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  345. %% retrieve
  346. undefined = syn:lookup(scope_ab, "scope_a"),
  347. undefined = rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a"]),
  348. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a"])),
  349. undefined = syn:lookup(scope_ab, "scope_a_alias"),
  350. undefined = rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a_alias"]),
  351. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a_alias"])),
  352. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, {remote_scoped_bc})),
  353. undefined = rpc:call(SlaveNode1, syn, lookup, [scope_bc, {remote_scoped_bc}]),
  354. undefined = rpc:call(SlaveNode2, syn, lookup, [scope_bc, {remote_scoped_bc}]),
  355. 0 = syn:registry_count(scope_ab),
  356. 0 = syn:registry_count(scope_ab, node()),
  357. 0 = syn:registry_count(scope_ab, SlaveNode1),
  358. 0 = syn:registry_count(scope_ab, SlaveNode2),
  359. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc)),
  360. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, node())),
  361. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode1)),
  362. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode2)),
  363. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab]),
  364. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, node()]),
  365. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode1]),
  366. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode2]),
  367. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  368. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  369. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  370. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  371. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab])),
  372. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, node()])),
  373. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode1])),
  374. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode2])),
  375. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  376. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  377. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  378. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  379. %% register
  380. ok = syn:register(scope_ab, "scope_a", Pid),
  381. ok = syn:register(scope_ab, "scope_a_alias", PidWithMeta, <<"with_meta">>),
  382. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:register(scope_bc, "scope_a", Pid)),
  383. {'EXIT', {{invalid_scope, non_existent_scope}, _}} = (catch syn:register(non_existent_scope, "scope_a", Pid)),
  384. ok = rpc:call(SlaveNode2, syn, register, [scope_bc, {remote_scoped_bc}, PidRemoteWithMetaOn1, <<"with_meta 1">>]),
  385. %% errors
  386. {error, taken} = syn:register(scope_ab, "scope_a", PidWithMeta),
  387. {error, not_alive} = syn:register(scope_ab, {"pid not alive"}, list_to_pid("<0.9999.0>")),
  388. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:register(scope_bc, "scope_a_noscope", Pid)),
  389. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:unregister(scope_bc, "scope_a_noscope")),
  390. LocalNode = node(),
  391. {badrpc, {'EXIT', {{invalid_remote_scope, scope_bc, LocalNode}, _}}} = (catch rpc:call(SlaveNode1, syn, register, [scope_bc, "pid-outside", Pid])),
  392. %% retrieve
  393. syn_test_suite_helper:assert_wait(
  394. {Pid, undefined},
  395. fun() -> syn:lookup(scope_ab, "scope_a") end
  396. ),
  397. syn_test_suite_helper:assert_wait(
  398. {Pid, undefined},
  399. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a"]) end
  400. ),
  401. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a"])),
  402. syn_test_suite_helper:assert_wait(
  403. {PidWithMeta, <<"with_meta">>},
  404. fun() -> syn:lookup(scope_ab, "scope_a_alias") end
  405. ),
  406. syn_test_suite_helper:assert_wait(
  407. {PidWithMeta, <<"with_meta">>},
  408. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a_alias"]) end
  409. ),
  410. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a_alias"])),
  411. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, {remote_scoped_bc})),
  412. syn_test_suite_helper:assert_wait(
  413. {PidRemoteWithMetaOn1, <<"with_meta 1">>},
  414. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, {remote_scoped_bc}]) end
  415. ),
  416. syn_test_suite_helper:assert_wait(
  417. {PidRemoteWithMetaOn1, <<"with_meta 1">>},
  418. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, {remote_scoped_bc}]) end
  419. ),
  420. 2 = syn:registry_count(scope_ab),
  421. 2 = syn:registry_count(scope_ab, node()),
  422. 0 = syn:registry_count(scope_ab, SlaveNode1),
  423. 0 = syn:registry_count(scope_ab, SlaveNode2),
  424. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc)),
  425. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, node())),
  426. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode1)),
  427. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode2)),
  428. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab]),
  429. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, node()]),
  430. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode1]),
  431. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode2]),
  432. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  433. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  434. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  435. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  436. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab])),
  437. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, node()])),
  438. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode1])),
  439. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode2])),
  440. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  441. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  442. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  443. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  444. %% re-register to edit meta
  445. ok = syn:register(scope_ab, "scope_a_alias", PidWithMeta, <<"with_meta_updated">>),
  446. syn_test_suite_helper:assert_wait(
  447. {PidWithMeta, <<"with_meta_updated">>},
  448. fun() -> syn:lookup(scope_ab, "scope_a_alias") end
  449. ),
  450. syn_test_suite_helper:assert_wait(
  451. {PidWithMeta, <<"with_meta_updated">>},
  452. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a_alias"]) end
  453. ),
  454. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a_alias"])),
  455. %% register remote
  456. ok = syn:register(scope_ab, "ab_on_1", PidRemoteWithMetaOn1, <<"ab-on-1">>),
  457. syn_test_suite_helper:assert_wait(
  458. {PidRemoteWithMetaOn1, <<"ab-on-1">>},
  459. fun() -> syn:lookup(scope_ab, "ab_on_1") end
  460. ),
  461. %% crash scope process to ensure that monitors get recreated & data received from other nodes
  462. syn_test_suite_helper:kill_process(syn_registry_scope_ab),
  463. syn_test_suite_helper:wait_process_name_ready(syn_registry_scope_ab),
  464. %% check remote has been sync'ed back
  465. syn_test_suite_helper:assert_wait(
  466. {PidRemoteWithMetaOn1, <<"ab-on-1">>},
  467. fun() -> syn:lookup(scope_ab, "ab_on_1") end
  468. ),
  469. %% kill process
  470. syn_test_suite_helper:kill_process(Pid),
  471. syn_test_suite_helper:kill_process(PidWithMeta),
  472. %% unregister processes
  473. {error, undefined} = (catch syn:unregister(scope_ab, <<"my proc with meta">>)),
  474. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:unregister(scope_bc, <<"my proc with meta">>)),
  475. ok = rpc:call(SlaveNode1, syn, unregister, [scope_bc, {remote_scoped_bc}]),
  476. %% retrieve
  477. syn_test_suite_helper:assert_wait(
  478. undefined,
  479. fun() -> syn:lookup(scope_ab, "scope_a") end
  480. ),
  481. syn_test_suite_helper:assert_wait(
  482. undefined,
  483. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a"]) end
  484. ),
  485. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a"])),
  486. syn_test_suite_helper:assert_wait(
  487. undefined,
  488. fun() -> syn:lookup(scope_ab, "scope_a_alias") end
  489. ),
  490. syn_test_suite_helper:assert_wait(
  491. undefined,
  492. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a_alias"]) end
  493. ),
  494. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a_alias"])),
  495. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, {remote_scoped_bc})),
  496. syn_test_suite_helper:assert_wait(
  497. undefined,
  498. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, {remote_scoped_bc}]) end
  499. ),
  500. syn_test_suite_helper:assert_wait(
  501. undefined,
  502. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, {remote_scoped_bc}]) end
  503. ),
  504. 1 = syn:registry_count(scope_ab),
  505. 0 = syn:registry_count(scope_ab, node()),
  506. 1 = syn:registry_count(scope_ab, SlaveNode1),
  507. 0 = syn:registry_count(scope_ab, SlaveNode2),
  508. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc)),
  509. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, node())),
  510. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode1)),
  511. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode2)),
  512. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab]),
  513. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, node()]),
  514. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode1]),
  515. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode2]),
  516. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  517. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  518. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  519. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  520. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab])),
  521. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, node()])),
  522. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode1])),
  523. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode2])),
  524. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  525. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  526. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  527. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  528. %% errors
  529. {error, undefined} = syn:unregister(scope_ab, {invalid_name}),
  530. %% (simulate race condition)
  531. Pid1 = syn_test_suite_helper:start_process(),
  532. Pid2 = syn_test_suite_helper:start_process(),
  533. ok = syn:register(scope_ab, <<"my proc">>, Pid1),
  534. syn_test_suite_helper:assert_wait(
  535. {Pid1, undefined},
  536. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, <<"my proc">>]) end
  537. ),
  538. remove_from_local_table(scope_ab, <<"my proc">>, Pid1),
  539. add_to_local_table(scope_ab, <<"my proc">>, Pid2, undefined, 0, undefined),
  540. {error, race_condition} = rpc:call(SlaveNode1, syn, unregister, [scope_ab, <<"my proc">>]).
  541. three_nodes_register_filter_unknown_node(Config) ->
  542. %% get slaves
  543. SlaveNode1 = proplists:get_value(syn_slave_1, Config),
  544. SlaveNode2 = proplists:get_value(syn_slave_2, Config),
  545. %% start syn on 1 and 2
  546. ok = rpc:call(SlaveNode1, syn, start, []),
  547. ok = rpc:call(SlaveNode2, syn, start, []),
  548. %% add scopes
  549. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_bc]]),
  550. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_bc]]),
  551. %% send sync message from out of scope node
  552. InvalidPid = syn_test_suite_helper:start_process(),
  553. {syn_registry_scope_bc, SlaveNode1} ! {'3.0', sync_register, <<"proc-name">>, InvalidPid, undefined, os:system_time(millisecond), normal},
  554. %% check
  555. undefined = rpc:call(SlaveNode1, syn, lookup, [scope_bc, <<"proc-name">>]).
  556. three_nodes_cluster_changes(Config) ->
  557. %% get slaves
  558. SlaveNode1 = proplists:get_value(syn_slave_1, Config),
  559. SlaveNode2 = proplists:get_value(syn_slave_2, Config),
  560. %% disconnect 1 from 2
  561. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  562. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  563. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  564. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  565. %% start syn on 1 and 2, nodes don't know of each other
  566. ok = rpc:call(SlaveNode1, syn, start, []),
  567. ok = rpc:call(SlaveNode2, syn, start, []),
  568. %% add scopes
  569. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  570. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  571. %% start processes
  572. PidRemoteOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  573. PidRemoteOn2 = syn_test_suite_helper:start_process(SlaveNode2),
  574. %% register
  575. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-1", PidRemoteOn1, "meta-1"]),
  576. ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-2", PidRemoteOn2, "meta-2"]),
  577. ok = rpc:call(SlaveNode1, syn, register, [scope_bc, "BC-proc-1", PidRemoteOn1, "meta-1"]),
  578. ok = rpc:call(SlaveNode1, syn, register, [scope_bc, "BC-proc-1 alias", PidRemoteOn1, "meta-1 alias"]),
  579. %% form full cluster
  580. ok = syn:start(),
  581. ok = syn:add_node_to_scopes([scope_all]),
  582. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  583. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  584. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  585. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  586. %% retrieve
  587. syn_test_suite_helper:assert_wait(
  588. {PidRemoteOn1, "meta-1"},
  589. fun() -> syn:lookup(scope_all, "proc-1") end
  590. ),
  591. syn_test_suite_helper:assert_wait(
  592. {PidRemoteOn1, "meta-1"},
  593. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-1"]) end
  594. ),
  595. syn_test_suite_helper:assert_wait(
  596. {PidRemoteOn1, "meta-1"},
  597. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-1"]) end
  598. ),
  599. syn_test_suite_helper:assert_wait(
  600. {PidRemoteOn2, "meta-2"},
  601. fun() -> syn:lookup(scope_all, "proc-2") end
  602. ),
  603. syn_test_suite_helper:assert_wait(
  604. {PidRemoteOn2, "meta-2"},
  605. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-2"]) end
  606. ),
  607. syn_test_suite_helper:assert_wait(
  608. {PidRemoteOn2, "meta-2"},
  609. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-2"]) end
  610. ),
  611. 2 = syn:registry_count(scope_all),
  612. 0 = syn:registry_count(scope_all, node()),
  613. 1 = syn:registry_count(scope_all, SlaveNode1),
  614. 1 = syn:registry_count(scope_all, SlaveNode2),
  615. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  616. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  617. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  618. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  619. 2 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  620. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  621. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  622. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  623. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, "BC-proc-1")),
  624. syn_test_suite_helper:assert_wait(
  625. {PidRemoteOn1, "meta-1"},
  626. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1"]) end
  627. ),
  628. syn_test_suite_helper:assert_wait(
  629. {PidRemoteOn1, "meta-1"},
  630. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1"]) end
  631. ),
  632. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, "BC-proc-1 alias")),
  633. syn_test_suite_helper:assert_wait(
  634. {PidRemoteOn1, "meta-1 alias"},
  635. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
  636. ),
  637. syn_test_suite_helper:assert_wait(
  638. {PidRemoteOn1, "meta-1 alias"},
  639. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
  640. ),
  641. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc)),
  642. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, node())),
  643. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode1)),
  644. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode2)),
  645. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  646. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  647. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  648. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  649. 2 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  650. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  651. 2 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  652. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  653. %% partial netsplit (1 cannot see 2)
  654. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  655. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  656. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  657. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  658. %% retrieve
  659. syn_test_suite_helper:assert_wait(
  660. {PidRemoteOn1, "meta-1"},
  661. fun() -> syn:lookup(scope_all, "proc-1") end
  662. ),
  663. syn_test_suite_helper:assert_wait(
  664. {PidRemoteOn1, "meta-1"},
  665. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-1"]) end
  666. ),
  667. syn_test_suite_helper:assert_wait(
  668. undefined,
  669. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-1"]) end
  670. ),
  671. syn_test_suite_helper:assert_wait(
  672. {PidRemoteOn2, "meta-2"},
  673. fun() -> syn:lookup(scope_all, "proc-2") end
  674. ),
  675. syn_test_suite_helper:assert_wait(
  676. undefined,
  677. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-2"]) end
  678. ),
  679. syn_test_suite_helper:assert_wait(
  680. {PidRemoteOn2, "meta-2"},
  681. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-2"]) end
  682. ),
  683. 2 = syn:registry_count(scope_all),
  684. 0 = syn:registry_count(scope_all, node()),
  685. 1 = syn:registry_count(scope_all, SlaveNode1),
  686. 1 = syn:registry_count(scope_all, SlaveNode2),
  687. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  688. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  689. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  690. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  691. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  692. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  693. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  694. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  695. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, "BC-proc-1")),
  696. syn_test_suite_helper:assert_wait(
  697. {PidRemoteOn1, "meta-1"},
  698. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1"]) end
  699. ),
  700. syn_test_suite_helper:assert_wait(
  701. undefined,
  702. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1"]) end
  703. ),
  704. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, "BC-proc-1 alias")),
  705. syn_test_suite_helper:assert_wait(
  706. {PidRemoteOn1, "meta-1 alias"},
  707. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
  708. ),
  709. syn_test_suite_helper:assert_wait(
  710. undefined,
  711. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
  712. ),
  713. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc)),
  714. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  715. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  716. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  717. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  718. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  719. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  720. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  721. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  722. %% re-join
  723. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  724. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  725. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  726. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  727. %% retrieve
  728. syn_test_suite_helper:assert_wait(
  729. {PidRemoteOn1, "meta-1"},
  730. fun() -> syn:lookup(scope_all, "proc-1") end
  731. ),
  732. syn_test_suite_helper:assert_wait(
  733. {PidRemoteOn1, "meta-1"},
  734. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-1"]) end
  735. ),
  736. syn_test_suite_helper:assert_wait(
  737. {PidRemoteOn1, "meta-1"},
  738. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-1"]) end
  739. ),
  740. syn_test_suite_helper:assert_wait(
  741. {PidRemoteOn2, "meta-2"},
  742. fun() -> syn:lookup(scope_all, "proc-2") end
  743. ),
  744. syn_test_suite_helper:assert_wait(
  745. {PidRemoteOn2, "meta-2"},
  746. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-2"]) end
  747. ),
  748. syn_test_suite_helper:assert_wait(
  749. {PidRemoteOn2, "meta-2"},
  750. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-2"]) end
  751. ),
  752. 2 = syn:registry_count(scope_all),
  753. 0 = syn:registry_count(scope_all, node()),
  754. 1 = syn:registry_count(scope_all, SlaveNode1),
  755. 1 = syn:registry_count(scope_all, SlaveNode2),
  756. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  757. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  758. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  759. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  760. 2 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  761. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  762. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  763. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  764. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, "BC-proc-1")),
  765. syn_test_suite_helper:assert_wait(
  766. {PidRemoteOn1, "meta-1"},
  767. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1"]) end
  768. ),
  769. syn_test_suite_helper:assert_wait(
  770. {PidRemoteOn1, "meta-1"},
  771. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1"]) end
  772. ),
  773. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, "BC-proc-1 alias")),
  774. syn_test_suite_helper:assert_wait(
  775. {PidRemoteOn1, "meta-1 alias"},
  776. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
  777. ),
  778. syn_test_suite_helper:assert_wait(
  779. {PidRemoteOn1, "meta-1 alias"},
  780. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
  781. ),
  782. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc)),
  783. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  784. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  785. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  786. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  787. 2 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  788. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  789. 2 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  790. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]).
  791. three_nodes_cluster_conflicts(Config) ->
  792. %% get slaves
  793. SlaveNode1 = proplists:get_value(syn_slave_1, Config),
  794. SlaveNode2 = proplists:get_value(syn_slave_2, Config),
  795. %% start syn on nodes
  796. ok = syn:start(),
  797. ok = rpc:call(SlaveNode1, syn, start, []),
  798. ok = rpc:call(SlaveNode2, syn, start, []),
  799. %% add scopes
  800. ok = syn:add_node_to_scopes([scope_all]),
  801. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  802. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  803. %% partial netsplit (1 cannot see 2)
  804. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  805. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  806. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  807. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  808. %% start conflict processes
  809. Pid2RemoteOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  810. Pid2RemoteOn2 = syn_test_suite_helper:start_process(SlaveNode2),
  811. %% --> conflict by netsplit
  812. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit", Pid2RemoteOn1, "meta-1"]),
  813. ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit", Pid2RemoteOn2, "meta-2"]),
  814. ok = rpc:call(SlaveNode1, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped", Pid2RemoteOn1, "meta-1"]),
  815. ok = rpc:call(SlaveNode2, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped", Pid2RemoteOn2, "meta-2"]),
  816. %% re-join
  817. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  818. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  819. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  820. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  821. %% retrieve
  822. syn_test_suite_helper:assert_wait(
  823. {Pid2RemoteOn2, "meta-2"},
  824. fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit") end
  825. ),
  826. syn_test_suite_helper:assert_wait(
  827. {Pid2RemoteOn2, "meta-2"},
  828. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit"]) end
  829. ),
  830. syn_test_suite_helper:assert_wait(
  831. {Pid2RemoteOn2, "meta-2"},
  832. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit"]) end
  833. ),
  834. 1 = syn:registry_count(scope_all),
  835. 0 = syn:registry_count(scope_all, node()),
  836. 0 = syn:registry_count(scope_all, SlaveNode1),
  837. 1 = syn:registry_count(scope_all, SlaveNode2),
  838. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  839. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  840. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  841. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  842. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  843. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  844. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  845. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  846. syn_test_suite_helper:assert_wait(
  847. {Pid2RemoteOn2, "meta-2"},
  848. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped"]) end
  849. ),
  850. syn_test_suite_helper:assert_wait(
  851. {Pid2RemoteOn2, "meta-2"},
  852. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped"]) end
  853. ),
  854. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  855. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  856. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  857. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  858. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  859. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  860. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  861. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  862. %% process alive
  863. syn_test_suite_helper:assert_wait(
  864. false,
  865. fun() -> rpc:call(SlaveNode1, erlang, is_process_alive, [Pid2RemoteOn1]) end
  866. ),
  867. syn_test_suite_helper:assert_wait(
  868. true,
  869. fun() -> rpc:call(SlaveNode2, erlang, is_process_alive, [Pid2RemoteOn2]) end
  870. ),
  871. %% --> conflict by race condition
  872. PidOnMaster = syn_test_suite_helper:start_process(),
  873. PidOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  874. rpc:call(SlaveNode1, syn_registry, add_to_local_table,
  875. [default, <<"my proc">>, PidOn1, "meta-2", erlang:system_time(), undefined]
  876. ),
  877. ok = syn:register(scope_all, <<"my proc">>, PidOnMaster, "meta-1"),
  878. %% retrieve
  879. syn_test_suite_helper:assert_wait(
  880. {PidOnMaster, "meta-1"},
  881. fun() -> syn:lookup(scope_all, <<"my proc">>) end
  882. ),
  883. syn_test_suite_helper:assert_wait(
  884. {PidOnMaster, "meta-1"},
  885. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, <<"my proc">>]) end
  886. ),
  887. syn_test_suite_helper:assert_wait(
  888. {PidOnMaster, "meta-1"},
  889. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, <<"my proc">>]) end
  890. ),
  891. %% NB: we can't check for process alive here because we injected the conflicting process in the DB
  892. %% -> it's not actually monitored
  893. ok.
  894. three_nodes_custom_event_handler_reg_unreg(Config) ->
  895. %% get slaves
  896. SlaveNode1 = proplists:get_value(syn_slave_1, Config),
  897. SlaveNode2 = proplists:get_value(syn_slave_2, Config),
  898. %% add custom handler for callbacks (using ENV)
  899. rpc:call(SlaveNode2, application, set_env, [syn, event_handler, syn_test_event_handler_callbacks]),
  900. %% start syn on nodes
  901. ok = syn:start(),
  902. ok = rpc:call(SlaveNode1, syn, start, []),
  903. ok = rpc:call(SlaveNode2, syn, start, []),
  904. %% add custom handler for callbacks (using method call)
  905. syn:set_event_handler(syn_test_event_handler_callbacks),
  906. rpc:call(SlaveNode1, syn, set_event_handler, [syn_test_event_handler_callbacks]),
  907. %% add scopes
  908. ok = syn:add_node_to_scopes([scope_all]),
  909. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all]]),
  910. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_all]]),
  911. %% init
  912. LocalNode = node(),
  913. %% start process
  914. Pid = syn_test_suite_helper:start_process(),
  915. Pid2 = syn_test_suite_helper:start_process(),
  916. %% ---> on registration
  917. ok = syn:register(scope_all, "proc-handler", Pid, {recipient, self(), <<"meta">>}),
  918. %% check callbacks called
  919. syn_test_suite_helper:assert_received_messages([
  920. {on_process_registered, LocalNode, scope_all, "proc-handler", Pid, <<"meta">>, normal},
  921. {on_process_registered, SlaveNode1, scope_all, "proc-handler", Pid, <<"meta">>, normal},
  922. {on_process_registered, SlaveNode2, scope_all, "proc-handler", Pid, <<"meta">>, normal}
  923. ]),
  924. %% registration from another node
  925. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-handler-2", Pid2, {recipient, self(), <<"meta-for-2">>}]),
  926. %% check callbacks called
  927. syn_test_suite_helper:assert_received_messages([
  928. {on_process_registered, LocalNode, scope_all, "proc-handler-2", Pid2, <<"meta-for-2">>, normal},
  929. {on_process_registered, SlaveNode1, scope_all, "proc-handler-2", Pid2, <<"meta-for-2">>, normal},
  930. {on_process_registered, SlaveNode2, scope_all, "proc-handler-2", Pid2, <<"meta-for-2">>, normal}
  931. ]),
  932. %% ---> on meta update
  933. ok = syn:register(scope_all, "proc-handler", Pid, {recipient, self(), <<"new-meta">>}),
  934. %% check callbacks called
  935. syn_test_suite_helper:assert_received_messages([
  936. {on_registry_process_updated, LocalNode, scope_all, "proc-handler", Pid, <<"new-meta">>, normal},
  937. {on_registry_process_updated, SlaveNode1, scope_all, "proc-handler", Pid, <<"new-meta">>, normal},
  938. {on_registry_process_updated, SlaveNode2, scope_all, "proc-handler", Pid, <<"new-meta">>, normal}
  939. ]),
  940. %% meta update from another node
  941. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-handler-2", Pid2, {recipient, self(), <<"meta-for-2-update">>}]),
  942. %% check callbacks called
  943. syn_test_suite_helper:assert_received_messages([
  944. {on_registry_process_updated, LocalNode, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal},
  945. {on_registry_process_updated, SlaveNode1, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal},
  946. {on_registry_process_updated, SlaveNode2, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal}
  947. ]),
  948. %% ---> on unregister
  949. ok = syn:unregister(scope_all, "proc-handler"),
  950. %% check callbacks called
  951. syn_test_suite_helper:assert_received_messages([
  952. {on_process_unregistered, LocalNode, scope_all, "proc-handler", Pid, <<"new-meta">>, normal},
  953. {on_process_unregistered, SlaveNode1, scope_all, "proc-handler", Pid, <<"new-meta">>, normal},
  954. {on_process_unregistered, SlaveNode2, scope_all, "proc-handler", Pid, <<"new-meta">>, normal}
  955. ]),
  956. %% unregister from another node
  957. ok = rpc:call(SlaveNode1, syn, unregister, [scope_all, "proc-handler-2"]),
  958. %% check callbacks called
  959. syn_test_suite_helper:assert_received_messages([
  960. {on_process_unregistered, LocalNode, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal},
  961. {on_process_unregistered, SlaveNode1, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal},
  962. {on_process_unregistered, SlaveNode2, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal}
  963. ]),
  964. %% clean & check
  965. syn_test_suite_helper:kill_process(Pid),
  966. %% no messages
  967. syn_test_suite_helper:assert_empty_queue(),
  968. %% ---> after a netsplit
  969. PidRemoteOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  970. ok = syn:register(scope_all, remote_on_1, PidRemoteOn1, {recipient, self(), <<"netsplit">>}),
  971. %% check callbacks called
  972. syn_test_suite_helper:assert_received_messages([
  973. {on_process_registered, LocalNode, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, normal},
  974. {on_process_registered, SlaveNode1, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, normal},
  975. {on_process_registered, SlaveNode2, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, normal}
  976. ]),
  977. %% partial netsplit (1 cannot see 2)
  978. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  979. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  980. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  981. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  982. %% check callbacks called
  983. syn_test_suite_helper:assert_received_messages([
  984. {on_process_unregistered, SlaveNode2, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, {syn_remote_scope_node_down, scope_all, SlaveNode1}}
  985. ]),
  986. %% ---> after a re-join
  987. %% re-join
  988. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  989. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  990. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  991. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  992. %% check callbacks called
  993. syn_test_suite_helper:assert_received_messages([
  994. {on_process_registered, SlaveNode2, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, {syn_remote_scope_node_up, scope_all, SlaveNode1}}
  995. ]),
  996. %% clean
  997. syn_test_suite_helper:kill_process(PidRemoteOn1),
  998. %% check callbacks called
  999. syn_test_suite_helper:assert_received_messages([
  1000. {on_process_unregistered, LocalNode, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, killed},
  1001. {on_process_unregistered, SlaveNode1, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, killed},
  1002. {on_process_unregistered, SlaveNode2, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, killed}
  1003. ]),
  1004. %% ---> after a conflict resolution
  1005. %% partial netsplit (1 cannot see 2)
  1006. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  1007. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1008. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  1009. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  1010. %% start conflict processes
  1011. Pid2RemoteOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  1012. Pid2RemoteOn2 = syn_test_suite_helper:start_process(SlaveNode2),
  1013. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict", Pid2RemoteOn1, {recipient, self(), <<"meta-1">>}]),
  1014. ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict", Pid2RemoteOn2, {recipient, self(), <<"meta-2">>}]),
  1015. %% check callbacks called
  1016. syn_test_suite_helper:assert_received_messages([
  1017. {on_process_registered, LocalNode, scope_all, "proc-confict", Pid2RemoteOn1, <<"meta-1">>, normal},
  1018. {on_process_unregistered, LocalNode, scope_all, "proc-confict", Pid2RemoteOn1, <<"meta-1">>, normal},
  1019. {on_process_registered, LocalNode, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, normal},
  1020. {on_process_registered, SlaveNode1, scope_all, "proc-confict", Pid2RemoteOn1, <<"meta-1">>, normal},
  1021. {on_process_registered, SlaveNode2, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, normal}
  1022. ]),
  1023. %% re-join
  1024. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  1025. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1026. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  1027. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  1028. %% check callbacks called
  1029. syn_test_suite_helper:assert_received_messages([
  1030. {on_process_unregistered, SlaveNode1, scope_all, "proc-confict", Pid2RemoteOn1, <<"meta-1">>, syn_conflict_resolution},
  1031. {on_process_registered, SlaveNode1, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, syn_conflict_resolution}
  1032. ]),
  1033. %% kill
  1034. syn_test_suite_helper:kill_process(Pid2RemoteOn1),
  1035. syn_test_suite_helper:kill_process(Pid2RemoteOn2),
  1036. %% check callbacks called
  1037. syn_test_suite_helper:assert_received_messages([
  1038. {on_process_unregistered, LocalNode, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, killed},
  1039. {on_process_unregistered, SlaveNode1, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, killed},
  1040. {on_process_unregistered, SlaveNode2, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, killed}
  1041. ]),
  1042. %% ---> don't call on monitor rebuild
  1043. %% crash the scope process on local
  1044. syn_test_suite_helper:kill_process(syn_registry_scope_all),
  1045. syn_test_suite_helper:wait_process_name_ready(syn_registry_scope_all),
  1046. %% no messages
  1047. syn_test_suite_helper:assert_empty_queue(),
  1048. %% ---> call if process died during the scope process crash
  1049. TransientPid = syn_test_suite_helper:start_process(),
  1050. ok = syn:register(scope_all, "transient-pid", TransientPid, {recipient, self(), "transient-meta"}),
  1051. %% check callbacks called
  1052. syn_test_suite_helper:assert_received_messages([
  1053. {on_process_registered, LocalNode, scope_all, "transient-pid", TransientPid, "transient-meta", normal},
  1054. {on_process_registered, SlaveNode1, scope_all, "transient-pid", TransientPid, "transient-meta", normal},
  1055. {on_process_registered, SlaveNode2, scope_all, "transient-pid", TransientPid, "transient-meta", normal}
  1056. ]),
  1057. %% crash the scope process & fake a died process on local
  1058. InvalidPid = list_to_pid("<0.9999.0>"),
  1059. add_to_local_table(scope_all, "invalid-pid", InvalidPid, {recipient, self(), "invalid-meta"}, 0, undefined),
  1060. syn_test_suite_helper:kill_process(syn_registry_scope_all),
  1061. %% check callbacks called
  1062. syn_test_suite_helper:assert_received_messages([
  1063. {on_process_unregistered, LocalNode, scope_all, "invalid-pid", InvalidPid, "invalid-meta", undefined},
  1064. {on_process_unregistered, SlaveNode1, scope_all, "transient-pid", TransientPid, "transient-meta", {syn_remote_scope_node_down, scope_all, LocalNode}},
  1065. {on_process_unregistered, SlaveNode2, scope_all, "transient-pid", TransientPid, "transient-meta", {syn_remote_scope_node_down, scope_all, LocalNode}},
  1066. {on_process_registered, SlaveNode1, scope_all, "transient-pid", TransientPid, "transient-meta", {syn_remote_scope_node_up, scope_all, LocalNode}},
  1067. {on_process_registered, SlaveNode2, scope_all, "transient-pid", TransientPid, "transient-meta", {syn_remote_scope_node_up, scope_all, LocalNode}}
  1068. ]).
  1069. three_nodes_custom_event_handler_conflict_resolution(Config) ->
  1070. %% get slaves
  1071. SlaveNode1 = proplists:get_value(syn_slave_1, Config),
  1072. SlaveNode2 = proplists:get_value(syn_slave_2, Config),
  1073. %% add custom handler for resolution & scopes (using ENV)
  1074. rpc:call(SlaveNode2, application, set_env, [syn, event_handler, syn_test_event_handler_resolution]),
  1075. rpc:call(SlaveNode2, application, set_env, [syn, scopes, [scope_all, scope_bc]]),
  1076. %% start syn on nodes
  1077. ok = syn:start(),
  1078. ok = rpc:call(SlaveNode1, syn, start, []),
  1079. ok = rpc:call(SlaveNode2, syn, start, []),
  1080. %% add custom handler for resolution (using method call)
  1081. syn:set_event_handler(syn_test_event_handler_resolution),
  1082. rpc:call(SlaveNode1, syn, set_event_handler, [syn_test_event_handler_resolution]),
  1083. %% add scopes
  1084. ok = syn:add_node_to_scopes([scope_all]),
  1085. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  1086. %% current node
  1087. TestPid = self(),
  1088. LocalNode = node(),
  1089. %% partial netsplit (1 cannot see 2)
  1090. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  1091. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1092. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  1093. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  1094. %% start conflict processes
  1095. PidOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  1096. PidOn2 = syn_test_suite_helper:start_process(SlaveNode2),
  1097. %% --> conflict by netsplit
  1098. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit-custom", PidOn1, {recipient, TestPid, keepthis}]),
  1099. ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit-custom", PidOn2, {recipient, TestPid, "meta-2"}]),
  1100. ok = rpc:call(SlaveNode1, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, {recipient, TestPid, keepthis}]),
  1101. ok = rpc:call(SlaveNode2, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn2, {recipient, TestPid, "meta-2"}]),
  1102. %% check callbacks
  1103. syn_test_suite_helper:assert_received_messages([
  1104. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
  1105. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
  1106. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn2, "meta-2", normal},
  1107. {on_process_registered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
  1108. {on_process_registered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom", PidOn2, "meta-2", normal},
  1109. {on_process_registered, SlaveNode1, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, keepthis, normal},
  1110. {on_process_registered, SlaveNode2, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn2, "meta-2", normal}
  1111. ]),
  1112. %% re-join
  1113. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  1114. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1115. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  1116. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  1117. %% retrieve
  1118. syn_test_suite_helper:assert_wait(
  1119. {PidOn1, {recipient, TestPid, keepthis}},
  1120. fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit-custom") end
  1121. ),
  1122. syn_test_suite_helper:assert_wait(
  1123. {PidOn1, {recipient, TestPid, keepthis}},
  1124. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom"]) end
  1125. ),
  1126. syn_test_suite_helper:assert_wait(
  1127. {PidOn1, {recipient, TestPid, keepthis}},
  1128. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom"]) end
  1129. ),
  1130. 1 = syn:registry_count(scope_all),
  1131. 0 = syn:registry_count(scope_all, node()),
  1132. 1 = syn:registry_count(scope_all, SlaveNode1),
  1133. 0 = syn:registry_count(scope_all, SlaveNode2),
  1134. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  1135. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  1136. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  1137. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  1138. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  1139. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  1140. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  1141. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  1142. syn_test_suite_helper:assert_wait(
  1143. {PidOn1, {recipient, TestPid, keepthis}},
  1144. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]) end
  1145. ),
  1146. syn_test_suite_helper:assert_wait(
  1147. {PidOn1, {recipient, TestPid, keepthis}},
  1148. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]) end
  1149. ),
  1150. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  1151. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  1152. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  1153. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  1154. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  1155. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  1156. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  1157. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  1158. syn_test_suite_helper:assert_received_messages([
  1159. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn2, "meta-2", syn_conflict_resolution},
  1160. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, syn_conflict_resolution},
  1161. {on_process_unregistered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom", PidOn2, "meta-2", syn_conflict_resolution},
  1162. {on_process_registered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, syn_conflict_resolution},
  1163. {on_process_unregistered, SlaveNode2, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn2, "meta-2", syn_conflict_resolution},
  1164. {on_process_registered, SlaveNode2, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, keepthis, syn_conflict_resolution}
  1165. ]),
  1166. %% process alive (discarded process does not get killed with a custom handler)
  1167. syn_test_suite_helper:assert_wait(
  1168. true,
  1169. fun() -> rpc:call(SlaveNode1, erlang, is_process_alive, [PidOn1]) end
  1170. ),
  1171. syn_test_suite_helper:assert_wait(
  1172. true,
  1173. fun() -> rpc:call(SlaveNode2, erlang, is_process_alive, [PidOn2]) end
  1174. ),
  1175. %% clean up default scope
  1176. syn:unregister(scope_all, "proc-confict-by-netsplit-custom"),
  1177. ok = rpc:call(SlaveNode1, syn, unregister, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]),
  1178. %% retrieve
  1179. syn_test_suite_helper:assert_wait(
  1180. undefined,
  1181. fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit-custom") end
  1182. ),
  1183. syn_test_suite_helper:assert_wait(
  1184. undefined,
  1185. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom"]) end
  1186. ),
  1187. syn_test_suite_helper:assert_wait(
  1188. undefined,
  1189. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom"]) end
  1190. ),
  1191. syn_test_suite_helper:assert_wait(
  1192. undefined,
  1193. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]) end
  1194. ),
  1195. syn_test_suite_helper:assert_wait(
  1196. undefined,
  1197. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]) end
  1198. ),
  1199. %% check callbacks
  1200. syn_test_suite_helper:assert_received_messages([
  1201. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
  1202. {on_process_unregistered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
  1203. {on_process_unregistered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
  1204. {on_process_unregistered, SlaveNode1, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, keepthis, normal},
  1205. {on_process_unregistered, SlaveNode2, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, keepthis, normal}
  1206. ]),
  1207. %% --> conflict by netsplit, which returns invalid pid
  1208. %% partial netsplit (1 cannot see 2)
  1209. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  1210. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1211. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  1212. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  1213. %% register with meta with no 'keepthis' element
  1214. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, {recipient, TestPid, "meta-1"}]),
  1215. ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, {recipient, TestPid, "meta-2"}]),
  1216. %% check callbacks
  1217. syn_test_suite_helper:assert_received_messages([
  1218. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, "meta-1", normal},
  1219. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, "meta-1", normal},
  1220. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, "meta-2", normal},
  1221. {on_process_registered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, "meta-1", normal},
  1222. {on_process_registered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, "meta-2", normal}
  1223. ]),
  1224. %% re-join
  1225. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  1226. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1227. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  1228. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  1229. %% retrieve (names get freed)
  1230. syn_test_suite_helper:assert_wait(
  1231. undefined,
  1232. fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit-custom-other-pid") end
  1233. ),
  1234. syn_test_suite_helper:assert_wait(
  1235. undefined,
  1236. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom-other-pid"]) end
  1237. ),
  1238. syn_test_suite_helper:assert_wait(
  1239. undefined,
  1240. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom-other-pid"]) end
  1241. ),
  1242. 0 = syn:registry_count(scope_all),
  1243. 0 = syn:registry_count(scope_all, node()),
  1244. 0 = syn:registry_count(scope_all, SlaveNode1),
  1245. 0 = syn:registry_count(scope_all, SlaveNode2),
  1246. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  1247. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  1248. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  1249. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  1250. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  1251. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  1252. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  1253. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  1254. %% check callbacks
  1255. syn_test_suite_helper:assert_received_messages([
  1256. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, "meta-2", syn_conflict_resolution},
  1257. {on_process_unregistered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, "meta-1", syn_conflict_resolution},
  1258. {on_process_unregistered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, "meta-2", syn_conflict_resolution}
  1259. ]),
  1260. %% process alive (discarded process does not get killed with a custom handler)
  1261. syn_test_suite_helper:assert_wait(
  1262. true,
  1263. fun() -> rpc:call(SlaveNode1, erlang, is_process_alive, [PidOn1]) end
  1264. ),
  1265. syn_test_suite_helper:assert_wait(
  1266. true,
  1267. fun() -> rpc:call(SlaveNode2, erlang, is_process_alive, [PidOn2]) end
  1268. ),
  1269. %% --> conflict by netsplit, which crashes
  1270. %% partial netsplit (1 cannot see 2)
  1271. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  1272. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1273. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  1274. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  1275. %% register with meta with no 'crash' element
  1276. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, {recipient, TestPid, crash}]),
  1277. ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, {recipient, TestPid, crash}]),
  1278. %% check callbacks
  1279. syn_test_suite_helper:assert_received_messages([
  1280. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, crash, normal},
  1281. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, crash, normal},
  1282. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, crash, normal},
  1283. {on_process_registered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, crash, normal},
  1284. {on_process_registered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, crash, normal}
  1285. ]),
  1286. %% re-join
  1287. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  1288. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1289. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  1290. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  1291. %% retrieve (names get freed)
  1292. syn_test_suite_helper:assert_wait(
  1293. undefined,
  1294. fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit-custom-crash") end
  1295. ),
  1296. syn_test_suite_helper:assert_wait(
  1297. undefined,
  1298. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom-crash"]) end
  1299. ),
  1300. syn_test_suite_helper:assert_wait(
  1301. undefined,
  1302. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom-crash"]) end
  1303. ),
  1304. 0 = syn:registry_count(scope_all),
  1305. 0 = syn:registry_count(scope_all, node()),
  1306. 0 = syn:registry_count(scope_all, SlaveNode1),
  1307. 0 = syn:registry_count(scope_all, SlaveNode2),
  1308. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  1309. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  1310. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  1311. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  1312. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  1313. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  1314. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  1315. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  1316. %% check callbacks
  1317. syn_test_suite_helper:assert_received_messages([
  1318. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, crash, syn_conflict_resolution},
  1319. {on_process_unregistered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, crash, syn_conflict_resolution},
  1320. {on_process_unregistered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, crash, syn_conflict_resolution}
  1321. ]),
  1322. %% process alive (discarded process does not get killed with a custom handler)
  1323. syn_test_suite_helper:assert_wait(
  1324. true,
  1325. fun() -> rpc:call(SlaveNode1, erlang, is_process_alive, [PidOn1]) end
  1326. ),
  1327. syn_test_suite_helper:assert_wait(
  1328. true,
  1329. fun() -> rpc:call(SlaveNode2, erlang, is_process_alive, [PidOn2]) end
  1330. ).
  1331. three_nodes_update(Config) ->
  1332. %% get slaves
  1333. SlaveNode1 = proplists:get_value(syn_slave_1, Config),
  1334. SlaveNode2 = proplists:get_value(syn_slave_2, Config),
  1335. %% start syn
  1336. ok = syn:start(),
  1337. ok = rpc:call(SlaveNode1, syn, start, []),
  1338. ok = rpc:call(SlaveNode2, syn, start, []),
  1339. %% add scopes
  1340. ok = syn:add_node_to_scopes([scope_all]),
  1341. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  1342. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  1343. %% start processes
  1344. Pid = syn_test_suite_helper:start_process(),
  1345. PidOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  1346. %% init
  1347. TestPid = self(),
  1348. LocalNode = node(),
  1349. %% register
  1350. ok = syn:register(scope_all, "my-proc", Pid, {recipient, TestPid, 10}),
  1351. %% add custom handler for resolution (using method call)
  1352. syn:set_event_handler(syn_test_event_handler_callbacks),
  1353. rpc:call(SlaveNode1, syn, set_event_handler, [syn_test_event_handler_callbacks]),
  1354. rpc:call(SlaveNode2, syn, set_event_handler, [syn_test_event_handler_callbacks]),
  1355. %% errors
  1356. {error, undefined} = syn:update_registry(scope_all, "unknown", fun(_IPid, ExistingMeta) -> ExistingMeta end),
  1357. {error, {update_fun, {badarith, _}}} = syn:update_registry(scope_all, "my-proc", fun(_IPid, _IMeta) -> 1/0 end),
  1358. %% retrieve
  1359. syn_test_suite_helper:assert_wait(
  1360. {Pid, {recipient, TestPid, 10}},
  1361. fun() -> syn:lookup(scope_all, "my-proc") end
  1362. ),
  1363. syn_test_suite_helper:assert_wait(
  1364. {Pid, {recipient, TestPid, 10}},
  1365. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "my-proc"]) end
  1366. ),
  1367. syn_test_suite_helper:assert_wait(
  1368. {Pid, {recipient, TestPid, 10}},
  1369. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "my-proc"]) end
  1370. ),
  1371. %% update
  1372. {ok, {Pid, {recipient, TestPid, 20}}} = syn:update_registry(scope_all, "my-proc", fun(IPid, {recipient, TestPid0, Count}) ->
  1373. IPid = Pid,
  1374. {recipient, TestPid0, Count * 2}
  1375. end),
  1376. %% retrieve
  1377. syn_test_suite_helper:assert_wait(
  1378. {Pid, {recipient, TestPid, 20}},
  1379. fun() -> syn:lookup(scope_all, "my-proc") end
  1380. ),
  1381. syn_test_suite_helper:assert_wait(
  1382. {Pid, {recipient, TestPid, 20}},
  1383. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "my-proc"]) end
  1384. ),
  1385. syn_test_suite_helper:assert_wait(
  1386. {Pid, {recipient, TestPid, 20}},
  1387. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "my-proc"]) end
  1388. ),
  1389. %% check callbacks called
  1390. syn_test_suite_helper:assert_received_messages([
  1391. {on_registry_process_updated, LocalNode, scope_all, "my-proc", Pid, 20, normal},
  1392. {on_registry_process_updated, SlaveNode1, scope_all, "my-proc", Pid, 20, normal},
  1393. {on_registry_process_updated, SlaveNode2, scope_all, "my-proc", Pid, 20, normal}
  1394. ]),
  1395. %% register on remote
  1396. ok = syn:register(scope_all, "my-proc-on-1", PidOn1, {recipient, TestPid, 1000}),
  1397. %% check callbacks called
  1398. syn_test_suite_helper:assert_received_messages([
  1399. {on_process_registered, LocalNode, scope_all, "my-proc-on-1", PidOn1, 1000, normal},
  1400. {on_process_registered, SlaveNode1, scope_all, "my-proc-on-1", PidOn1, 1000, normal},
  1401. {on_process_registered, SlaveNode2, scope_all, "my-proc-on-1", PidOn1, 1000, normal}
  1402. ]),
  1403. %% update on remote
  1404. {ok, {PidOn1, {recipient, TestPid, 1001}}} = syn:update_registry(scope_all, "my-proc-on-1", fun(_IPid, {recipient, TestPid0, Count}) ->
  1405. {recipient, TestPid0, Count + 1}
  1406. end),
  1407. %% retrieve
  1408. syn_test_suite_helper:assert_wait(
  1409. {PidOn1, {recipient, TestPid, 1001}},
  1410. fun() -> syn:lookup(scope_all, "my-proc-on-1") end
  1411. ),
  1412. syn_test_suite_helper:assert_wait(
  1413. {PidOn1, {recipient, TestPid, 1001}},
  1414. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "my-proc-on-1"]) end
  1415. ),
  1416. syn_test_suite_helper:assert_wait(
  1417. {PidOn1, {recipient, TestPid, 1001}},
  1418. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "my-proc-on-1"]) end
  1419. ),
  1420. %% check callbacks called
  1421. syn_test_suite_helper:assert_received_messages([
  1422. {on_registry_process_updated, LocalNode, scope_all, "my-proc-on-1", PidOn1, 1001, normal},
  1423. {on_registry_process_updated, SlaveNode1, scope_all, "my-proc-on-1", PidOn1, 1001, normal},
  1424. {on_registry_process_updated, SlaveNode2, scope_all, "my-proc-on-1", PidOn1, 1001, normal}
  1425. ]).
  1426. four_nodes_concurrency(Config) ->
  1427. %% get slaves
  1428. SlaveNode1 = proplists:get_value(syn_slave_1, Config),
  1429. SlaveNode2 = proplists:get_value(syn_slave_2, Config),
  1430. SlaveNode3 = proplists:get_value(syn_slave_3, Config),
  1431. %% start syn on nodes
  1432. ok = syn:start(),
  1433. ok = rpc:call(SlaveNode1, syn, start, []),
  1434. ok = rpc:call(SlaveNode2, syn, start, []),
  1435. ok = rpc:call(SlaveNode3, syn, start, []),
  1436. %% add scopes
  1437. ok = syn:add_node_to_scopes([scope_all]),
  1438. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all]]),
  1439. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_all]]),
  1440. ok = rpc:call(SlaveNode3, syn, add_node_to_scopes, [[scope_all]]),
  1441. %% ref
  1442. TestPid = self(),
  1443. Iterations = 250,
  1444. %% concurrent test
  1445. WorkerFun = fun() ->
  1446. lists:foreach(fun(_) ->
  1447. %% start pid
  1448. Pid = syn_test_suite_helper:start_process(),
  1449. RandomMeta = rand:uniform(99999),
  1450. %% loop
  1451. case syn:register(scope_all, <<"concurrent">>, Pid, RandomMeta) of
  1452. ok ->
  1453. ok;
  1454. {error, taken} ->
  1455. case syn:unregister(scope_all, <<"concurrent">>) of
  1456. {error, undefined} ->
  1457. ok;
  1458. {error, race_condition} ->
  1459. ok;
  1460. ok ->
  1461. syn:register(scope_all, <<"concurrent">>, Pid, RandomMeta)
  1462. end
  1463. end,
  1464. %% random kill
  1465. case rand:uniform(10) of
  1466. 1 -> exit(Pid, kill);
  1467. _ -> ok
  1468. end,
  1469. %% random sleep
  1470. RndTime = rand:uniform(30),
  1471. timer:sleep(RndTime)
  1472. end, lists:seq(1, Iterations)),
  1473. TestPid ! {done, node()}
  1474. end,
  1475. %% spawn concurrent
  1476. LocalNode = node(),
  1477. spawn(LocalNode, WorkerFun),
  1478. spawn(SlaveNode1, WorkerFun),
  1479. spawn(SlaveNode2, WorkerFun),
  1480. spawn(SlaveNode3, WorkerFun),
  1481. %% wait for workers done
  1482. syn_test_suite_helper:assert_received_messages([
  1483. {done, LocalNode},
  1484. {done, SlaveNode1},
  1485. {done, SlaveNode2},
  1486. {done, SlaveNode3}
  1487. ]),
  1488. %% check results are same across network
  1489. syn_test_suite_helper:assert_wait(
  1490. 1,
  1491. fun() ->
  1492. ResultPidLocal = syn:lookup(scope_all, <<"concurrent">>),
  1493. ResultPidOn1 = rpc:call(SlaveNode1, syn, lookup, [scope_all, <<"concurrent">>]),
  1494. ResultPidOn2 = rpc:call(SlaveNode2, syn, lookup, [scope_all, <<"concurrent">>]),
  1495. ResultPidOn3 = rpc:call(SlaveNode3, syn, lookup, [scope_all, <<"concurrent">>]),
  1496. %% if unique set is of 1 element then they all contain the same result
  1497. Ordset = ordsets:from_list([ResultPidLocal, ResultPidOn1, ResultPidOn2, ResultPidOn3]),
  1498. ordsets:size(Ordset)
  1499. end
  1500. ).
  1501. %% ===================================================================
  1502. %% Internal
  1503. %% ===================================================================
  1504. add_to_local_table(Scope, Name, Pid, Meta, Time, MRef) ->
  1505. TableByName = syn_backbone:get_table_name(syn_registry_by_name, Scope),
  1506. TableByPid = syn_backbone:get_table_name(syn_registry_by_pid, Scope),
  1507. syn_registry:add_to_local_table(Name, Pid, Meta, Time, MRef, TableByName, TableByPid).
  1508. remove_from_local_table(Scope, Name, Pid) ->
  1509. TableByName = syn_backbone:get_table_name(syn_registry_by_name, Scope),
  1510. TableByPid = syn_backbone:get_table_name(syn_registry_by_pid, Scope),
  1511. syn_registry:remove_from_local_table(Name, Pid, TableByName, TableByPid).