syn_registry_SUITE.erl 81 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712
  1. %% ==========================================================================================================
  2. %% Syn - A global Process Registry and Process Group manager.
  3. %%
  4. %% The MIT License (MIT)
  5. %%
  6. %% Copyright (c) 2015-2021 Roberto Ostinelli <roberto@ostinelli.net> and Neato Robotics, Inc.
  7. %%
  8. %% Permission is hereby granted, free of charge, to any person obtaining a copy
  9. %% of this software and associated documentation files (the "Software"), to deal
  10. %% in the Software without restriction, including without limitation the rights
  11. %% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  12. %% copies of the Software, and to permit persons to whom the Software is
  13. %% furnished to do so, subject to the following conditions:
  14. %%
  15. %% The above copyright notice and this permission notice shall be included in
  16. %% all copies or substantial portions of the Software.
  17. %%
  18. %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19. %% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20. %% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21. %% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22. %% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  23. %% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  24. %% THE SOFTWARE.
  25. %% ==========================================================================================================
  26. -module(syn_registry_SUITE).
  27. %% callbacks
  28. -export([all/0]).
  29. -export([init_per_suite/1, end_per_suite/1]).
  30. -export([groups/0, init_per_group/2, end_per_group/2]).
  31. -export([init_per_testcase/2, end_per_testcase/2]).
  32. %% tests
  33. -export([
  34. one_node_via_register_unregister/1,
  35. one_node_via_register_unregister_with_metadata/1,
  36. one_node_strict_mode/1
  37. ]).
  38. -export([
  39. three_nodes_discover/1,
  40. three_nodes_register_unregister_and_monitor/1,
  41. three_nodes_register_filter_unknown_node/1,
  42. three_nodes_cluster_changes/1,
  43. three_nodes_cluster_conflicts/1,
  44. three_nodes_custom_event_handler_reg_unreg/1,
  45. three_nodes_custom_event_handler_conflict_resolution/1,
  46. three_nodes_update/1
  47. ]).
  48. -export([
  49. four_nodes_concurrency/1
  50. ]).
  51. %% include
  52. -include_lib("common_test/include/ct.hrl").
  53. -include_lib("syn/src/syn.hrl").
  54. %% ===================================================================
  55. %% Callbacks
  56. %% ===================================================================
  57. %% -------------------------------------------------------------------
  58. %% Function: all() -> GroupsAndTestCases | {skip,Reason}
  59. %% GroupsAndTestCases = [{group,GroupName} | TestCase]
  60. %% GroupName = atom()
  61. %% TestCase = atom()
  62. %% Reason = any()
  63. %% -------------------------------------------------------------------
  64. all() ->
  65. [
  66. {group, one_node_registry},
  67. {group, three_nodes_registry},
  68. {group, four_nodes_registry}
  69. ].
  70. %% -------------------------------------------------------------------
  71. %% Function: groups() -> [Group]
  72. %% Group = {GroupName,Properties,GroupsAndTestCases}
  73. %% GroupName = atom()
  74. %% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
  75. %% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
  76. %% TestCase = atom()
  77. %% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
  78. %% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
  79. %% repeat_until_any_ok | repeat_until_any_fail
  80. %% N = integer() | forever
  81. %% -------------------------------------------------------------------
  82. groups() ->
  83. [
  84. {one_node_registry, [shuffle], [
  85. one_node_via_register_unregister,
  86. one_node_via_register_unregister_with_metadata,
  87. one_node_strict_mode
  88. ]},
  89. {three_nodes_registry, [shuffle], [
  90. three_nodes_discover,
  91. three_nodes_register_unregister_and_monitor,
  92. three_nodes_register_filter_unknown_node,
  93. three_nodes_cluster_changes,
  94. three_nodes_cluster_conflicts,
  95. three_nodes_custom_event_handler_reg_unreg,
  96. three_nodes_custom_event_handler_conflict_resolution,
  97. three_nodes_update
  98. ]},
  99. {four_nodes_registry, [shuffle], [
  100. four_nodes_concurrency
  101. ]}
  102. ].
  103. %% -------------------------------------------------------------------
  104. %% Function: init_per_suite(Config0) ->
  105. %% Config1 | {skip,Reason} |
  106. %% {skip_and_save,Reason,Config1}
  107. %% Config0 = Config1 = [tuple()]
  108. %% Reason = any()
  109. %% -------------------------------------------------------------------
  110. init_per_suite(Config) ->
  111. Config.
  112. %% -------------------------------------------------------------------
  113. %% Function: end_per_suite(Config0) -> void() | {save_config,Config1}
  114. %% Config0 = Config1 = [tuple()]
  115. %% -------------------------------------------------------------------
  116. end_per_suite(_Config) ->
  117. ok.
  118. %% -------------------------------------------------------------------
  119. %% Function: init_per_group(GroupName, Config0) ->
  120. %% Config1 | {skip,Reason} |
  121. %% {skip_and_save,Reason,Config1}
  122. %% GroupName = atom()
  123. %% Config0 = Config1 = [tuple()]
  124. %% Reason = any()
  125. %% -------------------------------------------------------------------
  126. init_per_group(three_nodes_registry, Config) ->
  127. case syn_test_suite_helper:init_cluster(3) of
  128. {error_initializing_cluster, Other} ->
  129. end_per_group(three_nodes_registry, Config),
  130. {skip, Other};
  131. NodesConfig ->
  132. NodesConfig ++ Config
  133. end;
  134. init_per_group(four_nodes_registry, Config) ->
  135. case syn_test_suite_helper:init_cluster(4) of
  136. {error_initializing_cluster, Other} ->
  137. end_per_group(four_nodes_registry, Config),
  138. {skip, Other};
  139. NodesConfig ->
  140. NodesConfig ++ Config
  141. end;
  142. init_per_group(_GroupName, Config) ->
  143. Config.
  144. %% -------------------------------------------------------------------
  145. %% Function: end_per_group(GroupName, Config0) ->
  146. %% void() | {save_config,Config1}
  147. %% GroupName = atom()
  148. %% Config0 = Config1 = [tuple()]
  149. %% -------------------------------------------------------------------
  150. end_per_group(three_nodes_registry, Config) ->
  151. syn_test_suite_helper:end_cluster(3, Config);
  152. end_per_group(four_nodes_registry, Config) ->
  153. syn_test_suite_helper:end_cluster(4, Config);
  154. end_per_group(_GroupName, _Config) ->
  155. syn_test_suite_helper:clean_after_test().
  156. %% -------------------------------------------------------------------
  157. %% Function: init_per_testcase(TestCase, Config0) ->
  158. %% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
  159. %% TestCase = atom()
  160. %% Config0 = Config1 = [tuple()]
  161. %% Reason = any()
  162. %% -------------------------------------------------------------------
  163. init_per_testcase(TestCase, Config) ->
  164. ct:pal("Starting test: ~p", [TestCase]),
  165. Config.
  166. %% -------------------------------------------------------------------
  167. %% Function: end_per_testcase(TestCase, Config0) ->
  168. %% void() | {save_config,Config1} | {fail,Reason}
  169. %% TestCase = atom()
  170. %% Config0 = Config1 = [tuple()]
  171. %% Reason = any()
  172. %% -------------------------------------------------------------------
  173. end_per_testcase(_, _Config) ->
  174. syn_test_suite_helper:clean_after_test().
  175. %% ===================================================================
  176. %% Tests
  177. %% ===================================================================
  178. one_node_via_register_unregister(_Config) ->
  179. %% start syn
  180. ok = syn:start(),
  181. %% ---> scope
  182. syn:add_node_to_scopes([scope]),
  183. %% start gen server via syn
  184. GenServerNameCustom = {scope, <<"my proc">>},
  185. TupleCustom = {via, syn, GenServerNameCustom},
  186. {ok, PidCustom} = syn_test_gen_server:start_link(TupleCustom),
  187. %% retrieve
  188. {PidCustom, undefined} = syn:lookup(scope, <<"my proc">>),
  189. PidCustom = syn:whereis_name(GenServerNameCustom),
  190. %% call
  191. pong = syn_test_gen_server:ping(TupleCustom),
  192. %% send via syn
  193. syn:send(GenServerNameCustom, {self(), send_ping}),
  194. syn_test_suite_helper:assert_received_messages([
  195. reply_pong
  196. ]),
  197. %% stop server
  198. syn_test_gen_server:stop(TupleCustom),
  199. %% retrieve
  200. syn_test_suite_helper:assert_wait(
  201. undefined,
  202. fun() -> syn:lookup(scope, <<"my proc">>) end
  203. ),
  204. %% send via syn
  205. {badarg, {GenServerNameCustom, anything}} = (catch syn:send(GenServerNameCustom, anything)).
  206. one_node_via_register_unregister_with_metadata(_Config) ->
  207. %% start syn
  208. ok = syn:start(),
  209. syn:add_node_to_scopes([scope]),
  210. %% start gen server via syn
  211. GenServerNameCustomMeta = {scope, <<"my proc">>, my_metadata},
  212. TupleCustomMeta = {via, syn, GenServerNameCustomMeta},
  213. {ok, PidCustom} = syn_test_gen_server:start_link(TupleCustomMeta),
  214. %% retrieve
  215. {PidCustom, my_metadata} = syn:lookup(scope, <<"my proc">>),
  216. PidCustom = syn:whereis_name(GenServerNameCustomMeta),
  217. PidCustom = syn:whereis_name({scope, <<"my proc">>}),
  218. %% call
  219. pong = syn_test_gen_server:ping(TupleCustomMeta),
  220. %% send via syn
  221. syn:send(GenServerNameCustomMeta, {self(), send_ping}),
  222. syn_test_suite_helper:assert_received_messages([
  223. reply_pong
  224. ]),
  225. %% stop server
  226. syn_test_gen_server:stop(TupleCustomMeta),
  227. %% retrieve
  228. syn_test_suite_helper:assert_wait(
  229. undefined,
  230. fun() -> syn:lookup(scope, <<"my proc">>) end
  231. ),
  232. %% send via syn
  233. {badarg, {GenServerNameCustomMeta, anything}} = (catch syn:send(GenServerNameCustomMeta, anything)).
  234. one_node_strict_mode(_Config) ->
  235. %% start syn
  236. ok = syn:start(),
  237. syn:add_node_to_scopes([scope]),
  238. %% strict mode enabled
  239. application:set_env(syn, strict_mode, true),
  240. %% start process
  241. Pid = syn_test_suite_helper:start_process(),
  242. {error, not_self} = syn:register(scope, "strict-true", Pid, metadata),
  243. Self = self(),
  244. ok = syn:register(scope, "strict-true", Self, metadata),
  245. ok = syn:register(scope, "strict-true", Self, new_metadata),
  246. {Self, new_metadata} = syn:lookup(scope, "strict-true"),
  247. ok = syn:register(scope, "strict-true", Self),
  248. {Self, undefined} = syn:lookup(scope, "strict-true").
  249. three_nodes_discover(Config) ->
  250. %% get slaves
  251. SlaveNode1 = proplists:get_value(syn_slave_1, Config),
  252. SlaveNode2 = proplists:get_value(syn_slave_2, Config),
  253. %% add scopes partially with ENV
  254. ok = rpc:call(SlaveNode2, application, set_env, [syn, scopes, [scope_all]]),
  255. %% start syn on nodes
  256. ok = syn:start(),
  257. ok = rpc:call(SlaveNode1, syn, start, []),
  258. ok = rpc:call(SlaveNode2, syn, start, []),
  259. %% add scopes
  260. ok = syn:add_node_to_scopes([scope_ab]),
  261. ok = syn:add_node_to_scopes([scope_all]),
  262. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_ab, scope_bc, scope_all]]),
  263. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_bc, scope_c]]),
  264. %% subcluster_nodes should return invalid errors
  265. {'EXIT', {{invalid_scope, custom_abcdef}, _}} = (catch syn_registry:subcluster_nodes(custom_abcdef)),
  266. %% check
  267. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
  268. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1, SlaveNode2]),
  269. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
  270. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
  271. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
  272. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_bc, [SlaveNode1]),
  273. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_c, []),
  274. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_all, [node(), SlaveNode1]),
  275. %% disconnect node 2 (node 1 can still see node 2)
  276. syn_test_suite_helper:disconnect_node(SlaveNode2),
  277. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1]),
  278. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  279. %% check
  280. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
  281. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1]),
  282. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
  283. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
  284. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
  285. %% reconnect node 2
  286. syn_test_suite_helper:connect_node(SlaveNode2),
  287. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  288. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  289. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  290. %% check
  291. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
  292. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1, SlaveNode2]),
  293. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
  294. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
  295. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
  296. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_bc, [SlaveNode1]),
  297. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_c, []),
  298. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_all, [node(), SlaveNode1]),
  299. %% crash a scope process on 2
  300. rpc:call(SlaveNode2, syn_test_suite_helper, kill_process, [syn_registry_scope_bc]),
  301. rpc:call(SlaveNode2, syn_test_suite_helper, wait_process_name_ready, [syn_registry_scope_bc]),
  302. %% check
  303. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
  304. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1, SlaveNode2]),
  305. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
  306. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
  307. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
  308. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_bc, [SlaveNode1]),
  309. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_c, []),
  310. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_all, [node(), SlaveNode1]),
  311. %% crash scopes supervisor on local
  312. syn_test_suite_helper:kill_process(syn_scopes_sup),
  313. syn_test_suite_helper:wait_process_name_ready(syn_registry_scope_ab),
  314. syn_test_suite_helper:wait_process_name_ready(syn_registry_scope_all),
  315. %% check
  316. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_ab, [SlaveNode1]),
  317. syn_test_suite_helper:assert_registry_scope_subcluster(node(), scope_all, [SlaveNode1, SlaveNode2]),
  318. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_ab, [node()]),
  319. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_bc, [SlaveNode2]),
  320. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode1, scope_all, [node(), SlaveNode2]),
  321. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_bc, [SlaveNode1]),
  322. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_c, []),
  323. syn_test_suite_helper:assert_registry_scope_subcluster(SlaveNode2, scope_all, [node(), SlaveNode1]).
  324. three_nodes_register_unregister_and_monitor(Config) ->
  325. %% get slaves
  326. SlaveNode1 = proplists:get_value(syn_slave_1, Config),
  327. SlaveNode2 = proplists:get_value(syn_slave_2, Config),
  328. %% start syn on nodes
  329. ok = syn:start(),
  330. ok = rpc:call(SlaveNode1, syn, start, []),
  331. ok = rpc:call(SlaveNode2, syn, start, []),
  332. %% add scopes
  333. ok = syn:add_node_to_scopes([scope_ab]),
  334. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_ab, scope_bc]]),
  335. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_bc]]),
  336. %% start processes
  337. Pid = syn_test_suite_helper:start_process(),
  338. PidWithMeta = syn_test_suite_helper:start_process(),
  339. PidRemoteWithMetaOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  340. %% retrieve
  341. undefined = syn:lookup(scope_ab, "scope_a"),
  342. undefined = rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a"]),
  343. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a"])),
  344. undefined = syn:lookup(scope_ab, "scope_a_alias"),
  345. undefined = rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a_alias"]),
  346. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a_alias"])),
  347. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, {remote_scoped_bc})),
  348. undefined = rpc:call(SlaveNode1, syn, lookup, [scope_bc, {remote_scoped_bc}]),
  349. undefined = rpc:call(SlaveNode2, syn, lookup, [scope_bc, {remote_scoped_bc}]),
  350. 0 = syn:registry_count(scope_ab),
  351. 0 = syn:registry_count(scope_ab, node()),
  352. 0 = syn:registry_count(scope_ab, SlaveNode1),
  353. 0 = syn:registry_count(scope_ab, SlaveNode2),
  354. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc)),
  355. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, node())),
  356. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode1)),
  357. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode2)),
  358. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab]),
  359. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, node()]),
  360. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode1]),
  361. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode2]),
  362. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  363. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  364. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  365. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  366. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab])),
  367. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, node()])),
  368. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode1])),
  369. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode2])),
  370. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  371. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  372. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  373. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  374. %% register
  375. ok = syn:register(scope_ab, "scope_a", Pid),
  376. ok = syn:register(scope_ab, "scope_a_alias", PidWithMeta, <<"with_meta">>),
  377. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:register(scope_bc, "scope_a", Pid)),
  378. {'EXIT', {{invalid_scope, non_existent_scope}, _}} = (catch syn:register(non_existent_scope, "scope_a", Pid)),
  379. ok = rpc:call(SlaveNode2, syn, register, [scope_bc, {remote_scoped_bc}, PidRemoteWithMetaOn1, <<"with_meta 1">>]),
  380. %% errors
  381. {error, taken} = syn:register(scope_ab, "scope_a", PidWithMeta),
  382. {error, not_alive} = syn:register(scope_ab, {"pid not alive"}, list_to_pid("<0.9999.0>")),
  383. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:register(scope_bc, "scope_a_noscope", Pid)),
  384. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:unregister(scope_bc, "scope_a_noscope")),
  385. LocalNode = node(),
  386. {badrpc, {'EXIT', {{invalid_remote_scope, scope_bc, LocalNode}, _}}} = (catch rpc:call(SlaveNode1, syn, register, [scope_bc, "pid-outside", Pid])),
  387. %% retrieve
  388. syn_test_suite_helper:assert_wait(
  389. {Pid, undefined},
  390. fun() -> syn:lookup(scope_ab, "scope_a") end
  391. ),
  392. syn_test_suite_helper:assert_wait(
  393. {Pid, undefined},
  394. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a"]) end
  395. ),
  396. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a"])),
  397. syn_test_suite_helper:assert_wait(
  398. {PidWithMeta, <<"with_meta">>},
  399. fun() -> syn:lookup(scope_ab, "scope_a_alias") end
  400. ),
  401. syn_test_suite_helper:assert_wait(
  402. {PidWithMeta, <<"with_meta">>},
  403. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a_alias"]) end
  404. ),
  405. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a_alias"])),
  406. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, {remote_scoped_bc})),
  407. syn_test_suite_helper:assert_wait(
  408. {PidRemoteWithMetaOn1, <<"with_meta 1">>},
  409. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, {remote_scoped_bc}]) end
  410. ),
  411. syn_test_suite_helper:assert_wait(
  412. {PidRemoteWithMetaOn1, <<"with_meta 1">>},
  413. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, {remote_scoped_bc}]) end
  414. ),
  415. 2 = syn:registry_count(scope_ab),
  416. 2 = syn:registry_count(scope_ab, node()),
  417. 0 = syn:registry_count(scope_ab, SlaveNode1),
  418. 0 = syn:registry_count(scope_ab, SlaveNode2),
  419. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc)),
  420. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, node())),
  421. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode1)),
  422. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode2)),
  423. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab]),
  424. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, node()]),
  425. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode1]),
  426. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode2]),
  427. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  428. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  429. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  430. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  431. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab])),
  432. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, node()])),
  433. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode1])),
  434. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode2])),
  435. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  436. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  437. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  438. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  439. %% re-register to edit meta
  440. ok = syn:register(scope_ab, "scope_a_alias", PidWithMeta, <<"with_meta_updated">>),
  441. syn_test_suite_helper:assert_wait(
  442. {PidWithMeta, <<"with_meta_updated">>},
  443. fun() -> syn:lookup(scope_ab, "scope_a_alias") end
  444. ),
  445. syn_test_suite_helper:assert_wait(
  446. {PidWithMeta, <<"with_meta_updated">>},
  447. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a_alias"]) end
  448. ),
  449. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a_alias"])),
  450. %% register remote
  451. ok = syn:register(scope_ab, "ab_on_1", PidRemoteWithMetaOn1, <<"ab-on-1">>),
  452. syn_test_suite_helper:assert_wait(
  453. {PidRemoteWithMetaOn1, <<"ab-on-1">>},
  454. fun() -> syn:lookup(scope_ab, "ab_on_1") end
  455. ),
  456. %% crash scope process to ensure that monitors get recreated & data received from other nodes
  457. syn_test_suite_helper:kill_process(syn_registry_scope_ab),
  458. syn_test_suite_helper:wait_process_name_ready(syn_registry_scope_ab),
  459. %% check remote has been sync'ed back
  460. syn_test_suite_helper:assert_wait(
  461. {PidRemoteWithMetaOn1, <<"ab-on-1">>},
  462. fun() -> syn:lookup(scope_ab, "ab_on_1") end
  463. ),
  464. %% kill process
  465. syn_test_suite_helper:kill_process(Pid),
  466. syn_test_suite_helper:kill_process(PidWithMeta),
  467. %% unregister processes
  468. {error, undefined} = (catch syn:unregister(scope_ab, <<"my proc with meta">>)),
  469. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:unregister(scope_bc, <<"my proc with meta">>)),
  470. ok = rpc:call(SlaveNode1, syn, unregister, [scope_bc, {remote_scoped_bc}]),
  471. %% retrieve
  472. syn_test_suite_helper:assert_wait(
  473. undefined,
  474. fun() -> syn:lookup(scope_ab, "scope_a") end
  475. ),
  476. syn_test_suite_helper:assert_wait(
  477. undefined,
  478. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a"]) end
  479. ),
  480. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a"])),
  481. syn_test_suite_helper:assert_wait(
  482. undefined,
  483. fun() -> syn:lookup(scope_ab, "scope_a_alias") end
  484. ),
  485. syn_test_suite_helper:assert_wait(
  486. undefined,
  487. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, "scope_a_alias"]) end
  488. ),
  489. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, lookup, [scope_ab, "scope_a_alias"])),
  490. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, {remote_scoped_bc})),
  491. syn_test_suite_helper:assert_wait(
  492. undefined,
  493. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, {remote_scoped_bc}]) end
  494. ),
  495. syn_test_suite_helper:assert_wait(
  496. undefined,
  497. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, {remote_scoped_bc}]) end
  498. ),
  499. 1 = syn:registry_count(scope_ab),
  500. 0 = syn:registry_count(scope_ab, node()),
  501. 1 = syn:registry_count(scope_ab, SlaveNode1),
  502. 0 = syn:registry_count(scope_ab, SlaveNode2),
  503. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc)),
  504. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, node())),
  505. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode1)),
  506. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode2)),
  507. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab]),
  508. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, node()]),
  509. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode1]),
  510. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_ab, SlaveNode2]),
  511. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  512. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  513. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  514. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  515. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab])),
  516. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, node()])),
  517. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode1])),
  518. {badrpc, {'EXIT', {{invalid_scope, scope_ab}, _}}} = (catch rpc:call(SlaveNode2, syn, registry_count, [scope_ab, SlaveNode2])),
  519. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  520. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  521. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  522. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  523. %% errors
  524. {error, undefined} = syn:unregister(scope_ab, {invalid_name}),
  525. %% (simulate race condition)
  526. Pid1 = syn_test_suite_helper:start_process(),
  527. Pid2 = syn_test_suite_helper:start_process(),
  528. ok = syn:register(scope_ab, <<"my proc">>, Pid1),
  529. syn_test_suite_helper:assert_wait(
  530. {Pid1, undefined},
  531. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_ab, <<"my proc">>]) end
  532. ),
  533. remove_from_local_table(scope_ab, <<"my proc">>, Pid1),
  534. add_to_local_table(scope_ab, <<"my proc">>, Pid2, undefined, 0, undefined),
  535. {error, race_condition} = rpc:call(SlaveNode1, syn, unregister, [scope_ab, <<"my proc">>]).
  536. three_nodes_register_filter_unknown_node(Config) ->
  537. %% get slaves
  538. SlaveNode1 = proplists:get_value(syn_slave_1, Config),
  539. SlaveNode2 = proplists:get_value(syn_slave_2, Config),
  540. %% start syn on 1 and 2
  541. ok = rpc:call(SlaveNode1, syn, start, []),
  542. ok = rpc:call(SlaveNode2, syn, start, []),
  543. %% add scopes
  544. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_bc]]),
  545. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_bc]]),
  546. %% send sync message from out of scope node
  547. InvalidPid = syn_test_suite_helper:start_process(),
  548. {syn_registry_scope_bc, SlaveNode1} ! {'3.0', sync_register, <<"proc-name">>, InvalidPid, undefined, os:system_time(millisecond), normal},
  549. %% check
  550. undefined = rpc:call(SlaveNode1, syn, lookup, [scope_bc, <<"proc-name">>]).
  551. three_nodes_cluster_changes(Config) ->
  552. %% get slaves
  553. SlaveNode1 = proplists:get_value(syn_slave_1, Config),
  554. SlaveNode2 = proplists:get_value(syn_slave_2, Config),
  555. %% disconnect 1 from 2
  556. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  557. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  558. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  559. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  560. %% start syn on 1 and 2, nodes don't know of each other
  561. ok = rpc:call(SlaveNode1, syn, start, []),
  562. ok = rpc:call(SlaveNode2, syn, start, []),
  563. %% add scopes
  564. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  565. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  566. %% start processes
  567. PidRemoteOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  568. PidRemoteOn2 = syn_test_suite_helper:start_process(SlaveNode2),
  569. %% register
  570. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-1", PidRemoteOn1, "meta-1"]),
  571. ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-2", PidRemoteOn2, "meta-2"]),
  572. ok = rpc:call(SlaveNode1, syn, register, [scope_bc, "BC-proc-1", PidRemoteOn1, "meta-1"]),
  573. ok = rpc:call(SlaveNode1, syn, register, [scope_bc, "BC-proc-1 alias", PidRemoteOn1, "meta-1 alias"]),
  574. %% form full cluster
  575. ok = syn:start(),
  576. ok = syn:add_node_to_scopes([scope_all]),
  577. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  578. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  579. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  580. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  581. %% retrieve
  582. syn_test_suite_helper:assert_wait(
  583. {PidRemoteOn1, "meta-1"},
  584. fun() -> syn:lookup(scope_all, "proc-1") end
  585. ),
  586. syn_test_suite_helper:assert_wait(
  587. {PidRemoteOn1, "meta-1"},
  588. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-1"]) end
  589. ),
  590. syn_test_suite_helper:assert_wait(
  591. {PidRemoteOn1, "meta-1"},
  592. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-1"]) end
  593. ),
  594. syn_test_suite_helper:assert_wait(
  595. {PidRemoteOn2, "meta-2"},
  596. fun() -> syn:lookup(scope_all, "proc-2") end
  597. ),
  598. syn_test_suite_helper:assert_wait(
  599. {PidRemoteOn2, "meta-2"},
  600. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-2"]) end
  601. ),
  602. syn_test_suite_helper:assert_wait(
  603. {PidRemoteOn2, "meta-2"},
  604. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-2"]) end
  605. ),
  606. 2 = syn:registry_count(scope_all),
  607. 0 = syn:registry_count(scope_all, node()),
  608. 1 = syn:registry_count(scope_all, SlaveNode1),
  609. 1 = syn:registry_count(scope_all, SlaveNode2),
  610. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  611. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  612. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  613. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  614. 2 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  615. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  616. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  617. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  618. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, "BC-proc-1")),
  619. syn_test_suite_helper:assert_wait(
  620. {PidRemoteOn1, "meta-1"},
  621. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1"]) end
  622. ),
  623. syn_test_suite_helper:assert_wait(
  624. {PidRemoteOn1, "meta-1"},
  625. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1"]) end
  626. ),
  627. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, "BC-proc-1 alias")),
  628. syn_test_suite_helper:assert_wait(
  629. {PidRemoteOn1, "meta-1 alias"},
  630. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
  631. ),
  632. syn_test_suite_helper:assert_wait(
  633. {PidRemoteOn1, "meta-1 alias"},
  634. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
  635. ),
  636. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc)),
  637. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, node())),
  638. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode1)),
  639. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc, SlaveNode2)),
  640. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  641. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  642. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  643. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  644. 2 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  645. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  646. 2 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  647. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  648. %% partial netsplit (1 cannot see 2)
  649. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  650. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  651. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  652. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  653. %% retrieve
  654. syn_test_suite_helper:assert_wait(
  655. {PidRemoteOn1, "meta-1"},
  656. fun() -> syn:lookup(scope_all, "proc-1") end
  657. ),
  658. syn_test_suite_helper:assert_wait(
  659. {PidRemoteOn1, "meta-1"},
  660. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-1"]) end
  661. ),
  662. syn_test_suite_helper:assert_wait(
  663. undefined,
  664. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-1"]) end
  665. ),
  666. syn_test_suite_helper:assert_wait(
  667. {PidRemoteOn2, "meta-2"},
  668. fun() -> syn:lookup(scope_all, "proc-2") end
  669. ),
  670. syn_test_suite_helper:assert_wait(
  671. undefined,
  672. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-2"]) end
  673. ),
  674. syn_test_suite_helper:assert_wait(
  675. {PidRemoteOn2, "meta-2"},
  676. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-2"]) end
  677. ),
  678. 2 = syn:registry_count(scope_all),
  679. 0 = syn:registry_count(scope_all, node()),
  680. 1 = syn:registry_count(scope_all, SlaveNode1),
  681. 1 = syn:registry_count(scope_all, SlaveNode2),
  682. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  683. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  684. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  685. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  686. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  687. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  688. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  689. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  690. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, "BC-proc-1")),
  691. syn_test_suite_helper:assert_wait(
  692. {PidRemoteOn1, "meta-1"},
  693. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1"]) end
  694. ),
  695. syn_test_suite_helper:assert_wait(
  696. undefined,
  697. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1"]) end
  698. ),
  699. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, "BC-proc-1 alias")),
  700. syn_test_suite_helper:assert_wait(
  701. {PidRemoteOn1, "meta-1 alias"},
  702. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
  703. ),
  704. syn_test_suite_helper:assert_wait(
  705. undefined,
  706. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
  707. ),
  708. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc)),
  709. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  710. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  711. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  712. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  713. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  714. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  715. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  716. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  717. %% re-join
  718. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  719. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  720. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  721. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  722. %% retrieve
  723. syn_test_suite_helper:assert_wait(
  724. {PidRemoteOn1, "meta-1"},
  725. fun() -> syn:lookup(scope_all, "proc-1") end
  726. ),
  727. syn_test_suite_helper:assert_wait(
  728. {PidRemoteOn1, "meta-1"},
  729. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-1"]) end
  730. ),
  731. syn_test_suite_helper:assert_wait(
  732. {PidRemoteOn1, "meta-1"},
  733. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-1"]) end
  734. ),
  735. syn_test_suite_helper:assert_wait(
  736. {PidRemoteOn2, "meta-2"},
  737. fun() -> syn:lookup(scope_all, "proc-2") end
  738. ),
  739. syn_test_suite_helper:assert_wait(
  740. {PidRemoteOn2, "meta-2"},
  741. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-2"]) end
  742. ),
  743. syn_test_suite_helper:assert_wait(
  744. {PidRemoteOn2, "meta-2"},
  745. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-2"]) end
  746. ),
  747. 2 = syn:registry_count(scope_all),
  748. 0 = syn:registry_count(scope_all, node()),
  749. 1 = syn:registry_count(scope_all, SlaveNode1),
  750. 1 = syn:registry_count(scope_all, SlaveNode2),
  751. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  752. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  753. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  754. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  755. 2 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  756. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  757. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  758. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  759. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, "BC-proc-1")),
  760. syn_test_suite_helper:assert_wait(
  761. {PidRemoteOn1, "meta-1"},
  762. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1"]) end
  763. ),
  764. syn_test_suite_helper:assert_wait(
  765. {PidRemoteOn1, "meta-1"},
  766. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1"]) end
  767. ),
  768. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:lookup(scope_bc, "BC-proc-1 alias")),
  769. syn_test_suite_helper:assert_wait(
  770. {PidRemoteOn1, "meta-1 alias"},
  771. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
  772. ),
  773. syn_test_suite_helper:assert_wait(
  774. {PidRemoteOn1, "meta-1 alias"},
  775. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "BC-proc-1 alias"]) end
  776. ),
  777. {'EXIT', {{invalid_scope, scope_bc}, _}} = (catch syn:registry_count(scope_bc)),
  778. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  779. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  780. 2 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  781. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  782. 2 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  783. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  784. 2 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  785. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]).
  786. three_nodes_cluster_conflicts(Config) ->
  787. %% get slaves
  788. SlaveNode1 = proplists:get_value(syn_slave_1, Config),
  789. SlaveNode2 = proplists:get_value(syn_slave_2, Config),
  790. %% start syn on nodes
  791. ok = syn:start(),
  792. ok = rpc:call(SlaveNode1, syn, start, []),
  793. ok = rpc:call(SlaveNode2, syn, start, []),
  794. %% add scopes
  795. ok = syn:add_node_to_scopes([scope_all]),
  796. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  797. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  798. %% partial netsplit (1 cannot see 2)
  799. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  800. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  801. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  802. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  803. %% start conflict processes
  804. Pid2RemoteOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  805. Pid2RemoteOn2 = syn_test_suite_helper:start_process(SlaveNode2),
  806. %% --> conflict by netsplit
  807. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit", Pid2RemoteOn1, "meta-1"]),
  808. ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit", Pid2RemoteOn2, "meta-2"]),
  809. ok = rpc:call(SlaveNode1, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped", Pid2RemoteOn1, "meta-1"]),
  810. ok = rpc:call(SlaveNode2, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped", Pid2RemoteOn2, "meta-2"]),
  811. %% re-join
  812. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  813. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  814. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  815. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  816. %% retrieve
  817. syn_test_suite_helper:assert_wait(
  818. {Pid2RemoteOn2, "meta-2"},
  819. fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit") end
  820. ),
  821. syn_test_suite_helper:assert_wait(
  822. {Pid2RemoteOn2, "meta-2"},
  823. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit"]) end
  824. ),
  825. syn_test_suite_helper:assert_wait(
  826. {Pid2RemoteOn2, "meta-2"},
  827. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit"]) end
  828. ),
  829. 1 = syn:registry_count(scope_all),
  830. 0 = syn:registry_count(scope_all, node()),
  831. 0 = syn:registry_count(scope_all, SlaveNode1),
  832. 1 = syn:registry_count(scope_all, SlaveNode2),
  833. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  834. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  835. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  836. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  837. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  838. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  839. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  840. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  841. syn_test_suite_helper:assert_wait(
  842. {Pid2RemoteOn2, "meta-2"},
  843. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped"]) end
  844. ),
  845. syn_test_suite_helper:assert_wait(
  846. {Pid2RemoteOn2, "meta-2"},
  847. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped"]) end
  848. ),
  849. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  850. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  851. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  852. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  853. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  854. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  855. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  856. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  857. %% process alive
  858. syn_test_suite_helper:assert_wait(
  859. false,
  860. fun() -> rpc:call(SlaveNode1, erlang, is_process_alive, [Pid2RemoteOn1]) end
  861. ),
  862. syn_test_suite_helper:assert_wait(
  863. true,
  864. fun() -> rpc:call(SlaveNode2, erlang, is_process_alive, [Pid2RemoteOn2]) end
  865. ),
  866. %% --> conflict by race condition
  867. PidOnMaster = syn_test_suite_helper:start_process(),
  868. PidOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  869. rpc:call(SlaveNode1, syn_registry, add_to_local_table,
  870. [default, <<"my proc">>, PidOn1, "meta-2", erlang:system_time(), undefined]
  871. ),
  872. ok = syn:register(scope_all, <<"my proc">>, PidOnMaster, "meta-1"),
  873. %% retrieve
  874. syn_test_suite_helper:assert_wait(
  875. {PidOnMaster, "meta-1"},
  876. fun() -> syn:lookup(scope_all, <<"my proc">>) end
  877. ),
  878. syn_test_suite_helper:assert_wait(
  879. {PidOnMaster, "meta-1"},
  880. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, <<"my proc">>]) end
  881. ),
  882. syn_test_suite_helper:assert_wait(
  883. {PidOnMaster, "meta-1"},
  884. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, <<"my proc">>]) end
  885. ),
  886. %% NB: we can't check for process alive here because we injected the conflicting process in the DB
  887. %% -> it's not actually monitored
  888. ok.
  889. three_nodes_custom_event_handler_reg_unreg(Config) ->
  890. %% get slaves
  891. SlaveNode1 = proplists:get_value(syn_slave_1, Config),
  892. SlaveNode2 = proplists:get_value(syn_slave_2, Config),
  893. %% add custom handler for callbacks (using ENV)
  894. rpc:call(SlaveNode2, application, set_env, [syn, event_handler, syn_test_event_handler_callbacks]),
  895. %% start syn on nodes
  896. ok = syn:start(),
  897. ok = rpc:call(SlaveNode1, syn, start, []),
  898. ok = rpc:call(SlaveNode2, syn, start, []),
  899. %% add custom handler for callbacks (using method call)
  900. syn:set_event_handler(syn_test_event_handler_callbacks),
  901. rpc:call(SlaveNode1, syn, set_event_handler, [syn_test_event_handler_callbacks]),
  902. %% add scopes
  903. ok = syn:add_node_to_scopes([scope_all]),
  904. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all]]),
  905. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_all]]),
  906. %% init
  907. LocalNode = node(),
  908. %% start process
  909. Pid = syn_test_suite_helper:start_process(),
  910. Pid2 = syn_test_suite_helper:start_process(),
  911. %% ---> on registration
  912. ok = syn:register(scope_all, "proc-handler", Pid, {recipient, self(), <<"meta">>}),
  913. %% check callbacks called
  914. syn_test_suite_helper:assert_received_messages([
  915. {on_process_registered, LocalNode, scope_all, "proc-handler", Pid, <<"meta">>, normal},
  916. {on_process_registered, SlaveNode1, scope_all, "proc-handler", Pid, <<"meta">>, normal},
  917. {on_process_registered, SlaveNode2, scope_all, "proc-handler", Pid, <<"meta">>, normal}
  918. ]),
  919. %% registration from another node
  920. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-handler-2", Pid2, {recipient, self(), <<"meta-for-2">>}]),
  921. %% check callbacks called
  922. syn_test_suite_helper:assert_received_messages([
  923. {on_process_registered, LocalNode, scope_all, "proc-handler-2", Pid2, <<"meta-for-2">>, normal},
  924. {on_process_registered, SlaveNode1, scope_all, "proc-handler-2", Pid2, <<"meta-for-2">>, normal},
  925. {on_process_registered, SlaveNode2, scope_all, "proc-handler-2", Pid2, <<"meta-for-2">>, normal}
  926. ]),
  927. %% ---> on meta update
  928. ok = syn:register(scope_all, "proc-handler", Pid, {recipient, self(), <<"new-meta">>}),
  929. %% check callbacks called
  930. syn_test_suite_helper:assert_received_messages([
  931. {on_registry_process_updated, LocalNode, scope_all, "proc-handler", Pid, <<"new-meta">>, normal},
  932. {on_registry_process_updated, SlaveNode1, scope_all, "proc-handler", Pid, <<"new-meta">>, normal},
  933. {on_registry_process_updated, SlaveNode2, scope_all, "proc-handler", Pid, <<"new-meta">>, normal}
  934. ]),
  935. %% meta update from another node
  936. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-handler-2", Pid2, {recipient, self(), <<"meta-for-2-update">>}]),
  937. %% check callbacks called
  938. syn_test_suite_helper:assert_received_messages([
  939. {on_registry_process_updated, LocalNode, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal},
  940. {on_registry_process_updated, SlaveNode1, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal},
  941. {on_registry_process_updated, SlaveNode2, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal}
  942. ]),
  943. %% ---> on unregister
  944. ok = syn:unregister(scope_all, "proc-handler"),
  945. %% check callbacks called
  946. syn_test_suite_helper:assert_received_messages([
  947. {on_process_unregistered, LocalNode, scope_all, "proc-handler", Pid, <<"new-meta">>, normal},
  948. {on_process_unregistered, SlaveNode1, scope_all, "proc-handler", Pid, <<"new-meta">>, normal},
  949. {on_process_unregistered, SlaveNode2, scope_all, "proc-handler", Pid, <<"new-meta">>, normal}
  950. ]),
  951. %% unregister from another node
  952. ok = rpc:call(SlaveNode1, syn, unregister, [scope_all, "proc-handler-2"]),
  953. %% check callbacks called
  954. syn_test_suite_helper:assert_received_messages([
  955. {on_process_unregistered, LocalNode, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal},
  956. {on_process_unregistered, SlaveNode1, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal},
  957. {on_process_unregistered, SlaveNode2, scope_all, "proc-handler-2", Pid2, <<"meta-for-2-update">>, normal}
  958. ]),
  959. %% clean & check
  960. syn_test_suite_helper:kill_process(Pid),
  961. %% no messages
  962. syn_test_suite_helper:assert_empty_queue(),
  963. %% ---> after a netsplit
  964. PidRemoteOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  965. ok = syn:register(scope_all, remote_on_1, PidRemoteOn1, {recipient, self(), <<"netsplit">>}),
  966. %% check callbacks called
  967. syn_test_suite_helper:assert_received_messages([
  968. {on_process_registered, LocalNode, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, normal},
  969. {on_process_registered, SlaveNode1, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, normal},
  970. {on_process_registered, SlaveNode2, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, normal}
  971. ]),
  972. %% partial netsplit (1 cannot see 2)
  973. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  974. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  975. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  976. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  977. %% check callbacks called
  978. syn_test_suite_helper:assert_received_messages([
  979. {on_process_unregistered, SlaveNode2, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, {syn_remote_scope_node_down, scope_all, SlaveNode1}}
  980. ]),
  981. %% ---> after a re-join
  982. %% re-join
  983. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  984. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  985. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  986. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  987. %% check callbacks called
  988. syn_test_suite_helper:assert_received_messages([
  989. {on_process_registered, SlaveNode2, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, {syn_remote_scope_node_up, scope_all, SlaveNode1}}
  990. ]),
  991. %% clean
  992. syn_test_suite_helper:kill_process(PidRemoteOn1),
  993. %% check callbacks called
  994. syn_test_suite_helper:assert_received_messages([
  995. {on_process_unregistered, LocalNode, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, killed},
  996. {on_process_unregistered, SlaveNode1, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, killed},
  997. {on_process_unregistered, SlaveNode2, scope_all, remote_on_1, PidRemoteOn1, <<"netsplit">>, killed}
  998. ]),
  999. %% ---> after a conflict resolution
  1000. %% partial netsplit (1 cannot see 2)
  1001. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  1002. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1003. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  1004. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  1005. %% start conflict processes
  1006. Pid2RemoteOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  1007. Pid2RemoteOn2 = syn_test_suite_helper:start_process(SlaveNode2),
  1008. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict", Pid2RemoteOn1, {recipient, self(), <<"meta-1">>}]),
  1009. ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict", Pid2RemoteOn2, {recipient, self(), <<"meta-2">>}]),
  1010. %% check callbacks called
  1011. syn_test_suite_helper:assert_received_messages([
  1012. {on_process_registered, LocalNode, scope_all, "proc-confict", Pid2RemoteOn1, <<"meta-1">>, normal},
  1013. {on_process_unregistered, LocalNode, scope_all, "proc-confict", Pid2RemoteOn1, <<"meta-1">>, normal},
  1014. {on_process_registered, LocalNode, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, normal},
  1015. {on_process_registered, SlaveNode1, scope_all, "proc-confict", Pid2RemoteOn1, <<"meta-1">>, normal},
  1016. {on_process_registered, SlaveNode2, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, normal}
  1017. ]),
  1018. %% re-join
  1019. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  1020. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1021. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  1022. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  1023. %% check callbacks called
  1024. syn_test_suite_helper:assert_received_messages([
  1025. {on_process_unregistered, SlaveNode1, scope_all, "proc-confict", Pid2RemoteOn1, <<"meta-1">>, syn_conflict_resolution},
  1026. {on_process_registered, SlaveNode1, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, syn_conflict_resolution}
  1027. ]),
  1028. %% kill
  1029. syn_test_suite_helper:kill_process(Pid2RemoteOn1),
  1030. syn_test_suite_helper:kill_process(Pid2RemoteOn2),
  1031. %% check callbacks called
  1032. syn_test_suite_helper:assert_received_messages([
  1033. {on_process_unregistered, LocalNode, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, killed},
  1034. {on_process_unregistered, SlaveNode1, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, killed},
  1035. {on_process_unregistered, SlaveNode2, scope_all, "proc-confict", Pid2RemoteOn2, <<"meta-2">>, killed}
  1036. ]),
  1037. %% ---> don't call on monitor rebuild
  1038. %% crash the scope process on local
  1039. syn_test_suite_helper:kill_process(syn_registry_scope_all),
  1040. syn_test_suite_helper:wait_process_name_ready(syn_registry_scope_all),
  1041. %% no messages
  1042. syn_test_suite_helper:assert_empty_queue(),
  1043. %% ---> call if process died during the scope process crash
  1044. TransientPid = syn_test_suite_helper:start_process(),
  1045. ok = syn:register(scope_all, "transient-pid", TransientPid, {recipient, self(), "transient-meta"}),
  1046. %% check callbacks called
  1047. syn_test_suite_helper:assert_received_messages([
  1048. {on_process_registered, LocalNode, scope_all, "transient-pid", TransientPid, "transient-meta", normal},
  1049. {on_process_registered, SlaveNode1, scope_all, "transient-pid", TransientPid, "transient-meta", normal},
  1050. {on_process_registered, SlaveNode2, scope_all, "transient-pid", TransientPid, "transient-meta", normal}
  1051. ]),
  1052. %% crash the scope process & fake a died process on local
  1053. InvalidPid = list_to_pid("<0.9999.0>"),
  1054. add_to_local_table(scope_all, "invalid-pid", InvalidPid, {recipient, self(), "invalid-meta"}, 0, undefined),
  1055. syn_test_suite_helper:kill_process(syn_registry_scope_all),
  1056. %% check callbacks called
  1057. syn_test_suite_helper:assert_received_messages([
  1058. {on_process_unregistered, LocalNode, scope_all, "invalid-pid", InvalidPid, "invalid-meta", undefined},
  1059. {on_process_unregistered, SlaveNode1, scope_all, "transient-pid", TransientPid, "transient-meta", {syn_remote_scope_node_down, scope_all, LocalNode}},
  1060. {on_process_unregistered, SlaveNode2, scope_all, "transient-pid", TransientPid, "transient-meta", {syn_remote_scope_node_down, scope_all, LocalNode}},
  1061. {on_process_registered, SlaveNode1, scope_all, "transient-pid", TransientPid, "transient-meta", {syn_remote_scope_node_up, scope_all, LocalNode}},
  1062. {on_process_registered, SlaveNode2, scope_all, "transient-pid", TransientPid, "transient-meta", {syn_remote_scope_node_up, scope_all, LocalNode}}
  1063. ]).
  1064. three_nodes_custom_event_handler_conflict_resolution(Config) ->
  1065. %% get slaves
  1066. SlaveNode1 = proplists:get_value(syn_slave_1, Config),
  1067. SlaveNode2 = proplists:get_value(syn_slave_2, Config),
  1068. %% add custom handler for resolution & scopes (using ENV)
  1069. rpc:call(SlaveNode2, application, set_env, [syn, event_handler, syn_test_event_handler_resolution]),
  1070. rpc:call(SlaveNode2, application, set_env, [syn, scopes, [scope_all, scope_bc]]),
  1071. %% start syn on nodes
  1072. ok = syn:start(),
  1073. ok = rpc:call(SlaveNode1, syn, start, []),
  1074. ok = rpc:call(SlaveNode2, syn, start, []),
  1075. %% add custom handler for resolution (using method call)
  1076. syn:set_event_handler(syn_test_event_handler_resolution),
  1077. rpc:call(SlaveNode1, syn, set_event_handler, [syn_test_event_handler_resolution]),
  1078. %% add scopes
  1079. ok = syn:add_node_to_scopes([scope_all]),
  1080. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  1081. %% current node
  1082. TestPid = self(),
  1083. LocalNode = node(),
  1084. %% partial netsplit (1 cannot see 2)
  1085. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  1086. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1087. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  1088. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  1089. %% start conflict processes
  1090. PidOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  1091. PidOn2 = syn_test_suite_helper:start_process(SlaveNode2),
  1092. %% --> conflict by netsplit
  1093. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit-custom", PidOn1, {recipient, TestPid, keepthis}]),
  1094. ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit-custom", PidOn2, {recipient, TestPid, "meta-2"}]),
  1095. ok = rpc:call(SlaveNode1, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, {recipient, TestPid, keepthis}]),
  1096. ok = rpc:call(SlaveNode2, syn, register, [scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn2, {recipient, TestPid, "meta-2"}]),
  1097. %% check callbacks
  1098. syn_test_suite_helper:assert_received_messages([
  1099. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
  1100. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
  1101. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn2, "meta-2", normal},
  1102. {on_process_registered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
  1103. {on_process_registered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom", PidOn2, "meta-2", normal},
  1104. {on_process_registered, SlaveNode1, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, keepthis, normal},
  1105. {on_process_registered, SlaveNode2, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn2, "meta-2", normal}
  1106. ]),
  1107. %% re-join
  1108. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  1109. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1110. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  1111. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  1112. %% retrieve
  1113. syn_test_suite_helper:assert_wait(
  1114. {PidOn1, {recipient, TestPid, keepthis}},
  1115. fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit-custom") end
  1116. ),
  1117. syn_test_suite_helper:assert_wait(
  1118. {PidOn1, {recipient, TestPid, keepthis}},
  1119. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom"]) end
  1120. ),
  1121. syn_test_suite_helper:assert_wait(
  1122. {PidOn1, {recipient, TestPid, keepthis}},
  1123. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom"]) end
  1124. ),
  1125. 1 = syn:registry_count(scope_all),
  1126. 0 = syn:registry_count(scope_all, node()),
  1127. 1 = syn:registry_count(scope_all, SlaveNode1),
  1128. 0 = syn:registry_count(scope_all, SlaveNode2),
  1129. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  1130. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  1131. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  1132. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  1133. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  1134. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  1135. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  1136. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  1137. syn_test_suite_helper:assert_wait(
  1138. {PidOn1, {recipient, TestPid, keepthis}},
  1139. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]) end
  1140. ),
  1141. syn_test_suite_helper:assert_wait(
  1142. {PidOn1, {recipient, TestPid, keepthis}},
  1143. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]) end
  1144. ),
  1145. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc]),
  1146. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, node()]),
  1147. 1 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode1]),
  1148. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_bc, SlaveNode2]),
  1149. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc]),
  1150. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, node()]),
  1151. 1 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode1]),
  1152. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_bc, SlaveNode2]),
  1153. syn_test_suite_helper:assert_received_messages([
  1154. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn2, "meta-2", syn_conflict_resolution},
  1155. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, syn_conflict_resolution},
  1156. {on_process_unregistered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom", PidOn2, "meta-2", syn_conflict_resolution},
  1157. {on_process_registered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, syn_conflict_resolution},
  1158. {on_process_unregistered, SlaveNode2, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn2, "meta-2", syn_conflict_resolution},
  1159. {on_process_registered, SlaveNode2, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, keepthis, syn_conflict_resolution}
  1160. ]),
  1161. %% process alive (discarded process does not get killed with a custom handler)
  1162. syn_test_suite_helper:assert_wait(
  1163. true,
  1164. fun() -> rpc:call(SlaveNode1, erlang, is_process_alive, [PidOn1]) end
  1165. ),
  1166. syn_test_suite_helper:assert_wait(
  1167. true,
  1168. fun() -> rpc:call(SlaveNode2, erlang, is_process_alive, [PidOn2]) end
  1169. ),
  1170. %% clean up default scope
  1171. syn:unregister(scope_all, "proc-confict-by-netsplit-custom"),
  1172. ok = rpc:call(SlaveNode1, syn, unregister, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]),
  1173. %% retrieve
  1174. syn_test_suite_helper:assert_wait(
  1175. undefined,
  1176. fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit-custom") end
  1177. ),
  1178. syn_test_suite_helper:assert_wait(
  1179. undefined,
  1180. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom"]) end
  1181. ),
  1182. syn_test_suite_helper:assert_wait(
  1183. undefined,
  1184. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom"]) end
  1185. ),
  1186. syn_test_suite_helper:assert_wait(
  1187. undefined,
  1188. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]) end
  1189. ),
  1190. syn_test_suite_helper:assert_wait(
  1191. undefined,
  1192. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_bc, "proc-confict-by-netsplit-scoped-custom"]) end
  1193. ),
  1194. %% check callbacks
  1195. syn_test_suite_helper:assert_received_messages([
  1196. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
  1197. {on_process_unregistered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
  1198. {on_process_unregistered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom", PidOn1, keepthis, normal},
  1199. {on_process_unregistered, SlaveNode1, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, keepthis, normal},
  1200. {on_process_unregistered, SlaveNode2, scope_bc, "proc-confict-by-netsplit-scoped-custom", PidOn1, keepthis, normal}
  1201. ]),
  1202. %% --> conflict by netsplit, which returns invalid pid
  1203. %% partial netsplit (1 cannot see 2)
  1204. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  1205. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1206. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  1207. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  1208. %% register with meta with no 'keepthis' element
  1209. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, {recipient, TestPid, "meta-1"}]),
  1210. ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, {recipient, TestPid, "meta-2"}]),
  1211. %% check callbacks
  1212. syn_test_suite_helper:assert_received_messages([
  1213. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, "meta-1", normal},
  1214. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, "meta-1", normal},
  1215. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, "meta-2", normal},
  1216. {on_process_registered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, "meta-1", normal},
  1217. {on_process_registered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, "meta-2", normal}
  1218. ]),
  1219. %% re-join
  1220. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  1221. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1222. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  1223. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  1224. %% retrieve (names get freed)
  1225. syn_test_suite_helper:assert_wait(
  1226. undefined,
  1227. fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit-custom-other-pid") end
  1228. ),
  1229. syn_test_suite_helper:assert_wait(
  1230. undefined,
  1231. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom-other-pid"]) end
  1232. ),
  1233. syn_test_suite_helper:assert_wait(
  1234. undefined,
  1235. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom-other-pid"]) end
  1236. ),
  1237. 0 = syn:registry_count(scope_all),
  1238. 0 = syn:registry_count(scope_all, node()),
  1239. 0 = syn:registry_count(scope_all, SlaveNode1),
  1240. 0 = syn:registry_count(scope_all, SlaveNode2),
  1241. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  1242. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  1243. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  1244. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  1245. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  1246. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  1247. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  1248. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  1249. %% check callbacks
  1250. syn_test_suite_helper:assert_received_messages([
  1251. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, "meta-2", syn_conflict_resolution},
  1252. {on_process_unregistered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn1, "meta-1", syn_conflict_resolution},
  1253. {on_process_unregistered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-other-pid", PidOn2, "meta-2", syn_conflict_resolution}
  1254. ]),
  1255. %% process alive (discarded process does not get killed with a custom handler)
  1256. syn_test_suite_helper:assert_wait(
  1257. true,
  1258. fun() -> rpc:call(SlaveNode1, erlang, is_process_alive, [PidOn1]) end
  1259. ),
  1260. syn_test_suite_helper:assert_wait(
  1261. true,
  1262. fun() -> rpc:call(SlaveNode2, erlang, is_process_alive, [PidOn2]) end
  1263. ),
  1264. %% --> conflict by netsplit, which crashes
  1265. %% partial netsplit (1 cannot see 2)
  1266. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  1267. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1268. syn_test_suite_helper:assert_cluster(SlaveNode1, [node()]),
  1269. syn_test_suite_helper:assert_cluster(SlaveNode2, [node()]),
  1270. %% register with meta with no 'crash' element
  1271. ok = rpc:call(SlaveNode1, syn, register, [scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, {recipient, TestPid, crash}]),
  1272. ok = rpc:call(SlaveNode2, syn, register, [scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, {recipient, TestPid, crash}]),
  1273. %% check callbacks
  1274. syn_test_suite_helper:assert_received_messages([
  1275. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, crash, normal},
  1276. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, crash, normal},
  1277. {on_process_registered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, crash, normal},
  1278. {on_process_registered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, crash, normal},
  1279. {on_process_registered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, crash, normal}
  1280. ]),
  1281. %% re-join
  1282. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  1283. syn_test_suite_helper:assert_cluster(node(), [SlaveNode1, SlaveNode2]),
  1284. syn_test_suite_helper:assert_cluster(SlaveNode1, [node(), SlaveNode2]),
  1285. syn_test_suite_helper:assert_cluster(SlaveNode2, [node(), SlaveNode1]),
  1286. %% retrieve (names get freed)
  1287. syn_test_suite_helper:assert_wait(
  1288. undefined,
  1289. fun() -> syn:lookup(scope_all, "proc-confict-by-netsplit-custom-crash") end
  1290. ),
  1291. syn_test_suite_helper:assert_wait(
  1292. undefined,
  1293. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom-crash"]) end
  1294. ),
  1295. syn_test_suite_helper:assert_wait(
  1296. undefined,
  1297. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "proc-confict-by-netsplit-custom-crash"]) end
  1298. ),
  1299. 0 = syn:registry_count(scope_all),
  1300. 0 = syn:registry_count(scope_all, node()),
  1301. 0 = syn:registry_count(scope_all, SlaveNode1),
  1302. 0 = syn:registry_count(scope_all, SlaveNode2),
  1303. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all]),
  1304. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, node()]),
  1305. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode1]),
  1306. 0 = rpc:call(SlaveNode1, syn, registry_count, [scope_all, SlaveNode2]),
  1307. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all]),
  1308. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, node()]),
  1309. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode1]),
  1310. 0 = rpc:call(SlaveNode2, syn, registry_count, [scope_all, SlaveNode2]),
  1311. %% check callbacks
  1312. syn_test_suite_helper:assert_received_messages([
  1313. {on_process_unregistered, LocalNode, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, crash, syn_conflict_resolution},
  1314. {on_process_unregistered, SlaveNode1, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn1, crash, syn_conflict_resolution},
  1315. {on_process_unregistered, SlaveNode2, scope_all, "proc-confict-by-netsplit-custom-crash", PidOn2, crash, syn_conflict_resolution}
  1316. ]),
  1317. %% process alive (discarded process does not get killed with a custom handler)
  1318. syn_test_suite_helper:assert_wait(
  1319. true,
  1320. fun() -> rpc:call(SlaveNode1, erlang, is_process_alive, [PidOn1]) end
  1321. ),
  1322. syn_test_suite_helper:assert_wait(
  1323. true,
  1324. fun() -> rpc:call(SlaveNode2, erlang, is_process_alive, [PidOn2]) end
  1325. ).
  1326. three_nodes_update(Config) ->
  1327. %% get slaves
  1328. SlaveNode1 = proplists:get_value(syn_slave_1, Config),
  1329. SlaveNode2 = proplists:get_value(syn_slave_2, Config),
  1330. %% start syn
  1331. ok = syn:start(),
  1332. ok = rpc:call(SlaveNode1, syn, start, []),
  1333. ok = rpc:call(SlaveNode2, syn, start, []),
  1334. %% add scopes
  1335. ok = syn:add_node_to_scopes([scope_all]),
  1336. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  1337. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_all, scope_bc]]),
  1338. %% start processes
  1339. Pid = syn_test_suite_helper:start_process(),
  1340. PidOn1 = syn_test_suite_helper:start_process(SlaveNode1),
  1341. %% init
  1342. TestPid = self(),
  1343. LocalNode = node(),
  1344. %% register
  1345. ok = syn:register(scope_all, "my-proc", Pid, {recipient, TestPid, 10}),
  1346. %% add custom handler for resolution (using method call)
  1347. syn:set_event_handler(syn_test_event_handler_callbacks),
  1348. rpc:call(SlaveNode1, syn, set_event_handler, [syn_test_event_handler_callbacks]),
  1349. rpc:call(SlaveNode2, syn, set_event_handler, [syn_test_event_handler_callbacks]),
  1350. %% errors
  1351. {error, undefined} = syn:update_registry(scope_all, "unknown", fun(_IPid, ExistingMeta) -> ExistingMeta end),
  1352. {error, {update_fun, {badarith, _}}} = syn:update_registry(scope_all, "my-proc", fun(_IPid, _IMeta) -> 1/0 end),
  1353. %% retrieve
  1354. syn_test_suite_helper:assert_wait(
  1355. {Pid, {recipient, TestPid, 10}},
  1356. fun() -> syn:lookup(scope_all, "my-proc") end
  1357. ),
  1358. syn_test_suite_helper:assert_wait(
  1359. {Pid, {recipient, TestPid, 10}},
  1360. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "my-proc"]) end
  1361. ),
  1362. syn_test_suite_helper:assert_wait(
  1363. {Pid, {recipient, TestPid, 10}},
  1364. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "my-proc"]) end
  1365. ),
  1366. %% update
  1367. {ok, {Pid, {recipient, TestPid, 20}}} = syn:update_registry(scope_all, "my-proc", fun(IPid, {recipient, TestPid0, Count}) ->
  1368. IPid = Pid,
  1369. {recipient, TestPid0, Count * 2}
  1370. end),
  1371. %% retrieve
  1372. syn_test_suite_helper:assert_wait(
  1373. {Pid, {recipient, TestPid, 20}},
  1374. fun() -> syn:lookup(scope_all, "my-proc") end
  1375. ),
  1376. syn_test_suite_helper:assert_wait(
  1377. {Pid, {recipient, TestPid, 20}},
  1378. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "my-proc"]) end
  1379. ),
  1380. syn_test_suite_helper:assert_wait(
  1381. {Pid, {recipient, TestPid, 20}},
  1382. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "my-proc"]) end
  1383. ),
  1384. %% check callbacks called
  1385. syn_test_suite_helper:assert_received_messages([
  1386. {on_registry_process_updated, LocalNode, scope_all, "my-proc", Pid, 20, normal},
  1387. {on_registry_process_updated, SlaveNode1, scope_all, "my-proc", Pid, 20, normal},
  1388. {on_registry_process_updated, SlaveNode2, scope_all, "my-proc", Pid, 20, normal}
  1389. ]),
  1390. %% register on remote
  1391. ok = syn:register(scope_all, "my-proc-on-1", PidOn1, {recipient, TestPid, 1000}),
  1392. %% check callbacks called
  1393. syn_test_suite_helper:assert_received_messages([
  1394. {on_process_registered, LocalNode, scope_all, "my-proc-on-1", PidOn1, 1000, normal},
  1395. {on_process_registered, SlaveNode1, scope_all, "my-proc-on-1", PidOn1, 1000, normal},
  1396. {on_process_registered, SlaveNode2, scope_all, "my-proc-on-1", PidOn1, 1000, normal}
  1397. ]),
  1398. %% update on remote
  1399. {ok, {PidOn1, {recipient, TestPid, 1001}}} = syn:update_registry(scope_all, "my-proc-on-1", fun(_IPid, {recipient, TestPid0, Count}) ->
  1400. {recipient, TestPid0, Count + 1}
  1401. end),
  1402. %% retrieve
  1403. syn_test_suite_helper:assert_wait(
  1404. {PidOn1, {recipient, TestPid, 1001}},
  1405. fun() -> syn:lookup(scope_all, "my-proc-on-1") end
  1406. ),
  1407. syn_test_suite_helper:assert_wait(
  1408. {PidOn1, {recipient, TestPid, 1001}},
  1409. fun() -> rpc:call(SlaveNode1, syn, lookup, [scope_all, "my-proc-on-1"]) end
  1410. ),
  1411. syn_test_suite_helper:assert_wait(
  1412. {PidOn1, {recipient, TestPid, 1001}},
  1413. fun() -> rpc:call(SlaveNode2, syn, lookup, [scope_all, "my-proc-on-1"]) end
  1414. ),
  1415. %% check callbacks called
  1416. syn_test_suite_helper:assert_received_messages([
  1417. {on_registry_process_updated, LocalNode, scope_all, "my-proc-on-1", PidOn1, 1001, normal},
  1418. {on_registry_process_updated, SlaveNode1, scope_all, "my-proc-on-1", PidOn1, 1001, normal},
  1419. {on_registry_process_updated, SlaveNode2, scope_all, "my-proc-on-1", PidOn1, 1001, normal}
  1420. ]).
  1421. four_nodes_concurrency(Config) ->
  1422. %% get slaves
  1423. SlaveNode1 = proplists:get_value(syn_slave_1, Config),
  1424. SlaveNode2 = proplists:get_value(syn_slave_2, Config),
  1425. SlaveNode3 = proplists:get_value(syn_slave_3, Config),
  1426. %% start syn on nodes
  1427. ok = syn:start(),
  1428. ok = rpc:call(SlaveNode1, syn, start, []),
  1429. ok = rpc:call(SlaveNode2, syn, start, []),
  1430. ok = rpc:call(SlaveNode3, syn, start, []),
  1431. %% add scopes
  1432. ok = syn:add_node_to_scopes([scope_all]),
  1433. ok = rpc:call(SlaveNode1, syn, add_node_to_scopes, [[scope_all]]),
  1434. ok = rpc:call(SlaveNode2, syn, add_node_to_scopes, [[scope_all]]),
  1435. ok = rpc:call(SlaveNode3, syn, add_node_to_scopes, [[scope_all]]),
  1436. %% ref
  1437. TestPid = self(),
  1438. Iterations = 250,
  1439. %% concurrent test
  1440. WorkerFun = fun() ->
  1441. lists:foreach(fun(_) ->
  1442. %% start pid
  1443. Pid = syn_test_suite_helper:start_process(),
  1444. RandomMeta = rand:uniform(99999),
  1445. %% loop
  1446. case syn:register(scope_all, <<"concurrent">>, Pid, RandomMeta) of
  1447. ok ->
  1448. ok;
  1449. {error, taken} ->
  1450. case syn:unregister(scope_all, <<"concurrent">>) of
  1451. {error, undefined} ->
  1452. ok;
  1453. {error, race_condition} ->
  1454. ok;
  1455. ok ->
  1456. syn:register(scope_all, <<"concurrent">>, Pid, RandomMeta)
  1457. end
  1458. end,
  1459. %% random kill
  1460. case rand:uniform(10) of
  1461. 1 -> exit(Pid, kill);
  1462. _ -> ok
  1463. end,
  1464. %% random sleep
  1465. RndTime = rand:uniform(30),
  1466. timer:sleep(RndTime)
  1467. end, lists:seq(1, Iterations)),
  1468. TestPid ! {done, node()}
  1469. end,
  1470. %% spawn concurrent
  1471. LocalNode = node(),
  1472. spawn(LocalNode, WorkerFun),
  1473. spawn(SlaveNode1, WorkerFun),
  1474. spawn(SlaveNode2, WorkerFun),
  1475. spawn(SlaveNode3, WorkerFun),
  1476. %% wait for workers done
  1477. syn_test_suite_helper:assert_received_messages([
  1478. {done, LocalNode},
  1479. {done, SlaveNode1},
  1480. {done, SlaveNode2},
  1481. {done, SlaveNode3}
  1482. ]),
  1483. %% check results are same across network
  1484. syn_test_suite_helper:assert_wait(
  1485. 1,
  1486. fun() ->
  1487. ResultPidLocal = syn:lookup(scope_all, <<"concurrent">>),
  1488. ResultPidOn1 = rpc:call(SlaveNode1, syn, lookup, [scope_all, <<"concurrent">>]),
  1489. ResultPidOn2 = rpc:call(SlaveNode2, syn, lookup, [scope_all, <<"concurrent">>]),
  1490. ResultPidOn3 = rpc:call(SlaveNode3, syn, lookup, [scope_all, <<"concurrent">>]),
  1491. %% if unique set is of 1 element then they all contain the same result
  1492. Ordset = ordsets:from_list([ResultPidLocal, ResultPidOn1, ResultPidOn2, ResultPidOn3]),
  1493. ordsets:size(Ordset)
  1494. end
  1495. ).
  1496. %% ===================================================================
  1497. %% Internal
  1498. %% ===================================================================
  1499. add_to_local_table(Scope, Name, Pid, Meta, Time, MRef) ->
  1500. TableByName = syn_backbone:get_table_name(syn_registry_by_name, Scope),
  1501. TableByPid = syn_backbone:get_table_name(syn_registry_by_pid, Scope),
  1502. syn_registry:add_to_local_table(Name, Pid, Meta, Time, MRef, TableByName, TableByPid).
  1503. remove_from_local_table(Scope, Name, Pid) ->
  1504. TableByName = syn_backbone:get_table_name(syn_registry_by_name, Scope),
  1505. TableByPid = syn_backbone:get_table_name(syn_registry_by_pid, Scope),
  1506. syn_registry:remove_from_local_table(Name, Pid, TableByName, TableByPid).