syn_registry_SUITE.erl 50 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137
  1. %% ==========================================================================================================
  2. %% Syn - A global Process Registry and Process Group manager.
  3. %%
  4. %% The MIT License (MIT)
  5. %%
  6. %% Copyright (c) 2015-2019 Roberto Ostinelli <roberto@ostinelli.net> and Neato Robotics, Inc.
  7. %%
  8. %% Permission is hereby granted, free of charge, to any person obtaining a copy
  9. %% of this software and associated documentation files (the "Software"), to deal
  10. %% in the Software without restriction, including without limitation the rights
  11. %% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  12. %% copies of the Software, and to permit persons to whom the Software is
  13. %% furnished to do so, subject to the following conditions:
  14. %%
  15. %% The above copyright notice and this permission notice shall be included in
  16. %% all copies or substantial portions of the Software.
  17. %%
  18. %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19. %% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20. %% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21. %% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22. %% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  23. %% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  24. %% THE SOFTWARE.
  25. %% ==========================================================================================================
  26. -module(syn_registry_SUITE).
  27. %% callbacks
  28. -export([all/0]).
  29. -export([init_per_suite/1, end_per_suite/1]).
  30. -export([groups/0, init_per_group/2, end_per_group/2]).
  31. -export([init_per_testcase/2, end_per_testcase/2]).
  32. %% tests
  33. -export([
  34. single_node_register_and_monitor/1,
  35. single_node_register_and_unregister/1,
  36. single_node_registration_errors/1,
  37. single_node_registry_count/1,
  38. single_node_register_gen_server/1,
  39. single_node_callback_on_process_exit/1,
  40. single_node_ensure_callback_process_exit_is_called_if_process_killed/1,
  41. single_node_monitor_after_registry_crash/1
  42. ]).
  43. -export([
  44. two_nodes_register_monitor_and_unregister/1,
  45. two_nodes_registry_count/1,
  46. two_nodes_registration_race_condition_conflict_resolution_keep_local/1,
  47. two_nodes_registration_race_condition_conflict_resolution_keep_remote/1,
  48. two_nodes_registration_race_condition_conflict_resolution_when_process_died/1,
  49. two_nodes_registry_full_cluster_sync_on_boot_node_added_later/1,
  50. two_nodes_registry_full_cluster_sync_on_boot_syn_started_later/1,
  51. two_nodes_reregister/1
  52. ]).
  53. -export([
  54. three_nodes_partial_netsplit_consistency/1,
  55. three_nodes_full_netsplit_consistency/1,
  56. three_nodes_start_syn_before_connecting_cluster_with_conflict/1,
  57. three_nodes_start_syn_before_connecting_cluster_with_custom_conflict_resolution/1,
  58. three_nodes_registration_race_condition_custom_conflict_resolution/1,
  59. three_nodes_anti_entropy/1,
  60. three_nodes_anti_entropy_manual/1,
  61. three_nodes_concurrent_registration_unregistration/1
  62. ]).
  63. %% support
  64. -export([
  65. start_syn_delayed_and_register_local_process/3,
  66. start_syn_delayed_with_custom_handler_register_local_process/4,
  67. seq_unregister_register/3
  68. ]).
  69. %% include
  70. -include_lib("common_test/include/ct.hrl").
  71. -include_lib("../src/syn.hrl").
  72. %% ===================================================================
  73. %% Callbacks
  74. %% ===================================================================
  75. %% -------------------------------------------------------------------
  76. %% Function: all() -> GroupsAndTestCases | {skip,Reason}
  77. %% GroupsAndTestCases = [{group,GroupName} | TestCase]
  78. %% GroupName = atom()
  79. %% TestCase = atom()
  80. %% Reason = any()
  81. %% -------------------------------------------------------------------
  82. all() ->
  83. [
  84. {group, single_node_process_registration},
  85. {group, two_nodes_process_registration},
  86. {group, three_nodes_process_registration}
  87. ].
  88. %% -------------------------------------------------------------------
  89. %% Function: groups() -> [Group]
  90. %% Group = {GroupName,Properties,GroupsAndTestCases}
  91. %% GroupName = atom()
  92. %% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
  93. %% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
  94. %% TestCase = atom()
  95. %% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
  96. %% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
  97. %% repeat_until_any_ok | repeat_until_any_fail
  98. %% N = integer() | forever
  99. %% -------------------------------------------------------------------
  100. groups() ->
  101. [
  102. {single_node_process_registration, [shuffle], [
  103. single_node_register_and_monitor,
  104. single_node_register_and_unregister,
  105. single_node_registration_errors,
  106. single_node_registry_count,
  107. single_node_register_gen_server,
  108. single_node_callback_on_process_exit,
  109. single_node_ensure_callback_process_exit_is_called_if_process_killed,
  110. single_node_monitor_after_registry_crash
  111. ]},
  112. {two_nodes_process_registration, [shuffle], [
  113. two_nodes_register_monitor_and_unregister,
  114. two_nodes_registry_count,
  115. two_nodes_registration_race_condition_conflict_resolution_keep_local,
  116. two_nodes_registration_race_condition_conflict_resolution_keep_remote,
  117. two_nodes_registration_race_condition_conflict_resolution_when_process_died,
  118. two_nodes_registry_full_cluster_sync_on_boot_node_added_later,
  119. two_nodes_registry_full_cluster_sync_on_boot_syn_started_later,
  120. two_nodes_reregister
  121. ]},
  122. {three_nodes_process_registration, [shuffle], [
  123. three_nodes_partial_netsplit_consistency,
  124. three_nodes_full_netsplit_consistency,
  125. three_nodes_start_syn_before_connecting_cluster_with_conflict,
  126. three_nodes_start_syn_before_connecting_cluster_with_custom_conflict_resolution,
  127. three_nodes_registration_race_condition_custom_conflict_resolution,
  128. three_nodes_anti_entropy,
  129. three_nodes_anti_entropy_manual,
  130. three_nodes_concurrent_registration_unregistration
  131. ]}
  132. ].
  133. %% -------------------------------------------------------------------
  134. %% Function: init_per_suite(Config0) ->
  135. %% Config1 | {skip,Reason} |
  136. %% {skip_and_save,Reason,Config1}
  137. %% Config0 = Config1 = [tuple()]
  138. %% Reason = any()
  139. %% -------------------------------------------------------------------
  140. init_per_suite(Config) ->
  141. Config.
  142. %% -------------------------------------------------------------------
  143. %% Function: end_per_suite(Config0) -> void() | {save_config,Config1}
  144. %% Config0 = Config1 = [tuple()]
  145. %% -------------------------------------------------------------------
  146. end_per_suite(_Config) ->
  147. ok.
  148. %% -------------------------------------------------------------------
  149. %% Function: init_per_group(GroupName, Config0) ->
  150. %% Config1 | {skip,Reason} |
  151. %% {skip_and_save,Reason,Config1}
  152. %% GroupName = atom()
  153. %% Config0 = Config1 = [tuple()]
  154. %% Reason = any()
  155. %% -------------------------------------------------------------------
  156. init_per_group(two_nodes_process_registration, Config) ->
  157. %% start slave
  158. {ok, SlaveNode} = syn_test_suite_helper:start_slave(syn_slave),
  159. %% config
  160. [{slave_node, SlaveNode} | Config];
  161. init_per_group(three_nodes_process_registration, Config) ->
  162. %% start slave
  163. {ok, SlaveNode1} = syn_test_suite_helper:start_slave(syn_slave_1),
  164. {ok, SlaveNode2} = syn_test_suite_helper:start_slave(syn_slave_2),
  165. %% config
  166. [{slave_node_1, SlaveNode1}, {slave_node_2, SlaveNode2} | Config];
  167. init_per_group(_GroupName, Config) ->
  168. Config.
  169. %% -------------------------------------------------------------------
  170. %% Function: end_per_group(GroupName, Config0) ->
  171. %% void() | {save_config,Config1}
  172. %% GroupName = atom()
  173. %% Config0 = Config1 = [tuple()]
  174. %% -------------------------------------------------------------------
  175. end_per_group(two_nodes_process_registration, Config) ->
  176. SlaveNode = proplists:get_value(slave_node, Config),
  177. syn_test_suite_helper:connect_node(SlaveNode),
  178. syn_test_suite_helper:clean_after_test(),
  179. syn_test_suite_helper:stop_slave(syn_slave),
  180. timer:sleep(1000);
  181. end_per_group(three_nodes_process_registration, Config) ->
  182. SlaveNode1 = proplists:get_value(slave_node_1, Config),
  183. syn_test_suite_helper:connect_node(SlaveNode1),
  184. SlaveNode2 = proplists:get_value(slave_node_2, Config),
  185. syn_test_suite_helper:connect_node(SlaveNode2),
  186. syn_test_suite_helper:clean_after_test(),
  187. syn_test_suite_helper:stop_slave(syn_slave_1),
  188. syn_test_suite_helper:stop_slave(syn_slave_2),
  189. timer:sleep(1000);
  190. end_per_group(_GroupName, _Config) ->
  191. syn_test_suite_helper:clean_after_test().
  192. %% -------------------------------------------------------------------
  193. %% Function: init_per_testcase(TestCase, Config0) ->
  194. %% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
  195. %% TestCase = atom()
  196. %% Config0 = Config1 = [tuple()]
  197. %% Reason = any()
  198. %% -------------------------------------------------------------------
  199. init_per_testcase(_TestCase, Config) ->
  200. Config.
  201. %% -------------------------------------------------------------------
  202. %% Function: end_per_testcase(TestCase, Config0) ->
  203. %% void() | {save_config,Config1} | {fail,Reason}
  204. %% TestCase = atom()
  205. %% Config0 = Config1 = [tuple()]
  206. %% Reason = any()
  207. %% -------------------------------------------------------------------
  208. end_per_testcase(_, _Config) ->
  209. syn_test_suite_helper:clean_after_test().
  210. %% ===================================================================
  211. %% Tests
  212. %% ===================================================================
  213. single_node_register_and_monitor(_Config) ->
  214. %% start
  215. ok = syn:start(),
  216. %% start processes
  217. Pid = syn_test_suite_helper:start_process(),
  218. PidWithMeta = syn_test_suite_helper:start_process(),
  219. %% retrieve
  220. undefined = syn:whereis(<<"my proc">>),
  221. %% register
  222. ok = syn:register(<<"my proc">>, Pid),
  223. ok = syn:register({"my proc 2"}, Pid),
  224. ok = syn:register(<<"my proc with meta">>, PidWithMeta, {meta, <<"meta">>}),
  225. %% retrieve
  226. Pid = syn:whereis(<<"my proc">>),
  227. Pid = syn:whereis({"my proc 2"}),
  228. {PidWithMeta, {meta, <<"meta">>}} = syn:whereis(<<"my proc with meta">>, with_meta),
  229. %% re-register
  230. ok = syn:register(<<"my proc with meta">>, PidWithMeta, {meta2, <<"meta2">>}),
  231. {PidWithMeta, {meta2, <<"meta2">>}} = syn:whereis(<<"my proc with meta">>, with_meta),
  232. %% kill process
  233. syn_test_suite_helper:kill_process(Pid),
  234. syn_test_suite_helper:kill_process(PidWithMeta),
  235. timer:sleep(100),
  236. %% retrieve
  237. undefined = syn:whereis(<<"my proc">>),
  238. undefined = syn:whereis({"my proc 2"}),
  239. undefined = syn:whereis(<<"my proc with meta">>).
  240. single_node_register_and_unregister(_Config) ->
  241. %% start
  242. ok = syn:start(),
  243. %% start process
  244. Pid = syn_test_suite_helper:start_process(),
  245. %% retrieve
  246. undefined = syn:whereis(<<"my proc">>),
  247. %% register
  248. ok = syn:register(<<"my proc">>, Pid),
  249. ok = syn:register(<<"my proc 2">>, Pid),
  250. %% retrieve
  251. Pid = syn:whereis(<<"my proc">>),
  252. Pid = syn:whereis(<<"my proc 2">>),
  253. %% unregister 1
  254. ok = syn:unregister(<<"my proc">>),
  255. %% retrieve
  256. undefined = syn:whereis(<<"my proc">>),
  257. Pid = syn:whereis(<<"my proc 2">>),
  258. %% unregister 2
  259. ok = syn:unregister(<<"my proc 2">>),
  260. {error, undefined} = syn:unregister(<<"my proc 2">>),
  261. %% retrieve
  262. undefined = syn:whereis(<<"my proc">>),
  263. undefined = syn:whereis(<<"my proc 2">>).
  264. single_node_registration_errors(_Config) ->
  265. %% start
  266. ok = syn:start(),
  267. %% start process
  268. Pid = syn_test_suite_helper:start_process(),
  269. Pid2 = syn_test_suite_helper:start_process(),
  270. %% register
  271. ok = syn:register(<<"my proc">>, Pid),
  272. {error, taken} = syn:register(<<"my proc">>, Pid2),
  273. %% kill processes
  274. syn_test_suite_helper:kill_process(Pid),
  275. syn_test_suite_helper:kill_process(Pid2),
  276. timer:sleep(100),
  277. %% retrieve
  278. undefined = syn:whereis(<<"my proc">>),
  279. %% try registering a dead pid
  280. {error, not_alive} = syn:register(<<"my proc">>, Pid).
  281. single_node_registry_count(_Config) ->
  282. %% start
  283. ok = syn:start(),
  284. %% start process
  285. Pid = syn_test_suite_helper:start_process(),
  286. Pid2 = syn_test_suite_helper:start_process(),
  287. PidUnregistered = syn_test_suite_helper:start_process(),
  288. %% register
  289. ok = syn:register(<<"my proc">>, Pid),
  290. ok = syn:register(<<"my proc 2">>, Pid2),
  291. %% count
  292. 2 = syn:registry_count(),
  293. 2 = syn:registry_count(node()),
  294. %% kill & unregister
  295. syn_test_suite_helper:kill_process(Pid),
  296. ok = syn:unregister(<<"my proc 2">>),
  297. syn_test_suite_helper:kill_process(PidUnregistered),
  298. timer:sleep(100),
  299. %% count
  300. 0 = syn:registry_count(),
  301. 0 = syn:registry_count(node()).
  302. single_node_register_gen_server(_Config) ->
  303. %% start
  304. ok = syn:start(),
  305. %% start gen server via syn
  306. {ok, Pid} = syn_test_gen_server:start_link(),
  307. %% retrieve
  308. Pid = syn:whereis(syn_test_gen_server),
  309. %% call
  310. pong = syn_test_gen_server:ping(),
  311. %% send via syn
  312. syn:send(syn_test_gen_server, {self(), send_ping}),
  313. receive
  314. send_pong -> ok
  315. after 1000 ->
  316. ok = did_not_receive_gen_server_pong
  317. end,
  318. %% stop server
  319. syn_test_gen_server:stop(),
  320. timer:sleep(200),
  321. %% retrieve
  322. undefined = syn:whereis(syn_test_gen_server),
  323. %% send via syn
  324. {badarg, {syn_test_gen_server, anything}} = (catch syn:send(syn_test_gen_server, anything)).
  325. single_node_callback_on_process_exit(_Config) ->
  326. %% use custom handler
  327. syn_test_suite_helper:use_custom_handler(),
  328. %% start
  329. ok = syn:start(),
  330. %% start process
  331. Pid = syn_test_suite_helper:start_process(),
  332. Pid2 = syn_test_suite_helper:start_process(),
  333. %% register
  334. TestPid = self(),
  335. ok = syn:register(<<"my proc">>, Pid, {pid, TestPid}),
  336. ok = syn:register(<<"my proc - alternate">>, Pid, {pid_alternate, TestPid}),
  337. ok = syn:register(<<"my proc 2">>, Pid2, {pid2, TestPid}),
  338. %% kill 1
  339. syn_test_suite_helper:kill_process(Pid),
  340. receive
  341. {received_event_on, pid} ->
  342. ok;
  343. {received_event_on, pid2} ->
  344. ok = callback_on_process_exit_was_received_by_pid2
  345. after 1000 ->
  346. ok = callback_on_process_exit_was_not_received_by_pid
  347. end,
  348. receive
  349. {received_event_on, pid_alternate} ->
  350. ok;
  351. {received_event_on, pid2} ->
  352. ok = callback_on_process_exit_was_received_by_pid2
  353. after 1000 ->
  354. ok = callback_on_process_exit_was_not_received_by_pid
  355. end,
  356. %% unregister & kill 2
  357. ok = syn:unregister(<<"my proc 2">>),
  358. syn_test_suite_helper:kill_process(Pid2),
  359. receive
  360. {received_event_on, pid2} ->
  361. ok = callback_on_process_exit_was_received_by_pid2
  362. after 1000 ->
  363. ok
  364. end.
  365. single_node_ensure_callback_process_exit_is_called_if_process_killed(_Config) ->
  366. Name = <<"my proc">>,
  367. %% use custom handler
  368. syn_test_suite_helper:use_custom_handler(),
  369. %% start
  370. ok = syn:start(),
  371. %% start process
  372. Pid = syn_test_suite_helper:start_process(),
  373. %% register
  374. TestPid = self(),
  375. ok = syn:register(Name, Pid, {some_meta, TestPid}),
  376. %% remove from table to simulate conflict resolution
  377. syn_registry:remove_from_local_table(Name, TestPid),
  378. %% kill
  379. exit(Pid, {syn_resolve_kill, Name, {some_meta, TestPid}}),
  380. receive
  381. {received_event_on, some_meta} ->
  382. ok
  383. after 1000 ->
  384. ok = callback_on_process_exit_was_not_received_by_pid
  385. end.
  386. single_node_monitor_after_registry_crash(_Config) ->
  387. %% start
  388. ok = syn:start(),
  389. %% start processes
  390. Pid = syn_test_suite_helper:start_process(),
  391. %% register
  392. ok = syn:register(<<"my proc">>, Pid),
  393. %% kill registry
  394. exit(whereis(syn_registry), kill),
  395. timer:sleep(200),
  396. %% retrieve
  397. Pid = syn:whereis(<<"my proc">>),
  398. %% kill process
  399. syn_test_suite_helper:kill_process(Pid),
  400. timer:sleep(200),
  401. %% retrieve
  402. undefined = syn:whereis(<<"my proc 2">>).
  403. two_nodes_register_monitor_and_unregister(Config) ->
  404. %% get slave
  405. SlaveNode = proplists:get_value(slave_node, Config),
  406. %% start
  407. ok = syn:start(),
  408. ok = rpc:call(SlaveNode, syn, start, []),
  409. timer:sleep(100),
  410. %% start processes
  411. LocalPid = syn_test_suite_helper:start_process(),
  412. RemotePid = syn_test_suite_helper:start_process(SlaveNode),
  413. RemotePidRegRemote = syn_test_suite_helper:start_process(SlaveNode),
  414. %% retrieve
  415. undefined = syn:whereis(<<"local proc">>),
  416. undefined = syn:whereis(<<"remote proc">>),
  417. undefined = syn:whereis(<<"remote proc reg_remote">>),
  418. undefined = rpc:call(SlaveNode, syn, whereis, [<<"local proc">>]),
  419. undefined = rpc:call(SlaveNode, syn, whereis, [<<"remote proc">>]),
  420. undefined = rpc:call(SlaveNode, syn, whereis, [<<"remote proc reg_remote">>]),
  421. %% register
  422. ok = syn:register(<<"local proc">>, LocalPid),
  423. ok = syn:register(<<"remote proc">>, RemotePid),
  424. ok = rpc:call(SlaveNode, syn, register, [<<"remote proc reg_remote">>, RemotePidRegRemote]),
  425. timer:sleep(500),
  426. %% retrieve
  427. LocalPid = syn:whereis(<<"local proc">>),
  428. RemotePid = syn:whereis(<<"remote proc">>),
  429. RemotePidRegRemote = syn:whereis(<<"remote proc reg_remote">>),
  430. LocalPid = rpc:call(SlaveNode, syn, whereis, [<<"local proc">>]),
  431. RemotePid = rpc:call(SlaveNode, syn, whereis, [<<"remote proc">>]),
  432. RemotePidRegRemote = rpc:call(SlaveNode, syn, whereis, [<<"remote proc reg_remote">>]),
  433. %% kill & unregister processes
  434. syn_test_suite_helper:kill_process(LocalPid),
  435. ok = syn:unregister(<<"remote proc">>),
  436. syn_test_suite_helper:kill_process(RemotePidRegRemote),
  437. timer:sleep(100),
  438. %% retrieve
  439. undefined = syn:whereis(<<"local proc">>),
  440. undefined = syn:whereis(<<"remote proc">>),
  441. undefined = syn:whereis(<<"remote proc reg_remote">>),
  442. undefined = rpc:call(SlaveNode, syn, whereis, [<<"local proc">>]),
  443. undefined = rpc:call(SlaveNode, syn, whereis, [<<"remote proc">>]),
  444. undefined = rpc:call(SlaveNode, syn, whereis, [<<"remote proc reg_remote">>]).
  445. two_nodes_registry_count(Config) ->
  446. %% get slave
  447. SlaveNode = proplists:get_value(slave_node, Config),
  448. %% start
  449. ok = syn:start(),
  450. ok = rpc:call(SlaveNode, syn, start, []),
  451. timer:sleep(100),
  452. %% start processes
  453. LocalPid = syn_test_suite_helper:start_process(),
  454. RemotePid = syn_test_suite_helper:start_process(SlaveNode),
  455. RemotePidRegRemote = syn_test_suite_helper:start_process(SlaveNode),
  456. _PidUnregistered = syn_test_suite_helper:start_process(),
  457. %% register
  458. ok = syn:register(<<"local proc">>, LocalPid),
  459. ok = syn:register(<<"remote proc">>, RemotePid),
  460. ok = rpc:call(SlaveNode, syn, register, [<<"remote proc reg_remote">>, RemotePidRegRemote]),
  461. timer:sleep(500),
  462. %% count
  463. 3 = syn:registry_count(),
  464. 1 = syn:registry_count(node()),
  465. 2 = syn:registry_count(SlaveNode),
  466. %% kill & unregister processes
  467. syn_test_suite_helper:kill_process(LocalPid),
  468. ok = syn:unregister(<<"remote proc">>),
  469. syn_test_suite_helper:kill_process(RemotePidRegRemote),
  470. timer:sleep(100),
  471. %% count
  472. 0 = syn:registry_count(),
  473. 0 = syn:registry_count(node()),
  474. 0 = syn:registry_count(SlaveNode).
  475. two_nodes_registration_race_condition_conflict_resolution_keep_local(Config) ->
  476. ConflictingName = "COMMON",
  477. %% get slaves
  478. SlaveNode = proplists:get_value(slave_node, Config),
  479. %% start syn on nodes
  480. ok = syn:start(),
  481. ok = rpc:call(SlaveNode, syn, start, []),
  482. timer:sleep(1000),
  483. %% start processes
  484. Pid0 = syn_test_suite_helper:start_process(),
  485. Pid1 = syn_test_suite_helper:start_process(SlaveNode),
  486. %% inject into syn to simulate concurrent registration
  487. ok = syn_registry:add_to_local_table(ConflictingName, Pid0, node(), undefined),
  488. %% register on slave node to trigger conflict resolution on master node
  489. ok = rpc:call(SlaveNode, syn, register, [ConflictingName, Pid1, SlaveNode]),
  490. timer:sleep(1000),
  491. %% check metadata, resolution happens on master node
  492. Node = node(),
  493. {Pid0, Node} = syn:whereis(ConflictingName, with_meta),
  494. {Pid0, Node} = rpc:call(SlaveNode, syn, whereis, [ConflictingName, with_meta]),
  495. %% check that other processes are not alive because syn killed them
  496. true = is_process_alive(Pid0),
  497. false = rpc:call(SlaveNode, erlang, is_process_alive, [Pid1]).
  498. two_nodes_registration_race_condition_conflict_resolution_keep_remote(Config) ->
  499. ConflictingName = "COMMON",
  500. %% get slaves
  501. SlaveNode = proplists:get_value(slave_node, Config),
  502. %% use customer handler
  503. syn_test_suite_helper:use_custom_handler(),
  504. rpc:call(SlaveNode, syn_test_suite_helper, use_custom_handler, []),
  505. %% start syn on nodes
  506. ok = syn:start(),
  507. ok = rpc:call(SlaveNode, syn, start, []),
  508. timer:sleep(1000),
  509. %% start processes
  510. Pid0 = syn_test_suite_helper:start_process(),
  511. Pid1 = syn_test_suite_helper:start_process(SlaveNode),
  512. %% inject into syn to simulate concurrent registration
  513. ok = syn_registry:add_to_local_table(ConflictingName, Pid0, node(), undefined),
  514. %% register on slave node to trigger conflict resolution on master node
  515. ok = rpc:call(SlaveNode, syn, register, [ConflictingName, Pid1, keep_this_one]),
  516. timer:sleep(1000),
  517. %% check metadata, resolution happens on master node
  518. {Pid1, keep_this_one} = syn:whereis(ConflictingName, with_meta),
  519. {Pid1, keep_this_one} = rpc:call(SlaveNode, syn, whereis, [ConflictingName, with_meta]),
  520. %% check that other processes are not alive because syn killed them
  521. true = is_process_alive(Pid0),
  522. true = rpc:call(SlaveNode, erlang, is_process_alive, [Pid1]).
  523. two_nodes_registration_race_condition_conflict_resolution_when_process_died(Config) ->
  524. ConflictingName = "COMMON",
  525. %% get slaves
  526. SlaveNode = proplists:get_value(slave_node, Config),
  527. %% use customer handler
  528. syn_test_suite_helper:use_custom_handler(),
  529. rpc:call(SlaveNode, syn_test_suite_helper, use_custom_handler, []),
  530. %% start syn on nodes
  531. ok = syn:start(),
  532. ok = rpc:call(SlaveNode, syn, start, []),
  533. timer:sleep(100),
  534. %% start processes
  535. Pid0 = syn_test_suite_helper:start_process(),
  536. Pid1 = syn_test_suite_helper:start_process(SlaveNode),
  537. %% inject into syn to simulate concurrent registration
  538. syn_registry:add_to_local_table(ConflictingName, Pid0, keep_this_one, undefined),
  539. %% kill process
  540. syn_test_suite_helper:kill_process(Pid0),
  541. timer:sleep(250),
  542. %% register to trigger conflict resolution
  543. ok = rpc:call(SlaveNode, syn, register, [ConflictingName, Pid1, SlaveNode]),
  544. timer:sleep(1000),
  545. %% check
  546. {Pid1, SlaveNode} = syn:whereis(ConflictingName, with_meta),
  547. {Pid1, SlaveNode} = rpc:call(SlaveNode, syn, whereis, [ConflictingName, with_meta]),
  548. %% check that process is alive
  549. true = rpc:call(SlaveNode, erlang, is_process_alive, [Pid1]).
  550. two_nodes_registry_full_cluster_sync_on_boot_node_added_later(_Config) ->
  551. %% stop slave
  552. syn_test_suite_helper:stop_slave(syn_slave),
  553. %% start syn on local node
  554. ok = syn:start(),
  555. %% start process
  556. Pid = syn_test_suite_helper:start_process(),
  557. %% register
  558. ok = syn:register(<<"proc">>, Pid),
  559. %% start remote node and syn
  560. {ok, SlaveNode} = syn_test_suite_helper:start_slave(syn_slave),
  561. ok = rpc:call(SlaveNode, syn, start, []),
  562. timer:sleep(1000),
  563. %% check
  564. Pid = syn:whereis(<<"proc">>),
  565. Pid = rpc:call(SlaveNode, syn, whereis, [<<"proc">>]).
  566. two_nodes_registry_full_cluster_sync_on_boot_syn_started_later(Config) ->
  567. %% get slaves
  568. SlaveNode = proplists:get_value(slave_node, Config),
  569. %% start syn on local node
  570. ok = syn:start(),
  571. %% start process
  572. Pid = syn_test_suite_helper:start_process(),
  573. %% register
  574. ok = syn:register(<<"proc">>, Pid),
  575. %% start ib remote syn
  576. ok = rpc:call(SlaveNode, syn, start, []),
  577. timer:sleep(500),
  578. %% check
  579. Pid = syn:whereis(<<"proc">>),
  580. Pid = rpc:call(SlaveNode, syn, whereis, [<<"proc">>]).
  581. two_nodes_reregister(Config) ->
  582. Name = "common name",
  583. %% get slave
  584. SlaveNode = proplists:get_value(slave_node, Config),
  585. %% start
  586. ok = syn:start(),
  587. ok = rpc:call(SlaveNode, syn, start, []),
  588. timer:sleep(100),
  589. %% start processes
  590. PidLocal = syn_test_suite_helper:start_process(),
  591. PidRemote = syn_test_suite_helper:start_process(SlaveNode),
  592. ok = rpc:call(SlaveNode, syn, register, [Name, PidRemote]),
  593. %% fast unreg-reg
  594. ok = syn:reregister(Name, PidLocal),
  595. timer:sleep(1000),
  596. PidLocal = syn:whereis(Name),
  597. ok = rpc:call(SlaveNode, syn, reregister, [Name, PidRemote, some_meta]),
  598. timer:sleep(1000),
  599. {PidRemote, some_meta} = syn:whereis(Name, with_meta).
  600. three_nodes_partial_netsplit_consistency(Config) ->
  601. %% get slaves
  602. SlaveNode1 = proplists:get_value(slave_node_1, Config),
  603. SlaveNode2 = proplists:get_value(slave_node_2, Config),
  604. %% start syn on nodes
  605. ok = syn:start(),
  606. ok = rpc:call(SlaveNode1, syn, start, []),
  607. ok = rpc:call(SlaveNode2, syn, start, []),
  608. timer:sleep(100),
  609. %% start processes
  610. Pid0 = syn_test_suite_helper:start_process(),
  611. Pid0Changed = syn_test_suite_helper:start_process(),
  612. Pid1 = syn_test_suite_helper:start_process(SlaveNode1),
  613. Pid2 = syn_test_suite_helper:start_process(SlaveNode2),
  614. timer:sleep(100),
  615. %% retrieve
  616. undefined = syn:whereis(<<"proc0">>),
  617. undefined = syn:whereis(<<"proc0-changed">>),
  618. undefined = syn:whereis(<<"proc1">>),
  619. undefined = syn:whereis(<<"proc2">>),
  620. undefined = rpc:call(SlaveNode1, syn, whereis, [<<"proc0">>]),
  621. undefined = rpc:call(SlaveNode1, syn, whereis, [<<"proc0-changed">>]),
  622. undefined = rpc:call(SlaveNode1, syn, whereis, [<<"proc1">>]),
  623. undefined = rpc:call(SlaveNode1, syn, whereis, [<<"proc2">>]),
  624. undefined = rpc:call(SlaveNode2, syn, whereis, [<<"proc0">>]),
  625. undefined = rpc:call(SlaveNode2, syn, whereis, [<<"proc0-changed">>]),
  626. undefined = rpc:call(SlaveNode2, syn, whereis, [<<"proc1">>]),
  627. undefined = rpc:call(SlaveNode2, syn, whereis, [<<"proc2">>]),
  628. %% register (mix nodes)
  629. ok = rpc:call(SlaveNode2, syn, register, [<<"proc0">>, Pid0]),
  630. ok = syn:register(<<"proc1">>, Pid1),
  631. ok = rpc:call(SlaveNode1, syn, register, [<<"proc2">>, Pid2]),
  632. ok = rpc:call(SlaveNode1, syn, register, [<<"proc0-changed">>, Pid0Changed]),
  633. timer:sleep(200),
  634. %% retrieve
  635. Pid0 = syn:whereis(<<"proc0">>),
  636. Pid0Changed = syn:whereis(<<"proc0-changed">>),
  637. Pid1 = syn:whereis(<<"proc1">>),
  638. Pid2 = syn:whereis(<<"proc2">>),
  639. Pid0 = rpc:call(SlaveNode1, syn, whereis, [<<"proc0">>]),
  640. Pid0Changed = rpc:call(SlaveNode1, syn, whereis, [<<"proc0-changed">>]),
  641. Pid1 = rpc:call(SlaveNode1, syn, whereis, [<<"proc1">>]),
  642. Pid2 = rpc:call(SlaveNode1, syn, whereis, [<<"proc2">>]),
  643. Pid0 = rpc:call(SlaveNode2, syn, whereis, [<<"proc0">>]),
  644. Pid0Changed = rpc:call(SlaveNode2, syn, whereis, [<<"proc0-changed">>]),
  645. Pid1 = rpc:call(SlaveNode2, syn, whereis, [<<"proc1">>]),
  646. Pid2 = rpc:call(SlaveNode2, syn, whereis, [<<"proc2">>]),
  647. %% disconnect slave 2 from main (slave 1 can still see slave 2)
  648. syn_test_suite_helper:disconnect_node(SlaveNode2),
  649. timer:sleep(500),
  650. %% retrieve
  651. Pid0 = syn:whereis(<<"proc0">>),
  652. Pid0Changed = syn:whereis(<<"proc0-changed">>),
  653. Pid1 = syn:whereis(<<"proc1">>),
  654. undefined = syn:whereis(<<"proc2">>), %% main has lost slave 2 so 'proc2' is removed
  655. Pid0 = rpc:call(SlaveNode1, syn, whereis, [<<"proc0">>]),
  656. Pid0Changed = rpc:call(SlaveNode1, syn, whereis, [<<"proc0-changed">>]),
  657. Pid1 = rpc:call(SlaveNode1, syn, whereis, [<<"proc1">>]),
  658. Pid2 = rpc:call(SlaveNode1, syn, whereis, [<<"proc2">>]), %% slave 1 still has slave 2 so 'proc2' is still there
  659. %% disconnect slave 1
  660. syn_test_suite_helper:disconnect_node(SlaveNode1),
  661. timer:sleep(500),
  662. %% unregister proc0-changed
  663. ok = syn:unregister(<<"proc0-changed">>),
  664. %% retrieve
  665. Pid0 = syn:whereis(<<"proc0">>),
  666. undefined = syn:whereis(<<"proc0-changed">>),
  667. undefined = syn:whereis(<<"proc1">>),
  668. undefined = syn:whereis(<<"proc2">>),
  669. %% reconnect all
  670. syn_test_suite_helper:connect_node(SlaveNode1),
  671. syn_test_suite_helper:connect_node(SlaveNode2),
  672. timer:sleep(5000),
  673. %% retrieve
  674. Pid0 = syn:whereis(<<"proc0">>),
  675. undefined = syn:whereis(<<"proc0-changed">>),
  676. Pid1 = syn:whereis(<<"proc1">>),
  677. Pid2 = syn:whereis(<<"proc2">>),
  678. Pid0 = rpc:call(SlaveNode1, syn, whereis, [<<"proc0">>]),
  679. undefined = rpc:call(SlaveNode1, syn, whereis, [<<"proc0-changed">>]),
  680. Pid1 = rpc:call(SlaveNode1, syn, whereis, [<<"proc1">>]),
  681. Pid2 = rpc:call(SlaveNode1, syn, whereis, [<<"proc2">>]),
  682. Pid0 = rpc:call(SlaveNode2, syn, whereis, [<<"proc0">>]),
  683. undefined = rpc:call(SlaveNode2, syn, whereis, [<<"proc0-changed">>]),
  684. Pid1 = rpc:call(SlaveNode2, syn, whereis, [<<"proc1">>]),
  685. Pid2 = rpc:call(SlaveNode2, syn, whereis, [<<"proc2">>]).
  686. three_nodes_full_netsplit_consistency(Config) ->
  687. %% get slaves
  688. SlaveNode1 = proplists:get_value(slave_node_1, Config),
  689. SlaveNode2 = proplists:get_value(slave_node_2, Config),
  690. %% start syn on nodes
  691. ok = syn:start(),
  692. ok = rpc:call(SlaveNode1, syn, start, []),
  693. ok = rpc:call(SlaveNode2, syn, start, []),
  694. timer:sleep(100),
  695. %% start processes
  696. Pid0 = syn_test_suite_helper:start_process(),
  697. Pid0Changed = syn_test_suite_helper:start_process(),
  698. Pid1 = syn_test_suite_helper:start_process(SlaveNode1),
  699. Pid2 = syn_test_suite_helper:start_process(SlaveNode2),
  700. timer:sleep(100),
  701. %% retrieve
  702. undefined = syn:whereis(<<"proc0">>),
  703. undefined = syn:whereis(<<"proc0-changed">>),
  704. undefined = syn:whereis(<<"proc1">>),
  705. undefined = syn:whereis(<<"proc2">>),
  706. undefined = rpc:call(SlaveNode1, syn, whereis, [<<"proc0">>]),
  707. undefined = rpc:call(SlaveNode1, syn, whereis, [<<"proc0-changed">>]),
  708. undefined = rpc:call(SlaveNode1, syn, whereis, [<<"proc1">>]),
  709. undefined = rpc:call(SlaveNode1, syn, whereis, [<<"proc2">>]),
  710. undefined = rpc:call(SlaveNode2, syn, whereis, [<<"proc0">>]),
  711. undefined = rpc:call(SlaveNode2, syn, whereis, [<<"proc0-changed">>]),
  712. undefined = rpc:call(SlaveNode2, syn, whereis, [<<"proc1">>]),
  713. undefined = rpc:call(SlaveNode2, syn, whereis, [<<"proc2">>]),
  714. %% register (mix nodes)
  715. ok = rpc:call(SlaveNode2, syn, register, [<<"proc0">>, Pid0]),
  716. ok = rpc:call(SlaveNode2, syn, register, [<<"proc0-changed">>, Pid0Changed]),
  717. ok = syn:register(<<"proc1">>, Pid1),
  718. ok = rpc:call(SlaveNode1, syn, register, [<<"proc2">>, Pid2]),
  719. timer:sleep(200),
  720. %% retrieve
  721. Pid0 = syn:whereis(<<"proc0">>),
  722. Pid0Changed = syn:whereis(<<"proc0-changed">>),
  723. Pid1 = syn:whereis(<<"proc1">>),
  724. Pid2 = syn:whereis(<<"proc2">>),
  725. Pid0 = rpc:call(SlaveNode1, syn, whereis, [<<"proc0">>]),
  726. Pid0Changed = rpc:call(SlaveNode1, syn, whereis, [<<"proc0-changed">>]),
  727. Pid1 = rpc:call(SlaveNode1, syn, whereis, [<<"proc1">>]),
  728. Pid2 = rpc:call(SlaveNode1, syn, whereis, [<<"proc2">>]),
  729. Pid0 = rpc:call(SlaveNode2, syn, whereis, [<<"proc0">>]),
  730. Pid0Changed = rpc:call(SlaveNode2, syn, whereis, [<<"proc0-changed">>]),
  731. Pid1 = rpc:call(SlaveNode2, syn, whereis, [<<"proc1">>]),
  732. Pid2 = rpc:call(SlaveNode2, syn, whereis, [<<"proc2">>]),
  733. %% disconnect slave 2 from main (slave 1 can still see slave 2)
  734. syn_test_suite_helper:disconnect_node(SlaveNode2),
  735. timer:sleep(500),
  736. %% retrieve
  737. Pid0 = syn:whereis(<<"proc0">>),
  738. Pid0Changed = syn:whereis(<<"proc0-changed">>),
  739. Pid1 = syn:whereis(<<"proc1">>),
  740. undefined = syn:whereis(<<"proc2">>), %% main has lost slave 2 so 'proc2' is removed
  741. Pid0 = rpc:call(SlaveNode1, syn, whereis, [<<"proc0">>]),
  742. Pid0Changed = rpc:call(SlaveNode1, syn, whereis, [<<"proc0-changed">>]),
  743. Pid1 = rpc:call(SlaveNode1, syn, whereis, [<<"proc1">>]),
  744. Pid2 = rpc:call(SlaveNode1, syn, whereis, [<<"proc2">>]), %% slave 1 still has slave 2 so 'proc2' is still there
  745. %% disconnect slave 2 from slave 1
  746. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  747. timer:sleep(500),
  748. %% retrieve
  749. Pid0 = syn:whereis(<<"proc0">>),
  750. Pid0Changed = syn:whereis(<<"proc0-changed">>),
  751. Pid1 = syn:whereis(<<"proc1">>),
  752. undefined = syn:whereis(<<"proc2">>), %% main has lost slave 2 so 'proc2' is removed
  753. undefined = syn:whereis(<<"proc2">>, with_meta),
  754. Pid0 = rpc:call(SlaveNode1, syn, whereis, [<<"proc0">>]),
  755. Pid0Changed = rpc:call(SlaveNode1, syn, whereis, [<<"proc0-changed">>]),
  756. Pid1 = rpc:call(SlaveNode1, syn, whereis, [<<"proc1">>]),
  757. undefined = rpc:call(SlaveNode1, syn, whereis, [<<"proc2">>]),
  758. %% disconnect slave 1
  759. syn_test_suite_helper:disconnect_node(SlaveNode1),
  760. timer:sleep(500),
  761. %% unregister
  762. ok = syn:unregister(<<"proc0-changed">>),
  763. %% retrieve
  764. Pid0 = syn:whereis(<<"proc0">>),
  765. undefined = syn:whereis(<<"proc0-changed">>),
  766. undefined = syn:whereis(<<"proc1">>),
  767. undefined = syn:whereis(<<"proc2">>),
  768. %% reconnect all
  769. syn_test_suite_helper:connect_node(SlaveNode1),
  770. syn_test_suite_helper:connect_node(SlaveNode2),
  771. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  772. timer:sleep(1500),
  773. %% retrieve
  774. Pid0 = syn:whereis(<<"proc0">>),
  775. undefined = syn:whereis(<<"proc0-changed">>),
  776. Pid1 = syn:whereis(<<"proc1">>),
  777. Pid2 = syn:whereis(<<"proc2">>),
  778. Pid0 = rpc:call(SlaveNode1, syn, whereis, [<<"proc0">>]),
  779. undefined = rpc:call(SlaveNode1, syn, whereis, [<<"proc0-changed">>]),
  780. Pid1 = rpc:call(SlaveNode1, syn, whereis, [<<"proc1">>]),
  781. Pid2 = rpc:call(SlaveNode1, syn, whereis, [<<"proc2">>]),
  782. Pid0 = rpc:call(SlaveNode2, syn, whereis, [<<"proc0">>]),
  783. undefined = rpc:call(SlaveNode2, syn, whereis, [<<"proc0-changed">>]),
  784. Pid1 = rpc:call(SlaveNode2, syn, whereis, [<<"proc1">>]),
  785. Pid2 = rpc:call(SlaveNode2, syn, whereis, [<<"proc2">>]).
  786. three_nodes_start_syn_before_connecting_cluster_with_conflict(Config) ->
  787. ConflictingName = "COMMON",
  788. %% get slaves
  789. SlaveNode1 = proplists:get_value(slave_node_1, Config),
  790. SlaveNode2 = proplists:get_value(slave_node_2, Config),
  791. %% start processes
  792. Pid0 = syn_test_suite_helper:start_process(),
  793. Pid1 = syn_test_suite_helper:start_process(SlaveNode1),
  794. Pid2 = syn_test_suite_helper:start_process(SlaveNode2),
  795. %% start delayed
  796. start_syn_delayed_and_register_local_process(ConflictingName, Pid0, 1500),
  797. rpc:cast(SlaveNode1, ?MODULE, start_syn_delayed_and_register_local_process, [ConflictingName, Pid1, 1500]),
  798. rpc:cast(SlaveNode2, ?MODULE, start_syn_delayed_and_register_local_process, [ConflictingName, Pid2, 1500]),
  799. timer:sleep(500),
  800. %% disconnect all
  801. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  802. syn_test_suite_helper:disconnect_node(SlaveNode1),
  803. syn_test_suite_helper:disconnect_node(SlaveNode2),
  804. timer:sleep(1500),
  805. [] = nodes(),
  806. %% reconnect all
  807. syn_test_suite_helper:connect_node(SlaveNode1),
  808. syn_test_suite_helper:connect_node(SlaveNode2),
  809. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  810. timer:sleep(1500),
  811. %% count
  812. 1 = syn:registry_count(),
  813. 1 = rpc:call(SlaveNode1, syn, registry_count, []),
  814. 1 = rpc:call(SlaveNode2, syn, registry_count, []),
  815. %% retrieve
  816. true = lists:member(syn:whereis(ConflictingName), [Pid0, Pid1, Pid2]),
  817. true = lists:member(rpc:call(SlaveNode1, syn, whereis, [ConflictingName]), [Pid0, Pid1, Pid2]),
  818. true = lists:member(rpc:call(SlaveNode2, syn, whereis, [ConflictingName]), [Pid0, Pid1, Pid2]),
  819. %% check metadata
  820. case syn:whereis(ConflictingName, with_meta) of
  821. {Pid0, Meta} ->
  822. CurrentNode = node(),
  823. %% check that other nodes' data corresponds
  824. {Pid0, CurrentNode} = rpc:call(SlaveNode1, syn, whereis, [ConflictingName, with_meta]),
  825. {Pid0, CurrentNode} = rpc:call(SlaveNode2, syn, whereis, [ConflictingName, with_meta]),
  826. %% check that other processes are not alive because syn killed them
  827. true = is_process_alive(Pid0),
  828. false = rpc:call(SlaveNode1, erlang, is_process_alive, [Pid1]),
  829. false = rpc:call(SlaveNode2, erlang, is_process_alive, [Pid2]);
  830. {Pid1, Meta} ->
  831. SlaveNode1 = Meta,
  832. %% check that other nodes' data corresponds
  833. {Pid1, Meta} = rpc:call(SlaveNode1, syn, whereis, [ConflictingName, with_meta]),
  834. {Pid1, Meta} = rpc:call(SlaveNode2, syn, whereis, [ConflictingName, with_meta]),
  835. %% check that other processes are not alive because syn killed them
  836. false = is_process_alive(Pid0),
  837. true = rpc:call(SlaveNode1, erlang, is_process_alive, [Pid1]),
  838. false = rpc:call(SlaveNode2, erlang, is_process_alive, [Pid2]);
  839. {Pid2, Meta} ->
  840. SlaveNode2 = Meta,
  841. %% check that other nodes' data corresponds
  842. {Pid2, Meta} = rpc:call(SlaveNode1, syn, whereis, [ConflictingName, with_meta]),
  843. {Pid2, Meta} = rpc:call(SlaveNode2, syn, whereis, [ConflictingName, with_meta]),
  844. %% check that other processes are not alive because syn killed them
  845. false = is_process_alive(Pid0),
  846. false = rpc:call(SlaveNode1, erlang, is_process_alive, [Pid1]),
  847. true = rpc:call(SlaveNode2, erlang, is_process_alive, [Pid2]);
  848. _ ->
  849. ok = no_process_is_registered_with_conflicting_name
  850. end.
  851. three_nodes_start_syn_before_connecting_cluster_with_custom_conflict_resolution(Config) ->
  852. ConflictingName = "COMMON",
  853. %% get slaves
  854. SlaveNode1 = proplists:get_value(slave_node_1, Config),
  855. SlaveNode2 = proplists:get_value(slave_node_2, Config),
  856. %% start processes
  857. Pid0 = syn_test_suite_helper:start_process(),
  858. Pid1 = syn_test_suite_helper:start_process(SlaveNode1),
  859. Pid2 = syn_test_suite_helper:start_process(SlaveNode2),
  860. %% start delayed
  861. start_syn_delayed_with_custom_handler_register_local_process(ConflictingName, Pid0, {node, node()}, 1500),
  862. rpc:cast(
  863. SlaveNode1,
  864. ?MODULE,
  865. start_syn_delayed_with_custom_handler_register_local_process,
  866. [ConflictingName, Pid1, keep_this_one, 1500])
  867. ,
  868. rpc:cast(
  869. SlaveNode2,
  870. ?MODULE,
  871. start_syn_delayed_with_custom_handler_register_local_process,
  872. [ConflictingName, Pid2, {node, SlaveNode2}, 1500]
  873. ),
  874. timer:sleep(500),
  875. %% disconnect all
  876. rpc:call(SlaveNode1, syn_test_suite_helper, disconnect_node, [SlaveNode2]),
  877. syn_test_suite_helper:disconnect_node(SlaveNode1),
  878. syn_test_suite_helper:disconnect_node(SlaveNode2),
  879. timer:sleep(1500),
  880. [] = nodes(),
  881. %% reconnect all
  882. syn_test_suite_helper:connect_node(SlaveNode1),
  883. syn_test_suite_helper:connect_node(SlaveNode2),
  884. rpc:call(SlaveNode1, syn_test_suite_helper, connect_node, [SlaveNode2]),
  885. timer:sleep(5000),
  886. %% count
  887. 1 = syn:registry_count(),
  888. 1 = rpc:call(SlaveNode1, syn, registry_count, []),
  889. 1 = rpc:call(SlaveNode2, syn, registry_count, []),
  890. %% retrieve
  891. true = lists:member(syn:whereis(ConflictingName), [Pid0, Pid1, Pid2]),
  892. true = lists:member(rpc:call(SlaveNode1, syn, whereis, [ConflictingName]), [Pid0, Pid1, Pid2]),
  893. true = lists:member(rpc:call(SlaveNode2, syn, whereis, [ConflictingName]), [Pid0, Pid1, Pid2]),
  894. %% check metadata that we kept the correct process on all nodes
  895. {Pid1, keep_this_one} = syn:whereis(ConflictingName, with_meta),
  896. {Pid1, keep_this_one} = rpc:call(SlaveNode1, syn, whereis, [ConflictingName, with_meta]),
  897. {Pid1, keep_this_one} = rpc:call(SlaveNode1, syn, whereis, [ConflictingName, with_meta]),
  898. %% check that other processes are still alive because we didn't kill them
  899. true = is_process_alive(Pid0),
  900. true = rpc:call(SlaveNode1, erlang, is_process_alive, [Pid1]),
  901. true = rpc:call(SlaveNode2, erlang, is_process_alive, [Pid2]).
  902. three_nodes_registration_race_condition_custom_conflict_resolution(Config) ->
  903. ConflictingName = "COMMON",
  904. %% get slaves
  905. SlaveNode1 = proplists:get_value(slave_node_1, Config),
  906. SlaveNode2 = proplists:get_value(slave_node_2, Config),
  907. %% use customer handler
  908. syn_test_suite_helper:use_custom_handler(),
  909. rpc:call(SlaveNode1, syn_test_suite_helper, use_custom_handler, []),
  910. rpc:call(SlaveNode2, syn_test_suite_helper, use_custom_handler, []),
  911. %% start syn on nodes
  912. ok = syn:start(),
  913. ok = rpc:call(SlaveNode1, syn, start, []),
  914. ok = rpc:call(SlaveNode2, syn, start, []),
  915. timer:sleep(500),
  916. %% start processes
  917. Pid0 = syn_test_suite_helper:start_process(),
  918. Pid1 = syn_test_suite_helper:start_process(SlaveNode1),
  919. Pid2 = syn_test_suite_helper:start_process(SlaveNode2),
  920. %% inject into syn to simulate concurrent registration
  921. ok = rpc:call(SlaveNode1, syn_registry, add_to_local_table, [ConflictingName, Pid1, keep_this_one, undefined]),
  922. %% register on master node to trigger conflict resolution
  923. ok = syn:register(ConflictingName, Pid0, node()),
  924. timer:sleep(1000),
  925. %% retrieve
  926. true = lists:member(syn:whereis(ConflictingName), [Pid0, Pid1, Pid2]),
  927. true = lists:member(rpc:call(SlaveNode1, syn, whereis, [ConflictingName]), [Pid0, Pid1, Pid2]),
  928. true = lists:member(rpc:call(SlaveNode2, syn, whereis, [ConflictingName]), [Pid0, Pid1, Pid2]),
  929. %% check metadata that we kept the correct process on all nodes
  930. {Pid1, keep_this_one} = syn:whereis(ConflictingName, with_meta),
  931. {Pid1, keep_this_one} = rpc:call(SlaveNode1, syn, whereis, [ConflictingName, with_meta]),
  932. {Pid1, keep_this_one} = rpc:call(SlaveNode1, syn, whereis, [ConflictingName, with_meta]),
  933. %% check that other processes are still alive because we didn't kill them
  934. true = is_process_alive(Pid0),
  935. true = rpc:call(SlaveNode1, erlang, is_process_alive, [Pid1]),
  936. true = rpc:call(SlaveNode2, erlang, is_process_alive, [Pid2]).
  937. three_nodes_anti_entropy(Config) ->
  938. %% get slaves
  939. SlaveNode1 = proplists:get_value(slave_node_1, Config),
  940. SlaveNode2 = proplists:get_value(slave_node_2, Config),
  941. %% use customer handler
  942. syn_test_suite_helper:use_custom_handler(),
  943. rpc:call(SlaveNode1, syn_test_suite_helper, use_custom_handler, []),
  944. rpc:call(SlaveNode2, syn_test_suite_helper, use_custom_handler, []),
  945. %% set anti-entropy with a very low interval (0.25 second)
  946. syn_test_suite_helper:use_anti_entropy(registry, 0.25),
  947. rpc:call(SlaveNode1, syn_test_suite_helper, use_anti_entropy, [registry, 0.25]),
  948. rpc:call(SlaveNode2, syn_test_suite_helper, use_anti_entropy, [registry, 0.25]),
  949. %% start syn on nodes
  950. ok = syn:start(),
  951. ok = rpc:call(SlaveNode1, syn, start, []),
  952. ok = rpc:call(SlaveNode2, syn, start, []),
  953. timer:sleep(100),
  954. %% start processes
  955. Pid0 = syn_test_suite_helper:start_process(),
  956. Pid1 = syn_test_suite_helper:start_process(SlaveNode1),
  957. Pid2 = syn_test_suite_helper:start_process(SlaveNode2),
  958. Pid0Conflict = syn_test_suite_helper:start_process(),
  959. Pid1Conflict = syn_test_suite_helper:start_process(SlaveNode1),
  960. Pid2Conflict = syn_test_suite_helper:start_process(SlaveNode2),
  961. timer:sleep(100),
  962. %% inject data to simulate latent conflicts
  963. ok = syn_registry:add_to_local_table("pid0", Pid0, node(), undefined),
  964. ok = rpc:call(SlaveNode1, syn_registry, add_to_local_table, ["pid1", Pid1, SlaveNode1, undefined]),
  965. ok = rpc:call(SlaveNode2, syn_registry, add_to_local_table, ["pid2", Pid2, SlaveNode2, undefined]),
  966. ok = syn_registry:add_to_local_table("conflict", Pid0Conflict, node(), undefined),
  967. ok = rpc:call(SlaveNode1, syn_registry, add_to_local_table, ["conflict", Pid1Conflict, keep_this_one, undefined]),
  968. ok = rpc:call(SlaveNode2, syn_registry, add_to_local_table, ["conflict", Pid2Conflict, SlaveNode2, undefined]),
  969. %% wait to let anti-entropy settle
  970. timer:sleep(5000),
  971. %% check
  972. Node = node(),
  973. {Pid0, Node} = syn:whereis("pid0", with_meta),
  974. {Pid1, SlaveNode1} = syn:whereis("pid1", with_meta),
  975. {Pid2, SlaveNode2} = syn:whereis("pid2", with_meta),
  976. {Pid1Conflict, keep_this_one} = syn:whereis("conflict", with_meta),
  977. {Pid0, Node} = rpc:call(SlaveNode1, syn, whereis, ["pid0", with_meta]),
  978. {Pid1, SlaveNode1} = rpc:call(SlaveNode1, syn, whereis, ["pid1", with_meta]),
  979. {Pid2, SlaveNode2} = rpc:call(SlaveNode1, syn, whereis, ["pid2", with_meta]),
  980. {Pid1Conflict, keep_this_one} = rpc:call(SlaveNode1, syn, whereis, ["conflict", with_meta]),
  981. {Pid0, Node} = rpc:call(SlaveNode2, syn, whereis, ["pid0", with_meta]),
  982. {Pid1, SlaveNode1} = rpc:call(SlaveNode2, syn, whereis, ["pid1", with_meta]),
  983. {Pid2, SlaveNode2} = rpc:call(SlaveNode2, syn, whereis, ["pid2", with_meta]),
  984. {Pid1Conflict, keep_this_one} = rpc:call(SlaveNode2, syn, whereis, ["conflict", with_meta]).
  985. three_nodes_anti_entropy_manual(Config) ->
  986. %% get slaves
  987. SlaveNode1 = proplists:get_value(slave_node_1, Config),
  988. SlaveNode2 = proplists:get_value(slave_node_2, Config),
  989. %% use customer handler
  990. syn_test_suite_helper:use_custom_handler(),
  991. rpc:call(SlaveNode1, syn_test_suite_helper, use_custom_handler, []),
  992. rpc:call(SlaveNode2, syn_test_suite_helper, use_custom_handler, []),
  993. %% start syn on nodes
  994. ok = syn:start(),
  995. ok = rpc:call(SlaveNode1, syn, start, []),
  996. ok = rpc:call(SlaveNode2, syn, start, []),
  997. timer:sleep(100),
  998. %% start processes
  999. Pid0 = syn_test_suite_helper:start_process(),
  1000. Pid1 = syn_test_suite_helper:start_process(SlaveNode1),
  1001. Pid2 = syn_test_suite_helper:start_process(SlaveNode2),
  1002. Pid0Conflict = syn_test_suite_helper:start_process(),
  1003. Pid1Conflict = syn_test_suite_helper:start_process(SlaveNode1),
  1004. Pid2Conflict = syn_test_suite_helper:start_process(SlaveNode2),
  1005. timer:sleep(100),
  1006. %% inject data to simulate latent conflicts
  1007. ok = syn_registry:add_to_local_table("pid0", Pid0, node(), undefined),
  1008. ok = rpc:call(SlaveNode1, syn_registry, add_to_local_table, ["pid1", Pid1, SlaveNode1, undefined]),
  1009. ok = rpc:call(SlaveNode2, syn_registry, add_to_local_table, ["pid2", Pid2, SlaveNode2, undefined]),
  1010. ok = syn_registry:add_to_local_table("conflict", Pid0Conflict, node(), undefined),
  1011. ok = rpc:call(SlaveNode1, syn_registry, add_to_local_table, ["conflict", Pid1Conflict, keep_this_one, undefined]),
  1012. ok = rpc:call(SlaveNode2, syn_registry, add_to_local_table, ["conflict", Pid2Conflict, SlaveNode2, undefined]),
  1013. %% call anti entropy
  1014. {error, not_remote_node} = syn:sync_from_node(registry, node()),
  1015. ok = syn:sync_from_node(registry, SlaveNode1),
  1016. ok = syn:sync_from_node(registry, SlaveNode2),
  1017. {error, not_remote_node} = rpc:call(SlaveNode1, syn, sync_from_node, [registry, SlaveNode1]),
  1018. ok = rpc:call(SlaveNode1, syn, sync_from_node, [registry, node()]),
  1019. ok = rpc:call(SlaveNode1, syn, sync_from_node, [registry, SlaveNode2]),
  1020. {error, not_remote_node} = rpc:call(SlaveNode2, syn, sync_from_node, [registry, SlaveNode2]),
  1021. ok = rpc:call(SlaveNode2, syn, sync_from_node, [registry, node()]),
  1022. ok = rpc:call(SlaveNode2, syn, sync_from_node, [registry, SlaveNode1]),
  1023. timer:sleep(500),
  1024. %% check
  1025. Node = node(),
  1026. {Pid0, Node} = syn:whereis("pid0", with_meta),
  1027. {Pid1, SlaveNode1} = syn:whereis("pid1", with_meta),
  1028. {Pid2, SlaveNode2} = syn:whereis("pid2", with_meta),
  1029. {Pid1Conflict, keep_this_one} = syn:whereis("conflict", with_meta),
  1030. {Pid0, Node} = rpc:call(SlaveNode1, syn, whereis, ["pid0", with_meta]),
  1031. {Pid1, SlaveNode1} = rpc:call(SlaveNode1, syn, whereis, ["pid1", with_meta]),
  1032. {Pid2, SlaveNode2} = rpc:call(SlaveNode1, syn, whereis, ["pid2", with_meta]),
  1033. {Pid1Conflict, keep_this_one} = rpc:call(SlaveNode1, syn, whereis, ["conflict", with_meta]),
  1034. {Pid0, Node} = rpc:call(SlaveNode2, syn, whereis, ["pid0", with_meta]),
  1035. {Pid1, SlaveNode1} = rpc:call(SlaveNode2, syn, whereis, ["pid1", with_meta]),
  1036. {Pid2, SlaveNode2} = rpc:call(SlaveNode2, syn, whereis, ["pid2", with_meta]),
  1037. {Pid1Conflict, keep_this_one} = rpc:call(SlaveNode2, syn, whereis, ["conflict", with_meta]).
  1038. three_nodes_concurrent_registration_unregistration(Config) ->
  1039. CommonName = "common-name",
  1040. %% get slaves
  1041. SlaveNode1 = proplists:get_value(slave_node_1, Config),
  1042. SlaveNode2 = proplists:get_value(slave_node_2, Config),
  1043. %% start syn on nodes
  1044. ok = syn:start(),
  1045. ok = rpc:call(SlaveNode1, syn, start, []),
  1046. ok = rpc:call(SlaveNode2, syn, start, []),
  1047. timer:sleep(100),
  1048. %% start processes
  1049. Pid0 = syn_test_suite_helper:start_process(),
  1050. Pid1 = syn_test_suite_helper:start_process(SlaveNode1),
  1051. Pid2 = syn_test_suite_helper:start_process(SlaveNode2),
  1052. timer:sleep(100),
  1053. %% register on 0
  1054. ok = syn:register(CommonName, Pid0, node()),
  1055. timer:sleep(250),
  1056. %% check
  1057. Node = node(),
  1058. {Pid0, Node} = syn:whereis(CommonName, with_meta),
  1059. {Pid0, Node} = rpc:call(SlaveNode1, syn, whereis, [CommonName, with_meta]),
  1060. {Pid0, Node} = rpc:call(SlaveNode2, syn, whereis, [CommonName, with_meta]),
  1061. %% simulate unregistration with inconsistent data
  1062. syn_registry:sync_unregister(SlaveNode1, Pid1, CommonName),
  1063. timer:sleep(250),
  1064. %% check
  1065. Node = node(),
  1066. {Pid0, Node} = syn:whereis(CommonName, with_meta),
  1067. {Pid0, Node} = rpc:call(SlaveNode1, syn, whereis, [CommonName, with_meta]),
  1068. {Pid0, Node} = rpc:call(SlaveNode2, syn, whereis, [CommonName, with_meta]).
  1069. %% ===================================================================
  1070. %% Internal
  1071. %% ===================================================================
  1072. start_syn_delayed_and_register_local_process(Name, Pid, Ms) ->
  1073. spawn(fun() ->
  1074. lists:foreach(fun(Node) ->
  1075. syn_test_suite_helper:disconnect_node(Node)
  1076. end, nodes()),
  1077. timer:sleep(Ms),
  1078. [] = nodes(),
  1079. %%
  1080. syn:start(),
  1081. ok = syn:register(Name, Pid, node())
  1082. end).
  1083. start_syn_delayed_with_custom_handler_register_local_process(Name, Pid, Meta, Ms) ->
  1084. spawn(fun() ->
  1085. lists:foreach(fun(Node) ->
  1086. syn_test_suite_helper:disconnect_node(Node)
  1087. end, nodes()),
  1088. timer:sleep(Ms),
  1089. [] = nodes(),
  1090. %% use customer handler
  1091. syn_test_suite_helper:use_custom_handler(),
  1092. %%
  1093. syn:start(),
  1094. ok = syn:register(Name, Pid, Meta)
  1095. end).
  1096. seq_unregister_register(Name, Pid, Meta) ->
  1097. syn:unregister(Name),
  1098. syn:register(Name, Pid, Meta).