syn_groups_SUITE.erl 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533
  1. %% ==========================================================================================================
  2. %% Syn - A global Process Registry and Process Group manager.
  3. %%
  4. %% The MIT License (MIT)
  5. %%
  6. %% Copyright (c) 2016 Roberto Ostinelli <roberto@ostinelli.net> and Neato Robotics, Inc.
  7. %%
  8. %% Permission is hereby granted, free of charge, to any person obtaining a copy
  9. %% of this software and associated documentation files (the "Software"), to deal
  10. %% in the Software without restriction, including without limitation the rights
  11. %% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  12. %% copies of the Software, and to permit persons to whom the Software is
  13. %% furnished to do so, subject to the following conditions:
  14. %%
  15. %% The above copyright notice and this permission notice shall be included in
  16. %% all copies or substantial portions of the Software.
  17. %%
  18. %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19. %% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20. %% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21. %% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22. %% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  23. %% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  24. %% THE SOFTWARE.
  25. %% ==========================================================================================================
  26. -module(syn_groups_SUITE).
  27. %% callbacks
  28. -export([all/0]).
  29. -export([init_per_suite/1, end_per_suite/1]).
  30. -export([groups/0, init_per_group/2, end_per_group/2]).
  31. -export([init_per_testcase/2, end_per_testcase/2]).
  32. %% tests
  33. -export([
  34. single_node_join/1,
  35. single_node_leave/1,
  36. single_node_kill/1,
  37. single_node_publish/1,
  38. single_node_multi_call/1,
  39. single_node_multi_call_when_recipient_crashes/1,
  40. single_node_meta/1,
  41. single_node_callback_on_process_exit/1
  42. ]).
  43. -export([
  44. two_nodes_kill/1,
  45. two_nodes_publish/1,
  46. two_nodes_multi_call/1
  47. ]).
  48. %% internals
  49. -export([recipient_loop/1]).
  50. -export([called_loop/1, called_loop_that_crashes/1]).
  51. -export([process_groups_process_exit_callback_dummy/4]).
  52. %% include
  53. -include_lib("common_test/include/ct.hrl").
  54. %% ===================================================================
  55. %% Callbacks
  56. %% ===================================================================
  57. %% -------------------------------------------------------------------
  58. %% Function: all() -> GroupsAndTestCases | {skip,Reason}
  59. %% GroupsAndTestCases = [{group,GroupName} | TestCase]
  60. %% GroupName = atom()
  61. %% TestCase = atom()
  62. %% Reason = term()
  63. %% -------------------------------------------------------------------
  64. all() ->
  65. [
  66. {group, single_node_process_groups},
  67. {group, two_nodes_process_groups}
  68. ].
  69. %% -------------------------------------------------------------------
  70. %% Function: groups() -> [Group]
  71. %% Group = {GroupName,Properties,GroupsAndTestCases}
  72. %% GroupName = atom()
  73. %% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
  74. %% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
  75. %% TestCase = atom()
  76. %% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
  77. %% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
  78. %% repeat_until_any_ok | repeat_until_any_fail
  79. %% N = integer() | forever
  80. %% -------------------------------------------------------------------
  81. groups() ->
  82. [
  83. {single_node_process_groups, [shuffle], [
  84. single_node_join,
  85. single_node_leave,
  86. single_node_kill,
  87. single_node_publish,
  88. single_node_multi_call,
  89. single_node_multi_call_when_recipient_crashes,
  90. single_node_meta,
  91. single_node_callback_on_process_exit
  92. ]},
  93. {two_nodes_process_groups, [shuffle], [
  94. two_nodes_kill,
  95. two_nodes_publish,
  96. two_nodes_multi_call
  97. ]}
  98. ].
  99. %% -------------------------------------------------------------------
  100. %% Function: init_per_suite(Config0) ->
  101. %% Config1 | {skip,Reason} |
  102. %% {skip_and_save,Reason,Config1}
  103. %% Config0 = Config1 = [tuple()]
  104. %% Reason = term()
  105. %% -------------------------------------------------------------------
  106. init_per_suite(Config) ->
  107. %% config
  108. [
  109. {slave_node_short_name, syn_slave}
  110. | Config
  111. ].
  112. %% -------------------------------------------------------------------
  113. %% Function: end_per_suite(Config0) -> void() | {save_config,Config1}
  114. %% Config0 = Config1 = [tuple()]
  115. %% -------------------------------------------------------------------
  116. end_per_suite(_Config) -> ok.
  117. %% -------------------------------------------------------------------
  118. %% Function: init_per_group(GroupName, Config0) ->
  119. %% Config1 | {skip,Reason} |
  120. %% {skip_and_save,Reason,Config1}
  121. %% GroupName = atom()
  122. %% Config0 = Config1 = [tuple()]
  123. %% Reason = term()
  124. %% -------------------------------------------------------------------
  125. init_per_group(two_nodes_process_groups, Config) ->
  126. %% start slave
  127. SlaveNodeShortName = proplists:get_value(slave_node_short_name, Config),
  128. {ok, SlaveNode} = syn_test_suite_helper:start_slave(SlaveNodeShortName),
  129. %% config
  130. [
  131. {slave_node, SlaveNode}
  132. | Config
  133. ];
  134. init_per_group(_GroupName, Config) -> Config.
  135. %% -------------------------------------------------------------------
  136. %% Function: end_per_group(GroupName, Config0) ->
  137. %% void() | {save_config,Config1}
  138. %% GroupName = atom()
  139. %% Config0 = Config1 = [tuple()]
  140. %% -------------------------------------------------------------------
  141. end_per_group(two_nodes_process_groups, Config) ->
  142. %% get slave node name
  143. SlaveNodeShortName = proplists:get_value(slave_node_short_name, Config),
  144. %% stop slave
  145. syn_test_suite_helper:stop_slave(SlaveNodeShortName);
  146. end_per_group(_GroupName, _Config) ->
  147. ok.
  148. % ----------------------------------------------------------------------------------------------------------
  149. % Function: init_per_testcase(TestCase, Config0) ->
  150. % Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
  151. % TestCase = atom()
  152. % Config0 = Config1 = [tuple()]
  153. % Reason = term()
  154. % ----------------------------------------------------------------------------------------------------------
  155. init_per_testcase(_TestCase, Config) ->
  156. Config.
  157. % ----------------------------------------------------------------------------------------------------------
  158. % Function: end_per_testcase(TestCase, Config0) ->
  159. % void() | {save_config,Config1} | {fail,Reason}
  160. % TestCase = atom()
  161. % Config0 = Config1 = [tuple()]
  162. % Reason = term()
  163. % ----------------------------------------------------------------------------------------------------------
  164. end_per_testcase(_TestCase, Config) ->
  165. %% get slave
  166. SlaveNode = proplists:get_value(slave_node, Config),
  167. syn_test_suite_helper:clean_after_test(SlaveNode).
  168. %% ===================================================================
  169. %% Tests
  170. %% ===================================================================
  171. single_node_join(_Config) ->
  172. %% set schema location
  173. application:set_env(mnesia, schema_location, ram),
  174. %% start
  175. ok = syn:start(),
  176. ok = syn:init(),
  177. %% start process
  178. Pid = syn_test_suite_helper:start_process(),
  179. %% retrieve
  180. [] = syn:get_members(<<"my group">>),
  181. false = syn:member(Pid, <<"my group">>),
  182. %% join
  183. ok = syn:join(<<"my group">>, Pid),
  184. %% allow to rejoin
  185. ok = syn:join(<<"my group">>, Pid),
  186. %% retrieve
  187. [Pid] = syn:get_members(<<"my group">>),
  188. true = syn:member(Pid, <<"my group">>),
  189. %% kill process
  190. syn_test_suite_helper:kill_process(Pid).
  191. single_node_leave(_Config) ->
  192. %% set schema location
  193. application:set_env(mnesia, schema_location, ram),
  194. %% start
  195. ok = syn:start(),
  196. ok = syn:init(),
  197. %% start process
  198. Pid = syn_test_suite_helper:start_process(),
  199. %% retrieve
  200. [] = syn:get_members(<<"my group">>),
  201. false = syn:member(Pid, <<"my group">>),
  202. %% join
  203. ok = syn:join(<<"my group">>, Pid),
  204. %% retrieve
  205. [Pid] = syn:get_members(<<"my group">>),
  206. true = syn:member(Pid, <<"my group">>),
  207. %% leave
  208. ok = syn:leave(<<"my group">>, Pid),
  209. %% retrieve
  210. [] = syn:get_members(<<"my group">>),
  211. false = syn:member(Pid, <<"my group">>),
  212. %% kill process
  213. syn_test_suite_helper:kill_process(Pid).
  214. single_node_kill(_Config) ->
  215. %% set schema location
  216. application:set_env(mnesia, schema_location, ram),
  217. %% start
  218. ok = syn:start(),
  219. ok = syn:init(),
  220. %% start process
  221. Pid = syn_test_suite_helper:start_process(),
  222. %% retrieve
  223. [] = syn:get_members(<<"my group 1">>),
  224. [] = syn:get_members(<<"my group 2">>),
  225. false = syn:member(Pid, <<"my group 1">>),
  226. false = syn:member(Pid, <<"my group 2">>),
  227. %% join
  228. ok = syn:join(<<"my group 1">>, Pid),
  229. ok = syn:join(<<"my group 2">>, Pid),
  230. %% retrieve
  231. [Pid] = syn:get_members(<<"my group 1">>),
  232. [Pid] = syn:get_members(<<"my group 2">>),
  233. true = syn:member(Pid, <<"my group 1">>),
  234. true = syn:member(Pid, <<"my group 2">>),
  235. %% kill process
  236. syn_test_suite_helper:kill_process(Pid),
  237. timer:sleep(100),
  238. %% retrieve
  239. [] = syn:get_members(<<"my group 1">>),
  240. [] = syn:get_members(<<"my group 2">>),
  241. false = syn:member(Pid, <<"my group 1">>),
  242. false = syn:member(Pid, <<"my group 2">>).
  243. single_node_publish(_Config) ->
  244. %% set schema location
  245. application:set_env(mnesia, schema_location, ram),
  246. %% start
  247. ok = syn:start(),
  248. ok = syn:init(),
  249. %% start processes
  250. ResultPid = self(),
  251. F = fun() -> recipient_loop(ResultPid) end,
  252. Pid1 = syn_test_suite_helper:start_process(F),
  253. Pid2 = syn_test_suite_helper:start_process(F),
  254. %% join
  255. ok = syn:join(<<"my group">>, Pid1),
  256. ok = syn:join(<<"my group">>, Pid2),
  257. %% publish
  258. syn:publish(<<"my group">>, {test, message}),
  259. %% check publish was received
  260. receive
  261. {received, Pid1, {test, message}} -> ok
  262. after 2000 ->
  263. ok = published_message_was_not_received_by_pid1
  264. end,
  265. receive
  266. {received, Pid2, {test, message}} -> ok
  267. after 2000 ->
  268. ok = published_message_was_not_received_by_pid2
  269. end,
  270. %% kill processes
  271. syn_test_suite_helper:kill_process(Pid1),
  272. syn_test_suite_helper:kill_process(Pid2).
  273. single_node_multi_call(_Config) ->
  274. %% set schema location
  275. application:set_env(mnesia, schema_location, ram),
  276. %% start
  277. ok = syn:start(),
  278. ok = syn:init(),
  279. %% start processes
  280. Pid1 = syn_test_suite_helper:start_process(fun() -> called_loop(pid1) end),
  281. Pid2 = syn_test_suite_helper:start_process(fun() -> called_loop(pid2) end),
  282. PidUnresponsive = syn_test_suite_helper:start_process(),
  283. %% register
  284. ok = syn:join(<<"my group">>, Pid1),
  285. ok = syn:join(<<"my group">>, Pid2),
  286. ok = syn:join(<<"my group">>, PidUnresponsive),
  287. %% call
  288. {Replies, BadPids} = syn:multi_call(<<"my group">>, get_pid_name),
  289. %% check responses
  290. 2 = length(Replies),
  291. pid1 = proplists:get_value(Pid1, Replies),
  292. pid2 = proplists:get_value(Pid2, Replies),
  293. [PidUnresponsive] = BadPids,
  294. %% kill processes
  295. syn_test_suite_helper:kill_process(Pid1),
  296. syn_test_suite_helper:kill_process(Pid2),
  297. syn_test_suite_helper:kill_process(PidUnresponsive).
  298. single_node_multi_call_when_recipient_crashes(_Config) ->
  299. %% set schema location
  300. application:set_env(mnesia, schema_location, ram),
  301. %% start
  302. ok = syn:start(),
  303. ok = syn:init(),
  304. %% start processes
  305. Pid1 = syn_test_suite_helper:start_process(fun() -> called_loop(pid1) end),
  306. Pid2 = syn_test_suite_helper:start_process(fun() -> called_loop(pid2) end),
  307. PidCrashes = syn_test_suite_helper:start_process(fun() -> called_loop_that_crashes(pid_crashes) end),
  308. %% register
  309. ok = syn:join(<<"my group">>, Pid1),
  310. ok = syn:join(<<"my group">>, Pid2),
  311. ok = syn:join(<<"my group">>, PidCrashes),
  312. %% call
  313. {Time, {Replies, BadPids}} = timer:tc(syn, multi_call, [<<"my group">>, get_pid_name]),
  314. %% check that pid2 was monitored, no need to wait for timeout
  315. true = Time / 1000 < 1000,
  316. %% check responses
  317. 2 = length(Replies),
  318. pid1 = proplists:get_value(Pid1, Replies),
  319. pid2 = proplists:get_value(Pid2, Replies),
  320. [PidCrashes] = BadPids,
  321. %% kill processes
  322. syn_test_suite_helper:kill_process(Pid1),
  323. syn_test_suite_helper:kill_process(Pid2).
  324. single_node_meta(_Config) ->
  325. %% set schema location
  326. application:set_env(mnesia, schema_location, ram),
  327. %% start
  328. ok = syn:start(),
  329. ok = syn:init(),
  330. %% start process
  331. Pid = syn_test_suite_helper:start_process(),
  332. %% retrieve
  333. [] = syn:get_members(<<"my group">>, with_meta),
  334. false = syn:member(Pid, <<"my group">>),
  335. %% join
  336. ok = syn:join(<<"my group">>, Pid, {some, meta}),
  337. %% retrieve
  338. [{Pid, {some, meta}}] = syn:get_members(<<"my group">>, with_meta),
  339. %% allow to rejoin to update meta
  340. ok = syn:join(<<"my group">>, Pid, {updated, meta}),
  341. %% retrieve
  342. [{Pid, {updated, meta}}] = syn:get_members(<<"my group">>, with_meta),
  343. %% leave
  344. ok = syn:leave(<<"my group">>, Pid),
  345. %% retrieve
  346. [] = syn:get_members(<<"my group">>),
  347. false = syn:member(Pid, <<"my group">>),
  348. %% kill process
  349. syn_test_suite_helper:kill_process(Pid).
  350. single_node_callback_on_process_exit(_Config) ->
  351. CurrentNode = node(),
  352. %% set schema location
  353. application:set_env(mnesia, schema_location, ram),
  354. %% load configuration variables from syn-test.config => this defines the callback
  355. syn_test_suite_helper:set_environment_variables(),
  356. %% start
  357. ok = syn:start(),
  358. ok = syn:init(),
  359. %% register global process
  360. ResultPid = self(),
  361. global:register_name(syn_process_groups_SUITE_result, ResultPid),
  362. %% start process
  363. Pid = syn_test_suite_helper:start_process(),
  364. %% register
  365. ok = syn:join(<<"my group">>, Pid, {some, meta, 1}),
  366. ok = syn:join(<<"my other group">>, Pid, {some, meta, 2}),
  367. %% kill process
  368. syn_test_suite_helper:kill_process(Pid),
  369. %% check callback were triggered
  370. receive
  371. {exited, CurrentNode, <<"my group">>, Pid, {some, meta, 1}, killed} -> ok
  372. after 2000 ->
  373. ok = process_groups_exit_callback_was_not_called_from_local_node
  374. end,
  375. receive
  376. {exited, CurrentNode, <<"my other group">>, Pid, {some, meta, 2}, killed} -> ok
  377. after 2000 ->
  378. ok = process_groups_exit_callback_was_not_called_from_local_node
  379. end,
  380. %% unregister
  381. global:unregister_name(syn_process_groups_SUITE_result).
  382. two_nodes_kill(Config) ->
  383. %% get slave
  384. SlaveNode = proplists:get_value(slave_node, Config),
  385. %% set schema location
  386. application:set_env(mnesia, schema_location, ram),
  387. rpc:call(SlaveNode, mnesia, schema_location, [ram]),
  388. %% start
  389. ok = syn:start(),
  390. ok = syn:init(),
  391. ok = rpc:call(SlaveNode, syn, start, []),
  392. ok = rpc:call(SlaveNode, syn, init, []),
  393. timer:sleep(100),
  394. %% start processes
  395. PidLocal = syn_test_suite_helper:start_process(),
  396. PidSlave = syn_test_suite_helper:start_process(SlaveNode),
  397. %% retrieve
  398. [] = syn:get_members(<<"my group">>),
  399. false = syn:member(PidLocal, <<"my group">>),
  400. false = syn:member(PidSlave, <<"my group">>),
  401. [] = rpc:call(SlaveNode, syn, get_members, [<<"my group">>]),
  402. false = rpc:call(SlaveNode, syn, member, [PidLocal, <<"my group">>]),
  403. false = rpc:call(SlaveNode, syn, member, [PidSlave, <<"my group">>]),
  404. %% register
  405. ok = syn:join(<<"my group">>, PidSlave),
  406. ok = rpc:call(SlaveNode, syn, join, [<<"my group">>, PidLocal]),
  407. %% retrieve, pid should have the same order in all nodes
  408. [PidSlave, PidLocal] = syn:get_members(<<"my group">>),
  409. [PidSlave, PidLocal] = rpc:call(SlaveNode, syn, get_members, [<<"my group">>]),
  410. %% kill processes
  411. syn_test_suite_helper:kill_process(PidLocal),
  412. syn_test_suite_helper:kill_process(PidSlave),
  413. timer:sleep(100),
  414. %% retrieve
  415. [] = syn:get_members(<<"my group">>),
  416. false = syn:member(PidLocal, <<"my group">>),
  417. false = syn:member(PidSlave, <<"my group">>),
  418. [] = rpc:call(SlaveNode, syn, get_members, [<<"my group">>]),
  419. false = rpc:call(SlaveNode, syn, member, [PidLocal, <<"my group">>]),
  420. false = rpc:call(SlaveNode, syn, member, [PidSlave, <<"my group">>]).
  421. two_nodes_publish(Config) ->
  422. %% get slave
  423. SlaveNode = proplists:get_value(slave_node, Config),
  424. %% set schema location
  425. application:set_env(mnesia, schema_location, ram),
  426. rpc:call(SlaveNode, mnesia, schema_location, [ram]),
  427. %% start
  428. ok = syn:start(),
  429. ok = syn:init(),
  430. ok = rpc:call(SlaveNode, syn, start, []),
  431. ok = rpc:call(SlaveNode, syn, init, []),
  432. timer:sleep(100),
  433. %% start process
  434. ResultPid = self(),
  435. F = fun() -> recipient_loop(ResultPid) end,
  436. PidLocal = syn_test_suite_helper:start_process(F),
  437. PidSlave = syn_test_suite_helper:start_process(SlaveNode, F),
  438. %% register
  439. ok = syn:join(<<"my group">>, PidSlave),
  440. ok = rpc:call(SlaveNode, syn, join, [<<"my group">>, PidLocal]),
  441. %% publish
  442. syn:publish(<<"my group">>, {test, message}),
  443. %% check publish was received
  444. receive
  445. {received, PidLocal, {test, message}} -> ok
  446. after 2000 ->
  447. ok = published_message_was_not_received_by_pidlocal
  448. end,
  449. receive
  450. {received, PidSlave, {test, message}} -> ok
  451. after 2000 ->
  452. ok = published_message_was_not_received_by_pidslave
  453. end,
  454. %% kill processes
  455. syn_test_suite_helper:kill_process(PidLocal),
  456. syn_test_suite_helper:kill_process(PidSlave).
  457. two_nodes_multi_call(Config) ->
  458. %% get slave
  459. SlaveNode = proplists:get_value(slave_node, Config),
  460. %% set schema location
  461. application:set_env(mnesia, schema_location, ram),
  462. rpc:call(SlaveNode, mnesia, schema_location, [ram]),
  463. %% start
  464. ok = syn:start(),
  465. ok = syn:init(),
  466. ok = rpc:call(SlaveNode, syn, start, []),
  467. ok = rpc:call(SlaveNode, syn, init, []),
  468. timer:sleep(100),
  469. %% start processes
  470. PidLocal = syn_test_suite_helper:start_process(fun() -> called_loop(pid1) end),
  471. PidSlave = syn_test_suite_helper:start_process(SlaveNode, fun() -> called_loop(pid2) end),
  472. PidUnresponsive = syn_test_suite_helper:start_process(),
  473. %% register
  474. ok = syn:join(<<"my group">>, PidLocal),
  475. ok = syn:join(<<"my group">>, PidSlave),
  476. ok = syn:join(<<"my group">>, PidUnresponsive),
  477. timer:sleep(100),
  478. %% call
  479. {Replies, BadPids} = syn:multi_call(<<"my group">>, get_pid_name, 3000),
  480. %% check responses
  481. 2 = length(Replies),
  482. pid1 = proplists:get_value(PidLocal, Replies),
  483. pid2 = proplists:get_value(PidSlave, Replies),
  484. [PidUnresponsive] = BadPids,
  485. %% kill processes
  486. syn_test_suite_helper:kill_process(PidLocal),
  487. syn_test_suite_helper:kill_process(PidSlave),
  488. syn_test_suite_helper:kill_process(PidUnresponsive).
  489. %% ===================================================================
  490. %% Internal
  491. %% ===================================================================
  492. recipient_loop(Pid) ->
  493. receive
  494. Message -> Pid ! {received, self(), Message}
  495. end.
  496. called_loop(PidName) ->
  497. receive
  498. {syn_multi_call, CallerPid, get_pid_name} -> syn:multi_call_reply(CallerPid, PidName)
  499. end.
  500. called_loop_that_crashes(_PidName) ->
  501. receive
  502. {syn_multi_call, _CallerPid, get_pid_name} -> exit(recipient_crashed_on_purpose)
  503. end.
  504. process_groups_process_exit_callback_dummy(Name, Pid, Meta, Reason) ->
  505. global:send(syn_process_groups_SUITE_result, {exited, node(), Name, Pid, Meta, Reason}).