1: %% 
    2: %% %CopyrightBegin%
    3: %%
    4: %% Copyright Ericsson AB 2008-2011. All Rights Reserved.
    5: %%
    6: %% The contents of this file are subject to the Erlang Public License,
    7: %% Version 1.1, (the "License"); you may not use this file except in
    8: %% compliance with the License. You should have received a copy of the
    9: %% Erlang Public License along with this software. If not, it can be
   10: %% retrieved online at http://www.erlang.org/.
   11: %%
   12: %% Software distributed under the License is distributed on an "AS IS"
   13: %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
   14: %% the License for the specific language governing rights and limitations
   15: %% under the License.
   16: %%
   17: %% %CopyrightEnd%
   18: %%
   19: 
   20: 
   21: %%%-------------------------------------------------------------------
   22: %%% File    : scheduler_SUITE.erl
   23: %%% Author  : Rickard Green
   24: %%% Description : 
   25: %%%
   26: %%% Created : 27 Oct 2008 by Rickard Green
   27: %%%-------------------------------------------------------------------
   28: -module(scheduler_SUITE).
   29: 
   30: 
   31: %-define(line_trace, 1).
   32: 
   33: -include_lib("test_server/include/test_server.hrl").
   34: 
   35: %-compile(export_all).
   36: -export([all/0, suite/0,groups/0,init_per_suite/1, 
   37: 	 init_per_group/2,end_per_group/2, 
   38: 	 init_per_testcase/2, end_per_testcase/2, end_per_suite/1]).
   39: 
   40: -export([equal/1,
   41: 	 few_low/1,
   42: 	 many_low/1,
   43: 	 equal_with_part_time_high/1,
   44: 	 equal_with_part_time_max/1,
   45: 	 equal_and_high_with_part_time_max/1,
   46: 	 equal_with_high/1,
   47: 	 equal_with_high_max/1,
   48: 	 bound_process/1,
   49: 	
   50: 	 scheduler_bind_types/1,
   51: 	 cpu_topology/1,
   52: 	 update_cpu_info/1,
   53: 	 sct_cmd/1,
   54: 	 sbt_cmd/1,
   55: 	 scheduler_threads/1,
   56: 	 scheduler_suspend/1,
   57: 	 reader_groups/1]).
   58: 
   59: -define(DEFAULT_TIMEOUT, ?t:minutes(15)).
   60: 
   61: -define(MIN_SCHEDULER_TEST_TIMEOUT, ?t:minutes(1)).
   62: 
   63: suite() -> [{ct_hooks,[ts_install_cth]}].
   64: 
   65: all() -> 
   66:     [equal, few_low, many_low, equal_with_part_time_high,
   67:      equal_with_part_time_max,
   68:      equal_and_high_with_part_time_max, equal_with_high,
   69:      equal_with_high_max, bound_process,
   70:      {group, scheduler_bind}, scheduler_threads, scheduler_suspend,
   71:      reader_groups].
   72: 
   73: groups() -> 
   74:     [{scheduler_bind, [],
   75:       [scheduler_bind_types, cpu_topology, update_cpu_info,
   76:        sct_cmd, sbt_cmd]}].
   77: 
   78: init_per_suite(Config) ->
   79:     Config.
   80: 
   81: end_per_suite(Config) ->
   82:     catch erts_debug:set_internal_state(available_internal_state, false),
   83:     Config.
   84: 
   85: init_per_group(_GroupName, Config) ->
   86:     Config.
   87: 
   88: end_per_group(_GroupName, Config) ->
   89:     Config.
   90: 
   91: init_per_testcase(update_cpu_info, Config) ->
   92:     case os:find_executable("taskset") of
   93: 	false ->
   94: 	    {skip,"Could not find 'taskset' in path"};
   95: 	_ ->
   96: 	    init_per_tc(update_cpu_info, Config)
   97:     end;
   98: init_per_testcase(Case, Config) when is_list(Config) ->
   99:     init_per_tc(Case, Config).
  100: 
  101: init_per_tc(Case, Config) ->
  102:     Dog = ?t:timetrap(?DEFAULT_TIMEOUT),
  103:     process_flag(priority, max),
  104:     erlang:display({'------------', ?MODULE, Case, '------------'}),
  105:     OkRes = ok,
  106:     [{watchdog, Dog}, {testcase, Case}, {ok_res, OkRes} |Config].
  107: 
  108: end_per_testcase(_Case, Config) when is_list(Config) ->
  109:     Dog = ?config(watchdog, Config),
  110:     ?t:timetrap_cancel(Dog),
  111:     ok.
  112: 
  113: -define(ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED, (2000*2000)).
  114: -define(DEFAULT_TEST_REDS_PER_SCHED, 200000000).
  115: 
  116: %%
  117: %% Test cases
  118: %%
  119: 
  120: equal(Config) when is_list(Config) ->
  121:     low_normal_test(Config, 500, 500).
  122: 
  123: few_low(Config) when is_list(Config) ->
  124:     low_normal_test(Config, 1000, 2*active_schedulers()).
  125: 
  126: many_low(Config) when is_list(Config) ->
  127:     low_normal_test(Config, 2*active_schedulers(), 1000).
  128: 
  129: low_normal_test(Config, NW, LW) ->
  130:     ?line Tracer = start_tracer(),
  131:     ?line Low = workers(LW, low),
  132:     ?line Normal = workers(NW, normal),
  133:     ?line Res = do_it(Tracer, Low, Normal, [], []),
  134:     ?line chk_result(Res, LW, NW, 0, 0, true, false, false),
  135:     ?line workers_exit([Low, Normal]),
  136:     ?line ok(Res, Config).
  137: 
  138: equal_with_part_time_high(Config) when is_list(Config) ->
  139:     ?line NW = 500,
  140:     ?line LW = 500,
  141:     ?line HW = 1,
  142:     ?line Tracer = start_tracer(),
  143:     ?line Normal = workers(NW, normal),
  144:     ?line Low = workers(LW, low),
  145:     ?line High = part_time_workers(HW, high),
  146:     ?line Res = do_it(Tracer, Low, Normal, High, []),
  147:     ?line chk_result(Res, LW, NW, HW, 0, true, true, false),
  148:     ?line workers_exit([Low, Normal, High]),
  149:     ?line ok(Res, Config).
  150: 
  151: equal_and_high_with_part_time_max(Config) when is_list(Config) ->
  152:     ?line NW = 500,
  153:     ?line LW = 500,
  154:     ?line HW = 500,
  155:     ?line MW = 1,
  156:     ?line Tracer = start_tracer(),
  157:     ?line Low = workers(LW, low),
  158:     ?line Normal = workers(NW, normal),
  159:     ?line High = workers(HW, high),
  160:     ?line Max = part_time_workers(MW, max),
  161:     ?line Res = do_it(Tracer, Low, Normal, High, Max),
  162:     ?line chk_result(Res, LW, NW, HW, MW, false, true, true),
  163:     ?line workers_exit([Low, Normal, Max]),
  164:     ?line ok(Res, Config).
  165: 
  166: equal_with_part_time_max(Config) when is_list(Config) ->
  167:     ?line NW = 500,
  168:     ?line LW = 500,
  169:     ?line MW = 1,
  170:     ?line Tracer = start_tracer(),
  171:     ?line Low = workers(LW, low),
  172:     ?line Normal = workers(NW, normal),
  173:     ?line Max = part_time_workers(MW, max),
  174:     ?line Res = do_it(Tracer, Low, Normal, [], Max),
  175:     ?line chk_result(Res, LW, NW, 0, MW, true, false, true),
  176:     ?line workers_exit([Low, Normal, Max]),
  177:     ?line ok(Res, Config).
  178: 
  179: equal_with_high(Config) when is_list(Config) ->
  180:     ?line NW = 500,
  181:     ?line LW = 500,
  182:     ?line HW = 1,
  183:     ?line Tracer = start_tracer(),
  184:     ?line Low = workers(LW, low),
  185:     ?line Normal = workers(NW, normal),
  186:     ?line High = workers(HW, high),
  187:     ?line Res = do_it(Tracer, Low, Normal, High, []),
  188:     ?line LNExe = case active_schedulers() of
  189: 		      S when S =< HW -> false;
  190: 		      _ -> true
  191: 		  end,
  192:     ?line chk_result(Res, LW, NW, HW, 0, LNExe, true, false),
  193:     ?line workers_exit([Low, Normal, High]),
  194:     ?line ok(Res, Config).
  195: 
  196: equal_with_high_max(Config) when is_list(Config) ->
  197:     ?line NW = 500,
  198:     ?line LW = 500,
  199:     ?line HW = 1,
  200:     ?line MW = 1,
  201:     ?line Tracer = start_tracer(),
  202:     ?line Normal = workers(NW, normal),
  203:     ?line Low = workers(LW, low),
  204:     ?line High = workers(HW, high),
  205:     ?line Max = workers(MW, max),
  206:     ?line Res = do_it(Tracer, Low, Normal, High, Max),
  207:     ?line {LNExe, HExe} = case active_schedulers() of
  208: 			      S when S =< MW -> {false, false};
  209: 			      S when S =< (MW + HW) -> {false, true};
  210: 			      _ -> {true, true}
  211: 			  end,
  212:     ?line chk_result(Res, LW, NW, HW, MW, LNExe, HExe, true),
  213:     ?line workers_exit([Low, Normal, Max]),
  214:     ?line ok(Res, Config).
  215: 
  216: bound_process(Config) when is_list(Config) ->
  217:     case erlang:system_info(run_queues) == erlang:system_info(schedulers) of
  218: 	true ->
  219: 	    ?line NStartBase = 20000,
  220: 	    ?line NStart = case {erlang:system_info(debug_compiled),
  221: 				 erlang:system_info(lock_checking)} of
  222: 			       {true, true} -> NStartBase div 100;
  223: 			       {_, true} -> NStartBase div 10;
  224: 			       _ -> NStartBase
  225: 			   end,
  226: 	    ?line MStart = 100,
  227: 	    ?line Seq = lists:seq(1, 100),
  228: 	    ?line Tester = self(),
  229: 	    ?line Procs = lists:map(
  230: 			    fun (N) when N rem 2 == 0 ->
  231: 				    spawn_opt(fun () ->
  232: 						      bound_loop(NStart,
  233: 								 NStart,
  234: 								 MStart,
  235: 								 1),
  236: 						      Tester ! {self(), done}
  237: 					      end,
  238: 					      [{scheduler, 1}, link]);
  239: 				(_N) ->
  240: 				    spawn_link(fun () ->
  241: 						       bound_loop(NStart,
  242: 								  NStart,
  243: 								  MStart,
  244: 								  false),
  245: 						       Tester ! {self(), done}
  246: 					       end)
  247: 			    end,
  248: 			    Seq),
  249: 	    ?line lists:foreach(fun (P) -> receive {P, done} -> ok end end,
  250: 				Procs),
  251: 	    ?line ok;
  252: 	false ->
  253: 	    {skipped, "Functionality not supported"}
  254:     end.
  255: 
  256: bound_loop(_, 0, 0, _) ->
  257:     ok;
  258: bound_loop(NS, 0, M, false) ->
  259:     bound_loop(NS, NS, M-1, false);
  260: bound_loop(NS, N, M, false) ->
  261:     erlang:system_info(scheduler_id),
  262:     bound_loop(NS, N-1, M, false);
  263: bound_loop(NS, 0, M, Sched) ->
  264:     NewSched = (Sched rem erlang:system_info(schedulers_online)) + 1,
  265:     Sched = process_flag(scheduler, NewSched),
  266:     NewSched = erlang:system_info(scheduler_id),
  267:     bound_loop(NS, NS, M-1, NewSched);
  268: bound_loop(NS, N, M, Sched) ->
  269:     Sched = erlang:system_info(scheduler_id),
  270:     bound_loop(NS, N-1, M, Sched).
  271: 
  272: 
  273: -define(TOPOLOGY_A_CMD,
  274: 	"+sct"
  275: 	"L0-1t0-1c0p0n0"
  276: 	":L2-3t0-1c1p0n0"
  277: 	":L4-5t0-1c0p1n0"
  278: 	":L6-7t0-1c1p1n0"
  279: 	":L8-9t0-1c0p2n1"
  280: 	":L10-11t0-1c1p2n1"
  281: 	":L12-13t0-1c0p3n1"
  282: 	":L14-15t0-1c1p3n1").
  283: 
  284: -define(TOPOLOGY_A_TERM,
  285: 	[{node,[{processor,[{core,[{thread,{logical,0}},
  286: 				   {thread,{logical,1}}]},
  287: 			    {core,[{thread,{logical,2}},
  288: 				   {thread,{logical,3}}]}]},
  289: 		{processor,[{core,[{thread,{logical,4}},
  290: 				   {thread,{logical,5}}]},
  291: 			    {core,[{thread,{logical,6}},
  292: 				   {thread,{logical,7}}]}]}]},
  293: 	 {node,[{processor,[{core,[{thread,{logical,8}},
  294: 				   {thread,{logical,9}}]},
  295: 			    {core,[{thread,{logical,10}},
  296: 				   {thread,{logical,11}}]}]},
  297: 		{processor,[{core,[{thread,{logical,12}},
  298: 				   {thread,{logical,13}}]},
  299: 			    {core,[{thread,{logical,14}},
  300: 				   {thread,{logical,15}}]}]}]}]).
  301: 
  302: -define(TOPOLOGY_B_CMD,
  303: 	"+sct"
  304: 	"L0-1t0-1c0n0p0"
  305: 	":L2-3t0-1c1n0p0"
  306: 	":L4-5t0-1c2n1p0"
  307: 	":L6-7t0-1c3n1p0"
  308: 	":L8-9t0-1c0n2p1"
  309: 	":L10-11t0-1c1n2p1"
  310: 	":L12-13t0-1c2n3p1"
  311: 	":L14-15t0-1c3n3p1").
  312: 
  313: -define(TOPOLOGY_B_TERM,
  314: 	[{processor,[{node,[{core,[{thread,{logical,0}},
  315: 				   {thread,{logical,1}}]},
  316: 			    {core,[{thread,{logical,2}},
  317: 				   {thread,{logical,3}}]}]},
  318: 		     {node,[{core,[{thread,{logical,4}},
  319: 				   {thread,{logical,5}}]},
  320: 			    {core,[{thread,{logical,6}},
  321: 				   {thread,{logical,7}}]}]}]},
  322: 	 {processor,[{node,[{core,[{thread,{logical,8}},
  323: 				   {thread,{logical,9}}]},
  324: 			    {core,[{thread,{logical,10}},
  325: 				   {thread,{logical,11}}]}]},
  326: 		     {node,[{core,[{thread,{logical,12}},
  327: 				   {thread,{logical,13}}]},
  328: 			    {core,[{thread,{logical,14}},
  329: 				   {thread,{logical,15}}]}]}]}]).
  330: 
  331: -define(TOPOLOGY_C_TERM,
  332: 	[{node,[{processor,[{core,[{thread,{logical,0}},
  333: 				   {thread,{logical,1}}]},
  334: 			    {core,[{thread,{logical,2}},
  335: 				   {thread,{logical,3}}]}]},
  336: 		{processor,[{core,[{thread,{logical,4}},
  337: 				   {thread,{logical,5}}]},
  338: 			    {core,[{thread,{logical,6}},
  339: 				   {thread,{logical,7}}]}]}]},
  340: 	 {processor,[{node,[{core,[{thread,{logical,8}},
  341: 				   {thread,{logical,9}}]},
  342: 			    {core,[{thread,{logical,10}},
  343: 				   {thread,{logical,11}}]}]},
  344: 		     {node,[{core,[{thread,{logical,12}},
  345: 				   {thread,{logical,13}}]},
  346: 			    {core,[{thread,{logical,14}},
  347: 				   {thread,{logical,15}}]}]}]},
  348: 	 {node,[{processor,[{core,[{thread,{logical,16}},
  349: 				   {thread,{logical,17}}]},
  350: 			    {core,[{thread,{logical,18}},
  351: 				   {thread,{logical,19}}]}]},
  352: 		{processor,[{core,[{thread,{logical,20}},
  353: 				   {thread,{logical,21}}]},
  354: 			    {core,[{thread,{logical,22}},
  355: 				   {thread,{logical,23}}]}]}]},
  356: 	 {processor,[{node,[{core,[{thread,{logical,24}},
  357: 				   {thread,{logical,25}}]},
  358: 			    {core,[{thread,{logical,26}},
  359: 				   {thread,{logical,27}}]}]},
  360: 		     {node,[{core,[{thread,{logical,28}},
  361: 				   {thread,{logical,29}}]},
  362: 			    {core,[{thread,{logical,30}},
  363: 				   {thread,{logical,31}}]}]}]}]).
  364: 
  365: 
  366: -define(TOPOLOGY_C_CMD,
  367: 	"+sct"
  368: 	"L0-1t0-1c0p0n0"
  369: 	":L2-3t0-1c1p0n0"
  370: 	":L4-5t0-1c0p1n0"
  371: 	":L6-7t0-1c1p1n0"
  372: 	":L8-9t0-1c0n1p2"
  373: 	":L10-11t0-1c1n1p2"
  374: 	":L12-13t0-1c2n2p2"
  375: 	":L14-15t0-1c3n2p2"
  376: 	":L16-17t0-1c0p3n3"
  377: 	":L18-19t0-1c1p3n3"
  378: 	":L20-21t0-1c0p4n3"
  379: 	":L22-23t0-1c1p4n3"
  380: 	":L24-25t0-1c0n4p5"
  381: 	":L26-27t0-1c1n4p5"
  382: 	":L28-29t0-1c2n5p5"
  383: 	":L30-31t0-1c3n5p5").
  384: 
  385: -define(TOPOLOGY_D_TERM,
  386: 	[{processor,[{node,[{core,[{thread,{logical,0}},
  387: 				   {thread,{logical,1}}]},
  388: 			    {core,[{thread,{logical,2}},
  389: 				   {thread,{logical,3}}]}]},
  390: 		     {node,[{core,[{thread,{logical,4}},
  391: 				   {thread,{logical,5}}]},
  392: 			    {core,[{thread,{logical,6}},
  393: 				   {thread,{logical,7}}]}]}]},
  394: 	 {node,[{processor,[{core,[{thread,{logical,8}},
  395: 				   {thread,{logical,9}}]},
  396: 			    {core,[{thread,{logical,10}},
  397: 				   {thread,{logical,11}}]}]},
  398: 		{processor,[{core,[{thread,{logical,12}},
  399: 				   {thread,{logical,13}}]},
  400: 			    {core,[{thread,{logical,14}},
  401: 				   {thread,{logical,15}}]}]}]},
  402: 	 {processor,[{node,[{core,[{thread,{logical,16}},
  403: 				   {thread,{logical,17}}]},
  404: 			    {core,[{thread,{logical,18}},
  405: 				   {thread,{logical,19}}]}]},
  406: 		     {node,[{core,[{thread,{logical,20}},
  407: 				   {thread,{logical,21}}]},
  408: 			    {core,[{thread,{logical,22}},
  409: 				   {thread,{logical,23}}]}]}]},
  410: 	 {node,[{processor,[{core,[{thread,{logical,24}},
  411: 				   {thread,{logical,25}}]},
  412: 			    {core,[{thread,{logical,26}},
  413: 				   {thread,{logical,27}}]}]},
  414: 		{processor,[{core,[{thread,{logical,28}},
  415: 				   {thread,{logical,29}}]},
  416: 			    {core,[{thread,{logical,30}},
  417: 				   {thread,{logical,31}}]}]}]}]).
  418: 
  419: -define(TOPOLOGY_D_CMD,
  420: 	"+sct"
  421: 	"L0-1t0-1c0n0p0"
  422: 	":L2-3t0-1c1n0p0"
  423: 	":L4-5t0-1c2n1p0"
  424: 	":L6-7t0-1c3n1p0"
  425: 	":L8-9t0-1c0p1n2"
  426: 	":L10-11t0-1c1p1n2"
  427: 	":L12-13t0-1c0p2n2"
  428: 	":L14-15t0-1c1p2n2"
  429: 	":L16-17t0-1c0n3p3"
  430: 	":L18-19t0-1c1n3p3"
  431: 	":L20-21t0-1c2n4p3"
  432: 	":L22-23t0-1c3n4p3"
  433: 	":L24-25t0-1c0p4n5"
  434: 	":L26-27t0-1c1p4n5"
  435: 	":L28-29t0-1c0p5n5"
  436: 	":L30-31t0-1c1p5n5").
  437: 
  438: -define(TOPOLOGY_E_CMD,
  439: 	"+sct"
  440: 	"L0-1t0-1c0p0n0"
  441: 	":L2-3t0-1c1p0n0"
  442: 	":L4-5t0-1c2p0n0"
  443: 	":L6-7t0-1c3p0n0"
  444: 	":L8-9t0-1c0p1n1"
  445: 	":L10-11t0-1c1p1n1"
  446: 	":L12-13t0-1c2p1n1"
  447: 	":L14-15t0-1c3p1n1").
  448: 
  449: -define(TOPOLOGY_E_TERM,
  450: 	[{node,[{processor,[{core,[{thread,{logical,0}},
  451: 				   {thread,{logical,1}}]},
  452: 			    {core,[{thread,{logical,2}},
  453: 				   {thread,{logical,3}}]},
  454: 			    {core,[{thread,{logical,4}},
  455: 				   {thread,{logical,5}}]},
  456: 			    {core,[{thread,{logical,6}},
  457: 				   {thread,{logical,7}}]}]}]},
  458: 	 {node,[{processor,[{core,[{thread,{logical,8}},
  459: 				   {thread,{logical,9}}]},
  460: 			    {core,[{thread,{logical,10}},
  461: 				   {thread,{logical,11}}]},
  462: 			    {core,[{thread,{logical,12}},
  463: 				   {thread,{logical,13}}]},
  464: 			    {core,[{thread,{logical,14}},
  465: 				   {thread,{logical,15}}]}]}]}]).
  466: 
  467: -define(TOPOLOGY_F_CMD,
  468: 	"+sct"
  469: 	"L0-1t0-1c0n0p0"
  470: 	":L2-3t0-1c1n0p0"
  471: 	":L4-5t0-1c2n0p0"
  472: 	":L6-7t0-1c3n0p0"
  473: 	":L8-9t0-1c4n1p0"
  474: 	":L10-11t0-1c5n1p0"
  475: 	":L12-13t0-1c6n1p0"
  476: 	":L14-15t0-1c7n1p0"
  477: 	":L16-17t0-1c8n2p0"
  478: 	":L18-19t0-1c9n2p0"
  479: 	":L20-21t0-1c10n2p0"
  480: 	":L22-23t0-1c11n2p0"
  481: 	":L24-25t0-1c12n3p0"
  482: 	":L26-27t0-1c13n3p0"
  483: 	":L28-29t0-1c14n3p0"
  484: 	":L30-31t0-1c15n3p0").
  485: 
  486: -define(TOPOLOGY_F_TERM,
  487: 	[{processor,[{node,[{core,[{thread,{logical,0}},
  488: 				   {thread,{logical,1}}]},
  489: 			    {core,[{thread,{logical,2}},
  490: 				   {thread,{logical,3}}]},
  491: 			    {core,[{thread,{logical,4}},
  492: 				   {thread,{logical,5}}]},
  493: 			    {core,[{thread,{logical,6}},
  494: 				   {thread,{logical,7}}]}]},
  495: 		     {node,[{core,[{thread,{logical,8}},
  496: 				   {thread,{logical,9}}]},
  497: 			    {core,[{thread,{logical,10}},
  498: 				   {thread,{logical,11}}]},
  499: 			    {core,[{thread,{logical,12}},
  500: 				   {thread,{logical,13}}]},
  501: 			    {core,[{thread,{logical,14}},
  502: 				   {thread,{logical,15}}]}]},
  503: 		     {node,[{core,[{thread,{logical,16}},
  504: 				   {thread,{logical,17}}]},
  505: 			    {core,[{thread,{logical,18}},
  506: 				   {thread,{logical,19}}]},
  507: 			    {core,[{thread,{logical,20}},
  508: 				   {thread,{logical,21}}]},
  509: 			    {core,[{thread,{logical,22}},
  510: 				   {thread,{logical,23}}]}]},
  511: 		     {node,[{core,[{thread,{logical,24}},
  512: 				   {thread,{logical,25}}]},
  513: 			    {core,[{thread,{logical,26}},
  514: 				   {thread,{logical,27}}]},
  515: 			    {core,[{thread,{logical,28}},
  516: 				   {thread,{logical,29}}]},
  517: 			    {core,[{thread,{logical,30}},
  518: 				   {thread,{logical,31}}]}]}]}]).
  519: 
  520: bindings(Node, BindType) ->
  521:     Parent = self(),
  522:     Ref = make_ref(),
  523:     Pid = spawn_link(Node,
  524: 		     fun () ->
  525: 			     enable_internal_state(),
  526: 			     Res = (catch erts_debug:get_internal_state(
  527: 					    {fake_scheduler_bindings,
  528: 					     BindType})),
  529: 			     Parent ! {Ref, Res}
  530: 		     end),
  531:     receive
  532: 	{Ref, Res} ->
  533: 	    ?t:format("~p: ~p~n", [BindType, Res]),
  534: 	    unlink(Pid),
  535: 	    Res
  536:     end.
  537: 
  538: scheduler_bind_types(Config) when is_list(Config) ->
  539:     ?line OldRelFlags = clear_erl_rel_flags(),
  540:     try
  541: 	scheduler_bind_types_test(Config,
  542: 				  ?TOPOLOGY_A_TERM,
  543: 				  ?TOPOLOGY_A_CMD,
  544: 				  a),
  545: 	scheduler_bind_types_test(Config,
  546: 				  ?TOPOLOGY_B_TERM,
  547: 				  ?TOPOLOGY_B_CMD,
  548: 				  b),
  549: 	scheduler_bind_types_test(Config,
  550: 				  ?TOPOLOGY_C_TERM,
  551: 				  ?TOPOLOGY_C_CMD,
  552: 				  c),
  553: 	scheduler_bind_types_test(Config,
  554: 				  ?TOPOLOGY_D_TERM,
  555: 				  ?TOPOLOGY_D_CMD,
  556: 				  d),
  557: 	scheduler_bind_types_test(Config,
  558: 				  ?TOPOLOGY_E_TERM,
  559: 				  ?TOPOLOGY_E_CMD,
  560: 				  e),
  561: 	scheduler_bind_types_test(Config,
  562: 				  ?TOPOLOGY_F_TERM,
  563: 				  ?TOPOLOGY_F_CMD,
  564: 				  f)
  565:     after
  566: 	restore_erl_rel_flags(OldRelFlags)
  567:     end,
  568:     ?line ok.
  569: 
  570: scheduler_bind_types_test(Config, Topology, CmdLine, TermLetter) ->
  571:     ?line ?t:format("Testing (~p): ~p~n", [TermLetter, Topology]),
  572:     ?line {ok, Node0} = start_node(Config),
  573:     ?line _ = rpc:call(Node0, erlang, system_flag, [cpu_topology, Topology]),
  574:     ?line cmp(Topology, rpc:call(Node0, erlang, system_info, [cpu_topology])),
  575:     ?line check_bind_types(Node0, TermLetter),
  576:     ?line stop_node(Node0),
  577:     ?line {ok, Node1} = start_node(Config, CmdLine),
  578:     ?line cmp(Topology, rpc:call(Node1, erlang, system_info, [cpu_topology])),
  579:     ?line check_bind_types(Node1, TermLetter),
  580:     ?line stop_node(Node1).
  581: 
  582: check_bind_types(Node, a) ->
  583:     ?line {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}
  584: 	= bindings(Node, no_spread),
  585:     ?line {0,2,4,6,8,10,12,14,1,3,5,7,9,11,13,15}
  586: 	= bindings(Node, thread_spread),
  587:     ?line {0,4,8,12,2,6,10,14,1,5,9,13,3,7,11,15}
  588: 	= bindings(Node, processor_spread),
  589:     ?line {0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15}
  590: 	= bindings(Node, spread),
  591:     ?line {0,2,4,6,1,3,5,7,8,10,12,14,9,11,13,15}
  592: 	= bindings(Node, no_node_thread_spread),
  593:     ?line {0,4,2,6,1,5,3,7,8,12,10,14,9,13,11,15}
  594: 	= bindings(Node, no_node_processor_spread),
  595:     ?line {0,4,2,6,8,12,10,14,1,5,3,7,9,13,11,15}
  596: 	= bindings(Node, thread_no_node_processor_spread),
  597:     ?line {0,4,2,6,8,12,10,14,1,5,3,7,9,13,11,15}
  598: 	= bindings(Node, default_bind),
  599:     ?line ok;
  600: check_bind_types(Node, b) ->
  601:     ?line {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}
  602: 	= bindings(Node, no_spread),
  603:     ?line {0,2,4,6,8,10,12,14,1,3,5,7,9,11,13,15}
  604: 	= bindings(Node, thread_spread),
  605:     ?line {0,8,2,10,4,12,6,14,1,9,3,11,5,13,7,15}
  606: 	= bindings(Node, processor_spread),
  607:     ?line {0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15}
  608: 	= bindings(Node, spread),
  609:     ?line {0,2,1,3,4,6,5,7,8,10,9,11,12,14,13,15}
  610: 	= bindings(Node, no_node_thread_spread),
  611:     ?line {0,2,1,3,4,6,5,7,8,10,9,11,12,14,13,15}
  612: 	= bindings(Node, no_node_processor_spread),
  613:     ?line {0,2,4,6,8,10,12,14,1,3,5,7,9,11,13,15}
  614: 	= bindings(Node, thread_no_node_processor_spread),
  615:     ?line {0,2,4,6,8,10,12,14,1,3,5,7,9,11,13,15}
  616: 	= bindings(Node, default_bind),
  617:     ?line ok;
  618: check_bind_types(Node, c) ->
  619:     ?line {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,
  620: 	   25,26,27,28,29,30,31} = bindings(Node, no_spread),
  621:     ?line {0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,1,3,5,7,9,11,13,15,
  622: 	   17,19,21,23,25,27,29,31} = bindings(Node, thread_spread),
  623:     ?line {0,4,8,16,20,24,2,6,10,18,22,26,12,28,14,30,1,5,9,17,21,25,
  624: 	   3,7,11,19,23,27,13,29,15,31} = bindings(Node, processor_spread),
  625:     ?line {0,8,16,24,4,20,12,28,2,10,18,26,6,22,14,30,1,9,17,25,5,21,13,29,3,11,
  626: 	   19,27,7,23,15,31} = bindings(Node, spread),
  627:     ?line {0,2,4,6,1,3,5,7,8,10,9,11,12,14,13,15,16,18,20,22,17,19,21,23,24,26,
  628: 	   25,27,28,30,29,31} = bindings(Node, no_node_thread_spread),
  629:     ?line {0,4,2,6,1,5,3,7,8,10,9,11,12,14,13,15,16,20,18,22,17,21,19,23,24,26,
  630: 	   25,27,28,30,29,31} = bindings(Node, no_node_processor_spread),
  631:     ?line {0,4,2,6,8,10,12,14,16,20,18,22,24,26,28,30,1,5,3,7,9,11,13,15,17,21,
  632: 	   19,23,25,27,29,31} = bindings(Node, thread_no_node_processor_spread),
  633:     ?line {0,4,2,6,8,10,12,14,16,20,18,22,24,26,28,30,1,5,3,7,9,11,13,15,17,21,
  634: 	   19,23,25,27,29,31} = bindings(Node, default_bind),
  635:     ?line ok;
  636: check_bind_types(Node, d) ->
  637:     ?line {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,
  638: 	   25,26,27,28,29,30,31} = bindings(Node, no_spread),
  639:     ?line {0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,1,3,5,7,9,11,13,15,
  640: 	   17,19,21,23,25,27,29,31} = bindings(Node, thread_spread),
  641:     ?line {0,8,12,16,24,28,2,10,14,18,26,30,4,20,6,22,1,9,13,17,25,29,3,11,15,
  642: 	   19,27,31,5,21,7,23} = bindings(Node, processor_spread),
  643:     ?line {0,8,16,24,12,28,4,20,2,10,18,26,14,30,6,22,1,9,17,25,13,29,5,21,3,11,
  644: 	   19,27,15,31,7,23} = bindings(Node, spread),
  645:     ?line {0,2,1,3,4,6,5,7,8,10,12,14,9,11,13,15,16,18,17,19,20,22,21,23,24,26,
  646: 	   28,30,25,27,29,31} = bindings(Node, no_node_thread_spread),
  647:     ?line {0,2,1,3,4,6,5,7,8,12,10,14,9,13,11,15,16,18,17,19,20,22,21,23,24,28,
  648: 	   26,30,25,29,27,31} = bindings(Node, no_node_processor_spread),
  649:     ?line {0,2,4,6,8,12,10,14,16,18,20,22,24,28,26,30,1,3,5,7,9,13,11,15,17,19,
  650: 	   21,23,25,29,27,31} = bindings(Node, thread_no_node_processor_spread),
  651:     ?line {0,2,4,6,8,12,10,14,16,18,20,22,24,28,26,30,1,3,5,7,9,13,11,15,17,19,
  652: 	   21,23,25,29,27,31} = bindings(Node, default_bind),
  653:     ?line ok;
  654: check_bind_types(Node, e) ->
  655:     ?line {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}
  656: 	= bindings(Node, no_spread),
  657:     ?line {0,2,4,6,8,10,12,14,1,3,5,7,9,11,13,15}
  658: 	= bindings(Node, thread_spread),
  659:     ?line {0,8,2,10,4,12,6,14,1,9,3,11,5,13,7,15}
  660: 	= bindings(Node, processor_spread),
  661:     ?line {0,8,2,10,4,12,6,14,1,9,3,11,5,13,7,15}
  662: 	= bindings(Node, spread),
  663:     ?line {0,2,4,6,1,3,5,7,8,10,12,14,9,11,13,15}
  664: 	= bindings(Node, no_node_thread_spread),
  665:     ?line {0,2,4,6,1,3,5,7,8,10,12,14,9,11,13,15}
  666: 	= bindings(Node, no_node_processor_spread),
  667:     ?line {0,2,4,6,8,10,12,14,1,3,5,7,9,11,13,15}
  668: 	= bindings(Node, thread_no_node_processor_spread),
  669:     ?line {0,2,4,6,8,10,12,14,1,3,5,7,9,11,13,15}
  670: 	= bindings(Node, default_bind),
  671:     ?line ok;
  672: check_bind_types(Node, f) ->
  673:     ?line {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,
  674: 	   25,26,27,28,29,30,31} = bindings(Node, no_spread),
  675:     ?line {0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,1,3,5,7,9,11,13,15,
  676: 	   17,19,21,23,25,27,29,31} = bindings(Node, thread_spread),
  677:     ?line {0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,1,3,5,7,9,11,13,
  678: 	   15,17,19,21,23,25,27,29,31} = bindings(Node, processor_spread),
  679:     ?line {0,8,16,24,2,10,18,26,4,12,20,28,6,14,22,30,1,9,17,25,3,11,19,27,5,13,
  680: 	   21,29,7,15,23,31} = bindings(Node, spread),
  681:     ?line {0,2,4,6,1,3,5,7,8,10,12,14,9,11,13,15,16,18,20,22,17,19,21,23,24,26,
  682: 	   28,30,25,27,29,31} = bindings(Node, no_node_thread_spread),
  683:     ?line {0,2,4,6,1,3,5,7,8,10,12,14,9,11,13,15,16,18,20,22,17,19,21,23,24,26,
  684: 	   28,30,25,27,29,31} = bindings(Node, no_node_processor_spread),
  685:     ?line {0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,1,3,5,7,9,11,13,15,17,19,
  686: 	   21,23,25,27,29,31} = bindings(Node, thread_no_node_processor_spread),
  687:     ?line {0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,1,3,5,7,9,11,13,15,17,19,
  688: 	   21,23,25,27,29,31} = bindings(Node, default_bind),
  689:     ?line ok;
  690: check_bind_types(Node, _) ->
  691:     ?line bindings(Node, no_spread),
  692:     ?line bindings(Node, thread_spread),
  693:     ?line bindings(Node, processor_spread),
  694:     ?line bindings(Node, spread),
  695:     ?line bindings(Node, no_node_thread_spread),
  696:     ?line bindings(Node, no_node_processor_spread),
  697:     ?line bindings(Node, thread_no_node_processor_spread),
  698:     ?line bindings(Node, default_bind),
  699:     ?line ok.
  700: 
  701: cpu_topology(Config) when is_list(Config) ->
  702:     ?line OldRelFlags = clear_erl_rel_flags(),
  703:     try
  704: 	?line cpu_topology_test(
  705: 		Config,
  706: 		[{node,[{processor,[{core,{logical,0}},
  707: 				    {core,{logical,1}}]}]},
  708: 		 {processor,[{node,[{core,{logical,2}},
  709: 				    {core,{logical,3}}]}]},
  710: 		 {node,[{processor,[{core,{logical,4}},
  711: 				    {core,{logical,5}}]}]},
  712: 		 {processor,[{node,[{core,{logical,6}},
  713: 				    {core,{logical,7}}]}]}],
  714: 		"+sct "
  715: 		"L0-1c0-1p0n0"
  716: 		":L2-3c0-1n1p1"
  717: 		":L4-5c0-1p2n2"
  718: 		":L6-7c0-1n3p3"),
  719: 	?line cpu_topology_test(
  720: 		Config,
  721: 		[{node,[{processor,[{core,{logical,0}},
  722: 				    {core,{logical,1}}]},
  723: 			{processor,[{core,{logical,2}},
  724: 				    {core,{logical,3}}]}]},
  725: 		 {processor,[{node,[{core,{logical,4}},
  726: 				    {core,{logical,5}}]},
  727: 			     {node,[{core,{logical,6}},
  728: 				    {core,{logical,7}}]}]},
  729: 		 {node,[{processor,[{core,{logical,8}},
  730: 				    {core,{logical,9}}]},
  731: 			{processor,[{core,{logical,10}},
  732: 				    {core,{logical,11}}]}]},
  733: 		 {processor,[{node,[{core,{logical,12}},
  734: 				    {core,{logical,13}}]},
  735: 			     {node,[{core,{logical,14}},
  736: 				    {core,{logical,15}}]}]}],
  737: 		"+sct "
  738: 		"L0-1c0-1p0n0"
  739: 		":L2-3c0-1p1n0"
  740: 		":L4-5c0-1n1p2"
  741: 		":L6-7c2-3n2p2"
  742: 		":L8-9c0-1p3n3"
  743: 		":L10-11c0-1p4n3"
  744: 		":L12-13c0-1n4p5"
  745: 		":L14-15c2-3n5p5"),
  746: 	?line cpu_topology_test(
  747: 		Config,
  748: 		[{node,[{processor,[{core,{logical,0}},
  749: 				    {core,{logical,1}}]}]},
  750: 		 {processor,[{node,[{core,{logical,2}},
  751: 				    {core,{logical,3}}]}]},
  752: 		 {processor,[{node,[{core,{logical,4}},
  753: 				    {core,{logical,5}}]}]},
  754: 		 {node,[{processor,[{core,{logical,6}},
  755: 				    {core,{logical,7}}]}]},
  756: 		 {node,[{processor,[{core,{logical,8}},
  757: 				    {core,{logical,9}}]}]},
  758: 		 {processor,[{node,[{core,{logical,10}},
  759: 				    {core,{logical,11}}]}]}],
  760: 		"+sct "
  761: 		"L0-1c0-1p0n0"
  762: 		":L2-3c0-1n1p1"
  763: 		":L4-5c0-1n2p2"
  764: 		":L6-7c0-1p3n3"
  765: 		":L8-9c0-1p4n4"
  766: 		":L10-11c0-1n5p5")
  767:     after
  768: 	restore_erl_rel_flags(OldRelFlags)
  769:     end,
  770:     ?line ok.
  771: 
  772: cpu_topology_test(Config, Topology, Cmd) ->
  773:     ?line ?t:format("Testing~n ~p~n ~p~n", [Topology, Cmd]),
  774:     ?line cpu_topology_bif_test(Config, Topology),
  775:     ?line cpu_topology_cmdline_test(Config, Topology, Cmd),
  776:     ?line ok.
  777: 
  778: cpu_topology_bif_test(_Config, false) ->
  779:     ?line ok;
  780: cpu_topology_bif_test(Config, Topology) ->
  781:     ?line {ok, Node} = start_node(Config),
  782:     ?line _ = rpc:call(Node, erlang, system_flag, [cpu_topology, Topology]),
  783:     ?line cmp(Topology, rpc:call(Node, erlang, system_info, [cpu_topology])),
  784:     ?line stop_node(Node),
  785:     ?line ok.
  786: 
  787: cpu_topology_cmdline_test(_Config, _Topology, false) ->
  788:     ?line ok;
  789: cpu_topology_cmdline_test(Config, Topology, Cmd) ->
  790:     ?line {ok, Node} = start_node(Config, Cmd),
  791:     ?line cmp(Topology, rpc:call(Node, erlang, system_info, [cpu_topology])),
  792:     ?line stop_node(Node),
  793:     ?line ok.
  794: 
  795: update_cpu_info(Config) when is_list(Config) ->
  796:     ?line OldOnline = erlang:system_info(schedulers_online),
  797:     ?line OldAff = get_affinity_mask(),
  798:     ?line ?t:format("START - Affinity mask: ~p - Schedulers online: ~p - Scheduler bindings: ~p~n",
  799: 		    [OldAff, OldOnline, erlang:system_info(scheduler_bindings)]),
  800:     ?line case {erlang:system_info(logical_processors_available), OldAff} of
  801: 	      {Avail, _} when Avail == unknown; OldAff == unknown ->
  802: 		  %% Nothing much to test; just a smoke test
  803: 		  case erlang:system_info(update_cpu_info) of
  804: 		      unchanged -> ?line ok;
  805: 		      changed -> ?line ok
  806: 		  end;
  807: 	      _ ->
  808: 		  try
  809: 		      ?line adjust_schedulers_online(),
  810: 		      case erlang:system_info(schedulers_online) of
  811: 			  1 ->
  812: 			      %% Nothing much to test; just a smoke test
  813: 			      ?line ok;
  814: 			  Onln0 ->
  815: 			      %% unset least significant bit
  816: 			      ?line Aff = (OldAff band (OldAff - 1)),
  817: 			      ?line set_affinity_mask(Aff),
  818: 			      ?line Onln1 = Onln0 - 1,
  819: 			      ?line case adjust_schedulers_online() of
  820: 					{Onln0, Onln1} ->
  821: 					    ?line Onln1 = erlang:system_info(schedulers_online),
  822: 					    ?line receive after 500 -> ok end,
  823: 					    ?line ?t:format("TEST - Affinity mask: ~p - Schedulers online: ~p - Scheduler bindings: ~p~n",
  824: 							    [Aff, Onln1, erlang:system_info(scheduler_bindings)]),
  825: 					    ?line unchanged = adjust_schedulers_online(),
  826: 					    ?line ok;
  827: 					Fail ->
  828: 					    ?line ?t:fail(Fail)
  829: 				    end
  830: 		      end
  831: 		  after
  832: 		      set_affinity_mask(OldAff),
  833: 		      adjust_schedulers_online(),
  834: 		      erlang:system_flag(schedulers_online, OldOnline),
  835: 		      receive after 500 -> ok end,
  836: 		      ?t:format("END - Affinity mask: ~p - Schedulers online: ~p - Scheduler bindings: ~p~n",
  837: 				[get_affinity_mask(),
  838: 				 erlang:system_info(schedulers_online),
  839: 				 erlang:system_info(scheduler_bindings)])
  840: 		  end
  841: 	  end.
  842: 
  843: adjust_schedulers_online() ->
  844:     case erlang:system_info(update_cpu_info) of
  845: 	unchanged ->
  846: 	    unchanged;
  847: 	changed ->
  848: 	    Avail = erlang:system_info(logical_processors_available),
  849: 	    {erlang:system_flag(schedulers_online, Avail), Avail}
  850:     end.
  851: 
  852: read_affinity(Data) ->
  853:     Exp = "pid " ++ os:getpid() ++ "'s current affinity mask",
  854:     case string:tokens(Data, ":") of
  855: 	[Exp, DirtyAffinityStr] ->
  856: 	    AffinityStr = string:strip(string:strip(DirtyAffinityStr,
  857: 						    both, $ ),
  858: 				       both, $\n),
  859: 	    case catch erlang:list_to_integer(AffinityStr, 16) of
  860: 		Affinity when is_integer(Affinity) ->
  861: 		    Affinity;
  862: 		_ ->
  863: 		    bad
  864: 	    end;
  865: 	_ ->
  866: 	    bad
  867:     end.
  868: 
  869: get_affinity_mask(Port, Status, Affinity) when Status == unknown;
  870: 					       Affinity == unknown ->
  871:     receive
  872: 	{Port,{data, Data}} ->
  873: 	    get_affinity_mask(Port, Status, read_affinity(Data));
  874: 	{Port,{exit_status,S}} ->
  875: 	    get_affinity_mask(Port, S, Affinity)
  876:     end;
  877: get_affinity_mask(_Port, _Status, bad) ->
  878:     unknown;
  879: get_affinity_mask(_Port, _Status, Affinity) ->
  880:     Affinity.
  881: 
  882: get_affinity_mask() ->
  883:     case ?t:os_type() of
  884: 	{unix, linux} ->
  885: 	    case catch open_port({spawn, "taskset -p " ++ os:getpid()},
  886: 				 [exit_status]) of
  887: 		Port when is_port(Port) ->
  888: 		    get_affinity_mask(Port, unknown, unknown);
  889: 		_ ->
  890: 		    unknown
  891: 	    end;
  892: 	_ ->
  893: 	    unknown
  894:     end.
  895: 
  896: set_affinity_mask(Port, unknown) ->
  897:     receive
  898: 	{Port,{data, _}} ->
  899: 	    set_affinity_mask(Port, unknown);
  900: 	{Port,{exit_status,Status}} ->
  901: 	    set_affinity_mask(Port, Status)
  902:     end;
  903: set_affinity_mask(Port, Status) ->
  904:     receive
  905: 	{Port,{data, _}} ->
  906: 	    set_affinity_mask(Port, unknown)
  907:     after 0 ->
  908: 	    Status
  909:     end.
  910: 
  911: set_affinity_mask(Mask) ->
  912:     Cmd = lists:flatten(["taskset -p ",
  913: 			 io_lib:format("~.16b", [Mask]),
  914: 			 " ",
  915: 			 os:getpid()]),
  916:     case catch open_port({spawn, Cmd}, [exit_status]) of
  917: 	Port when is_port(Port) ->
  918: 	    case set_affinity_mask(Port, unknown) of
  919: 		0 -> ok;
  920: 		_ -> exit(failed_to_set_affinity)
  921: 	    end;
  922: 	_ ->
  923: 	    exit(failed_to_set_affinity)
  924:     end.
  925: 
  926: sct_cmd(Config) when is_list(Config) ->
  927:     ?line Topology = ?TOPOLOGY_A_TERM,
  928:     ?line OldRelFlags = clear_erl_rel_flags(),
  929:     try
  930: 	?line {ok, Node} = start_node(Config, ?TOPOLOGY_A_CMD),
  931: 	?line cmp(Topology,
  932: 		  rpc:call(Node, erlang, system_info, [cpu_topology])),
  933: 	?line cmp(Topology,
  934: 		  rpc:call(Node, erlang, system_flag, [cpu_topology, Topology])),
  935: 	?line cmp(Topology,
  936: 		  rpc:call(Node, erlang, system_info, [cpu_topology])),
  937: 	?line stop_node(Node)
  938:     after
  939: 	restore_erl_rel_flags(OldRelFlags)
  940:     end,
  941:     ?line ok.
  942: 
  943: -define(BIND_TYPES,
  944: 	[{"u", unbound},
  945: 	 {"ns", no_spread},
  946: 	 {"ts", thread_spread},
  947: 	 {"ps", processor_spread},
  948: 	 {"s", spread},
  949: 	 {"nnts", no_node_thread_spread},
  950: 	 {"nnps", no_node_processor_spread},
  951: 	 {"tnnps", thread_no_node_processor_spread},
  952: 	 {"db", thread_no_node_processor_spread}]).
  953: 
  954: sbt_cmd(Config) when is_list(Config) ->
  955:     Bind = try
  956: 	      OldVal = erlang:system_flag(scheduler_bind_type, default_bind),
  957: 	      erlang:system_flag(scheduler_bind_type, OldVal),
  958: 	      go_for_it
  959: 	  catch
  960: 	      error:notsup -> notsup;
  961: 		error:_ -> go_for_it
  962: 	  end,
  963:     case Bind of
  964: 	notsup ->
  965: 	    ?line {skipped, "Binding of schedulers not supported"};
  966: 	go_for_it ->
  967: 	    CpuTCmd = case erlang:system_info({cpu_topology,detected}) of
  968: 			  undefined ->
  969: 			      case os:type() of
  970: 				  linux ->
  971: 				      case erlang:system_info(logical_processors) of
  972: 					  1 ->
  973: 					      "+sctL0";
  974: 					  N when is_integer(N) ->
  975: 					      NS = integer_to_list(N-1),
  976: 					      "+sctL0-"++NS++"p0-"++NS;
  977: 					  _ ->
  978: 					      false
  979: 				      end;
  980: 				  _ ->
  981: 				      false
  982: 			      end;
  983: 			  _ ->
  984: 			      ""
  985: 		      end,
  986: 	    case CpuTCmd of
  987: 		false ->
  988: 		    ?line {skipped, "Don't know how to create cpu topology"};
  989: 		_ ->
  990: 		    case erlang:system_info(logical_processors) of
  991: 			LP when is_integer(LP) ->
  992: 			    OldRelFlags = clear_erl_rel_flags(),
  993: 			    try
  994: 				lists:foreach(fun ({ClBt, Bt}) ->
  995: 						      ?line sbt_test(Config,
  996: 								     CpuTCmd,
  997: 								     ClBt,
  998: 								     Bt,
  999: 								     LP)
 1000: 					      end,
 1001: 					      ?BIND_TYPES)
 1002: 			    after
 1003: 				restore_erl_rel_flags(OldRelFlags)
 1004: 			    end,
 1005: 			    ?line ok;
 1006: 			_ ->
 1007: 			    ?line {skipped,
 1008: 				   "Don't know the amount of logical processors"}
 1009: 		    end
 1010: 	    end
 1011:     end.
 1012: 
 1013: sbt_test(Config, CpuTCmd, ClBt, Bt, LP) ->
 1014:     ?line ?t:format("Testing +sbt ~s (~p)~n", [ClBt, Bt]),
 1015:     ?line LPS = integer_to_list(LP),
 1016:     ?line Cmd = CpuTCmd++" +sbt "++ClBt++" +S"++LPS++":"++LPS,
 1017:     ?line {ok, Node} = start_node(Config, Cmd),
 1018:     ?line Bt = rpc:call(Node,
 1019: 			erlang,
 1020: 			system_info,
 1021: 			[scheduler_bind_type]),
 1022:     ?line SB = rpc:call(Node,
 1023: 			erlang,
 1024: 			system_info,
 1025: 			[scheduler_bindings]),
 1026:     ?line ?t:format("scheduler bindings: ~p~n", [SB]),
 1027:     ?line BS = case {Bt, erlang:system_info(logical_processors_available)} of
 1028: 		   {unbound, _} -> 0;
 1029: 		   {_, Int} when is_integer(Int) -> Int;
 1030: 		   {_, _} -> LP
 1031: 		end,
 1032:     ?line lists:foldl(fun (S, 0) ->
 1033: 			      ?line unbound = S,
 1034: 			      0;
 1035: 			  (S, N) ->
 1036: 			      ?line true = is_integer(S),
 1037: 			      N-1
 1038: 		      end,
 1039: 		      BS,
 1040: 		      tuple_to_list(SB)),
 1041:     ?line stop_node(Node),
 1042:     ?line ok.
 1043: 
 1044: scheduler_threads(Config) when is_list(Config) ->
 1045:     SmpSupport = erlang:system_info(smp_support),
 1046:     {Sched, SchedOnln, _} = get_sstate(Config, ""),
 1047:     %% Configure half the number of both the scheduler threads and
 1048:     %% the scheduler threads online.
 1049:     {HalfSched, HalfSchedOnln} = case SmpSupport of
 1050:                                      false -> {1,1};
 1051:                                      true ->
 1052:                                          {Sched div 2,
 1053:                                           SchedOnln div 2}
 1054:                                  end,
 1055:     {HalfSched, HalfSchedOnln, _} = get_sstate(Config, "+SP 50:50"),
 1056:     %% Use +S to configure 4x the number of scheduler threads and
 1057:     %% 4x the number of scheduler threads online, but alter that
 1058:     %% setting using +SP to 50% scheduler threads and 25% scheduler
 1059:     %% threads online. The result should be 2x scheduler threads and
 1060:     %% 1x scheduler threads online.
 1061:     TwiceSched = case SmpSupport of
 1062:                      false -> 1;
 1063:                      true -> Sched*2
 1064:                  end,
 1065:     FourSched = integer_to_list(Sched*4),
 1066:     FourSchedOnln = integer_to_list(SchedOnln*4),
 1067:     CombinedCmd1 = "+S "++FourSched++":"++FourSchedOnln++" +SP50:25",
 1068:     {TwiceSched, SchedOnln, _} = get_sstate(Config, CombinedCmd1),
 1069:     %% Now do the same test but with the +S and +SP options in the
 1070:     %% opposite order, since order shouldn't matter.
 1071:     CombinedCmd2 = "+SP50:25 +S "++FourSched++":"++FourSchedOnln,
 1072:     {TwiceSched, SchedOnln, _} = get_sstate(Config, CombinedCmd2),
 1073:     %% Apply two +SP options to make sure the second overrides the first
 1074:     TwoCmd = "+SP 25:25 +SP 100:100",
 1075:     {Sched, SchedOnln, _} = get_sstate(Config, TwoCmd),
 1076:     %% Configure 50% of scheduler threads online only
 1077:     {Sched, HalfSchedOnln, _} = get_sstate(Config, "+SP:50"),
 1078:     %% Configure 2x scheduler threads only
 1079:     {TwiceSched, SchedOnln, _} = get_sstate(Config, "+SP 200"),
 1080:     %% Test resetting the scheduler counts
 1081:     ResetCmd = "+S "++FourSched++":"++FourSchedOnln++" +S 0:0",
 1082:     {Sched, SchedOnln, _} = get_sstate(Config, ResetCmd),
 1083:     %% Test negative +S settings, but only for SMP-enabled emulators
 1084:     case SmpSupport of
 1085:         false -> ok;
 1086:         true ->
 1087:             SchedMinus1 = Sched-1,
 1088:             SchedOnlnMinus1 = SchedOnln-1,
 1089:             {SchedMinus1, SchedOnlnMinus1, _} = get_sstate(Config, "+S -1"),
 1090:             {Sched, SchedOnlnMinus1, _} = get_sstate(Config, "+S :-1"),
 1091:             {SchedMinus1, SchedOnlnMinus1, _} = get_sstate(Config, "+S -1:-1")
 1092:     end,
 1093:     ok.
 1094: 
 1095: get_sstate(Config, Cmd) ->
 1096:     {ok, Node} = start_node(Config, Cmd),
 1097:     [SState] = mcall(Node, [fun () ->
 1098:                                     erlang:system_info(schedulers_state)
 1099:                             end]),
 1100:     stop_node(Node),
 1101:     SState.
 1102: 
 1103: scheduler_suspend(Config) when is_list(Config) ->
 1104:     ?line Dog = ?t:timetrap(?t:minutes(5)),
 1105:     ?line lists:foreach(fun (S) -> scheduler_suspend_test(Config, S) end,
 1106: 			[64, 32, 16, default]),
 1107:     ?line ?t:timetrap_cancel(Dog),
 1108:     ?line ok.
 1109: scheduler_suspend_test(Config, Schedulers) ->
 1110:     ?line Cmd = case Schedulers of
 1111: 		    default ->
 1112: 			"";
 1113: 		    _ ->
 1114: 			S = integer_to_list(Schedulers),
 1115: 			"+S"++S++":"++S
 1116: 		end,
 1117:     ?line {ok, Node} = start_node(Config, Cmd),
 1118:     ?line [SState] = mcall(Node, [fun () ->
 1119: 					  erlang:system_info(schedulers_state)
 1120: 				  end]),
 1121:     ?line ?t:format("SState=~p~n", [SState]),
 1122:     ?line {Sched, SchedOnln, _SchedAvail} = SState,
 1123:     ?line true = is_integer(Sched),
 1124:     ?line [ok] = mcall(Node, [fun () -> sst0_loop(300) end]),
 1125:     ?line [ok] = mcall(Node, [fun () -> sst1_loop(300) end]),
 1126:     ?line [ok] = mcall(Node, [fun () -> sst2_loop(300) end]),
 1127:     ?line [ok, ok, ok, ok, ok] = mcall(Node,
 1128: 				       [fun () -> sst0_loop(200) end,
 1129: 					fun () -> sst1_loop(200) end,
 1130: 					fun () -> sst2_loop(200) end,
 1131: 					fun () -> sst2_loop(200) end,
 1132: 					fun () -> sst3_loop(Sched, 200) end]),
 1133:     ?line [SState] = mcall(Node, [fun () ->
 1134: 					  case Sched == SchedOnln of
 1135: 					      false ->
 1136: 						  Sched = erlang:system_flag(
 1137: 							    schedulers_online,
 1138: 							    SchedOnln);
 1139: 					      true ->
 1140: 						  ok
 1141: 					  end,
 1142: 					  erlang:system_info(schedulers_state)
 1143: 				  end]),
 1144:     ?line stop_node(Node),
 1145:     ?line ok.
 1146:     
 1147: 
 1148: sst0_loop(0) ->
 1149:     ok;
 1150: sst0_loop(N) ->
 1151:     erlang:system_flag(multi_scheduling, block),
 1152:     erlang:system_flag(multi_scheduling, unblock),
 1153:     erlang:yield(),
 1154:     sst0_loop(N-1).
 1155: 
 1156: sst1_loop(0) ->
 1157:     ok;
 1158: sst1_loop(N) ->
 1159:     erlang:system_flag(multi_scheduling, block),
 1160:     erlang:system_flag(multi_scheduling, unblock),
 1161:     sst1_loop(N-1).
 1162: 
 1163: sst2_loop(0) ->
 1164:     ok;
 1165: sst2_loop(N) ->
 1166:     erlang:system_flag(multi_scheduling, block),
 1167:     erlang:system_flag(multi_scheduling, block),
 1168:     erlang:system_flag(multi_scheduling, block),
 1169:     erlang:system_flag(multi_scheduling, unblock),
 1170:     erlang:system_flag(multi_scheduling, unblock),
 1171:     erlang:system_flag(multi_scheduling, unblock),
 1172:     sst2_loop(N-1).
 1173: 
 1174: sst3_loop(_S, 0) ->
 1175:     ok;
 1176: sst3_loop(S, N) ->
 1177:     erlang:system_flag(schedulers_online, (S div 2)+1),
 1178:     erlang:system_flag(schedulers_online, 1),
 1179:     erlang:system_flag(schedulers_online, (S div 2)+1),
 1180:     erlang:system_flag(schedulers_online, S),
 1181:     erlang:system_flag(schedulers_online, 1),
 1182:     erlang:system_flag(schedulers_online, S),
 1183:     sst3_loop(S, N-1).
 1184: 
 1185: reader_groups(Config) when is_list(Config) ->
 1186:     %% White box testing. These results are correct, but other results
 1187:     %% could be too...
 1188: 
 1189:     %% The actual tilepro64 topology
 1190:     CPUT0 = [{processor,[{node,[{core,{logical,0}},
 1191: 				{core,{logical,1}},
 1192: 				{core,{logical,2}},
 1193: 				{core,{logical,8}},
 1194: 				{core,{logical,9}},
 1195: 				{core,{logical,10}},
 1196: 				{core,{logical,11}},
 1197: 				{core,{logical,16}},
 1198: 				{core,{logical,17}},
 1199: 				{core,{logical,18}},
 1200: 				{core,{logical,19}},
 1201: 				{core,{logical,24}},
 1202: 				{core,{logical,25}},
 1203: 				{core,{logical,27}},
 1204: 				{core,{logical,29}}]},
 1205: 			 {node,[{core,{logical,3}},
 1206: 				{core,{logical,4}},
 1207: 				{core,{logical,5}},
 1208: 				{core,{logical,6}},
 1209: 				{core,{logical,7}},
 1210: 				{core,{logical,12}},
 1211: 				{core,{logical,13}},
 1212: 				{core,{logical,14}},
 1213: 				{core,{logical,15}},
 1214: 				{core,{logical,20}},
 1215: 				{core,{logical,21}},
 1216: 				{core,{logical,22}},
 1217: 				{core,{logical,23}},
 1218: 				{core,{logical,28}},
 1219: 				{core,{logical,30}}]},
 1220: 			 {node,[{core,{logical,31}},
 1221: 				{core,{logical,36}},
 1222: 				{core,{logical,37}},
 1223: 				{core,{logical,38}},
 1224: 				{core,{logical,44}},
 1225: 				{core,{logical,45}},
 1226: 				{core,{logical,46}},
 1227: 				{core,{logical,47}},
 1228: 				{core,{logical,51}},
 1229: 				{core,{logical,52}},
 1230: 				{core,{logical,53}},
 1231: 				{core,{logical,54}},
 1232: 				{core,{logical,55}},
 1233: 				{core,{logical,60}},
 1234: 				{core,{logical,61}}]},
 1235: 			 {node,[{core,{logical,26}},
 1236: 				{core,{logical,32}},
 1237: 				{core,{logical,33}},
 1238: 				{core,{logical,34}},
 1239: 				{core,{logical,35}},
 1240: 				{core,{logical,39}},
 1241: 				{core,{logical,40}},
 1242: 				{core,{logical,41}},
 1243: 				{core,{logical,42}},
 1244: 				{core,{logical,43}},
 1245: 				{core,{logical,48}},
 1246: 				{core,{logical,49}},
 1247: 				{core,{logical,50}},
 1248: 				{core,{logical,58}}]}]}],
 1249: 
 1250:     ?line [{0,1},{1,1},{2,1},{3,3},{4,3},{5,3},{6,3},{7,3},{8,1},{9,1},{10,1},
 1251: 	   {11,1},{12,3},{13,3},{14,4},{15,4},{16,2},{17,2},{18,2},{19,2},
 1252: 	   {20,4},{21,4},{22,4},{23,4},{24,2},{25,2},{26,7},{27,2},{28,4},
 1253: 	   {29,2},{30,4},{31,5},{32,7},{33,7},{34,7},{35,7},{36,5},{37,5},
 1254: 	   {38,5},{39,7},{40,7},{41,8},{42,8},{43,8},{44,5},{45,5},{46,5},
 1255: 	   {47,6},{48,8},{49,8},{50,8},{51,6},{52,6},{53,6},{54,6},{55,6},
 1256: 	   {58,8},{60,6},{61,6}]
 1257: 	= reader_groups_map(CPUT0, 8),
 1258: 
 1259:     CPUT1 = [n([p([c([t(l(0)),t(l(1)),t(l(2)),t(l(3))]),
 1260: 		   c([t(l(4)),t(l(5)),t(l(6)),t(l(7))]),
 1261: 		   c([t(l(8)),t(l(9)),t(l(10)),t(l(11))]),
 1262: 		   c([t(l(12)),t(l(13)),t(l(14)),t(l(15))])]),
 1263: 		p([c([t(l(16)),t(l(17)),t(l(18)),t(l(19))]),
 1264: 		   c([t(l(20)),t(l(21)),t(l(22)),t(l(23))]),
 1265: 		   c([t(l(24)),t(l(25)),t(l(26)),t(l(27))]),
 1266: 		   c([t(l(28)),t(l(29)),t(l(30)),t(l(31))])])]),
 1267: 	     n([p([c([t(l(32)),t(l(33)),t(l(34)),t(l(35))]),
 1268: 		   c([t(l(36)),t(l(37)),t(l(38)),t(l(39))]),
 1269: 		   c([t(l(40)),t(l(41)),t(l(42)),t(l(43))]),
 1270: 		   c([t(l(44)),t(l(45)),t(l(46)),t(l(47))])]),
 1271: 		p([c([t(l(48)),t(l(49)),t(l(50)),t(l(51))]),
 1272: 		   c([t(l(52)),t(l(53)),t(l(54)),t(l(55))]),
 1273: 		   c([t(l(56)),t(l(57)),t(l(58)),t(l(59))]),
 1274: 		   c([t(l(60)),t(l(61)),t(l(62)),t(l(63))])])]),
 1275: 	     n([p([c([t(l(64)),t(l(65)),t(l(66)),t(l(67))]),
 1276: 		   c([t(l(68)),t(l(69)),t(l(70)),t(l(71))]),
 1277: 		   c([t(l(72)),t(l(73)),t(l(74)),t(l(75))]),
 1278: 		   c([t(l(76)),t(l(77)),t(l(78)),t(l(79))])]),
 1279: 		p([c([t(l(80)),t(l(81)),t(l(82)),t(l(83))]),
 1280: 		   c([t(l(84)),t(l(85)),t(l(86)),t(l(87))]),
 1281: 		   c([t(l(88)),t(l(89)),t(l(90)),t(l(91))]),
 1282: 		   c([t(l(92)),t(l(93)),t(l(94)),t(l(95))])])]),
 1283: 	     n([p([c([t(l(96)),t(l(97)),t(l(98)),t(l(99))]),
 1284: 		   c([t(l(100)),t(l(101)),t(l(102)),t(l(103))]),
 1285: 		   c([t(l(104)),t(l(105)),t(l(106)),t(l(107))]),
 1286: 		   c([t(l(108)),t(l(109)),t(l(110)),t(l(111))])]),
 1287: 		p([c([t(l(112)),t(l(113)),t(l(114)),t(l(115))]),
 1288: 		   c([t(l(116)),t(l(117)),t(l(118)),t(l(119))]),
 1289: 		   c([t(l(120)),t(l(121)),t(l(122)),t(l(123))]),
 1290: 		   c([t(l(124)),t(l(125)),t(l(126)),t(l(127))])])])],
 1291: 
 1292:     ?line [{0,1},{1,1},{2,1},{3,1},{4,2},{5,2},{6,2},{7,2},{8,3},{9,3},
 1293: 	   {10,3},{11,3},{12,4},{13,4},{14,4},{15,4},{16,5},{17,5},{18,5},
 1294: 	   {19,5},{20,6},{21,6},{22,6},{23,6},{24,7},{25,7},{26,7},{27,7},
 1295: 	   {28,8},{29,8},{30,8},{31,8},{32,9},{33,9},{34,9},{35,9},{36,10},
 1296: 	   {37,10},{38,10},{39,10},{40,11},{41,11},{42,11},{43,11},{44,12},
 1297: 	   {45,12},{46,12},{47,12},{48,13},{49,13},{50,13},{51,13},{52,14},
 1298: 	   {53,14},{54,14},{55,14},{56,15},{57,15},{58,15},{59,15},{60,16},
 1299: 	   {61,16},{62,16},{63,16},{64,17},{65,17},{66,17},{67,17},{68,18},
 1300: 	   {69,18},{70,18},{71,18},{72,19},{73,19},{74,19},{75,19},{76,20},
 1301: 	   {77,20},{78,20},{79,20},{80,21},{81,21},{82,21},{83,21},{84,22},
 1302: 	   {85,22},{86,22},{87,22},{88,23},{89,23},{90,23},{91,23},{92,24},
 1303: 	   {93,24},{94,24},{95,24},{96,25},{97,25},{98,25},{99,25},{100,26},
 1304: 	   {101,26},{102,26},{103,26},{104,27},{105,27},{106,27},{107,27},
 1305: 	   {108,28},{109,28},{110,28},{111,28},{112,29},{113,29},{114,29},
 1306: 	   {115,29},{116,30},{117,30},{118,30},{119,30},{120,31},{121,31},
 1307: 	   {122,31},{123,31},{124,32},{125,32},{126,32},{127,32}]
 1308: 	= reader_groups_map(CPUT1, 128),
 1309: 
 1310:     ?line [{0,1},{1,1},{2,1},{3,1},{4,1},{5,1},{6,1},{7,1},{8,1},{9,1},{10,1},
 1311: 	   {11,1},{12,1},{13,1},{14,1},{15,1},{16,1},{17,1},{18,1},{19,1},
 1312: 	   {20,1},{21,1},{22,1},{23,1},{24,1},{25,1},{26,1},{27,1},{28,1},
 1313: 	   {29,1},{30,1},{31,1},{32,1},{33,1},{34,1},{35,1},{36,1},{37,1},
 1314: 	   {38,1},{39,1},{40,1},{41,1},{42,1},{43,1},{44,1},{45,1},{46,1},
 1315: 	   {47,1},{48,1},{49,1},{50,1},{51,1},{52,1},{53,1},{54,1},{55,1},
 1316: 	   {56,1},{57,1},{58,1},{59,1},{60,1},{61,1},{62,1},{63,1},{64,2},
 1317: 	   {65,2},{66,2},{67,2},{68,2},{69,2},{70,2},{71,2},{72,2},{73,2},
 1318: 	   {74,2},{75,2},{76,2},{77,2},{78,2},{79,2},{80,2},{81,2},{82,2},
 1319: 	   {83,2},{84,2},{85,2},{86,2},{87,2},{88,2},{89,2},{90,2},{91,2},
 1320: 	   {92,2},{93,2},{94,2},{95,2},{96,2},{97,2},{98,2},{99,2},{100,2},
 1321: 	   {101,2},{102,2},{103,2},{104,2},{105,2},{106,2},{107,2},{108,2},
 1322: 	   {109,2},{110,2},{111,2},{112,2},{113,2},{114,2},{115,2},{116,2},
 1323: 	   {117,2},{118,2},{119,2},{120,2},{121,2},{122,2},{123,2},{124,2},
 1324: 	   {125,2},{126,2},{127,2}]
 1325: 	= reader_groups_map(CPUT1, 2),
 1326: 
 1327:     ?line [{0,1},{1,1},{2,1},{3,1},{4,2},{5,2},{6,2},{7,2},{8,3},{9,3},{10,3},
 1328: 	   {11,3},{12,3},{13,3},{14,3},{15,3},{16,4},{17,4},{18,4},{19,4},
 1329: 	   {20,4},{21,4},{22,4},{23,4},{24,5},{25,5},{26,5},{27,5},{28,5},
 1330: 	   {29,5},{30,5},{31,5},{32,6},{33,6},{34,6},{35,6},{36,6},{37,6},
 1331: 	   {38,6},{39,6},{40,7},{41,7},{42,7},{43,7},{44,7},{45,7},{46,7},
 1332: 	   {47,7},{48,8},{49,8},{50,8},{51,8},{52,8},{53,8},{54,8},{55,8},
 1333: 	   {56,9},{57,9},{58,9},{59,9},{60,9},{61,9},{62,9},{63,9},{64,10},
 1334: 	   {65,10},{66,10},{67,10},{68,10},{69,10},{70,10},{71,10},{72,11},
 1335: 	   {73,11},{74,11},{75,11},{76,11},{77,11},{78,11},{79,11},{80,12},
 1336: 	   {81,12},{82,12},{83,12},{84,12},{85,12},{86,12},{87,12},{88,13},
 1337: 	   {89,13},{90,13},{91,13},{92,13},{93,13},{94,13},{95,13},{96,14},
 1338: 	   {97,14},{98,14},{99,14},{100,14},{101,14},{102,14},{103,14},
 1339: 	   {104,15},{105,15},{106,15},{107,15},{108,15},{109,15},{110,15},
 1340: 	   {111,15},{112,16},{113,16},{114,16},{115,16},{116,16},{117,16},
 1341: 	   {118,16},{119,16},{120,17},{121,17},{122,17},{123,17},{124,17},
 1342: 	   {125,17},{126,17},{127,17}]
 1343: 	= reader_groups_map(CPUT1, 17),
 1344: 
 1345:     ?line [{0,1},{1,1},{2,1},{3,1},{4,1},{5,1},{6,1},{7,1},{8,1},{9,1},{10,1},
 1346: 	   {11,1},{12,1},{13,1},{14,1},{15,1},{16,2},{17,2},{18,2},{19,2},
 1347: 	   {20,2},{21,2},{22,2},{23,2},{24,2},{25,2},{26,2},{27,2},{28,2},
 1348: 	   {29,2},{30,2},{31,2},{32,3},{33,3},{34,3},{35,3},{36,3},{37,3},
 1349: 	   {38,3},{39,3},{40,3},{41,3},{42,3},{43,3},{44,3},{45,3},{46,3},
 1350: 	   {47,3},{48,4},{49,4},{50,4},{51,4},{52,4},{53,4},{54,4},{55,4},
 1351: 	   {56,4},{57,4},{58,4},{59,4},{60,4},{61,4},{62,4},{63,4},{64,5},
 1352: 	   {65,5},{66,5},{67,5},{68,5},{69,5},{70,5},{71,5},{72,5},{73,5},
 1353: 	   {74,5},{75,5},{76,5},{77,5},{78,5},{79,5},{80,6},{81,6},{82,6},
 1354: 	   {83,6},{84,6},{85,6},{86,6},{87,6},{88,6},{89,6},{90,6},{91,6},
 1355: 	   {92,6},{93,6},{94,6},{95,6},{96,7},{97,7},{98,7},{99,7},{100,7},
 1356: 	   {101,7},{102,7},{103,7},{104,7},{105,7},{106,7},{107,7},{108,7},
 1357: 	   {109,7},{110,7},{111,7},{112,7},{113,7},{114,7},{115,7},{116,7},
 1358: 	   {117,7},{118,7},{119,7},{120,7},{121,7},{122,7},{123,7},{124,7},
 1359: 	   {125,7},{126,7},{127,7}]
 1360: 	= reader_groups_map(CPUT1, 7),
 1361: 
 1362:     ?line CPUT2 = [p([c(l(0)),c(l(1)),c(l(2)),c(l(3)),c(l(4))]),
 1363: 		   p([t(l(5)),t(l(6)),t(l(7)),t(l(8)),t(l(9))]),
 1364: 		   p([t(l(10))]),
 1365: 		   p([c(l(11)),c(l(12)),c(l(13))]),
 1366: 		   p([c(l(14)),c(l(15))])],
 1367: 
 1368:     ?line [{0,1},{1,1},{2,1},{3,1},{4,1},
 1369: 	   {5,2},{6,2},{7,2},{8,2},{9,2},
 1370: 	   {10,3},
 1371: 	   {11,4},{12,4},{13,4},
 1372: 	   {14,5},{15,5}] = reader_groups_map(CPUT2, 5),
 1373: 
 1374: 
 1375:     ?line [{0,1},{1,1},{2,2},{3,2},{4,2},
 1376: 	   {5,3},{6,3},{7,3},{8,3},{9,3},
 1377: 	   {10,4},
 1378: 	   {11,5},{12,5},{13,5},
 1379: 	   {14,6},{15,6}] = reader_groups_map(CPUT2, 6),
 1380: 
 1381:     ?line [{0,1},{1,1},{2,2},{3,2},{4,2},
 1382: 	   {5,3},{6,3},{7,3},{8,3},{9,3},
 1383: 	   {10,4},
 1384: 	   {11,5},{12,6},{13,6},
 1385: 	   {14,7},{15,7}] = reader_groups_map(CPUT2, 7),
 1386: 
 1387:     ?line [{0,1},{1,1},{2,2},{3,2},{4,2},
 1388: 	   {5,3},{6,3},{7,3},{8,3},{9,3},
 1389: 	   {10,4},
 1390: 	   {11,5},{12,6},{13,6},
 1391: 	   {14,7},{15,8}] = reader_groups_map(CPUT2, 8),
 1392: 
 1393:     ?line [{0,1},{1,2},{2,2},{3,3},{4,3},
 1394: 	   {5,4},{6,4},{7,4},{8,4},{9,4},
 1395: 	   {10,5},
 1396: 	   {11,6},{12,7},{13,7},
 1397: 	   {14,8},{15,9}] = reader_groups_map(CPUT2, 9),
 1398: 
 1399:     ?line [{0,1},{1,2},{2,2},{3,3},{4,3},
 1400: 	   {5,4},{6,4},{7,4},{8,4},{9,4},
 1401: 	   {10,5},
 1402: 	   {11,6},{12,7},{13,8},
 1403: 	   {14,9},{15,10}] = reader_groups_map(CPUT2, 10),
 1404: 
 1405:     ?line [{0,1},{1,2},{2,3},{3,4},{4,4},
 1406: 	   {5,5},{6,5},{7,5},{8,5},{9,5},
 1407: 	   {10,6},
 1408: 	   {11,7},{12,8},{13,9},
 1409: 	   {14,10},{15,11}] = reader_groups_map(CPUT2, 11),
 1410: 
 1411:     ?line [{0,1},{1,2},{2,3},{3,4},{4,5},
 1412: 	   {5,6},{6,6},{7,6},{8,6},{9,6},
 1413: 	   {10,7},
 1414: 	   {11,8},{12,9},{13,10},
 1415: 	   {14,11},{15,12}] = reader_groups_map(CPUT2, 100),
 1416: 
 1417:     CPUT3 = [p([t(l(5)),t(l(6)),t(l(7)),t(l(8)),t(l(9))]),
 1418: 	     p([t(l(10))]),
 1419: 	     p([c(l(11)),c(l(12)),c(l(13))]),
 1420: 	     p([c(l(14)),c(l(15))]),
 1421: 	     p([c(l(0)),c(l(1)),c(l(2)),c(l(3)),c(l(4))])],
 1422: 
 1423:     ?line [{0,5},{1,5},{2,6},{3,6},{4,6},
 1424: 	   {5,1},{6,1},{7,1},{8,1},{9,1},
 1425: 	   {10,2},{11,3},{12,3},{13,3},
 1426: 	   {14,4},{15,4}] = reader_groups_map(CPUT3, 6),
 1427: 
 1428:     CPUT4 = [p([t(l(0)),t(l(1)),t(l(2)),t(l(3)),t(l(4))]),
 1429: 	     p([t(l(5))]),
 1430: 	     p([c(l(6)),c(l(7)),c(l(8))]),
 1431: 	     p([c(l(9)),c(l(10))]),
 1432: 	     p([c(l(11)),c(l(12)),c(l(13)),c(l(14)),c(l(15))])],
 1433: 
 1434:     ?line [{0,1},{1,1},{2,1},{3,1},{4,1},
 1435: 	   {5,2},
 1436: 	   {6,3},{7,3},{8,3},
 1437: 	   {9,4},{10,4},
 1438: 	   {11,5},{12,5},{13,6},{14,6},{15,6}] = reader_groups_map(CPUT4, 6),
 1439: 
 1440:     ?line [{0,1},{1,1},{2,1},{3,1},{4,1},
 1441: 	   {5,2},
 1442: 	   {6,3},{7,4},{8,4},
 1443: 	   {9,5},{10,5},
 1444: 	   {11,6},{12,6},{13,7},{14,7},{15,7}] = reader_groups_map(CPUT4, 7),
 1445: 
 1446:     ?line [{0,1},{65535,2}] = reader_groups_map([c(l(0)),c(l(65535))], 10),
 1447: 
 1448:     ?line ok.
 1449:     
 1450: 
 1451: reader_groups_map(CPUT, Groups) ->
 1452:     Old = erlang:system_info({cpu_topology, defined}),
 1453:     erlang:system_flag(cpu_topology, CPUT),
 1454:     enable_internal_state(),
 1455:     Res = erts_debug:get_internal_state({reader_groups_map, Groups}),
 1456:     erlang:system_flag(cpu_topology, Old),
 1457:     lists:sort(Res).
 1458: 
 1459: %%
 1460: %% Utils
 1461: %%
 1462: 
 1463: l(Id) ->
 1464:     {logical, Id}.
 1465: 
 1466: t(X) ->
 1467:     {thread, X}.
 1468: 
 1469: c(X) ->
 1470:     {core, X}.
 1471: 
 1472: p(X) ->
 1473:     {processor, X}.
 1474: 
 1475: n(X) ->
 1476:     {node, X}.
 1477: 
 1478: mcall(Node, Funs) ->
 1479:     Parent = self(),
 1480:     Refs = lists:map(fun (Fun) ->
 1481: 			     Ref = make_ref(),
 1482: 			     spawn_link(Node,
 1483: 					fun () ->
 1484: 						Res = Fun(),
 1485: 						unlink(Parent),
 1486: 						Parent ! {Ref, Res}
 1487: 					end),
 1488: 			     Ref
 1489: 		     end, Funs),
 1490:     lists:map(fun (Ref) ->
 1491: 		      receive
 1492: 			  {Ref, Res} ->
 1493: 			      Res
 1494: 		      end
 1495: 	      end, Refs).
 1496: 
 1497: erl_rel_flag_var() ->
 1498:     "ERL_"++erlang:system_info(otp_release)++"_FLAGS".
 1499: 
 1500: clear_erl_rel_flags() ->
 1501:     EnvVar = erl_rel_flag_var(),
 1502:     case os:getenv(EnvVar) of
 1503: 	false ->
 1504: 	    false;
 1505: 	Value ->
 1506: 	    os:putenv(EnvVar, ""),
 1507: 	    Value
 1508:     end.
 1509: 
 1510: restore_erl_rel_flags(false) ->
 1511:     ok;
 1512: restore_erl_rel_flags(OldValue) ->
 1513:     os:putenv(erl_rel_flag_var(), OldValue),
 1514:     ok.
 1515: 
 1516: ok(too_slow, _Config) ->
 1517:     {comment, "Too slow system to do any actual testing..."};
 1518: ok(_Res, Config) ->
 1519:     ?config(ok_res, Config).
 1520: 
 1521: chk_result(too_slow,
 1522: 	   _LWorkers,
 1523: 	   _NWorkers,
 1524: 	   _HWorkers,
 1525: 	   _MWorkers,
 1526: 	   _LNShouldWork,
 1527: 	   _HShouldWork,
 1528: 	   _MShouldWork) ->
 1529:     ?line ok;
 1530: chk_result([{low, L, Lmin, _Lmax},
 1531: 	    {normal, N, Nmin, _Nmax},
 1532: 	    {high, H, Hmin, _Hmax},
 1533: 	    {max, M, Mmin, _Mmax}] = Res,
 1534: 	   LWorkers,
 1535: 	   NWorkers,
 1536: 	   HWorkers,
 1537: 	   MWorkers,
 1538: 	   LNShouldWork,
 1539: 	   HShouldWork,
 1540: 	   MShouldWork) ->
 1541:     ?line ?t:format("~p~n", [Res]),
 1542:     ?line Relax = relax_limits(),
 1543:     case {L, N} of
 1544: 	{0, 0} ->
 1545: 	    ?line false = LNShouldWork;
 1546: 	_ ->
 1547: 	    ?line {LminRatioLim,
 1548: 		   NminRatioLim,
 1549: 		   LNRatioLimMin,
 1550: 		   LNRatioLimMax} = case Relax of
 1551: 					false -> {0.5, 0.5, 0.05, 0.25};
 1552: 					true -> {0.05, 0.05, 0.01, 0.4}
 1553: 				    end,
 1554: 	    ?line Lavg = L/LWorkers,
 1555: 	    ?line Navg = N/NWorkers,
 1556: 	    ?line Ratio = Lavg/Navg,
 1557: 	    ?line LminRatio = Lmin/Lavg,
 1558: 	    ?line NminRatio = Nmin/Navg,
 1559: 	    ?line ?t:format("low min ratio=~p~n"
 1560: 			    "normal min ratio=~p~n"
 1561: 			    "low avg=~p~n"
 1562: 			    "normal avg=~p~n"
 1563: 			    "low/normal ratio=~p~n",
 1564: 			    [LminRatio, NminRatio, Lavg, Navg, Ratio]),
 1565: 	    erlang:display({low_min_ratio, LminRatio}),
 1566: 	    erlang:display({normal_min_ratio, NminRatio}),
 1567: 	    erlang:display({low_avg, Lavg}),
 1568: 	    erlang:display({normal_avg, Navg}),
 1569: 	    erlang:display({low_normal_ratio, Ratio}),
 1570: 	    ?line chk_lim(LminRatioLim, LminRatio, 1.0, low_min_ratio),
 1571: 	    ?line chk_lim(NminRatioLim, NminRatio, 1.0, normal_min_ratio),
 1572: 	    ?line chk_lim(LNRatioLimMin, Ratio, LNRatioLimMax, low_normal_ratio),
 1573: 	    ?line true = LNShouldWork,
 1574: 	    ?line ok
 1575:     end,
 1576:     case H of
 1577: 	0 ->
 1578: 	    ?line false = HShouldWork;
 1579: 	_ ->
 1580: 	    ?line HminRatioLim = case Relax of
 1581: 				     false -> 0.5;
 1582: 				     true -> 0.1
 1583: 				 end,
 1584: 	    ?line Havg = H/HWorkers,
 1585: 	    ?line HminRatio = Hmin/Havg,
 1586: 	    erlang:display({high_min_ratio, HminRatio}),
 1587: 	    ?line chk_lim(HminRatioLim, HminRatio, 1.0, high_min_ratio),
 1588: 	    ?line true = HShouldWork,
 1589: 	    ?line ok
 1590:     end,
 1591:     case M of
 1592: 	0 ->
 1593: 	    ?line false = MShouldWork;
 1594: 	_ ->
 1595: 	    ?line MminRatioLim = case Relax of
 1596: 				     false -> 0.5;
 1597: 				     true -> 0.1
 1598: 				 end,
 1599: 	    ?line Mavg = M/MWorkers,
 1600: 	    ?line MminRatio = Mmin/Mavg,
 1601: 	    erlang:display({max_min_ratio, MminRatio}),
 1602: 	    ?line chk_lim(MminRatioLim, MminRatio, 1.0, max_min_ratio),
 1603: 	    ?line true = MShouldWork,
 1604: 	    ?line ok
 1605:     end,
 1606:     ?line ok.
 1607: 
 1608: 	    
 1609: 	    
 1610: chk_lim(Min, V, Max, _What) when Min =< V, V =< Max ->
 1611:     ok;
 1612: chk_lim(_Min, V, _Max, What) ->
 1613:     ?t:fail({bad, What, V}).
 1614: 
 1615: snd(_Msg, []) ->
 1616:     [];
 1617: snd(Msg, [P|Ps]) ->
 1618:     P ! Msg,
 1619:     Ps.
 1620: 
 1621: relax_limits() ->
 1622:     case strange_system_scale() of
 1623: 	Scale when Scale > 1 ->
 1624: 	    ?t:format("Relaxing limits~n", []),
 1625: 	    true;
 1626: 	_ ->
 1627: 	    false
 1628:     end.
 1629: 
 1630: strange_system_scale() ->
 1631:     S0 = 1,
 1632:     S1 = case erlang:system_info(schedulers_online)
 1633: 	     > erlang:system_info(logical_processors) of
 1634: 	     true -> S0*2;
 1635: 	     false -> S0
 1636: 	 end,
 1637:     S2 = case erlang:system_info(debug_compiled) of
 1638: 	     true -> S1*10;
 1639: 	     false ->
 1640: 		 case erlang:system_info(lock_checking) of
 1641: 		     true -> S1*2;
 1642: 		     false -> S1
 1643: 		 end
 1644: 	 end,
 1645:     S3 = case lock_counting() of
 1646: 	     true -> S2*2;
 1647: 	     false -> S2
 1648: 	 end,
 1649:     S3.
 1650: 
 1651: lock_counting() ->
 1652:     lock_counting(erlang:system_info(system_version)).
 1653: 
 1654: lock_counting([]) ->
 1655:     false;
 1656: lock_counting([$[,$l,$o,$c,$k,$-,$c,$o,$u,$n,$t,$i,$n,$g,$],_]) ->
 1657:     true;
 1658: lock_counting([_C|Cs]) ->
 1659:     lock_counting(Cs).
 1660: 
 1661: go_work([], [], [], []) ->
 1662:     [];
 1663: go_work(L, N, [], []) ->
 1664:     go_work(snd(go_work, L), snd(go_work, N), [], []);
 1665: go_work(L, N, H, []) ->
 1666:     go_work(L, N, snd(go_work, H), []);
 1667: go_work(L, N, H, M) ->
 1668:     go_work(L, N, H, snd(go_work, M)).
 1669: 
 1670: stop_work([], [], [], []) ->
 1671:     [];
 1672: stop_work([], [], [], M) ->
 1673:     stop_work([], [], [], snd(stop_work, M));
 1674: stop_work([], [], H, M) ->
 1675:     stop_work([], [], snd(stop_work, H), M);
 1676: stop_work(L, N, H, M) ->
 1677:     stop_work(snd(stop_work, L), snd(stop_work, N), H, M).
 1678: 
 1679: wait_balance(N) when is_integer(N) ->
 1680:     case erlang:system_info(schedulers_active) of
 1681: 	1 ->
 1682: 	    done;
 1683: 	_ ->
 1684: 	    erts_debug:set_internal_state(available_internal_state,true),
 1685: 	    Start = erts_debug:get_internal_state(nbalance),
 1686: 	    End = (Start + N) band ((1 bsl (8*erlang:system_info(wordsize)))-1),
 1687: 	    wait_balance(Start, End),
 1688: 	    erts_debug:set_internal_state(available_internal_state,false)
 1689:     end.
 1690: 
 1691: wait_balance(Start, End) ->
 1692:     X = erts_debug:get_internal_state(nbalance),
 1693:     case End =< X of
 1694: 	true ->
 1695: 	    case Start =< End of
 1696: 		true ->
 1697: 		    done;
 1698: 		false ->
 1699: 		    case X < Start of
 1700: 			true ->
 1701: 			    done;
 1702: 			false ->
 1703: 			    receive after 250 -> ok end,
 1704: 			    wait_balance(Start, End)
 1705: 		    end
 1706: 	    end;
 1707: 	false ->
 1708: 	    receive after 250 -> ok end,
 1709: 	    wait_balance(Start, End)
 1710:     end.
 1711: 
 1712: wait_reds(RedsLimit, Timeout) ->
 1713:     Stop = erlang:start_timer(Timeout, self(), stop),
 1714:     statistics(reductions),
 1715:     wait_reds(0, RedsLimit, Stop).
 1716: 
 1717: wait_reds(Reds, RedsLimit, Stop) when Reds < RedsLimit ->
 1718:     receive
 1719: 	{timeout, Stop, stop} ->
 1720: 	    erlang:display(timeout),
 1721: 	    erlang:display({reduction_limit, RedsLimit}),
 1722: 	    erlang:display({reductions, Reds}),
 1723: 	    done
 1724:     after 10000 ->
 1725: 	    {_, NewReds} = statistics(reductions),
 1726: 	    wait_reds(NewReds+Reds, RedsLimit, Stop)
 1727:     end;
 1728: wait_reds(Reds, RedsLimit, Stop) when is_reference(Stop) ->
 1729:     erlang:cancel_timer(Stop),
 1730:     receive {timeout, Stop, stop} -> ok after 0 -> ok end,
 1731:     wait_reds(Reds, RedsLimit, false);
 1732: wait_reds(Reds, RedsLimit, _Stop) ->
 1733:     erlang:display({reduction_limit, RedsLimit}),
 1734:     erlang:display({reductions, Reds}),
 1735:     done.
 1736: 
 1737: do_it(Tracer, Low, Normal, High, Max) ->
 1738:     do_it(Tracer, Low, Normal, High, Max, ?DEFAULT_TEST_REDS_PER_SCHED).
 1739: 
 1740: do_it(Tracer, Low, Normal, High, Max, RedsPerSchedLimit) ->
 1741:     OldPrio = process_flag(priority, max),
 1742:     go_work(Low, Normal, High, Max),
 1743:     StartWait = now(),
 1744:     %% Give the emulator a chance to balance the load...
 1745:     wait_balance(5),
 1746:     EndWait = now(),
 1747:     BalanceWait = timer:now_diff(EndWait,StartWait) div 1000,
 1748:     erlang:display({balance_wait, BalanceWait}),
 1749:     Timeout = ?DEFAULT_TIMEOUT - ?t:minutes(4) - BalanceWait,
 1750:     Res = case Timeout < ?MIN_SCHEDULER_TEST_TIMEOUT of
 1751: 	      true ->
 1752: 		  stop_work(Low, Normal, High, Max),
 1753: 		  too_slow;
 1754: 	      false ->
 1755: 		  set_tracing(true, Tracer, normal, Normal),
 1756: 		  set_tracing(true, Tracer, low, Low),
 1757: 		  set_tracing(true, Tracer, high, High),
 1758: 		  set_tracing(true, Tracer, max, Max),
 1759: 		  wait_reds(RedsPerSchedLimit
 1760: 			    * erlang:system_info(schedulers_online),
 1761: 			    Timeout),
 1762: 		  set_tracing(false, Tracer, normal, Normal),
 1763: 		  set_tracing(false, Tracer, low, Low),
 1764: 		  set_tracing(false, Tracer, high, High),
 1765: 		  set_tracing(false, Tracer, max, Max),
 1766: 		  stop_work(Low, Normal, High, Max),
 1767: 		  get_trace_result(Tracer)
 1768: 	  end,
 1769:     process_flag(priority, OldPrio),
 1770:     Res.
 1771: 
 1772: workers_exit([]) ->
 1773:     ok;
 1774: workers_exit([P|Ps]) when is_pid(P) ->
 1775:     Mon = erlang:monitor(process, P),
 1776:     unlink(P),
 1777:     exit(P, kill),
 1778:     workers_exit(Ps),
 1779:     receive {'DOWN', Mon, process, P, _} -> ok end,
 1780:     ok;
 1781: workers_exit([[]]) ->
 1782:     ok;
 1783: workers_exit([Ps|Pss])  ->
 1784:     workers_exit(Ps),
 1785:     workers_exit(Pss).
 1786: 
 1787: do_work(PartTime) ->
 1788:     lists:reverse(lists:seq(1, 50)),
 1789:     receive stop_work -> receive after infinity -> ok end after 0 -> ok end,
 1790:     case PartTime of
 1791: 	true -> receive after 1 -> ok end;
 1792: 	false -> ok
 1793:     end,
 1794:     do_work(PartTime).
 1795: 
 1796: workers(N, _Prio, _PartTime) when N =< 0 ->
 1797:     [];
 1798: workers(N, Prio, PartTime) ->
 1799:     Parent = self(),
 1800:     W = spawn_opt(fun () ->
 1801: 			  Parent ! {ready, self()},
 1802: 			  receive
 1803: 			      go_work ->
 1804: 				  do_work(PartTime)
 1805: 			  end
 1806: 		  end,
 1807: 		  [{priority, Prio}, link]),
 1808:     Ws = workers(N-1, Prio, PartTime),
 1809:     receive {ready, W} -> ok end,
 1810:     [W|Ws].
 1811: 
 1812: workers(N, Prio) ->
 1813:     workers(N, Prio, false).
 1814: 
 1815: part_time_workers(N, Prio) ->
 1816:     workers(N, Prio, true).
 1817: 
 1818: tracer(Low, Normal, High, Max) ->
 1819:     receive
 1820: 	{tracees, Prio, Tracees} ->
 1821: 	    save_tracees(Prio, Tracees),
 1822: 	    case Prio of
 1823: 		low -> tracer(Tracees++Low, Normal, High, Max);
 1824: 		normal -> tracer(Low, Tracees++Normal, High, Max);
 1825: 		high -> tracer(Low, Normal, Tracees++High, Max);
 1826: 		max -> tracer(Low, Normal, High, Tracees++Max)
 1827: 	    end;
 1828: 	{get_result, Ref, Who} ->
 1829: 	    Delivered = erlang:trace_delivered(all),
 1830: 	    receive
 1831: 		{trace_delivered, all, Delivered} ->
 1832: 		    ok
 1833: 	    end,
 1834: 	    {Lc, Nc, Hc, Mc} = read_trace(),
 1835: 	    GetMinMax
 1836: 		= fun (Prio, Procs) ->
 1837: 			  LargeNum = 1 bsl 64,
 1838: 			  case lists:foldl(fun (P, {Mn, Mx} = MnMx) ->
 1839: 						   {Prio, C} = get(P),
 1840: 						   case C < Mn of
 1841: 						       true ->
 1842: 							   case C > Mx of
 1843: 							       true ->
 1844: 								   {C, C};
 1845: 							       false ->
 1846: 								   {C, Mx}
 1847: 							   end;
 1848: 						       false ->
 1849: 							   case C > Mx of
 1850: 							       true -> {Mn, C};
 1851: 							       false -> MnMx
 1852: 							   end
 1853: 						   end
 1854: 					   end,
 1855: 					   {LargeNum, 0},
 1856: 					   Procs) of
 1857: 			      {LargeNum, 0} -> {0, 0};
 1858: 			      Res -> Res
 1859: 			  end
 1860: 		  end,
 1861: 	    {Lmin, Lmax} = GetMinMax(low, Low),
 1862: 	    {Nmin, Nmax} = GetMinMax(normal, Normal),
 1863: 	    {Hmin, Hmax} = GetMinMax(high, High),
 1864: 	    {Mmin, Mmax} = GetMinMax(max, Max),
 1865: 	    Who ! {trace_result, Ref, [{low, Lc, Lmin, Lmax},
 1866: 				       {normal, Nc, Nmin, Nmax},
 1867: 				       {high, Hc, Hmin, Hmax},
 1868: 				       {max, Mc, Mmin, Mmax}]}
 1869:     end.
 1870: 
 1871: read_trace() ->
 1872:     read_trace(0,0,0,0).
 1873: 
 1874: read_trace(Low, Normal, High, Max) ->
 1875:     receive
 1876: 	{trace, Proc, in, _} ->
 1877: 	    {Prio, Count} = get(Proc),
 1878: 	    put(Proc, {Prio, Count+1}),
 1879: 	    case Prio of
 1880: 		low -> read_trace(Low+1, Normal, High, Max);
 1881: 		normal -> read_trace(Low, Normal+1, High, Max);
 1882: 		high -> read_trace(Low, Normal, High+1, Max);
 1883: 		max -> read_trace(Low, Normal, High, Max+1)
 1884: 	    end;
 1885: 	{trace, _Proc, out, _} ->
 1886: 	    read_trace(Low, Normal, High, Max)
 1887:     after 0 ->
 1888: 	    {Low, Normal, High, Max}
 1889:     end.
 1890: 
 1891: save_tracees(_Prio, []) ->
 1892:     ok;
 1893: save_tracees(Prio, [T|Ts]) ->
 1894:     put(T, {Prio, 0}),
 1895:     save_tracees(Prio, Ts).
 1896: 	    
 1897: start_tracer() ->
 1898:     Tracer = spawn_link(fun () -> tracer([], [], [], []) end),
 1899:     true = erlang:suspend_process(Tracer),
 1900:     Tracer.
 1901: 
 1902: get_trace_result(Tracer) ->
 1903:     erlang:resume_process(Tracer),
 1904:     Ref = make_ref(),
 1905:     Tracer ! {get_result, Ref, self()},
 1906:     receive
 1907: 	{trace_result, Ref, Res} ->
 1908: 	    Res
 1909:     end.
 1910: 	
 1911: 
 1912: set_tracing(_On, _Tracer, _Prio, []) ->
 1913:     ok;
 1914: set_tracing(true, Tracer, Prio, Pids) ->
 1915:     Tracer ! {tracees, Prio, Pids},
 1916:     set_tracing(true, Tracer, Pids);
 1917: set_tracing(false, Tracer, _Prio, Pids) ->
 1918:     set_tracing(false, Tracer, Pids).
 1919: 
 1920: set_tracing(_On, _Tracer, []) ->
 1921:     ok;
 1922: set_tracing(On, Tracer, [Pid|Pids]) ->
 1923:     1 = erlang:trace(Pid, On, [running, {tracer, Tracer}]),
 1924:     set_tracing(On, Tracer, Pids).
 1925:     
 1926: active_schedulers() ->
 1927:     case erlang:system_info(schedulers_online) of
 1928: 	1 ->
 1929: 	    1;
 1930: 	N ->
 1931: 	    case erlang:system_info(multi_scheduling) of
 1932: 		blocked -> 1;
 1933: 		enabled -> N
 1934: 	    end
 1935:     end.
 1936:     
 1937: start_node(Config) ->
 1938:     start_node(Config, "").
 1939: 
 1940: start_node(Config, Args) when is_list(Config) ->
 1941:     ?line Pa = filename:dirname(code:which(?MODULE)),
 1942:     ?line {A, B, C} = now(),
 1943:     ?line Name = list_to_atom(atom_to_list(?MODULE)
 1944: 			      ++ "-"
 1945: 			      ++ atom_to_list(?config(testcase, Config))
 1946: 			      ++ "-"
 1947: 			      ++ integer_to_list(A)
 1948: 			      ++ "-"
 1949: 			      ++ integer_to_list(B)
 1950: 			      ++ "-"
 1951: 			      ++ integer_to_list(C)),
 1952:     ?line ?t:start_node(Name, slave, [{args, "-pa "++Pa++" "++Args}]).
 1953: 
 1954: stop_node(Node) ->
 1955:     ?t:stop_node(Node).
 1956: 
 1957: 
 1958: enable_internal_state() ->
 1959:     case catch erts_debug:get_internal_state(available_internal_state) of
 1960: 	true -> true;
 1961: 	_ -> erts_debug:set_internal_state(available_internal_state, true)
 1962:     end.
 1963: 
 1964: cmp(X, X) ->
 1965:     ok;
 1966: cmp(X, Y) ->
 1967:     ?t:format("cmp failed:~n X=~p~n Y=~p~n", [X,Y]),
 1968:     cmp_aux(X, Y).
 1969: 
 1970: 
 1971: cmp_aux([X0|Y0], [X1|Y1]) ->
 1972:     cmp_aux(X0, X1),
 1973:     cmp_aux(Y0, Y1);
 1974: cmp_aux(T0, T1) when is_tuple(T0), is_tuple(T1), size(T0) == size(T1) ->
 1975:     cmp_tuple(T0, T1, 1, size(T0));
 1976: cmp_aux(X, X) ->
 1977:     ok;
 1978: cmp_aux(F0, F1) ->
 1979:     ?t:fail({no_match, F0, F1}).
 1980: 
 1981: cmp_tuple(_T0, _T1, N, Sz) when N > Sz ->
 1982:     ok;
 1983: cmp_tuple(T0, T1, N, Sz) ->
 1984:     cmp_aux(element(N, T0), element(N, T1)),
 1985:     cmp_tuple(T0, T1, N+1, Sz).