1: %% 2: %% %CopyrightBegin% 3: %% 4: %% Copyright Ericsson AB 2001-2011. All Rights Reserved. 5: %% 6: %% The contents of this file are subject to the Erlang Public License, 7: %% Version 1.1, (the "License"); you may not use this file except in 8: %% compliance with the License. You should have received a copy of the 9: %% Erlang Public License along with this software. If not, it can be 10: %% retrieved online at http://www.erlang.org/. 11: %% 12: %% Software distributed under the License is distributed on an "AS IS" 13: %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 14: %% the License for the specific language governing rights and limitations 15: %% under the License. 16: %% 17: %% %CopyrightEnd% 18: %% 19: -module(file_sorter_SUITE). 20: 21: %-define(debug, true). 22: 23: -ifdef(debug). 24: -define(format(S, A), io:format(S, A)). 25: -define(line, put(line, ?LINE), ). 26: -define(config(X,Y), foo). 27: -define(t,test_server). 28: -define(privdir(_), "./file_sorter_SUITE_priv"). 29: -else. 30: -include_lib("test_server/include/test_server.hrl"). 31: -define(format(S, A), ok). 32: -define(privdir(Conf), ?config(priv_dir, Conf)). 33: -endif. 34: 35: -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 36: init_per_group/2,end_per_group/2, basic/1, badarg/1, 37: term_sort/1, term_keysort/1, 38: binary_term_sort/1, binary_term_keysort/1, 39: binary_sort/1, 40: term_merge/1, term_keymerge/1, 41: binary_term_merge/1, binary_term_keymerge/1, 42: binary_merge/1, 43: term_check/1, term_keycheck/1, 44: binary_term_check/1, binary_term_keycheck/1, 45: binary_check/1, 46: inout/1, misc/1, many/1]). 47: 48: -export([init_per_testcase/2, end_per_testcase/2]). 49: 50: init_per_testcase(_Case, Config) -> 51: Dog=?t:timetrap(?t:minutes(2)), 52: [{watchdog, Dog}|Config]. 53: 54: end_per_testcase(_Case, Config) -> 55: Dog=?config(watchdog, Config), 56: test_server:timetrap_cancel(Dog), 57: ok. 58: 59: suite() -> [{ct_hooks,[ts_install_cth]}]. 60: 61: all() -> 62: [basic, badarg, term_sort, term_keysort, 63: binary_term_sort, binary_term_keysort, binary_sort, 64: term_merge, term_keymerge, binary_term_merge, 65: binary_term_keymerge, binary_merge, term_check, 66: binary_term_keycheck, binary_term_check, 67: binary_term_keycheck, binary_check, inout, misc, many]. 68: 69: groups() -> 70: []. 71: 72: init_per_suite(Config) -> 73: Config. 74: 75: end_per_suite(_Config) -> 76: ok. 77: 78: init_per_group(_GroupName, Config) -> 79: Config. 80: 81: end_per_group(_GroupName, Config) -> 82: Config. 83: 84: 85: basic(doc) -> 86: ["Basic test case."]; 87: basic(suite) -> 88: []; 89: basic(Config) when is_list(Config) -> 90: Fmt = binary, 91: Arg = {format,Fmt}, 92: Foo = outfile("foo", Config), 93: P0 = pps(), 94: 95: ?line F1s = [F1] = to_files([[]], Fmt, Config), 96: ?line ok = file_sorter:sort(F1), 97: ?line [] = from_files(F1, Fmt), 98: ?line ok = file_sorter:keysort(17, F1), 99: ?line [] = from_files(F1, Fmt), 100: ?line ok = file_sorter:merge(F1s, Foo), 101: ?line [] = from_files(Foo, Fmt), 102: ?line delete_files(Foo), 103: ?line ok = file_sorter:keymerge(17, F1s, Foo), 104: ?line [] = from_files(Foo, Fmt), 105: ?line delete_files([Foo | F1s]), 106: 107: ?line [F2] = to_files([[foo,bar]], Fmt, Config), 108: ?line ok = file_sorter:sort([F2], F2, Arg), 109: ?line [bar,foo] = from_files(F2, Fmt), 110: ?line delete_files(F2), 111: 112: ?line Fs1 = to_files([[foo],[bar]], Fmt, Config), 113: ?line ok = file_sorter:sort(Fs1, Foo, Arg), 114: ?line [bar,foo] = from_files(Foo, Fmt), 115: ?line delete_files(Foo), 116: ?line ok = file_sorter:merge(Fs1, Foo, Arg), 117: ?line [bar,foo] = from_files(Foo, Fmt), 118: ?line delete_files([Foo | Fs1]), 119: 120: ?line Fmt2 = binary_term, 121: ?line Arg2 = {format, Fmt2}, 122: ?line [F3] = to_files([[{foo,1},{bar,2}]], Fmt2, Config), 123: ?line ok = file_sorter:keysort([2], [F3], F3, Arg2), 124: ?line [{foo,1},{bar,2}] = from_files(F3, Fmt2), 125: ?line delete_files(F3), 126: 127: ?line Fs2 = to_files([[{foo,1}],[{bar,2}]], Fmt2, Config), 128: ?line ok = file_sorter:keysort(1, Fs2, Foo, Arg2), 129: ?line [{bar,2},{foo,1}] = from_files(Foo, Fmt2), 130: ?line delete_files(Foo), 131: ?line ok = file_sorter:keymerge(1, Fs2, Foo, Arg2), 132: ?line [{bar,2},{foo,1}] = from_files(Foo, Fmt2), 133: ?line delete_files([Foo | Fs2]), 134: 135: ?line true = P0 =:= pps(), 136: 137: ok. 138: 139: badarg(doc) -> 140: ["Call functions with bad arguments."]; 141: badarg(suite) -> 142: []; 143: badarg(Config) when is_list(Config) -> 144: PrivDir = ?privdir(Config), 145: BadFile = filename:join(PrivDir, "not_a_file"), 146: ABadFile = filename:absname(BadFile), 147: ?line file:delete(BadFile), 148: ?line {error,{file_error,ABadFile,enoent}} = 149: file_sorter:sort(BadFile), 150: ?line {'EXIT', {{badarg, {flipp}}, _}} = 151: (catch file_sorter:sort({flipp})), 152: ?line {error,{file_error,ABadFile,enoent}} = 153: file_sorter:keysort(1, BadFile), 154: ?line {'EXIT', {{badarg, {flipp}}, _}} = 155: (catch file_sorter:keysort(1, {flipp})), 156: 157: ?line {'EXIT', {{badarg, {flipp}}, _}} = 158: (catch file_sorter:merge([{flipp}],foo)), 159: ?line {error,{file_error,ABadFile,enoent}} = 160: file_sorter:keymerge(1,[BadFile],foo), 161: ?line {'EXIT', {{badarg, {flipp}}, _}} = 162: (catch file_sorter:keymerge(1,[{flipp}],foo)), 163: ?line {'EXIT', {{badarg, _}, _}} = 164: (catch file_sorter:merge(fun(X) -> X end, foo)), 165: ?line {'EXIT', {{badarg, _}, _}} = 166: (catch file_sorter:keymerge(1, fun(X) -> X end, foo)), 167: 168: ?line {error,{file_error,ABadFile,enoent}} = 169: file_sorter:check(BadFile), 170: ?line {'EXIT', {{badarg, {flipp}}, _}} = 171: (catch file_sorter:check({flipp})), 172: ?line {error,{file_error,ABadFile,enoent}} = 173: file_sorter:keycheck(1, BadFile), 174: ?line {'EXIT', {{badarg, {flipp}}, _}} = 175: (catch file_sorter:keycheck(1, {flipp})), 176: ?line {'EXIT', {{badarg, {flipp}}, _}} = 177: (catch file_sorter:check([{flipp}],foo)), 178: ?line {'EXIT', {{badarg, {flipp}}, _}} = 179: (catch file_sorter:keycheck(1,[{flipp}],foo)), 180: ?line {'EXIT', {{badarg, _}, _}} = 181: (catch file_sorter:check(fun(X) -> X end, foo)), 182: ?line {'EXIT', {{badarg, _}, _}} = 183: (catch file_sorter:keycheck(1, fun(X) -> X end, foo)), 184: 185: ?line Fs1 = to_files([[1,2,3]], binary_term, Config), 186: ?line {'EXIT', {{badarg, flipp}, _}} = 187: (catch file_sorter:check(Fs1 ++ flipp, [])), 188: [F1] = Fs1, 189: ?line {error,{file_error,_,_}} = 190: file_sorter:sort(Fs1, foo, [{tmpdir,F1},{size,0}]), 191: ?line delete_files(Fs1), 192: ?line Fs2 = to_files([[1,2,3]], binary_term, Config), 193: {error,{file_error,_,enoent}} = 194: file_sorter:sort(Fs2, foo, [{tmpdir,filename:absname(BadFile)}, 195: {size,0}]), 196: ?line delete_files(Fs2), 197: 198: ?line {'EXIT', {{badarg, bad}, _}} = 199: (catch file_sorter:check([], [{format,term} | bad])), 200: ?line {'EXIT', {{badarg, [{flipp}]}, _}} = 201: (catch file_sorter:check([{flipp}])), 202: ?line {'EXIT', {{badarg, {flipp}}, _}} = 203: (catch file_sorter:keycheck(1, {flipp})), 204: ?line {'EXIT', {{badarg, [{flipp}]}, _}} = 205: (catch file_sorter:keycheck(2, [{flipp}])), 206: ?line {error,{file_error,_,eisdir}} = file_sorter:keycheck(1, []), 207: ?line {'EXIT', {{badarg, kp}, _}} = (catch file_sorter:keycheck(kp, [])), 208: ?line {'EXIT', {{badarg, kp}, _}} = 209: (catch file_sorter:keycheck([1, kp], [])), 210: ?line {'EXIT', {{badarg, kp}, _}} = 211: (catch file_sorter:keycheck([1 | kp], [])), 212: ?line {'EXIT', {{badarg, []}, _}} = (catch file_sorter:keycheck([], [])), 213: ?line {'EXIT', {{badarg, {format, foo}}, _}} = 214: (catch file_sorter:check([], {format,foo})), 215: ?line {'EXIT', {{badarg, not_an_option}, _}} = 216: (catch file_sorter:keycheck(7, [], [not_an_option])), 217: ?line {'EXIT', {{badarg, format}, _}} = 218: (catch file_sorter:keycheck(1, [], [{format, binary}])), 219: ?line {'EXIT', {{badarg, order}, _}} = 220: (catch file_sorter:keycheck(1, [], [{order, fun compare/2}])), 221: 222: ?line do_badarg(fun(I, O) -> file_sorter:sort(I, O) end, 223: fun(Kp, I, O) -> file_sorter:keysort(Kp, I, O) end, 224: BadFile), 225: ?line do_badarg_opt(fun(I, O, X) -> file_sorter:sort(I, O, X) end, 226: fun(Kp, I, O, X) -> file_sorter:keysort(Kp, I, O, X) 227: end), 228: ?line do_badarg(fun(I, O) -> file_sorter:merge(I, O) end, 229: fun(Kp, I, O) -> file_sorter:keymerge(Kp, I, O) end, 230: BadFile), 231: ?line do_badarg_opt(fun(I, O, X) -> file_sorter:merge(I, O, X) end, 232: fun(Kp, I, O, X) -> file_sorter:keymerge(Kp, I, O, X) 233: end). 234: 235: do_badarg(F, KF, BadFile) -> 236: [Char | _] = BadFile, 237: AFlipp = filename:absname(flipp), 238: ?line {error,{file_error,AFlipp,enoent}} = F([flipp | flopp], foo), 239: ?line {'EXIT', {{badarg, {foo,bar}}, _}} = (catch F([], {foo,bar})), 240: ?line {'EXIT', {{badarg, Char}, _}} = (catch F(BadFile, [])), 241: ?line {'EXIT', {{badarg, {flipp}}, _}} = (catch F({flipp}, [])), 242: 243: ?line {'EXIT', {{badarg, Char}, _}} = (catch KF(1, BadFile, [])), 244: ?line {'EXIT', {{badarg, {flipp}}, _}} = (catch KF(1, {flipp}, [])), 245: ?line {error,{file_error,AFlipp,enoent}} = 246: KF(2, [flipp | flopp], foo), 247: ?line {'EXIT', {{badarg, {foo,bar}}, _}} = (catch KF(1, [], {foo,bar})), 248: ?line {'EXIT', {{badarg, kp}, _}} = (catch KF(kp, [], foo)), 249: ?line {'EXIT', {{badarg, kp}, _}} = (catch KF([1, kp], [], foo)), 250: ?line {'EXIT', {{badarg, kp}, _}} = (catch KF([1 | kp], [], foo)), 251: ?line {'EXIT', {{badarg, []}, _}} = (catch KF([], [], foo)), 252: ok. 253: 254: do_badarg_opt(F, KF) -> 255: AFlipp = filename:absname(flipp), 256: ?line {error,{file_error,AFlipp,enoent}} = 257: F([flipp | flopp], foo, []), 258: ?line {'EXIT', {{badarg, {flipp}}, _}} = (catch F([{flipp}], foo, [])), 259: ?line {'EXIT', {{badarg, {out,put}}, _}} = (catch F([], {out,put}, [])), 260: ?line {'EXIT', {{badarg, not_an_option}, _}} = 261: (catch F([], foo, [not_an_option])), 262: ?line {'EXIT', {{badarg, {format, foo}}, _}} = 263: (catch F([], foo, {format,foo})), 264: ?line {'EXIT', {{badarg, {size,foo}}, _}} = (catch F([], foo, {size,foo})), 265: 266: ?line {'EXIT', {{badarg, {size, -1}}, _}} = (catch F([], foo, {size,-1})), 267: ?line {'EXIT', {{badarg, {no_files, foo}}, _}} = 268: (catch F([], foo, {no_files,foo})), 269: ?line {'EXIT', {{badarg, {no_files, 1}}, _}} = 270: (catch F([], foo, {no_files,1})), 271: ?line {'EXIT', {{badarg, 1}, _}} = (catch F([], foo, {tmpdir,1})), 272: ?line {'EXIT', {{badarg, {order,1}}, _}} = (catch F([], foo, {order,1})), 273: ?line {'EXIT', {{badarg, {compressed, flopp}}, _}} = 274: (catch F([], foo, {compressed,flopp})), 275: ?line {'EXIT', {{badarg, {unique,flopp}}, _}} = 276: (catch F([], foo, {unique,flopp})), 277: ?line {'EXIT', {{badarg, {header,foo}}, _}} = 278: (catch F([], foo, {header,foo})), 279: ?line {'EXIT', {{badarg, {header, 0}}, _}} = 280: (catch F([], foo, {header,0})), 281: ?line {'EXIT', {{badarg, {header, 1 bsl 35}}, _}} = 282: (catch F([], foo, {header,1 bsl 35})), 283: ?line {'EXIT', {{badarg, header}, _}} = 284: (catch F([], foo, [{header,1},{format,term}])), 285: 286: ?line {'EXIT', {{badarg, not_an_option}, _}} = 287: (catch KF(7, [], foo, [not_an_option])), 288: ?line {'EXIT', {{badarg,format}, _}} = 289: (catch KF(1, [], foo, [{format, binary}])), 290: ?line {'EXIT', {{badarg, order}, _}} = 291: (catch KF(1, [], foo, [{order, fun compare/2}])), 292: ?line {'EXIT', {{badarg, {flipp}}, _}} = 293: (catch KF(2, [{flipp}], foo,[])), 294: ?line {error,{file_error,AFlipp,enoent}} = 295: KF(2, [flipp | flopp], foo,[]), 296: ?line {'EXIT', {{badarg, {out, put}}, _}} = 297: (catch KF(1, [], {out,put}, [])), 298: ?line {'EXIT', {{badarg, kp}, _}} = (catch KF(kp, [], foo, [])), 299: ?line {'EXIT', {{badarg, kp}, _}} = (catch KF([1, kp], [], foo, [])), 300: ?line {'EXIT', {{badarg, kp}, _}} = (catch KF([1 | kp], [], foo, [])), 301: ok. 302: 303: term_sort(doc) -> 304: ["Sort terms on files."]; 305: term_sort(suite) -> 306: []; 307: term_sort(Config) when is_list(Config) -> 308: ?line sort(term, [{compressed,false}], Config), 309: ?line sort(term, [{order, fun compare/2}], Config), 310: ?line sort(term, [{order, ascending}, {compressed,true}], Config), 311: ?line sort(term, [{order, descending}], Config), 312: ok. 313: 314: term_keysort(doc) -> 315: ["Keysort terms on files."]; 316: term_keysort(suite) -> 317: []; 318: term_keysort(Config) when is_list(Config) -> 319: ?line keysort(term, [{tmpdir, ""}], Config), 320: ?line keysort(term, [{order,descending}], Config), 321: ok. 322: 323: binary_term_sort(doc) -> 324: ["Sort binary terms on files."]; 325: binary_term_sort(suite) -> 326: []; 327: binary_term_sort(Config) when is_list(Config) -> 328: PrivDir = ?privdir(Config), 329: ?line sort({2, binary_term}, [], Config), 330: ?line sort(binary_term, [{tmpdir, list_to_atom(PrivDir)}], Config), 331: ?line sort(binary_term, [{tmpdir,PrivDir}], Config), 332: ?line sort({3,binary_term}, [{order, fun compare/2}], Config), 333: ?line sort(binary_term, [{order, fun compare/2}], Config), 334: ?line sort(binary_term, [{order,descending}], Config), 335: ok. 336: 337: binary_term_keysort(doc) -> 338: ["Keysort binary terms on files."]; 339: binary_term_keysort(suite) -> 340: []; 341: binary_term_keysort(Config) when is_list(Config) -> 342: ?line keysort({3, binary_term}, [], Config), 343: ?line keysort(binary_term, [], Config), 344: ?line keysort(binary_term, [{order,descending}], Config), 345: ok. 346: 347: binary_sort(doc) -> 348: ["Sort binaries on files."]; 349: binary_sort(suite) -> 350: []; 351: binary_sort(Config) when is_list(Config) -> 352: PrivDir = ?privdir(Config), 353: ?line sort({2, binary}, [], Config), 354: ?line sort(binary, [{tmpdir, list_to_atom(PrivDir)}], Config), 355: ?line sort(binary, [{tmpdir,PrivDir}], Config), 356: ?line sort({3,binary}, [{order, fun compare/2}], Config), 357: ?line sort(binary, [{order, fun compare/2}], Config), 358: ?line sort(binary, [{order,descending}], Config), 359: ok. 360: 361: term_merge(doc) -> 362: ["Merge terms on files."]; 363: term_merge(suite) -> 364: []; 365: term_merge(Config) when is_list(Config) -> 366: ?line merge(term, [{order, fun compare/2}], Config), 367: ?line merge(term, [{order, ascending}, {compressed,true}], Config), 368: ?line merge(term, [{order, descending}, {compressed,false}], Config), 369: ok. 370: 371: term_keymerge(doc) -> 372: ["Keymerge terms on files."]; 373: term_keymerge(suite) -> 374: []; 375: term_keymerge(Config) when is_list(Config) -> 376: ?line keymerge(term, [], Config), 377: ?line keymerge(term, [{order, descending}], Config), 378: ?line funmerge(term, [], Config), 379: ok. 380: 381: binary_term_merge(doc) -> 382: ["Merge binary terms on files."]; 383: binary_term_merge(suite) -> 384: []; 385: binary_term_merge(Config) when is_list(Config) -> 386: ?line merge(binary_term, [], Config), 387: ?line merge({7, binary_term}, [], Config), 388: ?line merge({3, binary_term}, [{order, fun compare/2}], Config), 389: ok. 390: 391: binary_term_keymerge(doc) -> 392: ["Keymerge binary terms on files."]; 393: binary_term_keymerge(suite) -> 394: []; 395: binary_term_keymerge(Config) when is_list(Config) -> 396: ?line keymerge({3, binary_term}, [], Config), 397: ?line keymerge(binary_term, [], Config), 398: ?line funmerge({3, binary_term}, [], Config), 399: ?line funmerge(binary_term, [], Config), 400: ok. 401: 402: binary_merge(doc) -> 403: ["Merge binaries on files."]; 404: binary_merge(suite) -> 405: []; 406: binary_merge(Config) when is_list(Config) -> 407: ?line merge(binary, [], Config), 408: ?line merge({7, binary}, [], Config), 409: ?line merge({3, binary}, [{order, fun compare/2}], Config), 410: ok. 411: 412: term_check(doc) -> 413: ["Check terms on files."]; 414: term_check(suite) -> 415: []; 416: term_check(Config) when is_list(Config) -> 417: ?line check(term, Config), 418: ok. 419: 420: binary_term_check(doc) -> 421: ["Check binary terms on files."]; 422: binary_term_check(suite) -> 423: []; 424: binary_term_check(Config) when is_list(Config) -> 425: ?line check(binary_term, Config), 426: ok. 427: 428: term_keycheck(doc) -> 429: ["Keycheck terms on files."]; 430: term_keycheck(suite) -> 431: []; 432: term_keycheck(Config) when is_list(Config) -> 433: ?line keycheck(term, Config), 434: ok. 435: 436: binary_term_keycheck(doc) -> 437: ["Keycheck binary terms on files."]; 438: binary_term_keycheck(suite) -> 439: []; 440: binary_term_keycheck(Config) when is_list(Config) -> 441: ?line keycheck(binary_term, Config), 442: ok. 443: 444: binary_check(doc) -> 445: ["Check binary terms on files."]; 446: binary_check(suite) -> 447: []; 448: binary_check(Config) when is_list(Config) -> 449: ?line check(binary, Config), 450: ok. 451: 452: inout(doc) -> 453: ["Funs as input or output."]; 454: inout(suite) -> 455: []; 456: inout(Config) when is_list(Config) -> 457: BTF = {format, binary_term}, 458: Foo = outfile("foo", Config), 459: 460: %% Input is fun. 461: End = fun(read) -> end_of_input end, 462: 463: IF1 = fun(read) -> {[1,7,5], End} end, 464: ?line ok = file_sorter:sort(IF1, Foo, [{format, term}]), 465: %% 'close' is called, but the return value is caught and ignored. 466: IF2 = fun(read) -> {[1,2,3], fun(close) -> throw(ignored) end} end, 467: ?line {error, bad_object} = file_sorter:sort(IF2, Foo, BTF), 468: 469: IF3 = fun(no_match) -> foo end, 470: ?line {'EXIT', {function_clause, _}} = 471: (catch file_sorter:sort(IF3, Foo)), 472: IF4 = fun(read) -> throw(my_message) end, 473: ?line my_message = (catch file_sorter:sort(IF4, Foo)), 474: IF5 = fun(read) -> {error, my_error} end, 475: ?line {error, my_error} = file_sorter:sort(IF5, Foo), 476: 477: %% Output is fun. 478: ?line {error, bad_object} = 479: file_sorter:sort(IF2, fun(close) -> ignored end, BTF), 480: Args = [{format, term}], 481: ?line {error, bad_object} = 482: file_sorter:keysort(1, IF2, fun(close) -> ignored end, Args), 483: OF1 = fun(close) -> fine; (L) when is_list(L) -> fun(close) -> nice end end, 484: ?line nice = file_sorter:sort(IF1, OF1, Args), 485: OF2 = fun(_) -> my_return end, 486: ?line my_return = file_sorter:sort(IF1, OF2, Args), 487: OF3 = fun(_) -> throw(my_message) end, 488: ?line my_message = (catch file_sorter:sort(IF1, OF3, Args)), 489: OF4 = fun(no_match) -> foo end, 490: ?line {'EXIT', {function_clause, _}} = 491: (catch file_sorter:sort(IF1, OF4, Args)), 492: 493: ?line P0 = pps(), 494: ?line Fs1 = to_files([[3,1,2,5,4], [8,3,10]], term, Config), 495: ?line error = file_sorter:sort(Fs1, fun(_) -> error end, Args), 496: ?line delete_files(Fs1), 497: 498: ?line true = P0 =:= pps(), 499: 500: %% Passing a value from the input functions to the output functions. 501: IFV1 = fun(read) -> {end_of_input, 17} end, 502: OFV1 = fun({value, Value}) -> ofv(Value, []) end, 503: ?line {17, []} = file_sorter:sort(IFV1, OFV1, Args), 504: 505: %% Output is not a fun. The value returned by input funs is ignored. 506: %% OTP-5009. 507: ?line ok = file_sorter:sort(IFV1, Foo, [{format,term}]), 508: ?line [] = from_files(Foo, term), 509: ?line delete_files(Foo), 510: 511: ok. 512: 513: ofv(Value, A) -> 514: fun(close) -> 515: {Value, lists:append(lists:reverse(A))}; 516: (L) when is_list(L) -> 517: ofv(Value, [L | A]) 518: end. 519: 520: many(doc) -> 521: ["Many temporary files."]; 522: many(suite) -> 523: []; 524: many(Config) when is_list(Config) -> 525: Foo = outfile("foo", Config), 526: PrivDir = ?privdir(Config), 527: P0 = pps(), 528: 529: Args = [{format, term}], 530: L1 = lists:map(fun(I) -> {one, two, three, I} end, lists:seq(1,1000)), 531: L2 = lists:map(fun(I) -> {four, five, six, I} end, lists:seq(1,1000)), 532: ?line Fs2 = to_files([L1, L2], term, Config), 533: ?line ok = file_sorter:sort(Fs2, Foo, [{size,1000} | Args]), 534: ?line R = lists:sort(L1++L2), 535: ?line R = from_files(Foo, term), 536: ?line 2000 = length(R), 537: ?line ok = file_sorter:sort(Fs2, Foo, [{no_files,4},{size,1000} | Args]), 538: ?line R = from_files(Foo, term), 539: ?line ok = 540: file_sorter:sort(Fs2, Foo, 541: [{no_files,4},{size,1000},{order,descending} | Args]), 542: ?line true = lists:reverse(R) =:= from_files(Foo, term), 543: ?line ok = 544: file_sorter:sort(Fs2, Foo, 545: [{no_files,4},{size,1000}, 546: {order,fun compare/2} | Args]), 547: ?line R = from_files(Foo, term), 548: ?line ok = file_sorter:keysort(4, Fs2, Foo, 549: [{no_files,4},{size,1000} | Args]), 550: ?line RK = lists:keysort(4, L1++L2), 551: ?line RK = from_files(Foo, term), 552: ?line delete_files(Foo), 553: ?line ok = 554: file_sorter:keysort(4, Fs2, Foo, 555: [{no_files,4},{size,1000},{order,descending} | Args]), 556: ?line true = lists:reverse(RK) =:= from_files(Foo, term), 557: ?line delete_files(Foo), 558: ?line ok = file_sorter:keysort(4, Fs2, Foo, 559: [{size,500},{order,descending} | Args]), 560: ?line true = lists:reverse(RK) =:= from_files(Foo, term), 561: ?line delete_files(Foo), 562: ?line error = file_sorter:sort(Fs2, fun(_) -> error end, 563: [{tmpdir, PrivDir}, {no_files,3}, 564: {size,10000} | Args]), 565: 566: TmpDir = filename:join(PrivDir, "tmpdir"), 567: file:del_dir(TmpDir), 568: ?line ok = file:make_dir(TmpDir), 569: ?line case os:type() of 570: {unix, _} -> 571: ?line ok = file:change_mode(TmpDir, 8#0000), 572: ?line {error, {file_error, _,_}} = 573: file_sorter:sort(Fs2, fun(_M) -> foo end, 574: [{no_files,3},{size,10000}, 575: {tmpdir,TmpDir} | Args]); 576: _ -> 577: true 578: end, 579: ?line ok = file:del_dir(TmpDir), 580: delete_files(Fs2), 581: ?line true = P0 =:= pps(), 582: ok. 583: 584: misc(doc) -> 585: ["Some other tests."]; 586: misc(suite) -> 587: []; 588: misc(Config) when is_list(Config) -> 589: BTF = {format, binary_term}, 590: Foo = outfile("foo", Config), 591: FFoo = filename:absname(Foo), 592: P0 = pps(), 593: 594: ?line [File] = Fs1 = to_files([[1,3,2]], term, Config), 595: ?line ok = file:write_file(Foo,<<>>), 596: ?line case os:type() of 597: {unix, _} -> 598: ok = file:change_mode(Foo, 8#0000), 599: {error,{file_error,FFoo,eacces}} = 600: file_sorter:sort(Fs1, Foo, {format,term}); 601: _ -> 602: true 603: end, 604: ?line file:delete(Foo), 605: ?line NoBytes = 16, % RAM memory will never get this big, or? 606: ?line ALot = (1 bsl (NoBytes*8)) - 1, 607: ?line ok = file:write_file(File, <<ALot:NoBytes/unit:8,"foobar">>), 608: FFile = filename:absname(File), 609: ?line {error, {bad_object,FFile}} = 610: file_sorter:sort(Fs1, Foo, [BTF, {header, 20}]), 611: ?line ok = file:write_file(File, <<30:32,"foobar">>), 612: ?line {error, {premature_eof, FFile}} = file_sorter:sort(Fs1, Foo, BTF), 613: ?line ok = file:write_file(File, <<6:32,"foobar">>), 614: ?line {error, {bad_object,FFile}} = file_sorter:sort(Fs1, Foo, BTF), 615: ?line case os:type() of 616: {unix, _} -> 617: ok = file:change_mode(File, 8#0000), 618: {error, {file_error,FFile,eacces}} = 619: file_sorter:sort(Fs1, Foo), 620: {error, {file_error,FFile,eacces}} = 621: file_sorter:sort(Fs1, Foo, {format, binary_term}); 622: _ -> 623: true 624: end, 625: ?line delete_files(Fs1), 626: ?line true = P0 =:= pps(), 627: 628: %% bigger than chunksize 629: ?line E1 = <<32000:32, 10:256000>>, 630: ?line E2 = <<32000:32, 5:256000>>, 631: ?line E3 = <<32000:32, 8:256000>>, 632: ?line ok = file:write_file(Foo, [E1, E2, E3]), 633: ?line ok = file_sorter:sort([Foo], Foo, [{format,binary},{size,10000}]), 634: ?line ok = file_sorter:sort([Foo], Foo, [{format,fun(X) -> X end}, 635: {size,10000}]), 636: ?line Es = list_to_binary([E2,E3,E1]), 637: ?line {ok, Es} = file:read_file(Foo), 638: ?line delete_files(Foo), 639: ?line true = P0 =:= pps(), 640: 641: %% keysort more than one element 642: L = [{c,1,a},{c,2,b},{c,3,c},{b,1,c},{b,2,b},{b,3,a},{a,1,a},{a,2,b}, 643: {a,3,c}], 644: ?line Fs2 = to_files([L], binary_term, Config), 645: ?line ok = file_sorter:keysort([2,3], Fs2, Foo, {format, binary_term}), 646: ?line KS2_1 = from_files(Foo, binary_term), 647: ?line KS2_2 = lists:keysort(2,lists:keysort(3, L)), 648: ?line KS2_1 = KS2_2, 649: ?line ok = file_sorter:keysort([2,3], Fs2, Foo, 650: [{format, binary_term},{size,5}]), 651: ?line KS2_3 = from_files(Foo, binary_term), 652: ?line KS2_3 = KS2_2, 653: ?line ok = file_sorter:keysort([2,3,1], Fs2, Foo, {format, binary_term}), 654: ?line KS3_1 = from_files(Foo, binary_term), 655: ?line KS3_2 = lists:keysort(2, lists:keysort(3,lists:keysort(1, L))), 656: ?line KS3_1 = KS3_2, 657: ?line ok = file_sorter:keysort([2,3,1], Fs2, Foo, 658: [{format, binary_term},{size,5}]), 659: ?line KS3_3 = from_files(Foo, binary_term), 660: ?line KS3_3 = KS3_2, 661: ?line delete_files([Foo | Fs2]), 662: ?line true = P0 =:= pps(), 663: 664: %% bigger than chunksize 665: %% Assumes that CHUNKSIZE = 16384. Illustrates that the Last argument 666: %% of merge_files/5 is necessary. 667: ?line EP1 = erlang:make_tuple(2728,foo), 668: ?line EP2 = lists:duplicate(2729,qqq), 669: ?line LL = [EP1, EP2, EP1, EP2, EP1, EP2], 670: ?line Fs3 = to_files([LL], binary, Config), 671: ?line ok = file_sorter:sort(Fs3, Foo, [{format,binary}, {unique,true}]), 672: ?line [EP1,EP2] = from_files(Foo, binary), 673: ?line delete_files(Foo), 674: ?line ok = file_sorter:sort(Fs3, Foo, 675: [{format,binary_term}, {unique,true}, 676: {size,30000}]), 677: ?line [EP1,EP2] = from_files(Foo, binary_term), 678: ?line delete_files([Foo | Fs3]), 679: 680: ?line true = P0 =:= pps(), 681: 682: ?line BE1 = <<20000:32, 17:160000>>, 683: ?line BE2 = <<20000:32, 1717:160000>>, 684: ?line ok = file:write_file(Foo, [BE1,BE2,BE1,BE2]), 685: ?line ok = file_sorter:sort([Foo], Foo, [{format,binary}, 686: {size,10000}, 687: {unique,true}]), 688: ?line BEs = list_to_binary([BE1, BE2]), 689: ?line {ok, BEs} = file:read_file(Foo), 690: ?line delete_files(Foo), 691: ?line true = P0 =:= pps(), 692: 693: ?line Fs4 = to_files([[7,4,1]], binary_term, Config), 694: ?line {error, {bad_term, _}} = file_sorter:sort(Fs4, Foo, {format, term}), 695: ?line delete_files([Foo | Fs4]), 696: ?line true = P0 =:= pps(), 697: 698: ok. 699: 700: %%% 701: %%% Utilities. 702: %%% 703: 704: sort(Fmt, XArgs, Config) -> 705: Args = make_args(Fmt, [{size,5} | XArgs]), 706: TmpArgs = [{tmpdir,?privdir(Config)} | Args], 707: Foo = outfile("foo", Config), 708: 709: %% Input is a fun. Output is a fun. 710: ?line [] = file_sorter:sort(input([], 2, Fmt), output([], Fmt), Args), 711: ?line L1 = [3,1,2,5,4], 712: ?line S1 = file_sorter:sort(input(L1, 2, Fmt), output([], Fmt), TmpArgs), 713: ?line S1 = rev(lists:sort(L1), TmpArgs), 714: 715: %% Input is a file. Output is a fun. 716: ?line [] = file_sorter:sort([], output([], Fmt), Args), 717: ?line L2 = [3,1,2,5,4], 718: ?line Fs1 = to_files([L2], Fmt, Config), 719: ?line S2 = file_sorter:sort(Fs1, output([], Fmt), TmpArgs), 720: ?line S2 = rev(lists:sort(L2), TmpArgs), 721: ?line delete_files(Fs1), 722: 723: %% Input is a file. Output is a file 724: ?line ok = file_sorter:sort([], Foo, Args), 725: ?line [] = from_files(Foo, Fmt), 726: ?line delete_files(Foo), 727: ?line ok = file_sorter:sort([], Foo, [{unique,true} | Args]), 728: ?line [] = from_files(Foo, Fmt), 729: ?line delete_files(Foo), 730: ?line L3 = [3,1,2,5,4,6], 731: ?line Fs2 = to_files([L3], Fmt, Config), 732: ?line ok = file_sorter:sort(Fs2, Foo, Args), 733: ?line true = rev(lists:sort(L3), Args) =:= from_files(Foo, Fmt), 734: ?line delete_files([Foo | Fs2]), 735: ?line L4 = [1,3,4,1,2,5,4,5,6], 736: ?line Fs3 = to_files([L4], Fmt, Config), 737: ?line ok = file_sorter:sort(Fs3, Foo, Args++[{unique,true}, 738: {size,100000}]), 739: ?line true = rev(lists:usort(L4), Args) =:= from_files(Foo, Fmt), 740: ?line delete_files(Foo), 741: ?line ok = file_sorter:sort(Fs3, Foo, Args++[{unique,true}]), 742: ?line true = rev(lists:usort(L4), Args) =:= from_files(Foo, Fmt), 743: ?line delete_files([Foo | Fs3]), 744: 745: %% Input is a fun. Output is a file. 746: ?line ok = file_sorter:sort(input([], 2, Fmt), Foo, Args), 747: ?line [] = from_files(Foo, Fmt), 748: ?line delete_files(Foo), 749: ?line L5 = [3,1,2,5,4,7], 750: ?line ok = file_sorter:sort(input(L5, 2, Fmt), Foo, Args), 751: ?line true = rev(lists:sort(L5), Args) =:= from_files(Foo, Fmt), 752: ?line delete_files(Foo), 753: 754: %% Removing duplicate keys. 755: KFun = key_compare(2), 756: L6 = [{5,e},{2,b},{3,c},{1,a},{4,d}] ++ [{2,c},{1,b},{4,a}], 757: KUArgs = lists:keydelete(order, 1, Args) ++ 758: [{unique, true}, {order, KFun},{size,100000}], 759: ?line ok = file_sorter:sort(input(L6, 2, Fmt), Foo, KUArgs), 760: ?line true = rev(lists:ukeysort(2, L6), KUArgs) =:= from_files(Foo, Fmt), 761: KArgs = lists:keydelete(unique, 1, KUArgs), 762: ?line ok = file_sorter:sort(input(L6, 2, Fmt), Foo, KArgs), 763: ?line true = rev(lists:keysort(2, L6), KArgs) =:= from_files(Foo, Fmt), 764: 765: %% Removing duplicate keys. Again. 766: KUArgs2 = lists:keydelete(order, 1, Args) ++ 767: [{unique, true}, {order, KFun},{size,5}], 768: ?line ok = file_sorter:sort(input(L6, 2, Fmt), Foo, KUArgs2), 769: ?line true = rev(lists:ukeysort(2, L6), KUArgs2) =:= from_files(Foo, Fmt), 770: KArgs2 = lists:keydelete(unique, 1, KUArgs2), 771: ?line ok = file_sorter:sort(input(L6, 2, Fmt), Foo, KArgs2), 772: ?line true = rev(lists:keysort(2, L6), KArgs2) =:= from_files(Foo, Fmt), 773: ?line delete_files(Foo), 774: 775: ok. 776: 777: keysort(Fmt, XArgs, Config) -> 778: Args = make_args(Fmt, [{size,50}, {no_files, 2} | XArgs]), 779: TmpArgs = Args ++ [{tmpdir,?privdir(Config)}], 780: Foo = outfile("foo", Config), 781: 782: %% Input is files. Output is a file. 783: ?line ok = file_sorter:keysort(2, [], Foo, Args), 784: ?line [] = from_files(Foo, Fmt), 785: ?line delete_files(Foo), 786: ?line ok = file_sorter:keysort(2, [], Foo, [{unique,true} | Args]), 787: ?line [] = from_files(Foo, Fmt), 788: ?line delete_files(Foo), 789: ?line L0 = [{a,2},{a,1},{a,2},{a,2},{a,1},{a,2},{a,2},{a,3}], 790: ?line Fs0 = to_files([L0], Fmt, Config), 791: ?line S = rev(lists:ukeysort(1, L0), Args), 792: ?line ok = 793: file_sorter:keysort(1, Fs0, Foo, Args ++ [{unique,true}, 794: {size,100000}]), 795: ?line S = from_files(Foo, Fmt), 796: ?line ok = 797: file_sorter:keysort(1, Fs0, Foo, Args ++ [{unique,true}, 798: {size,5}]), 799: ?line S = from_files(Foo, Fmt), 800: ?line ok = file_sorter:keysort(1, Fs0, Foo, Args ++ [{unique,true}]), 801: ?line S = from_files(Foo, Fmt), 802: ?line delete_files([Foo | Fs0]), 803: ?line L11 = [{a,1,x4},{b,2,x4},{c,3,x4}], 804: ?line L21 = [{a,1,x3},{b,2,x3},{c,3,x3}], 805: ?line L31 = [{a,1,x2},{b,2,x2},{c,3,x2}], 806: ?line L41 = [{a,1,x1},{b,2,x1},{c,3,x1}], 807: ?line All = [L11, L21, L31, L41], 808: ?line AllFlat = lists:append(All), 809: ?line Sorted = rev(lists:keysort(2, AllFlat), Args), 810: ?line Fs1 = to_files(All, Fmt, Config), 811: ?line ok = file_sorter:keysort(2, Fs1, Foo, Args), 812: ?line Sorted = from_files(Foo, Fmt), 813: ?line delete_files(Foo), 814: 815: %% Input is files. Output is a fun. 816: ?line [] = file_sorter:keysort(2, [], output([], Fmt), Args), 817: ?line KS1 = file_sorter:keysort(2, Fs1, output([], Fmt), TmpArgs), 818: ?line Sorted = KS1, 819: ?line delete_files(Fs1), 820: 821: %% Input is a fun. Output is a file. 822: ?line ok = file_sorter:keysort(2, input([], 2, Fmt), Foo, Args), 823: ?line [] = from_files(Foo, Fmt), 824: ?line delete_files(Foo), 825: ?line ok = file_sorter:keysort(2, input(AllFlat, 4, Fmt), Foo, Args), 826: ?line Sorted = from_files(Foo, Fmt), 827: ?line delete_files(Foo), 828: 829: %% Input is a fun. Output is a fun. 830: ?line [] = file_sorter:keysort(2, input([], 2, Fmt), output([], Fmt),Args), 831: ?line KS2 = 832: file_sorter:keysort(2, input(AllFlat, 4, Fmt), output([], Fmt), 833: TmpArgs), 834: ?line Sorted = KS2, 835: ok. 836: 837: merge(Fmt, XArgs, Config) -> 838: Args = make_args(Fmt, [{size,5} | XArgs]), 839: Foo = outfile("foo", Config), 840: 841: %% Input is a file. Output is a fun. 842: ?line [] = file_sorter:merge([], output([], Fmt), Args), 843: ?line L2 = [[1,3,5],[2,4,5]], 844: ?line Fs1 = to_files(L2, Fmt, Config), 845: ?line S2 = file_sorter:sort(Fs1, output([], Fmt), Args), 846: ?line S2 = rev(lists:sort(lists:append(L2)), Args), 847: ?line delete_files(Fs1), 848: 849: %% Input is a file. Output is a file 850: ?line ok = file_sorter:merge([], Foo, Args), 851: ?line [] = from_files(Foo, Fmt), 852: ?line delete_files(Foo), 853: ?line ok = file_sorter:merge([], Foo, [{unique,true} | Args]), 854: ?line [] = from_files(Foo, Fmt), 855: ?line delete_files(Foo), 856: ?line L31 = [1,2,3], 857: ?line L32 = [2,3,4], 858: ?line L33 = [4,5,6], 859: ?line L3r = [L31, L32, L33], 860: ?line L3 = [rev(L31,Args), rev(L32,Args), rev(L33,Args)], 861: ?line Fs2 = to_files(L3, Fmt, Config), 862: ?line ok = file_sorter:merge(Fs2, Foo, Args), 863: ?line true = rev(lists:merge(L3r), Args) =:= from_files(Foo, Fmt), 864: ?line ok = file_sorter:merge(Fs2, Foo, Args++[{unique,true}, 865: {size,100000}]), 866: ?line true = rev(lists:umerge(L3r), Args) =:= from_files(Foo, Fmt), 867: ?line delete_files(Foo), 868: ?line ok = file_sorter:merge(Fs2, Foo, Args++[{unique,true}]), 869: ?line true = rev(lists:umerge(L3r), Args) =:= from_files(Foo, Fmt), 870: ?line delete_files([Foo | Fs2]), 871: 872: ok. 873: 874: keymerge(Fmt, XArgs, Config) -> 875: Args = make_args(Fmt, [{size,50}, {no_files, 2} | XArgs]), 876: Foo = outfile("foo", Config), 877: 878: %% Input is files. Output is a file. 879: ?line ok = file_sorter:keymerge(2, [], Foo, Args), 880: ?line [] = from_files(Foo, Fmt), 881: ?line delete_files(Foo), 882: ?line ok = file_sorter:keymerge(2, [], Foo, [{unique,true} | Args]), 883: ?line [] = from_files(Foo, Fmt), 884: ?line delete_files(Foo), 885: ?line L0 = [rev([{a,1},{a,2}], Args), rev([{a,2},{a,1},{a,3}], Args)], 886: ?line Fs0 = to_files(L0, Fmt, Config), 887: ?line delete_files(Foo), 888: ?line ok = file_sorter:keymerge(1, Fs0, Foo, Args ++ [{unique,false}]), 889: ?line S2 = rev([{a,1},{a,2},{a,2},{a,1},{a,3}], Args), 890: ?line S2 = from_files(Foo, Fmt), 891: ?line delete_files([Foo | Fs0]), 892: ?line L11 = [{a,1,x4},{b,2,x4},{c,3,x4}], 893: ?line L21 = [{a,1,x3},{b,2,x3},{c,3,x3}], 894: ?line L31 = [{a,1,x2},{b,2,x2},{c,3,x2}], 895: ?line L41 = [{a,1,x1},{b,2,x1},{c,3,x1}], 896: ?line All = 897: [rev(L11, Args), rev(L21, Args), rev(L31, Args), rev(L41, Args)], 898: ?line Merged1 = lists:keymerge(2, L11, L21), 899: ?line Merged2 = lists:keymerge(2, L31, L41), 900: ?line Merged = rev(lists:keymerge(2, Merged1, Merged2), Args), 901: ?line Fs1 = to_files(All, Fmt, Config), 902: ?line ok = file_sorter:keymerge(2, Fs1, Foo, Args), 903: ?line Merged = from_files(Foo, Fmt), 904: 905: fun() -> 906: UArgs = [{unique,true} | Args], 907: ?line UMerged1 = lists:ukeymerge(2, L11, L21), 908: ?line UMerged2 = lists:ukeymerge(2, L31, L41), 909: ?line UMerged = rev(lists:ukeymerge(2, UMerged1, UMerged2), Args), 910: ?line ok = file_sorter:keymerge(2, Fs1, Foo, UArgs), 911: ?line UMerged = from_files(Foo, Fmt), 912: UArgs2 = make_args(Fmt, [{unique,true}, {size,50} | XArgs]), 913: ?line ok = file_sorter:keymerge(2, Fs1, Foo, UArgs2), 914: ?line UMerged = from_files(Foo, Fmt), 915: ?line List = rev([{a,1,x4},{b,2,x4},{c,3,x4}], Args), 916: ?line FsL = to_files([List], Fmt, Config), 917: ?line ok = file_sorter:keymerge(2, FsL, Foo, UArgs), 918: ?line List = from_files(Foo, Fmt), 919: ?line List1 = [{a,1,x4},{b,2,x4},{c,3,x4}], 920: ?line List2 = [{a,3,x4},{b,4,x4},{c,5,x4}], 921: ?line FsLL = to_files([rev(List1, Args), rev(List2, Args)], Fmt, Config), 922: ?line ok = file_sorter:keymerge(2, FsLL, Foo, UArgs), 923: ?line List1_2 = rev(lists:ukeymerge(2, List1, List2), Args), 924: ?line List1_2 = from_files(Foo, Fmt), 925: ?line delete_files(Foo) 926: end(), 927: 928: %% Input is files. Output is a fun. 929: ?line Fs3 = to_files(All, Fmt, Config), 930: ?line [] = file_sorter:keysort(2, [], output([], Fmt), Args), 931: ?line KS1 = file_sorter:keymerge(2, Fs3, output([], Fmt), Args), 932: ?line Merged = KS1, 933: ?line delete_files([Foo | Fs3]), 934: 935: ?line L2 = [[{a,1}],[{a,2}],[{a,3}],[{a,4}],[{a,5}],[{a,6}],[{a,7}]], 936: ?line Fs2 = to_files(L2, Fmt, Config), 937: ?line M = file_sorter:keymerge(1, Fs2, output([], Fmt), Args), 938: ?line M = rev(lists:append(L2), Args), 939: ?line delete_files(Fs2), 940: 941: ?line LL1 = [{d,4},{e,5},{f,6}], 942: ?line LL2 = [{a,1},{b,2},{c,3}], 943: ?line LL3 = [{j,10},{k,11},{l,12}], 944: ?line LL4 = [{g,7},{h,8},{i,9}], 945: ?line LL5 = [{p,16},{q,17},{r,18}], 946: ?line LL6 = [{m,13},{n,14},{o,15}], 947: ?line LLAll = [rev(LL1, Args),rev(LL2, Args),rev(LL3, Args), 948: rev(LL4, Args),rev(LL5, Args),rev(LL6, Args)], 949: ?line FsLL6 = to_files(LLAll, Fmt, Config), 950: ?line LL = rev(lists:sort(lists:append(LLAll)), Args), 951: ?line ok = file_sorter:keymerge(1, FsLL6, Foo, Args), 952: ?line LL = from_files(Foo, Fmt), 953: ?line ok = file_sorter:keymerge(1, FsLL6, Foo, [{unique,true} | Args]), 954: ?line LL = from_files(Foo, Fmt), 955: ?line delete_files([Foo | FsLL6]), 956: 957: ok. 958: 959: funmerge(Fmt, XArgs, Config) -> 960: KComp = key_compare(2), 961: Args = make_args(Fmt, [{order,KComp},{size,5}, {no_files, 5} | XArgs]), 962: UArgs = [{unique,true} | Args], 963: Foo = outfile(foo, Config), 964: 965: ?line EFs = to_files([[]], Fmt, Config), 966: ?line ok = file_sorter:merge(EFs, Foo, UArgs), 967: ?line [] = from_files(Foo, Fmt), 968: delete_files([Foo | EFs]), 969: 970: ?line L11 = [{a,1,x4},{b,2,x4},{c,3,x4}], 971: ?line L21 = [{a,1,x3},{b,2,x3},{c,3,x3}], 972: ?line L31 = [{a,1,x2},{b,2,x2},{c,3,x2}], 973: ?line L41 = [{a,1,x1},{b,2,x1},{c,3,x1}], 974: ?line CAll = [L11, L21, L31, L41], 975: ?line CMerged1 = lists:merge(KComp, L11, L21), 976: ?line CMerged2 = lists:merge(KComp, L31, L41), 977: ?line CMerged = lists:merge(KComp, CMerged1, CMerged2), 978: ?line CFs1 = to_files(CAll, Fmt, Config), 979: ?line ok = file_sorter:merge(CFs1, Foo, Args), 980: ?line CMerged = from_files(Foo, Fmt), 981: 982: Args4 = make_args(Fmt, [{size,50} | XArgs]), 983: ?line ok = file_sorter:merge(CFs1, Foo, [{order,KComp} | Args4]), 984: ?line CMerged = from_files(Foo, Fmt), 985: 986: ?line UMerged1 = lists:umerge(KComp, L11, L21), 987: ?line UMerged2 = lists:umerge(KComp, L31, L41), 988: ?line UMerged = lists:umerge(KComp, UMerged1, UMerged2), 989: ?line ok = file_sorter:merge(CFs1, Foo, [{order,KComp} | UArgs]), 990: ?line UMerged = from_files(Foo, Fmt), 991: UArgs2 = 992: lists:keydelete(order, 1, 993: make_args(Fmt, [{unique,true}, {size,50} | XArgs])), 994: ?line ok = file_sorter:merge(CFs1, Foo, [{order,KComp} | UArgs2]), 995: ?line UMerged = from_files(Foo, Fmt), 996: ?line delete_files(Foo), 997: 998: ?line List1 = [{a,1,x4},{b,2,x4},{c,3,x4}], 999: ?line List2 = [{a,3,x4},{b,4,x4},{c,5,x4}], 1000: ?line List3 = [{a,5,x4},{b,6,x4},{c,7,x4}], 1001: ?line FsLL = to_files([List1, List2, List3], Fmt, Config), 1002: ?line ok = file_sorter:merge(FsLL, Foo, Args), 1003: ?line List1_2 = lists:merge(KComp,lists:merge(KComp,List1,List2),List3), 1004: ?line List1_2 = from_files(Foo, Fmt), 1005: ?line ok = file_sorter:merge(FsLL, Foo, [{order,KComp} | UArgs]), 1006: ?line UList1_2 = 1007: lists:umerge(KComp,lists:umerge(KComp, List1, List2),List3), 1008: ?line UList1_2 = from_files(Foo, Fmt), 1009: ?line delete_files([Foo | CFs1]), 1010: 1011: fun() -> 1012: ?line LL1 = [{d,4},{e,5},{f,6}], 1013: ?line LL2 = [{a,1},{b,2},{c,3}], 1014: ?line LL3 = [{j,10},{k,11},{l,12}], 1015: ?line LL4 = [{g,7},{h,8},{i,9}], 1016: ?line LL5 = [{p,16},{q,17},{r,18}], 1017: ?line LL6 = [{m,13},{n,14},{o,15}], 1018: ?line LLAll = [LL1,LL2,LL3,LL4,LL5,LL6], 1019: ?line FsLL6 = to_files(LLAll, Fmt, Config), 1020: ?line LL = lists:sort(lists:append(LLAll)), 1021: ?line ok = file_sorter:merge(FsLL6, Foo, Args), 1022: ?line LL = from_files(Foo, Fmt), 1023: ?line ok = file_sorter:merge(FsLL6, Foo, UArgs), 1024: ?line LL = from_files(Foo, Fmt), 1025: ?line delete_files([Foo | FsLL6]) 1026: end(), 1027: 1028: fun() -> 1029: ?line RLL1 = [{b,2},{h,8},{n,14}], 1030: ?line RLL2 = [{a,1},{g,7},{m,13}], 1031: ?line RLL3 = [{d,4},{j,10},{p,16}], 1032: ?line RLL4 = [{c,3},{i,9},{o,15}], 1033: ?line RLL5 = [{f,6},{l,12},{r,18}], 1034: ?line RLL6 = [{e,5},{k,11},{q,17}], 1035: ?line RLLAll = [RLL1,RLL2,RLL3,RLL4,RLL5,RLL6], 1036: ?line RFsLL6 = to_files(RLLAll, Fmt, Config), 1037: ?line RLL = lists:sort(lists:append(RLLAll)), 1038: ?line ok = file_sorter:merge(RFsLL6, Foo, Args), 1039: ?line RLL = from_files(Foo, Fmt), 1040: ?line ok = file_sorter:merge(RFsLL6, Foo, UArgs), 1041: ?line RLL = from_files(Foo, Fmt), 1042: ?line delete_files([Foo | RFsLL6]) 1043: end(), 1044: 1045: ok. 1046: 1047: check(Fmt, Config) -> 1048: Args0 = make_args(Fmt, [{size,5}]), 1049: Args = Args0 ++ [{tmpdir,?privdir(Config)}], 1050: 1051: Fun = fun compare/2, 1052: 1053: L1 = [3,1,2,5,4], 1054: [F1_0] = Fs1 = to_files([L1], Fmt, Config), 1055: F1 = filename:absname(F1_0), 1056: ?line {ok, [{F1,2,1}]} = file_sorter:check(Fs1, Args), 1057: ?line {ok, [{F1,2,1}]} = file_sorter:check(Fs1, [{order,Fun} | Args]), 1058: ?line {ok, [{F1,2,1}]} = file_sorter:check(Fs1, [{unique,true} | Args]), 1059: ?line {ok, [{F1,2,1}]} = 1060: file_sorter:check(Fs1, [{order,Fun},{unique,true} | Args]), 1061: ?line {ok, [{F1,3,2}]} = 1062: file_sorter:check(Fs1, [{order,descending} | Args]), 1063: ?line {ok, [{F1,3,2}]} = 1064: file_sorter:check(Fs1, [{unique,true},{order,descending} | Args]), 1065: ?line delete_files(Fs1), 1066: 1067: L2 = [[1,2,2,3,3,4,5,5],[5,5,4,3,3,2,2,1]], 1068: [F2_0,F3_0] = Fs2 = to_files(L2, Fmt, Config), 1069: F2 = filename:absname(F2_0), 1070: F3 = filename:absname(F3_0), 1071: ?line {ok, [{F3,3,4}]} = file_sorter:check(Fs2, Args), 1072: ?line {ok, [{F3,3,4}]} = file_sorter:check(Fs2, [{order,Fun} | Args]), 1073: ?line {ok, [{F2,3,2},{F3,2,5}]} = 1074: file_sorter:check(Fs2, [{unique, true} | Args]), 1075: ?line {ok, [{F2,3,2},{F3,2,5}]} = 1076: file_sorter:check(Fs2, [{order,Fun},{unique, true} | Args]), 1077: ?line {ok, [{F2,2,2}]} = 1078: file_sorter:check(Fs2, [{order,descending} | Args]), 1079: ?line {ok, [{F2,2,2},{F3,2,5}]} = 1080: file_sorter:check(Fs2, [{unique,true},{order,descending} | Args]), 1081: ?line delete_files(Fs2), 1082: 1083: L3 = [1,2,3,4], 1084: ?line Fs3 = to_files([L3], Fmt, Config), 1085: ?line {ok, []} = file_sorter:check(Fs3, [{unique,true} | Args]), 1086: ?line {ok, []} = 1087: file_sorter:check(Fs3, [{unique,true},{order,Fun} | Args]), 1088: ?line delete_files(Fs3), 1089: 1090: %% big objects 1091: ?line T1 = erlang:make_tuple(10000,foo), 1092: ?line T2 = erlang:make_tuple(10000,bar), 1093: ?line L4 = [T1,T2], 1094: ?line [FF_0] = Fs4 = to_files([L4], Fmt, Config), 1095: FF = filename:absname(FF_0), 1096: ?line {ok, [{FF,2,T2}]} = file_sorter:check(Fs4, [{unique,true} | Args]), 1097: ?line delete_files(Fs4), 1098: 1099: CFun = key_compare(2), 1100: L10 = [[{1,a},{2,b},T10_1={1,b},{3,c}], [{1,b},T10_2={2,a}]], 1101: [F10_0,F11_0] = Fs10 = to_files(L10, Fmt, Config), 1102: F10_1 = filename:absname(F10_0), 1103: F11_1 = filename:absname(F11_0), 1104: ?line {ok, [{F10_1,3,T10_1},{F11_1,2,T10_2}]} = 1105: file_sorter:check(Fs10, [{unique,true},{order,CFun} | Args]), 1106: ?line delete_files(Fs10), 1107: 1108: ok. 1109: 1110: keycheck(Fmt, Config) -> 1111: Args0 = make_args(Fmt, [{size,5}]), 1112: Args = Args0 ++ [{tmpdir,?privdir(Config)}], 1113: 1114: ?line L1 = [[{a,1},{b,2}], [{c,2},{b,1},{a,3}]], 1115: ?line [F1_0,F2_0] = Fs1 = to_files(L1, Fmt, Config), 1116: F1 = filename:absname(F1_0), 1117: F2 = filename:absname(F2_0), 1118: ?line {ok, [{F2,2,{b,1}}]} = file_sorter:keycheck(1, Fs1, Args), 1119: ?line {ok, [{F2,2,{b,1}}]} = 1120: file_sorter:keycheck(1, Fs1, [{unique,true} | Args]), 1121: ?line {ok, [{F1,2,{b,2}}]} = 1122: file_sorter:keycheck(1, Fs1, [{order,descending},{unique,true} | Args]), 1123: ?line delete_files(Fs1), 1124: 1125: L2 = [[{a,1},{a,2},{a,2},{b,2}], [{c,2},{b,1},{b,2},{b,2},{a,3}]], 1126: ?line [F3_0,F4_0] = Fs2 = to_files(L2, Fmt, Config), 1127: F3 = filename:absname(F3_0), 1128: F4 = filename:absname(F4_0), 1129: ?line {ok, [{F4,2,{b,1}}]} = file_sorter:keycheck(1, Fs2, Args), 1130: ?line {ok, [{F3,2,{a,2}},{F4,2,{b,1}}]} = 1131: file_sorter:keycheck(1, Fs2, [{unique,true} | Args]), 1132: ?line {ok, [{F3,4,{b,2}}]} = 1133: file_sorter:keycheck(1, Fs2, [{order,descending} | Args]), 1134: ?line {ok, [{F3,2,{a,2}},{F4,3,{b,2}}]} = 1135: file_sorter:keycheck(1, Fs2, 1136: [{order,descending},{unique,true} | Args]), 1137: ?line delete_files(Fs2), 1138: 1139: ok. 1140: 1141: rev(L, Args) -> 1142: case lists:member({order, descending}, Args) of 1143: true -> 1144: lists:reverse(L); 1145: false -> 1146: L 1147: end. 1148: 1149: make_args({HL, Fmt}, Args) -> 1150: make_args(Fmt, [{header, HL} | Args]); 1151: make_args(Fmt, Args) -> 1152: [{format, Fmt} | Args]. 1153: 1154: compare(X, Y) -> 1155: X =< Y. 1156: 1157: key_compare(I) -> 1158: fun(X, Y) -> 1159: element(I, bin_to_term(X)) =< element(I, bin_to_term(Y)) 1160: end. 1161: 1162: bin_to_term(B) when is_binary(B) -> binary_to_term(B); 1163: bin_to_term(T) -> T. 1164: 1165: -define(CHUNKSIZE, 8096). 1166: 1167: pps() -> 1168: erlang:ports(). 1169: 1170: input(L, N, term) -> 1171: input(L, N); 1172: input(L, N, {_HL, Format}) when Format =:= binary_term; Format =:= binary -> 1173: binput(L, N); 1174: input(L, N, Format) when Format =:= binary_term; Format =:= binary -> 1175: binput(L, N). 1176: 1177: binput(L, N) -> 1178: Bs = lists:map(fun(T) -> term_to_binary(T) end, L), 1179: input(Bs, N). 1180: 1181: input(L, N) -> 1182: fun(close) -> 1183: ok; 1184: (read) -> 1185: case L of 1186: [] -> end_of_input; 1187: _ -> 1188: R = lists:sublist(L, N), 1189: NL = lists:nthtail(length(R), L), 1190: {R, input(NL, N)} 1191: end 1192: end. 1193: 1194: output(L, term) -> 1195: output(L); 1196: output(L, {_HL, Format}) when Format =:= binary_term; Format =:= binary -> 1197: boutput(L); 1198: output(L, Format) when Format =:= binary_term; Format =:= binary -> 1199: boutput(L). 1200: 1201: output(A) -> 1202: fun(close) -> 1203: lists:append(lists:reverse(A)); 1204: (L) when is_list(L) -> 1205: output([L | A]) 1206: end. 1207: 1208: boutput(A) -> 1209: fun(close) -> 1210: Bs = lists:append(lists:reverse(A)), 1211: lists:map(fun(B) -> binary_to_term(B) end, Bs); 1212: (L) when is_list(L) -> 1213: boutput([L | A]) 1214: end. 1215: 1216: outfile(Name, Config) -> 1217: list_to_atom(filename:join(?privdir(Config), Name)). 1218: 1219: %% [[term()]] -> [filename()] 1220: to_files(Lists, term, Config) -> 1221: terms_to_files(Lists, Config); 1222: to_files(Lists, Format, Config) when Format =:= binary_term; 1223: Format =:= binary -> 1224: bins_to_files(Lists, 4, Config); 1225: to_files(Lists, {HL, Format}, Config) when Format =:= binary_term; 1226: Format =:= binary -> 1227: bins_to_files(Lists, HL, Config). 1228: 1229: %% [[term()]] -> [filename()] 1230: terms_to_files(Lists, Config) -> 1231: PrivDir = ?privdir(Config), 1232: terms_to_files(Lists, PrivDir, 1). 1233: 1234: terms_to_files([L | Ls], PrivDir, N) -> 1235: F = lists:concat([?MODULE, '_', N]), 1236: File = filename:join(PrivDir, F), 1237: {ok, Fd} = file:open(File, [write]), 1238: write_terms(Fd, L), 1239: file:close(Fd), 1240: [list_to_atom(File) | terms_to_files(Ls, PrivDir, N+1)]; 1241: terms_to_files([], _PrivDir, _N) -> 1242: []. 1243: 1244: write_terms(Fd, [T | Ts]) -> 1245: io:format(Fd, "~p.~n", [T]), 1246: write_terms(Fd, Ts); 1247: write_terms(_Fd, []) -> 1248: ok. 1249: 1250: %% [[term()]] -> [filename()] 1251: bins_to_files(Lists, HL, Config) -> 1252: PrivDir = ?privdir(Config), 1253: bins_to_files(Lists, PrivDir, 1, HL). 1254: 1255: bins_to_files([L | Fs], PrivDir, N, HL) -> 1256: F = lists:concat([?MODULE, '_', N]), 1257: File = filename:join(PrivDir, F), 1258: {ok, Fd} = file:open(File, [raw,binary,write]), 1259: write_bins(Fd, L, HL), 1260: file:close(Fd), 1261: [list_to_atom(File) | bins_to_files(Fs, PrivDir, N+1, HL)]; 1262: bins_to_files([], _PrivDir, _N, _HL) -> 1263: []. 1264: 1265: write_bins(Fd, [T | Ts], HL) -> 1266: B = term_to_binary(T), 1267: Sz = byte_size(B), 1268: ok = file:write(Fd, [<<Sz:HL/unit:8>>, B]), 1269: write_bins(Fd, Ts, HL); 1270: write_bins(_Fd, [], _HL) -> 1271: ok. 1272: 1273: %% [filename()] -> [[term()]] or filename() -> [term()] 1274: from_files(Files, term) -> 1275: terms_from_files(Files); 1276: from_files(Files, Format) when Format =:= binary_term; Format =:= binary -> 1277: bins_from_files(Files, 4); 1278: from_files(Files, {HL, Format}) when Format =:= binary_term; 1279: Format =:= binary -> 1280: bins_from_files(Files, HL). 1281: 1282: %% [filename()] -> [[term()]] or filename() -> [term()] 1283: terms_from_files(File) when is_atom(File) -> 1284: [Terms] = terms_from_files([File]), 1285: Terms; 1286: terms_from_files(Files) -> 1287: lists:map(fun(F) -> terms_from_file(F) end, Files). 1288: 1289: terms_from_file(File) -> 1290: {ok, Fd} = file:open(File, [read,compressed]), 1291: terms_from_file(Fd, []). 1292: 1293: terms_from_file(Fd, L) -> 1294: case io:read(Fd, '') of 1295: {ok, Term} -> 1296: terms_from_file(Fd, [Term | L]); 1297: eof -> 1298: file:close(Fd), 1299: lists:reverse(L) 1300: end. 1301: 1302: %% [filename()] -> [[term()]] 1303: bins_from_files(File, HL) when is_atom(File) -> 1304: [Bins] = bins_from_files([File], HL), 1305: Bins; 1306: bins_from_files(Files, HL) -> 1307: lists:map(fun(F) -> collect(F, HL) end, Files). 1308: 1309: delete_files(File) when is_atom(File) -> 1310: file:delete(File); 1311: delete_files(Files) -> 1312: lists:foreach(fun(F) -> file:delete(F) end, Files). 1313: 1314: %%% 1315: %%% Collects binaries converted to terms in a list. Not very efficient. 1316: %%% 1317: collect(F, HL) -> 1318: {ok, Fd} = file:open(F, [read, binary, raw, compressed]), 1319: R = (catch c(Fd, <<>>, 0, ?CHUNKSIZE, HL, [])), 1320: file:close(Fd), 1321: R. 1322: 1323: c(Fd, Bin0, Size0, NoBytes, HL, L) -> 1324: case file:read(Fd, NoBytes) of 1325: {ok, Bin} -> 1326: Size = Size0 + byte_size(Bin), 1327: NBin = list_to_binary([Bin0, Bin]), 1328: c1(Fd, NBin, Size, HL, L); 1329: eof when Size0 =:= 0 -> 1330: lists:reverse(L); 1331: eof -> 1332: test_server:fail({error, premature_eof}); 1333: Error -> 1334: test_server:fail(Error) 1335: end. 1336: 1337: c1(Fd, B, BinSize, HL, L) -> 1338: case B of 1339: <<Size:HL/unit:8, Bin/binary>> -> 1340: if 1341: Size > BinSize - HL, Size > ?CHUNKSIZE -> 1342: c(Fd, B, BinSize, Size + HL, HL, L); 1343: Size > BinSize - HL -> 1344: c(Fd, B, BinSize, ?CHUNKSIZE, HL, L); 1345: true -> 1346: <<BinTerm:Size/binary, R/binary>> = Bin, 1347: E = case catch binary_to_term(BinTerm) of 1348: {'EXIT', _} -> 1349: test_server:fail({error, bad_object}); 1350: Term -> 1351: Term 1352: end, 1353: NBinSize = BinSize - HL - Size, 1354: c1(Fd, R, NBinSize, HL, [E | L]) 1355: end; 1356: _ -> 1357: c(Fd, B, BinSize, ?CHUNKSIZE, HL, L) 1358: end.