1: %% -*- coding: utf-8 -*- 2: %% 3: %% %CopyrightBegin% 4: %% 5: %% Copyright Ericsson AB 1998-2013. All Rights Reserved. 6: %% 7: %% The contents of this file are subject to the Erlang Public License, 8: %% Version 1.1, (the "License"); you may not use this file except in 9: %% compliance with the License. You should have received a copy of the 10: %% Erlang Public License along with this software. If not, it can be 11: %% retrieved online at http://www.erlang.org/. 12: %% 13: %% Software distributed under the License is distributed on an "AS IS" 14: %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See 15: %% the License for the specific language governing rights and limitations 16: %% under the License. 17: %% 18: %% %CopyrightEnd% 19: 20: -module(erl_scan_SUITE). 21: -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, 22: init_per_group/2,end_per_group/2]). 23: 24: -export([ error_1/1, error_2/1, iso88591/1, otp_7810/1, otp_10302/1, 25: otp_10990/1, otp_10992/1]). 26: 27: -import(lists, [nth/2,flatten/1]). 28: -import(io_lib, [print/1]). 29: 30: %% 31: %% Define to run outside of test server 32: %% 33: %-define(STANDALONE,1). 34: 35: -ifdef(STANDALONE). 36: -compile(export_all). 37: -define(line, put(line, ?LINE), ). 38: -define(config(A,B),config(A,B)). 39: -define(t, test_server). 40: %% config(priv_dir, _) -> 41: %% "."; 42: %% config(data_dir, _) -> 43: %% ".". 44: -else. 45: -include_lib("test_server/include/test_server.hrl"). 46: -export([init_per_testcase/2, end_per_testcase/2]). 47: 48: init_per_testcase(_Case, Config) when is_list(Config) -> 49: ?line Dog=test_server:timetrap(test_server:seconds(1200)), 50: [{watchdog, Dog}|Config]. 51: 52: end_per_testcase(_Case, Config) -> 53: Dog=?config(watchdog, Config), 54: test_server:timetrap_cancel(Dog), 55: ok. 56: -endif. 57: 58: % Default timetrap timeout (set in init_per_testcase). 59: -define(default_timeout, ?t:minutes(1)). 60: 61: suite() -> [{ct_hooks,[ts_install_cth]}]. 62: 63: all() -> 64: [{group, error}, iso88591, otp_7810, otp_10302, otp_10990, otp_10992]. 65: 66: groups() -> 67: [{error, [], [error_1, error_2]}]. 68: 69: init_per_suite(Config) -> 70: Config. 71: 72: end_per_suite(_Config) -> 73: ok. 74: 75: init_per_group(_GroupName, Config) -> 76: Config. 77: 78: end_per_group(_GroupName, Config) -> 79: Config. 80: 81: 82: 83: error_1(doc) -> 84: ["(OTP-2347)"]; 85: error_1(suite) -> 86: []; 87: error_1(Config) when is_list(Config) -> 88: ?line {error, _, _} = erl_scan:string("'a"), 89: ok. 90: 91: error_2(doc) -> 92: ["Checks that format_error works on the error cases."]; 93: error_2(suite) -> 94: []; 95: error_2(Config) when is_list(Config) -> 96: ?line lists:foreach(fun check/1, error_cases()), 97: ok. 98: 99: error_cases() -> 100: ["'a", 101: "\"a", 102: "'\\", 103: "\"\\", 104: "$", 105: "$\\", 106: "2.3e", 107: "2.3e-", 108: "91#9" 109: ]. 110: 111: assert_type(N, integer) when is_integer(N) -> 112: ok; 113: assert_type(N, atom) when is_atom(N) -> 114: ok. 115: 116: check(String) -> 117: Error = erl_scan:string(String), 118: check_error(Error, erl_scan). 119: 120: %%% (This should be useful for all format_error functions.) 121: check_error({error, Info, EndLine}, Module0) -> 122: {ErrorLine, Module, Desc} = Info, 123: true = (Module == Module0), 124: assert_type(EndLine, integer), 125: assert_type(ErrorLine, integer), 126: true = (ErrorLine =< EndLine), 127: String = lists:flatten(Module0:format_error(Desc)), 128: true = io_lib:printable_list(String). 129: 130: iso88591(doc) -> ["Tests the support for ISO-8859-1 i.e Latin-1"]; 131: iso88591(suite) -> []; 132: iso88591(Config) when is_list(Config) -> 133: ?line ok = 134: case catch begin 135: %% Some atom and variable names 136: V1s = [$Á,$á,$é,$ë], 137: V2s = [$N,$ä,$r], 138: A1s = [$h,$ä,$r], 139: A2s = [$ö,$r,$e], 140: %% Test parsing atom and variable characters. 141: {ok,Ts1,_} = erl_scan:string(V1s ++ " " ++ V2s ++ 142: "\327" ++ 143: A1s ++ " " ++ A2s), 144: V1s = atom_to_list(element(3, nth(1, Ts1))), 145: V2s = atom_to_list(element(3, nth(2, Ts1))), 146: A1s = atom_to_list(element(3, nth(4, Ts1))), 147: A2s = atom_to_list(element(3, nth(5, Ts1))), 148: %% Test printing atoms 149: A1s = flatten(print(element(3, nth(4, Ts1)))), 150: A2s = flatten(print(element(3, nth(5, Ts1)))), 151: %% Test parsing and printing strings. 152: S1 = V1s ++ "\327" ++ A1s ++ "\250" ++ A2s, 153: S1s = "\"" ++ S1 ++ "\"", 154: {ok,Ts2,_} = erl_scan:string(S1s), 155: S1 = element(3, nth(1, Ts2)), 156: S1s = flatten(print(element(3, nth(1, Ts2)))), 157: ok %It all worked 158: end of 159: {'EXIT',R} -> %Something went wrong! 160: {error,R}; 161: ok -> ok %Aok 162: end. 163: 164: otp_7810(doc) -> 165: ["OTP-7810. White spaces, comments, and more.."]; 166: otp_7810(suite) -> 167: []; 168: otp_7810(Config) when is_list(Config) -> 169: ?line ok = reserved_words(), 170: ?line ok = atoms(), 171: ?line ok = punctuations(), 172: ?line ok = comments(), 173: ?line ok = errors(), 174: ?line ok = integers(), 175: ?line ok = base_integers(), 176: ?line ok = floats(), 177: ?line ok = dots(), 178: ?line ok = chars(), 179: ?line ok = variables(), 180: ?line ok = eof(), 181: ?line ok = illegal(), 182: ?line ok = crashes(), 183: 184: ?line ok = options(), 185: ?line ok = token_info(), 186: ?line ok = column_errors(), 187: ?line ok = white_spaces(), 188: 189: ?line ok = unicode(), 190: 191: ?line ok = more_chars(), 192: ?line ok = more_options(), 193: ?line ok = attributes_info(), 194: ?line ok = set_attribute(), 195: 196: ok. 197: 198: reserved_words() -> 199: L = ['after', 'begin', 'case', 'try', 'cond', 'catch', 200: 'andalso', 'orelse', 'end', 'fun', 'if', 'let', 'of', 201: 'receive', 'when', 'bnot', 'not', 'div', 202: 'rem', 'band', 'and', 'bor', 'bxor', 'bsl', 'bsr', 203: 'or', 'xor'], 204: [begin 205: ?line {RW, true} = {RW, erl_scan:reserved_word(RW)}, 206: S = atom_to_list(RW), 207: Ts = [{RW,1}], 208: ?line test_string(S, Ts) 209: end || RW <- L], 210: ok. 211: 212: 213: atoms() -> 214: ?line test_string("a 215: b", [{atom,1,a},{atom,2,b}]), 216: ?line test_string("'a b'", [{atom,1,'a b'}]), 217: ?line test_string("a", [{atom,1,a}]), 218: ?line test_string("a@2", [{atom,1,a@2}]), 219: ?line test_string([39,65,200,39], [{atom,1,'AÈ'}]), 220: ?line test_string("ärlig östen", [{atom,1,ärlig},{atom,1,östen}]), 221: ?line {ok,[{atom,_,'$a'}],{1,6}} = 222: erl_scan:string("'$\\a'", {1,1}), 223: ?line test("'$\\a'"), 224: ok. 225: 226: punctuations() -> 227: L = ["<<", "<-", "<=", "<", ">>", ">=", ">", "->", "--", 228: "-", "++", "+", "=:=", "=/=", "=<", "==", "=", "/=", 229: "/", "||", "|", ":-", "::", ":"], 230: %% One token at a time: 231: [begin 232: W = list_to_atom(S), 233: Ts = [{W,1}], 234: ?line test_string(S, Ts) 235: end || S <- L], 236: Three = ["/=:=", "<=:=", "==:=", ">=:="], % three tokens... 237: No = Three ++ L, 238: SL0 = [{S1++S2,{-length(S1),S1,S2}} || 239: S1 <- L, 240: S2 <- L, 241: not lists:member(S1++S2, No)], 242: SL = family_list(SL0), 243: %% Two tokens. When there are several answers, the one with 244: %% the longest first token is chosen: 245: %% [the special case "=<<" is among the tested ones] 246: [begin 247: W1 = list_to_atom(S1), 248: W2 = list_to_atom(S2), 249: Ts = [{W1,1},{W2,1}], 250: ?line test_string(S, Ts) 251: end || {S,[{_,S1,S2}|_]} <- SL], 252: 253: PTs1 = [{'!',1},{'(',1},{')',1},{',',1},{';',1},{'=',1},{'[',1}, 254: {']',1},{'{',1},{'|',1},{'}',1}], 255: ?line test_string("!(),;=[]{|}", PTs1), 256: 257: PTs2 = [{'#',1},{'&',1},{'*',1},{'+',1},{'/',1}, 258: {':',1},{'<',1},{'>',1},{'?',1},{'@',1}, 259: {'\\',1},{'^',1},{'`',1},{'~',1}], 260: ?line test_string("#&*+/:<>?@\\^`~", PTs2), 261: 262: ?line test_string(".. ", [{'..',1}]), 263: ?line test("1 .. 2"), 264: ?line test_string("...", [{'...',1}]), 265: ok. 266: 267: comments() -> 268: ?line test("a %%\n b"), 269: ?line {ok,[],1} = erl_scan:string("%"), 270: ?line test("a %%\n b"), 271: ?line {ok,[{atom,_,a},{atom,_,b}],{2,3}} = 272: erl_scan:string("a %%\n b",{1,1}), 273: ?line {ok,[{atom,_,a},{comment,_,"%%"},{atom,_,b}],{2,3}} = 274: erl_scan:string("a %%\n b",{1,1}, [return_comments]), 275: ?line {ok,[{atom,_,a}, 276: {white_space,_," "}, 277: {white_space,_,"\n "}, 278: {atom,_,b}], 279: {2,3}} = 280: erl_scan:string("a %%\n b",{1,1},[return_white_spaces]), 281: ?line {ok,[{atom,_,a}, 282: {white_space,_," "}, 283: {comment,_,"%%"}, 284: {white_space,_,"\n "}, 285: {atom,_,b}], 286: {2,3}} = erl_scan:string("a %%\n b",{1,1},[return]), 287: ok. 288: 289: errors() -> 290: ?line {error,{1,erl_scan,{string,$',"qa"}},1} = erl_scan:string("'qa"), %' 291: ?line {error,{1,erl_scan,{string,$","str"}},1} = %" 292: erl_scan:string("\"str"), %" 293: ?line {error,{1,erl_scan,char},1} = erl_scan:string("$"), 294: ?line test_string([34,65,200,34], [{string,1,"AÈ"}]), 295: ?line test_string("\\", [{'\\',1}]), 296: ?line {'EXIT',_} = 297: (catch {foo, erl_scan:string('$\\a', {1,1})}), % type error 298: ?line {'EXIT',_} = 299: (catch {foo, erl_scan:tokens([], '$\\a', {1,1})}), % type error 300: 301: ?line "{a,tuple}" = erl_scan:format_error({a,tuple}), 302: ok. 303: 304: integers() -> 305: [begin 306: I = list_to_integer(S), 307: Ts = [{integer,1,I}], 308: ?line test_string(S, Ts) 309: end || S <- [[N] || N <- lists:seq($0, $9)] ++ ["2323","000"] ], 310: ok. 311: 312: base_integers() -> 313: [begin 314: B = list_to_integer(BS), 315: I = erlang:list_to_integer(S, B), 316: Ts = [{integer,1,I}], 317: ?line test_string(BS++"#"++S, Ts) 318: end || {BS,S} <- [{"2","11"}, {"5","23234"}, {"12","05a"}, 319: {"16","abcdef"}, {"16","ABCDEF"}] ], 320: 321: ?line {error,{1,erl_scan,{base,1}},1} = erl_scan:string("1#000"), 322: 323: ?line test_string("12#bc", [{integer,1,11},{atom,1,c}]), 324: 325: [begin 326: Str = BS ++ "#" ++ S, 327: ?line {error,{1,erl_scan,{illegal,integer}},1} = 328: erl_scan:string(Str) 329: end || {BS,S} <- [{"3","3"},{"15","f"}, {"12","c"}] ], 330: 331: ?line {ok,[{integer,1,239},{'@',1}],1} = erl_scan:string("16#ef@"), 332: ?line {ok,[{integer,1,14},{atom,1,g@}],1} = erl_scan:string("16#eg@"), 333: 334: ok. 335: 336: floats() -> 337: [begin 338: F = list_to_float(FS), 339: Ts = [{float,1,F}], 340: ?line test_string(FS, Ts) 341: end || FS <- ["1.0","001.17","3.31200","1.0e0","1.0E17", 342: "34.21E-18", "17.0E+14"]], 343: ?line test_string("1.e2", [{integer,1,1},{'.',1},{atom,1,e2}]), 344: 345: ?line {error,{1,erl_scan,{illegal,float}},1} = 346: erl_scan:string("1.0e400"), 347: [begin 348: ?line {error,{1,erl_scan,{illegal,float}},1} = erl_scan:string(S) 349: end || S <- ["1.14Ea"]], 350: 351: ok. 352: 353: dots() -> 354: Dot = [{".", {ok,[{dot,1}],1}}, 355: {". ", {ok,[{dot,1}],1}}, 356: {".\n", {ok,[{dot,1}],2}}, 357: {".%", {ok,[{dot,1}],1}}, 358: {".\210",{ok,[{dot,1}],1}}, 359: {".% öh",{ok,[{dot,1}],1}}, 360: {".%\n", {ok,[{dot,1}],2}}, 361: {".$", {error,{1,erl_scan,char},1}}, 362: {".$\\", {error,{1,erl_scan,char},1}}, 363: {".a", {ok,[{'.',1},{atom,1,a}],1}} 364: ], 365: ?line [R = erl_scan:string(S) || {S, R} <- Dot], 366: 367: ?line {ok,[{dot,_}=T1],{1,2}} = erl_scan:string(".", {1,1}, text), 368: ?line [{column,1},{length,1},{line,1},{text,"."}] = 369: erl_scan:token_info(T1, [column, length, line, text]), 370: ?line {ok,[{dot,_}=T2],{1,3}} = erl_scan:string(".%", {1,1}, text), 371: ?line [{column,1},{length,1},{line,1},{text,"."}] = 372: erl_scan:token_info(T2, [column, length, line, text]), 373: ?line {ok,[{dot,_}=T3],{1,6}} = 374: erl_scan:string(".% öh", {1,1}, text), 375: ?line [{column,1},{length,1},{line,1},{text,"."}] = 376: erl_scan:token_info(T3, [column, length, line, text]), 377: ?line {error,{{1,2},erl_scan,char},{1,3}} = 378: erl_scan:string(".$", {1,1}), 379: ?line {error,{{1,2},erl_scan,char},{1,4}} = 380: erl_scan:string(".$\\", {1,1}), 381: 382: ?line test(". "), 383: ?line test(". "), 384: ?line test(".\n"), 385: ?line test(".\n\n"), 386: ?line test(".\n\r"), 387: ?line test(".\n\n\n"), 388: ?line test(".\210"), 389: ?line test(".%\n"), 390: ?line test(".a"), 391: 392: ?line test("%. \n. "), 393: ?line {more,C} = erl_scan:tokens([], "%. ",{1,1}, return), 394: ?line {done,{ok,[{comment,_,"%. "}, 395: {white_space,_,"\n"}, 396: {dot,_}], 397: {2,3}}, ""} = 398: erl_scan:tokens(C, "\n. ", {1,1}, return), % any loc, any options 399: 400: ?line [test_string(S, R) || 401: {S, R} <- [{".$\n", [{'.',1},{char,1,$\n}]}, 402: {"$\\\n", [{char,1,$\n}]}, 403: {"'\\\n'", [{atom,1,'\n'}]}, 404: {"$\n", [{char,1,$\n}]}] ], 405: ok. 406: 407: chars() -> 408: [begin 409: L = lists:flatten(io_lib:format("$\\~.8b", [C])), 410: Ts = [{char,1,C}], 411: ?line test_string(L, Ts) 412: end || C <- lists:seq(0, 255)], 413: 414: %% Leading zeroes... 415: [begin 416: L = lists:flatten(io_lib:format("$\\~3.8.0b", [C])), 417: Ts = [{char,1,C}], 418: ?line test_string(L, Ts) 419: end || C <- lists:seq(0, 255)], 420: 421: %% $\^\n now increments the line... 422: [begin 423: L = "$\\^" ++ [C], 424: Ts = [{char,1,C band 2#11111}], 425: ?line test_string(L, Ts) 426: end || C <- lists:seq(0, 255)], 427: 428: [begin 429: L = "$\\" ++ [C], 430: Ts = [{char,1,V}], 431: ?line test_string(L, Ts) 432: end || {C,V} <- [{$n,$\n}, {$r,$\r}, {$t,$\t}, {$v,$\v}, 433: {$b,$\b}, {$f,$\f}, {$e,$\e}, {$s,$\s}, 434: {$d,$\d}]], 435: 436: EC = [$\n,$\r,$\t,$\v,$\b,$\f,$\e,$\s,$\d], 437: Ds = lists:seq($0, $9), 438: X = [$^,$n,$r,$t,$v,$b,$f,$e,$s,$d], 439: New = [${,$x], 440: No = EC ++ Ds ++ X ++ New, 441: [begin 442: L = "$\\" ++ [C], 443: Ts = [{char,1,C}], 444: ?line test_string(L, Ts) 445: end || C <- lists:seq(0, 255) -- No], 446: 447: [begin 448: L = "'$\\" ++ [C] ++ "'", 449: Ts = [{atom,1,list_to_atom("$"++[C])}], 450: ?line test_string(L, Ts) 451: end || C <- lists:seq(0, 255) -- No], 452: 453: ?line test_string("\"\\013a\\\n\"", [{string,1,"\va\n"}]), 454: 455: ?line test_string("'\n'", [{atom,1,'\n'}]), 456: ?line test_string("\"\n\a\"", [{string,1,"\na"}]), 457: 458: %% No escape 459: [begin 460: L = "$" ++ [C], 461: Ts = [{char,1,C}], 462: ?line test_string(L, Ts) 463: end || C <- lists:seq(0, 255) -- (No ++ [$\\])], 464: ?line test_string("$\n", [{char,1,$\n}]), 465: 466: ?line {error,{{1,1},erl_scan,char},{1,4}} = 467: erl_scan:string("$\\^",{1,1}), 468: ?line test_string("$\\\n", [{char,1,$\n}]), 469: %% Robert's scanner returns line 1: 470: ?line test_string("$\\\n", [{char,1,$\n}]), 471: ?line test_string("$\n\n", [{char,1,$\n}]), 472: ?line test("$\n\n"), 473: ok. 474: 475: 476: variables() -> 477: ?line test_string(" \237_Aouåeiyäö", [{var,1,'_Aouåeiyäö'}]), 478: ?line test_string("A_b_c@", [{var,1,'A_b_c@'}]), 479: ?line test_string("V@2", [{var,1,'V@2'}]), 480: ?line test_string("ABDÀ", [{var,1,'ABDÀ'}]), 481: ?line test_string("Ärlig Östen", [{var,1,'Ärlig'},{var,1,'Östen'}]), 482: ok. 483: 484: eof() -> 485: ?line {done,{eof,1},eof} = erl_scan:tokens([], eof, 1), 486: {more, C1} = erl_scan:tokens([]," \n", 1), 487: ?line {done,{eof,2},eof} = erl_scan:tokens(C1, eof, 1), 488: {more, C2} = erl_scan:tokens([], "abra", 1), 489: %% An error before R13A. 490: %% ?line {done,Err={error,{1,erl_scan,scan},1},eof} = 491: ?line {done,{ok,[{atom,1,abra}],1},eof} = 492: erl_scan:tokens(C2, eof, 1), 493: 494: %% With column. 495: ?line {more, C3} = erl_scan:tokens([]," \n",{1,1}), 496: ?line {done,{eof,{2,1}},eof} = erl_scan:tokens(C3, eof, 1), 497: {more, C4} = erl_scan:tokens([], "abra", {1,1}), 498: %% An error before R13A. 499: %% ?line {done,{error,{{1,1},erl_scan,scan},{1,5}},eof} = 500: ?line {done,{ok,[{atom,_,abra}],{1,5}},eof} = 501: erl_scan:tokens(C4, eof, 1), 502: 503: %% Robert's scanner returns "" as LeftoverChars; 504: %% the R12B scanner returns eof as LeftoverChars: (eof is correct) 505: ?line {more, C5} = erl_scan:tokens([], "a", 1), 506: %% An error before R13A. 507: %% ?line {done,{error,{1,erl_scan,scan},1},eof} = 508: ?line {done,{ok,[{atom,1,a}],1},eof} = 509: erl_scan:tokens(C5,eof,1), 510: 511: %% A dot followed by eof is special: 512: ?line {more, C} = erl_scan:tokens([], "a.", 1), 513: ?line {done,{ok,[{atom,1,a},{dot,1}],1},eof} = erl_scan:tokens(C,eof,1), 514: ?line {ok,[{atom,1,foo},{dot,1}],1} = erl_scan:string("foo."), 515: 516: ok. 517: 518: illegal() -> 519: Atom = lists:duplicate(1000, $a), 520: ?line {error,{1,erl_scan,{illegal,atom}},1} = erl_scan:string(Atom), 521: ?line {done,{error,{1,erl_scan,{illegal,atom}},1},". "} = 522: erl_scan:tokens([], Atom++". ", 1), 523: QAtom = "'" ++ Atom ++ "'", 524: ?line {error,{1,erl_scan,{illegal,atom}},1} = erl_scan:string(QAtom), 525: ?line {done,{error,{1,erl_scan,{illegal,atom}},1},". "} = 526: erl_scan:tokens([], QAtom++". ", 1), 527: Var = lists:duplicate(1000, $A), 528: ?line {error,{1,erl_scan,{illegal,var}},1} = erl_scan:string(Var), 529: ?line {done,{error,{1,erl_scan,{illegal,var}},1},". "} = 530: erl_scan:tokens([], Var++". ", 1), 531: Float = "1" ++ lists:duplicate(400, $0) ++ ".0", 532: ?line {error,{1,erl_scan,{illegal,float}},1} = erl_scan:string(Float), 533: ?line {done,{error,{1,erl_scan,{illegal,float}},1},". "} = 534: erl_scan:tokens([], Float++". ", 1), 535: String = "\"43\\x{aaaaaa}34\"", 536: ?line {error,{1,erl_scan,{illegal,character}},1} = erl_scan:string(String), 537: ?line {done,{error,{1,erl_scan,{illegal,character}},1},"34\". "} = 538: %% Would be nice if `34\"' were skipped... 539: %% Maybe, but then the LeftOverChars would not be the characters 540: %% immediately following the end location of the error. 541: erl_scan:tokens([], String++". ", 1), 542: 543: ?line {error,{{1,1},erl_scan,{illegal,atom}},{1,1001}} = 544: erl_scan:string(Atom, {1,1}), 545: ?line {done,{error,{{1,5},erl_scan,{illegal,atom}},{1,1005}},". "} = 546: erl_scan:tokens([], "foo "++Atom++". ", {1,1}), 547: ?line {error,{{1,1},erl_scan,{illegal,atom}},{1,1003}} = 548: erl_scan:string(QAtom, {1,1}), 549: ?line {done,{error,{{1,5},erl_scan,{illegal,atom}},{1,1007}},". "} = 550: erl_scan:tokens([], "foo "++QAtom++". ", {1,1}), 551: ?line {error,{{1,1},erl_scan,{illegal,var}},{1,1001}} = 552: erl_scan:string(Var, {1,1}), 553: ?line {done,{error,{{1,5},erl_scan,{illegal,var}},{1,1005}},". "} = 554: erl_scan:tokens([], "foo "++Var++". ", {1,1}), 555: ?line {error,{{1,1},erl_scan,{illegal,float}},{1,404}} = 556: erl_scan:string(Float, {1,1}), 557: ?line {done,{error,{{1,5},erl_scan,{illegal,float}},{1,408}},". "} = 558: erl_scan:tokens([], "foo "++Float++". ", {1,1}), 559: ?line {error,{{1,4},erl_scan,{illegal,character}},{1,14}} = 560: erl_scan:string(String, {1,1}), 561: ?line {done,{error,{{1,4},erl_scan,{illegal,character}},{1,14}},"34\". "} = 562: erl_scan:tokens([], String++". ", {1,1}), 563: ok. 564: 565: crashes() -> 566: ?line {'EXIT',_} = (catch {foo, erl_scan:string([-1])}), % type error 567: ?line {'EXIT',_} = (catch {foo, erl_scan:string("$"++[-1])}), 568: ?line {'EXIT',_} = (catch {foo, erl_scan:string("$\\"++[-1])}), 569: ?line {'EXIT',_} = (catch {foo, erl_scan:string("$\\^"++[-1])}), 570: ?line {'EXIT',_} = (catch {foo, erl_scan:string([$",-1,$"],{1,1})}), 571: ?line {'EXIT',_} = (catch {foo, erl_scan:string("\"\\v"++[-1,$"])}), %$" 572: ?line {'EXIT',_} = (catch {foo, erl_scan:string([$",-1,$"])}), 573: ?line {'EXIT',_} = (catch {foo, erl_scan:string("% foo"++[-1])}), 574: ?line {'EXIT',_} = 575: (catch {foo, erl_scan:string("% foo"++[-1],{1,1})}), 576: 577: ?line {'EXIT',_} = (catch {foo, erl_scan:string([a])}), % type error 578: ?line {'EXIT',_} = (catch {foo, erl_scan:string("$"++[a])}), 579: ?line {'EXIT',_} = (catch {foo, erl_scan:string("$\\"++[a])}), 580: ?line {'EXIT',_} = (catch {foo, erl_scan:string("$\\^"++[a])}), 581: ?line {'EXIT',_} = (catch {foo, erl_scan:string([$",a,$"],{1,1})}), 582: ?line {'EXIT',_} = (catch {foo, erl_scan:string("\"\\v"++[a,$"])}), %$" 583: ?line {'EXIT',_} = (catch {foo, erl_scan:string([$",a,$"])}), 584: ?line {'EXIT',_} = (catch {foo, erl_scan:string("% foo"++[a])}), 585: ?line {'EXIT',_} = 586: (catch {foo, erl_scan:string("% foo"++[a],{1,1})}), 587: 588: ?line {'EXIT',_} = (catch {foo, erl_scan:string([3.0])}), % type error 589: 590: ok. 591: 592: options() -> 593: %% line and column are not options, but tested here 594: ?line {ok,[{atom,1,foo},{white_space,1," "},{comment,1,"% bar"}], 1} = 595: erl_scan:string("foo % bar", 1, return), 596: ?line {ok,[{atom,1,foo},{white_space,1," "}],1} = 597: erl_scan:string("foo % bar", 1, return_white_spaces), 598: ?line {ok,[{atom,1,foo},{comment,1,"% bar"}],1} = 599: erl_scan:string("foo % bar", 1, return_comments), 600: ?line {ok,[{atom,17,foo}],17} = 601: erl_scan:string("foo % bar", 17), 602: ?line {'EXIT',{function_clause,_}} = 603: (catch {foo, 604: erl_scan:string("foo % bar", {a,1}, [])}), % type error 605: ?line {ok,[{atom,_,foo}],{17,18}} = 606: erl_scan:string("foo % bar", {17,9}, []), 607: ?line {'EXIT',{function_clause,_}} = 608: (catch {foo, 609: erl_scan:string("foo % bar", {1,0}, [])}), % type error 610: ?line {ok,[{foo,1}],1} = 611: erl_scan:string("foo % bar",1, [{reserved_word_fun, 612: fun(W) -> W =:= foo end}]), 613: ?line {'EXIT',{badarg,_}} = 614: (catch {foo, 615: erl_scan:string("foo % bar",1, % type error 616: [{reserved_word_fun, 617: fun(W,_) -> W =:= foo end}])}), 618: ok. 619: 620: more_options() -> 621: ?line {ok,[{atom,A1,foo}],{19,20}} = 622: erl_scan:string("foo", {19,17},[]), 623: ?line [{column,17},{line,19}] = erl_scan:attributes_info(A1), 624: ?line {done,{ok,[{atom,A2,foo},{dot,_}],{19,22}},[]} = 625: erl_scan:tokens([], "foo. ", {19,17}, [bad_opt]), % type error 626: ?line [{column,17},{line,19}] = erl_scan:attributes_info(A2), 627: ?line {ok,[{atom,A3,foo}],{19,20}} = 628: erl_scan:string("foo", {19,17},[text]), 629: ?line [{column,17},{length,3},{line,19},{text,"foo"}] = 630: erl_scan:attributes_info(A3), 631: 632: ?line {ok,[{atom,A4,foo}],1} = erl_scan:string("foo", 1, [text]), 633: ?line [{length,3},{line,1},{text,"foo"}] = erl_scan:attributes_info(A4), 634: 635: ok. 636: 637: token_info() -> 638: ?line {ok,[T1],_} = erl_scan:string("foo", {1,18}, [text]), 639: {'EXIT',{badarg,_}} = 640: (catch {foo, erl_scan:token_info(T1, foo)}), % type error 641: ?line {line,1} = erl_scan:token_info(T1, line), 642: ?line {column,18} = erl_scan:token_info(T1, column), 643: ?line {length,3} = erl_scan:token_info(T1, length), 644: ?line {text,"foo"} = erl_scan:token_info(T1, text), 645: ?line [{category,atom},{column,18},{length,3},{line,1}, 646: {symbol,foo},{text,"foo"}] = 647: erl_scan:token_info(T1), 648: ?line [{length,3},{column,18}] = 649: erl_scan:token_info(T1, [length, column]), 650: ?line [{location,{1,18}}] = 651: erl_scan:token_info(T1, [location]), 652: ?line {category,atom} = erl_scan:token_info(T1, category), 653: ?line [{symbol,foo}] = erl_scan:token_info(T1, [symbol]), 654: 655: ?line {ok,[T2],_} = erl_scan:string("foo", 1, []), 656: ?line {line,1} = erl_scan:token_info(T2, line), 657: ?line undefined = erl_scan:token_info(T2, column), 658: ?line undefined = erl_scan:token_info(T2, length), 659: ?line undefined = erl_scan:token_info(T2, text), 660: ?line {location,1} = erl_scan:token_info(T2, location), 661: ?line [{category,atom},{line,1},{symbol,foo}] = erl_scan:token_info(T2), 662: ?line [{line,1}] = erl_scan:token_info(T2, [length, line]), 663: 664: ?line {ok,[T3],_} = erl_scan:string("=", 1, []), 665: ?line [{line,1}] = erl_scan:token_info(T3, [column, line]), 666: ?line {category,'='} = erl_scan:token_info(T3, category), 667: ?line [{symbol,'='}] = erl_scan:token_info(T3, [symbol]), 668: ok. 669: 670: attributes_info() -> 671: ?line {'EXIT',_} = 672: (catch {foo,erl_scan:attributes_info(foo)}), % type error 673: ?line [{line,18}] = erl_scan:attributes_info(18), 674: ?line {location,19} = erl_scan:attributes_info(19, location), 675: ?line {ok,[{atom,A0,foo}],_} = erl_scan:string("foo", 19, [text]), 676: ?line {location,19} = erl_scan:attributes_info(A0, location), 677: 678: ?line {ok,[{atom,A3,foo}],_} = erl_scan:string("foo", {1,3}, [text]), 679: ?line {line,1} = erl_scan:attributes_info(A3, line), 680: ?line {column,3} = erl_scan:attributes_info(A3, column), 681: ?line {location,{1,3}} = erl_scan:attributes_info(A3, location), 682: ?line {text,"foo"} = erl_scan:attributes_info(A3, text), 683: 684: ?line {ok,[{atom,A4,foo}],_} = erl_scan:string("foo", 2, [text]), 685: ?line {line,2} = erl_scan:attributes_info(A4, line), 686: ?line undefined = erl_scan:attributes_info(A4, column), 687: ?line {location,2} = erl_scan:attributes_info(A4, location), 688: ?line {text,"foo"} = erl_scan:attributes_info(A4, text), 689: 690: ?line {ok,[{atom,A5,foo}],_} = erl_scan:string("foo", {1,3}, []), 691: ?line {line,1} = erl_scan:attributes_info(A5, line), 692: ?line {column,3} = erl_scan:attributes_info(A5, column), 693: ?line {location,{1,3}} = erl_scan:attributes_info(A5, location), 694: ?line undefined = erl_scan:attributes_info(A5, text), 695: 696: ?line undefined = erl_scan:attributes_info([], line), % type error 697: 698: ok. 699: 700: set_attribute() -> 701: F = fun(Line) -> -Line end, 702: ?line -2 = erl_scan:set_attribute(line, 2, F), 703: ?line {ok,[{atom,A1,foo}],_} = erl_scan:string("foo", {9,17}), 704: ?line A2 = erl_scan:set_attribute(line, A1, F), 705: ?line {line,-9} = erl_scan:attributes_info(A2, line), 706: ?line {location,{-9,17}} = erl_scan:attributes_info(A2, location), 707: ?line [{line,-9},{column,17}] = 708: erl_scan:attributes_info(A2, [line,column,text]), 709: 710: F2 = fun(Line) -> {17,Line} end, 711: ?line Attr1 = erl_scan:set_attribute(line, 2, F2), 712: ?line {line,{17,2}} = erl_scan:attributes_info(Attr1, line), 713: ?line undefined = erl_scan:attributes_info(Attr1, column), 714: ?line {location,{17,2}} = % a bit mixed up 715: erl_scan:attributes_info(Attr1, location), 716: 717: ?line A3 = erl_scan:set_attribute(line, A1, F2), 718: ?line {line,{17,9}} = erl_scan:attributes_info(A3, line), 719: ?line {location,{{17,9},17}} = erl_scan:attributes_info(A3, location), 720: ?line [{line,{17,9}},{column,17}] = 721: erl_scan:attributes_info(A3, [line,column,text]), 722: 723: ?line {ok,[{atom,A4,foo}],_} = erl_scan:string("foo", {9,17}, [text]), 724: ?line A5 = erl_scan:set_attribute(line, A4, F), 725: ?line {line,-9} = erl_scan:attributes_info(A5, line), 726: ?line {location,{-9,17}} = erl_scan:attributes_info(A5, location), 727: ?line [{line,-9},{column,17},{text,"foo"}] = 728: erl_scan:attributes_info(A5, [line,column,text]), 729: 730: ?line {ok,[{atom,A6,foo}],_} = erl_scan:string("foo", 11, [text]), 731: ?line A7 = erl_scan:set_attribute(line, A6, F2), 732: ?line {line,{17,11}} = erl_scan:attributes_info(A7, line), 733: ?line {location,{17,11}} = % mixed up 734: erl_scan:attributes_info(A7, location), 735: ?line [{line,{17,11}},{text,"foo"}] = 736: erl_scan:attributes_info(A7, [line,column,text]), 737: 738: ?line {'EXIT',_} = 739: (catch {foo, erl_scan:set_attribute(line, [], F2)}), % type error 740: ?line {'EXIT',{badarg,_}} = 741: (catch {foo, erl_scan:set_attribute(column, [], F2)}), % type error 742: 743: %% OTP-9412 744: ?line 8 = erl_scan:set_attribute(line, [{line,{nos,'X',8}}], 745: fun({nos,_V,VL}) -> VL end), 746: ok. 747: 748: column_errors() -> 749: ?line {error,{{1,1},erl_scan,{string,$',""}},{1,3}} = % $' 750: erl_scan:string("'\\",{1,1}), 751: ?line {error,{{1,1},erl_scan,{string,$",""}},{1,3}} = % $" 752: erl_scan:string("\"\\",{1,1}), 753: 754: ?line {error,{{1,1},erl_scan,{string,$',""}},{1,2}} = % $' 755: erl_scan:string("'",{1,1}), 756: ?line {error,{{1,1},erl_scan,{string,$",""}},{1,2}} = % $" 757: erl_scan:string("\"",{1,1}), 758: 759: ?line {error,{{1,1},erl_scan,char},{1,2}} = 760: erl_scan:string("$",{1,1}), 761: 762: ?line {error,{{1,2},erl_scan,{string,$',"1234567890123456"}},{1,20}} = %' 763: erl_scan:string(" '12345678901234567", {1,1}), 764: ?line {error,{{1,2},erl_scan,{string,$',"123456789012345 "}}, {1,20}} = %' 765: erl_scan:string(" '123456789012345\\s", {1,1}), 766: ?line {error,{{1,2},erl_scan,{string,$","1234567890123456"}},{1,20}} = %" 767: erl_scan:string(" \"12345678901234567", {1,1}), 768: ?line {error,{{1,2},erl_scan,{string,$","123456789012345 "}}, {1,20}} = %" 769: erl_scan:string(" \"123456789012345\\s", {1,1}), 770: ?line {error,{{1,2},erl_scan,{string,$',"1234567890123456"}},{2,1}} = %' 771: erl_scan:string(" '12345678901234567\n", {1,1}), 772: ok. 773: 774: white_spaces() -> 775: ?line {ok,[{white_space,_,"\r"}, 776: {white_space,_," "}, 777: {atom,_,a}, 778: {white_space,_,"\n"}], 779: _} = erl_scan:string("\r a\n", {1,1}, return), 780: ?line test("\r a\n"), 781: L = "{\"a\nb\", \"a\\nb\",\nabc\r,def}.\n\n", 782: ?line {ok,[{'{',_}, 783: {string,_,"a\nb"}, 784: {',',_}, 785: {white_space,_," "}, 786: {string,_,"a\nb"}, 787: {',',_}, 788: {white_space,_,"\n"}, 789: {atom,_,abc}, 790: {white_space,_,"\r"}, 791: {',',_}, 792: {atom,_,def}, 793: {'}',_}, 794: {dot,_}, 795: {white_space,_,"\n"}], 796: _} = erl_scan:string(L, {1,1}, return), 797: ?line test(L), 798: ?line test("\"\n\"\n"), 799: ?line test("\n\r\n"), 800: ?line test("\n\r"), 801: ?line test("\r\n"), 802: ?line test("\n\f"), 803: ?line [test(lists:duplicate(N, $\t)) || N <- lists:seq(1, 20)], 804: ?line [test([$\n|lists:duplicate(N, $\t)]) || N <- lists:seq(1, 20)], 805: ?line [test(lists:duplicate(N, $\s)) || N <- lists:seq(1, 20)], 806: ?line [test([$\n|lists:duplicate(N, $\s)]) || N <- lists:seq(1, 20)], 807: ?line test("\v\f\n\v "), 808: ?line test("\n\e\n\b\f\n\da\n"), 809: ok. 810: 811: unicode() -> 812: ?line {ok,[{char,1,83},{integer,1,45}],1} = 813: erl_scan:string("$\\12345"), % not unicode 814: 815: ?line {error,{1,erl_scan,{illegal,character}},1} = 816: erl_scan:string([1089]), 817: ?line {error,{{1,1},erl_scan,{illegal,character}},{1,2}} = 818: erl_scan:string([1089], {1,1}), 819: ?line {error,{1,erl_scan,{illegal,atom}},1} = 820: erl_scan:string("'a"++[1089]++"b'", 1), 821: ?line {error,{{1,1},erl_scan,{illegal,atom}},{1,6}} = 822: erl_scan:string("'a"++[1089]++"b'", {1,1}), 823: ?line test("\"a"++[1089]++"b\""), 824: ?line {ok,[{char,1,1}],1} = 825: erl_scan:string([$$,$\\,$^,1089], 1), 826: 827: ?line {error,{1,erl_scan,Error},1} = 828: erl_scan:string("\"qa\x{aaa}", 1), 829: ?line "unterminated string starting with \"qa"++[2730]++"\"" = 830: erl_scan:format_error(Error), 831: ?line {error,{{1,1},erl_scan,_},{1,11}} = 832: erl_scan:string("\"qa\\x{aaa}",{1,1}), 833: ?line {error,{{1,1},erl_scan,{illegal,atom}},{1,12}} = 834: erl_scan:string("'qa\\x{aaa}'",{1,1}), 835: 836: ?line {ok,[{char,1,1089}],1} = 837: erl_scan:string([$$,1089], 1), 838: ?line {ok,[{char,1,1089}],1} = 839: erl_scan:string([$$,$\\,1089], 1), 840: 841: Qs = "$\\x{aaa}", 842: ?line {ok,[{char,1,$\x{aaa}}],1} = 843: erl_scan:string(Qs, 1), 844: ?line {ok,[Q2],{1,9}} = 845: erl_scan:string("$\\x{aaa}", {1,1}, [text]), 846: ?line [{category,char},{column,1},{length,8}, 847: {line,1},{symbol,16#aaa},{text,Qs}] = 848: erl_scan:token_info(Q2), 849: 850: U1 = "\"\\x{aaa}\"", 851: {ok, 852: [{string,[{line,1},{column,1},{text,"\"\\x{aaa}\""}],[2730]}], 853: {1,10}} = erl_scan:string(U1, {1,1}, [text]), 854: {ok,[{string,1,[2730]}],1} = erl_scan:string(U1, 1), 855: 856: U2 = "\"\\x41\\x{fff}\\x42\"", 857: {ok,[{string,1,[$\x41,$\x{fff},$\x42]}],1} = erl_scan:string(U2, 1), 858: 859: U3 = "\"a\n\\x{fff}\n\"", 860: {ok,[{string,1,[$a,$\n,$\x{fff},$\n]}],3} = erl_scan:string(U3, 1), 861: 862: U4 = "\"\\^\n\\x{aaa}\\^\n\"", 863: {ok,[{string,1,[$\n,$\x{aaa},$\n]}],3} = erl_scan:string(U4, 1), 864: 865: %% Keep these tests: 866: ?line test(Qs), 867: ?line test(U1), 868: ?line test(U2), 869: ?line test(U3), 870: ?line test(U4), 871: 872: Str1 = "\"ab" ++ [1089] ++ "cd\"", 873: {ok,[{string,1,[$a,$b,1089,$c,$d]}],1} = erl_scan:string(Str1, 1), 874: {ok,[{string,{1,1},[$a,$b,1089,$c,$d]}],{1,8}} = 875: erl_scan:string(Str1, {1,1}), 876: ?line test(Str1), 877: Comment = "%% "++[1089], 878: {ok,[{comment,1,[$%,$%,$\s,1089]}],1} = 879: erl_scan:string(Comment, 1, [return]), 880: {ok,[{comment,{1,1},[$%,$%,$\s,1089]}],{1,5}} = 881: erl_scan:string(Comment, {1,1}, [return]), 882: ok. 883: 884: more_chars() -> 885: %% Due to unicode, the syntax has been incompatibly augmented: 886: %% $\x{...}, $\xHH 887: 888: %% All kinds of tests... 889: ?line {ok,[{char,_,123}],{1,4}} = 890: erl_scan:string("$\\{",{1,1}), 891: ?line {more, C1} = erl_scan:tokens([], "$\\{", {1,1}), 892: ?line {done,{ok,[{char,_,123}],{1,4}},eof} = 893: erl_scan:tokens(C1, eof, 1), 894: ?line {ok,[{char,1,123},{atom,1,a},{'}',1}],1} = 895: erl_scan:string("$\\{a}"), 896: 897: ?line {error,{{1,1},erl_scan,char},{1,4}} = 898: erl_scan:string("$\\x", {1,1}), 899: ?line {error,{{1,1},erl_scan,char},{1,5}} = 900: erl_scan:string("$\\x{",{1,1}), 901: ?line {more, C3} = erl_scan:tokens([], "$\\x", {1,1}), 902: ?line {done,{error,{{1,1},erl_scan,char},{1,4}},eof} = 903: erl_scan:tokens(C3, eof, 1), 904: ?line {error,{{1,1},erl_scan,char},{1,5}} = 905: erl_scan:string("$\\x{",{1,1}), 906: ?line {more, C2} = erl_scan:tokens([], "$\\x{", {1,1}), 907: ?line {done,{error,{{1,1},erl_scan,char},{1,5}},eof} = 908: erl_scan:tokens(C2, eof, 1), 909: ?line {error,{1,erl_scan,{illegal,character}},1} = 910: erl_scan:string("$\\x{g}"), 911: ?line {error,{{1,1},erl_scan,{illegal,character}},{1,5}} = 912: erl_scan:string("$\\x{g}", {1,1}), 913: ?line {error,{{1,1},erl_scan,{illegal,character}},{1,6}} = 914: erl_scan:string("$\\x{}",{1,1}), 915: 916: ?line test("\"\\{0}\""), 917: ?line test("\"\\x{0}\""), 918: ?line test("\'\\{0}\'"), 919: ?line test("\'\\x{0}\'"), 920: 921: ?line {error,{{2,3},erl_scan,{illegal,character}},{2,6}} = 922: erl_scan:string("\"ab \n $\\x{g}\"",{1,1}), 923: ?line {error,{{2,3},erl_scan,{illegal,character}},{2,6}} = 924: erl_scan:string("\'ab \n $\\x{g}\'",{1,1}), 925: 926: ?line test("$\\{34}"), 927: ?line test("$\\x{34}"), 928: ?line test("$\\{377}"), 929: ?line test("$\\x{FF}"), 930: ?line test("$\\{400}"), 931: ?line test("$\\x{100}"), 932: ?line test("$\\x{10FFFF}"), 933: ?line test("$\\x{10ffff}"), 934: ?line test("\"$\n \\{1}\""), 935: ?line {error,{1,erl_scan,{illegal,character}},1} = 936: erl_scan:string("$\\x{110000}"), 937: ?line {error,{{1,1},erl_scan,{illegal,character}},{1,12}} = 938: erl_scan:string("$\\x{110000}", {1,1}), 939: 940: ?line {error,{{1,1},erl_scan,{illegal,character}},{1,4}} = 941: erl_scan:string("$\\xfg", {1,1}), 942: 943: ?line test("$\\xffg"), 944: 945: ?line {error,{{1,1},erl_scan,{illegal,character}},{1,4}} = 946: erl_scan:string("$\\xg", {1,1}), 947: ok. 948: 949: otp_10302(doc) -> 950: "OTP-10302. Unicode characters scanner/parser."; 951: otp_10302(suite) -> 952: []; 953: otp_10302(Config) when is_list(Config) -> 954: %% From unicode(): 955: {error,{1,erl_scan,{illegal,atom}},1} = 956: erl_scan:string("'a"++[1089]++"b'", 1), 957: {error,{{1,1},erl_scan,{illegal,atom}},{1,12}} = 958: erl_scan:string("'qa\\x{aaa}'",{1,1}), 959: 960: {ok,[{char,1,1089}],1} = erl_scan:string([$$,1089], 1), 961: {ok,[{char,1,1089}],1} = erl_scan:string([$$,$\\,1089],1), 962: 963: Qs = "$\\x{aaa}", 964: {ok,[{char,1,2730}],1} = erl_scan:string(Qs,1), 965: {ok,[Q2],{1,9}} = erl_scan:string(Qs,{1,1},[text]), 966: [{category,char},{column,1},{length,8}, 967: {line,1},{symbol,16#aaa},{text,Qs}] = 968: erl_scan:token_info(Q2), 969: 970: Tags = [category, column, length, line, symbol, text], 971: 972: U1 = "\"\\x{aaa}\"", 973: {ok,[T1],{1,10}} = erl_scan:string(U1, {1,1}, [text]), 974: [{category,string},{column,1},{length,9},{line,1}, 975: {symbol,[16#aaa]},{text,U1}] = erl_scan:token_info(T1, Tags), 976: 977: U2 = "\"\\x41\\x{fff}\\x42\"", 978: {ok,[{string,1,[65,4095,66]}],1} = erl_scan:string(U2, 1), 979: 980: U3 = "\"a\n\\x{fff}\n\"", 981: {ok,[{string,1,[97,10,4095,10]}],3} = erl_scan:string(U3, 1), 982: 983: U4 = "\"\\^\n\\x{aaa}\\^\n\"", 984: {ok,[{string,1,[10,2730,10]}],3} = erl_scan:string(U4, 1,[]), 985: 986: Str1 = "\"ab" ++ [1089] ++ "cd\"", 987: {ok,[{string,1,[97,98,1089,99,100]}],1} = 988: erl_scan:string(Str1,1), 989: {ok,[{string,{1,1},[97,98,1089,99,100]}],{1,8}} = 990: erl_scan:string(Str1, {1,1}), 991: 992: OK1 = 16#D800-1, 993: OK2 = 16#DFFF+1, 994: OK3 = 16#FFFE-1, 995: OK4 = 16#FFFF+1, 996: OKL = [OK1,OK2,OK3,OK4], 997: 998: Illegal1 = 16#D800, 999: Illegal2 = 16#DFFF, 1000: Illegal3 = 16#FFFE, 1001: Illegal4 = 16#FFFF, 1002: IllegalL = [Illegal1,Illegal2,Illegal3,Illegal4], 1003: 1004: [{ok,[{comment,1,[$%,$%,$\s,OK]}],1} = 1005: erl_scan:string("%% "++[OK], 1, [return]) || 1006: OK <- OKL], 1007: {ok,[{comment,_,[$%,$%,$\s,OK1]}],{1,5}} = 1008: erl_scan:string("%% "++[OK1], {1,1}, [return]), 1009: [{error,{1,erl_scan,{illegal,character}},1} = 1010: erl_scan:string("%% "++[Illegal], 1, [return]) || 1011: Illegal <- IllegalL], 1012: {error,{{1,1},erl_scan,{illegal,character}},{1,5}} = 1013: erl_scan:string("%% "++[Illegal1], {1,1}, [return]), 1014: 1015: [{ok,[],1} = erl_scan:string("%% "++[OK], 1, []) || 1016: OK <- OKL], 1017: {ok,[],{1,5}} = erl_scan:string("%% "++[OK1], {1,1}, []), 1018: [{error,{1,erl_scan,{illegal,character}},1} = 1019: erl_scan:string("%% "++[Illegal], 1, []) || 1020: Illegal <- IllegalL], 1021: {error,{{1,1},erl_scan,{illegal,character}},{1,5}} = 1022: erl_scan:string("%% "++[Illegal1], {1,1}, []), 1023: 1024: [{ok,[{string,{1,1},[OK]}],{1,4}} = 1025: erl_scan:string("\""++[OK]++"\"",{1,1}) || 1026: OK <- OKL], 1027: [{error,{{1,2},erl_scan,{illegal,character}},{1,3}} = 1028: erl_scan:string("\""++[OK]++"\"",{1,1}) || 1029: OK <- IllegalL], 1030: 1031: [{error,{{1,1},erl_scan,{illegal,character}},{1,2}} = 1032: erl_scan:string([Illegal],{1,1}) || 1033: Illegal <- IllegalL], 1034: 1035: {ok,[{char,{1,1},OK1}],{1,3}} = 1036: erl_scan:string([$$,OK1],{1,1}), 1037: {error,{{1,1},erl_scan,{illegal,character}},{1,2}} = 1038: erl_scan:string([$$,Illegal1],{1,1}), 1039: 1040: {ok,[{char,{1,1},OK1}],{1,4}} = 1041: erl_scan:string([$$,$\\,OK1],{1,1}), 1042: {error,{{1,1},erl_scan,{illegal,character}},{1,4}} = 1043: erl_scan:string([$$,$\\,Illegal1],{1,1}), 1044: 1045: {ok,[{string,{1,1},[55295]}],{1,5}} = 1046: erl_scan:string("\"\\"++[OK1]++"\"",{1,1}), 1047: {error,{{1,2},erl_scan,{illegal,character}},{1,4}} = 1048: erl_scan:string("\"\\"++[Illegal1]++"\"",{1,1}), 1049: 1050: {ok,[{char,{1,1},OK1}],{1,10}} = 1051: erl_scan:string("$\\x{D7FF}",{1,1}), 1052: {error,{{1,1},erl_scan,{illegal,character}},{1,10}} = 1053: erl_scan:string("$\\x{D800}",{1,1}), 1054: 1055: %% Not erl_scan, but erl_parse. 1056: {integer,0,1} = erl_parse:abstract(1), 1057: Float = 3.14, {float,0,Float} = erl_parse:abstract(Float), 1058: {nil,0} = erl_parse:abstract([]), 1059: {bin,0, 1060: [{bin_element,0,{integer,0,1},default,default}, 1061: {bin_element,0,{integer,0,2},default,default}]} = 1062: erl_parse:abstract(<<1,2>>), 1063: {cons,0,{tuple,0,[{atom,0,a}]},{atom,0,b}} = 1064: erl_parse:abstract([{a} | b]), 1065: {string,0,"str"} = erl_parse:abstract("str"), 1066: {cons,0, 1067: {integer,0,$a}, 1068: {cons,0,{integer,0,1024},{string,0,"c"}}} = 1069: erl_parse:abstract("a"++[1024]++"c"), 1070: 1071: Line = 17, 1072: {integer,Line,1} = erl_parse:abstract(1, Line), 1073: Float = 3.14, {float,Line,Float} = erl_parse:abstract(Float, Line), 1074: {nil,Line} = erl_parse:abstract([], Line), 1075: {bin,Line, 1076: [{bin_element,Line,{integer,Line,1},default,default}, 1077: {bin_element,Line,{integer,Line,2},default,default}]} = 1078: erl_parse:abstract(<<1,2>>, Line), 1079: {cons,Line,{tuple,Line,[{atom,Line,a}]},{atom,Line,b}} = 1080: erl_parse:abstract([{a} | b], Line), 1081: {string,Line,"str"} = erl_parse:abstract("str", Line), 1082: {cons,Line, 1083: {integer,Line,$a}, 1084: {cons,Line,{integer,Line,1024},{string,Line,"c"}}} = 1085: erl_parse:abstract("a"++[1024]++"c", Line), 1086: 1087: Opts1 = [{line,17}], 1088: {integer,Line,1} = erl_parse:abstract(1, Opts1), 1089: Float = 3.14, {float,Line,Float} = erl_parse:abstract(Float, Opts1), 1090: {nil,Line} = erl_parse:abstract([], Opts1), 1091: {bin,Line, 1092: [{bin_element,Line,{integer,Line,1},default,default}, 1093: {bin_element,Line,{integer,Line,2},default,default}]} = 1094: erl_parse:abstract(<<1,2>>, Opts1), 1095: {cons,Line,{tuple,Line,[{atom,Line,a}]},{atom,Line,b}} = 1096: erl_parse:abstract([{a} | b], Opts1), 1097: {string,Line,"str"} = erl_parse:abstract("str", Opts1), 1098: {cons,Line, 1099: {integer,Line,$a}, 1100: {cons,Line,{integer,Line,1024},{string,Line,"c"}}} = 1101: erl_parse:abstract("a"++[1024]++"c", Opts1), 1102: 1103: [begin 1104: {integer,Line,1} = erl_parse:abstract(1, Opts2), 1105: Float = 3.14, {float,Line,Float} = erl_parse:abstract(Float, Opts2), 1106: {nil,Line} = erl_parse:abstract([], Opts2), 1107: {bin,Line, 1108: [{bin_element,Line,{integer,Line,1},default,default}, 1109: {bin_element,Line,{integer,Line,2},default,default}]} = 1110: erl_parse:abstract(<<1,2>>, Opts2), 1111: {cons,Line,{tuple,Line,[{atom,Line,a}]},{atom,Line,b}} = 1112: erl_parse:abstract([{a} | b], Opts2), 1113: {string,Line,"str"} = erl_parse:abstract("str", Opts2), 1114: {string,Line,[97,1024,99]} = 1115: erl_parse:abstract("a"++[1024]++"c", Opts2) 1116: end || Opts2 <- [[{encoding,unicode},{line,Line}], 1117: [{encoding,utf8},{line,Line}]]], 1118: 1119: {cons,0, 1120: {integer,0,97}, 1121: {cons,0,{integer,0,1024},{string,0,"c"}}} = 1122: erl_parse:abstract("a"++[1024]++"c", [{encoding,latin1}]), 1123: ok. 1124: 1125: otp_10990(doc) -> 1126: "OTP-10990. Floating point number in input string."; 1127: otp_10990(suite) -> 1128: []; 1129: otp_10990(Config) when is_list(Config) -> 1130: {'EXIT',_} = (catch {foo, erl_scan:string([$",42.0,$"],1)}), 1131: ok. 1132: 1133: otp_10992(doc) -> 1134: "OTP-10992. List of floats to abstract format."; 1135: otp_10992(suite) -> 1136: []; 1137: otp_10992(Config) when is_list(Config) -> 1138: {cons,0,{float,0,42.0},{nil,0}} = 1139: erl_parse:abstract([42.0], [{encoding,unicode}]), 1140: {cons,0,{float,0,42.0},{nil,0}} = 1141: erl_parse:abstract([42.0], [{encoding,utf8}]), 1142: {cons,0,{integer,0,65},{cons,0,{float,0,42.0},{nil,0}}} = 1143: erl_parse:abstract([$A,42.0], [{encoding,unicode}]), 1144: {cons,0,{integer,0,65},{cons,0,{float,0,42.0},{nil,0}}} = 1145: erl_parse:abstract([$A,42.0], [{encoding,utf8}]), 1146: ok. 1147: 1148: test_string(String, Expected) -> 1149: {ok, Expected, _End} = erl_scan:string(String), 1150: test(String). 1151: 1152: %% test_string(String, Expected, StartLocation, Options) -> 1153: %% {ok, Expected, _End} = erl_scan:string(String, StartLocation, Options), 1154: %% test(String). 1155: 1156: %% There are no checks of the tags... 1157: test(String) -> 1158: %% io:format("Testing `~ts'~n", [String]), 1159: [{Tokens, End}, 1160: {Wtokens, Wend}, 1161: {Ctokens, Cend}, 1162: {CWtokens, CWend}, 1163: {CWtokens2, _}] = 1164: [scan_string_with_column(String, X) || 1165: X <- [[], 1166: [return_white_spaces], 1167: [return_comments], 1168: [return], 1169: [return]]], % for white space compaction test 1170: 1171: {end1,End,Wend} = {end1,Wend,End}, 1172: {end2,Wend,Cend} = {end2,Cend,Wend}, 1173: {end3,Cend,CWend} = {end3,CWend,Cend}, 1174: 1175: %% Test that the tokens that are common to two token lists are identical. 1176: {none,Tokens} = {none, filter_tokens(CWtokens, [white_space,comment])}, 1177: {comments,Ctokens} = 1178: {comments,filter_tokens(CWtokens, [white_space])}, 1179: {white_spaces,Wtokens} = 1180: {white_spaces,filter_tokens(CWtokens, [comment])}, 1181: 1182: %% Use token attributes to extract parts from the original string, 1183: %% and check that the parts are identical to the token strings. 1184: {Line,Column} = test_decorated_tokens(String, CWtokens), 1185: {deco,{Line,Column},End} = {deco,End,{Line,Column}}, 1186: 1187: %% Almost the same again: concat texts to get the original: 1188: Text = get_text(CWtokens), 1189: {text,Text,String} = {text,String,Text}, 1190: 1191: %% Test that white spaces occupy less heap than the worst case. 1192: ok = test_white_space_compaction(CWtokens, CWtokens2), 1193: 1194: %% Test that white newlines are always first in text: 1195: WhiteTokens = select_tokens(CWtokens, [white_space]), 1196: ok = newlines_first(WhiteTokens), 1197: 1198: %% Line attribute only: 1199: [Simple,Wsimple,Csimple,WCsimple] = Simples = 1200: [element(2, erl_scan:string(String, 1, Opts)) || 1201: Opts <- [[], 1202: [return_white_spaces], 1203: [return_comments], 1204: [return]]], 1205: {consistent,true} = {consistent,consistent_attributes(Simples)}, 1206: {simple_wc,WCsimple} = {simple_wc,simplify(CWtokens)}, 1207: {simple,Simple} = {simple,filter_tokens(WCsimple, [white_space,comment])}, 1208: {simple_c,Csimple} = {simple_c,filter_tokens(WCsimple, [white_space])}, 1209: {simple_w,Wsimple} = {simple_w,filter_tokens(WCsimple, [comment])}, 1210: 1211: %% Line attribute only, with text: 1212: [SimpleTxt,WsimpleTxt,CsimpleTxt,WCsimpleTxt] = SimplesTxt = 1213: [element(2, erl_scan:string(String, 1, [text|Opts])) || 1214: Opts <- [[], 1215: [return_white_spaces], 1216: [return_comments], 1217: [return]]], 1218: TextTxt = get_text(WCsimpleTxt), 1219: {text_txt,TextTxt,String} = {text_txt,String,TextTxt}, 1220: {consistent_txt,true} = 1221: {consistent_txt,consistent_attributes(SimplesTxt)}, 1222: {simple_txt,SimpleTxt} = 1223: {simple_txt,filter_tokens(WCsimpleTxt, [white_space,comment])}, 1224: {simple_c_txt,CsimpleTxt} = 1225: {simple_c_txt,filter_tokens(WCsimpleTxt, [white_space])}, 1226: {simple_w_txt,WsimpleTxt} = 1227: {simple_w_txt,filter_tokens(WCsimpleTxt, [comment])}, 1228: 1229: ok. 1230: 1231: test_white_space_compaction(Tokens, Tokens2) when Tokens =:= Tokens2 -> 1232: [WS, WS2] = [select_tokens(Ts, [white_space]) || Ts <- [Tokens, Tokens2]], 1233: test_wsc(WS, WS2). 1234: 1235: test_wsc([], []) -> 1236: ok; 1237: test_wsc([Token|Tokens], [Token2|Tokens2]) -> 1238: [Text, Text2] = [Text || 1239: {text, Text} <- 1240: [erl_scan:token_info(T, text) || 1241: T <- [Token, Token2]]], 1242: Sz = erts_debug:size(Text), 1243: Sz2 = erts_debug:size({Text, Text2}), 1244: IsCompacted = Sz2 < 2*Sz+erts_debug:size({a,a}), 1245: ToBeCompacted = is_compacted(Text), 1246: if 1247: IsCompacted =:= ToBeCompacted -> 1248: test_wsc(Tokens, Tokens2); 1249: true -> 1250: {compaction_error, Token} 1251: end. 1252: 1253: is_compacted("\r") -> 1254: true; 1255: is_compacted("\n\r") -> 1256: true; 1257: is_compacted("\n\f") -> 1258: true; 1259: is_compacted([$\n|String]) -> 1260: all_spaces(String) 1261: orelse 1262: all_tabs(String); 1263: is_compacted(String) -> 1264: all_spaces(String) 1265: orelse 1266: all_tabs(String). 1267: 1268: all_spaces(L) -> 1269: all_same(L, $\s). 1270: 1271: all_tabs(L) -> 1272: all_same(L, $\t). 1273: 1274: all_same(L, Char) -> 1275: lists:all(fun(C) -> C =:= Char end, L). 1276: 1277: newlines_first([]) -> 1278: ok; 1279: newlines_first([Token|Tokens]) -> 1280: {text,Text} = erl_scan:token_info(Token, text), 1281: Nnls = length([C || C <- Text, C =:= $\n]), 1282: OK = case Text of 1283: [$\n|_] -> 1284: Nnls =:= 1; 1285: _ -> 1286: Nnls =:= 0 1287: end, 1288: if 1289: OK -> newlines_first(Tokens); 1290: true -> OK 1291: end. 1292: 1293: filter_tokens(Tokens, Tags) -> 1294: lists:filter(fun(T) -> not lists:member(element(1, T), Tags) end, Tokens). 1295: 1296: select_tokens(Tokens, Tags) -> 1297: lists:filter(fun(T) -> lists:member(element(1, T), Tags) end, Tokens). 1298: 1299: simplify([Token|Tokens]) -> 1300: {line,Line} = erl_scan:token_info(Token, line), 1301: [setelement(2, Token, Line) | simplify(Tokens)]; 1302: simplify([]) -> 1303: []. 1304: 1305: get_text(Tokens) -> 1306: lists:flatten( 1307: [T || 1308: Token <- Tokens, 1309: ({text,T} = erl_scan:token_info(Token, text)) =/= []]). 1310: 1311: test_decorated_tokens(String, Tokens) -> 1312: ToksAttrs = token_attrs(Tokens), 1313: test_strings(ToksAttrs, String, 1, 1). 1314: 1315: token_attrs(Tokens) -> 1316: [{L,C,Len,T} || 1317: Token <- Tokens, 1318: ([{line,L},{column,C},{length,Len},{text,T}] = 1319: erl_scan:token_info(Token, [line,column,length,text])) =/= []]. 1320: 1321: test_strings([], _S, Line, Column) -> 1322: {Line,Column}; 1323: test_strings([{L,C,Len,T}=Attr|Attrs], String0, Line0, Column0) -> 1324: {String1, Column1} = skip_newlines(String0, L, Line0, Column0), 1325: String = skip_chars(String1, C-Column1), 1326: {Str,Rest} = lists:split(Len, String), 1327: if 1328: Str =:= T -> 1329: {Line,Column} = string_newlines(T, L, C), 1330: test_strings(Attrs, Rest, Line, Column); 1331: true -> 1332: {token_error, Attr, Str} 1333: end. 1334: 1335: skip_newlines(String, Line, Line, Column) -> 1336: {String, Column}; 1337: skip_newlines([$\n|String], L, Line, _Column) -> 1338: skip_newlines(String, L, Line+1, 1); 1339: skip_newlines([_|String], L, Line, Column) -> 1340: skip_newlines(String, L, Line, Column+1). 1341: 1342: skip_chars(String, 0) -> 1343: String; 1344: skip_chars([_|String], N) -> 1345: skip_chars(String, N-1). 1346: 1347: string_newlines([$\n|String], Line, _Column) -> 1348: string_newlines(String, Line+1, 1); 1349: string_newlines([], Line, Column) -> 1350: {Line, Column}; 1351: string_newlines([_|String], Line, Column) -> 1352: string_newlines(String, Line, Column+1). 1353: 1354: scan_string_with_column(String, Options0) -> 1355: Options = [text | Options0], 1356: StartLoc = {1, 1}, 1357: {ok, Ts1, End1} = erl_scan:string(String, StartLoc, Options), 1358: TString = String ++ ". ", 1359: {ok,Ts2,End2} = scan_tokens(TString, Options, [], StartLoc), 1360: {ok, Ts3, End3} = 1361: scan_tokens_1({more, []}, TString, Options, [], StartLoc), 1362: {end_2,End2,End3} = {end_2,End3,End2}, 1363: {EndLine1,EndColumn1} = End1, 1364: End2 = {EndLine1,EndColumn1+2}, 1365: {ts_1,Ts2,Ts3} = {ts_1,Ts3,Ts2}, 1366: Ts2 = Ts1 ++ [lists:last(Ts2)], 1367: 1368: %% Attributes are keylists, but have no text. 1369: {ok, Ts7, End7} = erl_scan:string(String, {1,1}, Options), 1370: {ok, Ts8, End8} = scan_tokens(TString, Options, [], {1,1}), 1371: {end1, End1} = {end1, End7}, 1372: {end2, End2} = {end2, End8}, 1373: Ts8 = Ts7 ++ [lists:last(Ts8)], 1374: {cons,true} = {cons,consistent_attributes([Ts1,Ts2,Ts3,Ts7,Ts8])}, 1375: 1376: {Ts1, End1}. 1377: 1378: scan_tokens(String, Options, Rs, Location) -> 1379: case erl_scan:tokens([], String, Location, Options) of 1380: {done, {ok,Ts,End}, ""} -> 1381: {ok, lists:append(lists:reverse([Ts|Rs])), End}; 1382: {done, {ok,Ts,End}, Rest} -> 1383: scan_tokens(Rest, Options, [Ts|Rs], End) 1384: end. 1385: 1386: scan_tokens_1({done, {ok,Ts,End}, ""}, "", _Options, Rs, _Location) -> 1387: {ok,lists:append(lists:reverse([Ts|Rs])),End}; 1388: scan_tokens_1({done, {ok,Ts,End}, Rest}, Cs, Options, Rs, _Location) -> 1389: scan_tokens_1({more,[]}, Rest++Cs, Options, [Ts|Rs], End); 1390: scan_tokens_1({more, Cont}, [C | Cs], Options, Rs, Loc) -> 1391: R = erl_scan:tokens(Cont, [C], Loc, Options), 1392: scan_tokens_1(R, Cs, Options, Rs, Loc). 1393: 1394: consistent_attributes([]) -> 1395: true; 1396: consistent_attributes([Ts | TsL]) -> 1397: L = [T || T <- Ts, is_integer(element(2, T))], 1398: case L of 1399: [] -> 1400: TagsL = [[Tag || {Tag,_} <- 1401: erl_scan:attributes_info(element(2, T))] || 1402: T <- Ts], 1403: case lists:usort(TagsL) of 1404: [_] -> 1405: consistent_attributes(TsL); 1406: [] when Ts =:= [] -> 1407: consistent_attributes(TsL); 1408: _ -> 1409: Ts 1410: end; 1411: Ts -> 1412: consistent_attributes(TsL); 1413: _ -> 1414: Ts 1415: end. 1416: 1417: family_list(L) -> 1418: sofs:to_external(family(L)). 1419: 1420: family(L) -> 1421: sofs:relation_to_family(sofs:relation(L)).